mirror of
https://gitee.com/mateos/mateclaw.git
synced 2026-09-14 03:33:43 +08:00
feat(wiki): add image-to-text vision SPI with DashScope provider
This commit is contained in:
parent
b26a4ee4d1
commit
088ffcaee1
@ -1,7 +1,7 @@
|
|||||||
package vip.mate.tool.image;
|
package vip.mate.tool.image;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 图片生成能力枚举
|
* 图片相关能力枚举(生成方向 + 解析方向)。
|
||||||
*
|
*
|
||||||
* @author MateClaw Team
|
* @author MateClaw Team
|
||||||
*/
|
*/
|
||||||
@ -11,5 +11,8 @@ public enum ImageCapability {
|
|||||||
TEXT_TO_IMAGE,
|
TEXT_TO_IMAGE,
|
||||||
|
|
||||||
/** 图片编辑 / 风格转换 */
|
/** 图片编辑 / 风格转换 */
|
||||||
IMAGE_EDIT
|
IMAGE_EDIT,
|
||||||
|
|
||||||
|
/** Image to text — vision-in pipeline (factual caption + best-effort visible text). */
|
||||||
|
IMAGE_TO_TEXT
|
||||||
}
|
}
|
||||||
|
|||||||
@ -0,0 +1,59 @@
|
|||||||
|
package vip.mate.tool.image.vision;
|
||||||
|
|
||||||
|
import vip.mate.system.model.SystemSettingsDTO;
|
||||||
|
import vip.mate.tool.image.ImageCapability;
|
||||||
|
|
||||||
|
import java.util.Set;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* SPI for image-to-text providers.
|
||||||
|
*
|
||||||
|
* <p>Symmetric counterpart of {@link vip.mate.tool.image.ImageGenerationProvider}.
|
||||||
|
* Implementations are auto-discovered as Spring beans and selected by
|
||||||
|
* {@link ImageVisionService} according to {@link #autoDetectOrder()}
|
||||||
|
* (lower wins) — typically: regional default first, premium fallbacks last.
|
||||||
|
*
|
||||||
|
* @author MateClaw Team
|
||||||
|
*/
|
||||||
|
public interface ImageVisionProvider {
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Stable identifier for the SPI registry.
|
||||||
|
*
|
||||||
|
* <p>Convention: {@code <vendor>-vision} (e.g. {@code dashscope-vision},
|
||||||
|
* {@code openai-vision}, {@code claude-vision}).
|
||||||
|
*/
|
||||||
|
String id();
|
||||||
|
|
||||||
|
/** Display label for admin UI. */
|
||||||
|
String label();
|
||||||
|
|
||||||
|
/** Whether the provider needs an API key configured. */
|
||||||
|
boolean requiresCredential();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Auto-detect priority. Lower wins. Conventional bands:
|
||||||
|
* <ul>
|
||||||
|
* <li>10–19: regional default (lowest cost, primary path)</li>
|
||||||
|
* <li>20–29: universal fallback</li>
|
||||||
|
* <li>30+: premium / niche fallback</li>
|
||||||
|
* </ul>
|
||||||
|
*/
|
||||||
|
int autoDetectOrder();
|
||||||
|
|
||||||
|
/** Capabilities advertised; today only {@link ImageCapability#IMAGE_TO_TEXT}. */
|
||||||
|
Set<ImageCapability> capabilities();
|
||||||
|
|
||||||
|
/** Reports whether the provider is currently usable (key configured + reachable). */
|
||||||
|
boolean isAvailable(SystemSettingsDTO settings);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Synchronously caption an image.
|
||||||
|
*
|
||||||
|
* <p>Throws on failure — the caller (typically {@link ImageVisionService})
|
||||||
|
* decides whether to retry / fall back / surface to user.
|
||||||
|
*
|
||||||
|
* @return non-null populated {@link VisionResult}
|
||||||
|
*/
|
||||||
|
VisionResult caption(VisionRequest request, SystemSettingsDTO settings);
|
||||||
|
}
|
||||||
@ -0,0 +1,185 @@
|
|||||||
|
package vip.mate.tool.image.vision;
|
||||||
|
|
||||||
|
import lombok.RequiredArgsConstructor;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.springframework.stereotype.Service;
|
||||||
|
import vip.mate.exception.MateClawException;
|
||||||
|
import vip.mate.system.featureflag.FeatureFlagService;
|
||||||
|
import vip.mate.system.model.SystemSettingsDTO;
|
||||||
|
import vip.mate.system.service.SystemSettingService;
|
||||||
|
import vip.mate.wiki.metrics.WikiMetrics;
|
||||||
|
import vip.mate.wiki.model.WikiImageCaptionCacheEntity;
|
||||||
|
import vip.mate.wiki.service.WikiImageCaptionCacheService;
|
||||||
|
|
||||||
|
import java.security.MessageDigest;
|
||||||
|
import java.security.NoSuchAlgorithmException;
|
||||||
|
import java.time.Duration;
|
||||||
|
import java.time.Instant;
|
||||||
|
import java.time.LocalDateTime;
|
||||||
|
import java.time.ZoneId;
|
||||||
|
import java.util.Comparator;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Optional;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Routing + caching coordinator for the image-to-text pipeline.
|
||||||
|
*
|
||||||
|
* <p>End-to-end flow:
|
||||||
|
* <ol>
|
||||||
|
* <li>Short-circuit when the {@code wiki.ocr.enabled} feature flag is off.</li>
|
||||||
|
* <li>Hash the image bytes (SHA-256) and probe the caption cache. A cache
|
||||||
|
* hit returns immediately and bumps the row's {@code hit_count} as a
|
||||||
|
* side effect.</li>
|
||||||
|
* <li>On miss, walk providers in {@link ImageVisionProvider#autoDetectOrder()}
|
||||||
|
* order. The first available provider that returns a non-null caption
|
||||||
|
* wins. Failures are logged and the next provider is tried.</li>
|
||||||
|
* <li>Persist the winning caption to the cache (race-tolerant) and emit
|
||||||
|
* success metrics.</li>
|
||||||
|
* <li>If every provider fails, throw a {@link MateClawException} carrying
|
||||||
|
* the i18n key {@code err.wiki.vision.all_failed}.</li>
|
||||||
|
* </ol>
|
||||||
|
*
|
||||||
|
* @author MateClaw Team
|
||||||
|
*/
|
||||||
|
@Slf4j
|
||||||
|
@Service
|
||||||
|
@RequiredArgsConstructor
|
||||||
|
public class ImageVisionService {
|
||||||
|
|
||||||
|
private static final String FEATURE_FLAG = "wiki.ocr.enabled";
|
||||||
|
|
||||||
|
private final List<ImageVisionProvider> providers;
|
||||||
|
private final WikiImageCaptionCacheService cacheService;
|
||||||
|
private final SystemSettingService systemSettingService;
|
||||||
|
private final FeatureFlagService featureFlagService;
|
||||||
|
private final WikiMetrics metrics;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Captions an image, going through cache → provider chain.
|
||||||
|
*
|
||||||
|
* @throws MateClawException with key {@code err.wiki.vision.disabled} when the feature flag is off
|
||||||
|
* @throws MateClawException with key {@code err.wiki.vision.no_provider} when no provider is available
|
||||||
|
* @throws MateClawException with key {@code err.wiki.vision.all_failed} when every provider failed
|
||||||
|
*/
|
||||||
|
public VisionResult caption(VisionRequest request) {
|
||||||
|
if (request == null || request.getImageBytes() == null
|
||||||
|
|| request.getImageBytes().length == 0) {
|
||||||
|
throw new IllegalArgumentException("VisionRequest with image bytes is required");
|
||||||
|
}
|
||||||
|
if (!featureFlagService.isEnabled(FEATURE_FLAG)) {
|
||||||
|
throw new MateClawException("err.wiki.vision.disabled",
|
||||||
|
"Image vision pipeline is currently disabled");
|
||||||
|
}
|
||||||
|
|
||||||
|
String sha256 = sha256Hex(request.getImageBytes());
|
||||||
|
|
||||||
|
// 1. Cache check.
|
||||||
|
Optional<WikiImageCaptionCacheEntity> cached = cacheService.lookup(sha256);
|
||||||
|
if (cached.isPresent()) {
|
||||||
|
metrics.recordVisionCacheHit(true);
|
||||||
|
log.debug("[Vision] cache hit sha={}", shortSha(sha256));
|
||||||
|
return fromEntity(cached.get());
|
||||||
|
}
|
||||||
|
metrics.recordVisionCacheHit(false);
|
||||||
|
|
||||||
|
// 2. Provider walk.
|
||||||
|
SystemSettingsDTO settings = systemSettingService.getSettings();
|
||||||
|
List<ImageVisionProvider> available = providers.stream()
|
||||||
|
.filter(p -> {
|
||||||
|
try {
|
||||||
|
return p.isAvailable(settings);
|
||||||
|
} catch (Exception e) {
|
||||||
|
log.debug("[Vision] availability check threw for provider={}: {}",
|
||||||
|
p.id(), e.getMessage());
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.sorted(Comparator.comparingInt(ImageVisionProvider::autoDetectOrder))
|
||||||
|
.toList();
|
||||||
|
if (available.isEmpty()) {
|
||||||
|
throw new MateClawException("err.wiki.vision.no_provider",
|
||||||
|
"No image vision provider is configured");
|
||||||
|
}
|
||||||
|
|
||||||
|
VisionResult winner = null;
|
||||||
|
Throwable lastError = null;
|
||||||
|
for (ImageVisionProvider provider : available) {
|
||||||
|
long startNanos = System.nanoTime();
|
||||||
|
try {
|
||||||
|
winner = provider.caption(request, settings);
|
||||||
|
metrics.recordVisionCall(provider.id(), true,
|
||||||
|
Duration.ofNanos(System.nanoTime() - startNanos));
|
||||||
|
if (winner != null) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
} catch (Exception e) {
|
||||||
|
metrics.recordVisionCall(provider.id(), false,
|
||||||
|
Duration.ofNanos(System.nanoTime() - startNanos));
|
||||||
|
log.warn("[Vision] provider={} failed sha={}: {}",
|
||||||
|
provider.id(), shortSha(sha256), e.getMessage());
|
||||||
|
lastError = e;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (winner == null) {
|
||||||
|
String why = lastError != null ? lastError.getMessage() : "all providers returned null";
|
||||||
|
throw new MateClawException("err.wiki.vision.all_failed",
|
||||||
|
"All image vision providers failed: " + why);
|
||||||
|
}
|
||||||
|
|
||||||
|
// 3. Persist to cache.
|
||||||
|
cacheService.persist(toEntity(sha256, winner, request.getMimeType()));
|
||||||
|
|
||||||
|
return winner;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ==================== internal ====================
|
||||||
|
|
||||||
|
private static VisionResult fromEntity(WikiImageCaptionCacheEntity row) {
|
||||||
|
Instant captured = row.getCapturedAt() == null
|
||||||
|
? Instant.now()
|
||||||
|
: row.getCapturedAt().atZone(ZoneId.systemDefault()).toInstant();
|
||||||
|
return VisionResult.builder()
|
||||||
|
.caption(row.getCaption())
|
||||||
|
.visibleText(row.getVisibleText())
|
||||||
|
.providerId(row.getProviderId())
|
||||||
|
.model(row.getCaptureModel())
|
||||||
|
.capturedAt(captured)
|
||||||
|
.durationMs(row.getDurationMs() != null ? row.getDurationMs() : 0L)
|
||||||
|
.build();
|
||||||
|
}
|
||||||
|
|
||||||
|
private static WikiImageCaptionCacheEntity toEntity(String sha256, VisionResult result,
|
||||||
|
String mimeType) {
|
||||||
|
WikiImageCaptionCacheEntity row = new WikiImageCaptionCacheEntity();
|
||||||
|
row.setImageSha256(sha256);
|
||||||
|
row.setCaption(result.getCaption());
|
||||||
|
row.setVisibleText(result.getVisibleText());
|
||||||
|
row.setMimeType(mimeType);
|
||||||
|
row.setCaptureModel(result.getModel());
|
||||||
|
row.setProviderId(result.getProviderId());
|
||||||
|
row.setDurationMs(result.getDurationMs());
|
||||||
|
row.setHitCount(0L);
|
||||||
|
row.setCapturedAt(result.getCapturedAt() != null
|
||||||
|
? LocalDateTime.ofInstant(result.getCapturedAt(), ZoneId.systemDefault())
|
||||||
|
: LocalDateTime.now());
|
||||||
|
return row;
|
||||||
|
}
|
||||||
|
|
||||||
|
static String sha256Hex(byte[] bytes) {
|
||||||
|
try {
|
||||||
|
byte[] digest = MessageDigest.getInstance("SHA-256").digest(bytes);
|
||||||
|
StringBuilder sb = new StringBuilder(64);
|
||||||
|
for (byte b : digest) {
|
||||||
|
sb.append(String.format("%02x", b));
|
||||||
|
}
|
||||||
|
return sb.toString();
|
||||||
|
} catch (NoSuchAlgorithmException e) {
|
||||||
|
throw new IllegalStateException("SHA-256 unavailable on this JVM", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static String shortSha(String sha) {
|
||||||
|
return sha.substring(0, Math.min(8, sha.length()));
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -0,0 +1,30 @@
|
|||||||
|
package vip.mate.tool.image.vision;
|
||||||
|
|
||||||
|
import lombok.Builder;
|
||||||
|
import lombok.Data;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Surrounding text fed to the vision provider for context-aware captioning.
|
||||||
|
*
|
||||||
|
* <p>When at least one side is non-blank the provider switches from the
|
||||||
|
* factual-only prompt to the context-aware prompt, which asks the model
|
||||||
|
* to flag images that look unrelated to the surrounding text (so callers
|
||||||
|
* can mark them off-topic in downstream pipelines).
|
||||||
|
*
|
||||||
|
* @author MateClaw Team
|
||||||
|
*/
|
||||||
|
@Data
|
||||||
|
@Builder
|
||||||
|
public class VisionContext {
|
||||||
|
|
||||||
|
/** Up to ~500 chars of text that appeared immediately before the image. */
|
||||||
|
private String beforeText;
|
||||||
|
|
||||||
|
/** Up to ~500 chars of text that appeared immediately after the image. */
|
||||||
|
private String afterText;
|
||||||
|
|
||||||
|
public boolean hasContext() {
|
||||||
|
return (beforeText != null && !beforeText.isBlank())
|
||||||
|
|| (afterText != null && !afterText.isBlank());
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -0,0 +1,35 @@
|
|||||||
|
package vip.mate.tool.image.vision;
|
||||||
|
|
||||||
|
import lombok.Builder;
|
||||||
|
import lombok.Data;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Input to one image-to-text call.
|
||||||
|
*
|
||||||
|
* <p>The image is supplied as raw bytes plus a MIME type; callers are
|
||||||
|
* responsible for any format conversion before invoking. The optional
|
||||||
|
* {@link VisionContext} carries surrounding text so the provider can
|
||||||
|
* decide whether the image is on-topic and produce a context-aware
|
||||||
|
* caption.
|
||||||
|
*
|
||||||
|
* @author MateClaw Team
|
||||||
|
*/
|
||||||
|
@Data
|
||||||
|
@Builder
|
||||||
|
public class VisionRequest {
|
||||||
|
|
||||||
|
/** Raw image bytes (PNG / JPEG / WebP / ...). Required. */
|
||||||
|
private byte[] imageBytes;
|
||||||
|
|
||||||
|
/** MIME type, e.g. {@code image/png}. Required. */
|
||||||
|
private String mimeType;
|
||||||
|
|
||||||
|
/** Optional surrounding-text context for context-aware captioning. */
|
||||||
|
private VisionContext context;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Caller hint: when set, ask the provider for this specific model name.
|
||||||
|
* Provider may ignore the hint if the model is unsupported.
|
||||||
|
*/
|
||||||
|
private String preferModel;
|
||||||
|
}
|
||||||
@ -0,0 +1,45 @@
|
|||||||
|
package vip.mate.tool.image.vision;
|
||||||
|
|
||||||
|
import lombok.Builder;
|
||||||
|
import lombok.Data;
|
||||||
|
|
||||||
|
import java.time.Instant;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Outcome of one successful image-to-text call.
|
||||||
|
*
|
||||||
|
* <p>Producers must populate {@link #caption}, {@link #providerId},
|
||||||
|
* {@link #model} and {@link #capturedAt} at minimum. The other fields
|
||||||
|
* are optional and allow callers to render richer UI.
|
||||||
|
*
|
||||||
|
* @author MateClaw Team
|
||||||
|
*/
|
||||||
|
@Data
|
||||||
|
@Builder
|
||||||
|
public class VisionResult {
|
||||||
|
|
||||||
|
/** 2-4 sentence factual description of the image (primary output). */
|
||||||
|
private String caption;
|
||||||
|
|
||||||
|
/** Best-effort recovery of any text rendered inside the image; may be null. */
|
||||||
|
private String visibleText;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* True when the context-aware prompt determined the image is unrelated
|
||||||
|
* to the surrounding text. Caller decides what to do with off-topic
|
||||||
|
* captions (downrank / hide / annotate). False (or null) when context
|
||||||
|
* was not supplied.
|
||||||
|
*/
|
||||||
|
private boolean offTopic;
|
||||||
|
|
||||||
|
/** SPI-registry id of the provider that produced this result. */
|
||||||
|
private String providerId;
|
||||||
|
|
||||||
|
/** Vendor-specific model identifier (e.g. {@code qwen-vl-max}). */
|
||||||
|
private String model;
|
||||||
|
|
||||||
|
private Instant capturedAt;
|
||||||
|
|
||||||
|
/** Wall-clock duration of the round-trip, in milliseconds. */
|
||||||
|
private long durationMs;
|
||||||
|
}
|
||||||
@ -0,0 +1,221 @@
|
|||||||
|
package vip.mate.tool.image.vision.provider;
|
||||||
|
|
||||||
|
import cn.hutool.http.HttpRequest;
|
||||||
|
import cn.hutool.http.HttpResponse;
|
||||||
|
import com.fasterxml.jackson.databind.JsonNode;
|
||||||
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
|
import com.fasterxml.jackson.databind.node.ArrayNode;
|
||||||
|
import com.fasterxml.jackson.databind.node.ObjectNode;
|
||||||
|
import lombok.RequiredArgsConstructor;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.springframework.stereotype.Component;
|
||||||
|
import vip.mate.agent.prompt.PromptLoader;
|
||||||
|
import vip.mate.exception.MateClawException;
|
||||||
|
import vip.mate.llm.model.ModelProviderEntity;
|
||||||
|
import vip.mate.llm.service.ModelProviderService;
|
||||||
|
import vip.mate.system.model.SystemSettingsDTO;
|
||||||
|
import vip.mate.tool.image.ImageCapability;
|
||||||
|
import vip.mate.tool.image.vision.ImageVisionProvider;
|
||||||
|
import vip.mate.tool.image.vision.VisionContext;
|
||||||
|
import vip.mate.tool.image.vision.VisionRequest;
|
||||||
|
import vip.mate.tool.image.vision.VisionResult;
|
||||||
|
|
||||||
|
import java.time.Duration;
|
||||||
|
import java.time.Instant;
|
||||||
|
import java.util.Base64;
|
||||||
|
import java.util.Set;
|
||||||
|
import java.util.regex.Matcher;
|
||||||
|
import java.util.regex.Pattern;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* DashScope vision provider — uses qwen-vl-max via the OpenAI-compatible
|
||||||
|
* endpoint at {@code /compatible-mode/v1/chat/completions}.
|
||||||
|
*
|
||||||
|
* <p>Default for the regional production rollout: API keys are typically
|
||||||
|
* available (DASHSCOPE_API_KEY is mandatory for the rest of the platform)
|
||||||
|
* and per-image cost is the lowest of the supported vendors.
|
||||||
|
*
|
||||||
|
* <p>Implementation deliberately uses Hutool {@code HttpRequest} rather
|
||||||
|
* than building a Spring AI {@code ChatClient} — the call shape is a
|
||||||
|
* one-shot request/response that doesn't benefit from the heavyweight
|
||||||
|
* agent-style observability / retry / cache machinery in
|
||||||
|
* {@code AgentOpenAiCompatibleChatModelBuilder}.
|
||||||
|
*
|
||||||
|
* @author MateClaw Team
|
||||||
|
*/
|
||||||
|
@Slf4j
|
||||||
|
@Component
|
||||||
|
@RequiredArgsConstructor
|
||||||
|
public class DashScopeVisionProvider implements ImageVisionProvider {
|
||||||
|
|
||||||
|
private static final String PROVIDER_ID = "dashscope-vision";
|
||||||
|
private static final String DASHSCOPE_PROVIDER_KEY = "dashscope";
|
||||||
|
private static final String DEFAULT_MODEL = "qwen-vl-max";
|
||||||
|
private static final String DEFAULT_BASE_URL = "https://dashscope.aliyuncs.com/compatible-mode/v1";
|
||||||
|
private static final int CALL_TIMEOUT_MS = 60_000;
|
||||||
|
private static final Pattern OFF_TOPIC_PREFIX = Pattern.compile("^\\s*\\[OFF-TOPIC\\]\\s*", Pattern.CASE_INSENSITIVE);
|
||||||
|
|
||||||
|
private final ModelProviderService modelProviderService;
|
||||||
|
private final ObjectMapper objectMapper;
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String id() {
|
||||||
|
return PROVIDER_ID;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String label() {
|
||||||
|
return "DashScope qwen-vl";
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean requiresCredential() {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public int autoDetectOrder() {
|
||||||
|
return 10;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Set<ImageCapability> capabilities() {
|
||||||
|
return Set.of(ImageCapability.IMAGE_TO_TEXT);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean isAvailable(SystemSettingsDTO settings) {
|
||||||
|
try {
|
||||||
|
return modelProviderService.isProviderConfigured(DASHSCOPE_PROVIDER_KEY);
|
||||||
|
} catch (Exception e) {
|
||||||
|
log.debug("[Vision][dashscope] availability check failed: {}", e.getMessage());
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public VisionResult caption(VisionRequest request, SystemSettingsDTO settings) {
|
||||||
|
ModelProviderEntity provider;
|
||||||
|
try {
|
||||||
|
provider = modelProviderService.getProviderConfig(DASHSCOPE_PROVIDER_KEY);
|
||||||
|
} catch (Exception e) {
|
||||||
|
throw new MateClawException("err.wiki.vision.no_provider",
|
||||||
|
"DashScope provider is not configured");
|
||||||
|
}
|
||||||
|
String apiKey = provider.getApiKey();
|
||||||
|
if (apiKey == null || apiKey.isBlank()) {
|
||||||
|
throw new MateClawException("err.wiki.vision.no_provider",
|
||||||
|
"DashScope API key is missing");
|
||||||
|
}
|
||||||
|
String baseUrl = (provider.getBaseUrl() == null || provider.getBaseUrl().isBlank())
|
||||||
|
? DEFAULT_BASE_URL
|
||||||
|
: provider.getBaseUrl();
|
||||||
|
String model = (request.getPreferModel() == null || request.getPreferModel().isBlank())
|
||||||
|
? DEFAULT_MODEL
|
||||||
|
: request.getPreferModel();
|
||||||
|
|
||||||
|
String prompt = buildPrompt(request.getContext());
|
||||||
|
String imageDataUrl = buildDataUrl(request.getMimeType(), request.getImageBytes());
|
||||||
|
|
||||||
|
ObjectNode body = objectMapper.createObjectNode();
|
||||||
|
body.put("model", model);
|
||||||
|
body.put("temperature", 0.0);
|
||||||
|
body.put("max_tokens", 4096);
|
||||||
|
|
||||||
|
ArrayNode messages = body.putArray("messages");
|
||||||
|
ObjectNode userMsg = messages.addObject();
|
||||||
|
userMsg.put("role", "user");
|
||||||
|
ArrayNode content = userMsg.putArray("content");
|
||||||
|
content.addObject().put("type", "text").put("text", prompt);
|
||||||
|
ObjectNode imagePart = content.addObject();
|
||||||
|
imagePart.put("type", "image_url");
|
||||||
|
imagePart.putObject("image_url").put("url", imageDataUrl);
|
||||||
|
|
||||||
|
long startNanos = System.nanoTime();
|
||||||
|
HttpResponse response;
|
||||||
|
try {
|
||||||
|
response = HttpRequest.post(baseUrl + "/chat/completions")
|
||||||
|
.header("Authorization", "Bearer " + apiKey)
|
||||||
|
.header("Content-Type", "application/json")
|
||||||
|
.body(body.toString())
|
||||||
|
.timeout(CALL_TIMEOUT_MS)
|
||||||
|
.execute();
|
||||||
|
} catch (Exception e) {
|
||||||
|
throw new MateClawException("err.wiki.vision.provider_failed",
|
||||||
|
"DashScope vision call failed: " + e.getMessage());
|
||||||
|
}
|
||||||
|
long durationMs = Duration.ofNanos(System.nanoTime() - startNanos).toMillis();
|
||||||
|
|
||||||
|
if (response.getStatus() != 200) {
|
||||||
|
String errBody = response.body();
|
||||||
|
log.warn("[Vision][dashscope] HTTP {} body={}", response.getStatus(), truncate(errBody, 400));
|
||||||
|
throw new MateClawException("err.wiki.vision.provider_failed",
|
||||||
|
"DashScope vision returned HTTP " + response.getStatus());
|
||||||
|
}
|
||||||
|
|
||||||
|
String text;
|
||||||
|
try {
|
||||||
|
JsonNode root = objectMapper.readTree(response.body());
|
||||||
|
text = root.path("choices").path(0).path("message").path("content").asText("").trim();
|
||||||
|
} catch (Exception e) {
|
||||||
|
throw new MateClawException("err.wiki.vision.provider_failed",
|
||||||
|
"DashScope vision response was unparseable: " + e.getMessage());
|
||||||
|
}
|
||||||
|
if (text.isEmpty()) {
|
||||||
|
throw new MateClawException("err.wiki.vision.provider_failed",
|
||||||
|
"DashScope vision returned an empty caption");
|
||||||
|
}
|
||||||
|
|
||||||
|
Matcher offTopic = OFF_TOPIC_PREFIX.matcher(text);
|
||||||
|
boolean isOffTopic = offTopic.find();
|
||||||
|
String caption = isOffTopic ? offTopic.replaceFirst("").trim() : text;
|
||||||
|
|
||||||
|
return VisionResult.builder()
|
||||||
|
.caption(caption)
|
||||||
|
.visibleText(extractVisibleTextHeuristic(caption))
|
||||||
|
.offTopic(isOffTopic)
|
||||||
|
.providerId(PROVIDER_ID)
|
||||||
|
.model(model)
|
||||||
|
.capturedAt(Instant.now())
|
||||||
|
.durationMs(durationMs)
|
||||||
|
.build();
|
||||||
|
}
|
||||||
|
|
||||||
|
private String buildPrompt(VisionContext context) {
|
||||||
|
if (context != null && context.hasContext()) {
|
||||||
|
String template = PromptLoader.loadPrompt("wiki/vision-caption-context-aware");
|
||||||
|
String before = (context.getBeforeText() == null || context.getBeforeText().isBlank())
|
||||||
|
? "(none)" : context.getBeforeText().trim();
|
||||||
|
String after = (context.getAfterText() == null || context.getAfterText().isBlank())
|
||||||
|
? "(none)" : context.getAfterText().trim();
|
||||||
|
return template.replace("{before}", before).replace("{after}", after);
|
||||||
|
}
|
||||||
|
return PromptLoader.loadPrompt("wiki/vision-caption-factual");
|
||||||
|
}
|
||||||
|
|
||||||
|
private static String buildDataUrl(String mimeType, byte[] bytes) {
|
||||||
|
String mime = (mimeType == null || mimeType.isBlank()) ? "image/png" : mimeType;
|
||||||
|
return "data:" + mime + ";base64," + Base64.getEncoder().encodeToString(bytes);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Heuristic: pull obvious quoted strings as "visible text". This is a
|
||||||
|
* placeholder until a stronger OCR-grade extraction lands; for the
|
||||||
|
* factual prompt the model already inlines text verbatim into the
|
||||||
|
* caption, so the cache row's caption alone is enough for retrieval.
|
||||||
|
*/
|
||||||
|
private static String extractVisibleTextHeuristic(String caption) {
|
||||||
|
Matcher m = Pattern.compile("\"([^\"]{3,})\"").matcher(caption);
|
||||||
|
StringBuilder sb = new StringBuilder();
|
||||||
|
while (m.find()) {
|
||||||
|
if (sb.length() > 0) sb.append('\n');
|
||||||
|
sb.append(m.group(1));
|
||||||
|
}
|
||||||
|
return sb.length() == 0 ? null : sb.toString();
|
||||||
|
}
|
||||||
|
|
||||||
|
private static String truncate(String s, int max) {
|
||||||
|
if (s == null) return "";
|
||||||
|
return s.length() > max ? s.substring(0, max) + "…" : s;
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -273,3 +273,9 @@ agent.limit_exceeded.empty_context=\uff08\u5c1a\u672a\u6536\u96c6\u5230\u5de5\u5
|
|||||||
cron.tasks_conversation.title=📋 定时任务
|
cron.tasks_conversation.title=📋 定时任务
|
||||||
cron.run_header.scheduled=定时触发
|
cron.run_header.scheduled=定时触发
|
||||||
cron.run_header.manual=手动触发
|
cron.run_header.manual=手动触发
|
||||||
|
|
||||||
|
# --- Wiki vision-in pipeline ---
|
||||||
|
err.wiki.vision.disabled=图片识别功能未启用
|
||||||
|
err.wiki.vision.no_provider=未配置可用的图片识别 provider
|
||||||
|
err.wiki.vision.provider_failed=图片识别 provider 调用失败
|
||||||
|
err.wiki.vision.all_failed=所有图片识别 provider 调用失败
|
||||||
|
|||||||
@ -280,3 +280,9 @@ agent.limit_exceeded.empty_context=(No tool call results collected yet.)
|
|||||||
cron.tasks_conversation.title=📋 Scheduled Tasks
|
cron.tasks_conversation.title=📋 Scheduled Tasks
|
||||||
cron.run_header.scheduled=scheduled
|
cron.run_header.scheduled=scheduled
|
||||||
cron.run_header.manual=manual
|
cron.run_header.manual=manual
|
||||||
|
|
||||||
|
# --- Wiki vision-in pipeline ---
|
||||||
|
err.wiki.vision.disabled=Image vision pipeline is currently disabled
|
||||||
|
err.wiki.vision.no_provider=No image vision provider is configured
|
||||||
|
err.wiki.vision.provider_failed=Image vision provider call failed
|
||||||
|
err.wiki.vision.all_failed=All image vision providers failed
|
||||||
|
|||||||
@ -0,0 +1,11 @@
|
|||||||
|
Describe this image factually for a knowledge-base index. Surrounding text below tells you what topic the image relates to.
|
||||||
|
|
||||||
|
--- Text before image ---
|
||||||
|
{before}
|
||||||
|
|
||||||
|
--- Text after image ---
|
||||||
|
{after}
|
||||||
|
|
||||||
|
--- End surrounding text ---
|
||||||
|
|
||||||
|
If the image is clearly NOT relevant to the surrounding text, prepend "[OFF-TOPIC] " to your response. Otherwise, describe factually: visible text verbatim, chart axes and values, diagram structure, key visual elements. 2 to 4 sentences. Plain text only — no markdown, no preamble.
|
||||||
@ -0,0 +1 @@
|
|||||||
|
Describe this image factually for a knowledge-base index. Include: any visible text verbatim, chart axes and values, diagram structure (boxes/arrows/labels), key visual elements. Do NOT speculate or editorialize. 2 to 4 sentences. Output plain text only — no markdown, no preamble.
|
||||||
Loading…
Reference in New Issue
Block a user