mirror of
https://gitee.com/mateos/mateclaw.git
synced 2026-09-13 03:13:41 +08:00
feat(wiki): add image-to-text vision SPI with DashScope provider
This commit is contained in:
matevip
parent
b26a4ee4d1
commit
088ffcaee1
@ -1,7 +1,7 @@
|
||||
package vip.mate.tool.image;
|
||||
|
||||
/**
|
||||
* 图片生成能力枚举
|
||||
* 图片相关能力枚举（生成方向 + 解析方向）。
|
||||
*
|
||||
* @author MateClaw Team
|
||||
*/
|
||||
@ -11,5 +11,8 @@ public enum ImageCapability {
|
||||
TEXT_TO_IMAGE,
|
||||
|
||||
/** 图片编辑 / 风格转换 */
|
||||
IMAGE_EDIT
|
||||
IMAGE_EDIT,
|
||||
|
||||
/** Image to text — vision-in pipeline (factual caption + best-effort visible text). */
|
||||
IMAGE_TO_TEXT
|
||||
}
|
||||
|
||||
@ -0,0 +1,59 @@
|
||||
package vip.mate.tool.image.vision;
|
||||
|
||||
import vip.mate.system.model.SystemSettingsDTO;
|
||||
import vip.mate.tool.image.ImageCapability;
|
||||
|
||||
import java.util.Set;
|
||||
|
||||
/**
|
||||
* SPI for image-to-text providers.
|
||||
*
|
||||
* <p>Symmetric counterpart of {@link vip.mate.tool.image.ImageGenerationProvider}.
|
||||
* Implementations are auto-discovered as Spring beans and selected by
|
||||
* {@link ImageVisionService} according to {@link #autoDetectOrder()}
|
||||
* (lower wins) — typically: regional default first, premium fallbacks last.
|
||||
*
|
||||
* @author MateClaw Team
|
||||
*/
|
||||
public interface ImageVisionProvider {
|
||||
|
||||
/**
|
||||
* Stable identifier for the SPI registry.
|
||||
*
|
||||
* <p>Convention: {@code <vendor>-vision} (e.g. {@code dashscope-vision},
|
||||
* {@code openai-vision}, {@code claude-vision}).
|
||||
*/
|
||||
String id();
|
||||
|
||||
/** Display label for admin UI. */
|
||||
String label();
|
||||
|
||||
/** Whether the provider needs an API key configured. */
|
||||
boolean requiresCredential();
|
||||
|
||||
/**
|
||||
* Auto-detect priority. Lower wins. Conventional bands:
|
||||
* <ul>
|
||||
* <li>10–19: regional default (lowest cost, primary path)</li>
|
||||
* <li>20–29: universal fallback</li>
|
||||
* <li>30+: premium / niche fallback</li>
|
||||
* </ul>
|
||||
*/
|
||||
int autoDetectOrder();
|
||||
|
||||
/** Capabilities advertised; today only {@link ImageCapability#IMAGE_TO_TEXT}. */
|
||||
Set<ImageCapability> capabilities();
|
||||
|
||||
/** Reports whether the provider is currently usable (key configured + reachable). */
|
||||
boolean isAvailable(SystemSettingsDTO settings);
|
||||
|
||||
/**
|
||||
* Synchronously caption an image.
|
||||
*
|
||||
* <p>Throws on failure — the caller (typically {@link ImageVisionService})
|
||||
* decides whether to retry / fall back / surface to user.
|
||||
*
|
||||
* @return non-null populated {@link VisionResult}
|
||||
*/
|
||||
VisionResult caption(VisionRequest request, SystemSettingsDTO settings);
|
||||
}
|
||||
@ -0,0 +1,185 @@
|
||||
package vip.mate.tool.image.vision;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.stereotype.Service;
|
||||
import vip.mate.exception.MateClawException;
|
||||
import vip.mate.system.featureflag.FeatureFlagService;
|
||||
import vip.mate.system.model.SystemSettingsDTO;
|
||||
import vip.mate.system.service.SystemSettingService;
|
||||
import vip.mate.wiki.metrics.WikiMetrics;
|
||||
import vip.mate.wiki.model.WikiImageCaptionCacheEntity;
|
||||
import vip.mate.wiki.service.WikiImageCaptionCacheService;
|
||||
|
||||
import java.security.MessageDigest;
|
||||
import java.security.NoSuchAlgorithmException;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.ZoneId;
|
||||
import java.util.Comparator;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
|
||||
/**
|
||||
* Routing + caching coordinator for the image-to-text pipeline.
|
||||
*
|
||||
* <p>End-to-end flow:
|
||||
* <ol>
|
||||
* <li>Short-circuit when the {@code wiki.ocr.enabled} feature flag is off.</li>
|
||||
* <li>Hash the image bytes (SHA-256) and probe the caption cache. A cache
|
||||
* hit returns immediately and bumps the row's {@code hit_count} as a
|
||||
* side effect.</li>
|
||||
* <li>On miss, walk providers in {@link ImageVisionProvider#autoDetectOrder()}
|
||||
* order. The first available provider that returns a non-null caption
|
||||
* wins. Failures are logged and the next provider is tried.</li>
|
||||
* <li>Persist the winning caption to the cache (race-tolerant) and emit
|
||||
* success metrics.</li>
|
||||
* <li>If every provider fails, throw a {@link MateClawException} carrying
|
||||
* the i18n key {@code err.wiki.vision.all_failed}.</li>
|
||||
* </ol>
|
||||
*
|
||||
* @author MateClaw Team
|
||||
*/
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class ImageVisionService {
|
||||
|
||||
private static final String FEATURE_FLAG = "wiki.ocr.enabled";
|
||||
|
||||
private final List<ImageVisionProvider> providers;
|
||||
private final WikiImageCaptionCacheService cacheService;
|
||||
private final SystemSettingService systemSettingService;
|
||||
private final FeatureFlagService featureFlagService;
|
||||
private final WikiMetrics metrics;
|
||||
|
||||
/**
|
||||
* Captions an image, going through cache → provider chain.
|
||||
*
|
||||
* @throws MateClawException with key {@code err.wiki.vision.disabled} when the feature flag is off
|
||||
* @throws MateClawException with key {@code err.wiki.vision.no_provider} when no provider is available
|
||||
* @throws MateClawException with key {@code err.wiki.vision.all_failed} when every provider failed
|
||||
*/
|
||||
public VisionResult caption(VisionRequest request) {
|
||||
if (request == null || request.getImageBytes() == null
|
||||
|| request.getImageBytes().length == 0) {
|
||||
throw new IllegalArgumentException("VisionRequest with image bytes is required");
|
||||
}
|
||||
if (!featureFlagService.isEnabled(FEATURE_FLAG)) {
|
||||
throw new MateClawException("err.wiki.vision.disabled",
|
||||
"Image vision pipeline is currently disabled");
|
||||
}
|
||||
|
||||
String sha256 = sha256Hex(request.getImageBytes());
|
||||
|
||||
// 1. Cache check.
|
||||
Optional<WikiImageCaptionCacheEntity> cached = cacheService.lookup(sha256);
|
||||
if (cached.isPresent()) {
|
||||
metrics.recordVisionCacheHit(true);
|
||||
log.debug("[Vision] cache hit sha={}", shortSha(sha256));
|
||||
return fromEntity(cached.get());
|
||||
}
|
||||
metrics.recordVisionCacheHit(false);
|
||||
|
||||
// 2. Provider walk.
|
||||
SystemSettingsDTO settings = systemSettingService.getSettings();
|
||||
List<ImageVisionProvider> available = providers.stream()
|
||||
.filter(p -> {
|
||||
try {
|
||||
return p.isAvailable(settings);
|
||||
} catch (Exception e) {
|
||||
log.debug("[Vision] availability check threw for provider={}: {}",
|
||||
p.id(), e.getMessage());
|
||||
return false;
|
||||
}
|
||||
})
|
||||
.sorted(Comparator.comparingInt(ImageVisionProvider::autoDetectOrder))
|
||||
.toList();
|
||||
if (available.isEmpty()) {
|
||||
throw new MateClawException("err.wiki.vision.no_provider",
|
||||
"No image vision provider is configured");
|
||||
}
|
||||
|
||||
VisionResult winner = null;
|
||||
Throwable lastError = null;
|
||||
for (ImageVisionProvider provider : available) {
|
||||
long startNanos = System.nanoTime();
|
||||
try {
|
||||
winner = provider.caption(request, settings);
|
||||
metrics.recordVisionCall(provider.id(), true,
|
||||
Duration.ofNanos(System.nanoTime() - startNanos));
|
||||
if (winner != null) {
|
||||
break;
|
||||
}
|
||||
} catch (Exception e) {
|
||||
metrics.recordVisionCall(provider.id(), false,
|
||||
Duration.ofNanos(System.nanoTime() - startNanos));
|
||||
log.warn("[Vision] provider={} failed sha={}: {}",
|
||||
provider.id(), shortSha(sha256), e.getMessage());
|
||||
lastError = e;
|
||||
}
|
||||
}
|
||||
|
||||
if (winner == null) {
|
||||
String why = lastError != null ? lastError.getMessage() : "all providers returned null";
|
||||
throw new MateClawException("err.wiki.vision.all_failed",
|
||||
"All image vision providers failed: " + why);
|
||||
}
|
||||
|
||||
// 3. Persist to cache.
|
||||
cacheService.persist(toEntity(sha256, winner, request.getMimeType()));
|
||||
|
||||
return winner;
|
||||
}
|
||||
|
||||
// ==================== internal ====================
|
||||
|
||||
private static VisionResult fromEntity(WikiImageCaptionCacheEntity row) {
|
||||
Instant captured = row.getCapturedAt() == null
|
||||
? Instant.now()
|
||||
: row.getCapturedAt().atZone(ZoneId.systemDefault()).toInstant();
|
||||
return VisionResult.builder()
|
||||
.caption(row.getCaption())
|
||||
.visibleText(row.getVisibleText())
|
||||
.providerId(row.getProviderId())
|
||||
.model(row.getCaptureModel())
|
||||
.capturedAt(captured)
|
||||
.durationMs(row.getDurationMs() != null ? row.getDurationMs() : 0L)
|
||||
.build();
|
||||
}
|
||||
|
||||
private static WikiImageCaptionCacheEntity toEntity(String sha256, VisionResult result,
|
||||
String mimeType) {
|
||||
WikiImageCaptionCacheEntity row = new WikiImageCaptionCacheEntity();
|
||||
row.setImageSha256(sha256);
|
||||
row.setCaption(result.getCaption());
|
||||
row.setVisibleText(result.getVisibleText());
|
||||
row.setMimeType(mimeType);
|
||||
row.setCaptureModel(result.getModel());
|
||||
row.setProviderId(result.getProviderId());
|
||||
row.setDurationMs(result.getDurationMs());
|
||||
row.setHitCount(0L);
|
||||
row.setCapturedAt(result.getCapturedAt() != null
|
||||
? LocalDateTime.ofInstant(result.getCapturedAt(), ZoneId.systemDefault())
|
||||
: LocalDateTime.now());
|
||||
return row;
|
||||
}
|
||||
|
||||
static String sha256Hex(byte[] bytes) {
|
||||
try {
|
||||
byte[] digest = MessageDigest.getInstance("SHA-256").digest(bytes);
|
||||
StringBuilder sb = new StringBuilder(64);
|
||||
for (byte b : digest) {
|
||||
sb.append(String.format("%02x", b));
|
||||
}
|
||||
return sb.toString();
|
||||
} catch (NoSuchAlgorithmException e) {
|
||||
throw new IllegalStateException("SHA-256 unavailable on this JVM", e);
|
||||
}
|
||||
}
|
||||
|
||||
private static String shortSha(String sha) {
|
||||
return sha.substring(0, Math.min(8, sha.length()));
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,30 @@
|
||||
package vip.mate.tool.image.vision;
|
||||
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
|
||||
/**
|
||||
* Surrounding text fed to the vision provider for context-aware captioning.
|
||||
*
|
||||
* <p>When at least one side is non-blank the provider switches from the
|
||||
* factual-only prompt to the context-aware prompt, which asks the model
|
||||
* to flag images that look unrelated to the surrounding text (so callers
|
||||
* can mark them off-topic in downstream pipelines).
|
||||
*
|
||||
* @author MateClaw Team
|
||||
*/
|
||||
@Data
|
||||
@Builder
|
||||
public class VisionContext {
|
||||
|
||||
/** Up to ~500 chars of text that appeared immediately before the image. */
|
||||
private String beforeText;
|
||||
|
||||
/** Up to ~500 chars of text that appeared immediately after the image. */
|
||||
private String afterText;
|
||||
|
||||
public boolean hasContext() {
|
||||
return (beforeText != null && !beforeText.isBlank())
|
||||
|| (afterText != null && !afterText.isBlank());
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,35 @@
|
||||
package vip.mate.tool.image.vision;
|
||||
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
|
||||
/**
|
||||
* Input to one image-to-text call.
|
||||
*
|
||||
* <p>The image is supplied as raw bytes plus a MIME type; callers are
|
||||
* responsible for any format conversion before invoking. The optional
|
||||
* {@link VisionContext} carries surrounding text so the provider can
|
||||
* decide whether the image is on-topic and produce a context-aware
|
||||
* caption.
|
||||
*
|
||||
* @author MateClaw Team
|
||||
*/
|
||||
@Data
|
||||
@Builder
|
||||
public class VisionRequest {
|
||||
|
||||
/** Raw image bytes (PNG / JPEG / WebP / ...). Required. */
|
||||
private byte[] imageBytes;
|
||||
|
||||
/** MIME type, e.g. {@code image/png}. Required. */
|
||||
private String mimeType;
|
||||
|
||||
/** Optional surrounding-text context for context-aware captioning. */
|
||||
private VisionContext context;
|
||||
|
||||
/**
|
||||
* Caller hint: when set, ask the provider for this specific model name.
|
||||
* Provider may ignore the hint if the model is unsupported.
|
||||
*/
|
||||
private String preferModel;
|
||||
}
|
||||
@ -0,0 +1,45 @@
|
||||
package vip.mate.tool.image.vision;
|
||||
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
|
||||
import java.time.Instant;
|
||||
|
||||
/**
|
||||
* Outcome of one successful image-to-text call.
|
||||
*
|
||||
* <p>Producers must populate {@link #caption}, {@link #providerId},
|
||||
* {@link #model} and {@link #capturedAt} at minimum. The other fields
|
||||
* are optional and allow callers to render richer UI.
|
||||
*
|
||||
* @author MateClaw Team
|
||||
*/
|
||||
@Data
|
||||
@Builder
|
||||
public class VisionResult {
|
||||
|
||||
/** 2-4 sentence factual description of the image (primary output). */
|
||||
private String caption;
|
||||
|
||||
/** Best-effort recovery of any text rendered inside the image; may be null. */
|
||||
private String visibleText;
|
||||
|
||||
/**
|
||||
* True when the context-aware prompt determined the image is unrelated
|
||||
* to the surrounding text. Caller decides what to do with off-topic
|
||||
* captions (downrank / hide / annotate). False (or null) when context
|
||||
* was not supplied.
|
||||
*/
|
||||
private boolean offTopic;
|
||||
|
||||
/** SPI-registry id of the provider that produced this result. */
|
||||
private String providerId;
|
||||
|
||||
/** Vendor-specific model identifier (e.g. {@code qwen-vl-max}). */
|
||||
private String model;
|
||||
|
||||
private Instant capturedAt;
|
||||
|
||||
/** Wall-clock duration of the round-trip, in milliseconds. */
|
||||
private long durationMs;
|
||||
}
|
||||
@ -0,0 +1,221 @@
|
||||
package vip.mate.tool.image.vision.provider;
|
||||
|
||||
import cn.hutool.http.HttpRequest;
|
||||
import cn.hutool.http.HttpResponse;
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.fasterxml.jackson.databind.node.ArrayNode;
|
||||
import com.fasterxml.jackson.databind.node.ObjectNode;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.stereotype.Component;
|
||||
import vip.mate.agent.prompt.PromptLoader;
|
||||
import vip.mate.exception.MateClawException;
|
||||
import vip.mate.llm.model.ModelProviderEntity;
|
||||
import vip.mate.llm.service.ModelProviderService;
|
||||
import vip.mate.system.model.SystemSettingsDTO;
|
||||
import vip.mate.tool.image.ImageCapability;
|
||||
import vip.mate.tool.image.vision.ImageVisionProvider;
|
||||
import vip.mate.tool.image.vision.VisionContext;
|
||||
import vip.mate.tool.image.vision.VisionRequest;
|
||||
import vip.mate.tool.image.vision.VisionResult;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.util.Base64;
|
||||
import java.util.Set;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
/**
|
||||
* DashScope vision provider — uses qwen-vl-max via the OpenAI-compatible
|
||||
* endpoint at {@code /compatible-mode/v1/chat/completions}.
|
||||
*
|
||||
* <p>Default for the regional production rollout: API keys are typically
|
||||
* available (DASHSCOPE_API_KEY is mandatory for the rest of the platform)
|
||||
* and per-image cost is the lowest of the supported vendors.
|
||||
*
|
||||
* <p>Implementation deliberately uses Hutool {@code HttpRequest} rather
|
||||
* than building a Spring AI {@code ChatClient} — the call shape is a
|
||||
* one-shot request/response that doesn't benefit from the heavyweight
|
||||
* agent-style observability / retry / cache machinery in
|
||||
* {@code AgentOpenAiCompatibleChatModelBuilder}.
|
||||
*
|
||||
* @author MateClaw Team
|
||||
*/
|
||||
@Slf4j
|
||||
@Component
|
||||
@RequiredArgsConstructor
|
||||
public class DashScopeVisionProvider implements ImageVisionProvider {
|
||||
|
||||
private static final String PROVIDER_ID = "dashscope-vision";
|
||||
private static final String DASHSCOPE_PROVIDER_KEY = "dashscope";
|
||||
private static final String DEFAULT_MODEL = "qwen-vl-max";
|
||||
private static final String DEFAULT_BASE_URL = "https://dashscope.aliyuncs.com/compatible-mode/v1";
|
||||
private static final int CALL_TIMEOUT_MS = 60_000;
|
||||
private static final Pattern OFF_TOPIC_PREFIX = Pattern.compile("^\\s*\\[OFF-TOPIC\\]\\s*", Pattern.CASE_INSENSITIVE);
|
||||
|
||||
private final ModelProviderService modelProviderService;
|
||||
private final ObjectMapper objectMapper;
|
||||
|
||||
@Override
|
||||
public String id() {
|
||||
return PROVIDER_ID;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String label() {
|
||||
return "DashScope qwen-vl";
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean requiresCredential() {
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int autoDetectOrder() {
|
||||
return 10;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Set<ImageCapability> capabilities() {
|
||||
return Set.of(ImageCapability.IMAGE_TO_TEXT);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isAvailable(SystemSettingsDTO settings) {
|
||||
try {
|
||||
return modelProviderService.isProviderConfigured(DASHSCOPE_PROVIDER_KEY);
|
||||
} catch (Exception e) {
|
||||
log.debug("[Vision][dashscope] availability check failed: {}", e.getMessage());
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public VisionResult caption(VisionRequest request, SystemSettingsDTO settings) {
|
||||
ModelProviderEntity provider;
|
||||
try {
|
||||
provider = modelProviderService.getProviderConfig(DASHSCOPE_PROVIDER_KEY);
|
||||
} catch (Exception e) {
|
||||
throw new MateClawException("err.wiki.vision.no_provider",
|
||||
"DashScope provider is not configured");
|
||||
}
|
||||
String apiKey = provider.getApiKey();
|
||||
if (apiKey == null || apiKey.isBlank()) {
|
||||
throw new MateClawException("err.wiki.vision.no_provider",
|
||||
"DashScope API key is missing");
|
||||
}
|
||||
String baseUrl = (provider.getBaseUrl() == null || provider.getBaseUrl().isBlank())
|
||||
? DEFAULT_BASE_URL
|
||||
: provider.getBaseUrl();
|
||||
String model = (request.getPreferModel() == null || request.getPreferModel().isBlank())
|
||||
? DEFAULT_MODEL
|
||||
: request.getPreferModel();
|
||||
|
||||
String prompt = buildPrompt(request.getContext());
|
||||
String imageDataUrl = buildDataUrl(request.getMimeType(), request.getImageBytes());
|
||||
|
||||
ObjectNode body = objectMapper.createObjectNode();
|
||||
body.put("model", model);
|
||||
body.put("temperature", 0.0);
|
||||
body.put("max_tokens", 4096);
|
||||
|
||||
ArrayNode messages = body.putArray("messages");
|
||||
ObjectNode userMsg = messages.addObject();
|
||||
userMsg.put("role", "user");
|
||||
ArrayNode content = userMsg.putArray("content");
|
||||
content.addObject().put("type", "text").put("text", prompt);
|
||||
ObjectNode imagePart = content.addObject();
|
||||
imagePart.put("type", "image_url");
|
||||
imagePart.putObject("image_url").put("url", imageDataUrl);
|
||||
|
||||
long startNanos = System.nanoTime();
|
||||
HttpResponse response;
|
||||
try {
|
||||
response = HttpRequest.post(baseUrl + "/chat/completions")
|
||||
.header("Authorization", "Bearer " + apiKey)
|
||||
.header("Content-Type", "application/json")
|
||||
.body(body.toString())
|
||||
.timeout(CALL_TIMEOUT_MS)
|
||||
.execute();
|
||||
} catch (Exception e) {
|
||||
throw new MateClawException("err.wiki.vision.provider_failed",
|
||||
"DashScope vision call failed: " + e.getMessage());
|
||||
}
|
||||
long durationMs = Duration.ofNanos(System.nanoTime() - startNanos).toMillis();
|
||||
|
||||
if (response.getStatus() != 200) {
|
||||
String errBody = response.body();
|
||||
log.warn("[Vision][dashscope] HTTP {} body={}", response.getStatus(), truncate(errBody, 400));
|
||||
throw new MateClawException("err.wiki.vision.provider_failed",
|
||||
"DashScope vision returned HTTP " + response.getStatus());
|
||||
}
|
||||
|
||||
String text;
|
||||
try {
|
||||
JsonNode root = objectMapper.readTree(response.body());
|
||||
text = root.path("choices").path(0).path("message").path("content").asText("").trim();
|
||||
} catch (Exception e) {
|
||||
throw new MateClawException("err.wiki.vision.provider_failed",
|
||||
"DashScope vision response was unparseable: " + e.getMessage());
|
||||
}
|
||||
if (text.isEmpty()) {
|
||||
throw new MateClawException("err.wiki.vision.provider_failed",
|
||||
"DashScope vision returned an empty caption");
|
||||
}
|
||||
|
||||
Matcher offTopic = OFF_TOPIC_PREFIX.matcher(text);
|
||||
boolean isOffTopic = offTopic.find();
|
||||
String caption = isOffTopic ? offTopic.replaceFirst("").trim() : text;
|
||||
|
||||
return VisionResult.builder()
|
||||
.caption(caption)
|
||||
.visibleText(extractVisibleTextHeuristic(caption))
|
||||
.offTopic(isOffTopic)
|
||||
.providerId(PROVIDER_ID)
|
||||
.model(model)
|
||||
.capturedAt(Instant.now())
|
||||
.durationMs(durationMs)
|
||||
.build();
|
||||
}
|
||||
|
||||
private String buildPrompt(VisionContext context) {
|
||||
if (context != null && context.hasContext()) {
|
||||
String template = PromptLoader.loadPrompt("wiki/vision-caption-context-aware");
|
||||
String before = (context.getBeforeText() == null || context.getBeforeText().isBlank())
|
||||
? "(none)" : context.getBeforeText().trim();
|
||||
String after = (context.getAfterText() == null || context.getAfterText().isBlank())
|
||||
? "(none)" : context.getAfterText().trim();
|
||||
return template.replace("{before}", before).replace("{after}", after);
|
||||
}
|
||||
return PromptLoader.loadPrompt("wiki/vision-caption-factual");
|
||||
}
|
||||
|
||||
private static String buildDataUrl(String mimeType, byte[] bytes) {
|
||||
String mime = (mimeType == null || mimeType.isBlank()) ? "image/png" : mimeType;
|
||||
return "data:" + mime + ";base64," + Base64.getEncoder().encodeToString(bytes);
|
||||
}
|
||||
|
||||
/**
|
||||
* Heuristic: pull obvious quoted strings as "visible text". This is a
|
||||
* placeholder until a stronger OCR-grade extraction lands; for the
|
||||
* factual prompt the model already inlines text verbatim into the
|
||||
* caption, so the cache row's caption alone is enough for retrieval.
|
||||
*/
|
||||
private static String extractVisibleTextHeuristic(String caption) {
|
||||
Matcher m = Pattern.compile("\"([^\"]{3,})\"").matcher(caption);
|
||||
StringBuilder sb = new StringBuilder();
|
||||
while (m.find()) {
|
||||
if (sb.length() > 0) sb.append('\n');
|
||||
sb.append(m.group(1));
|
||||
}
|
||||
return sb.length() == 0 ? null : sb.toString();
|
||||
}
|
||||
|
||||
private static String truncate(String s, int max) {
|
||||
if (s == null) return "";
|
||||
return s.length() > max ? s.substring(0, max) + "…" : s;
|
||||
}
|
||||
}
|
||||
@ -273,3 +273,9 @@ agent.limit_exceeded.empty_context=\uff08\u5c1a\u672a\u6536\u96c6\u5230\u5de5\u5
|
||||
cron.tasks_conversation.title=📋 定时任务
|
||||
cron.run_header.scheduled=定时触发
|
||||
cron.run_header.manual=手动触发
|
||||
|
||||
# --- Wiki vision-in pipeline ---
|
||||
err.wiki.vision.disabled=图片识别功能未启用
|
||||
err.wiki.vision.no_provider=未配置可用的图片识别 provider
|
||||
err.wiki.vision.provider_failed=图片识别 provider 调用失败
|
||||
err.wiki.vision.all_failed=所有图片识别 provider 调用失败
|
||||
|
||||
@ -280,3 +280,9 @@ agent.limit_exceeded.empty_context=(No tool call results collected yet.)
|
||||
cron.tasks_conversation.title=📋 Scheduled Tasks
|
||||
cron.run_header.scheduled=scheduled
|
||||
cron.run_header.manual=manual
|
||||
|
||||
# --- Wiki vision-in pipeline ---
|
||||
err.wiki.vision.disabled=Image vision pipeline is currently disabled
|
||||
err.wiki.vision.no_provider=No image vision provider is configured
|
||||
err.wiki.vision.provider_failed=Image vision provider call failed
|
||||
err.wiki.vision.all_failed=All image vision providers failed
|
||||
|
||||
@ -0,0 +1,11 @@
|
||||
Describe this image factually for a knowledge-base index. Surrounding text below tells you what topic the image relates to.
|
||||
|
||||
--- Text before image ---
|
||||
{before}
|
||||
|
||||
--- Text after image ---
|
||||
{after}
|
||||
|
||||
--- End surrounding text ---
|
||||
|
||||
If the image is clearly NOT relevant to the surrounding text, prepend "[OFF-TOPIC] " to your response. Otherwise, describe factually: visible text verbatim, chart axes and values, diagram structure, key visual elements. 2 to 4 sentences. Plain text only — no markdown, no preamble.
|
||||
@ -0,0 +1 @@
|
||||
Describe this image factually for a knowledge-base index. Include: any visible text verbatim, chart axes and values, diagram structure (boxes/arrows/labels), key visual elements. Do NOT speculate or editorialize. 2 to 4 sentences. Output plain text only — no markdown, no preamble.
|
||||
Loading…
Reference in New Issue
Block a user