diff --git a/mateclaw-server/src/main/java/vip/mate/tool/builtin/ImageGenerateTool.java b/mateclaw-server/src/main/java/vip/mate/tool/builtin/ImageGenerateTool.java index a15c4538..7041bf78 100644 --- a/mateclaw-server/src/main/java/vip/mate/tool/builtin/ImageGenerateTool.java +++ b/mateclaw-server/src/main/java/vip/mate/tool/builtin/ImageGenerateTool.java @@ -13,6 +13,7 @@ import vip.mate.task.AsyncTaskService; import vip.mate.task.model.AsyncTaskInfo; import vip.mate.tool.image.*; +import java.util.ArrayList; import java.util.List; import java.util.StringJoiner; @@ -30,19 +31,23 @@ public class ImageGenerateTool { private final ImageProviderRegistry providerRegistry; private final SystemSettingService systemSettingService; private final AsyncTaskService asyncTaskService; + private final ImageReferenceLoader imageReferenceLoader; @vip.mate.tool.ConcurrencyUnsafe("creates async tasks and persists generated artifacts; provider rate limits also forbid parallel calls") - @Tool(description = "Image generation tool. Supports actions: generate (default), list (show available providers), " - + "status (check task status). Some providers are async (30s-2min), results auto-displayed in conversation.") + @Tool(description = "Image generation tool. Supports actions: generate (default — text-to-image, OR image-edit when " + + "image/images parameters are set), list (show available providers/models), status (check task status). " + + "Reference images may be local paths, http(s) URLs, data: URLs, or msg:: for an attachment " + + "from an earlier conversation message. Async providers take 30s-2min; results auto-display in the conversation.") public String image_generate( @ToolParam(description = "Action type: generate, list, status. Default: generate", required = false) String action, @ToolParam(description = "Image content description, be detailed (required for generate)", required = false) String prompt, + @ToolParam(description = "Single reference image for edit mode. Path / http(s) URL / data: URL / msg:[:]", required = false) String image, + @ToolParam(description = "Multiple reference images for edit mode (provider caps the count). Same formats as 'image'.", required = false) List images, @ToolParam(description = "Image size: 1024x1024 / 1024x1792 / 1792x1024", required = false) String size, @ToolParam(description = "Aspect ratio: 1:1 / 16:9 / 9:16, default 1:1", required = false) String aspectRatio, @ToolParam(description = "Generation count (1-4), default 1", required = false) Integer count, @ToolParam(description = "Model name (optional)", required = false) String model, @ToolParam(description = "Task ID to check status (for status action)", required = false) String taskId, - // RFC-063r §2.5: ToolContext is hidden from the LLM by JsonSchemaGenerator. @Nullable ToolContext ctx ) { String normalizedAction = (action == null || action.isBlank()) ? "generate" : action.trim().toLowerCase(); @@ -50,7 +55,7 @@ public class ImageGenerateTool { return switch (normalizedAction) { case "list" -> handleListAction(); case "status" -> handleStatusAction(taskId, ctx); - default -> handleGenerateAction(prompt, size, aspectRatio, count, model, ctx); + default -> handleGenerateAction(prompt, image, images, size, aspectRatio, count, model, ctx); }; } @@ -120,7 +125,8 @@ public class ImageGenerateTool { // ==================== action=generate ==================== - private String handleGenerateAction(String prompt, String size, String aspectRatio, + private String handleGenerateAction(String prompt, String image, List images, + String size, String aspectRatio, Integer count, String model, @Nullable ToolContext ctx) { String conversationId = ToolExecutionContext.conversationId(ctx); String username = ToolExecutionContext.username(ctx); @@ -133,12 +139,33 @@ public class ImageGenerateTool { return "错误：prompt 为必填参数，请描述你想要生成的图片内容"; } + // Combine the singular and plural forms — the agent picks whichever is + // ergonomic. Order: image (first) then images[]. + List referenceInputs = new ArrayList<>(); + if (image != null && !image.isBlank()) { + referenceInputs.add(image); + } + if (images != null) { + for (String s : images) { + if (s != null && !s.isBlank()) referenceInputs.add(s); + } + } + + List inputImages; + try { + inputImages = imageReferenceLoader.loadAll(referenceInputs, conversationId); + } catch (Exception e) { + log.warn("[ImageGenerateTool] Failed to load reference images: {}", e.getMessage()); + return "错误：无法加载参考图片：" + e.getMessage(); + } + ImageGenerationRequest request = ImageGenerationRequest.builder() .prompt(prompt) .size(size) .aspectRatio(aspectRatio != null ? aspectRatio : "1:1") .count(count != null ? count : 1) .model(model) + .inputImages(inputImages) .build(); ImageGenerationResult result = imageGenerationService.submitGeneration( diff --git a/mateclaw-server/src/main/java/vip/mate/tool/image/ImageGenerationRequest.java b/mateclaw-server/src/main/java/vip/mate/tool/image/ImageGenerationRequest.java index 752ddb6a..07f982e3 100644 --- a/mateclaw-server/src/main/java/vip/mate/tool/image/ImageGenerationRequest.java +++ b/mateclaw-server/src/main/java/vip/mate/tool/image/ImageGenerationRequest.java @@ -3,10 +3,11 @@ package vip.mate.tool.image; import lombok.Builder; import lombok.Data; +import java.util.List; import java.util.Map; /** - * 图片生成统一请求 + * Unified image-generation request. * * @author MateClaw Team */ @@ -14,30 +15,36 @@ import java.util.Map; @Builder public class ImageGenerationRequest { - /** 图片内容描述 */ + /** Prompt describing the desired image. */ private String prompt; - /** 生成模式（由 runtime 自动推断） */ + /** Generation mode (inferred by the runtime when null). */ private ImageCapability mode; - /** 指定模型名称（可选，provider 有默认值） */ + /** Model id; provider supplies a default when null/blank. */ private String model; - /** 图片尺寸：1024x1024 / 1024x1792 / 1792x1024 等 */ + /** Pixel size like {@code 1024x1024} / {@code 1024x1792}. */ @Builder.Default private String size = "1024x1024"; - /** 画面比例：1:1 / 16:9 / 9:16 */ + /** Aspect ratio: {@code 1:1} / {@code 16:9} / {@code 9:16}. */ @Builder.Default private String aspectRatio = "1:1"; - /** 生成数量 */ + /** Number of images to return. */ @Builder.Default private Integer count = 1; - /** 参考图片 URL（IMAGE_EDIT 模式） */ - private String referenceImageUrl; + /** + * Reference images for edit / image-to-image flows. Loaded as in-memory + * buffers so providers can either inline base64, upload via multipart, or + * forward as a URL — without each provider re-implementing path/URL/data + * resolution. + */ + @Builder.Default + private List inputImages = List.of(); - /** provider 特有的额外参数 */ + /** Provider-specific extras forwarded as-is. */ private Map extraParams; } diff --git a/mateclaw-server/src/main/java/vip/mate/tool/image/ImageGenerationService.java b/mateclaw-server/src/main/java/vip/mate/tool/image/ImageGenerationService.java index cbb76cb8..2098daed 100644 --- a/mateclaw-server/src/main/java/vip/mate/tool/image/ImageGenerationService.java +++ b/mateclaw-server/src/main/java/vip/mate/tool/image/ImageGenerationService.java @@ -279,7 +279,7 @@ public class ImageGenerationService { } private ImageCapability inferMode(ImageGenerationRequest request) { - if (request.getReferenceImageUrl() != null && !request.getReferenceImageUrl().isBlank()) { + if (request.getInputImages() != null && !request.getInputImages().isEmpty()) { return ImageCapability.IMAGE_EDIT; } return ImageCapability.TEXT_TO_IMAGE; diff --git a/mateclaw-server/src/main/java/vip/mate/tool/image/ImageModelSpec.java b/mateclaw-server/src/main/java/vip/mate/tool/image/ImageModelSpec.java new file mode 100644 index 00000000..e4a92469 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/image/ImageModelSpec.java @@ -0,0 +1,57 @@ +package vip.mate.tool.image; + +import lombok.Builder; +import lombok.Singular; + +import java.util.Map; +import java.util.Set; + +/** + * Per-model descriptor that drives payload construction without {@code if/else} + * chains inside provider classes. Adding a new model = adding a new spec entry. + * + *

Three things make this configuration-driven: + *

    + *
  • {@code endpoint} chooses which provider URL to hit. A single provider + * (e.g. DashScope) can host both an async legacy endpoint and a unified + * multimodal endpoint — the spec routes per model.
    • + *
  • {@code transport} ({@link Transport#SYNC} / {@link Transport#ASYNC}) + * lets the provider pick between immediate-return and submit+poll without + * hard-coding the choice.
    • + *
  • {@code supports} acts as a payload key whitelist. Build the full payload + * freely, then filter against {@code supports} so models never receive + * fields they reject.
    • + *
+ * + * @author MateClaw Team + */ +@Builder +public record ImageModelSpec( + String id, + String displayName, + String endpoint, + Transport transport, + SizeStyle sizeStyle, + @Singular("sizeMapping") Map sizeMap, + @Singular("defaultParam") Map defaults, + @Singular Set supports, + @Singular Set modes, + int maxInputImages, + int maxCount +) { + + public enum Transport { + /** Provider returns image bytes / URL in the same HTTP response. */ + SYNC, + /** Provider returns a task id; caller polls a status endpoint. */ + ASYNC + } + + public boolean supportsEdit() { + return modes != null && modes.contains(ImageCapability.IMAGE_EDIT); + } + + public boolean supportsGenerate() { + return modes != null && modes.contains(ImageCapability.TEXT_TO_IMAGE); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/tool/image/ImageProviderCapabilities.java b/mateclaw-server/src/main/java/vip/mate/tool/image/ImageProviderCapabilities.java index 5d79124b..cdec5780 100644 --- a/mateclaw-server/src/main/java/vip/mate/tool/image/ImageProviderCapabilities.java +++ b/mateclaw-server/src/main/java/vip/mate/tool/image/ImageProviderCapabilities.java @@ -7,7 +7,15 @@ import java.util.List; import java.util.Set; /** - * 图片生成 Provider 细粒度能力声明 + * Image generation provider capability declaration. + * + *

The flat top-level fields ({@code supportedSizes}, {@code aspectRatios}, + * {@code maxCount}, {@code modes}) describe the provider's combined surface + * area and remain in use by callers that don't need per-mode granularity. + * Newer code should consult the structured {@link Generate} / {@link Edit} / + * {@link Geometry} / {@link Output} fields, which let the picker show + * "edit supports up to N reference images" or "generate accepts these + * formats" without conflating the two modes. * * @author MateClaw Team */ @@ -15,29 +23,87 @@ import java.util.Set; @Builder public class ImageProviderCapabilities { - /** 支持的生成模式 */ + /** Combined modes the provider supports across all its models. */ @Builder.Default private Set modes = Set.of(ImageCapability.TEXT_TO_IMAGE); - /** 支持的图片尺寸，如 ["1024x1024", "1024x1792"] */ + /** Union of pixel sizes accepted by any model under this provider. */ @Builder.Default private List supportedSizes = List.of("1024x1024"); - /** 支持的画面比例 */ + /** Union of aspect ratio presets accepted by any model under this provider. */ @Builder.Default private List aspectRatios = List.of("1:1", "16:9", "9:16"); - /** 最大生成数量 */ + /** Largest {@code n} (image count) any model under this provider accepts. */ @Builder.Default private int maxCount = 1; - /** 默认模型 */ + /** Default model id. */ private String defaultModel; - /** 可用模型列表 */ + /** All callable model ids. */ @Builder.Default private List models = List.of(); + /** Per-mode generate capabilities. Optional — falls back to flat fields when absent. */ + private Generate generate; + + /** Per-mode edit capabilities. {@code null} or {@code enabled=false} means edits unsupported. */ + private Edit edit; + + /** Geometry surface (sizes / aspect ratios). Optional. */ + private Geometry geometry; + + /** Output knobs (formats, qualities, backgrounds). Optional. */ + private Output output; + + @Data + @Builder + public static class Generate { + @Builder.Default + private int maxCount = 1; + @Builder.Default + private boolean supportsSize = true; + @Builder.Default + private boolean supportsAspectRatio = true; + } + + @Data + @Builder + public static class Edit { + @Builder.Default + private boolean enabled = false; + @Builder.Default + private int maxCount = 1; + @Builder.Default + private int maxInputImages = 1; + @Builder.Default + private boolean supportsSize = true; + @Builder.Default + private boolean supportsAspectRatio = true; + } + + @Data + @Builder + public static class Geometry { + @Builder.Default + private List sizes = List.of(); + @Builder.Default + private List aspectRatios = List.of(); + } + + @Data + @Builder + public static class Output { + @Builder.Default + private List formats = List.of(); + @Builder.Default + private List qualities = List.of(); + @Builder.Default + private List backgrounds = List.of(); + } + /** * Match the requested size against supported sizes by area only. * Orientation-blind — prefer {@link #normalizeSize(String, String)} when an diff --git a/mateclaw-server/src/main/java/vip/mate/tool/image/ImageReference.java b/mateclaw-server/src/main/java/vip/mate/tool/image/ImageReference.java new file mode 100644 index 00000000..87b5adec --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/image/ImageReference.java @@ -0,0 +1,21 @@ +package vip.mate.tool.image; + +/** + * In-memory image reference used for image-edit / image-to-image generation requests. + *

+ * The loader normalizes any of the agent-facing input forms (local paths, http(s) + * URLs, {@code data:} URLs, conversation message refs) into this single shape so + * providers receive bytes, mime type, and file name regardless of origin. + * + * @param data raw image bytes + * @param mimeType e.g. {@code image/png} + * @param fileName logical name (best-effort, may be synthesized) + * @param origin trace string identifying where the bytes came from + * ({@code path:/x.png}, {@code url:https://...}, {@code data-url}, + * {@code msg::}). Used for logging / audit, not + * forwarded to providers. + * + * @author MateClaw Team + */ +public record ImageReference(byte[] data, String mimeType, String fileName, String origin) { +} diff --git a/mateclaw-server/src/main/java/vip/mate/tool/image/ImageReferenceLoader.java b/mateclaw-server/src/main/java/vip/mate/tool/image/ImageReferenceLoader.java new file mode 100644 index 00000000..613d23ab --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/image/ImageReferenceLoader.java @@ -0,0 +1,297 @@ +package vip.mate.tool.image; + +import cn.hutool.http.HttpUtil; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import vip.mate.workspace.conversation.ConversationService; +import vip.mate.workspace.conversation.model.MessageContentPart; +import vip.mate.workspace.conversation.model.MessageEntity; + +import java.io.IOException; +import java.net.URI; +import java.net.URLDecoder; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Base64; +import java.util.List; + +/** + * Resolves agent-supplied image reference strings into in-memory + * {@link ImageReference} buffers. Five input forms are accepted: + * + *

    + *
  1. Local filesystem path: {@code /abs/path.png}, {@code ./rel.png}, + * {@code ~/x.png}, or {@code file://...}.
    2. + *
  2. Data URL: {@code data:image/png;base64,...} (base64 or URL-encoded body).
    3. + *
  3. HTTP(S) URL: downloaded with size + content-type guard.
    4. + *
  4. Conversation message reference: {@code msg::} — + * resolves to the local path stored on a {@link MessageContentPart} of + * type {@code image} on the named message. This is the channel an agent + * uses to forward a user-uploaded image into the image edit tool, so a + * non-vision model can still operate on attachments it cannot "see".
    5. + *
  5. Workspace-relative path: passed through as a regular path; the caller + * is expected to anchor it to the active workspace before invocation.
    6. + *
+ * + * @author MateClaw Team + */ +@Slf4j +@Component +@RequiredArgsConstructor +public class ImageReferenceLoader { + + private static final long MAX_REFERENCE_BYTES = 20L * 1024 * 1024; + private static final int HTTP_TIMEOUT_MS = 30_000; + + private final ConversationService conversationService; + + /** + * Resolve a list of input strings; null / blank entries are skipped. + * The caller is expected to enforce per-provider {@code maxInputImages} + * before calling. + */ + public List loadAll(List inputs, String conversationId) throws IOException { + if (inputs == null || inputs.isEmpty()) { + return List.of(); + } + List out = new ArrayList<>(inputs.size()); + for (String raw : inputs) { + if (raw == null || raw.isBlank()) { + continue; + } + out.add(load(raw.trim(), conversationId)); + } + return out; + } + + /** Resolve a single reference string. */ + public ImageReference load(String input, String conversationId) throws IOException { + if (input == null || input.isBlank()) { + throw new IOException("image reference is blank"); + } + String trimmed = input.trim(); + + if (trimmed.startsWith("data:")) { + return loadDataUrl(trimmed); + } + if (trimmed.startsWith("msg:")) { + return loadConversationMessageRef(trimmed, conversationId); + } + if (trimmed.startsWith("http://") || trimmed.startsWith("https://")) { + return loadHttpUrl(trimmed); + } + return loadFilePath(trimmed); + } + + // ==================== form: local path / file:// ==================== + + private ImageReference loadFilePath(String input) throws IOException { + String pathStr = input.startsWith("file://") ? input.substring("file://".length()) : input; + if (pathStr.startsWith("~")) { + pathStr = System.getProperty("user.home") + pathStr.substring(1); + } + Path p = Paths.get(pathStr); + if (!Files.exists(p)) { + throw new IOException("Image file not found: " + pathStr); + } + if (Files.size(p) > MAX_REFERENCE_BYTES) { + throw new IOException("Image exceeds 20MB limit: " + pathStr); + } + byte[] data = Files.readAllBytes(p); + String mime = inferMimeFromName(p.getFileName().toString()); + return new ImageReference(data, mime, p.getFileName().toString(), "path:" + p); + } + + // ==================== form: data: URL ==================== + + private ImageReference loadDataUrl(String dataUrl) throws IOException { + int comma = dataUrl.indexOf(','); + if (comma < 0) { + throw new IOException("Malformed data URL: missing comma"); + } + String header = dataUrl.substring("data:".length(), comma); + String body = dataUrl.substring(comma + 1); + boolean isBase64 = header.toLowerCase().contains(";base64"); + String mime = isBase64 + ? header.substring(0, header.toLowerCase().indexOf(";base64")) + : (header.contains(";") ? header.substring(0, header.indexOf(';')) : header); + if (mime == null || mime.isBlank()) { + mime = "image/png"; + } + byte[] data; + try { + data = isBase64 + ? Base64.getDecoder().decode(body) + : URLDecoder.decode(body, StandardCharsets.UTF_8).getBytes(StandardCharsets.UTF_8); + } catch (IllegalArgumentException e) { + throw new IOException("Invalid base64 in data URL: " + e.getMessage(), e); + } + if (data.length > MAX_REFERENCE_BYTES) { + throw new IOException("Image exceeds 20MB limit (data URL)"); + } + return new ImageReference(data, mime, "inline." + extensionFor(mime), "data-url"); + } + + // ==================== form: http(s) URL ==================== + + private ImageReference loadHttpUrl(String url) throws IOException { + URI uri = URI.create(url); + String host = uri.getHost(); + if (host == null) { + throw new IOException("URL has no host: " + url); + } + // Conservative SSRF guard: reject obvious internal targets. Refine later + // if the project gains a dedicated SsrFPolicy module. + String lowered = host.toLowerCase(); + if (lowered.equals("localhost") + || lowered.equals("127.0.0.1") + || lowered.startsWith("10.") + || lowered.startsWith("192.168.") + || lowered.startsWith("169.254.") + || lowered.startsWith("172.")) { + throw new IOException("Refusing to download image from internal host: " + host); + } + try { + byte[] data = HttpUtil.createGet(url).timeout(HTTP_TIMEOUT_MS).execute().bodyBytes(); + if (data == null || data.length == 0) { + throw new IOException("Empty response downloading image from " + url); + } + if (data.length > MAX_REFERENCE_BYTES) { + throw new IOException("Image exceeds 20MB limit: " + url); + } + String fileName = guessFileNameFromUrl(url); + String mime = inferMimeFromName(fileName); + return new ImageReference(data, mime, fileName, "url:" + url); + } catch (Exception e) { + throw new IOException("Failed to download image " + url + ": " + e.getMessage(), e); + } + } + + // ==================== form: msg:: ==================== + + private ImageReference loadConversationMessageRef(String ref, String conversationId) throws IOException { + // ref shape: "msg:" (first image part) or "msg::" + String body = ref.substring("msg:".length()); + String[] parts = body.split(":", 2); + long messageId; + try { + messageId = Long.parseLong(parts[0]); + } catch (NumberFormatException e) { + throw new IOException("Invalid msg: ref, expected msg:[:]: " + ref); + } + Integer wantedIdx = null; + if (parts.length == 2 && !parts[1].isBlank()) { + try { + wantedIdx = Integer.parseInt(parts[1]); + } catch (NumberFormatException e) { + throw new IOException("Invalid part index in: " + ref); + } + } + if (conversationId == null || conversationId.isBlank()) { + throw new IOException("Cannot resolve msg: reference without an active conversation"); + } + MessageEntity message = findMessageInConversation(conversationId, messageId); + if (message == null) { + throw new IOException("Message " + messageId + " not found in conversation " + conversationId); + } + List contentParts = conversationService.parseMessageParts(message); + MessageContentPart picked = pickImagePart(contentParts, wantedIdx); + if (picked == null) { + throw new IOException("No image part on message " + messageId + + (wantedIdx != null ? " at index " + wantedIdx : "")); + } + Path filePath = resolveLocalPath(picked); + if (filePath == null) { + throw new IOException("Message " + messageId + " image part has no local path: " + + picked.getFileName()); + } + if (Files.size(filePath) > MAX_REFERENCE_BYTES) { + throw new IOException("Image exceeds 20MB limit: " + filePath); + } + byte[] data = Files.readAllBytes(filePath); + String mime = picked.getContentType(); + if (mime == null || mime.isBlank() || "image/*".equals(mime)) { + mime = inferMimeFromName(picked.getFileName()); + } + String fileName = picked.getFileName() != null ? picked.getFileName() : filePath.getFileName().toString(); + return new ImageReference(data, mime, fileName, ref); + } + + private MessageEntity findMessageInConversation(String conversationId, long messageId) { + List all = conversationService.listMessages(conversationId); + for (MessageEntity m : all) { + if (m.getId() != null && m.getId() == messageId) { + return m; + } + } + return null; + } + + private MessageContentPart pickImagePart(List parts, Integer wantedIdx) { + if (parts == null || parts.isEmpty()) { + return null; + } + if (wantedIdx != null) { + int seen = 0; + for (MessageContentPart p : parts) { + if (p == null || !"image".equals(p.getType())) continue; + if (seen == wantedIdx) { + return p; + } + seen++; + } + return null; + } + for (MessageContentPart p : parts) { + if (p != null && "image".equals(p.getType())) { + return p; + } + } + return null; + } + + private Path resolveLocalPath(MessageContentPart part) { + if (part.getPath() != null && !part.getPath().isBlank()) { + Path p = Paths.get(part.getPath()); + if (Files.exists(p)) return p; + } + if (part.getStoredName() != null && !part.getStoredName().isBlank()) { + Path p = Paths.get(part.getStoredName()); + if (Files.exists(p)) return p; + } + return null; + } + + // ==================== shared helpers ==================== + + private static String inferMimeFromName(String name) { + if (name == null) return "image/png"; + String lower = name.toLowerCase(); + if (lower.endsWith(".jpg") || lower.endsWith(".jpeg")) return "image/jpeg"; + if (lower.endsWith(".webp")) return "image/webp"; + if (lower.endsWith(".gif")) return "image/gif"; + if (lower.endsWith(".bmp")) return "image/bmp"; + return "image/png"; + } + + private static String extensionFor(String mime) { + return switch (mime.toLowerCase().trim()) { + case "image/jpeg", "image/jpg" -> "jpg"; + case "image/webp" -> "webp"; + case "image/gif" -> "gif"; + case "image/bmp" -> "bmp"; + default -> "png"; + }; + } + + private static String guessFileNameFromUrl(String url) { + String stripped = url.split("\\?", 2)[0]; + int slash = stripped.lastIndexOf('/'); + String tail = slash >= 0 ? stripped.substring(slash + 1) : stripped; + return tail.isBlank() ? "remote.png" : tail; + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/tool/image/PayloadBuilder.java b/mateclaw-server/src/main/java/vip/mate/tool/image/PayloadBuilder.java new file mode 100644 index 00000000..a69ee163 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/image/PayloadBuilder.java @@ -0,0 +1,172 @@ +package vip.mate.tool.image; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; + +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Set; + +/** + * Configuration-driven payload builder for image-generation providers. + * + *

Without this, every provider class collects an {@code if/else} chain + * mapping (model id) → (request shape, sizing dialect, available knobs). Each + * new model in a family forces another branch. With it, each provider holds a + * static {@code Map}; the builder consults that spec + * for sizing dialect, default parameters, and a {@code supports} whitelist of + * payload keys. Values not in the whitelist are dropped at the end so the API + * never sees keys it would reject. + * + *

Sizing dialect handling: + *

    + *
  • {@link SizeStyle#LITERAL_DIMENSION} — output {@code "1024x1024"} or + * a separator-replaced form (e.g. DashScope wants {@code "1024*1024"} — + * the spec's sizeMap can carry the alternative).
    • + *
  • {@link SizeStyle#ASPECT_RATIO} — output {@code "1:1"} / {@code "16:9"}.
    • + *
  • {@link SizeStyle#PRESET_NAME} — output the model-native preset + * (e.g. {@code square_hd}). The spec's sizeMap drives the lookup keyed + * by orientation token (landscape / square / portrait).
    • + *
+ * + * @author MateClaw Team + */ +public final class PayloadBuilder { + + private final ImageModelSpec spec; + private final Map entries = new LinkedHashMap<>(); + + private PayloadBuilder(ImageModelSpec spec) { + this.spec = spec; + if (spec.defaults() != null) { + entries.putAll(spec.defaults()); + } + } + + public static PayloadBuilder from(ImageModelSpec spec) { + return new PayloadBuilder(spec); + } + + public PayloadBuilder withPrompt(String prompt) { + if (prompt != null) { + entries.put("prompt", prompt); + } + return this; + } + + public PayloadBuilder withCount(Integer count) { + if (count != null && count > 0) { + entries.put("n", Math.min(count, Math.max(1, spec.maxCount() == 0 ? count : spec.maxCount()))); + } + return this; + } + + /** + * Translate the unified {@code size} / {@code aspectRatio} inputs to whichever + * key/value pair this model expects. The spec's {@link SizeStyle} drives + * which key is set; the spec's sizeMap (orientation → native value) drives + * the value when the caller did not pass an exact match. + */ + public PayloadBuilder withSize(String requestedSize, String requestedAspectRatio) { + SizeStyle style = spec.sizeStyle(); + if (style == null) { + return this; + } + Map sizeMap = spec.sizeMap(); + switch (style) { + case LITERAL_DIMENSION -> entries.put("size", + resolveLiteralDimension(requestedSize, requestedAspectRatio, sizeMap)); + case ASPECT_RATIO -> entries.put("aspect_ratio", + resolveAspectRatio(requestedAspectRatio, sizeMap)); + case PRESET_NAME -> entries.put("image_size", + resolvePreset(requestedAspectRatio, sizeMap)); + } + return this; + } + + public PayloadBuilder withSeed(Integer seed) { + if (seed != null) { + entries.put("seed", seed); + } + return this; + } + + public PayloadBuilder put(String key, Object value) { + if (value != null) { + entries.put(key, value); + } + return this; + } + + /** + * Produce a Jackson {@link ObjectNode} containing only the keys this model's + * {@code supports} whitelist allows. Empty whitelist means "passthrough". + */ + public ObjectNode toJsonNode(ObjectMapper mapper) { + ObjectNode out = mapper.createObjectNode(); + Set supports = spec.supports(); + boolean filter = supports != null && !supports.isEmpty(); + for (Map.Entry e : entries.entrySet()) { + if (filter && !supports.contains(e.getKey())) { + continue; + } + out.set(e.getKey(), mapper.valueToTree(e.getValue())); + } + return out; + } + + /** Read-only view of accumulated entries (post defaults / pre supports filter). */ + public Map entries() { + return Map.copyOf(entries); + } + + // ==================== size resolution ==================== + + private String resolveLiteralDimension(String requestedSize, String aspectRatio, + Map sizeMap) { + if (requestedSize != null && !requestedSize.isBlank()) { + // Allow the spec's sizeMap to translate (e.g. "1024x1024" -> "1024*1024"). + String mapped = sizeMap == null ? null : sizeMap.get(requestedSize); + return mapped != null ? mapped : requestedSize; + } + String orientation = orientationOf(aspectRatio); + if (sizeMap != null && sizeMap.containsKey(orientation)) { + return sizeMap.get(orientation); + } + return "1024x1024"; + } + + private String resolveAspectRatio(String requested, Map sizeMap) { + if (requested != null && !requested.isBlank()) { + String mapped = sizeMap == null ? null : sizeMap.get(requested); + return mapped != null ? mapped : requested; + } + return "1:1"; + } + + private String resolvePreset(String aspectRatio, Map sizeMap) { + String orientation = orientationOf(aspectRatio); + if (sizeMap != null && sizeMap.containsKey(orientation)) { + return sizeMap.get(orientation); + } + return "square_hd"; + } + + private static String orientationOf(String aspectRatio) { + if (aspectRatio == null || aspectRatio.isBlank()) { + return "square"; + } + String[] parts = aspectRatio.split(":"); + if (parts.length != 2) { + return "square"; + } + try { + double w = Double.parseDouble(parts[0].trim()); + double h = Double.parseDouble(parts[1].trim()); + if (w == h) return "square"; + return w > h ? "landscape" : "portrait"; + } catch (NumberFormatException e) { + return "square"; + } + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/tool/image/SizeStyle.java b/mateclaw-server/src/main/java/vip/mate/tool/image/SizeStyle.java new file mode 100644 index 00000000..3b2247c9 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/image/SizeStyle.java @@ -0,0 +1,27 @@ +package vip.mate.tool.image; + +/** + * Describes how a particular image-generation model expects its size to be + * expressed. Three families cover all current providers: + * + *
    + *
  • {@link #LITERAL_DIMENSION} — explicit width/height string ({@code 1024x1024}, + * {@code 1536*1024}). Used by DashScope, OpenAI DALL-E, MiniMax.
    • + *
  • {@link #ASPECT_RATIO} — preset enum like {@code 16:9} or {@code 1:1}. + * Used by Gemini / nano-banana style APIs.
    • + *
  • {@link #PRESET_NAME} — provider-specific preset label + * ({@code square_hd}, {@code landscape_16_9}). Used by fal.ai's flux, + * z-image, qwen-image families.
    • + *
+ * + * Each {@link ImageModelSpec} declares one style and provides the sizeMap that + * translates the unified {@code aspectRatio} input ({@code landscape} / + * {@code square} / {@code portrait} or a literal ratio) to the model-native form. + * + * @author MateClaw Team + */ +public enum SizeStyle { + LITERAL_DIMENSION, + ASPECT_RATIO, + PRESET_NAME +} diff --git a/mateclaw-server/src/main/java/vip/mate/tool/image/provider/DashScopeImageModels.java b/mateclaw-server/src/main/java/vip/mate/tool/image/provider/DashScopeImageModels.java new file mode 100644 index 00000000..d010cd22 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/image/provider/DashScopeImageModels.java @@ -0,0 +1,220 @@ +package vip.mate.tool.image.provider; + +import vip.mate.tool.image.ImageCapability; +import vip.mate.tool.image.ImageModelSpec; +import vip.mate.tool.image.SizeStyle; + +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Set; + +/** + * Catalog of DashScope-served image generation / editing models, organised so + * that adding a new model is a one-line spec entry. + * + *

Two transport families are present: + *

    + *
  • Async legacy ({@link #LEGACY_ASYNC_ENDPOINT}) — wanx 2.0/2.1 and + * wan 2.2/2.5 turbo/plus models that exclusively do text-to-image. The + * caller submits and polls {@code /api/v1/tasks/{id}}.
    • + *
  • Sync multimodal ({@link #MULTIMODAL_ENDPOINT}) — wan 2.6/2.7, + * qwen-image, qwen-image-edit, z-image. Uses the OpenAI-style + * {@code messages.content[]} array and returns the generated image URL + * in the same response.
    • + *
+ * + * @author MateClaw Team + */ +final class DashScopeImageModels { + + static final String LEGACY_ASYNC_ENDPOINT = + "https://dashscope.aliyuncs.com/api/v1/services/aigc/image-generation/generation"; + static final String MULTIMODAL_ENDPOINT = + "https://dashscope.aliyuncs.com/api/v1/services/aigc/multimodal-generation/generation"; + static final String TASKS_ENDPOINT_PREFIX = + "https://dashscope.aliyuncs.com/api/v1/tasks/"; + + /** + * Default model when the request does not name one. + * + *

Kept on the legacy turbo so existing accounts that have not enrolled in + * the newer wan/qwen-image families do not see breakage. Callers that want + * edit support must name a model explicitly (e.g. {@code wan2.7-image} or + * {@code qwen-image-edit}) — the registry's edit-capability resolution then + * routes correctly. + */ + static final String DEFAULT_MODEL = "wanx2.1-t2i-turbo"; + + /** + * Default model when an edit-capable spec is required but the request did + * not name one. Used by the provider when the request carries + * {@code inputImages} but the named model lacks {@link ImageCapability#IMAGE_EDIT}. + */ + static final String DEFAULT_EDIT_MODEL = "wan2.7-image"; + + private static final Map ASPECT_LITERAL_SIZES = Map.of( + "1:1", "1024x1024", + "16:9", "1280x720", + "9:16", "720x1280", + "landscape", "1280x720", + "square", "1024x1024", + "portrait", "720x1280" + ); + + private static final Map ASPECT_LITERAL_SIZES_2K = Map.of( + "1:1", "2048x2048", + "16:9", "2560x1440", + "9:16", "1440x2560", + "landscape", "2560x1440", + "square", "2048x2048", + "portrait", "1440x2560" + ); + + private DashScopeImageModels() {} + + private static final Map CATALOG = buildCatalog(); + + static Map all() { + return CATALOG; + } + + static ImageModelSpec get(String id) { + if (id == null || id.isBlank()) { + return CATALOG.get(DEFAULT_MODEL); + } + return CATALOG.getOrDefault(id, CATALOG.get(DEFAULT_MODEL)); + } + + private static Map buildCatalog() { + Map m = new LinkedHashMap<>(); + + // ========== Legacy async text-to-image (image-generation/generation) ========== + // No edit support; keeps backward compatibility for users on existing model ids. + addAsyncT2I(m, "wanx2.1-t2i-turbo"); + addAsyncT2I(m, "wanx2.1-t2i-plus"); + addAsyncT2I(m, "wanx2.0-t2i-turbo"); + addAsyncT2I(m, "wan2.2-t2i-flash"); + addAsyncT2I(m, "wan2.2-t2i-plus"); + addAsyncT2I(m, "wan2.5-t2i-preview"); + + // ========== Sync multimodal text-to-image only (multimodal-generation) ========== + m.put("z-image-turbo", ImageModelSpec.builder() + .id("z-image-turbo") + .displayName("Z-Image Turbo (fastest)") + .endpoint(MULTIMODAL_ENDPOINT) + .transport(ImageModelSpec.Transport.SYNC) + .sizeStyle(SizeStyle.LITERAL_DIMENSION) + .sizeMap(ASPECT_LITERAL_SIZES) + .modes(Set.of(ImageCapability.TEXT_TO_IMAGE)) + .supports(Set.of("size", "seed", "prompt_extend")) + .maxCount(1) + .maxInputImages(0) + .build()); + + // ========== Sync multimodal text-to-image + edit (qwen-image series) ========== + addQwenImage(m, "qwen-image-2.0"); + addQwenImage(m, "qwen-image-2.0-pro"); + addQwenImageEdit(m, "qwen-image-edit"); + addQwenImageEdit(m, "qwen-image-edit-plus"); + addQwenImageEdit(m, "qwen-image-edit-max"); + + // ========== Sync multimodal text-to-image + edit (wan2.6 / 2.7 image) ========== + m.put("wan2.6-t2i", ImageModelSpec.builder() + .id("wan2.6-t2i") + .displayName("Wan 2.6 (sync T2I)") + .endpoint(MULTIMODAL_ENDPOINT) + .transport(ImageModelSpec.Transport.SYNC) + .sizeStyle(SizeStyle.LITERAL_DIMENSION) + .sizeMap(ASPECT_LITERAL_SIZES) + .modes(Set.of(ImageCapability.TEXT_TO_IMAGE)) + .supports(Set.of("size", "n", "seed", "negative_prompt", "prompt_extend", "watermark")) + .maxCount(4) + .maxInputImages(0) + .build()); + + m.put("wan2.7-image", ImageModelSpec.builder() + .id("wan2.7-image") + .displayName("Wan 2.7 Image (T2I + edit, up to 2K)") + .endpoint(MULTIMODAL_ENDPOINT) + .transport(ImageModelSpec.Transport.SYNC) + .sizeStyle(SizeStyle.LITERAL_DIMENSION) + .sizeMap(ASPECT_LITERAL_SIZES) + .modes(Set.of(ImageCapability.TEXT_TO_IMAGE, ImageCapability.IMAGE_EDIT)) + .supports(Set.of("size", "n", "seed", "negative_prompt", "prompt_extend", "watermark")) + .maxCount(4) + .maxInputImages(3) + .build()); + + m.put("wan2.7-image-pro", ImageModelSpec.builder() + .id("wan2.7-image-pro") + .displayName("Wan 2.7 Image Pro (T2I + edit, up to 4K)") + .endpoint(MULTIMODAL_ENDPOINT) + .transport(ImageModelSpec.Transport.SYNC) + .sizeStyle(SizeStyle.LITERAL_DIMENSION) + .sizeMap(ASPECT_LITERAL_SIZES_2K) + .modes(Set.of(ImageCapability.TEXT_TO_IMAGE, ImageCapability.IMAGE_EDIT)) + .supports(Set.of("size", "n", "seed", "negative_prompt", "prompt_extend", "watermark")) + .maxCount(4) + .maxInputImages(3) + .build()); + + return Map.copyOf(m); + } + + // ------------------- helper builders ------------------- + + private static void addAsyncT2I(Map m, String id) { + m.put(id, ImageModelSpec.builder() + .id(id) + .displayName(id + " (async legacy T2I)") + .endpoint(LEGACY_ASYNC_ENDPOINT) + .transport(ImageModelSpec.Transport.ASYNC) + .sizeStyle(SizeStyle.LITERAL_DIMENSION) + // Legacy endpoint uses '*' as the size separator; sizeMap stores + // the native form so PayloadBuilder can pass it through. + .sizeMapping("1:1", "1024*1024") + .sizeMapping("16:9", "1280*720") + .sizeMapping("9:16", "720*1280") + .sizeMapping("landscape", "1280*720") + .sizeMapping("square", "1024*1024") + .sizeMapping("portrait", "720*1280") + .sizeMapping("1024x1024", "1024*1024") + .sizeMapping("1280x720", "1280*720") + .sizeMapping("720x1280", "720*1280") + .modes(Set.of(ImageCapability.TEXT_TO_IMAGE)) + .supports(Set.of("size", "n")) + .maxCount(4) + .maxInputImages(0) + .build()); + } + + private static void addQwenImage(Map m, String id) { + m.put(id, ImageModelSpec.builder() + .id(id) + .displayName(id + " (T2I)") + .endpoint(MULTIMODAL_ENDPOINT) + .transport(ImageModelSpec.Transport.SYNC) + .sizeStyle(SizeStyle.LITERAL_DIMENSION) + .sizeMap(ASPECT_LITERAL_SIZES_2K) + .modes(Set.of(ImageCapability.TEXT_TO_IMAGE)) + .supports(Set.of("size", "n", "seed", "negative_prompt", "prompt_extend", "watermark")) + .maxCount(4) + .maxInputImages(0) + .build()); + } + + private static void addQwenImageEdit(Map m, String id) { + m.put(id, ImageModelSpec.builder() + .id(id) + .displayName(id + " (image edit)") + .endpoint(MULTIMODAL_ENDPOINT) + .transport(ImageModelSpec.Transport.SYNC) + .sizeStyle(SizeStyle.LITERAL_DIMENSION) + .sizeMap(ASPECT_LITERAL_SIZES_2K) + .modes(Set.of(ImageCapability.IMAGE_EDIT)) + .supports(Set.of("size", "n", "seed", "negative_prompt", "prompt_extend", "watermark")) + .maxCount(4) + .maxInputImages(3) + .build()); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/tool/image/provider/DashScopeImageProvider.java b/mateclaw-server/src/main/java/vip/mate/tool/image/provider/DashScopeImageProvider.java index d812db7c..60499cd4 100644 --- a/mateclaw-server/src/main/java/vip/mate/tool/image/provider/DashScopeImageProvider.java +++ b/mateclaw-server/src/main/java/vip/mate/tool/image/provider/DashScopeImageProvider.java @@ -4,6 +4,7 @@ import cn.hutool.http.HttpRequest; import cn.hutool.http.HttpResponse; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -11,17 +12,38 @@ import org.springframework.stereotype.Component; import vip.mate.llm.service.ModelProviderService; import vip.mate.system.model.SystemSettingsDTO; import vip.mate.task.AsyncTaskService.TaskPollResult; -import vip.mate.tool.image.*; +import vip.mate.tool.image.ImageCapability; +import vip.mate.tool.image.ImageGenerationProvider; +import vip.mate.tool.image.ImageGenerationRequest; +import vip.mate.tool.image.ImageModelSpec; +import vip.mate.tool.image.ImageProviderCapabilities; +import vip.mate.tool.image.ImageReference; +import vip.mate.tool.image.ImageSubmitResult; +import vip.mate.tool.image.PayloadBuilder; +import java.util.ArrayList; +import java.util.Base64; import java.util.List; import java.util.Set; /** - * DashScope 图片生成 Provider — 支持通义万相 Wanx 系列 - *

- * 异步模式：提交后返回 taskId，需轮询获取结果。 - * 复用已有的 DashScope LLM provider 的 API Key。 - * API 文档: https://help.aliyun.com/zh/model-studio/developer-reference/text-to-image + * DashScope image provider — routes per-model between two transports: + * + *

    + *
  • Async legacy ({@code services/aigc/image-generation/generation}) + * for the wanx 2.0/2.1, wan 2.2/2.5 turbo/plus families. Submit returns a + * task id; the caller polls {@code /api/v1/tasks/{id}} until + * SUCCEEDED.
    • + *
  • Sync multimodal ({@code services/aigc/multimodal-generation/generation}) + * for wan 2.6/2.7 image, qwen-image, qwen-image-edit, z-image. The + * generated image URL is returned in the same response. This endpoint + * also accepts inline reference images, enabling the image edit / + * image-to-image flow.
    • + *
+ * + * The model catalog ({@link DashScopeImageModels}) drives endpoint selection, + * payload shape, and the {@code supports} whitelist — adding a new model is a + * one-line spec entry. * * @author MateClaw Team */ @@ -33,9 +55,6 @@ public class DashScopeImageProvider implements ImageGenerationProvider { private final ModelProviderService modelProviderService; private final ObjectMapper objectMapper; - private static final String BASE_URL = "https://dashscope.aliyuncs.com/api/v1"; - private static final String DEFAULT_MODEL = "wanx2.1-t2i-turbo"; - @Override public String id() { return "dashscope"; @@ -43,7 +62,7 @@ public class DashScopeImageProvider implements ImageGenerationProvider { @Override public String label() { - return "DashScope (通义万相)"; + return "DashScope (Tongyi Wanxiang / Qwen-Image)"; } @Override @@ -58,18 +77,32 @@ public class DashScopeImageProvider implements ImageGenerationProvider { @Override public Set capabilities() { - return Set.of(ImageCapability.TEXT_TO_IMAGE); + return Set.of(ImageCapability.TEXT_TO_IMAGE, ImageCapability.IMAGE_EDIT); } @Override public ImageProviderCapabilities detailedCapabilities() { + List modelIds = new ArrayList<>(DashScopeImageModels.all().keySet()); return ImageProviderCapabilities.builder() .modes(capabilities()) - .supportedSizes(List.of("1024x1024", "720x1280", "1280x720")) + .supportedSizes(List.of( + "1024x1024", "1280x720", "720x1280", + "2048x2048", "2560x1440", "1440x2560")) .aspectRatios(List.of("1:1", "16:9", "9:16")) .maxCount(4) - .defaultModel(DEFAULT_MODEL) - .models(List.of("wanx2.1-t2i-turbo", "wanx-v1")) + .defaultModel(DashScopeImageModels.DEFAULT_MODEL) + .models(modelIds) + .generate(ImageProviderCapabilities.Generate.builder() + .maxCount(4).supportsSize(true).supportsAspectRatio(true).build()) + .edit(ImageProviderCapabilities.Edit.builder() + .enabled(true).maxCount(4).maxInputImages(3) + .supportsSize(true).supportsAspectRatio(true).build()) + .geometry(ImageProviderCapabilities.Geometry.builder() + .sizes(List.of( + "1024x1024", "1280x720", "720x1280", + "2048x2048", "2560x1440", "1440x2560")) + .aspectRatios(List.of("1:1", "16:9", "9:16")) + .build()) .build(); } @@ -86,51 +119,16 @@ public class DashScopeImageProvider implements ImageGenerationProvider { public ImageSubmitResult submit(ImageGenerationRequest request, SystemSettingsDTO config) { String apiKey = getDashScopeApiKey(); if (apiKey == null) { - return ImageSubmitResult.failure(id(), "DashScope API Key 未配置"); + return ImageSubmitResult.failure(id(), "DashScope API Key not configured"); } + ImageModelSpec spec = resolveSpec(request); try { - String model = request.getModel() != null && !request.getModel().isBlank() - ? request.getModel() : DEFAULT_MODEL; - - ObjectNode body = objectMapper.createObjectNode(); - body.put("model", model); - - ObjectNode input = body.putObject("input"); - input.put("prompt", request.getPrompt()); - - ObjectNode parameters = body.putObject("parameters"); - // request.size already normalized by ImageGenerationService to one of supportedSizes. - // DashScope API uses '*' separator instead of 'x'. - String size = request.getSize(); - if (size != null && !size.isBlank()) { - parameters.put("size", size.replace("x", "*")); - } - int count = request.getCount() != null ? Math.min(request.getCount(), 4) : 1; - parameters.put("n", count); - - HttpResponse response = HttpRequest.post(BASE_URL + "/services/aigc/text2image/image-synthesis") - .header("Authorization", "Bearer " + apiKey) - .header("Content-Type", "application/json") - .header("X-DashScope-Async", "enable") - .body(body.toString()) - .timeout(30_000) - .execute(); - - JsonNode result = objectMapper.readTree(response.body()); - - if (response.getStatus() == 200 && result.has("output")) { - String taskId = result.path("output").path("task_id").asText(); - log.info("[DashScope Image] Submitted task: {} (model={})", taskId, model); - return ImageSubmitResult.asyncSuccess(taskId, id()); - } else { - String errMsg = result.has("message") ? result.get("message").asText() - : "HTTP " + response.getStatus(); - log.warn("[DashScope Image] Submit failed: {}", errMsg); - return ImageSubmitResult.failure(id(), errMsg); - } + return spec.transport() == ImageModelSpec.Transport.SYNC + ? submitSyncMultimodal(request, spec, apiKey) + : submitAsyncLegacy(request, spec, apiKey); } catch (Exception e) { - log.error("[DashScope Image] Submit error: {}", e.getMessage(), e); + log.error("[DashScope Image] Submit error (model={}): {}", spec.id(), e.getMessage(), e); return ImageSubmitResult.failure(id(), e.getMessage()); } } @@ -139,11 +137,10 @@ public class DashScopeImageProvider implements ImageGenerationProvider { public TaskPollResult checkStatus(String providerTaskId, SystemSettingsDTO config) { String apiKey = getDashScopeApiKey(); if (apiKey == null) { - return TaskPollResult.failed("DashScope API Key 未配置"); + return TaskPollResult.failed("DashScope API Key not configured"); } - try { - HttpResponse response = HttpRequest.get(BASE_URL + "/tasks/" + providerTaskId) + HttpResponse response = HttpRequest.get(DashScopeImageModels.TASKS_ENDPOINT_PREFIX + providerTaskId) .header("Authorization", "Bearer " + apiKey) .timeout(15_000) .execute(); @@ -154,11 +151,11 @@ public class DashScopeImageProvider implements ImageGenerationProvider { return switch (taskStatus) { case "SUCCEEDED" -> { - String imageUrl = extractImageUrl(output); + String imageUrl = extractLegacyImageUrl(output); yield TaskPollResult.imageSucceeded(imageUrl, output.toString()); } case "FAILED" -> { - String errMsg = output.has("message") ? output.get("message").asText() : "任务失败"; + String errMsg = output.has("message") ? output.get("message").asText() : "task failed"; yield TaskPollResult.failed(errMsg); } case "RUNNING" -> TaskPollResult.running(null); @@ -170,6 +167,156 @@ public class DashScopeImageProvider implements ImageGenerationProvider { } } + // ==================== spec resolution ==================== + + /** + * Pick the model spec for this request. When the request asks for image + * editing but names a model that doesn't support edits (or names nothing), + * fall back to {@link DashScopeImageModels#DEFAULT_EDIT_MODEL} so the call + * doesn't silently degrade to a text-only generation. + */ + private ImageModelSpec resolveSpec(ImageGenerationRequest request) { + boolean wantsEdit = request.getInputImages() != null && !request.getInputImages().isEmpty(); + String requested = request.getModel(); + ImageModelSpec spec = DashScopeImageModels.get(requested); + if (wantsEdit && !spec.supportsEdit()) { + ImageModelSpec edit = DashScopeImageModels.get(DashScopeImageModels.DEFAULT_EDIT_MODEL); + log.info("[DashScope Image] Model {} lacks edit support; routing to {}", spec.id(), edit.id()); + return edit; + } + return spec; + } + + // ==================== sync multimodal-generation ==================== + + private ImageSubmitResult submitSyncMultimodal(ImageGenerationRequest request, + ImageModelSpec spec, + String apiKey) throws Exception { + ObjectNode body = objectMapper.createObjectNode(); + body.put("model", spec.id()); + + // input.messages[].content[] — image blocks first when editing, + // followed by the text prompt block. + ObjectNode input = body.putObject("input"); + ArrayNode messages = input.putArray("messages"); + ObjectNode userMsg = messages.addObject(); + userMsg.put("role", "user"); + ArrayNode content = userMsg.putArray("content"); + + if (request.getInputImages() != null) { + for (ImageReference ref : request.getInputImages()) { + ObjectNode imgPart = content.addObject(); + imgPart.put("image", toDataUrl(ref)); + } + } + ObjectNode textPart = content.addObject(); + textPart.put("text", request.getPrompt() == null ? "" : request.getPrompt()); + + // parameters block — built and filtered against the model's supports set. + ObjectNode parameters = PayloadBuilder.from(spec) + .withSize(request.getSize(), request.getAspectRatio()) + .withCount(request.getCount()) + .toJsonNode(objectMapper); + body.set("parameters", parameters); + + HttpResponse response = HttpRequest.post(spec.endpoint()) + .header("Authorization", "Bearer " + apiKey) + .header("Content-Type", "application/json") + .body(body.toString()) + .timeout(180_000) + .execute(); + + JsonNode result = objectMapper.readTree(response.body()); + if (response.getStatus() != 200) { + String errMsg = result.has("message") ? result.get("message").asText() : "HTTP " + response.getStatus(); + log.warn("[DashScope Image] Sync submit failed (model={}): {}", spec.id(), errMsg); + return ImageSubmitResult.failure(id(), errMsg); + } + + List imageUrls = extractMultimodalImageUrls(result); + if (imageUrls.isEmpty()) { + return ImageSubmitResult.failure(id(), "Multimodal response carried no image URL"); + } + log.info("[DashScope Image] Sync generated {} image(s) (model={})", imageUrls.size(), spec.id()); + return ImageSubmitResult.syncSuccess(id(), imageUrls); + } + + private List extractMultimodalImageUrls(JsonNode result) { + List urls = new ArrayList<>(); + JsonNode choices = result.path("output").path("choices"); + if (!choices.isArray()) { + return urls; + } + for (JsonNode choice : choices) { + JsonNode parts = choice.path("message").path("content"); + if (!parts.isArray()) continue; + for (JsonNode part : parts) { + String url = part.path("image").asText(null); + if (url != null && !url.isBlank()) { + urls.add(url); + } + } + } + return urls; + } + + // ==================== async legacy image-generation ==================== + + private ImageSubmitResult submitAsyncLegacy(ImageGenerationRequest request, + ImageModelSpec spec, + String apiKey) throws Exception { + ObjectNode body = objectMapper.createObjectNode(); + body.put("model", spec.id()); + + ObjectNode input = body.putObject("input"); + input.put("prompt", request.getPrompt() == null ? "" : request.getPrompt()); + + ObjectNode parameters = PayloadBuilder.from(spec) + .withSize(request.getSize(), request.getAspectRatio()) + .withCount(request.getCount()) + .toJsonNode(objectMapper); + body.set("parameters", parameters); + + HttpResponse response = HttpRequest.post(spec.endpoint()) + .header("Authorization", "Bearer " + apiKey) + .header("Content-Type", "application/json") + .header("X-DashScope-Async", "enable") + .body(body.toString()) + .timeout(30_000) + .execute(); + + JsonNode result = objectMapper.readTree(response.body()); + if (response.getStatus() == 200 && result.has("output")) { + String taskId = result.path("output").path("task_id").asText(); + log.info("[DashScope Image] Async submitted task {} (model={})", taskId, spec.id()); + return ImageSubmitResult.asyncSuccess(taskId, id()); + } + String errMsg = result.has("message") ? result.get("message").asText() + : "HTTP " + response.getStatus(); + log.warn("[DashScope Image] Async submit failed (model={}): {}", spec.id(), errMsg); + return ImageSubmitResult.failure(id(), errMsg); + } + + private String extractLegacyImageUrl(JsonNode output) { + JsonNode results = output.path("results"); + if (results.isArray() && !results.isEmpty()) { + JsonNode first = results.get(0); + String url = first.path("url").asText(null); + if (url == null || url.isBlank()) { + url = first.path("image").asText(null); + } + return url; + } + return null; + } + + // ==================== shared helpers ==================== + + private String toDataUrl(ImageReference ref) { + String mime = ref.mimeType() == null || ref.mimeType().isBlank() ? "image/png" : ref.mimeType(); + return "data:" + mime + ";base64," + Base64.getEncoder().encodeToString(ref.data()); + } + private String getDashScopeApiKey() { try { var providerEntity = modelProviderService.getProviderConfig("dashscope"); @@ -178,12 +325,4 @@ public class DashScopeImageProvider implements ImageGenerationProvider { return null; } } - - private String extractImageUrl(JsonNode output) { - JsonNode results = output.path("results"); - if (results.isArray() && !results.isEmpty()) { - return results.get(0).path("url").asText(null); - } - return null; - } } diff --git a/mateclaw-server/src/main/java/vip/mate/tool/video/provider/DashScopeVideoProvider.java b/mateclaw-server/src/main/java/vip/mate/tool/video/provider/DashScopeVideoProvider.java index e71bb868..64509e90 100644 --- a/mateclaw-server/src/main/java/vip/mate/tool/video/provider/DashScopeVideoProvider.java +++ b/mateclaw-server/src/main/java/vip/mate/tool/video/provider/DashScopeVideoProvider.java @@ -4,6 +4,7 @@ import cn.hutool.http.HttpRequest; import cn.hutool.http.HttpResponse; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -11,16 +12,35 @@ import org.springframework.stereotype.Component; import vip.mate.llm.service.ModelProviderService; import vip.mate.system.model.SystemSettingsDTO; import vip.mate.task.AsyncTaskService.TaskPollResult; -import vip.mate.tool.video.*; +import vip.mate.tool.video.VideoCapability; +import vip.mate.tool.video.VideoGenerationProvider; +import vip.mate.tool.video.VideoGenerationRequest; +import vip.mate.tool.video.VideoProviderCapabilities; +import vip.mate.tool.video.VideoSubmitResult; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; import java.util.Set; /** - * DashScope 视频生成 Provider — 支持通义万相 Wan 2.5 / Wanx 2.1 - *

- * 复用已有的 DashScope LLM provider 的 API Key。 - * API 文档: https://help.aliyun.com/zh/model-studio/developer-reference/video-generation + * DashScope video provider — supports two payload families on the same async + * task model, selected per model id: + * + *

    + *
  • Legacy ({@code services/aigc/video-generation/generation}) for + * wanx 2.1 and wan 2.5 turbo lines. Body uses {@code input.img_url} for + * image-to-video and {@code parameters.size} for sizing.
    • + *
  • Unified video-synthesis + * ({@code services/aigc/video-generation/video-synthesis}) for wan 2.7 + * and the happyhorse t2v line. Body uses {@code input.media[]} for the + * first frame plus {@code parameters.resolution} + {@code parameters.ratio} + * for sizing.
    • + *
+ * + * Routing is data-driven: each model is registered with its endpoint, body + * shape, and capability set; submit/build code consults the spec rather than + * branching on model id strings. * * @author MateClaw Team */ @@ -33,9 +53,52 @@ public class DashScopeVideoProvider implements VideoGenerationProvider { private final ObjectMapper objectMapper; private static final String BASE_URL = "https://dashscope.aliyuncs.com/api/v1"; + private static final String LEGACY_ENDPOINT = BASE_URL + "/services/aigc/video-generation/generation"; + private static final String UNIFIED_ENDPOINT = BASE_URL + "/services/aigc/video-generation/video-synthesis"; + private static final String TASKS_ENDPOINT_PREFIX = BASE_URL + "/tasks/"; + private static final String DEFAULT_T2V_MODEL = "wan2.5-t2v-turbo"; private static final String DEFAULT_I2V_MODEL = "wan2.5-i2v-turbo"; + private enum BodyShape { + /** input.img_url + parameters.size("1280*720") + parameters.duration. */ + LEGACY, + /** input.media[].first_frame + parameters.resolution + parameters.ratio + parameters.duration. */ + UNIFIED + } + + private record ModelSpec( + String id, + String endpoint, + BodyShape bodyShape, + Set modes + ) {} + + private static final Map MODELS = buildCatalog(); + + private static Map buildCatalog() { + Map m = new LinkedHashMap<>(); + // Legacy line — text-to-video + m.put("wan2.5-t2v-turbo", new ModelSpec("wan2.5-t2v-turbo", + LEGACY_ENDPOINT, BodyShape.LEGACY, Set.of(VideoCapability.GENERATE))); + m.put("wanx2.1-t2v-turbo", new ModelSpec("wanx2.1-t2v-turbo", + LEGACY_ENDPOINT, BodyShape.LEGACY, Set.of(VideoCapability.GENERATE))); + // Legacy line — image-to-video + m.put("wan2.5-i2v-turbo", new ModelSpec("wan2.5-i2v-turbo", + LEGACY_ENDPOINT, BodyShape.LEGACY, Set.of(VideoCapability.IMAGE_TO_VIDEO))); + m.put("wanx2.1-i2v-turbo", new ModelSpec("wanx2.1-i2v-turbo", + LEGACY_ENDPOINT, BodyShape.LEGACY, Set.of(VideoCapability.IMAGE_TO_VIDEO))); + // Unified video-synthesis line — wan 2.7 + m.put("wan2.7-t2v-2026-04-25", new ModelSpec("wan2.7-t2v-2026-04-25", + UNIFIED_ENDPOINT, BodyShape.UNIFIED, Set.of(VideoCapability.GENERATE))); + m.put("wan2.7-i2v-2026-04-25", new ModelSpec("wan2.7-i2v-2026-04-25", + UNIFIED_ENDPOINT, BodyShape.UNIFIED, Set.of(VideoCapability.IMAGE_TO_VIDEO))); + // Unified video-synthesis line — happyhorse text-to-video + m.put("happyhorse-1.0-t2v", new ModelSpec("happyhorse-1.0-t2v", + UNIFIED_ENDPOINT, BodyShape.UNIFIED, Set.of(VideoCapability.GENERATE))); + return Map.copyOf(m); + } + @Override public String id() { return "dashscope"; @@ -43,7 +106,7 @@ public class DashScopeVideoProvider implements VideoGenerationProvider { @Override public String label() { - return "DashScope (通义万相)"; + return "DashScope (Tongyi Wanxiang / HappyHorse)"; } @Override @@ -66,10 +129,10 @@ public class DashScopeVideoProvider implements VideoGenerationProvider { return VideoProviderCapabilities.builder() .modes(capabilities()) .aspectRatios(List.of("16:9", "9:16", "1:1")) - .supportedDurations(List.of(5, 10)) - .maxDurationSeconds(10) + .supportedDurations(List.of(2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15)) + .maxDurationSeconds(15) .defaultModel(DEFAULT_T2V_MODEL) - .models(List.of("wan2.5-t2v-turbo", "wan2.5-i2v-turbo", "wanx2.1-t2v-turbo", "wanx2.1-i2v-turbo")) + .models(List.copyOf(MODELS.keySet())) .build(); } @@ -86,14 +149,12 @@ public class DashScopeVideoProvider implements VideoGenerationProvider { public VideoSubmitResult submit(VideoGenerationRequest request, SystemSettingsDTO config) { String apiKey = getDashScopeApiKey(); if (apiKey == null) { - return VideoSubmitResult.failure(id(), "DashScope API Key 未配置"); + return VideoSubmitResult.failure(id(), "DashScope API Key not configured"); } - + ModelSpec spec = resolveSpec(request); try { - String model = resolveModel(request); - ObjectNode body = buildRequestBody(request, model); - - HttpResponse response = HttpRequest.post(BASE_URL + "/services/aigc/video-generation/generation") + ObjectNode body = buildRequestBody(request, spec); + HttpResponse response = HttpRequest.post(spec.endpoint()) .header("Authorization", "Bearer " + apiKey) .header("Content-Type", "application/json") .header("X-DashScope-Async", "enable") @@ -102,19 +163,17 @@ public class DashScopeVideoProvider implements VideoGenerationProvider { .execute(); JsonNode result = objectMapper.readTree(response.body()); - if (response.getStatus() == 200 && result.has("output")) { String taskId = result.path("output").path("task_id").asText(); - log.info("[DashScope Video] Submitted task: {} (model={})", taskId, model); + log.info("[DashScope Video] Submitted task {} (model={})", taskId, spec.id()); return VideoSubmitResult.success(taskId, id()); - } else { - String errMsg = result.has("message") ? result.get("message").asText() - : "HTTP " + response.getStatus(); - log.warn("[DashScope Video] Submit failed: {}", errMsg); - return VideoSubmitResult.failure(id(), errMsg); } + String errMsg = result.has("message") ? result.get("message").asText() + : "HTTP " + response.getStatus(); + log.warn("[DashScope Video] Submit failed (model={}): {}", spec.id(), errMsg); + return VideoSubmitResult.failure(id(), errMsg); } catch (Exception e) { - log.error("[DashScope Video] Submit error: {}", e.getMessage(), e); + log.error("[DashScope Video] Submit error (model={}): {}", spec.id(), e.getMessage(), e); return VideoSubmitResult.failure(id(), e.getMessage()); } } @@ -123,11 +182,10 @@ public class DashScopeVideoProvider implements VideoGenerationProvider { public TaskPollResult checkStatus(String providerTaskId, SystemSettingsDTO config) { String apiKey = getDashScopeApiKey(); if (apiKey == null) { - return TaskPollResult.failed("DashScope API Key 未配置"); + return TaskPollResult.failed("DashScope API Key not configured"); } - try { - HttpResponse response = HttpRequest.get(BASE_URL + "/tasks/" + providerTaskId) + HttpResponse response = HttpRequest.get(TASKS_ENDPOINT_PREFIX + providerTaskId) .header("Authorization", "Bearer " + apiKey) .timeout(15_000) .execute(); @@ -135,14 +193,13 @@ public class DashScopeVideoProvider implements VideoGenerationProvider { JsonNode result = objectMapper.readTree(response.body()); JsonNode output = result.path("output"); String taskStatus = output.path("task_status").asText(); - return switch (taskStatus) { case "SUCCEEDED" -> { String videoUrl = extractVideoUrl(output); yield TaskPollResult.succeeded(videoUrl, null, output.toString()); } case "FAILED" -> { - String errMsg = output.has("message") ? output.get("message").asText() : "任务失败"; + String errMsg = output.has("message") ? output.get("message").asText() : "task failed"; yield TaskPollResult.failed(errMsg); } case "RUNNING" -> TaskPollResult.running(null); @@ -150,11 +207,107 @@ public class DashScopeVideoProvider implements VideoGenerationProvider { }; } catch (Exception e) { log.error("[DashScope Video] Poll error for task {}: {}", providerTaskId, e.getMessage()); - return null; // 轮询异常不终止，等下次重试 + return null; } } - // ==================== 内部方法 ==================== + // ==================== spec resolution ==================== + + private ModelSpec resolveSpec(VideoGenerationRequest request) { + String requested = request.getModel(); + if (requested != null && !requested.isBlank() && MODELS.containsKey(requested)) { + return MODELS.get(requested); + } + // Fall back to a default by mode. + String defaultId = request.getMode() == VideoCapability.IMAGE_TO_VIDEO + ? DEFAULT_I2V_MODEL : DEFAULT_T2V_MODEL; + return MODELS.get(defaultId); + } + + // ==================== body building ==================== + + private ObjectNode buildRequestBody(VideoGenerationRequest request, ModelSpec spec) { + return switch (spec.bodyShape()) { + case LEGACY -> buildLegacyBody(request, spec); + case UNIFIED -> buildUnifiedBody(request, spec); + }; + } + + private ObjectNode buildLegacyBody(VideoGenerationRequest request, ModelSpec spec) { + ObjectNode body = objectMapper.createObjectNode(); + body.put("model", spec.id()); + + ObjectNode input = body.putObject("input"); + input.put("prompt", request.getPrompt() == null ? "" : request.getPrompt()); + if (spec.modes().contains(VideoCapability.IMAGE_TO_VIDEO) + && request.getImageUrl() != null && !request.getImageUrl().isBlank()) { + input.put("img_url", request.getImageUrl()); + } + + ObjectNode parameters = body.putObject("parameters"); + String size = aspectRatioToLegacySize(request.getAspectRatio()); + if (size != null) { + parameters.put("size", size); + } + if (request.getDurationSeconds() != null) { + parameters.put("duration", String.valueOf(request.getDurationSeconds())); + } + return body; + } + + private ObjectNode buildUnifiedBody(VideoGenerationRequest request, ModelSpec spec) { + ObjectNode body = objectMapper.createObjectNode(); + body.put("model", spec.id()); + + ObjectNode input = body.putObject("input"); + input.put("prompt", request.getPrompt() == null ? "" : request.getPrompt()); + if (spec.modes().contains(VideoCapability.IMAGE_TO_VIDEO) + && request.getImageUrl() != null && !request.getImageUrl().isBlank()) { + ArrayNode media = input.putArray("media"); + ObjectNode firstFrame = media.addObject(); + firstFrame.put("type", "first_frame"); + firstFrame.put("url", request.getImageUrl()); + } + + ObjectNode parameters = body.putObject("parameters"); + String resolution = aspectRatioToUnifiedResolution(request.getAspectRatio()); + parameters.put("resolution", resolution); + if (request.getAspectRatio() != null && !request.getAspectRatio().isBlank()) { + parameters.put("ratio", request.getAspectRatio()); + } + if (request.getDurationSeconds() != null) { + // Unified endpoint expects the duration as an integer. + parameters.put("duration", request.getDurationSeconds()); + } + return body; + } + + private String aspectRatioToLegacySize(String aspectRatio) { + if (aspectRatio == null) return null; + return switch (aspectRatio) { + case "16:9" -> "1280*720"; + case "9:16" -> "720*1280"; + case "1:1" -> "720*720"; + default -> null; + }; + } + + private String aspectRatioToUnifiedResolution(String aspectRatio) { + // Default to 720P; the unified endpoint also accepts 1080P. Callers that + // want to override should pass it via extraParams in a future iteration. + return "720P"; + } + + private String extractVideoUrl(JsonNode output) { + if (output.has("video_url")) { + return output.get("video_url").asText(); + } + JsonNode results = output.path("results"); + if (results.isArray() && !results.isEmpty()) { + return results.get(0).path("url").asText(null); + } + return null; + } private String getDashScopeApiKey() { try { @@ -164,59 +317,4 @@ public class DashScopeVideoProvider implements VideoGenerationProvider { return null; } } - - private String resolveModel(VideoGenerationRequest request) { - if (request.getModel() != null && !request.getModel().isBlank()) { - return request.getModel(); - } - return request.getMode() == VideoCapability.IMAGE_TO_VIDEO - ? DEFAULT_I2V_MODEL : DEFAULT_T2V_MODEL; - } - - private ObjectNode buildRequestBody(VideoGenerationRequest request, String model) { - ObjectNode body = objectMapper.createObjectNode(); - body.put("model", model); - - ObjectNode input = body.putObject("input"); - input.put("prompt", request.getPrompt()); - - if (request.getMode() == VideoCapability.IMAGE_TO_VIDEO && request.getImageUrl() != null) { - input.put("img_url", request.getImageUrl()); - } - - ObjectNode parameters = body.putObject("parameters"); - if (request.getAspectRatio() != null) { - // DashScope 使用 size 参数，如 "1280*720" - String size = aspectRatioToSize(request.getAspectRatio()); - if (size != null) { - parameters.put("size", size); - } - } - if (request.getDurationSeconds() != null) { - parameters.put("duration", String.valueOf(request.getDurationSeconds())); - } - - return body; - } - - private String aspectRatioToSize(String aspectRatio) { - return switch (aspectRatio) { - case "16:9" -> "1280*720"; - case "9:16" -> "720*1280"; - case "1:1" -> "720*720"; - default -> null; - }; - } - - private String extractVideoUrl(JsonNode output) { - JsonNode results = output.path("results"); - if (results.isArray() && !results.isEmpty()) { - return results.get(0).path("url").asText(null); - } - // 有些模型返回 video_url - if (output.has("video_url")) { - return output.get("video_url").asText(); - } - return null; - } }