mirror of
https://gitee.com/mateos/mateclaw.git
synced 2026-09-13 03:13:41 +08:00
feat(tool/image,video): image edit support + new DashScope models (#75)
This commit is contained in:
matevip
parent
0cbe271f6f
commit
3f289da6b7
@ -13,6 +13,7 @@ import vip.mate.task.AsyncTaskService;
|
||||
import vip.mate.task.model.AsyncTaskInfo;
|
||||
import vip.mate.tool.image.*;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.StringJoiner;
|
||||
|
||||
@ -30,19 +31,23 @@ public class ImageGenerateTool {
|
||||
private final ImageProviderRegistry providerRegistry;
|
||||
private final SystemSettingService systemSettingService;
|
||||
private final AsyncTaskService asyncTaskService;
|
||||
private final ImageReferenceLoader imageReferenceLoader;
|
||||
|
||||
@vip.mate.tool.ConcurrencyUnsafe("creates async tasks and persists generated artifacts; provider rate limits also forbid parallel calls")
|
||||
@Tool(description = "Image generation tool. Supports actions: generate (default), list (show available providers), "
|
||||
+ "status (check task status). Some providers are async (30s-2min), results auto-displayed in conversation.")
|
||||
@Tool(description = "Image generation tool. Supports actions: generate (default — text-to-image, OR image-edit when "
|
||||
+ "image/images parameters are set), list (show available providers/models), status (check task status). "
|
||||
+ "Reference images may be local paths, http(s) URLs, data: URLs, or msg:<messageId>:<idx> for an attachment "
|
||||
+ "from an earlier conversation message. Async providers take 30s-2min; results auto-display in the conversation.")
|
||||
public String image_generate(
|
||||
@ToolParam(description = "Action type: generate, list, status. Default: generate", required = false) String action,
|
||||
@ToolParam(description = "Image content description, be detailed (required for generate)", required = false) String prompt,
|
||||
@ToolParam(description = "Single reference image for edit mode. Path / http(s) URL / data: URL / msg:<messageId>[:<idx>]", required = false) String image,
|
||||
@ToolParam(description = "Multiple reference images for edit mode (provider caps the count). Same formats as 'image'.", required = false) List<String> images,
|
||||
@ToolParam(description = "Image size: 1024x1024 / 1024x1792 / 1792x1024", required = false) String size,
|
||||
@ToolParam(description = "Aspect ratio: 1:1 / 16:9 / 9:16, default 1:1", required = false) String aspectRatio,
|
||||
@ToolParam(description = "Generation count (1-4), default 1", required = false) Integer count,
|
||||
@ToolParam(description = "Model name (optional)", required = false) String model,
|
||||
@ToolParam(description = "Task ID to check status (for status action)", required = false) String taskId,
|
||||
// RFC-063r §2.5: ToolContext is hidden from the LLM by JsonSchemaGenerator.
|
||||
@Nullable ToolContext ctx
|
||||
) {
|
||||
String normalizedAction = (action == null || action.isBlank()) ? "generate" : action.trim().toLowerCase();
|
||||
@ -50,7 +55,7 @@ public class ImageGenerateTool {
|
||||
return switch (normalizedAction) {
|
||||
case "list" -> handleListAction();
|
||||
case "status" -> handleStatusAction(taskId, ctx);
|
||||
default -> handleGenerateAction(prompt, size, aspectRatio, count, model, ctx);
|
||||
default -> handleGenerateAction(prompt, image, images, size, aspectRatio, count, model, ctx);
|
||||
};
|
||||
}
|
||||
|
||||
@ -120,7 +125,8 @@ public class ImageGenerateTool {
|
||||
|
||||
// ==================== action=generate ====================
|
||||
|
||||
private String handleGenerateAction(String prompt, String size, String aspectRatio,
|
||||
private String handleGenerateAction(String prompt, String image, List<String> images,
|
||||
String size, String aspectRatio,
|
||||
Integer count, String model, @Nullable ToolContext ctx) {
|
||||
String conversationId = ToolExecutionContext.conversationId(ctx);
|
||||
String username = ToolExecutionContext.username(ctx);
|
||||
@ -133,12 +139,33 @@ public class ImageGenerateTool {
|
||||
return "错误：prompt 为必填参数，请描述你想要生成的图片内容";
|
||||
}
|
||||
|
||||
// Combine the singular and plural forms — the agent picks whichever is
|
||||
// ergonomic. Order: image (first) then images[].
|
||||
List<String> referenceInputs = new ArrayList<>();
|
||||
if (image != null && !image.isBlank()) {
|
||||
referenceInputs.add(image);
|
||||
}
|
||||
if (images != null) {
|
||||
for (String s : images) {
|
||||
if (s != null && !s.isBlank()) referenceInputs.add(s);
|
||||
}
|
||||
}
|
||||
|
||||
List<ImageReference> inputImages;
|
||||
try {
|
||||
inputImages = imageReferenceLoader.loadAll(referenceInputs, conversationId);
|
||||
} catch (Exception e) {
|
||||
log.warn("[ImageGenerateTool] Failed to load reference images: {}", e.getMessage());
|
||||
return "错误：无法加载参考图片：" + e.getMessage();
|
||||
}
|
||||
|
||||
ImageGenerationRequest request = ImageGenerationRequest.builder()
|
||||
.prompt(prompt)
|
||||
.size(size)
|
||||
.aspectRatio(aspectRatio != null ? aspectRatio : "1:1")
|
||||
.count(count != null ? count : 1)
|
||||
.model(model)
|
||||
.inputImages(inputImages)
|
||||
.build();
|
||||
|
||||
ImageGenerationResult result = imageGenerationService.submitGeneration(
|
||||
|
||||
@ -3,10 +3,11 @@ package vip.mate.tool.image;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* 图片生成统一请求
|
||||
* Unified image-generation request.
|
||||
*
|
||||
* @author MateClaw Team
|
||||
*/
|
||||
@ -14,30 +15,36 @@ import java.util.Map;
|
||||
@Builder
|
||||
public class ImageGenerationRequest {
|
||||
|
||||
/** 图片内容描述 */
|
||||
/** Prompt describing the desired image. */
|
||||
private String prompt;
|
||||
|
||||
/** 生成模式（由 runtime 自动推断） */
|
||||
/** Generation mode (inferred by the runtime when null). */
|
||||
private ImageCapability mode;
|
||||
|
||||
/** 指定模型名称（可选，provider 有默认值） */
|
||||
/** Model id; provider supplies a default when null/blank. */
|
||||
private String model;
|
||||
|
||||
/** 图片尺寸：1024x1024 / 1024x1792 / 1792x1024 等 */
|
||||
/** Pixel size like {@code 1024x1024} / {@code 1024x1792}. */
|
||||
@Builder.Default
|
||||
private String size = "1024x1024";
|
||||
|
||||
/** 画面比例：1:1 / 16:9 / 9:16 */
|
||||
/** Aspect ratio: {@code 1:1} / {@code 16:9} / {@code 9:16}. */
|
||||
@Builder.Default
|
||||
private String aspectRatio = "1:1";
|
||||
|
||||
/** 生成数量 */
|
||||
/** Number of images to return. */
|
||||
@Builder.Default
|
||||
private Integer count = 1;
|
||||
|
||||
/** 参考图片 URL（IMAGE_EDIT 模式） */
|
||||
private String referenceImageUrl;
|
||||
/**
|
||||
* Reference images for edit / image-to-image flows. Loaded as in-memory
|
||||
* buffers so providers can either inline base64, upload via multipart, or
|
||||
* forward as a URL — without each provider re-implementing path/URL/data
|
||||
* resolution.
|
||||
*/
|
||||
@Builder.Default
|
||||
private List<ImageReference> inputImages = List.of();
|
||||
|
||||
/** provider 特有的额外参数 */
|
||||
/** Provider-specific extras forwarded as-is. */
|
||||
private Map<String, Object> extraParams;
|
||||
}
|
||||
|
||||
@ -279,7 +279,7 @@ public class ImageGenerationService {
|
||||
}
|
||||
|
||||
private ImageCapability inferMode(ImageGenerationRequest request) {
|
||||
if (request.getReferenceImageUrl() != null && !request.getReferenceImageUrl().isBlank()) {
|
||||
if (request.getInputImages() != null && !request.getInputImages().isEmpty()) {
|
||||
return ImageCapability.IMAGE_EDIT;
|
||||
}
|
||||
return ImageCapability.TEXT_TO_IMAGE;
|
||||
|
||||
@ -0,0 +1,57 @@
|
||||
package vip.mate.tool.image;
|
||||
|
||||
import lombok.Builder;
|
||||
import lombok.Singular;
|
||||
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
/**
|
||||
* Per-model descriptor that drives payload construction without {@code if/else}
|
||||
* chains inside provider classes. Adding a new model = adding a new spec entry.
|
||||
*
|
||||
* <p>Three things make this configuration-driven:
|
||||
* <ul>
|
||||
* <li>{@code endpoint} chooses which provider URL to hit. A single provider
|
||||
* (e.g. DashScope) can host both an async legacy endpoint and a unified
|
||||
* multimodal endpoint — the spec routes per model.</li>
|
||||
* <li>{@code transport} ({@link Transport#SYNC} / {@link Transport#ASYNC})
|
||||
* lets the provider pick between immediate-return and submit+poll without
|
||||
* hard-coding the choice.</li>
|
||||
* <li>{@code supports} acts as a payload key whitelist. Build the full payload
|
||||
* freely, then filter against {@code supports} so models never receive
|
||||
* fields they reject.</li>
|
||||
* </ul>
|
||||
*
|
||||
* @author MateClaw Team
|
||||
*/
|
||||
@Builder
|
||||
public record ImageModelSpec(
|
||||
String id,
|
||||
String displayName,
|
||||
String endpoint,
|
||||
Transport transport,
|
||||
SizeStyle sizeStyle,
|
||||
@Singular("sizeMapping") Map<String, String> sizeMap,
|
||||
@Singular("defaultParam") Map<String, Object> defaults,
|
||||
@Singular Set<String> supports,
|
||||
@Singular Set<ImageCapability> modes,
|
||||
int maxInputImages,
|
||||
int maxCount
|
||||
) {
|
||||
|
||||
public enum Transport {
|
||||
/** Provider returns image bytes / URL in the same HTTP response. */
|
||||
SYNC,
|
||||
/** Provider returns a task id; caller polls a status endpoint. */
|
||||
ASYNC
|
||||
}
|
||||
|
||||
public boolean supportsEdit() {
|
||||
return modes != null && modes.contains(ImageCapability.IMAGE_EDIT);
|
||||
}
|
||||
|
||||
public boolean supportsGenerate() {
|
||||
return modes != null && modes.contains(ImageCapability.TEXT_TO_IMAGE);
|
||||
}
|
||||
}
|
||||
@ -7,7 +7,15 @@ import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
/**
|
||||
* 图片生成 Provider 细粒度能力声明
|
||||
* Image generation provider capability declaration.
|
||||
*
|
||||
* <p>The flat top-level fields ({@code supportedSizes}, {@code aspectRatios},
|
||||
* {@code maxCount}, {@code modes}) describe the provider's combined surface
|
||||
* area and remain in use by callers that don't need per-mode granularity.
|
||||
* Newer code should consult the structured {@link Generate} / {@link Edit} /
|
||||
* {@link Geometry} / {@link Output} fields, which let the picker show
|
||||
* "edit supports up to N reference images" or "generate accepts these
|
||||
* formats" without conflating the two modes.
|
||||
*
|
||||
* @author MateClaw Team
|
||||
*/
|
||||
@ -15,29 +23,87 @@ import java.util.Set;
|
||||
@Builder
|
||||
public class ImageProviderCapabilities {
|
||||
|
||||
/** 支持的生成模式 */
|
||||
/** Combined modes the provider supports across all its models. */
|
||||
@Builder.Default
|
||||
private Set<ImageCapability> modes = Set.of(ImageCapability.TEXT_TO_IMAGE);
|
||||
|
||||
/** 支持的图片尺寸，如 ["1024x1024", "1024x1792"] */
|
||||
/** Union of pixel sizes accepted by any model under this provider. */
|
||||
@Builder.Default
|
||||
private List<String> supportedSizes = List.of("1024x1024");
|
||||
|
||||
/** 支持的画面比例 */
|
||||
/** Union of aspect ratio presets accepted by any model under this provider. */
|
||||
@Builder.Default
|
||||
private List<String> aspectRatios = List.of("1:1", "16:9", "9:16");
|
||||
|
||||
/** 最大生成数量 */
|
||||
/** Largest {@code n} (image count) any model under this provider accepts. */
|
||||
@Builder.Default
|
||||
private int maxCount = 1;
|
||||
|
||||
/** 默认模型 */
|
||||
/** Default model id. */
|
||||
private String defaultModel;
|
||||
|
||||
/** 可用模型列表 */
|
||||
/** All callable model ids. */
|
||||
@Builder.Default
|
||||
private List<String> models = List.of();
|
||||
|
||||
/** Per-mode generate capabilities. Optional — falls back to flat fields when absent. */
|
||||
private Generate generate;
|
||||
|
||||
/** Per-mode edit capabilities. {@code null} or {@code enabled=false} means edits unsupported. */
|
||||
private Edit edit;
|
||||
|
||||
/** Geometry surface (sizes / aspect ratios). Optional. */
|
||||
private Geometry geometry;
|
||||
|
||||
/** Output knobs (formats, qualities, backgrounds). Optional. */
|
||||
private Output output;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
public static class Generate {
|
||||
@Builder.Default
|
||||
private int maxCount = 1;
|
||||
@Builder.Default
|
||||
private boolean supportsSize = true;
|
||||
@Builder.Default
|
||||
private boolean supportsAspectRatio = true;
|
||||
}
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
public static class Edit {
|
||||
@Builder.Default
|
||||
private boolean enabled = false;
|
||||
@Builder.Default
|
||||
private int maxCount = 1;
|
||||
@Builder.Default
|
||||
private int maxInputImages = 1;
|
||||
@Builder.Default
|
||||
private boolean supportsSize = true;
|
||||
@Builder.Default
|
||||
private boolean supportsAspectRatio = true;
|
||||
}
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
public static class Geometry {
|
||||
@Builder.Default
|
||||
private List<String> sizes = List.of();
|
||||
@Builder.Default
|
||||
private List<String> aspectRatios = List.of();
|
||||
}
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
public static class Output {
|
||||
@Builder.Default
|
||||
private List<String> formats = List.of();
|
||||
@Builder.Default
|
||||
private List<String> qualities = List.of();
|
||||
@Builder.Default
|
||||
private List<String> backgrounds = List.of();
|
||||
}
|
||||
|
||||
/**
|
||||
* Match the requested size against supported sizes by area only.
|
||||
* Orientation-blind — prefer {@link #normalizeSize(String, String)} when an
|
||||
|
||||
@ -0,0 +1,21 @@
|
||||
package vip.mate.tool.image;
|
||||
|
||||
/**
|
||||
* In-memory image reference used for image-edit / image-to-image generation requests.
|
||||
* <p>
|
||||
* The loader normalizes any of the agent-facing input forms (local paths, http(s)
|
||||
* URLs, {@code data:} URLs, conversation message refs) into this single shape so
|
||||
* providers receive bytes, mime type, and file name regardless of origin.
|
||||
*
|
||||
* @param data raw image bytes
|
||||
* @param mimeType e.g. {@code image/png}
|
||||
* @param fileName logical name (best-effort, may be synthesized)
|
||||
* @param origin trace string identifying where the bytes came from
|
||||
* ({@code path:/x.png}, {@code url:https://...}, {@code data-url},
|
||||
* {@code msg:<msgId>:<idx>}). Used for logging / audit, not
|
||||
* forwarded to providers.
|
||||
*
|
||||
* @author MateClaw Team
|
||||
*/
|
||||
public record ImageReference(byte[] data, String mimeType, String fileName, String origin) {
|
||||
}
|
||||
@ -0,0 +1,297 @@
|
||||
package vip.mate.tool.image;
|
||||
|
||||
import cn.hutool.http.HttpUtil;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.stereotype.Component;
|
||||
import vip.mate.workspace.conversation.ConversationService;
|
||||
import vip.mate.workspace.conversation.model.MessageContentPart;
|
||||
import vip.mate.workspace.conversation.model.MessageEntity;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.URI;
|
||||
import java.net.URLDecoder;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Base64;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* Resolves agent-supplied image reference strings into in-memory
|
||||
* {@link ImageReference} buffers. Five input forms are accepted:
|
||||
*
|
||||
* <ol>
|
||||
* <li>Local filesystem path: {@code /abs/path.png}, {@code ./rel.png},
|
||||
* {@code ~/x.png}, or {@code file://...}.</li>
|
||||
* <li>Data URL: {@code data:image/png;base64,...} (base64 or URL-encoded body).</li>
|
||||
* <li>HTTP(S) URL: downloaded with size + content-type guard.</li>
|
||||
* <li>Conversation message reference: {@code msg:<messageId>:<partIndex>} —
|
||||
* resolves to the local path stored on a {@link MessageContentPart} of
|
||||
* type {@code image} on the named message. This is the channel an agent
|
||||
* uses to forward a user-uploaded image into the image edit tool, so a
|
||||
* non-vision model can still operate on attachments it cannot "see".</li>
|
||||
* <li>Workspace-relative path: passed through as a regular path; the caller
|
||||
* is expected to anchor it to the active workspace before invocation.</li>
|
||||
* </ol>
|
||||
*
|
||||
* @author MateClaw Team
|
||||
*/
|
||||
@Slf4j
|
||||
@Component
|
||||
@RequiredArgsConstructor
|
||||
public class ImageReferenceLoader {
|
||||
|
||||
private static final long MAX_REFERENCE_BYTES = 20L * 1024 * 1024;
|
||||
private static final int HTTP_TIMEOUT_MS = 30_000;
|
||||
|
||||
private final ConversationService conversationService;
|
||||
|
||||
/**
|
||||
* Resolve a list of input strings; null / blank entries are skipped.
|
||||
* The caller is expected to enforce per-provider {@code maxInputImages}
|
||||
* before calling.
|
||||
*/
|
||||
public List<ImageReference> loadAll(List<String> inputs, String conversationId) throws IOException {
|
||||
if (inputs == null || inputs.isEmpty()) {
|
||||
return List.of();
|
||||
}
|
||||
List<ImageReference> out = new ArrayList<>(inputs.size());
|
||||
for (String raw : inputs) {
|
||||
if (raw == null || raw.isBlank()) {
|
||||
continue;
|
||||
}
|
||||
out.add(load(raw.trim(), conversationId));
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Resolve a single reference string. */
|
||||
public ImageReference load(String input, String conversationId) throws IOException {
|
||||
if (input == null || input.isBlank()) {
|
||||
throw new IOException("image reference is blank");
|
||||
}
|
||||
String trimmed = input.trim();
|
||||
|
||||
if (trimmed.startsWith("data:")) {
|
||||
return loadDataUrl(trimmed);
|
||||
}
|
||||
if (trimmed.startsWith("msg:")) {
|
||||
return loadConversationMessageRef(trimmed, conversationId);
|
||||
}
|
||||
if (trimmed.startsWith("http://") || trimmed.startsWith("https://")) {
|
||||
return loadHttpUrl(trimmed);
|
||||
}
|
||||
return loadFilePath(trimmed);
|
||||
}
|
||||
|
||||
// ==================== form: local path / file:// ====================
|
||||
|
||||
private ImageReference loadFilePath(String input) throws IOException {
|
||||
String pathStr = input.startsWith("file://") ? input.substring("file://".length()) : input;
|
||||
if (pathStr.startsWith("~")) {
|
||||
pathStr = System.getProperty("user.home") + pathStr.substring(1);
|
||||
}
|
||||
Path p = Paths.get(pathStr);
|
||||
if (!Files.exists(p)) {
|
||||
throw new IOException("Image file not found: " + pathStr);
|
||||
}
|
||||
if (Files.size(p) > MAX_REFERENCE_BYTES) {
|
||||
throw new IOException("Image exceeds 20MB limit: " + pathStr);
|
||||
}
|
||||
byte[] data = Files.readAllBytes(p);
|
||||
String mime = inferMimeFromName(p.getFileName().toString());
|
||||
return new ImageReference(data, mime, p.getFileName().toString(), "path:" + p);
|
||||
}
|
||||
|
||||
// ==================== form: data: URL ====================
|
||||
|
||||
private ImageReference loadDataUrl(String dataUrl) throws IOException {
|
||||
int comma = dataUrl.indexOf(',');
|
||||
if (comma < 0) {
|
||||
throw new IOException("Malformed data URL: missing comma");
|
||||
}
|
||||
String header = dataUrl.substring("data:".length(), comma);
|
||||
String body = dataUrl.substring(comma + 1);
|
||||
boolean isBase64 = header.toLowerCase().contains(";base64");
|
||||
String mime = isBase64
|
||||
? header.substring(0, header.toLowerCase().indexOf(";base64"))
|
||||
: (header.contains(";") ? header.substring(0, header.indexOf(';')) : header);
|
||||
if (mime == null || mime.isBlank()) {
|
||||
mime = "image/png";
|
||||
}
|
||||
byte[] data;
|
||||
try {
|
||||
data = isBase64
|
||||
? Base64.getDecoder().decode(body)
|
||||
: URLDecoder.decode(body, StandardCharsets.UTF_8).getBytes(StandardCharsets.UTF_8);
|
||||
} catch (IllegalArgumentException e) {
|
||||
throw new IOException("Invalid base64 in data URL: " + e.getMessage(), e);
|
||||
}
|
||||
if (data.length > MAX_REFERENCE_BYTES) {
|
||||
throw new IOException("Image exceeds 20MB limit (data URL)");
|
||||
}
|
||||
return new ImageReference(data, mime, "inline." + extensionFor(mime), "data-url");
|
||||
}
|
||||
|
||||
// ==================== form: http(s) URL ====================
|
||||
|
||||
private ImageReference loadHttpUrl(String url) throws IOException {
|
||||
URI uri = URI.create(url);
|
||||
String host = uri.getHost();
|
||||
if (host == null) {
|
||||
throw new IOException("URL has no host: " + url);
|
||||
}
|
||||
// Conservative SSRF guard: reject obvious internal targets. Refine later
|
||||
// if the project gains a dedicated SsrFPolicy module.
|
||||
String lowered = host.toLowerCase();
|
||||
if (lowered.equals("localhost")
|
||||
|| lowered.equals("127.0.0.1")
|
||||
|| lowered.startsWith("10.")
|
||||
|| lowered.startsWith("192.168.")
|
||||
|| lowered.startsWith("169.254.")
|
||||
|| lowered.startsWith("172.")) {
|
||||
throw new IOException("Refusing to download image from internal host: " + host);
|
||||
}
|
||||
try {
|
||||
byte[] data = HttpUtil.createGet(url).timeout(HTTP_TIMEOUT_MS).execute().bodyBytes();
|
||||
if (data == null || data.length == 0) {
|
||||
throw new IOException("Empty response downloading image from " + url);
|
||||
}
|
||||
if (data.length > MAX_REFERENCE_BYTES) {
|
||||
throw new IOException("Image exceeds 20MB limit: " + url);
|
||||
}
|
||||
String fileName = guessFileNameFromUrl(url);
|
||||
String mime = inferMimeFromName(fileName);
|
||||
return new ImageReference(data, mime, fileName, "url:" + url);
|
||||
} catch (Exception e) {
|
||||
throw new IOException("Failed to download image " + url + ": " + e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
|
||||
// ==================== form: msg:<messageId>:<partIndex> ====================
|
||||
|
||||
private ImageReference loadConversationMessageRef(String ref, String conversationId) throws IOException {
|
||||
// ref shape: "msg:<messageId>" (first image part) or "msg:<messageId>:<partIndex>"
|
||||
String body = ref.substring("msg:".length());
|
||||
String[] parts = body.split(":", 2);
|
||||
long messageId;
|
||||
try {
|
||||
messageId = Long.parseLong(parts[0]);
|
||||
} catch (NumberFormatException e) {
|
||||
throw new IOException("Invalid msg: ref, expected msg:<messageId>[:<idx>]: " + ref);
|
||||
}
|
||||
Integer wantedIdx = null;
|
||||
if (parts.length == 2 && !parts[1].isBlank()) {
|
||||
try {
|
||||
wantedIdx = Integer.parseInt(parts[1]);
|
||||
} catch (NumberFormatException e) {
|
||||
throw new IOException("Invalid part index in: " + ref);
|
||||
}
|
||||
}
|
||||
if (conversationId == null || conversationId.isBlank()) {
|
||||
throw new IOException("Cannot resolve msg: reference without an active conversation");
|
||||
}
|
||||
MessageEntity message = findMessageInConversation(conversationId, messageId);
|
||||
if (message == null) {
|
||||
throw new IOException("Message " + messageId + " not found in conversation " + conversationId);
|
||||
}
|
||||
List<MessageContentPart> contentParts = conversationService.parseMessageParts(message);
|
||||
MessageContentPart picked = pickImagePart(contentParts, wantedIdx);
|
||||
if (picked == null) {
|
||||
throw new IOException("No image part on message " + messageId
|
||||
+ (wantedIdx != null ? " at index " + wantedIdx : ""));
|
||||
}
|
||||
Path filePath = resolveLocalPath(picked);
|
||||
if (filePath == null) {
|
||||
throw new IOException("Message " + messageId + " image part has no local path: "
|
||||
+ picked.getFileName());
|
||||
}
|
||||
if (Files.size(filePath) > MAX_REFERENCE_BYTES) {
|
||||
throw new IOException("Image exceeds 20MB limit: " + filePath);
|
||||
}
|
||||
byte[] data = Files.readAllBytes(filePath);
|
||||
String mime = picked.getContentType();
|
||||
if (mime == null || mime.isBlank() || "image/*".equals(mime)) {
|
||||
mime = inferMimeFromName(picked.getFileName());
|
||||
}
|
||||
String fileName = picked.getFileName() != null ? picked.getFileName() : filePath.getFileName().toString();
|
||||
return new ImageReference(data, mime, fileName, ref);
|
||||
}
|
||||
|
||||
private MessageEntity findMessageInConversation(String conversationId, long messageId) {
|
||||
List<MessageEntity> all = conversationService.listMessages(conversationId);
|
||||
for (MessageEntity m : all) {
|
||||
if (m.getId() != null && m.getId() == messageId) {
|
||||
return m;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private MessageContentPart pickImagePart(List<MessageContentPart> parts, Integer wantedIdx) {
|
||||
if (parts == null || parts.isEmpty()) {
|
||||
return null;
|
||||
}
|
||||
if (wantedIdx != null) {
|
||||
int seen = 0;
|
||||
for (MessageContentPart p : parts) {
|
||||
if (p == null || !"image".equals(p.getType())) continue;
|
||||
if (seen == wantedIdx) {
|
||||
return p;
|
||||
}
|
||||
seen++;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
for (MessageContentPart p : parts) {
|
||||
if (p != null && "image".equals(p.getType())) {
|
||||
return p;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private Path resolveLocalPath(MessageContentPart part) {
|
||||
if (part.getPath() != null && !part.getPath().isBlank()) {
|
||||
Path p = Paths.get(part.getPath());
|
||||
if (Files.exists(p)) return p;
|
||||
}
|
||||
if (part.getStoredName() != null && !part.getStoredName().isBlank()) {
|
||||
Path p = Paths.get(part.getStoredName());
|
||||
if (Files.exists(p)) return p;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
// ==================== shared helpers ====================
|
||||
|
||||
private static String inferMimeFromName(String name) {
|
||||
if (name == null) return "image/png";
|
||||
String lower = name.toLowerCase();
|
||||
if (lower.endsWith(".jpg") || lower.endsWith(".jpeg")) return "image/jpeg";
|
||||
if (lower.endsWith(".webp")) return "image/webp";
|
||||
if (lower.endsWith(".gif")) return "image/gif";
|
||||
if (lower.endsWith(".bmp")) return "image/bmp";
|
||||
return "image/png";
|
||||
}
|
||||
|
||||
private static String extensionFor(String mime) {
|
||||
return switch (mime.toLowerCase().trim()) {
|
||||
case "image/jpeg", "image/jpg" -> "jpg";
|
||||
case "image/webp" -> "webp";
|
||||
case "image/gif" -> "gif";
|
||||
case "image/bmp" -> "bmp";
|
||||
default -> "png";
|
||||
};
|
||||
}
|
||||
|
||||
private static String guessFileNameFromUrl(String url) {
|
||||
String stripped = url.split("\\?", 2)[0];
|
||||
int slash = stripped.lastIndexOf('/');
|
||||
String tail = slash >= 0 ? stripped.substring(slash + 1) : stripped;
|
||||
return tail.isBlank() ? "remote.png" : tail;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,172 @@
|
||||
package vip.mate.tool.image;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.fasterxml.jackson.databind.node.ObjectNode;
|
||||
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
/**
|
||||
* Configuration-driven payload builder for image-generation providers.
|
||||
*
|
||||
* <p>Without this, every provider class collects an {@code if/else} chain
|
||||
* mapping (model id) → (request shape, sizing dialect, available knobs). Each
|
||||
* new model in a family forces another branch. With it, each provider holds a
|
||||
* static {@code Map<modelId, ImageModelSpec>}; the builder consults that spec
|
||||
* for sizing dialect, default parameters, and a {@code supports} whitelist of
|
||||
* payload keys. Values not in the whitelist are dropped at the end so the API
|
||||
* never sees keys it would reject.
|
||||
*
|
||||
* <p>Sizing dialect handling:
|
||||
* <ul>
|
||||
* <li>{@link SizeStyle#LITERAL_DIMENSION} — output {@code "1024x1024"} or
|
||||
* a separator-replaced form (e.g. DashScope wants {@code "1024*1024"} —
|
||||
* the spec's sizeMap can carry the alternative).</li>
|
||||
* <li>{@link SizeStyle#ASPECT_RATIO} — output {@code "1:1"} / {@code "16:9"}.</li>
|
||||
* <li>{@link SizeStyle#PRESET_NAME} — output the model-native preset
|
||||
* (e.g. {@code square_hd}). The spec's sizeMap drives the lookup keyed
|
||||
* by orientation token (landscape / square / portrait).</li>
|
||||
* </ul>
|
||||
*
|
||||
* @author MateClaw Team
|
||||
*/
|
||||
public final class PayloadBuilder {
|
||||
|
||||
private final ImageModelSpec spec;
|
||||
private final Map<String, Object> entries = new LinkedHashMap<>();
|
||||
|
||||
private PayloadBuilder(ImageModelSpec spec) {
|
||||
this.spec = spec;
|
||||
if (spec.defaults() != null) {
|
||||
entries.putAll(spec.defaults());
|
||||
}
|
||||
}
|
||||
|
||||
public static PayloadBuilder from(ImageModelSpec spec) {
|
||||
return new PayloadBuilder(spec);
|
||||
}
|
||||
|
||||
public PayloadBuilder withPrompt(String prompt) {
|
||||
if (prompt != null) {
|
||||
entries.put("prompt", prompt);
|
||||
}
|
||||
return this;
|
||||
}
|
||||
|
||||
public PayloadBuilder withCount(Integer count) {
|
||||
if (count != null && count > 0) {
|
||||
entries.put("n", Math.min(count, Math.max(1, spec.maxCount() == 0 ? count : spec.maxCount())));
|
||||
}
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Translate the unified {@code size} / {@code aspectRatio} inputs to whichever
|
||||
* key/value pair this model expects. The spec's {@link SizeStyle} drives
|
||||
* which key is set; the spec's sizeMap (orientation → native value) drives
|
||||
* the value when the caller did not pass an exact match.
|
||||
*/
|
||||
public PayloadBuilder withSize(String requestedSize, String requestedAspectRatio) {
|
||||
SizeStyle style = spec.sizeStyle();
|
||||
if (style == null) {
|
||||
return this;
|
||||
}
|
||||
Map<String, String> sizeMap = spec.sizeMap();
|
||||
switch (style) {
|
||||
case LITERAL_DIMENSION -> entries.put("size",
|
||||
resolveLiteralDimension(requestedSize, requestedAspectRatio, sizeMap));
|
||||
case ASPECT_RATIO -> entries.put("aspect_ratio",
|
||||
resolveAspectRatio(requestedAspectRatio, sizeMap));
|
||||
case PRESET_NAME -> entries.put("image_size",
|
||||
resolvePreset(requestedAspectRatio, sizeMap));
|
||||
}
|
||||
return this;
|
||||
}
|
||||
|
||||
public PayloadBuilder withSeed(Integer seed) {
|
||||
if (seed != null) {
|
||||
entries.put("seed", seed);
|
||||
}
|
||||
return this;
|
||||
}
|
||||
|
||||
public PayloadBuilder put(String key, Object value) {
|
||||
if (value != null) {
|
||||
entries.put(key, value);
|
||||
}
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Produce a Jackson {@link ObjectNode} containing only the keys this model's
|
||||
* {@code supports} whitelist allows. Empty whitelist means "passthrough".
|
||||
*/
|
||||
public ObjectNode toJsonNode(ObjectMapper mapper) {
|
||||
ObjectNode out = mapper.createObjectNode();
|
||||
Set<String> supports = spec.supports();
|
||||
boolean filter = supports != null && !supports.isEmpty();
|
||||
for (Map.Entry<String, Object> e : entries.entrySet()) {
|
||||
if (filter && !supports.contains(e.getKey())) {
|
||||
continue;
|
||||
}
|
||||
out.set(e.getKey(), mapper.valueToTree(e.getValue()));
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Read-only view of accumulated entries (post defaults / pre supports filter). */
|
||||
public Map<String, Object> entries() {
|
||||
return Map.copyOf(entries);
|
||||
}
|
||||
|
||||
// ==================== size resolution ====================
|
||||
|
||||
private String resolveLiteralDimension(String requestedSize, String aspectRatio,
|
||||
Map<String, String> sizeMap) {
|
||||
if (requestedSize != null && !requestedSize.isBlank()) {
|
||||
// Allow the spec's sizeMap to translate (e.g. "1024x1024" -> "1024*1024").
|
||||
String mapped = sizeMap == null ? null : sizeMap.get(requestedSize);
|
||||
return mapped != null ? mapped : requestedSize;
|
||||
}
|
||||
String orientation = orientationOf(aspectRatio);
|
||||
if (sizeMap != null && sizeMap.containsKey(orientation)) {
|
||||
return sizeMap.get(orientation);
|
||||
}
|
||||
return "1024x1024";
|
||||
}
|
||||
|
||||
private String resolveAspectRatio(String requested, Map<String, String> sizeMap) {
|
||||
if (requested != null && !requested.isBlank()) {
|
||||
String mapped = sizeMap == null ? null : sizeMap.get(requested);
|
||||
return mapped != null ? mapped : requested;
|
||||
}
|
||||
return "1:1";
|
||||
}
|
||||
|
||||
private String resolvePreset(String aspectRatio, Map<String, String> sizeMap) {
|
||||
String orientation = orientationOf(aspectRatio);
|
||||
if (sizeMap != null && sizeMap.containsKey(orientation)) {
|
||||
return sizeMap.get(orientation);
|
||||
}
|
||||
return "square_hd";
|
||||
}
|
||||
|
||||
private static String orientationOf(String aspectRatio) {
|
||||
if (aspectRatio == null || aspectRatio.isBlank()) {
|
||||
return "square";
|
||||
}
|
||||
String[] parts = aspectRatio.split(":");
|
||||
if (parts.length != 2) {
|
||||
return "square";
|
||||
}
|
||||
try {
|
||||
double w = Double.parseDouble(parts[0].trim());
|
||||
double h = Double.parseDouble(parts[1].trim());
|
||||
if (w == h) return "square";
|
||||
return w > h ? "landscape" : "portrait";
|
||||
} catch (NumberFormatException e) {
|
||||
return "square";
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,27 @@
|
||||
package vip.mate.tool.image;
|
||||
|
||||
/**
|
||||
* Describes how a particular image-generation model expects its size to be
|
||||
* expressed. Three families cover all current providers:
|
||||
*
|
||||
* <ul>
|
||||
* <li>{@link #LITERAL_DIMENSION} — explicit width/height string ({@code 1024x1024},
|
||||
* {@code 1536*1024}). Used by DashScope, OpenAI DALL-E, MiniMax.</li>
|
||||
* <li>{@link #ASPECT_RATIO} — preset enum like {@code 16:9} or {@code 1:1}.
|
||||
* Used by Gemini / nano-banana style APIs.</li>
|
||||
* <li>{@link #PRESET_NAME} — provider-specific preset label
|
||||
* ({@code square_hd}, {@code landscape_16_9}). Used by fal.ai's flux,
|
||||
* z-image, qwen-image families.</li>
|
||||
* </ul>
|
||||
*
|
||||
* Each {@link ImageModelSpec} declares one style and provides the sizeMap that
|
||||
* translates the unified {@code aspectRatio} input ({@code landscape} /
|
||||
* {@code square} / {@code portrait} or a literal ratio) to the model-native form.
|
||||
*
|
||||
* @author MateClaw Team
|
||||
*/
|
||||
public enum SizeStyle {
|
||||
LITERAL_DIMENSION,
|
||||
ASPECT_RATIO,
|
||||
PRESET_NAME
|
||||
}
|
||||
@ -0,0 +1,220 @@
|
||||
package vip.mate.tool.image.provider;
|
||||
|
||||
import vip.mate.tool.image.ImageCapability;
|
||||
import vip.mate.tool.image.ImageModelSpec;
|
||||
import vip.mate.tool.image.SizeStyle;
|
||||
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
/**
|
||||
* Catalog of DashScope-served image generation / editing models, organised so
|
||||
* that adding a new model is a one-line spec entry.
|
||||
*
|
||||
* <p>Two transport families are present:
|
||||
* <ul>
|
||||
* <li><b>Async legacy</b> ({@link #LEGACY_ASYNC_ENDPOINT}) — wanx 2.0/2.1 and
|
||||
* wan 2.2/2.5 turbo/plus models that exclusively do text-to-image. The
|
||||
* caller submits and polls {@code /api/v1/tasks/{id}}.</li>
|
||||
* <li><b>Sync multimodal</b> ({@link #MULTIMODAL_ENDPOINT}) — wan 2.6/2.7,
|
||||
* qwen-image, qwen-image-edit, z-image. Uses the OpenAI-style
|
||||
* {@code messages.content[]} array and returns the generated image URL
|
||||
* in the same response.</li>
|
||||
* </ul>
|
||||
*
|
||||
* @author MateClaw Team
|
||||
*/
|
||||
final class DashScopeImageModels {
|
||||
|
||||
static final String LEGACY_ASYNC_ENDPOINT =
|
||||
"https://dashscope.aliyuncs.com/api/v1/services/aigc/image-generation/generation";
|
||||
static final String MULTIMODAL_ENDPOINT =
|
||||
"https://dashscope.aliyuncs.com/api/v1/services/aigc/multimodal-generation/generation";
|
||||
static final String TASKS_ENDPOINT_PREFIX =
|
||||
"https://dashscope.aliyuncs.com/api/v1/tasks/";
|
||||
|
||||
/**
|
||||
* Default model when the request does not name one.
|
||||
*
|
||||
* <p>Kept on the legacy turbo so existing accounts that have not enrolled in
|
||||
* the newer wan/qwen-image families do not see breakage. Callers that want
|
||||
* edit support must name a model explicitly (e.g. {@code wan2.7-image} or
|
||||
* {@code qwen-image-edit}) — the registry's edit-capability resolution then
|
||||
* routes correctly.
|
||||
*/
|
||||
static final String DEFAULT_MODEL = "wanx2.1-t2i-turbo";
|
||||
|
||||
/**
|
||||
* Default model when an edit-capable spec is required but the request did
|
||||
* not name one. Used by the provider when the request carries
|
||||
* {@code inputImages} but the named model lacks {@link ImageCapability#IMAGE_EDIT}.
|
||||
*/
|
||||
static final String DEFAULT_EDIT_MODEL = "wan2.7-image";
|
||||
|
||||
private static final Map<String, String> ASPECT_LITERAL_SIZES = Map.of(
|
||||
"1:1", "1024x1024",
|
||||
"16:9", "1280x720",
|
||||
"9:16", "720x1280",
|
||||
"landscape", "1280x720",
|
||||
"square", "1024x1024",
|
||||
"portrait", "720x1280"
|
||||
);
|
||||
|
||||
private static final Map<String, String> ASPECT_LITERAL_SIZES_2K = Map.of(
|
||||
"1:1", "2048x2048",
|
||||
"16:9", "2560x1440",
|
||||
"9:16", "1440x2560",
|
||||
"landscape", "2560x1440",
|
||||
"square", "2048x2048",
|
||||
"portrait", "1440x2560"
|
||||
);
|
||||
|
||||
private DashScopeImageModels() {}
|
||||
|
||||
private static final Map<String, ImageModelSpec> CATALOG = buildCatalog();
|
||||
|
||||
static Map<String, ImageModelSpec> all() {
|
||||
return CATALOG;
|
||||
}
|
||||
|
||||
static ImageModelSpec get(String id) {
|
||||
if (id == null || id.isBlank()) {
|
||||
return CATALOG.get(DEFAULT_MODEL);
|
||||
}
|
||||
return CATALOG.getOrDefault(id, CATALOG.get(DEFAULT_MODEL));
|
||||
}
|
||||
|
||||
private static Map<String, ImageModelSpec> buildCatalog() {
|
||||
Map<String, ImageModelSpec> m = new LinkedHashMap<>();
|
||||
|
||||
// ========== Legacy async text-to-image (image-generation/generation) ==========
|
||||
// No edit support; keeps backward compatibility for users on existing model ids.
|
||||
addAsyncT2I(m, "wanx2.1-t2i-turbo");
|
||||
addAsyncT2I(m, "wanx2.1-t2i-plus");
|
||||
addAsyncT2I(m, "wanx2.0-t2i-turbo");
|
||||
addAsyncT2I(m, "wan2.2-t2i-flash");
|
||||
addAsyncT2I(m, "wan2.2-t2i-plus");
|
||||
addAsyncT2I(m, "wan2.5-t2i-preview");
|
||||
|
||||
// ========== Sync multimodal text-to-image only (multimodal-generation) ==========
|
||||
m.put("z-image-turbo", ImageModelSpec.builder()
|
||||
.id("z-image-turbo")
|
||||
.displayName("Z-Image Turbo (fastest)")
|
||||
.endpoint(MULTIMODAL_ENDPOINT)
|
||||
.transport(ImageModelSpec.Transport.SYNC)
|
||||
.sizeStyle(SizeStyle.LITERAL_DIMENSION)
|
||||
.sizeMap(ASPECT_LITERAL_SIZES)
|
||||
.modes(Set.of(ImageCapability.TEXT_TO_IMAGE))
|
||||
.supports(Set.of("size", "seed", "prompt_extend"))
|
||||
.maxCount(1)
|
||||
.maxInputImages(0)
|
||||
.build());
|
||||
|
||||
// ========== Sync multimodal text-to-image + edit (qwen-image series) ==========
|
||||
addQwenImage(m, "qwen-image-2.0");
|
||||
addQwenImage(m, "qwen-image-2.0-pro");
|
||||
addQwenImageEdit(m, "qwen-image-edit");
|
||||
addQwenImageEdit(m, "qwen-image-edit-plus");
|
||||
addQwenImageEdit(m, "qwen-image-edit-max");
|
||||
|
||||
// ========== Sync multimodal text-to-image + edit (wan2.6 / 2.7 image) ==========
|
||||
m.put("wan2.6-t2i", ImageModelSpec.builder()
|
||||
.id("wan2.6-t2i")
|
||||
.displayName("Wan 2.6 (sync T2I)")
|
||||
.endpoint(MULTIMODAL_ENDPOINT)
|
||||
.transport(ImageModelSpec.Transport.SYNC)
|
||||
.sizeStyle(SizeStyle.LITERAL_DIMENSION)
|
||||
.sizeMap(ASPECT_LITERAL_SIZES)
|
||||
.modes(Set.of(ImageCapability.TEXT_TO_IMAGE))
|
||||
.supports(Set.of("size", "n", "seed", "negative_prompt", "prompt_extend", "watermark"))
|
||||
.maxCount(4)
|
||||
.maxInputImages(0)
|
||||
.build());
|
||||
|
||||
m.put("wan2.7-image", ImageModelSpec.builder()
|
||||
.id("wan2.7-image")
|
||||
.displayName("Wan 2.7 Image (T2I + edit, up to 2K)")
|
||||
.endpoint(MULTIMODAL_ENDPOINT)
|
||||
.transport(ImageModelSpec.Transport.SYNC)
|
||||
.sizeStyle(SizeStyle.LITERAL_DIMENSION)
|
||||
.sizeMap(ASPECT_LITERAL_SIZES)
|
||||
.modes(Set.of(ImageCapability.TEXT_TO_IMAGE, ImageCapability.IMAGE_EDIT))
|
||||
.supports(Set.of("size", "n", "seed", "negative_prompt", "prompt_extend", "watermark"))
|
||||
.maxCount(4)
|
||||
.maxInputImages(3)
|
||||
.build());
|
||||
|
||||
m.put("wan2.7-image-pro", ImageModelSpec.builder()
|
||||
.id("wan2.7-image-pro")
|
||||
.displayName("Wan 2.7 Image Pro (T2I + edit, up to 4K)")
|
||||
.endpoint(MULTIMODAL_ENDPOINT)
|
||||
.transport(ImageModelSpec.Transport.SYNC)
|
||||
.sizeStyle(SizeStyle.LITERAL_DIMENSION)
|
||||
.sizeMap(ASPECT_LITERAL_SIZES_2K)
|
||||
.modes(Set.of(ImageCapability.TEXT_TO_IMAGE, ImageCapability.IMAGE_EDIT))
|
||||
.supports(Set.of("size", "n", "seed", "negative_prompt", "prompt_extend", "watermark"))
|
||||
.maxCount(4)
|
||||
.maxInputImages(3)
|
||||
.build());
|
||||
|
||||
return Map.copyOf(m);
|
||||
}
|
||||
|
||||
// ------------------- helper builders -------------------
|
||||
|
||||
private static void addAsyncT2I(Map<String, ImageModelSpec> m, String id) {
|
||||
m.put(id, ImageModelSpec.builder()
|
||||
.id(id)
|
||||
.displayName(id + " (async legacy T2I)")
|
||||
.endpoint(LEGACY_ASYNC_ENDPOINT)
|
||||
.transport(ImageModelSpec.Transport.ASYNC)
|
||||
.sizeStyle(SizeStyle.LITERAL_DIMENSION)
|
||||
// Legacy endpoint uses '*' as the size separator; sizeMap stores
|
||||
// the native form so PayloadBuilder can pass it through.
|
||||
.sizeMapping("1:1", "1024*1024")
|
||||
.sizeMapping("16:9", "1280*720")
|
||||
.sizeMapping("9:16", "720*1280")
|
||||
.sizeMapping("landscape", "1280*720")
|
||||
.sizeMapping("square", "1024*1024")
|
||||
.sizeMapping("portrait", "720*1280")
|
||||
.sizeMapping("1024x1024", "1024*1024")
|
||||
.sizeMapping("1280x720", "1280*720")
|
||||
.sizeMapping("720x1280", "720*1280")
|
||||
.modes(Set.of(ImageCapability.TEXT_TO_IMAGE))
|
||||
.supports(Set.of("size", "n"))
|
||||
.maxCount(4)
|
||||
.maxInputImages(0)
|
||||
.build());
|
||||
}
|
||||
|
||||
private static void addQwenImage(Map<String, ImageModelSpec> m, String id) {
|
||||
m.put(id, ImageModelSpec.builder()
|
||||
.id(id)
|
||||
.displayName(id + " (T2I)")
|
||||
.endpoint(MULTIMODAL_ENDPOINT)
|
||||
.transport(ImageModelSpec.Transport.SYNC)
|
||||
.sizeStyle(SizeStyle.LITERAL_DIMENSION)
|
||||
.sizeMap(ASPECT_LITERAL_SIZES_2K)
|
||||
.modes(Set.of(ImageCapability.TEXT_TO_IMAGE))
|
||||
.supports(Set.of("size", "n", "seed", "negative_prompt", "prompt_extend", "watermark"))
|
||||
.maxCount(4)
|
||||
.maxInputImages(0)
|
||||
.build());
|
||||
}
|
||||
|
||||
private static void addQwenImageEdit(Map<String, ImageModelSpec> m, String id) {
|
||||
m.put(id, ImageModelSpec.builder()
|
||||
.id(id)
|
||||
.displayName(id + " (image edit)")
|
||||
.endpoint(MULTIMODAL_ENDPOINT)
|
||||
.transport(ImageModelSpec.Transport.SYNC)
|
||||
.sizeStyle(SizeStyle.LITERAL_DIMENSION)
|
||||
.sizeMap(ASPECT_LITERAL_SIZES_2K)
|
||||
.modes(Set.of(ImageCapability.IMAGE_EDIT))
|
||||
.supports(Set.of("size", "n", "seed", "negative_prompt", "prompt_extend", "watermark"))
|
||||
.maxCount(4)
|
||||
.maxInputImages(3)
|
||||
.build());
|
||||
}
|
||||
}
|
||||
@ -4,6 +4,7 @@ import cn.hutool.http.HttpRequest;
|
||||
import cn.hutool.http.HttpResponse;
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.fasterxml.jackson.databind.node.ArrayNode;
|
||||
import com.fasterxml.jackson.databind.node.ObjectNode;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
@ -11,17 +12,38 @@ import org.springframework.stereotype.Component;
|
||||
import vip.mate.llm.service.ModelProviderService;
|
||||
import vip.mate.system.model.SystemSettingsDTO;
|
||||
import vip.mate.task.AsyncTaskService.TaskPollResult;
|
||||
import vip.mate.tool.image.*;
|
||||
import vip.mate.tool.image.ImageCapability;
|
||||
import vip.mate.tool.image.ImageGenerationProvider;
|
||||
import vip.mate.tool.image.ImageGenerationRequest;
|
||||
import vip.mate.tool.image.ImageModelSpec;
|
||||
import vip.mate.tool.image.ImageProviderCapabilities;
|
||||
import vip.mate.tool.image.ImageReference;
|
||||
import vip.mate.tool.image.ImageSubmitResult;
|
||||
import vip.mate.tool.image.PayloadBuilder;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Base64;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
/**
|
||||
* DashScope 图片生成 Provider — 支持通义万相 Wanx 系列
|
||||
* <p>
|
||||
* 异步模式：提交后返回 taskId，需轮询获取结果。
|
||||
* 复用已有的 DashScope LLM provider 的 API Key。
|
||||
* API 文档: https://help.aliyun.com/zh/model-studio/developer-reference/text-to-image
|
||||
* DashScope image provider — routes per-model between two transports:
|
||||
*
|
||||
* <ul>
|
||||
* <li><b>Async legacy</b> ({@code services/aigc/image-generation/generation})
|
||||
* for the wanx 2.0/2.1, wan 2.2/2.5 turbo/plus families. Submit returns a
|
||||
* task id; the caller polls {@code /api/v1/tasks/{id}} until
|
||||
* SUCCEEDED.</li>
|
||||
* <li><b>Sync multimodal</b> ({@code services/aigc/multimodal-generation/generation})
|
||||
* for wan 2.6/2.7 image, qwen-image, qwen-image-edit, z-image. The
|
||||
* generated image URL is returned in the same response. This endpoint
|
||||
* also accepts inline reference images, enabling the image edit /
|
||||
* image-to-image flow.</li>
|
||||
* </ul>
|
||||
*
|
||||
* The model catalog ({@link DashScopeImageModels}) drives endpoint selection,
|
||||
* payload shape, and the {@code supports} whitelist — adding a new model is a
|
||||
* one-line spec entry.
|
||||
*
|
||||
* @author MateClaw Team
|
||||
*/
|
||||
@ -33,9 +55,6 @@ public class DashScopeImageProvider implements ImageGenerationProvider {
|
||||
private final ModelProviderService modelProviderService;
|
||||
private final ObjectMapper objectMapper;
|
||||
|
||||
private static final String BASE_URL = "https://dashscope.aliyuncs.com/api/v1";
|
||||
private static final String DEFAULT_MODEL = "wanx2.1-t2i-turbo";
|
||||
|
||||
@Override
|
||||
public String id() {
|
||||
return "dashscope";
|
||||
@ -43,7 +62,7 @@ public class DashScopeImageProvider implements ImageGenerationProvider {
|
||||
|
||||
@Override
|
||||
public String label() {
|
||||
return "DashScope (通义万相)";
|
||||
return "DashScope (Tongyi Wanxiang / Qwen-Image)";
|
||||
}
|
||||
|
||||
@Override
|
||||
@ -58,18 +77,32 @@ public class DashScopeImageProvider implements ImageGenerationProvider {
|
||||
|
||||
@Override
|
||||
public Set<ImageCapability> capabilities() {
|
||||
return Set.of(ImageCapability.TEXT_TO_IMAGE);
|
||||
return Set.of(ImageCapability.TEXT_TO_IMAGE, ImageCapability.IMAGE_EDIT);
|
||||
}
|
||||
|
||||
@Override
|
||||
public ImageProviderCapabilities detailedCapabilities() {
|
||||
List<String> modelIds = new ArrayList<>(DashScopeImageModels.all().keySet());
|
||||
return ImageProviderCapabilities.builder()
|
||||
.modes(capabilities())
|
||||
.supportedSizes(List.of("1024x1024", "720x1280", "1280x720"))
|
||||
.supportedSizes(List.of(
|
||||
"1024x1024", "1280x720", "720x1280",
|
||||
"2048x2048", "2560x1440", "1440x2560"))
|
||||
.aspectRatios(List.of("1:1", "16:9", "9:16"))
|
||||
.maxCount(4)
|
||||
.defaultModel(DEFAULT_MODEL)
|
||||
.models(List.of("wanx2.1-t2i-turbo", "wanx-v1"))
|
||||
.defaultModel(DashScopeImageModels.DEFAULT_MODEL)
|
||||
.models(modelIds)
|
||||
.generate(ImageProviderCapabilities.Generate.builder()
|
||||
.maxCount(4).supportsSize(true).supportsAspectRatio(true).build())
|
||||
.edit(ImageProviderCapabilities.Edit.builder()
|
||||
.enabled(true).maxCount(4).maxInputImages(3)
|
||||
.supportsSize(true).supportsAspectRatio(true).build())
|
||||
.geometry(ImageProviderCapabilities.Geometry.builder()
|
||||
.sizes(List.of(
|
||||
"1024x1024", "1280x720", "720x1280",
|
||||
"2048x2048", "2560x1440", "1440x2560"))
|
||||
.aspectRatios(List.of("1:1", "16:9", "9:16"))
|
||||
.build())
|
||||
.build();
|
||||
}
|
||||
|
||||
@ -86,51 +119,16 @@ public class DashScopeImageProvider implements ImageGenerationProvider {
|
||||
public ImageSubmitResult submit(ImageGenerationRequest request, SystemSettingsDTO config) {
|
||||
String apiKey = getDashScopeApiKey();
|
||||
if (apiKey == null) {
|
||||
return ImageSubmitResult.failure(id(), "DashScope API Key 未配置");
|
||||
return ImageSubmitResult.failure(id(), "DashScope API Key not configured");
|
||||
}
|
||||
|
||||
ImageModelSpec spec = resolveSpec(request);
|
||||
try {
|
||||
String model = request.getModel() != null && !request.getModel().isBlank()
|
||||
? request.getModel() : DEFAULT_MODEL;
|
||||
|
||||
ObjectNode body = objectMapper.createObjectNode();
|
||||
body.put("model", model);
|
||||
|
||||
ObjectNode input = body.putObject("input");
|
||||
input.put("prompt", request.getPrompt());
|
||||
|
||||
ObjectNode parameters = body.putObject("parameters");
|
||||
// request.size already normalized by ImageGenerationService to one of supportedSizes.
|
||||
// DashScope API uses '*' separator instead of 'x'.
|
||||
String size = request.getSize();
|
||||
if (size != null && !size.isBlank()) {
|
||||
parameters.put("size", size.replace("x", "*"));
|
||||
}
|
||||
int count = request.getCount() != null ? Math.min(request.getCount(), 4) : 1;
|
||||
parameters.put("n", count);
|
||||
|
||||
HttpResponse response = HttpRequest.post(BASE_URL + "/services/aigc/text2image/image-synthesis")
|
||||
.header("Authorization", "Bearer " + apiKey)
|
||||
.header("Content-Type", "application/json")
|
||||
.header("X-DashScope-Async", "enable")
|
||||
.body(body.toString())
|
||||
.timeout(30_000)
|
||||
.execute();
|
||||
|
||||
JsonNode result = objectMapper.readTree(response.body());
|
||||
|
||||
if (response.getStatus() == 200 && result.has("output")) {
|
||||
String taskId = result.path("output").path("task_id").asText();
|
||||
log.info("[DashScope Image] Submitted task: {} (model={})", taskId, model);
|
||||
return ImageSubmitResult.asyncSuccess(taskId, id());
|
||||
} else {
|
||||
String errMsg = result.has("message") ? result.get("message").asText()
|
||||
: "HTTP " + response.getStatus();
|
||||
log.warn("[DashScope Image] Submit failed: {}", errMsg);
|
||||
return ImageSubmitResult.failure(id(), errMsg);
|
||||
}
|
||||
return spec.transport() == ImageModelSpec.Transport.SYNC
|
||||
? submitSyncMultimodal(request, spec, apiKey)
|
||||
: submitAsyncLegacy(request, spec, apiKey);
|
||||
} catch (Exception e) {
|
||||
log.error("[DashScope Image] Submit error: {}", e.getMessage(), e);
|
||||
log.error("[DashScope Image] Submit error (model={}): {}", spec.id(), e.getMessage(), e);
|
||||
return ImageSubmitResult.failure(id(), e.getMessage());
|
||||
}
|
||||
}
|
||||
@ -139,11 +137,10 @@ public class DashScopeImageProvider implements ImageGenerationProvider {
|
||||
public TaskPollResult checkStatus(String providerTaskId, SystemSettingsDTO config) {
|
||||
String apiKey = getDashScopeApiKey();
|
||||
if (apiKey == null) {
|
||||
return TaskPollResult.failed("DashScope API Key 未配置");
|
||||
return TaskPollResult.failed("DashScope API Key not configured");
|
||||
}
|
||||
|
||||
try {
|
||||
HttpResponse response = HttpRequest.get(BASE_URL + "/tasks/" + providerTaskId)
|
||||
HttpResponse response = HttpRequest.get(DashScopeImageModels.TASKS_ENDPOINT_PREFIX + providerTaskId)
|
||||
.header("Authorization", "Bearer " + apiKey)
|
||||
.timeout(15_000)
|
||||
.execute();
|
||||
@ -154,11 +151,11 @@ public class DashScopeImageProvider implements ImageGenerationProvider {
|
||||
|
||||
return switch (taskStatus) {
|
||||
case "SUCCEEDED" -> {
|
||||
String imageUrl = extractImageUrl(output);
|
||||
String imageUrl = extractLegacyImageUrl(output);
|
||||
yield TaskPollResult.imageSucceeded(imageUrl, output.toString());
|
||||
}
|
||||
case "FAILED" -> {
|
||||
String errMsg = output.has("message") ? output.get("message").asText() : "任务失败";
|
||||
String errMsg = output.has("message") ? output.get("message").asText() : "task failed";
|
||||
yield TaskPollResult.failed(errMsg);
|
||||
}
|
||||
case "RUNNING" -> TaskPollResult.running(null);
|
||||
@ -170,6 +167,156 @@ public class DashScopeImageProvider implements ImageGenerationProvider {
|
||||
}
|
||||
}
|
||||
|
||||
// ==================== spec resolution ====================
|
||||
|
||||
/**
|
||||
* Pick the model spec for this request. When the request asks for image
|
||||
* editing but names a model that doesn't support edits (or names nothing),
|
||||
* fall back to {@link DashScopeImageModels#DEFAULT_EDIT_MODEL} so the call
|
||||
* doesn't silently degrade to a text-only generation.
|
||||
*/
|
||||
private ImageModelSpec resolveSpec(ImageGenerationRequest request) {
|
||||
boolean wantsEdit = request.getInputImages() != null && !request.getInputImages().isEmpty();
|
||||
String requested = request.getModel();
|
||||
ImageModelSpec spec = DashScopeImageModels.get(requested);
|
||||
if (wantsEdit && !spec.supportsEdit()) {
|
||||
ImageModelSpec edit = DashScopeImageModels.get(DashScopeImageModels.DEFAULT_EDIT_MODEL);
|
||||
log.info("[DashScope Image] Model {} lacks edit support; routing to {}", spec.id(), edit.id());
|
||||
return edit;
|
||||
}
|
||||
return spec;
|
||||
}
|
||||
|
||||
// ==================== sync multimodal-generation ====================
|
||||
|
||||
private ImageSubmitResult submitSyncMultimodal(ImageGenerationRequest request,
|
||||
ImageModelSpec spec,
|
||||
String apiKey) throws Exception {
|
||||
ObjectNode body = objectMapper.createObjectNode();
|
||||
body.put("model", spec.id());
|
||||
|
||||
// input.messages[].content[] — image blocks first when editing,
|
||||
// followed by the text prompt block.
|
||||
ObjectNode input = body.putObject("input");
|
||||
ArrayNode messages = input.putArray("messages");
|
||||
ObjectNode userMsg = messages.addObject();
|
||||
userMsg.put("role", "user");
|
||||
ArrayNode content = userMsg.putArray("content");
|
||||
|
||||
if (request.getInputImages() != null) {
|
||||
for (ImageReference ref : request.getInputImages()) {
|
||||
ObjectNode imgPart = content.addObject();
|
||||
imgPart.put("image", toDataUrl(ref));
|
||||
}
|
||||
}
|
||||
ObjectNode textPart = content.addObject();
|
||||
textPart.put("text", request.getPrompt() == null ? "" : request.getPrompt());
|
||||
|
||||
// parameters block — built and filtered against the model's supports set.
|
||||
ObjectNode parameters = PayloadBuilder.from(spec)
|
||||
.withSize(request.getSize(), request.getAspectRatio())
|
||||
.withCount(request.getCount())
|
||||
.toJsonNode(objectMapper);
|
||||
body.set("parameters", parameters);
|
||||
|
||||
HttpResponse response = HttpRequest.post(spec.endpoint())
|
||||
.header("Authorization", "Bearer " + apiKey)
|
||||
.header("Content-Type", "application/json")
|
||||
.body(body.toString())
|
||||
.timeout(180_000)
|
||||
.execute();
|
||||
|
||||
JsonNode result = objectMapper.readTree(response.body());
|
||||
if (response.getStatus() != 200) {
|
||||
String errMsg = result.has("message") ? result.get("message").asText() : "HTTP " + response.getStatus();
|
||||
log.warn("[DashScope Image] Sync submit failed (model={}): {}", spec.id(), errMsg);
|
||||
return ImageSubmitResult.failure(id(), errMsg);
|
||||
}
|
||||
|
||||
List<String> imageUrls = extractMultimodalImageUrls(result);
|
||||
if (imageUrls.isEmpty()) {
|
||||
return ImageSubmitResult.failure(id(), "Multimodal response carried no image URL");
|
||||
}
|
||||
log.info("[DashScope Image] Sync generated {} image(s) (model={})", imageUrls.size(), spec.id());
|
||||
return ImageSubmitResult.syncSuccess(id(), imageUrls);
|
||||
}
|
||||
|
||||
private List<String> extractMultimodalImageUrls(JsonNode result) {
|
||||
List<String> urls = new ArrayList<>();
|
||||
JsonNode choices = result.path("output").path("choices");
|
||||
if (!choices.isArray()) {
|
||||
return urls;
|
||||
}
|
||||
for (JsonNode choice : choices) {
|
||||
JsonNode parts = choice.path("message").path("content");
|
||||
if (!parts.isArray()) continue;
|
||||
for (JsonNode part : parts) {
|
||||
String url = part.path("image").asText(null);
|
||||
if (url != null && !url.isBlank()) {
|
||||
urls.add(url);
|
||||
}
|
||||
}
|
||||
}
|
||||
return urls;
|
||||
}
|
||||
|
||||
// ==================== async legacy image-generation ====================
|
||||
|
||||
private ImageSubmitResult submitAsyncLegacy(ImageGenerationRequest request,
|
||||
ImageModelSpec spec,
|
||||
String apiKey) throws Exception {
|
||||
ObjectNode body = objectMapper.createObjectNode();
|
||||
body.put("model", spec.id());
|
||||
|
||||
ObjectNode input = body.putObject("input");
|
||||
input.put("prompt", request.getPrompt() == null ? "" : request.getPrompt());
|
||||
|
||||
ObjectNode parameters = PayloadBuilder.from(spec)
|
||||
.withSize(request.getSize(), request.getAspectRatio())
|
||||
.withCount(request.getCount())
|
||||
.toJsonNode(objectMapper);
|
||||
body.set("parameters", parameters);
|
||||
|
||||
HttpResponse response = HttpRequest.post(spec.endpoint())
|
||||
.header("Authorization", "Bearer " + apiKey)
|
||||
.header("Content-Type", "application/json")
|
||||
.header("X-DashScope-Async", "enable")
|
||||
.body(body.toString())
|
||||
.timeout(30_000)
|
||||
.execute();
|
||||
|
||||
JsonNode result = objectMapper.readTree(response.body());
|
||||
if (response.getStatus() == 200 && result.has("output")) {
|
||||
String taskId = result.path("output").path("task_id").asText();
|
||||
log.info("[DashScope Image] Async submitted task {} (model={})", taskId, spec.id());
|
||||
return ImageSubmitResult.asyncSuccess(taskId, id());
|
||||
}
|
||||
String errMsg = result.has("message") ? result.get("message").asText()
|
||||
: "HTTP " + response.getStatus();
|
||||
log.warn("[DashScope Image] Async submit failed (model={}): {}", spec.id(), errMsg);
|
||||
return ImageSubmitResult.failure(id(), errMsg);
|
||||
}
|
||||
|
||||
private String extractLegacyImageUrl(JsonNode output) {
|
||||
JsonNode results = output.path("results");
|
||||
if (results.isArray() && !results.isEmpty()) {
|
||||
JsonNode first = results.get(0);
|
||||
String url = first.path("url").asText(null);
|
||||
if (url == null || url.isBlank()) {
|
||||
url = first.path("image").asText(null);
|
||||
}
|
||||
return url;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
// ==================== shared helpers ====================
|
||||
|
||||
private String toDataUrl(ImageReference ref) {
|
||||
String mime = ref.mimeType() == null || ref.mimeType().isBlank() ? "image/png" : ref.mimeType();
|
||||
return "data:" + mime + ";base64," + Base64.getEncoder().encodeToString(ref.data());
|
||||
}
|
||||
|
||||
private String getDashScopeApiKey() {
|
||||
try {
|
||||
var providerEntity = modelProviderService.getProviderConfig("dashscope");
|
||||
@ -178,12 +325,4 @@ public class DashScopeImageProvider implements ImageGenerationProvider {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private String extractImageUrl(JsonNode output) {
|
||||
JsonNode results = output.path("results");
|
||||
if (results.isArray() && !results.isEmpty()) {
|
||||
return results.get(0).path("url").asText(null);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
@ -4,6 +4,7 @@ import cn.hutool.http.HttpRequest;
|
||||
import cn.hutool.http.HttpResponse;
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.fasterxml.jackson.databind.node.ArrayNode;
|
||||
import com.fasterxml.jackson.databind.node.ObjectNode;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
@ -11,16 +12,35 @@ import org.springframework.stereotype.Component;
|
||||
import vip.mate.llm.service.ModelProviderService;
|
||||
import vip.mate.system.model.SystemSettingsDTO;
|
||||
import vip.mate.task.AsyncTaskService.TaskPollResult;
|
||||
import vip.mate.tool.video.*;
|
||||
import vip.mate.tool.video.VideoCapability;
|
||||
import vip.mate.tool.video.VideoGenerationProvider;
|
||||
import vip.mate.tool.video.VideoGenerationRequest;
|
||||
import vip.mate.tool.video.VideoProviderCapabilities;
|
||||
import vip.mate.tool.video.VideoSubmitResult;
|
||||
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
/**
|
||||
* DashScope 视频生成 Provider — 支持通义万相 Wan 2.5 / Wanx 2.1
|
||||
* <p>
|
||||
* 复用已有的 DashScope LLM provider 的 API Key。
|
||||
* API 文档: https://help.aliyun.com/zh/model-studio/developer-reference/video-generation
|
||||
* DashScope video provider — supports two payload families on the same async
|
||||
* task model, selected per model id:
|
||||
*
|
||||
* <ul>
|
||||
* <li><b>Legacy</b> ({@code services/aigc/video-generation/generation}) for
|
||||
* wanx 2.1 and wan 2.5 turbo lines. Body uses {@code input.img_url} for
|
||||
* image-to-video and {@code parameters.size} for sizing.</li>
|
||||
* <li><b>Unified video-synthesis</b>
|
||||
* ({@code services/aigc/video-generation/video-synthesis}) for wan 2.7
|
||||
* and the happyhorse t2v line. Body uses {@code input.media[]} for the
|
||||
* first frame plus {@code parameters.resolution} + {@code parameters.ratio}
|
||||
* for sizing.</li>
|
||||
* </ul>
|
||||
*
|
||||
* Routing is data-driven: each model is registered with its endpoint, body
|
||||
* shape, and capability set; submit/build code consults the spec rather than
|
||||
* branching on model id strings.
|
||||
*
|
||||
* @author MateClaw Team
|
||||
*/
|
||||
@ -33,9 +53,52 @@ public class DashScopeVideoProvider implements VideoGenerationProvider {
|
||||
private final ObjectMapper objectMapper;
|
||||
|
||||
private static final String BASE_URL = "https://dashscope.aliyuncs.com/api/v1";
|
||||
private static final String LEGACY_ENDPOINT = BASE_URL + "/services/aigc/video-generation/generation";
|
||||
private static final String UNIFIED_ENDPOINT = BASE_URL + "/services/aigc/video-generation/video-synthesis";
|
||||
private static final String TASKS_ENDPOINT_PREFIX = BASE_URL + "/tasks/";
|
||||
|
||||
private static final String DEFAULT_T2V_MODEL = "wan2.5-t2v-turbo";
|
||||
private static final String DEFAULT_I2V_MODEL = "wan2.5-i2v-turbo";
|
||||
|
||||
private enum BodyShape {
|
||||
/** input.img_url + parameters.size("1280*720") + parameters.duration. */
|
||||
LEGACY,
|
||||
/** input.media[].first_frame + parameters.resolution + parameters.ratio + parameters.duration. */
|
||||
UNIFIED
|
||||
}
|
||||
|
||||
private record ModelSpec(
|
||||
String id,
|
||||
String endpoint,
|
||||
BodyShape bodyShape,
|
||||
Set<VideoCapability> modes
|
||||
) {}
|
||||
|
||||
private static final Map<String, ModelSpec> MODELS = buildCatalog();
|
||||
|
||||
private static Map<String, ModelSpec> buildCatalog() {
|
||||
Map<String, ModelSpec> m = new LinkedHashMap<>();
|
||||
// Legacy line — text-to-video
|
||||
m.put("wan2.5-t2v-turbo", new ModelSpec("wan2.5-t2v-turbo",
|
||||
LEGACY_ENDPOINT, BodyShape.LEGACY, Set.of(VideoCapability.GENERATE)));
|
||||
m.put("wanx2.1-t2v-turbo", new ModelSpec("wanx2.1-t2v-turbo",
|
||||
LEGACY_ENDPOINT, BodyShape.LEGACY, Set.of(VideoCapability.GENERATE)));
|
||||
// Legacy line — image-to-video
|
||||
m.put("wan2.5-i2v-turbo", new ModelSpec("wan2.5-i2v-turbo",
|
||||
LEGACY_ENDPOINT, BodyShape.LEGACY, Set.of(VideoCapability.IMAGE_TO_VIDEO)));
|
||||
m.put("wanx2.1-i2v-turbo", new ModelSpec("wanx2.1-i2v-turbo",
|
||||
LEGACY_ENDPOINT, BodyShape.LEGACY, Set.of(VideoCapability.IMAGE_TO_VIDEO)));
|
||||
// Unified video-synthesis line — wan 2.7
|
||||
m.put("wan2.7-t2v-2026-04-25", new ModelSpec("wan2.7-t2v-2026-04-25",
|
||||
UNIFIED_ENDPOINT, BodyShape.UNIFIED, Set.of(VideoCapability.GENERATE)));
|
||||
m.put("wan2.7-i2v-2026-04-25", new ModelSpec("wan2.7-i2v-2026-04-25",
|
||||
UNIFIED_ENDPOINT, BodyShape.UNIFIED, Set.of(VideoCapability.IMAGE_TO_VIDEO)));
|
||||
// Unified video-synthesis line — happyhorse text-to-video
|
||||
m.put("happyhorse-1.0-t2v", new ModelSpec("happyhorse-1.0-t2v",
|
||||
UNIFIED_ENDPOINT, BodyShape.UNIFIED, Set.of(VideoCapability.GENERATE)));
|
||||
return Map.copyOf(m);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String id() {
|
||||
return "dashscope";
|
||||
@ -43,7 +106,7 @@ public class DashScopeVideoProvider implements VideoGenerationProvider {
|
||||
|
||||
@Override
|
||||
public String label() {
|
||||
return "DashScope (通义万相)";
|
||||
return "DashScope (Tongyi Wanxiang / HappyHorse)";
|
||||
}
|
||||
|
||||
@Override
|
||||
@ -66,10 +129,10 @@ public class DashScopeVideoProvider implements VideoGenerationProvider {
|
||||
return VideoProviderCapabilities.builder()
|
||||
.modes(capabilities())
|
||||
.aspectRatios(List.of("16:9", "9:16", "1:1"))
|
||||
.supportedDurations(List.of(5, 10))
|
||||
.maxDurationSeconds(10)
|
||||
.supportedDurations(List.of(2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15))
|
||||
.maxDurationSeconds(15)
|
||||
.defaultModel(DEFAULT_T2V_MODEL)
|
||||
.models(List.of("wan2.5-t2v-turbo", "wan2.5-i2v-turbo", "wanx2.1-t2v-turbo", "wanx2.1-i2v-turbo"))
|
||||
.models(List.copyOf(MODELS.keySet()))
|
||||
.build();
|
||||
}
|
||||
|
||||
@ -86,14 +149,12 @@ public class DashScopeVideoProvider implements VideoGenerationProvider {
|
||||
public VideoSubmitResult submit(VideoGenerationRequest request, SystemSettingsDTO config) {
|
||||
String apiKey = getDashScopeApiKey();
|
||||
if (apiKey == null) {
|
||||
return VideoSubmitResult.failure(id(), "DashScope API Key 未配置");
|
||||
return VideoSubmitResult.failure(id(), "DashScope API Key not configured");
|
||||
}
|
||||
|
||||
ModelSpec spec = resolveSpec(request);
|
||||
try {
|
||||
String model = resolveModel(request);
|
||||
ObjectNode body = buildRequestBody(request, model);
|
||||
|
||||
HttpResponse response = HttpRequest.post(BASE_URL + "/services/aigc/video-generation/generation")
|
||||
ObjectNode body = buildRequestBody(request, spec);
|
||||
HttpResponse response = HttpRequest.post(spec.endpoint())
|
||||
.header("Authorization", "Bearer " + apiKey)
|
||||
.header("Content-Type", "application/json")
|
||||
.header("X-DashScope-Async", "enable")
|
||||
@ -102,19 +163,17 @@ public class DashScopeVideoProvider implements VideoGenerationProvider {
|
||||
.execute();
|
||||
|
||||
JsonNode result = objectMapper.readTree(response.body());
|
||||
|
||||
if (response.getStatus() == 200 && result.has("output")) {
|
||||
String taskId = result.path("output").path("task_id").asText();
|
||||
log.info("[DashScope Video] Submitted task: {} (model={})", taskId, model);
|
||||
log.info("[DashScope Video] Submitted task {} (model={})", taskId, spec.id());
|
||||
return VideoSubmitResult.success(taskId, id());
|
||||
} else {
|
||||
String errMsg = result.has("message") ? result.get("message").asText()
|
||||
: "HTTP " + response.getStatus();
|
||||
log.warn("[DashScope Video] Submit failed: {}", errMsg);
|
||||
return VideoSubmitResult.failure(id(), errMsg);
|
||||
}
|
||||
String errMsg = result.has("message") ? result.get("message").asText()
|
||||
: "HTTP " + response.getStatus();
|
||||
log.warn("[DashScope Video] Submit failed (model={}): {}", spec.id(), errMsg);
|
||||
return VideoSubmitResult.failure(id(), errMsg);
|
||||
} catch (Exception e) {
|
||||
log.error("[DashScope Video] Submit error: {}", e.getMessage(), e);
|
||||
log.error("[DashScope Video] Submit error (model={}): {}", spec.id(), e.getMessage(), e);
|
||||
return VideoSubmitResult.failure(id(), e.getMessage());
|
||||
}
|
||||
}
|
||||
@ -123,11 +182,10 @@ public class DashScopeVideoProvider implements VideoGenerationProvider {
|
||||
public TaskPollResult checkStatus(String providerTaskId, SystemSettingsDTO config) {
|
||||
String apiKey = getDashScopeApiKey();
|
||||
if (apiKey == null) {
|
||||
return TaskPollResult.failed("DashScope API Key 未配置");
|
||||
return TaskPollResult.failed("DashScope API Key not configured");
|
||||
}
|
||||
|
||||
try {
|
||||
HttpResponse response = HttpRequest.get(BASE_URL + "/tasks/" + providerTaskId)
|
||||
HttpResponse response = HttpRequest.get(TASKS_ENDPOINT_PREFIX + providerTaskId)
|
||||
.header("Authorization", "Bearer " + apiKey)
|
||||
.timeout(15_000)
|
||||
.execute();
|
||||
@ -135,14 +193,13 @@ public class DashScopeVideoProvider implements VideoGenerationProvider {
|
||||
JsonNode result = objectMapper.readTree(response.body());
|
||||
JsonNode output = result.path("output");
|
||||
String taskStatus = output.path("task_status").asText();
|
||||
|
||||
return switch (taskStatus) {
|
||||
case "SUCCEEDED" -> {
|
||||
String videoUrl = extractVideoUrl(output);
|
||||
yield TaskPollResult.succeeded(videoUrl, null, output.toString());
|
||||
}
|
||||
case "FAILED" -> {
|
||||
String errMsg = output.has("message") ? output.get("message").asText() : "任务失败";
|
||||
String errMsg = output.has("message") ? output.get("message").asText() : "task failed";
|
||||
yield TaskPollResult.failed(errMsg);
|
||||
}
|
||||
case "RUNNING" -> TaskPollResult.running(null);
|
||||
@ -150,11 +207,107 @@ public class DashScopeVideoProvider implements VideoGenerationProvider {
|
||||
};
|
||||
} catch (Exception e) {
|
||||
log.error("[DashScope Video] Poll error for task {}: {}", providerTaskId, e.getMessage());
|
||||
return null; // 轮询异常不终止，等下次重试
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// ==================== 内部方法 ====================
|
||||
// ==================== spec resolution ====================
|
||||
|
||||
private ModelSpec resolveSpec(VideoGenerationRequest request) {
|
||||
String requested = request.getModel();
|
||||
if (requested != null && !requested.isBlank() && MODELS.containsKey(requested)) {
|
||||
return MODELS.get(requested);
|
||||
}
|
||||
// Fall back to a default by mode.
|
||||
String defaultId = request.getMode() == VideoCapability.IMAGE_TO_VIDEO
|
||||
? DEFAULT_I2V_MODEL : DEFAULT_T2V_MODEL;
|
||||
return MODELS.get(defaultId);
|
||||
}
|
||||
|
||||
// ==================== body building ====================
|
||||
|
||||
private ObjectNode buildRequestBody(VideoGenerationRequest request, ModelSpec spec) {
|
||||
return switch (spec.bodyShape()) {
|
||||
case LEGACY -> buildLegacyBody(request, spec);
|
||||
case UNIFIED -> buildUnifiedBody(request, spec);
|
||||
};
|
||||
}
|
||||
|
||||
private ObjectNode buildLegacyBody(VideoGenerationRequest request, ModelSpec spec) {
|
||||
ObjectNode body = objectMapper.createObjectNode();
|
||||
body.put("model", spec.id());
|
||||
|
||||
ObjectNode input = body.putObject("input");
|
||||
input.put("prompt", request.getPrompt() == null ? "" : request.getPrompt());
|
||||
if (spec.modes().contains(VideoCapability.IMAGE_TO_VIDEO)
|
||||
&& request.getImageUrl() != null && !request.getImageUrl().isBlank()) {
|
||||
input.put("img_url", request.getImageUrl());
|
||||
}
|
||||
|
||||
ObjectNode parameters = body.putObject("parameters");
|
||||
String size = aspectRatioToLegacySize(request.getAspectRatio());
|
||||
if (size != null) {
|
||||
parameters.put("size", size);
|
||||
}
|
||||
if (request.getDurationSeconds() != null) {
|
||||
parameters.put("duration", String.valueOf(request.getDurationSeconds()));
|
||||
}
|
||||
return body;
|
||||
}
|
||||
|
||||
private ObjectNode buildUnifiedBody(VideoGenerationRequest request, ModelSpec spec) {
|
||||
ObjectNode body = objectMapper.createObjectNode();
|
||||
body.put("model", spec.id());
|
||||
|
||||
ObjectNode input = body.putObject("input");
|
||||
input.put("prompt", request.getPrompt() == null ? "" : request.getPrompt());
|
||||
if (spec.modes().contains(VideoCapability.IMAGE_TO_VIDEO)
|
||||
&& request.getImageUrl() != null && !request.getImageUrl().isBlank()) {
|
||||
ArrayNode media = input.putArray("media");
|
||||
ObjectNode firstFrame = media.addObject();
|
||||
firstFrame.put("type", "first_frame");
|
||||
firstFrame.put("url", request.getImageUrl());
|
||||
}
|
||||
|
||||
ObjectNode parameters = body.putObject("parameters");
|
||||
String resolution = aspectRatioToUnifiedResolution(request.getAspectRatio());
|
||||
parameters.put("resolution", resolution);
|
||||
if (request.getAspectRatio() != null && !request.getAspectRatio().isBlank()) {
|
||||
parameters.put("ratio", request.getAspectRatio());
|
||||
}
|
||||
if (request.getDurationSeconds() != null) {
|
||||
// Unified endpoint expects the duration as an integer.
|
||||
parameters.put("duration", request.getDurationSeconds());
|
||||
}
|
||||
return body;
|
||||
}
|
||||
|
||||
private String aspectRatioToLegacySize(String aspectRatio) {
|
||||
if (aspectRatio == null) return null;
|
||||
return switch (aspectRatio) {
|
||||
case "16:9" -> "1280*720";
|
||||
case "9:16" -> "720*1280";
|
||||
case "1:1" -> "720*720";
|
||||
default -> null;
|
||||
};
|
||||
}
|
||||
|
||||
private String aspectRatioToUnifiedResolution(String aspectRatio) {
|
||||
// Default to 720P; the unified endpoint also accepts 1080P. Callers that
|
||||
// want to override should pass it via extraParams in a future iteration.
|
||||
return "720P";
|
||||
}
|
||||
|
||||
private String extractVideoUrl(JsonNode output) {
|
||||
if (output.has("video_url")) {
|
||||
return output.get("video_url").asText();
|
||||
}
|
||||
JsonNode results = output.path("results");
|
||||
if (results.isArray() && !results.isEmpty()) {
|
||||
return results.get(0).path("url").asText(null);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private String getDashScopeApiKey() {
|
||||
try {
|
||||
@ -164,59 +317,4 @@ public class DashScopeVideoProvider implements VideoGenerationProvider {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private String resolveModel(VideoGenerationRequest request) {
|
||||
if (request.getModel() != null && !request.getModel().isBlank()) {
|
||||
return request.getModel();
|
||||
}
|
||||
return request.getMode() == VideoCapability.IMAGE_TO_VIDEO
|
||||
? DEFAULT_I2V_MODEL : DEFAULT_T2V_MODEL;
|
||||
}
|
||||
|
||||
private ObjectNode buildRequestBody(VideoGenerationRequest request, String model) {
|
||||
ObjectNode body = objectMapper.createObjectNode();
|
||||
body.put("model", model);
|
||||
|
||||
ObjectNode input = body.putObject("input");
|
||||
input.put("prompt", request.getPrompt());
|
||||
|
||||
if (request.getMode() == VideoCapability.IMAGE_TO_VIDEO && request.getImageUrl() != null) {
|
||||
input.put("img_url", request.getImageUrl());
|
||||
}
|
||||
|
||||
ObjectNode parameters = body.putObject("parameters");
|
||||
if (request.getAspectRatio() != null) {
|
||||
// DashScope 使用 size 参数，如 "1280*720"
|
||||
String size = aspectRatioToSize(request.getAspectRatio());
|
||||
if (size != null) {
|
||||
parameters.put("size", size);
|
||||
}
|
||||
}
|
||||
if (request.getDurationSeconds() != null) {
|
||||
parameters.put("duration", String.valueOf(request.getDurationSeconds()));
|
||||
}
|
||||
|
||||
return body;
|
||||
}
|
||||
|
||||
private String aspectRatioToSize(String aspectRatio) {
|
||||
return switch (aspectRatio) {
|
||||
case "16:9" -> "1280*720";
|
||||
case "9:16" -> "720*1280";
|
||||
case "1:1" -> "720*720";
|
||||
default -> null;
|
||||
};
|
||||
}
|
||||
|
||||
private String extractVideoUrl(JsonNode output) {
|
||||
JsonNode results = output.path("results");
|
||||
if (results.isArray() && !results.isEmpty()) {
|
||||
return results.get(0).path("url").asText(null);
|
||||
}
|
||||
// 有些模型返回 video_url
|
||||
if (output.has("video_url")) {
|
||||
return output.get("video_url").asText();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user