mirror of
https://gitee.com/mateos/mateclaw.git
synced 2026-09-16 04:18:17 +08:00
feat(tool/image,video): image edit support + new DashScope models (#75)
This commit is contained in:
parent
0cbe271f6f
commit
3f289da6b7
@ -13,6 +13,7 @@ import vip.mate.task.AsyncTaskService;
|
|||||||
import vip.mate.task.model.AsyncTaskInfo;
|
import vip.mate.task.model.AsyncTaskInfo;
|
||||||
import vip.mate.tool.image.*;
|
import vip.mate.tool.image.*;
|
||||||
|
|
||||||
|
import java.util.ArrayList;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.StringJoiner;
|
import java.util.StringJoiner;
|
||||||
|
|
||||||
@ -30,19 +31,23 @@ public class ImageGenerateTool {
|
|||||||
private final ImageProviderRegistry providerRegistry;
|
private final ImageProviderRegistry providerRegistry;
|
||||||
private final SystemSettingService systemSettingService;
|
private final SystemSettingService systemSettingService;
|
||||||
private final AsyncTaskService asyncTaskService;
|
private final AsyncTaskService asyncTaskService;
|
||||||
|
private final ImageReferenceLoader imageReferenceLoader;
|
||||||
|
|
||||||
@vip.mate.tool.ConcurrencyUnsafe("creates async tasks and persists generated artifacts; provider rate limits also forbid parallel calls")
|
@vip.mate.tool.ConcurrencyUnsafe("creates async tasks and persists generated artifacts; provider rate limits also forbid parallel calls")
|
||||||
@Tool(description = "Image generation tool. Supports actions: generate (default), list (show available providers), "
|
@Tool(description = "Image generation tool. Supports actions: generate (default — text-to-image, OR image-edit when "
|
||||||
+ "status (check task status). Some providers are async (30s-2min), results auto-displayed in conversation.")
|
+ "image/images parameters are set), list (show available providers/models), status (check task status). "
|
||||||
|
+ "Reference images may be local paths, http(s) URLs, data: URLs, or msg:<messageId>:<idx> for an attachment "
|
||||||
|
+ "from an earlier conversation message. Async providers take 30s-2min; results auto-display in the conversation.")
|
||||||
public String image_generate(
|
public String image_generate(
|
||||||
@ToolParam(description = "Action type: generate, list, status. Default: generate", required = false) String action,
|
@ToolParam(description = "Action type: generate, list, status. Default: generate", required = false) String action,
|
||||||
@ToolParam(description = "Image content description, be detailed (required for generate)", required = false) String prompt,
|
@ToolParam(description = "Image content description, be detailed (required for generate)", required = false) String prompt,
|
||||||
|
@ToolParam(description = "Single reference image for edit mode. Path / http(s) URL / data: URL / msg:<messageId>[:<idx>]", required = false) String image,
|
||||||
|
@ToolParam(description = "Multiple reference images for edit mode (provider caps the count). Same formats as 'image'.", required = false) List<String> images,
|
||||||
@ToolParam(description = "Image size: 1024x1024 / 1024x1792 / 1792x1024", required = false) String size,
|
@ToolParam(description = "Image size: 1024x1024 / 1024x1792 / 1792x1024", required = false) String size,
|
||||||
@ToolParam(description = "Aspect ratio: 1:1 / 16:9 / 9:16, default 1:1", required = false) String aspectRatio,
|
@ToolParam(description = "Aspect ratio: 1:1 / 16:9 / 9:16, default 1:1", required = false) String aspectRatio,
|
||||||
@ToolParam(description = "Generation count (1-4), default 1", required = false) Integer count,
|
@ToolParam(description = "Generation count (1-4), default 1", required = false) Integer count,
|
||||||
@ToolParam(description = "Model name (optional)", required = false) String model,
|
@ToolParam(description = "Model name (optional)", required = false) String model,
|
||||||
@ToolParam(description = "Task ID to check status (for status action)", required = false) String taskId,
|
@ToolParam(description = "Task ID to check status (for status action)", required = false) String taskId,
|
||||||
// RFC-063r §2.5: ToolContext is hidden from the LLM by JsonSchemaGenerator.
|
|
||||||
@Nullable ToolContext ctx
|
@Nullable ToolContext ctx
|
||||||
) {
|
) {
|
||||||
String normalizedAction = (action == null || action.isBlank()) ? "generate" : action.trim().toLowerCase();
|
String normalizedAction = (action == null || action.isBlank()) ? "generate" : action.trim().toLowerCase();
|
||||||
@ -50,7 +55,7 @@ public class ImageGenerateTool {
|
|||||||
return switch (normalizedAction) {
|
return switch (normalizedAction) {
|
||||||
case "list" -> handleListAction();
|
case "list" -> handleListAction();
|
||||||
case "status" -> handleStatusAction(taskId, ctx);
|
case "status" -> handleStatusAction(taskId, ctx);
|
||||||
default -> handleGenerateAction(prompt, size, aspectRatio, count, model, ctx);
|
default -> handleGenerateAction(prompt, image, images, size, aspectRatio, count, model, ctx);
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -120,7 +125,8 @@ public class ImageGenerateTool {
|
|||||||
|
|
||||||
// ==================== action=generate ====================
|
// ==================== action=generate ====================
|
||||||
|
|
||||||
private String handleGenerateAction(String prompt, String size, String aspectRatio,
|
private String handleGenerateAction(String prompt, String image, List<String> images,
|
||||||
|
String size, String aspectRatio,
|
||||||
Integer count, String model, @Nullable ToolContext ctx) {
|
Integer count, String model, @Nullable ToolContext ctx) {
|
||||||
String conversationId = ToolExecutionContext.conversationId(ctx);
|
String conversationId = ToolExecutionContext.conversationId(ctx);
|
||||||
String username = ToolExecutionContext.username(ctx);
|
String username = ToolExecutionContext.username(ctx);
|
||||||
@ -133,12 +139,33 @@ public class ImageGenerateTool {
|
|||||||
return "错误:prompt 为必填参数,请描述你想要生成的图片内容";
|
return "错误:prompt 为必填参数,请描述你想要生成的图片内容";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Combine the singular and plural forms — the agent picks whichever is
|
||||||
|
// ergonomic. Order: image (first) then images[].
|
||||||
|
List<String> referenceInputs = new ArrayList<>();
|
||||||
|
if (image != null && !image.isBlank()) {
|
||||||
|
referenceInputs.add(image);
|
||||||
|
}
|
||||||
|
if (images != null) {
|
||||||
|
for (String s : images) {
|
||||||
|
if (s != null && !s.isBlank()) referenceInputs.add(s);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
List<ImageReference> inputImages;
|
||||||
|
try {
|
||||||
|
inputImages = imageReferenceLoader.loadAll(referenceInputs, conversationId);
|
||||||
|
} catch (Exception e) {
|
||||||
|
log.warn("[ImageGenerateTool] Failed to load reference images: {}", e.getMessage());
|
||||||
|
return "错误:无法加载参考图片:" + e.getMessage();
|
||||||
|
}
|
||||||
|
|
||||||
ImageGenerationRequest request = ImageGenerationRequest.builder()
|
ImageGenerationRequest request = ImageGenerationRequest.builder()
|
||||||
.prompt(prompt)
|
.prompt(prompt)
|
||||||
.size(size)
|
.size(size)
|
||||||
.aspectRatio(aspectRatio != null ? aspectRatio : "1:1")
|
.aspectRatio(aspectRatio != null ? aspectRatio : "1:1")
|
||||||
.count(count != null ? count : 1)
|
.count(count != null ? count : 1)
|
||||||
.model(model)
|
.model(model)
|
||||||
|
.inputImages(inputImages)
|
||||||
.build();
|
.build();
|
||||||
|
|
||||||
ImageGenerationResult result = imageGenerationService.submitGeneration(
|
ImageGenerationResult result = imageGenerationService.submitGeneration(
|
||||||
|
|||||||
@ -3,10 +3,11 @@ package vip.mate.tool.image;
|
|||||||
import lombok.Builder;
|
import lombok.Builder;
|
||||||
import lombok.Data;
|
import lombok.Data;
|
||||||
|
|
||||||
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 图片生成统一请求
|
* Unified image-generation request.
|
||||||
*
|
*
|
||||||
* @author MateClaw Team
|
* @author MateClaw Team
|
||||||
*/
|
*/
|
||||||
@ -14,30 +15,36 @@ import java.util.Map;
|
|||||||
@Builder
|
@Builder
|
||||||
public class ImageGenerationRequest {
|
public class ImageGenerationRequest {
|
||||||
|
|
||||||
/** 图片内容描述 */
|
/** Prompt describing the desired image. */
|
||||||
private String prompt;
|
private String prompt;
|
||||||
|
|
||||||
/** 生成模式(由 runtime 自动推断) */
|
/** Generation mode (inferred by the runtime when null). */
|
||||||
private ImageCapability mode;
|
private ImageCapability mode;
|
||||||
|
|
||||||
/** 指定模型名称(可选,provider 有默认值) */
|
/** Model id; provider supplies a default when null/blank. */
|
||||||
private String model;
|
private String model;
|
||||||
|
|
||||||
/** 图片尺寸:1024x1024 / 1024x1792 / 1792x1024 等 */
|
/** Pixel size like {@code 1024x1024} / {@code 1024x1792}. */
|
||||||
@Builder.Default
|
@Builder.Default
|
||||||
private String size = "1024x1024";
|
private String size = "1024x1024";
|
||||||
|
|
||||||
/** 画面比例:1:1 / 16:9 / 9:16 */
|
/** Aspect ratio: {@code 1:1} / {@code 16:9} / {@code 9:16}. */
|
||||||
@Builder.Default
|
@Builder.Default
|
||||||
private String aspectRatio = "1:1";
|
private String aspectRatio = "1:1";
|
||||||
|
|
||||||
/** 生成数量 */
|
/** Number of images to return. */
|
||||||
@Builder.Default
|
@Builder.Default
|
||||||
private Integer count = 1;
|
private Integer count = 1;
|
||||||
|
|
||||||
/** 参考图片 URL(IMAGE_EDIT 模式) */
|
/**
|
||||||
private String referenceImageUrl;
|
* Reference images for edit / image-to-image flows. Loaded as in-memory
|
||||||
|
* buffers so providers can either inline base64, upload via multipart, or
|
||||||
|
* forward as a URL — without each provider re-implementing path/URL/data
|
||||||
|
* resolution.
|
||||||
|
*/
|
||||||
|
@Builder.Default
|
||||||
|
private List<ImageReference> inputImages = List.of();
|
||||||
|
|
||||||
/** provider 特有的额外参数 */
|
/** Provider-specific extras forwarded as-is. */
|
||||||
private Map<String, Object> extraParams;
|
private Map<String, Object> extraParams;
|
||||||
}
|
}
|
||||||
|
|||||||
@ -279,7 +279,7 @@ public class ImageGenerationService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private ImageCapability inferMode(ImageGenerationRequest request) {
|
private ImageCapability inferMode(ImageGenerationRequest request) {
|
||||||
if (request.getReferenceImageUrl() != null && !request.getReferenceImageUrl().isBlank()) {
|
if (request.getInputImages() != null && !request.getInputImages().isEmpty()) {
|
||||||
return ImageCapability.IMAGE_EDIT;
|
return ImageCapability.IMAGE_EDIT;
|
||||||
}
|
}
|
||||||
return ImageCapability.TEXT_TO_IMAGE;
|
return ImageCapability.TEXT_TO_IMAGE;
|
||||||
|
|||||||
@ -0,0 +1,57 @@
|
|||||||
|
package vip.mate.tool.image;
|
||||||
|
|
||||||
|
import lombok.Builder;
|
||||||
|
import lombok.Singular;
|
||||||
|
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.Set;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Per-model descriptor that drives payload construction without {@code if/else}
|
||||||
|
* chains inside provider classes. Adding a new model = adding a new spec entry.
|
||||||
|
*
|
||||||
|
* <p>Three things make this configuration-driven:
|
||||||
|
* <ul>
|
||||||
|
* <li>{@code endpoint} chooses which provider URL to hit. A single provider
|
||||||
|
* (e.g. DashScope) can host both an async legacy endpoint and a unified
|
||||||
|
* multimodal endpoint — the spec routes per model.</li>
|
||||||
|
* <li>{@code transport} ({@link Transport#SYNC} / {@link Transport#ASYNC})
|
||||||
|
* lets the provider pick between immediate-return and submit+poll without
|
||||||
|
* hard-coding the choice.</li>
|
||||||
|
* <li>{@code supports} acts as a payload key whitelist. Build the full payload
|
||||||
|
* freely, then filter against {@code supports} so models never receive
|
||||||
|
* fields they reject.</li>
|
||||||
|
* </ul>
|
||||||
|
*
|
||||||
|
* @author MateClaw Team
|
||||||
|
*/
|
||||||
|
@Builder
|
||||||
|
public record ImageModelSpec(
|
||||||
|
String id,
|
||||||
|
String displayName,
|
||||||
|
String endpoint,
|
||||||
|
Transport transport,
|
||||||
|
SizeStyle sizeStyle,
|
||||||
|
@Singular("sizeMapping") Map<String, String> sizeMap,
|
||||||
|
@Singular("defaultParam") Map<String, Object> defaults,
|
||||||
|
@Singular Set<String> supports,
|
||||||
|
@Singular Set<ImageCapability> modes,
|
||||||
|
int maxInputImages,
|
||||||
|
int maxCount
|
||||||
|
) {
|
||||||
|
|
||||||
|
public enum Transport {
|
||||||
|
/** Provider returns image bytes / URL in the same HTTP response. */
|
||||||
|
SYNC,
|
||||||
|
/** Provider returns a task id; caller polls a status endpoint. */
|
||||||
|
ASYNC
|
||||||
|
}
|
||||||
|
|
||||||
|
public boolean supportsEdit() {
|
||||||
|
return modes != null && modes.contains(ImageCapability.IMAGE_EDIT);
|
||||||
|
}
|
||||||
|
|
||||||
|
public boolean supportsGenerate() {
|
||||||
|
return modes != null && modes.contains(ImageCapability.TEXT_TO_IMAGE);
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -7,7 +7,15 @@ import java.util.List;
|
|||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 图片生成 Provider 细粒度能力声明
|
* Image generation provider capability declaration.
|
||||||
|
*
|
||||||
|
* <p>The flat top-level fields ({@code supportedSizes}, {@code aspectRatios},
|
||||||
|
* {@code maxCount}, {@code modes}) describe the provider's combined surface
|
||||||
|
* area and remain in use by callers that don't need per-mode granularity.
|
||||||
|
* Newer code should consult the structured {@link Generate} / {@link Edit} /
|
||||||
|
* {@link Geometry} / {@link Output} fields, which let the picker show
|
||||||
|
* "edit supports up to N reference images" or "generate accepts these
|
||||||
|
* formats" without conflating the two modes.
|
||||||
*
|
*
|
||||||
* @author MateClaw Team
|
* @author MateClaw Team
|
||||||
*/
|
*/
|
||||||
@ -15,29 +23,87 @@ import java.util.Set;
|
|||||||
@Builder
|
@Builder
|
||||||
public class ImageProviderCapabilities {
|
public class ImageProviderCapabilities {
|
||||||
|
|
||||||
/** 支持的生成模式 */
|
/** Combined modes the provider supports across all its models. */
|
||||||
@Builder.Default
|
@Builder.Default
|
||||||
private Set<ImageCapability> modes = Set.of(ImageCapability.TEXT_TO_IMAGE);
|
private Set<ImageCapability> modes = Set.of(ImageCapability.TEXT_TO_IMAGE);
|
||||||
|
|
||||||
/** 支持的图片尺寸,如 ["1024x1024", "1024x1792"] */
|
/** Union of pixel sizes accepted by any model under this provider. */
|
||||||
@Builder.Default
|
@Builder.Default
|
||||||
private List<String> supportedSizes = List.of("1024x1024");
|
private List<String> supportedSizes = List.of("1024x1024");
|
||||||
|
|
||||||
/** 支持的画面比例 */
|
/** Union of aspect ratio presets accepted by any model under this provider. */
|
||||||
@Builder.Default
|
@Builder.Default
|
||||||
private List<String> aspectRatios = List.of("1:1", "16:9", "9:16");
|
private List<String> aspectRatios = List.of("1:1", "16:9", "9:16");
|
||||||
|
|
||||||
/** 最大生成数量 */
|
/** Largest {@code n} (image count) any model under this provider accepts. */
|
||||||
@Builder.Default
|
@Builder.Default
|
||||||
private int maxCount = 1;
|
private int maxCount = 1;
|
||||||
|
|
||||||
/** 默认模型 */
|
/** Default model id. */
|
||||||
private String defaultModel;
|
private String defaultModel;
|
||||||
|
|
||||||
/** 可用模型列表 */
|
/** All callable model ids. */
|
||||||
@Builder.Default
|
@Builder.Default
|
||||||
private List<String> models = List.of();
|
private List<String> models = List.of();
|
||||||
|
|
||||||
|
/** Per-mode generate capabilities. Optional — falls back to flat fields when absent. */
|
||||||
|
private Generate generate;
|
||||||
|
|
||||||
|
/** Per-mode edit capabilities. {@code null} or {@code enabled=false} means edits unsupported. */
|
||||||
|
private Edit edit;
|
||||||
|
|
||||||
|
/** Geometry surface (sizes / aspect ratios). Optional. */
|
||||||
|
private Geometry geometry;
|
||||||
|
|
||||||
|
/** Output knobs (formats, qualities, backgrounds). Optional. */
|
||||||
|
private Output output;
|
||||||
|
|
||||||
|
@Data
|
||||||
|
@Builder
|
||||||
|
public static class Generate {
|
||||||
|
@Builder.Default
|
||||||
|
private int maxCount = 1;
|
||||||
|
@Builder.Default
|
||||||
|
private boolean supportsSize = true;
|
||||||
|
@Builder.Default
|
||||||
|
private boolean supportsAspectRatio = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Data
|
||||||
|
@Builder
|
||||||
|
public static class Edit {
|
||||||
|
@Builder.Default
|
||||||
|
private boolean enabled = false;
|
||||||
|
@Builder.Default
|
||||||
|
private int maxCount = 1;
|
||||||
|
@Builder.Default
|
||||||
|
private int maxInputImages = 1;
|
||||||
|
@Builder.Default
|
||||||
|
private boolean supportsSize = true;
|
||||||
|
@Builder.Default
|
||||||
|
private boolean supportsAspectRatio = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Data
|
||||||
|
@Builder
|
||||||
|
public static class Geometry {
|
||||||
|
@Builder.Default
|
||||||
|
private List<String> sizes = List.of();
|
||||||
|
@Builder.Default
|
||||||
|
private List<String> aspectRatios = List.of();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Data
|
||||||
|
@Builder
|
||||||
|
public static class Output {
|
||||||
|
@Builder.Default
|
||||||
|
private List<String> formats = List.of();
|
||||||
|
@Builder.Default
|
||||||
|
private List<String> qualities = List.of();
|
||||||
|
@Builder.Default
|
||||||
|
private List<String> backgrounds = List.of();
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Match the requested size against supported sizes by area only.
|
* Match the requested size against supported sizes by area only.
|
||||||
* Orientation-blind — prefer {@link #normalizeSize(String, String)} when an
|
* Orientation-blind — prefer {@link #normalizeSize(String, String)} when an
|
||||||
|
|||||||
@ -0,0 +1,21 @@
|
|||||||
|
package vip.mate.tool.image;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* In-memory image reference used for image-edit / image-to-image generation requests.
|
||||||
|
* <p>
|
||||||
|
* The loader normalizes any of the agent-facing input forms (local paths, http(s)
|
||||||
|
* URLs, {@code data:} URLs, conversation message refs) into this single shape so
|
||||||
|
* providers receive bytes, mime type, and file name regardless of origin.
|
||||||
|
*
|
||||||
|
* @param data raw image bytes
|
||||||
|
* @param mimeType e.g. {@code image/png}
|
||||||
|
* @param fileName logical name (best-effort, may be synthesized)
|
||||||
|
* @param origin trace string identifying where the bytes came from
|
||||||
|
* ({@code path:/x.png}, {@code url:https://...}, {@code data-url},
|
||||||
|
* {@code msg:<msgId>:<idx>}). Used for logging / audit, not
|
||||||
|
* forwarded to providers.
|
||||||
|
*
|
||||||
|
* @author MateClaw Team
|
||||||
|
*/
|
||||||
|
public record ImageReference(byte[] data, String mimeType, String fileName, String origin) {
|
||||||
|
}
|
||||||
@ -0,0 +1,297 @@
|
|||||||
|
package vip.mate.tool.image;
|
||||||
|
|
||||||
|
import cn.hutool.http.HttpUtil;
|
||||||
|
import lombok.RequiredArgsConstructor;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.springframework.stereotype.Component;
|
||||||
|
import vip.mate.workspace.conversation.ConversationService;
|
||||||
|
import vip.mate.workspace.conversation.model.MessageContentPart;
|
||||||
|
import vip.mate.workspace.conversation.model.MessageEntity;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.net.URI;
|
||||||
|
import java.net.URLDecoder;
|
||||||
|
import java.nio.charset.StandardCharsets;
|
||||||
|
import java.nio.file.Files;
|
||||||
|
import java.nio.file.Path;
|
||||||
|
import java.nio.file.Paths;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.Base64;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Resolves agent-supplied image reference strings into in-memory
|
||||||
|
* {@link ImageReference} buffers. Five input forms are accepted:
|
||||||
|
*
|
||||||
|
* <ol>
|
||||||
|
* <li>Local filesystem path: {@code /abs/path.png}, {@code ./rel.png},
|
||||||
|
* {@code ~/x.png}, or {@code file://...}.</li>
|
||||||
|
* <li>Data URL: {@code data:image/png;base64,...} (base64 or URL-encoded body).</li>
|
||||||
|
* <li>HTTP(S) URL: downloaded with size + content-type guard.</li>
|
||||||
|
* <li>Conversation message reference: {@code msg:<messageId>:<partIndex>} —
|
||||||
|
* resolves to the local path stored on a {@link MessageContentPart} of
|
||||||
|
* type {@code image} on the named message. This is the channel an agent
|
||||||
|
* uses to forward a user-uploaded image into the image edit tool, so a
|
||||||
|
* non-vision model can still operate on attachments it cannot "see".</li>
|
||||||
|
* <li>Workspace-relative path: passed through as a regular path; the caller
|
||||||
|
* is expected to anchor it to the active workspace before invocation.</li>
|
||||||
|
* </ol>
|
||||||
|
*
|
||||||
|
* @author MateClaw Team
|
||||||
|
*/
|
||||||
|
@Slf4j
|
||||||
|
@Component
|
||||||
|
@RequiredArgsConstructor
|
||||||
|
public class ImageReferenceLoader {
|
||||||
|
|
||||||
|
private static final long MAX_REFERENCE_BYTES = 20L * 1024 * 1024;
|
||||||
|
private static final int HTTP_TIMEOUT_MS = 30_000;
|
||||||
|
|
||||||
|
private final ConversationService conversationService;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Resolve a list of input strings; null / blank entries are skipped.
|
||||||
|
* The caller is expected to enforce per-provider {@code maxInputImages}
|
||||||
|
* before calling.
|
||||||
|
*/
|
||||||
|
public List<ImageReference> loadAll(List<String> inputs, String conversationId) throws IOException {
|
||||||
|
if (inputs == null || inputs.isEmpty()) {
|
||||||
|
return List.of();
|
||||||
|
}
|
||||||
|
List<ImageReference> out = new ArrayList<>(inputs.size());
|
||||||
|
for (String raw : inputs) {
|
||||||
|
if (raw == null || raw.isBlank()) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
out.add(load(raw.trim(), conversationId));
|
||||||
|
}
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Resolve a single reference string. */
|
||||||
|
public ImageReference load(String input, String conversationId) throws IOException {
|
||||||
|
if (input == null || input.isBlank()) {
|
||||||
|
throw new IOException("image reference is blank");
|
||||||
|
}
|
||||||
|
String trimmed = input.trim();
|
||||||
|
|
||||||
|
if (trimmed.startsWith("data:")) {
|
||||||
|
return loadDataUrl(trimmed);
|
||||||
|
}
|
||||||
|
if (trimmed.startsWith("msg:")) {
|
||||||
|
return loadConversationMessageRef(trimmed, conversationId);
|
||||||
|
}
|
||||||
|
if (trimmed.startsWith("http://") || trimmed.startsWith("https://")) {
|
||||||
|
return loadHttpUrl(trimmed);
|
||||||
|
}
|
||||||
|
return loadFilePath(trimmed);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ==================== form: local path / file:// ====================
|
||||||
|
|
||||||
|
private ImageReference loadFilePath(String input) throws IOException {
|
||||||
|
String pathStr = input.startsWith("file://") ? input.substring("file://".length()) : input;
|
||||||
|
if (pathStr.startsWith("~")) {
|
||||||
|
pathStr = System.getProperty("user.home") + pathStr.substring(1);
|
||||||
|
}
|
||||||
|
Path p = Paths.get(pathStr);
|
||||||
|
if (!Files.exists(p)) {
|
||||||
|
throw new IOException("Image file not found: " + pathStr);
|
||||||
|
}
|
||||||
|
if (Files.size(p) > MAX_REFERENCE_BYTES) {
|
||||||
|
throw new IOException("Image exceeds 20MB limit: " + pathStr);
|
||||||
|
}
|
||||||
|
byte[] data = Files.readAllBytes(p);
|
||||||
|
String mime = inferMimeFromName(p.getFileName().toString());
|
||||||
|
return new ImageReference(data, mime, p.getFileName().toString(), "path:" + p);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ==================== form: data: URL ====================
|
||||||
|
|
||||||
|
private ImageReference loadDataUrl(String dataUrl) throws IOException {
|
||||||
|
int comma = dataUrl.indexOf(',');
|
||||||
|
if (comma < 0) {
|
||||||
|
throw new IOException("Malformed data URL: missing comma");
|
||||||
|
}
|
||||||
|
String header = dataUrl.substring("data:".length(), comma);
|
||||||
|
String body = dataUrl.substring(comma + 1);
|
||||||
|
boolean isBase64 = header.toLowerCase().contains(";base64");
|
||||||
|
String mime = isBase64
|
||||||
|
? header.substring(0, header.toLowerCase().indexOf(";base64"))
|
||||||
|
: (header.contains(";") ? header.substring(0, header.indexOf(';')) : header);
|
||||||
|
if (mime == null || mime.isBlank()) {
|
||||||
|
mime = "image/png";
|
||||||
|
}
|
||||||
|
byte[] data;
|
||||||
|
try {
|
||||||
|
data = isBase64
|
||||||
|
? Base64.getDecoder().decode(body)
|
||||||
|
: URLDecoder.decode(body, StandardCharsets.UTF_8).getBytes(StandardCharsets.UTF_8);
|
||||||
|
} catch (IllegalArgumentException e) {
|
||||||
|
throw new IOException("Invalid base64 in data URL: " + e.getMessage(), e);
|
||||||
|
}
|
||||||
|
if (data.length > MAX_REFERENCE_BYTES) {
|
||||||
|
throw new IOException("Image exceeds 20MB limit (data URL)");
|
||||||
|
}
|
||||||
|
return new ImageReference(data, mime, "inline." + extensionFor(mime), "data-url");
|
||||||
|
}
|
||||||
|
|
||||||
|
// ==================== form: http(s) URL ====================
|
||||||
|
|
||||||
|
private ImageReference loadHttpUrl(String url) throws IOException {
|
||||||
|
URI uri = URI.create(url);
|
||||||
|
String host = uri.getHost();
|
||||||
|
if (host == null) {
|
||||||
|
throw new IOException("URL has no host: " + url);
|
||||||
|
}
|
||||||
|
// Conservative SSRF guard: reject obvious internal targets. Refine later
|
||||||
|
// if the project gains a dedicated SsrFPolicy module.
|
||||||
|
String lowered = host.toLowerCase();
|
||||||
|
if (lowered.equals("localhost")
|
||||||
|
|| lowered.equals("127.0.0.1")
|
||||||
|
|| lowered.startsWith("10.")
|
||||||
|
|| lowered.startsWith("192.168.")
|
||||||
|
|| lowered.startsWith("169.254.")
|
||||||
|
|| lowered.startsWith("172.")) {
|
||||||
|
throw new IOException("Refusing to download image from internal host: " + host);
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
byte[] data = HttpUtil.createGet(url).timeout(HTTP_TIMEOUT_MS).execute().bodyBytes();
|
||||||
|
if (data == null || data.length == 0) {
|
||||||
|
throw new IOException("Empty response downloading image from " + url);
|
||||||
|
}
|
||||||
|
if (data.length > MAX_REFERENCE_BYTES) {
|
||||||
|
throw new IOException("Image exceeds 20MB limit: " + url);
|
||||||
|
}
|
||||||
|
String fileName = guessFileNameFromUrl(url);
|
||||||
|
String mime = inferMimeFromName(fileName);
|
||||||
|
return new ImageReference(data, mime, fileName, "url:" + url);
|
||||||
|
} catch (Exception e) {
|
||||||
|
throw new IOException("Failed to download image " + url + ": " + e.getMessage(), e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ==================== form: msg:<messageId>:<partIndex> ====================
|
||||||
|
|
||||||
|
private ImageReference loadConversationMessageRef(String ref, String conversationId) throws IOException {
|
||||||
|
// ref shape: "msg:<messageId>" (first image part) or "msg:<messageId>:<partIndex>"
|
||||||
|
String body = ref.substring("msg:".length());
|
||||||
|
String[] parts = body.split(":", 2);
|
||||||
|
long messageId;
|
||||||
|
try {
|
||||||
|
messageId = Long.parseLong(parts[0]);
|
||||||
|
} catch (NumberFormatException e) {
|
||||||
|
throw new IOException("Invalid msg: ref, expected msg:<messageId>[:<idx>]: " + ref);
|
||||||
|
}
|
||||||
|
Integer wantedIdx = null;
|
||||||
|
if (parts.length == 2 && !parts[1].isBlank()) {
|
||||||
|
try {
|
||||||
|
wantedIdx = Integer.parseInt(parts[1]);
|
||||||
|
} catch (NumberFormatException e) {
|
||||||
|
throw new IOException("Invalid part index in: " + ref);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (conversationId == null || conversationId.isBlank()) {
|
||||||
|
throw new IOException("Cannot resolve msg: reference without an active conversation");
|
||||||
|
}
|
||||||
|
MessageEntity message = findMessageInConversation(conversationId, messageId);
|
||||||
|
if (message == null) {
|
||||||
|
throw new IOException("Message " + messageId + " not found in conversation " + conversationId);
|
||||||
|
}
|
||||||
|
List<MessageContentPart> contentParts = conversationService.parseMessageParts(message);
|
||||||
|
MessageContentPart picked = pickImagePart(contentParts, wantedIdx);
|
||||||
|
if (picked == null) {
|
||||||
|
throw new IOException("No image part on message " + messageId
|
||||||
|
+ (wantedIdx != null ? " at index " + wantedIdx : ""));
|
||||||
|
}
|
||||||
|
Path filePath = resolveLocalPath(picked);
|
||||||
|
if (filePath == null) {
|
||||||
|
throw new IOException("Message " + messageId + " image part has no local path: "
|
||||||
|
+ picked.getFileName());
|
||||||
|
}
|
||||||
|
if (Files.size(filePath) > MAX_REFERENCE_BYTES) {
|
||||||
|
throw new IOException("Image exceeds 20MB limit: " + filePath);
|
||||||
|
}
|
||||||
|
byte[] data = Files.readAllBytes(filePath);
|
||||||
|
String mime = picked.getContentType();
|
||||||
|
if (mime == null || mime.isBlank() || "image/*".equals(mime)) {
|
||||||
|
mime = inferMimeFromName(picked.getFileName());
|
||||||
|
}
|
||||||
|
String fileName = picked.getFileName() != null ? picked.getFileName() : filePath.getFileName().toString();
|
||||||
|
return new ImageReference(data, mime, fileName, ref);
|
||||||
|
}
|
||||||
|
|
||||||
|
private MessageEntity findMessageInConversation(String conversationId, long messageId) {
|
||||||
|
List<MessageEntity> all = conversationService.listMessages(conversationId);
|
||||||
|
for (MessageEntity m : all) {
|
||||||
|
if (m.getId() != null && m.getId() == messageId) {
|
||||||
|
return m;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
private MessageContentPart pickImagePart(List<MessageContentPart> parts, Integer wantedIdx) {
|
||||||
|
if (parts == null || parts.isEmpty()) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
if (wantedIdx != null) {
|
||||||
|
int seen = 0;
|
||||||
|
for (MessageContentPart p : parts) {
|
||||||
|
if (p == null || !"image".equals(p.getType())) continue;
|
||||||
|
if (seen == wantedIdx) {
|
||||||
|
return p;
|
||||||
|
}
|
||||||
|
seen++;
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
for (MessageContentPart p : parts) {
|
||||||
|
if (p != null && "image".equals(p.getType())) {
|
||||||
|
return p;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
private Path resolveLocalPath(MessageContentPart part) {
|
||||||
|
if (part.getPath() != null && !part.getPath().isBlank()) {
|
||||||
|
Path p = Paths.get(part.getPath());
|
||||||
|
if (Files.exists(p)) return p;
|
||||||
|
}
|
||||||
|
if (part.getStoredName() != null && !part.getStoredName().isBlank()) {
|
||||||
|
Path p = Paths.get(part.getStoredName());
|
||||||
|
if (Files.exists(p)) return p;
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ==================== shared helpers ====================
|
||||||
|
|
||||||
|
private static String inferMimeFromName(String name) {
|
||||||
|
if (name == null) return "image/png";
|
||||||
|
String lower = name.toLowerCase();
|
||||||
|
if (lower.endsWith(".jpg") || lower.endsWith(".jpeg")) return "image/jpeg";
|
||||||
|
if (lower.endsWith(".webp")) return "image/webp";
|
||||||
|
if (lower.endsWith(".gif")) return "image/gif";
|
||||||
|
if (lower.endsWith(".bmp")) return "image/bmp";
|
||||||
|
return "image/png";
|
||||||
|
}
|
||||||
|
|
||||||
|
private static String extensionFor(String mime) {
|
||||||
|
return switch (mime.toLowerCase().trim()) {
|
||||||
|
case "image/jpeg", "image/jpg" -> "jpg";
|
||||||
|
case "image/webp" -> "webp";
|
||||||
|
case "image/gif" -> "gif";
|
||||||
|
case "image/bmp" -> "bmp";
|
||||||
|
default -> "png";
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private static String guessFileNameFromUrl(String url) {
|
||||||
|
String stripped = url.split("\\?", 2)[0];
|
||||||
|
int slash = stripped.lastIndexOf('/');
|
||||||
|
String tail = slash >= 0 ? stripped.substring(slash + 1) : stripped;
|
||||||
|
return tail.isBlank() ? "remote.png" : tail;
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -0,0 +1,172 @@
|
|||||||
|
package vip.mate.tool.image;
|
||||||
|
|
||||||
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
|
import com.fasterxml.jackson.databind.node.ObjectNode;
|
||||||
|
|
||||||
|
import java.util.LinkedHashMap;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.Set;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Configuration-driven payload builder for image-generation providers.
|
||||||
|
*
|
||||||
|
* <p>Without this, every provider class collects an {@code if/else} chain
|
||||||
|
* mapping (model id) → (request shape, sizing dialect, available knobs). Each
|
||||||
|
* new model in a family forces another branch. With it, each provider holds a
|
||||||
|
* static {@code Map<modelId, ImageModelSpec>}; the builder consults that spec
|
||||||
|
* for sizing dialect, default parameters, and a {@code supports} whitelist of
|
||||||
|
* payload keys. Values not in the whitelist are dropped at the end so the API
|
||||||
|
* never sees keys it would reject.
|
||||||
|
*
|
||||||
|
* <p>Sizing dialect handling:
|
||||||
|
* <ul>
|
||||||
|
* <li>{@link SizeStyle#LITERAL_DIMENSION} — output {@code "1024x1024"} or
|
||||||
|
* a separator-replaced form (e.g. DashScope wants {@code "1024*1024"} —
|
||||||
|
* the spec's sizeMap can carry the alternative).</li>
|
||||||
|
* <li>{@link SizeStyle#ASPECT_RATIO} — output {@code "1:1"} / {@code "16:9"}.</li>
|
||||||
|
* <li>{@link SizeStyle#PRESET_NAME} — output the model-native preset
|
||||||
|
* (e.g. {@code square_hd}). The spec's sizeMap drives the lookup keyed
|
||||||
|
* by orientation token (landscape / square / portrait).</li>
|
||||||
|
* </ul>
|
||||||
|
*
|
||||||
|
* @author MateClaw Team
|
||||||
|
*/
|
||||||
|
public final class PayloadBuilder {
|
||||||
|
|
||||||
|
private final ImageModelSpec spec;
|
||||||
|
private final Map<String, Object> entries = new LinkedHashMap<>();
|
||||||
|
|
||||||
|
private PayloadBuilder(ImageModelSpec spec) {
|
||||||
|
this.spec = spec;
|
||||||
|
if (spec.defaults() != null) {
|
||||||
|
entries.putAll(spec.defaults());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public static PayloadBuilder from(ImageModelSpec spec) {
|
||||||
|
return new PayloadBuilder(spec);
|
||||||
|
}
|
||||||
|
|
||||||
|
public PayloadBuilder withPrompt(String prompt) {
|
||||||
|
if (prompt != null) {
|
||||||
|
entries.put("prompt", prompt);
|
||||||
|
}
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
public PayloadBuilder withCount(Integer count) {
|
||||||
|
if (count != null && count > 0) {
|
||||||
|
entries.put("n", Math.min(count, Math.max(1, spec.maxCount() == 0 ? count : spec.maxCount())));
|
||||||
|
}
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Translate the unified {@code size} / {@code aspectRatio} inputs to whichever
|
||||||
|
* key/value pair this model expects. The spec's {@link SizeStyle} drives
|
||||||
|
* which key is set; the spec's sizeMap (orientation → native value) drives
|
||||||
|
* the value when the caller did not pass an exact match.
|
||||||
|
*/
|
||||||
|
public PayloadBuilder withSize(String requestedSize, String requestedAspectRatio) {
|
||||||
|
SizeStyle style = spec.sizeStyle();
|
||||||
|
if (style == null) {
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
Map<String, String> sizeMap = spec.sizeMap();
|
||||||
|
switch (style) {
|
||||||
|
case LITERAL_DIMENSION -> entries.put("size",
|
||||||
|
resolveLiteralDimension(requestedSize, requestedAspectRatio, sizeMap));
|
||||||
|
case ASPECT_RATIO -> entries.put("aspect_ratio",
|
||||||
|
resolveAspectRatio(requestedAspectRatio, sizeMap));
|
||||||
|
case PRESET_NAME -> entries.put("image_size",
|
||||||
|
resolvePreset(requestedAspectRatio, sizeMap));
|
||||||
|
}
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
public PayloadBuilder withSeed(Integer seed) {
|
||||||
|
if (seed != null) {
|
||||||
|
entries.put("seed", seed);
|
||||||
|
}
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
public PayloadBuilder put(String key, Object value) {
|
||||||
|
if (value != null) {
|
||||||
|
entries.put(key, value);
|
||||||
|
}
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Produce a Jackson {@link ObjectNode} containing only the keys this model's
|
||||||
|
* {@code supports} whitelist allows. Empty whitelist means "passthrough".
|
||||||
|
*/
|
||||||
|
public ObjectNode toJsonNode(ObjectMapper mapper) {
|
||||||
|
ObjectNode out = mapper.createObjectNode();
|
||||||
|
Set<String> supports = spec.supports();
|
||||||
|
boolean filter = supports != null && !supports.isEmpty();
|
||||||
|
for (Map.Entry<String, Object> e : entries.entrySet()) {
|
||||||
|
if (filter && !supports.contains(e.getKey())) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
out.set(e.getKey(), mapper.valueToTree(e.getValue()));
|
||||||
|
}
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Read-only view of accumulated entries (post defaults / pre supports filter). */
|
||||||
|
public Map<String, Object> entries() {
|
||||||
|
return Map.copyOf(entries);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ==================== size resolution ====================
|
||||||
|
|
||||||
|
private String resolveLiteralDimension(String requestedSize, String aspectRatio,
|
||||||
|
Map<String, String> sizeMap) {
|
||||||
|
if (requestedSize != null && !requestedSize.isBlank()) {
|
||||||
|
// Allow the spec's sizeMap to translate (e.g. "1024x1024" -> "1024*1024").
|
||||||
|
String mapped = sizeMap == null ? null : sizeMap.get(requestedSize);
|
||||||
|
return mapped != null ? mapped : requestedSize;
|
||||||
|
}
|
||||||
|
String orientation = orientationOf(aspectRatio);
|
||||||
|
if (sizeMap != null && sizeMap.containsKey(orientation)) {
|
||||||
|
return sizeMap.get(orientation);
|
||||||
|
}
|
||||||
|
return "1024x1024";
|
||||||
|
}
|
||||||
|
|
||||||
|
private String resolveAspectRatio(String requested, Map<String, String> sizeMap) {
|
||||||
|
if (requested != null && !requested.isBlank()) {
|
||||||
|
String mapped = sizeMap == null ? null : sizeMap.get(requested);
|
||||||
|
return mapped != null ? mapped : requested;
|
||||||
|
}
|
||||||
|
return "1:1";
|
||||||
|
}
|
||||||
|
|
||||||
|
private String resolvePreset(String aspectRatio, Map<String, String> sizeMap) {
|
||||||
|
String orientation = orientationOf(aspectRatio);
|
||||||
|
if (sizeMap != null && sizeMap.containsKey(orientation)) {
|
||||||
|
return sizeMap.get(orientation);
|
||||||
|
}
|
||||||
|
return "square_hd";
|
||||||
|
}
|
||||||
|
|
||||||
|
private static String orientationOf(String aspectRatio) {
|
||||||
|
if (aspectRatio == null || aspectRatio.isBlank()) {
|
||||||
|
return "square";
|
||||||
|
}
|
||||||
|
String[] parts = aspectRatio.split(":");
|
||||||
|
if (parts.length != 2) {
|
||||||
|
return "square";
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
double w = Double.parseDouble(parts[0].trim());
|
||||||
|
double h = Double.parseDouble(parts[1].trim());
|
||||||
|
if (w == h) return "square";
|
||||||
|
return w > h ? "landscape" : "portrait";
|
||||||
|
} catch (NumberFormatException e) {
|
||||||
|
return "square";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -0,0 +1,27 @@
|
|||||||
|
package vip.mate.tool.image;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Describes how a particular image-generation model expects its size to be
|
||||||
|
* expressed. Three families cover all current providers:
|
||||||
|
*
|
||||||
|
* <ul>
|
||||||
|
* <li>{@link #LITERAL_DIMENSION} — explicit width/height string ({@code 1024x1024},
|
||||||
|
* {@code 1536*1024}). Used by DashScope, OpenAI DALL-E, MiniMax.</li>
|
||||||
|
* <li>{@link #ASPECT_RATIO} — preset enum like {@code 16:9} or {@code 1:1}.
|
||||||
|
* Used by Gemini / nano-banana style APIs.</li>
|
||||||
|
* <li>{@link #PRESET_NAME} — provider-specific preset label
|
||||||
|
* ({@code square_hd}, {@code landscape_16_9}). Used by fal.ai's flux,
|
||||||
|
* z-image, qwen-image families.</li>
|
||||||
|
* </ul>
|
||||||
|
*
|
||||||
|
* Each {@link ImageModelSpec} declares one style and provides the sizeMap that
|
||||||
|
* translates the unified {@code aspectRatio} input ({@code landscape} /
|
||||||
|
* {@code square} / {@code portrait} or a literal ratio) to the model-native form.
|
||||||
|
*
|
||||||
|
* @author MateClaw Team
|
||||||
|
*/
|
||||||
|
public enum SizeStyle {
|
||||||
|
LITERAL_DIMENSION,
|
||||||
|
ASPECT_RATIO,
|
||||||
|
PRESET_NAME
|
||||||
|
}
|
||||||
@ -0,0 +1,220 @@
|
|||||||
|
package vip.mate.tool.image.provider;
|
||||||
|
|
||||||
|
import vip.mate.tool.image.ImageCapability;
|
||||||
|
import vip.mate.tool.image.ImageModelSpec;
|
||||||
|
import vip.mate.tool.image.SizeStyle;
|
||||||
|
|
||||||
|
import java.util.LinkedHashMap;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.Set;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Catalog of DashScope-served image generation / editing models, organised so
|
||||||
|
* that adding a new model is a one-line spec entry.
|
||||||
|
*
|
||||||
|
* <p>Two transport families are present:
|
||||||
|
* <ul>
|
||||||
|
* <li><b>Async legacy</b> ({@link #LEGACY_ASYNC_ENDPOINT}) — wanx 2.0/2.1 and
|
||||||
|
* wan 2.2/2.5 turbo/plus models that exclusively do text-to-image. The
|
||||||
|
* caller submits and polls {@code /api/v1/tasks/{id}}.</li>
|
||||||
|
* <li><b>Sync multimodal</b> ({@link #MULTIMODAL_ENDPOINT}) — wan 2.6/2.7,
|
||||||
|
* qwen-image, qwen-image-edit, z-image. Uses the OpenAI-style
|
||||||
|
* {@code messages.content[]} array and returns the generated image URL
|
||||||
|
* in the same response.</li>
|
||||||
|
* </ul>
|
||||||
|
*
|
||||||
|
* @author MateClaw Team
|
||||||
|
*/
|
||||||
|
final class DashScopeImageModels {
|
||||||
|
|
||||||
|
static final String LEGACY_ASYNC_ENDPOINT =
|
||||||
|
"https://dashscope.aliyuncs.com/api/v1/services/aigc/image-generation/generation";
|
||||||
|
static final String MULTIMODAL_ENDPOINT =
|
||||||
|
"https://dashscope.aliyuncs.com/api/v1/services/aigc/multimodal-generation/generation";
|
||||||
|
static final String TASKS_ENDPOINT_PREFIX =
|
||||||
|
"https://dashscope.aliyuncs.com/api/v1/tasks/";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Default model when the request does not name one.
|
||||||
|
*
|
||||||
|
* <p>Kept on the legacy turbo so existing accounts that have not enrolled in
|
||||||
|
* the newer wan/qwen-image families do not see breakage. Callers that want
|
||||||
|
* edit support must name a model explicitly (e.g. {@code wan2.7-image} or
|
||||||
|
* {@code qwen-image-edit}) — the registry's edit-capability resolution then
|
||||||
|
* routes correctly.
|
||||||
|
*/
|
||||||
|
static final String DEFAULT_MODEL = "wanx2.1-t2i-turbo";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Default model when an edit-capable spec is required but the request did
|
||||||
|
* not name one. Used by the provider when the request carries
|
||||||
|
* {@code inputImages} but the named model lacks {@link ImageCapability#IMAGE_EDIT}.
|
||||||
|
*/
|
||||||
|
static final String DEFAULT_EDIT_MODEL = "wan2.7-image";
|
||||||
|
|
||||||
|
private static final Map<String, String> ASPECT_LITERAL_SIZES = Map.of(
|
||||||
|
"1:1", "1024x1024",
|
||||||
|
"16:9", "1280x720",
|
||||||
|
"9:16", "720x1280",
|
||||||
|
"landscape", "1280x720",
|
||||||
|
"square", "1024x1024",
|
||||||
|
"portrait", "720x1280"
|
||||||
|
);
|
||||||
|
|
||||||
|
private static final Map<String, String> ASPECT_LITERAL_SIZES_2K = Map.of(
|
||||||
|
"1:1", "2048x2048",
|
||||||
|
"16:9", "2560x1440",
|
||||||
|
"9:16", "1440x2560",
|
||||||
|
"landscape", "2560x1440",
|
||||||
|
"square", "2048x2048",
|
||||||
|
"portrait", "1440x2560"
|
||||||
|
);
|
||||||
|
|
||||||
|
private DashScopeImageModels() {}
|
||||||
|
|
||||||
|
private static final Map<String, ImageModelSpec> CATALOG = buildCatalog();
|
||||||
|
|
||||||
|
static Map<String, ImageModelSpec> all() {
|
||||||
|
return CATALOG;
|
||||||
|
}
|
||||||
|
|
||||||
|
static ImageModelSpec get(String id) {
|
||||||
|
if (id == null || id.isBlank()) {
|
||||||
|
return CATALOG.get(DEFAULT_MODEL);
|
||||||
|
}
|
||||||
|
return CATALOG.getOrDefault(id, CATALOG.get(DEFAULT_MODEL));
|
||||||
|
}
|
||||||
|
|
||||||
|
private static Map<String, ImageModelSpec> buildCatalog() {
|
||||||
|
Map<String, ImageModelSpec> m = new LinkedHashMap<>();
|
||||||
|
|
||||||
|
// ========== Legacy async text-to-image (image-generation/generation) ==========
|
||||||
|
// No edit support; keeps backward compatibility for users on existing model ids.
|
||||||
|
addAsyncT2I(m, "wanx2.1-t2i-turbo");
|
||||||
|
addAsyncT2I(m, "wanx2.1-t2i-plus");
|
||||||
|
addAsyncT2I(m, "wanx2.0-t2i-turbo");
|
||||||
|
addAsyncT2I(m, "wan2.2-t2i-flash");
|
||||||
|
addAsyncT2I(m, "wan2.2-t2i-plus");
|
||||||
|
addAsyncT2I(m, "wan2.5-t2i-preview");
|
||||||
|
|
||||||
|
// ========== Sync multimodal text-to-image only (multimodal-generation) ==========
|
||||||
|
m.put("z-image-turbo", ImageModelSpec.builder()
|
||||||
|
.id("z-image-turbo")
|
||||||
|
.displayName("Z-Image Turbo (fastest)")
|
||||||
|
.endpoint(MULTIMODAL_ENDPOINT)
|
||||||
|
.transport(ImageModelSpec.Transport.SYNC)
|
||||||
|
.sizeStyle(SizeStyle.LITERAL_DIMENSION)
|
||||||
|
.sizeMap(ASPECT_LITERAL_SIZES)
|
||||||
|
.modes(Set.of(ImageCapability.TEXT_TO_IMAGE))
|
||||||
|
.supports(Set.of("size", "seed", "prompt_extend"))
|
||||||
|
.maxCount(1)
|
||||||
|
.maxInputImages(0)
|
||||||
|
.build());
|
||||||
|
|
||||||
|
// ========== Sync multimodal text-to-image + edit (qwen-image series) ==========
|
||||||
|
addQwenImage(m, "qwen-image-2.0");
|
||||||
|
addQwenImage(m, "qwen-image-2.0-pro");
|
||||||
|
addQwenImageEdit(m, "qwen-image-edit");
|
||||||
|
addQwenImageEdit(m, "qwen-image-edit-plus");
|
||||||
|
addQwenImageEdit(m, "qwen-image-edit-max");
|
||||||
|
|
||||||
|
// ========== Sync multimodal text-to-image + edit (wan2.6 / 2.7 image) ==========
|
||||||
|
m.put("wan2.6-t2i", ImageModelSpec.builder()
|
||||||
|
.id("wan2.6-t2i")
|
||||||
|
.displayName("Wan 2.6 (sync T2I)")
|
||||||
|
.endpoint(MULTIMODAL_ENDPOINT)
|
||||||
|
.transport(ImageModelSpec.Transport.SYNC)
|
||||||
|
.sizeStyle(SizeStyle.LITERAL_DIMENSION)
|
||||||
|
.sizeMap(ASPECT_LITERAL_SIZES)
|
||||||
|
.modes(Set.of(ImageCapability.TEXT_TO_IMAGE))
|
||||||
|
.supports(Set.of("size", "n", "seed", "negative_prompt", "prompt_extend", "watermark"))
|
||||||
|
.maxCount(4)
|
||||||
|
.maxInputImages(0)
|
||||||
|
.build());
|
||||||
|
|
||||||
|
m.put("wan2.7-image", ImageModelSpec.builder()
|
||||||
|
.id("wan2.7-image")
|
||||||
|
.displayName("Wan 2.7 Image (T2I + edit, up to 2K)")
|
||||||
|
.endpoint(MULTIMODAL_ENDPOINT)
|
||||||
|
.transport(ImageModelSpec.Transport.SYNC)
|
||||||
|
.sizeStyle(SizeStyle.LITERAL_DIMENSION)
|
||||||
|
.sizeMap(ASPECT_LITERAL_SIZES)
|
||||||
|
.modes(Set.of(ImageCapability.TEXT_TO_IMAGE, ImageCapability.IMAGE_EDIT))
|
||||||
|
.supports(Set.of("size", "n", "seed", "negative_prompt", "prompt_extend", "watermark"))
|
||||||
|
.maxCount(4)
|
||||||
|
.maxInputImages(3)
|
||||||
|
.build());
|
||||||
|
|
||||||
|
m.put("wan2.7-image-pro", ImageModelSpec.builder()
|
||||||
|
.id("wan2.7-image-pro")
|
||||||
|
.displayName("Wan 2.7 Image Pro (T2I + edit, up to 4K)")
|
||||||
|
.endpoint(MULTIMODAL_ENDPOINT)
|
||||||
|
.transport(ImageModelSpec.Transport.SYNC)
|
||||||
|
.sizeStyle(SizeStyle.LITERAL_DIMENSION)
|
||||||
|
.sizeMap(ASPECT_LITERAL_SIZES_2K)
|
||||||
|
.modes(Set.of(ImageCapability.TEXT_TO_IMAGE, ImageCapability.IMAGE_EDIT))
|
||||||
|
.supports(Set.of("size", "n", "seed", "negative_prompt", "prompt_extend", "watermark"))
|
||||||
|
.maxCount(4)
|
||||||
|
.maxInputImages(3)
|
||||||
|
.build());
|
||||||
|
|
||||||
|
return Map.copyOf(m);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ------------------- helper builders -------------------
|
||||||
|
|
||||||
|
private static void addAsyncT2I(Map<String, ImageModelSpec> m, String id) {
|
||||||
|
m.put(id, ImageModelSpec.builder()
|
||||||
|
.id(id)
|
||||||
|
.displayName(id + " (async legacy T2I)")
|
||||||
|
.endpoint(LEGACY_ASYNC_ENDPOINT)
|
||||||
|
.transport(ImageModelSpec.Transport.ASYNC)
|
||||||
|
.sizeStyle(SizeStyle.LITERAL_DIMENSION)
|
||||||
|
// Legacy endpoint uses '*' as the size separator; sizeMap stores
|
||||||
|
// the native form so PayloadBuilder can pass it through.
|
||||||
|
.sizeMapping("1:1", "1024*1024")
|
||||||
|
.sizeMapping("16:9", "1280*720")
|
||||||
|
.sizeMapping("9:16", "720*1280")
|
||||||
|
.sizeMapping("landscape", "1280*720")
|
||||||
|
.sizeMapping("square", "1024*1024")
|
||||||
|
.sizeMapping("portrait", "720*1280")
|
||||||
|
.sizeMapping("1024x1024", "1024*1024")
|
||||||
|
.sizeMapping("1280x720", "1280*720")
|
||||||
|
.sizeMapping("720x1280", "720*1280")
|
||||||
|
.modes(Set.of(ImageCapability.TEXT_TO_IMAGE))
|
||||||
|
.supports(Set.of("size", "n"))
|
||||||
|
.maxCount(4)
|
||||||
|
.maxInputImages(0)
|
||||||
|
.build());
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void addQwenImage(Map<String, ImageModelSpec> m, String id) {
|
||||||
|
m.put(id, ImageModelSpec.builder()
|
||||||
|
.id(id)
|
||||||
|
.displayName(id + " (T2I)")
|
||||||
|
.endpoint(MULTIMODAL_ENDPOINT)
|
||||||
|
.transport(ImageModelSpec.Transport.SYNC)
|
||||||
|
.sizeStyle(SizeStyle.LITERAL_DIMENSION)
|
||||||
|
.sizeMap(ASPECT_LITERAL_SIZES_2K)
|
||||||
|
.modes(Set.of(ImageCapability.TEXT_TO_IMAGE))
|
||||||
|
.supports(Set.of("size", "n", "seed", "negative_prompt", "prompt_extend", "watermark"))
|
||||||
|
.maxCount(4)
|
||||||
|
.maxInputImages(0)
|
||||||
|
.build());
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void addQwenImageEdit(Map<String, ImageModelSpec> m, String id) {
|
||||||
|
m.put(id, ImageModelSpec.builder()
|
||||||
|
.id(id)
|
||||||
|
.displayName(id + " (image edit)")
|
||||||
|
.endpoint(MULTIMODAL_ENDPOINT)
|
||||||
|
.transport(ImageModelSpec.Transport.SYNC)
|
||||||
|
.sizeStyle(SizeStyle.LITERAL_DIMENSION)
|
||||||
|
.sizeMap(ASPECT_LITERAL_SIZES_2K)
|
||||||
|
.modes(Set.of(ImageCapability.IMAGE_EDIT))
|
||||||
|
.supports(Set.of("size", "n", "seed", "negative_prompt", "prompt_extend", "watermark"))
|
||||||
|
.maxCount(4)
|
||||||
|
.maxInputImages(3)
|
||||||
|
.build());
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -4,6 +4,7 @@ import cn.hutool.http.HttpRequest;
|
|||||||
import cn.hutool.http.HttpResponse;
|
import cn.hutool.http.HttpResponse;
|
||||||
import com.fasterxml.jackson.databind.JsonNode;
|
import com.fasterxml.jackson.databind.JsonNode;
|
||||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
|
import com.fasterxml.jackson.databind.node.ArrayNode;
|
||||||
import com.fasterxml.jackson.databind.node.ObjectNode;
|
import com.fasterxml.jackson.databind.node.ObjectNode;
|
||||||
import lombok.RequiredArgsConstructor;
|
import lombok.RequiredArgsConstructor;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
@ -11,17 +12,38 @@ import org.springframework.stereotype.Component;
|
|||||||
import vip.mate.llm.service.ModelProviderService;
|
import vip.mate.llm.service.ModelProviderService;
|
||||||
import vip.mate.system.model.SystemSettingsDTO;
|
import vip.mate.system.model.SystemSettingsDTO;
|
||||||
import vip.mate.task.AsyncTaskService.TaskPollResult;
|
import vip.mate.task.AsyncTaskService.TaskPollResult;
|
||||||
import vip.mate.tool.image.*;
|
import vip.mate.tool.image.ImageCapability;
|
||||||
|
import vip.mate.tool.image.ImageGenerationProvider;
|
||||||
|
import vip.mate.tool.image.ImageGenerationRequest;
|
||||||
|
import vip.mate.tool.image.ImageModelSpec;
|
||||||
|
import vip.mate.tool.image.ImageProviderCapabilities;
|
||||||
|
import vip.mate.tool.image.ImageReference;
|
||||||
|
import vip.mate.tool.image.ImageSubmitResult;
|
||||||
|
import vip.mate.tool.image.PayloadBuilder;
|
||||||
|
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.Base64;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* DashScope 图片生成 Provider — 支持通义万相 Wanx 系列
|
* DashScope image provider — routes per-model between two transports:
|
||||||
* <p>
|
*
|
||||||
* 异步模式:提交后返回 taskId,需轮询获取结果。
|
* <ul>
|
||||||
* 复用已有的 DashScope LLM provider 的 API Key。
|
* <li><b>Async legacy</b> ({@code services/aigc/image-generation/generation})
|
||||||
* API 文档: https://help.aliyun.com/zh/model-studio/developer-reference/text-to-image
|
* for the wanx 2.0/2.1, wan 2.2/2.5 turbo/plus families. Submit returns a
|
||||||
|
* task id; the caller polls {@code /api/v1/tasks/{id}} until
|
||||||
|
* SUCCEEDED.</li>
|
||||||
|
* <li><b>Sync multimodal</b> ({@code services/aigc/multimodal-generation/generation})
|
||||||
|
* for wan 2.6/2.7 image, qwen-image, qwen-image-edit, z-image. The
|
||||||
|
* generated image URL is returned in the same response. This endpoint
|
||||||
|
* also accepts inline reference images, enabling the image edit /
|
||||||
|
* image-to-image flow.</li>
|
||||||
|
* </ul>
|
||||||
|
*
|
||||||
|
* The model catalog ({@link DashScopeImageModels}) drives endpoint selection,
|
||||||
|
* payload shape, and the {@code supports} whitelist — adding a new model is a
|
||||||
|
* one-line spec entry.
|
||||||
*
|
*
|
||||||
* @author MateClaw Team
|
* @author MateClaw Team
|
||||||
*/
|
*/
|
||||||
@ -33,9 +55,6 @@ public class DashScopeImageProvider implements ImageGenerationProvider {
|
|||||||
private final ModelProviderService modelProviderService;
|
private final ModelProviderService modelProviderService;
|
||||||
private final ObjectMapper objectMapper;
|
private final ObjectMapper objectMapper;
|
||||||
|
|
||||||
private static final String BASE_URL = "https://dashscope.aliyuncs.com/api/v1";
|
|
||||||
private static final String DEFAULT_MODEL = "wanx2.1-t2i-turbo";
|
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public String id() {
|
public String id() {
|
||||||
return "dashscope";
|
return "dashscope";
|
||||||
@ -43,7 +62,7 @@ public class DashScopeImageProvider implements ImageGenerationProvider {
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
public String label() {
|
public String label() {
|
||||||
return "DashScope (通义万相)";
|
return "DashScope (Tongyi Wanxiang / Qwen-Image)";
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@ -58,18 +77,32 @@ public class DashScopeImageProvider implements ImageGenerationProvider {
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
public Set<ImageCapability> capabilities() {
|
public Set<ImageCapability> capabilities() {
|
||||||
return Set.of(ImageCapability.TEXT_TO_IMAGE);
|
return Set.of(ImageCapability.TEXT_TO_IMAGE, ImageCapability.IMAGE_EDIT);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public ImageProviderCapabilities detailedCapabilities() {
|
public ImageProviderCapabilities detailedCapabilities() {
|
||||||
|
List<String> modelIds = new ArrayList<>(DashScopeImageModels.all().keySet());
|
||||||
return ImageProviderCapabilities.builder()
|
return ImageProviderCapabilities.builder()
|
||||||
.modes(capabilities())
|
.modes(capabilities())
|
||||||
.supportedSizes(List.of("1024x1024", "720x1280", "1280x720"))
|
.supportedSizes(List.of(
|
||||||
|
"1024x1024", "1280x720", "720x1280",
|
||||||
|
"2048x2048", "2560x1440", "1440x2560"))
|
||||||
.aspectRatios(List.of("1:1", "16:9", "9:16"))
|
.aspectRatios(List.of("1:1", "16:9", "9:16"))
|
||||||
.maxCount(4)
|
.maxCount(4)
|
||||||
.defaultModel(DEFAULT_MODEL)
|
.defaultModel(DashScopeImageModels.DEFAULT_MODEL)
|
||||||
.models(List.of("wanx2.1-t2i-turbo", "wanx-v1"))
|
.models(modelIds)
|
||||||
|
.generate(ImageProviderCapabilities.Generate.builder()
|
||||||
|
.maxCount(4).supportsSize(true).supportsAspectRatio(true).build())
|
||||||
|
.edit(ImageProviderCapabilities.Edit.builder()
|
||||||
|
.enabled(true).maxCount(4).maxInputImages(3)
|
||||||
|
.supportsSize(true).supportsAspectRatio(true).build())
|
||||||
|
.geometry(ImageProviderCapabilities.Geometry.builder()
|
||||||
|
.sizes(List.of(
|
||||||
|
"1024x1024", "1280x720", "720x1280",
|
||||||
|
"2048x2048", "2560x1440", "1440x2560"))
|
||||||
|
.aspectRatios(List.of("1:1", "16:9", "9:16"))
|
||||||
|
.build())
|
||||||
.build();
|
.build();
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -86,51 +119,16 @@ public class DashScopeImageProvider implements ImageGenerationProvider {
|
|||||||
public ImageSubmitResult submit(ImageGenerationRequest request, SystemSettingsDTO config) {
|
public ImageSubmitResult submit(ImageGenerationRequest request, SystemSettingsDTO config) {
|
||||||
String apiKey = getDashScopeApiKey();
|
String apiKey = getDashScopeApiKey();
|
||||||
if (apiKey == null) {
|
if (apiKey == null) {
|
||||||
return ImageSubmitResult.failure(id(), "DashScope API Key 未配置");
|
return ImageSubmitResult.failure(id(), "DashScope API Key not configured");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
ImageModelSpec spec = resolveSpec(request);
|
||||||
try {
|
try {
|
||||||
String model = request.getModel() != null && !request.getModel().isBlank()
|
return spec.transport() == ImageModelSpec.Transport.SYNC
|
||||||
? request.getModel() : DEFAULT_MODEL;
|
? submitSyncMultimodal(request, spec, apiKey)
|
||||||
|
: submitAsyncLegacy(request, spec, apiKey);
|
||||||
ObjectNode body = objectMapper.createObjectNode();
|
|
||||||
body.put("model", model);
|
|
||||||
|
|
||||||
ObjectNode input = body.putObject("input");
|
|
||||||
input.put("prompt", request.getPrompt());
|
|
||||||
|
|
||||||
ObjectNode parameters = body.putObject("parameters");
|
|
||||||
// request.size already normalized by ImageGenerationService to one of supportedSizes.
|
|
||||||
// DashScope API uses '*' separator instead of 'x'.
|
|
||||||
String size = request.getSize();
|
|
||||||
if (size != null && !size.isBlank()) {
|
|
||||||
parameters.put("size", size.replace("x", "*"));
|
|
||||||
}
|
|
||||||
int count = request.getCount() != null ? Math.min(request.getCount(), 4) : 1;
|
|
||||||
parameters.put("n", count);
|
|
||||||
|
|
||||||
HttpResponse response = HttpRequest.post(BASE_URL + "/services/aigc/text2image/image-synthesis")
|
|
||||||
.header("Authorization", "Bearer " + apiKey)
|
|
||||||
.header("Content-Type", "application/json")
|
|
||||||
.header("X-DashScope-Async", "enable")
|
|
||||||
.body(body.toString())
|
|
||||||
.timeout(30_000)
|
|
||||||
.execute();
|
|
||||||
|
|
||||||
JsonNode result = objectMapper.readTree(response.body());
|
|
||||||
|
|
||||||
if (response.getStatus() == 200 && result.has("output")) {
|
|
||||||
String taskId = result.path("output").path("task_id").asText();
|
|
||||||
log.info("[DashScope Image] Submitted task: {} (model={})", taskId, model);
|
|
||||||
return ImageSubmitResult.asyncSuccess(taskId, id());
|
|
||||||
} else {
|
|
||||||
String errMsg = result.has("message") ? result.get("message").asText()
|
|
||||||
: "HTTP " + response.getStatus();
|
|
||||||
log.warn("[DashScope Image] Submit failed: {}", errMsg);
|
|
||||||
return ImageSubmitResult.failure(id(), errMsg);
|
|
||||||
}
|
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
log.error("[DashScope Image] Submit error: {}", e.getMessage(), e);
|
log.error("[DashScope Image] Submit error (model={}): {}", spec.id(), e.getMessage(), e);
|
||||||
return ImageSubmitResult.failure(id(), e.getMessage());
|
return ImageSubmitResult.failure(id(), e.getMessage());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -139,11 +137,10 @@ public class DashScopeImageProvider implements ImageGenerationProvider {
|
|||||||
public TaskPollResult checkStatus(String providerTaskId, SystemSettingsDTO config) {
|
public TaskPollResult checkStatus(String providerTaskId, SystemSettingsDTO config) {
|
||||||
String apiKey = getDashScopeApiKey();
|
String apiKey = getDashScopeApiKey();
|
||||||
if (apiKey == null) {
|
if (apiKey == null) {
|
||||||
return TaskPollResult.failed("DashScope API Key 未配置");
|
return TaskPollResult.failed("DashScope API Key not configured");
|
||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
HttpResponse response = HttpRequest.get(BASE_URL + "/tasks/" + providerTaskId)
|
HttpResponse response = HttpRequest.get(DashScopeImageModels.TASKS_ENDPOINT_PREFIX + providerTaskId)
|
||||||
.header("Authorization", "Bearer " + apiKey)
|
.header("Authorization", "Bearer " + apiKey)
|
||||||
.timeout(15_000)
|
.timeout(15_000)
|
||||||
.execute();
|
.execute();
|
||||||
@ -154,11 +151,11 @@ public class DashScopeImageProvider implements ImageGenerationProvider {
|
|||||||
|
|
||||||
return switch (taskStatus) {
|
return switch (taskStatus) {
|
||||||
case "SUCCEEDED" -> {
|
case "SUCCEEDED" -> {
|
||||||
String imageUrl = extractImageUrl(output);
|
String imageUrl = extractLegacyImageUrl(output);
|
||||||
yield TaskPollResult.imageSucceeded(imageUrl, output.toString());
|
yield TaskPollResult.imageSucceeded(imageUrl, output.toString());
|
||||||
}
|
}
|
||||||
case "FAILED" -> {
|
case "FAILED" -> {
|
||||||
String errMsg = output.has("message") ? output.get("message").asText() : "任务失败";
|
String errMsg = output.has("message") ? output.get("message").asText() : "task failed";
|
||||||
yield TaskPollResult.failed(errMsg);
|
yield TaskPollResult.failed(errMsg);
|
||||||
}
|
}
|
||||||
case "RUNNING" -> TaskPollResult.running(null);
|
case "RUNNING" -> TaskPollResult.running(null);
|
||||||
@ -170,6 +167,156 @@ public class DashScopeImageProvider implements ImageGenerationProvider {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ==================== spec resolution ====================
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Pick the model spec for this request. When the request asks for image
|
||||||
|
* editing but names a model that doesn't support edits (or names nothing),
|
||||||
|
* fall back to {@link DashScopeImageModels#DEFAULT_EDIT_MODEL} so the call
|
||||||
|
* doesn't silently degrade to a text-only generation.
|
||||||
|
*/
|
||||||
|
private ImageModelSpec resolveSpec(ImageGenerationRequest request) {
|
||||||
|
boolean wantsEdit = request.getInputImages() != null && !request.getInputImages().isEmpty();
|
||||||
|
String requested = request.getModel();
|
||||||
|
ImageModelSpec spec = DashScopeImageModels.get(requested);
|
||||||
|
if (wantsEdit && !spec.supportsEdit()) {
|
||||||
|
ImageModelSpec edit = DashScopeImageModels.get(DashScopeImageModels.DEFAULT_EDIT_MODEL);
|
||||||
|
log.info("[DashScope Image] Model {} lacks edit support; routing to {}", spec.id(), edit.id());
|
||||||
|
return edit;
|
||||||
|
}
|
||||||
|
return spec;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ==================== sync multimodal-generation ====================
|
||||||
|
|
||||||
|
private ImageSubmitResult submitSyncMultimodal(ImageGenerationRequest request,
|
||||||
|
ImageModelSpec spec,
|
||||||
|
String apiKey) throws Exception {
|
||||||
|
ObjectNode body = objectMapper.createObjectNode();
|
||||||
|
body.put("model", spec.id());
|
||||||
|
|
||||||
|
// input.messages[].content[] — image blocks first when editing,
|
||||||
|
// followed by the text prompt block.
|
||||||
|
ObjectNode input = body.putObject("input");
|
||||||
|
ArrayNode messages = input.putArray("messages");
|
||||||
|
ObjectNode userMsg = messages.addObject();
|
||||||
|
userMsg.put("role", "user");
|
||||||
|
ArrayNode content = userMsg.putArray("content");
|
||||||
|
|
||||||
|
if (request.getInputImages() != null) {
|
||||||
|
for (ImageReference ref : request.getInputImages()) {
|
||||||
|
ObjectNode imgPart = content.addObject();
|
||||||
|
imgPart.put("image", toDataUrl(ref));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
ObjectNode textPart = content.addObject();
|
||||||
|
textPart.put("text", request.getPrompt() == null ? "" : request.getPrompt());
|
||||||
|
|
||||||
|
// parameters block — built and filtered against the model's supports set.
|
||||||
|
ObjectNode parameters = PayloadBuilder.from(spec)
|
||||||
|
.withSize(request.getSize(), request.getAspectRatio())
|
||||||
|
.withCount(request.getCount())
|
||||||
|
.toJsonNode(objectMapper);
|
||||||
|
body.set("parameters", parameters);
|
||||||
|
|
||||||
|
HttpResponse response = HttpRequest.post(spec.endpoint())
|
||||||
|
.header("Authorization", "Bearer " + apiKey)
|
||||||
|
.header("Content-Type", "application/json")
|
||||||
|
.body(body.toString())
|
||||||
|
.timeout(180_000)
|
||||||
|
.execute();
|
||||||
|
|
||||||
|
JsonNode result = objectMapper.readTree(response.body());
|
||||||
|
if (response.getStatus() != 200) {
|
||||||
|
String errMsg = result.has("message") ? result.get("message").asText() : "HTTP " + response.getStatus();
|
||||||
|
log.warn("[DashScope Image] Sync submit failed (model={}): {}", spec.id(), errMsg);
|
||||||
|
return ImageSubmitResult.failure(id(), errMsg);
|
||||||
|
}
|
||||||
|
|
||||||
|
List<String> imageUrls = extractMultimodalImageUrls(result);
|
||||||
|
if (imageUrls.isEmpty()) {
|
||||||
|
return ImageSubmitResult.failure(id(), "Multimodal response carried no image URL");
|
||||||
|
}
|
||||||
|
log.info("[DashScope Image] Sync generated {} image(s) (model={})", imageUrls.size(), spec.id());
|
||||||
|
return ImageSubmitResult.syncSuccess(id(), imageUrls);
|
||||||
|
}
|
||||||
|
|
||||||
|
private List<String> extractMultimodalImageUrls(JsonNode result) {
|
||||||
|
List<String> urls = new ArrayList<>();
|
||||||
|
JsonNode choices = result.path("output").path("choices");
|
||||||
|
if (!choices.isArray()) {
|
||||||
|
return urls;
|
||||||
|
}
|
||||||
|
for (JsonNode choice : choices) {
|
||||||
|
JsonNode parts = choice.path("message").path("content");
|
||||||
|
if (!parts.isArray()) continue;
|
||||||
|
for (JsonNode part : parts) {
|
||||||
|
String url = part.path("image").asText(null);
|
||||||
|
if (url != null && !url.isBlank()) {
|
||||||
|
urls.add(url);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return urls;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ==================== async legacy image-generation ====================
|
||||||
|
|
||||||
|
private ImageSubmitResult submitAsyncLegacy(ImageGenerationRequest request,
|
||||||
|
ImageModelSpec spec,
|
||||||
|
String apiKey) throws Exception {
|
||||||
|
ObjectNode body = objectMapper.createObjectNode();
|
||||||
|
body.put("model", spec.id());
|
||||||
|
|
||||||
|
ObjectNode input = body.putObject("input");
|
||||||
|
input.put("prompt", request.getPrompt() == null ? "" : request.getPrompt());
|
||||||
|
|
||||||
|
ObjectNode parameters = PayloadBuilder.from(spec)
|
||||||
|
.withSize(request.getSize(), request.getAspectRatio())
|
||||||
|
.withCount(request.getCount())
|
||||||
|
.toJsonNode(objectMapper);
|
||||||
|
body.set("parameters", parameters);
|
||||||
|
|
||||||
|
HttpResponse response = HttpRequest.post(spec.endpoint())
|
||||||
|
.header("Authorization", "Bearer " + apiKey)
|
||||||
|
.header("Content-Type", "application/json")
|
||||||
|
.header("X-DashScope-Async", "enable")
|
||||||
|
.body(body.toString())
|
||||||
|
.timeout(30_000)
|
||||||
|
.execute();
|
||||||
|
|
||||||
|
JsonNode result = objectMapper.readTree(response.body());
|
||||||
|
if (response.getStatus() == 200 && result.has("output")) {
|
||||||
|
String taskId = result.path("output").path("task_id").asText();
|
||||||
|
log.info("[DashScope Image] Async submitted task {} (model={})", taskId, spec.id());
|
||||||
|
return ImageSubmitResult.asyncSuccess(taskId, id());
|
||||||
|
}
|
||||||
|
String errMsg = result.has("message") ? result.get("message").asText()
|
||||||
|
: "HTTP " + response.getStatus();
|
||||||
|
log.warn("[DashScope Image] Async submit failed (model={}): {}", spec.id(), errMsg);
|
||||||
|
return ImageSubmitResult.failure(id(), errMsg);
|
||||||
|
}
|
||||||
|
|
||||||
|
private String extractLegacyImageUrl(JsonNode output) {
|
||||||
|
JsonNode results = output.path("results");
|
||||||
|
if (results.isArray() && !results.isEmpty()) {
|
||||||
|
JsonNode first = results.get(0);
|
||||||
|
String url = first.path("url").asText(null);
|
||||||
|
if (url == null || url.isBlank()) {
|
||||||
|
url = first.path("image").asText(null);
|
||||||
|
}
|
||||||
|
return url;
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ==================== shared helpers ====================
|
||||||
|
|
||||||
|
private String toDataUrl(ImageReference ref) {
|
||||||
|
String mime = ref.mimeType() == null || ref.mimeType().isBlank() ? "image/png" : ref.mimeType();
|
||||||
|
return "data:" + mime + ";base64," + Base64.getEncoder().encodeToString(ref.data());
|
||||||
|
}
|
||||||
|
|
||||||
private String getDashScopeApiKey() {
|
private String getDashScopeApiKey() {
|
||||||
try {
|
try {
|
||||||
var providerEntity = modelProviderService.getProviderConfig("dashscope");
|
var providerEntity = modelProviderService.getProviderConfig("dashscope");
|
||||||
@ -178,12 +325,4 @@ public class DashScopeImageProvider implements ImageGenerationProvider {
|
|||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private String extractImageUrl(JsonNode output) {
|
|
||||||
JsonNode results = output.path("results");
|
|
||||||
if (results.isArray() && !results.isEmpty()) {
|
|
||||||
return results.get(0).path("url").asText(null);
|
|
||||||
}
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|||||||
@ -4,6 +4,7 @@ import cn.hutool.http.HttpRequest;
|
|||||||
import cn.hutool.http.HttpResponse;
|
import cn.hutool.http.HttpResponse;
|
||||||
import com.fasterxml.jackson.databind.JsonNode;
|
import com.fasterxml.jackson.databind.JsonNode;
|
||||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
|
import com.fasterxml.jackson.databind.node.ArrayNode;
|
||||||
import com.fasterxml.jackson.databind.node.ObjectNode;
|
import com.fasterxml.jackson.databind.node.ObjectNode;
|
||||||
import lombok.RequiredArgsConstructor;
|
import lombok.RequiredArgsConstructor;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
@ -11,16 +12,35 @@ import org.springframework.stereotype.Component;
|
|||||||
import vip.mate.llm.service.ModelProviderService;
|
import vip.mate.llm.service.ModelProviderService;
|
||||||
import vip.mate.system.model.SystemSettingsDTO;
|
import vip.mate.system.model.SystemSettingsDTO;
|
||||||
import vip.mate.task.AsyncTaskService.TaskPollResult;
|
import vip.mate.task.AsyncTaskService.TaskPollResult;
|
||||||
import vip.mate.tool.video.*;
|
import vip.mate.tool.video.VideoCapability;
|
||||||
|
import vip.mate.tool.video.VideoGenerationProvider;
|
||||||
|
import vip.mate.tool.video.VideoGenerationRequest;
|
||||||
|
import vip.mate.tool.video.VideoProviderCapabilities;
|
||||||
|
import vip.mate.tool.video.VideoSubmitResult;
|
||||||
|
|
||||||
|
import java.util.LinkedHashMap;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* DashScope 视频生成 Provider — 支持通义万相 Wan 2.5 / Wanx 2.1
|
* DashScope video provider — supports two payload families on the same async
|
||||||
* <p>
|
* task model, selected per model id:
|
||||||
* 复用已有的 DashScope LLM provider 的 API Key。
|
*
|
||||||
* API 文档: https://help.aliyun.com/zh/model-studio/developer-reference/video-generation
|
* <ul>
|
||||||
|
* <li><b>Legacy</b> ({@code services/aigc/video-generation/generation}) for
|
||||||
|
* wanx 2.1 and wan 2.5 turbo lines. Body uses {@code input.img_url} for
|
||||||
|
* image-to-video and {@code parameters.size} for sizing.</li>
|
||||||
|
* <li><b>Unified video-synthesis</b>
|
||||||
|
* ({@code services/aigc/video-generation/video-synthesis}) for wan 2.7
|
||||||
|
* and the happyhorse t2v line. Body uses {@code input.media[]} for the
|
||||||
|
* first frame plus {@code parameters.resolution} + {@code parameters.ratio}
|
||||||
|
* for sizing.</li>
|
||||||
|
* </ul>
|
||||||
|
*
|
||||||
|
* Routing is data-driven: each model is registered with its endpoint, body
|
||||||
|
* shape, and capability set; submit/build code consults the spec rather than
|
||||||
|
* branching on model id strings.
|
||||||
*
|
*
|
||||||
* @author MateClaw Team
|
* @author MateClaw Team
|
||||||
*/
|
*/
|
||||||
@ -33,9 +53,52 @@ public class DashScopeVideoProvider implements VideoGenerationProvider {
|
|||||||
private final ObjectMapper objectMapper;
|
private final ObjectMapper objectMapper;
|
||||||
|
|
||||||
private static final String BASE_URL = "https://dashscope.aliyuncs.com/api/v1";
|
private static final String BASE_URL = "https://dashscope.aliyuncs.com/api/v1";
|
||||||
|
private static final String LEGACY_ENDPOINT = BASE_URL + "/services/aigc/video-generation/generation";
|
||||||
|
private static final String UNIFIED_ENDPOINT = BASE_URL + "/services/aigc/video-generation/video-synthesis";
|
||||||
|
private static final String TASKS_ENDPOINT_PREFIX = BASE_URL + "/tasks/";
|
||||||
|
|
||||||
private static final String DEFAULT_T2V_MODEL = "wan2.5-t2v-turbo";
|
private static final String DEFAULT_T2V_MODEL = "wan2.5-t2v-turbo";
|
||||||
private static final String DEFAULT_I2V_MODEL = "wan2.5-i2v-turbo";
|
private static final String DEFAULT_I2V_MODEL = "wan2.5-i2v-turbo";
|
||||||
|
|
||||||
|
private enum BodyShape {
|
||||||
|
/** input.img_url + parameters.size("1280*720") + parameters.duration. */
|
||||||
|
LEGACY,
|
||||||
|
/** input.media[].first_frame + parameters.resolution + parameters.ratio + parameters.duration. */
|
||||||
|
UNIFIED
|
||||||
|
}
|
||||||
|
|
||||||
|
private record ModelSpec(
|
||||||
|
String id,
|
||||||
|
String endpoint,
|
||||||
|
BodyShape bodyShape,
|
||||||
|
Set<VideoCapability> modes
|
||||||
|
) {}
|
||||||
|
|
||||||
|
private static final Map<String, ModelSpec> MODELS = buildCatalog();
|
||||||
|
|
||||||
|
private static Map<String, ModelSpec> buildCatalog() {
|
||||||
|
Map<String, ModelSpec> m = new LinkedHashMap<>();
|
||||||
|
// Legacy line — text-to-video
|
||||||
|
m.put("wan2.5-t2v-turbo", new ModelSpec("wan2.5-t2v-turbo",
|
||||||
|
LEGACY_ENDPOINT, BodyShape.LEGACY, Set.of(VideoCapability.GENERATE)));
|
||||||
|
m.put("wanx2.1-t2v-turbo", new ModelSpec("wanx2.1-t2v-turbo",
|
||||||
|
LEGACY_ENDPOINT, BodyShape.LEGACY, Set.of(VideoCapability.GENERATE)));
|
||||||
|
// Legacy line — image-to-video
|
||||||
|
m.put("wan2.5-i2v-turbo", new ModelSpec("wan2.5-i2v-turbo",
|
||||||
|
LEGACY_ENDPOINT, BodyShape.LEGACY, Set.of(VideoCapability.IMAGE_TO_VIDEO)));
|
||||||
|
m.put("wanx2.1-i2v-turbo", new ModelSpec("wanx2.1-i2v-turbo",
|
||||||
|
LEGACY_ENDPOINT, BodyShape.LEGACY, Set.of(VideoCapability.IMAGE_TO_VIDEO)));
|
||||||
|
// Unified video-synthesis line — wan 2.7
|
||||||
|
m.put("wan2.7-t2v-2026-04-25", new ModelSpec("wan2.7-t2v-2026-04-25",
|
||||||
|
UNIFIED_ENDPOINT, BodyShape.UNIFIED, Set.of(VideoCapability.GENERATE)));
|
||||||
|
m.put("wan2.7-i2v-2026-04-25", new ModelSpec("wan2.7-i2v-2026-04-25",
|
||||||
|
UNIFIED_ENDPOINT, BodyShape.UNIFIED, Set.of(VideoCapability.IMAGE_TO_VIDEO)));
|
||||||
|
// Unified video-synthesis line — happyhorse text-to-video
|
||||||
|
m.put("happyhorse-1.0-t2v", new ModelSpec("happyhorse-1.0-t2v",
|
||||||
|
UNIFIED_ENDPOINT, BodyShape.UNIFIED, Set.of(VideoCapability.GENERATE)));
|
||||||
|
return Map.copyOf(m);
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public String id() {
|
public String id() {
|
||||||
return "dashscope";
|
return "dashscope";
|
||||||
@ -43,7 +106,7 @@ public class DashScopeVideoProvider implements VideoGenerationProvider {
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
public String label() {
|
public String label() {
|
||||||
return "DashScope (通义万相)";
|
return "DashScope (Tongyi Wanxiang / HappyHorse)";
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@ -66,10 +129,10 @@ public class DashScopeVideoProvider implements VideoGenerationProvider {
|
|||||||
return VideoProviderCapabilities.builder()
|
return VideoProviderCapabilities.builder()
|
||||||
.modes(capabilities())
|
.modes(capabilities())
|
||||||
.aspectRatios(List.of("16:9", "9:16", "1:1"))
|
.aspectRatios(List.of("16:9", "9:16", "1:1"))
|
||||||
.supportedDurations(List.of(5, 10))
|
.supportedDurations(List.of(2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15))
|
||||||
.maxDurationSeconds(10)
|
.maxDurationSeconds(15)
|
||||||
.defaultModel(DEFAULT_T2V_MODEL)
|
.defaultModel(DEFAULT_T2V_MODEL)
|
||||||
.models(List.of("wan2.5-t2v-turbo", "wan2.5-i2v-turbo", "wanx2.1-t2v-turbo", "wanx2.1-i2v-turbo"))
|
.models(List.copyOf(MODELS.keySet()))
|
||||||
.build();
|
.build();
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -86,14 +149,12 @@ public class DashScopeVideoProvider implements VideoGenerationProvider {
|
|||||||
public VideoSubmitResult submit(VideoGenerationRequest request, SystemSettingsDTO config) {
|
public VideoSubmitResult submit(VideoGenerationRequest request, SystemSettingsDTO config) {
|
||||||
String apiKey = getDashScopeApiKey();
|
String apiKey = getDashScopeApiKey();
|
||||||
if (apiKey == null) {
|
if (apiKey == null) {
|
||||||
return VideoSubmitResult.failure(id(), "DashScope API Key 未配置");
|
return VideoSubmitResult.failure(id(), "DashScope API Key not configured");
|
||||||
}
|
}
|
||||||
|
ModelSpec spec = resolveSpec(request);
|
||||||
try {
|
try {
|
||||||
String model = resolveModel(request);
|
ObjectNode body = buildRequestBody(request, spec);
|
||||||
ObjectNode body = buildRequestBody(request, model);
|
HttpResponse response = HttpRequest.post(spec.endpoint())
|
||||||
|
|
||||||
HttpResponse response = HttpRequest.post(BASE_URL + "/services/aigc/video-generation/generation")
|
|
||||||
.header("Authorization", "Bearer " + apiKey)
|
.header("Authorization", "Bearer " + apiKey)
|
||||||
.header("Content-Type", "application/json")
|
.header("Content-Type", "application/json")
|
||||||
.header("X-DashScope-Async", "enable")
|
.header("X-DashScope-Async", "enable")
|
||||||
@ -102,19 +163,17 @@ public class DashScopeVideoProvider implements VideoGenerationProvider {
|
|||||||
.execute();
|
.execute();
|
||||||
|
|
||||||
JsonNode result = objectMapper.readTree(response.body());
|
JsonNode result = objectMapper.readTree(response.body());
|
||||||
|
|
||||||
if (response.getStatus() == 200 && result.has("output")) {
|
if (response.getStatus() == 200 && result.has("output")) {
|
||||||
String taskId = result.path("output").path("task_id").asText();
|
String taskId = result.path("output").path("task_id").asText();
|
||||||
log.info("[DashScope Video] Submitted task: {} (model={})", taskId, model);
|
log.info("[DashScope Video] Submitted task {} (model={})", taskId, spec.id());
|
||||||
return VideoSubmitResult.success(taskId, id());
|
return VideoSubmitResult.success(taskId, id());
|
||||||
} else {
|
|
||||||
String errMsg = result.has("message") ? result.get("message").asText()
|
|
||||||
: "HTTP " + response.getStatus();
|
|
||||||
log.warn("[DashScope Video] Submit failed: {}", errMsg);
|
|
||||||
return VideoSubmitResult.failure(id(), errMsg);
|
|
||||||
}
|
}
|
||||||
|
String errMsg = result.has("message") ? result.get("message").asText()
|
||||||
|
: "HTTP " + response.getStatus();
|
||||||
|
log.warn("[DashScope Video] Submit failed (model={}): {}", spec.id(), errMsg);
|
||||||
|
return VideoSubmitResult.failure(id(), errMsg);
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
log.error("[DashScope Video] Submit error: {}", e.getMessage(), e);
|
log.error("[DashScope Video] Submit error (model={}): {}", spec.id(), e.getMessage(), e);
|
||||||
return VideoSubmitResult.failure(id(), e.getMessage());
|
return VideoSubmitResult.failure(id(), e.getMessage());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -123,11 +182,10 @@ public class DashScopeVideoProvider implements VideoGenerationProvider {
|
|||||||
public TaskPollResult checkStatus(String providerTaskId, SystemSettingsDTO config) {
|
public TaskPollResult checkStatus(String providerTaskId, SystemSettingsDTO config) {
|
||||||
String apiKey = getDashScopeApiKey();
|
String apiKey = getDashScopeApiKey();
|
||||||
if (apiKey == null) {
|
if (apiKey == null) {
|
||||||
return TaskPollResult.failed("DashScope API Key 未配置");
|
return TaskPollResult.failed("DashScope API Key not configured");
|
||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
HttpResponse response = HttpRequest.get(BASE_URL + "/tasks/" + providerTaskId)
|
HttpResponse response = HttpRequest.get(TASKS_ENDPOINT_PREFIX + providerTaskId)
|
||||||
.header("Authorization", "Bearer " + apiKey)
|
.header("Authorization", "Bearer " + apiKey)
|
||||||
.timeout(15_000)
|
.timeout(15_000)
|
||||||
.execute();
|
.execute();
|
||||||
@ -135,14 +193,13 @@ public class DashScopeVideoProvider implements VideoGenerationProvider {
|
|||||||
JsonNode result = objectMapper.readTree(response.body());
|
JsonNode result = objectMapper.readTree(response.body());
|
||||||
JsonNode output = result.path("output");
|
JsonNode output = result.path("output");
|
||||||
String taskStatus = output.path("task_status").asText();
|
String taskStatus = output.path("task_status").asText();
|
||||||
|
|
||||||
return switch (taskStatus) {
|
return switch (taskStatus) {
|
||||||
case "SUCCEEDED" -> {
|
case "SUCCEEDED" -> {
|
||||||
String videoUrl = extractVideoUrl(output);
|
String videoUrl = extractVideoUrl(output);
|
||||||
yield TaskPollResult.succeeded(videoUrl, null, output.toString());
|
yield TaskPollResult.succeeded(videoUrl, null, output.toString());
|
||||||
}
|
}
|
||||||
case "FAILED" -> {
|
case "FAILED" -> {
|
||||||
String errMsg = output.has("message") ? output.get("message").asText() : "任务失败";
|
String errMsg = output.has("message") ? output.get("message").asText() : "task failed";
|
||||||
yield TaskPollResult.failed(errMsg);
|
yield TaskPollResult.failed(errMsg);
|
||||||
}
|
}
|
||||||
case "RUNNING" -> TaskPollResult.running(null);
|
case "RUNNING" -> TaskPollResult.running(null);
|
||||||
@ -150,11 +207,107 @@ public class DashScopeVideoProvider implements VideoGenerationProvider {
|
|||||||
};
|
};
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
log.error("[DashScope Video] Poll error for task {}: {}", providerTaskId, e.getMessage());
|
log.error("[DashScope Video] Poll error for task {}: {}", providerTaskId, e.getMessage());
|
||||||
return null; // 轮询异常不终止,等下次重试
|
return null;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// ==================== 内部方法 ====================
|
// ==================== spec resolution ====================
|
||||||
|
|
||||||
|
private ModelSpec resolveSpec(VideoGenerationRequest request) {
|
||||||
|
String requested = request.getModel();
|
||||||
|
if (requested != null && !requested.isBlank() && MODELS.containsKey(requested)) {
|
||||||
|
return MODELS.get(requested);
|
||||||
|
}
|
||||||
|
// Fall back to a default by mode.
|
||||||
|
String defaultId = request.getMode() == VideoCapability.IMAGE_TO_VIDEO
|
||||||
|
? DEFAULT_I2V_MODEL : DEFAULT_T2V_MODEL;
|
||||||
|
return MODELS.get(defaultId);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ==================== body building ====================
|
||||||
|
|
||||||
|
private ObjectNode buildRequestBody(VideoGenerationRequest request, ModelSpec spec) {
|
||||||
|
return switch (spec.bodyShape()) {
|
||||||
|
case LEGACY -> buildLegacyBody(request, spec);
|
||||||
|
case UNIFIED -> buildUnifiedBody(request, spec);
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private ObjectNode buildLegacyBody(VideoGenerationRequest request, ModelSpec spec) {
|
||||||
|
ObjectNode body = objectMapper.createObjectNode();
|
||||||
|
body.put("model", spec.id());
|
||||||
|
|
||||||
|
ObjectNode input = body.putObject("input");
|
||||||
|
input.put("prompt", request.getPrompt() == null ? "" : request.getPrompt());
|
||||||
|
if (spec.modes().contains(VideoCapability.IMAGE_TO_VIDEO)
|
||||||
|
&& request.getImageUrl() != null && !request.getImageUrl().isBlank()) {
|
||||||
|
input.put("img_url", request.getImageUrl());
|
||||||
|
}
|
||||||
|
|
||||||
|
ObjectNode parameters = body.putObject("parameters");
|
||||||
|
String size = aspectRatioToLegacySize(request.getAspectRatio());
|
||||||
|
if (size != null) {
|
||||||
|
parameters.put("size", size);
|
||||||
|
}
|
||||||
|
if (request.getDurationSeconds() != null) {
|
||||||
|
parameters.put("duration", String.valueOf(request.getDurationSeconds()));
|
||||||
|
}
|
||||||
|
return body;
|
||||||
|
}
|
||||||
|
|
||||||
|
private ObjectNode buildUnifiedBody(VideoGenerationRequest request, ModelSpec spec) {
|
||||||
|
ObjectNode body = objectMapper.createObjectNode();
|
||||||
|
body.put("model", spec.id());
|
||||||
|
|
||||||
|
ObjectNode input = body.putObject("input");
|
||||||
|
input.put("prompt", request.getPrompt() == null ? "" : request.getPrompt());
|
||||||
|
if (spec.modes().contains(VideoCapability.IMAGE_TO_VIDEO)
|
||||||
|
&& request.getImageUrl() != null && !request.getImageUrl().isBlank()) {
|
||||||
|
ArrayNode media = input.putArray("media");
|
||||||
|
ObjectNode firstFrame = media.addObject();
|
||||||
|
firstFrame.put("type", "first_frame");
|
||||||
|
firstFrame.put("url", request.getImageUrl());
|
||||||
|
}
|
||||||
|
|
||||||
|
ObjectNode parameters = body.putObject("parameters");
|
||||||
|
String resolution = aspectRatioToUnifiedResolution(request.getAspectRatio());
|
||||||
|
parameters.put("resolution", resolution);
|
||||||
|
if (request.getAspectRatio() != null && !request.getAspectRatio().isBlank()) {
|
||||||
|
parameters.put("ratio", request.getAspectRatio());
|
||||||
|
}
|
||||||
|
if (request.getDurationSeconds() != null) {
|
||||||
|
// Unified endpoint expects the duration as an integer.
|
||||||
|
parameters.put("duration", request.getDurationSeconds());
|
||||||
|
}
|
||||||
|
return body;
|
||||||
|
}
|
||||||
|
|
||||||
|
private String aspectRatioToLegacySize(String aspectRatio) {
|
||||||
|
if (aspectRatio == null) return null;
|
||||||
|
return switch (aspectRatio) {
|
||||||
|
case "16:9" -> "1280*720";
|
||||||
|
case "9:16" -> "720*1280";
|
||||||
|
case "1:1" -> "720*720";
|
||||||
|
default -> null;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private String aspectRatioToUnifiedResolution(String aspectRatio) {
|
||||||
|
// Default to 720P; the unified endpoint also accepts 1080P. Callers that
|
||||||
|
// want to override should pass it via extraParams in a future iteration.
|
||||||
|
return "720P";
|
||||||
|
}
|
||||||
|
|
||||||
|
private String extractVideoUrl(JsonNode output) {
|
||||||
|
if (output.has("video_url")) {
|
||||||
|
return output.get("video_url").asText();
|
||||||
|
}
|
||||||
|
JsonNode results = output.path("results");
|
||||||
|
if (results.isArray() && !results.isEmpty()) {
|
||||||
|
return results.get(0).path("url").asText(null);
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
private String getDashScopeApiKey() {
|
private String getDashScopeApiKey() {
|
||||||
try {
|
try {
|
||||||
@ -164,59 +317,4 @@ public class DashScopeVideoProvider implements VideoGenerationProvider {
|
|||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private String resolveModel(VideoGenerationRequest request) {
|
|
||||||
if (request.getModel() != null && !request.getModel().isBlank()) {
|
|
||||||
return request.getModel();
|
|
||||||
}
|
|
||||||
return request.getMode() == VideoCapability.IMAGE_TO_VIDEO
|
|
||||||
? DEFAULT_I2V_MODEL : DEFAULT_T2V_MODEL;
|
|
||||||
}
|
|
||||||
|
|
||||||
private ObjectNode buildRequestBody(VideoGenerationRequest request, String model) {
|
|
||||||
ObjectNode body = objectMapper.createObjectNode();
|
|
||||||
body.put("model", model);
|
|
||||||
|
|
||||||
ObjectNode input = body.putObject("input");
|
|
||||||
input.put("prompt", request.getPrompt());
|
|
||||||
|
|
||||||
if (request.getMode() == VideoCapability.IMAGE_TO_VIDEO && request.getImageUrl() != null) {
|
|
||||||
input.put("img_url", request.getImageUrl());
|
|
||||||
}
|
|
||||||
|
|
||||||
ObjectNode parameters = body.putObject("parameters");
|
|
||||||
if (request.getAspectRatio() != null) {
|
|
||||||
// DashScope 使用 size 参数,如 "1280*720"
|
|
||||||
String size = aspectRatioToSize(request.getAspectRatio());
|
|
||||||
if (size != null) {
|
|
||||||
parameters.put("size", size);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (request.getDurationSeconds() != null) {
|
|
||||||
parameters.put("duration", String.valueOf(request.getDurationSeconds()));
|
|
||||||
}
|
|
||||||
|
|
||||||
return body;
|
|
||||||
}
|
|
||||||
|
|
||||||
private String aspectRatioToSize(String aspectRatio) {
|
|
||||||
return switch (aspectRatio) {
|
|
||||||
case "16:9" -> "1280*720";
|
|
||||||
case "9:16" -> "720*1280";
|
|
||||||
case "1:1" -> "720*720";
|
|
||||||
default -> null;
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
private String extractVideoUrl(JsonNode output) {
|
|
||||||
JsonNode results = output.path("results");
|
|
||||||
if (results.isArray() && !results.isEmpty()) {
|
|
||||||
return results.get(0).path("url").asText(null);
|
|
||||||
}
|
|
||||||
// 有些模型返回 video_url
|
|
||||||
if (output.has("video_url")) {
|
|
||||||
return output.get("video_url").asText();
|
|
||||||
}
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user