diff --git a/mateclaw-server/src/main/java/vip/mate/agent/binding/service/AgentBindingService.java b/mateclaw-server/src/main/java/vip/mate/agent/binding/service/AgentBindingService.java index 97e1c08b..22095376 100644 --- a/mateclaw-server/src/main/java/vip/mate/agent/binding/service/AgentBindingService.java +++ b/mateclaw-server/src/main/java/vip/mate/agent/binding/service/AgentBindingService.java @@ -403,6 +403,11 @@ public class AgentBindingService { "image_generate", "music_generate", "video_generate", + // HTML → PNG rasteriser. Closes the loop for HTML-producing skills + // (architecture-diagram, infographics, dashboards) so IM channels + // can deliver the artifact as a native image instead of a file or + // a dead markdown link. + "render_html_image", // Universal capabilities the global system prompts (SOUL.md / // AGENTS.md / "Web Search Capability" / "File Reading Guidelines") // explicitly tell the LLM exist. Pre-Phase-2b they were globally diff --git a/mateclaw-server/src/main/java/vip/mate/tool/builtin/HtmlImageRenderTool.java b/mateclaw-server/src/main/java/vip/mate/tool/builtin/HtmlImageRenderTool.java new file mode 100644 index 00000000..2051f2e0 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/builtin/HtmlImageRenderTool.java @@ -0,0 +1,189 @@ +package vip.mate.tool.builtin; + +import com.microsoft.playwright.Browser; +import com.microsoft.playwright.BrowserContext; +import com.microsoft.playwright.BrowserType; +import com.microsoft.playwright.Page; +import com.microsoft.playwright.Playwright; +import com.microsoft.playwright.options.ScreenshotType; +import com.microsoft.playwright.options.WaitUntilState; +import lombok.extern.slf4j.Slf4j; +import org.springframework.ai.tool.annotation.Tool; +import org.springframework.ai.tool.annotation.ToolParam; +import org.springframework.stereotype.Component; +import vip.mate.tool.browser.BrowserLauncher; +import vip.mate.tool.document.FilenameSanitizer; +import vip.mate.tool.document.GeneratedFileCache; +import vip.mate.tool.document.GeneratedFileLink; +import vip.mate.tool.guard.WorkspacePathGuard; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; + +/** + * Render arbitrary HTML to a PNG and return a one-time download URL. + * + *

Bridges the gap between HTML-producing skills (architecture diagrams, + * infographics, dashboards) and IM channels whose native message types only + * accept rasterised images. The PNG is stashed in {@link GeneratedFileCache} + * with an {@code image/png} MIME so the per-channel sniff layer + * ({@code WeComChannelAdapter}, {@code DingTalkChannelAdapter}, …) uploads it + * as a native image attachment rather than a fallback file. + */ +@Slf4j +@Component +public class HtmlImageRenderTool { + + private static final String PNG_MIME = "image/png"; + private static final int DEFAULT_VIEWPORT_WIDTH = 1440; + private static final int DEFAULT_VIEWPORT_HEIGHT = 900; + private static final int MAX_VIEWPORT_DIMENSION = 4096; + private static final int SET_CONTENT_TIMEOUT_MS = 15_000; + + private final GeneratedFileCache cache; + + private volatile Playwright sharedPlaywright; + private final Object playwrightLock = new Object(); + + public HtmlImageRenderTool(GeneratedFileCache cache) { + this.cache = cache; + } + + @Tool(description = """ + Render HTML to a PNG image and return a one-time download URL. + + Use this whenever the user wants an HTML artifact (architecture + diagram, infographic, dashboard, mockup, ...) delivered as an + *image* — especially when the chat is happening on an IM channel + (WeCom / 企业微信, DingTalk, Feishu, Telegram, Discord) where users + cannot click through a raw HTML link. + + The returned URL is `/api/v1/files/generated/` with MIME + `image/png`. Channel adapters detect this MIME and upload the + bytes as a native image message, so the recipient sees an inline + picture rather than a file attachment. + + Typical workflow when paired with an HTML-producing skill: + 1. write_file(filePath="diagram.html", content="...") + 2. render_html_image(filePath="diagram.html", filename="diagram") + 3. return the markdown link to the user + + Or directly, without going through disk: + 1. render_html_image(html="...", filename="diagram") + + Exactly one of `filePath` or `html` must be supplied. The link is + valid for 10 minutes. + """) + public String render_html_image( + @ToolParam(description = "Path to an HTML file on disk (workspace-relative or absolute). Mutually exclusive with `html`.", required = false) + String filePath, + @ToolParam(description = "Inline HTML source. Mutually exclusive with `filePath`.", required = false) + String html, + @ToolParam(description = "Output filename without extension, e.g. 'architecture'") + String filename, + @ToolParam(description = "Viewport width in px (default 1440, max 4096)", required = false) + Integer width, + @ToolParam(description = "Viewport height in px (default 900, max 4096). Ignored when fullPage=true except as initial layout hint.", required = false) + Integer height, + @ToolParam(description = "Capture full scrollable page (default true). Set false to only capture the viewport.", required = false) + Boolean fullPage) { + + String source; + try { + source = resolveHtml(filePath, html); + } catch (IllegalArgumentException e) { + return "Error: " + e.getMessage(); + } catch (Exception e) { + log.error("[HtmlImageRender] failed to load HTML: {}", e.getMessage(), e); + return "Error: failed to load HTML — " + e.getMessage(); + } + + int vw = clampViewport(width, DEFAULT_VIEWPORT_WIDTH); + int vh = clampViewport(height, DEFAULT_VIEWPORT_HEIGHT); + boolean full = fullPage == null || fullPage; + String displayName = FilenameSanitizer.sanitize(filename, "image", ".png") + ".png"; + + byte[] pngBytes; + try { + pngBytes = renderToPng(source, vw, vh, full); + } catch (Exception e) { + log.error("[HtmlImageRender] render failed for {}: {}", displayName, e.getMessage(), e); + String hint = e.getMessage() != null && e.getMessage().contains("Executable doesn't exist") + ? " Hint: run `mvn exec:java -e -Dexec.mainClass=\"com.microsoft.playwright.CLI\" -Dexec.args=\"install chromium\"` to install the bundled browser." + : ""; + return "Render failed: " + e.getMessage() + hint; + } + + log.info("[HtmlImageRender] rendered {} ({} bytes, viewport={}x{}, fullPage={})", + displayName, pngBytes.length, vw, vh, full); + return GeneratedFileLink.resultZh(pngBytes, displayName, PNG_MIME, cache, "图片"); + } + + private String resolveHtml(String filePath, String inlineHtml) throws Exception { + boolean hasPath = filePath != null && !filePath.isBlank(); + boolean hasInline = inlineHtml != null && !inlineHtml.isBlank(); + if (hasPath == hasInline) { + throw new IllegalArgumentException( + "Provide exactly one of `filePath` or `html` (not both, not neither)."); + } + if (hasPath) { + Path path = WorkspacePathGuard.validatePath(filePath); + if (!Files.exists(path)) { + throw new IllegalArgumentException("HTML file not found: " + filePath); + } + if (Files.isDirectory(path)) { + throw new IllegalArgumentException("Path is a directory, not a file: " + filePath); + } + return Files.readString(path, StandardCharsets.UTF_8); + } + return inlineHtml; + } + + private byte[] renderToPng(String source, int viewportWidth, int viewportHeight, boolean fullPage) { + Playwright pw = getOrCreatePlaywright(); + BrowserType.LaunchOptions opts = new BrowserType.LaunchOptions() + .setHeadless(true) + .setArgs(BrowserLauncher.chromiumLaunchArgs()); + Browser browser = pw.chromium().launch(opts); + try { + BrowserContext ctx = browser.newContext(new Browser.NewContextOptions() + .setViewportSize(viewportWidth, viewportHeight) + .setDeviceScaleFactor(2.0)); + try { + Page page = ctx.newPage(); + page.setContent(source, new Page.SetContentOptions() + .setWaitUntil(WaitUntilState.NETWORKIDLE) + .setTimeout(SET_CONTENT_TIMEOUT_MS)); + return page.screenshot(new Page.ScreenshotOptions() + .setFullPage(fullPage) + .setType(ScreenshotType.PNG)); + } finally { + try { ctx.close(); } catch (Exception ignored) {} + } + } finally { + try { browser.close(); } catch (Exception ignored) {} + } + } + + /** + * Lazily create one Playwright instance per JVM. Playwright.create() + * spawns a Node.js child process and costs ~1–2 s; keeping the instance + * around means subsequent screenshots only pay the browser-launch cost. + */ + private Playwright getOrCreatePlaywright() { + Playwright local = sharedPlaywright; + if (local != null) return local; + synchronized (playwrightLock) { + if (sharedPlaywright == null) { + sharedPlaywright = Playwright.create(); + } + return sharedPlaywright; + } + } + + private static int clampViewport(Integer requested, int fallback) { + if (requested == null || requested <= 0) return fallback; + return Math.min(requested, MAX_VIEWPORT_DIMENSION); + } +} diff --git a/mateclaw-server/src/main/resources/skills/architecture-diagram/SKILL.md b/mateclaw-server/src/main/resources/skills/architecture-diagram/SKILL.md index f41af49e..ea209160 100644 --- a/mateclaw-server/src/main/resources/skills/architecture-diagram/SKILL.md +++ b/mateclaw-server/src/main/resources/skills/architecture-diagram/SKILL.md @@ -41,7 +41,8 @@ Based on [Cocoon AI's architecture-diagram-generator](https://github.com/Cocoon- 1. User describes their system architecture (components, connections, technologies) 2. Generate the HTML file following the design system below 3. Save with `write_file` to a `.html` file (e.g. `~/architecture-diagram.html`) -4. User opens in any browser — works offline, no dependencies +4. **If the user wants to view/share the diagram in chat (web console, WeCom / 企业微信, DingTalk, Feishu, Telegram, ...): call `render_html_image(filePath="", filename="")`** and return the markdown link it produces. IM channels can only deliver rasterised images natively, so a PNG is required for the diagram to appear inline rather than as a dead link or a file attachment. +5. Otherwise, the user opens the `.html` directly in a browser — works offline, no dependencies. ### Output Location @@ -50,9 +51,19 @@ Save diagrams to a user-specified path, or default to the current working direct ./[project-name]-architecture.html ``` -### Preview +### Delivering through chat / IM channels -After saving, suggest the user open it: +When the current channel is anything other than a local browser session, follow up `write_file` with: + +``` +render_html_image(filePath="./architecture-diagram.html", filename="architecture") +``` + +This returns a `/api/v1/files/generated/` URL with `image/png` MIME. The channel layer detects the image MIME and uploads the PNG as a native image message (so it renders inline in WeCom / DingTalk / Feishu / Telegram / Web). Without this step, an `.html` artifact reaches IM channels as either a dead markdown link or, at best, a non-previewable file attachment. + +### Local preview + +After saving, the user can open the `.html` directly: ```bash # macOS open ./my-architecture.html