mirror of
https://gitee.com/mateos/mateclaw.git
synced 2026-09-15 11:58:34 +08:00
feat(tool): add render_html_image to deliver HTML artifacts as native IM images
This commit is contained in:
matevip
parent
b74176ef66
commit
071ccff9bf
@ -403,6 +403,11 @@ public class AgentBindingService {
|
|||||||
"image_generate",
|
"image_generate",
|
||||||
"music_generate",
|
"music_generate",
|
||||||
"video_generate",
|
"video_generate",
|
||||||
|
// HTML → PNG rasteriser. Closes the loop for HTML-producing skills
|
||||||
|
// (architecture-diagram, infographics, dashboards) so IM channels
|
||||||
|
// can deliver the artifact as a native image instead of a file or
|
||||||
|
// a dead markdown link.
|
||||||
|
"render_html_image",
|
||||||
// Universal capabilities the global system prompts (SOUL.md /
|
// Universal capabilities the global system prompts (SOUL.md /
|
||||||
// AGENTS.md / "Web Search Capability" / "File Reading Guidelines")
|
// AGENTS.md / "Web Search Capability" / "File Reading Guidelines")
|
||||||
// explicitly tell the LLM exist. Pre-Phase-2b they were globally
|
// explicitly tell the LLM exist. Pre-Phase-2b they were globally
|
||||||
|
|||||||
@ -0,0 +1,189 @@
|
|||||||
|
package vip.mate.tool.builtin;
|
||||||
|
|
||||||
|
import com.microsoft.playwright.Browser;
|
||||||
|
import com.microsoft.playwright.BrowserContext;
|
||||||
|
import com.microsoft.playwright.BrowserType;
|
||||||
|
import com.microsoft.playwright.Page;
|
||||||
|
import com.microsoft.playwright.Playwright;
|
||||||
|
import com.microsoft.playwright.options.ScreenshotType;
|
||||||
|
import com.microsoft.playwright.options.WaitUntilState;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.springframework.ai.tool.annotation.Tool;
|
||||||
|
import org.springframework.ai.tool.annotation.ToolParam;
|
||||||
|
import org.springframework.stereotype.Component;
|
||||||
|
import vip.mate.tool.browser.BrowserLauncher;
|
||||||
|
import vip.mate.tool.document.FilenameSanitizer;
|
||||||
|
import vip.mate.tool.document.GeneratedFileCache;
|
||||||
|
import vip.mate.tool.document.GeneratedFileLink;
|
||||||
|
import vip.mate.tool.guard.WorkspacePathGuard;
|
||||||
|
|
||||||
|
import java.nio.charset.StandardCharsets;
|
||||||
|
import java.nio.file.Files;
|
||||||
|
import java.nio.file.Path;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Render arbitrary HTML to a PNG and return a one-time download URL.
|
||||||
|
*
|
||||||
|
* <p>Bridges the gap between HTML-producing skills (architecture diagrams,
|
||||||
|
* infographics, dashboards) and IM channels whose native message types only
|
||||||
|
* accept rasterised images. The PNG is stashed in {@link GeneratedFileCache}
|
||||||
|
* with an {@code image/png} MIME so the per-channel sniff layer
|
||||||
|
* ({@code WeComChannelAdapter}, {@code DingTalkChannelAdapter}, …) uploads it
|
||||||
|
* as a native image attachment rather than a fallback file.
|
||||||
|
*/
|
||||||
|
@Slf4j
|
||||||
|
@Component
|
||||||
|
public class HtmlImageRenderTool {
|
||||||
|
|
||||||
|
private static final String PNG_MIME = "image/png";
|
||||||
|
private static final int DEFAULT_VIEWPORT_WIDTH = 1440;
|
||||||
|
private static final int DEFAULT_VIEWPORT_HEIGHT = 900;
|
||||||
|
private static final int MAX_VIEWPORT_DIMENSION = 4096;
|
||||||
|
private static final int SET_CONTENT_TIMEOUT_MS = 15_000;
|
||||||
|
|
||||||
|
private final GeneratedFileCache cache;
|
||||||
|
|
||||||
|
private volatile Playwright sharedPlaywright;
|
||||||
|
private final Object playwrightLock = new Object();
|
||||||
|
|
||||||
|
public HtmlImageRenderTool(GeneratedFileCache cache) {
|
||||||
|
this.cache = cache;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Tool(description = """
|
||||||
|
Render HTML to a PNG image and return a one-time download URL.
|
||||||
|
|
||||||
|
Use this whenever the user wants an HTML artifact (architecture
|
||||||
|
diagram, infographic, dashboard, mockup, ...) delivered as an
|
||||||
|
*image* — especially when the chat is happening on an IM channel
|
||||||
|
(WeCom / 企业微信, DingTalk, Feishu, Telegram, Discord) where users
|
||||||
|
cannot click through a raw HTML link.
|
||||||
|
|
||||||
|
The returned URL is `/api/v1/files/generated/<id>` with MIME
|
||||||
|
`image/png`. Channel adapters detect this MIME and upload the
|
||||||
|
bytes as a native image message, so the recipient sees an inline
|
||||||
|
picture rather than a file attachment.
|
||||||
|
|
||||||
|
Typical workflow when paired with an HTML-producing skill:
|
||||||
|
1. write_file(filePath="diagram.html", content="<html>...")
|
||||||
|
2. render_html_image(filePath="diagram.html", filename="diagram")
|
||||||
|
3. return the markdown link to the user
|
||||||
|
|
||||||
|
Or directly, without going through disk:
|
||||||
|
1. render_html_image(html="<html>...", filename="diagram")
|
||||||
|
|
||||||
|
Exactly one of `filePath` or `html` must be supplied. The link is
|
||||||
|
valid for 10 minutes.
|
||||||
|
""")
|
||||||
|
public String render_html_image(
|
||||||
|
@ToolParam(description = "Path to an HTML file on disk (workspace-relative or absolute). Mutually exclusive with `html`.", required = false)
|
||||||
|
String filePath,
|
||||||
|
@ToolParam(description = "Inline HTML source. Mutually exclusive with `filePath`.", required = false)
|
||||||
|
String html,
|
||||||
|
@ToolParam(description = "Output filename without extension, e.g. 'architecture'")
|
||||||
|
String filename,
|
||||||
|
@ToolParam(description = "Viewport width in px (default 1440, max 4096)", required = false)
|
||||||
|
Integer width,
|
||||||
|
@ToolParam(description = "Viewport height in px (default 900, max 4096). Ignored when fullPage=true except as initial layout hint.", required = false)
|
||||||
|
Integer height,
|
||||||
|
@ToolParam(description = "Capture full scrollable page (default true). Set false to only capture the viewport.", required = false)
|
||||||
|
Boolean fullPage) {
|
||||||
|
|
||||||
|
String source;
|
||||||
|
try {
|
||||||
|
source = resolveHtml(filePath, html);
|
||||||
|
} catch (IllegalArgumentException e) {
|
||||||
|
return "Error: " + e.getMessage();
|
||||||
|
} catch (Exception e) {
|
||||||
|
log.error("[HtmlImageRender] failed to load HTML: {}", e.getMessage(), e);
|
||||||
|
return "Error: failed to load HTML — " + e.getMessage();
|
||||||
|
}
|
||||||
|
|
||||||
|
int vw = clampViewport(width, DEFAULT_VIEWPORT_WIDTH);
|
||||||
|
int vh = clampViewport(height, DEFAULT_VIEWPORT_HEIGHT);
|
||||||
|
boolean full = fullPage == null || fullPage;
|
||||||
|
String displayName = FilenameSanitizer.sanitize(filename, "image", ".png") + ".png";
|
||||||
|
|
||||||
|
byte[] pngBytes;
|
||||||
|
try {
|
||||||
|
pngBytes = renderToPng(source, vw, vh, full);
|
||||||
|
} catch (Exception e) {
|
||||||
|
log.error("[HtmlImageRender] render failed for {}: {}", displayName, e.getMessage(), e);
|
||||||
|
String hint = e.getMessage() != null && e.getMessage().contains("Executable doesn't exist")
|
||||||
|
? " Hint: run `mvn exec:java -e -Dexec.mainClass=\"com.microsoft.playwright.CLI\" -Dexec.args=\"install chromium\"` to install the bundled browser."
|
||||||
|
: "";
|
||||||
|
return "Render failed: " + e.getMessage() + hint;
|
||||||
|
}
|
||||||
|
|
||||||
|
log.info("[HtmlImageRender] rendered {} ({} bytes, viewport={}x{}, fullPage={})",
|
||||||
|
displayName, pngBytes.length, vw, vh, full);
|
||||||
|
return GeneratedFileLink.resultZh(pngBytes, displayName, PNG_MIME, cache, "图片");
|
||||||
|
}
|
||||||
|
|
||||||
|
private String resolveHtml(String filePath, String inlineHtml) throws Exception {
|
||||||
|
boolean hasPath = filePath != null && !filePath.isBlank();
|
||||||
|
boolean hasInline = inlineHtml != null && !inlineHtml.isBlank();
|
||||||
|
if (hasPath == hasInline) {
|
||||||
|
throw new IllegalArgumentException(
|
||||||
|
"Provide exactly one of `filePath` or `html` (not both, not neither).");
|
||||||
|
}
|
||||||
|
if (hasPath) {
|
||||||
|
Path path = WorkspacePathGuard.validatePath(filePath);
|
||||||
|
if (!Files.exists(path)) {
|
||||||
|
throw new IllegalArgumentException("HTML file not found: " + filePath);
|
||||||
|
}
|
||||||
|
if (Files.isDirectory(path)) {
|
||||||
|
throw new IllegalArgumentException("Path is a directory, not a file: " + filePath);
|
||||||
|
}
|
||||||
|
return Files.readString(path, StandardCharsets.UTF_8);
|
||||||
|
}
|
||||||
|
return inlineHtml;
|
||||||
|
}
|
||||||
|
|
||||||
|
private byte[] renderToPng(String source, int viewportWidth, int viewportHeight, boolean fullPage) {
|
||||||
|
Playwright pw = getOrCreatePlaywright();
|
||||||
|
BrowserType.LaunchOptions opts = new BrowserType.LaunchOptions()
|
||||||
|
.setHeadless(true)
|
||||||
|
.setArgs(BrowserLauncher.chromiumLaunchArgs());
|
||||||
|
Browser browser = pw.chromium().launch(opts);
|
||||||
|
try {
|
||||||
|
BrowserContext ctx = browser.newContext(new Browser.NewContextOptions()
|
||||||
|
.setViewportSize(viewportWidth, viewportHeight)
|
||||||
|
.setDeviceScaleFactor(2.0));
|
||||||
|
try {
|
||||||
|
Page page = ctx.newPage();
|
||||||
|
page.setContent(source, new Page.SetContentOptions()
|
||||||
|
.setWaitUntil(WaitUntilState.NETWORKIDLE)
|
||||||
|
.setTimeout(SET_CONTENT_TIMEOUT_MS));
|
||||||
|
return page.screenshot(new Page.ScreenshotOptions()
|
||||||
|
.setFullPage(fullPage)
|
||||||
|
.setType(ScreenshotType.PNG));
|
||||||
|
} finally {
|
||||||
|
try { ctx.close(); } catch (Exception ignored) {}
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
try { browser.close(); } catch (Exception ignored) {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Lazily create one Playwright instance per JVM. Playwright.create()
|
||||||
|
* spawns a Node.js child process and costs ~1–2 s; keeping the instance
|
||||||
|
* around means subsequent screenshots only pay the browser-launch cost.
|
||||||
|
*/
|
||||||
|
private Playwright getOrCreatePlaywright() {
|
||||||
|
Playwright local = sharedPlaywright;
|
||||||
|
if (local != null) return local;
|
||||||
|
synchronized (playwrightLock) {
|
||||||
|
if (sharedPlaywright == null) {
|
||||||
|
sharedPlaywright = Playwright.create();
|
||||||
|
}
|
||||||
|
return sharedPlaywright;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static int clampViewport(Integer requested, int fallback) {
|
||||||
|
if (requested == null || requested <= 0) return fallback;
|
||||||
|
return Math.min(requested, MAX_VIEWPORT_DIMENSION);
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -41,7 +41,8 @@ Based on [Cocoon AI's architecture-diagram-generator](https://github.com/Cocoon-
|
|||||||
1. User describes their system architecture (components, connections, technologies)
|
1. User describes their system architecture (components, connections, technologies)
|
||||||
2. Generate the HTML file following the design system below
|
2. Generate the HTML file following the design system below
|
||||||
3. Save with `write_file` to a `.html` file (e.g. `~/architecture-diagram.html`)
|
3. Save with `write_file` to a `.html` file (e.g. `~/architecture-diagram.html`)
|
||||||
4. User opens in any browser — works offline, no dependencies
|
4. **If the user wants to view/share the diagram in chat (web console, WeCom / 企业微信, DingTalk, Feishu, Telegram, ...): call `render_html_image(filePath="<the .html path>", filename="<name>")`** and return the markdown link it produces. IM channels can only deliver rasterised images natively, so a PNG is required for the diagram to appear inline rather than as a dead link or a file attachment.
|
||||||
|
5. Otherwise, the user opens the `.html` directly in a browser — works offline, no dependencies.
|
||||||
|
|
||||||
### Output Location
|
### Output Location
|
||||||
|
|
||||||
@ -50,9 +51,19 @@ Save diagrams to a user-specified path, or default to the current working direct
|
|||||||
./[project-name]-architecture.html
|
./[project-name]-architecture.html
|
||||||
```
|
```
|
||||||
|
|
||||||
### Preview
|
### Delivering through chat / IM channels
|
||||||
|
|
||||||
After saving, suggest the user open it:
|
When the current channel is anything other than a local browser session, follow up `write_file` with:
|
||||||
|
|
||||||
|
```
|
||||||
|
render_html_image(filePath="./architecture-diagram.html", filename="architecture")
|
||||||
|
```
|
||||||
|
|
||||||
|
This returns a `/api/v1/files/generated/<id>` URL with `image/png` MIME. The channel layer detects the image MIME and uploads the PNG as a native image message (so it renders inline in WeCom / DingTalk / Feishu / Telegram / Web). Without this step, an `.html` artifact reaches IM channels as either a dead markdown link or, at best, a non-previewable file attachment.
|
||||||
|
|
||||||
|
### Local preview
|
||||||
|
|
||||||
|
After saving, the user can open the `.html` directly:
|
||||||
```bash
|
```bash
|
||||||
# macOS
|
# macOS
|
||||||
open ./my-architecture.html
|
open ./my-architecture.html
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user