From 9e33782b7dea8758466af42115f729363da66947 Mon Sep 17 00:00:00 2001 From: MIST <52695829+MISTLXC@users.noreply.github.com> Date: Thu, 2 Jul 2026 09:25:16 +0800 Subject: [PATCH] =?UTF-8?q?feat(browser):=20=E6=94=BE=E5=BC=80=E5=86=85?= =?UTF-8?q?=E7=BD=91=E6=9C=8D=E5=8A=A1=E8=AE=BF=E9=97=AE=E9=99=90=E5=88=B6?= =?UTF-8?q?=EF=BC=8C=E6=96=B0=E5=A2=9E=E5=B1=80=E5=9F=9F=E7=BD=91=E9=83=A8?= =?UTF-8?q?=E7=BD=B2=E6=A8=A1=E5=BC=8F=E5=BC=80=E5=85=B3=20(#472)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(browser): 放开内网服务访问限制，新增局域网部署模式开关 ### 背景 局域网部署时，Agent 用浏览器工具访问 http://192.168.x.x:port 等内网服务会被默认 SSRF 严格模式拦截。 ### 方案 新增两个 .env 开关（默认 false，行为与改动前完全一致）： - PLAYWRIGHT_ALLOW_PRIVATE_NETWORK=true ：放行本地回环和私有 IP，云元数据端点仍拦截 - PLAYWRIGHT_IGNORE_HTTPS_ERRORS=true ：忽略 HTTPS 证书错误（自签证书场景） 顺带修复 IPv6 AWS IMDS 网段 fd00:ec2::/64 在严格模式下漏网的问题。 ### 验证 24 个单元测试全通过（UrlSafetyChecker 21 + BrowserProperties 3），覆盖严格/豁免两模式 + 4 个 check 重载 + IPv6 IMDS 网段。 ### 风险 开关仅作用于浏览器工具；公网部署务必保持 false。 * feat(browser): Playwright 超时可配 + snapshot 智能截断与 selector 作用域 ### 背景 Agent 用浏览器工具访问慢链路或大页面（超大表格）时遇到两类问题： 1. Playwright 默认 30s 超时不够用，且无法配置 2. snapshot 全量抓取页面文本，硬截断在 20000 字符处会切在元素中间，LLM 拿到残缺数据且不知道有截断 ### 方案 新增三个 .env 开关（默认值与改动前完全一致）： 开关 作用 PLAYWRIGHT_DEFAULT_TIMEOUT_SECONDS 单次操作超时（click / fill / waitForLoadState） PLAYWRIGHT_NAVIGATION_TIMEOUT_SECONDS 导航超时（page.navigate / load-state） PLAYWRIGHT_SNAPSHOT_MAX_LENGTH snapshot 文本截断长度 snapshot 改造： - 支持 selector 参数作用域到子树（之前只用于 click/type） - budget 机制按元素边界智能截断，不再切在 中间 - 返回 truncated:true + hint 引导 LLM 用 selector 重抓 - JSON 字段顺序优化：truncated/hint 放 content 前，确保框架 spill preview（head 800 chars）能切到 ### 验证 - 28 个单元测试全通过（BrowserPropertiesTest 7 + UrlSafetyCheckerTest 21） - IDE 诊断 0 错误 - 覆盖：默认值不变 + setter 往返 + 严格/豁免两模式 ### 兼容性 - 默认值保持 30s / 30s / 20000，行为与改动前完全一致 - selector 参数本就是 @ToolParam(required=false) ，LLM schema 无变化，只是描述更新引导 snapshot 场景也能用 - 不影响 webhook / image download 等其他 SSRF 守卫 ## 改动汇总 文件 改动 BrowserProperties.java +3 字段： defaultTimeoutSeconds / defaultNavigationTimeoutSeconds / snapshotMaxLength （默认 30/30/20000） BrowserLauncher.java 抽 applyContextDefaults(context) 在三处 context 创建点调用；补全 setIgnoreHTTPSErrors 在 wrapLocalBrowser 落地 BrowserUseTool.java 工具描述 + selector 描述引导 LLM 在 snapshot 场景用 selector；doSnapshot 改造支持 selector 参数 + budget 智能截断 + JSON 字段顺序（truncated/hint 放 content 前确保 spill preview 能切到） docker-compose.yml +3 开关： PLAYWRIGHT_DEFAULT_TIMEOUT_SECONDS / PLAYWRIGHT_NAVIGATION_TIMEOUT_SECONDS / PLAYWRIGHT_SNAPSHOT_MAX_LENGTH .env.example +3 开关，简短说明 BrowserPropertiesTest.java +4 测试覆盖新字段默认值和 setter ## 测试结果 ## 数据流论证的关键决策 决策 依据 truncated:true 和 hint 放在 JSON content 字段之前 ToolResultStorage.buildPreview 会把 >8000 chars 的结果截到 head 800 chars，放在前面确保 LLM 看到 selector 参数描述从 "for click/type" 改为明确说 "OPTIONAL for snapshot" LLM 读 JSON schema 时按描述判断参数用途，原描述误导 LLM 不在 snapshot 用 selector 截断从硬 substring(0, N) 改为 JS budget 机制递归累计 避免切在 订单号 ABC 中间，截断发生在 TEXT_NODE 完整段或下一个子元素开始之前 用 ElementHandle.querySelector + evaluate 替代字符串拼接 selector 进 JS 防止 selector 注入（selector 含特殊字符如引号、反斜杠） 三处 context 创建点统一调 applyContextDefaults 确保 CDP / external-CDP / 本地 launch 三条路径都应用配置的 timeout ## 临时改动还原 文件 改动 还原状态 mateclaw-server/pom.xml 临时加 maven-compiler-plugin + Lombok annotation processor ✅ 已删除，恢复原始状态 ## 未测项 项 原因 doSnapshot 的 JS budget 逻辑 需启动真 Playwright + 大页面，单元测试范围外 BrowserLauncher.applyContextDefaults 是否真的影响 page.click 行为 同上，集成测试范围 LLM 是否真的会按 hint 用 selector 重调 取决于 LLM 推理能力，需端到端测试 --- .env.example | 14 ++ docker-compose.yml | 24 +++ .../mate/tool/browser/BrowserLauncher.java | 33 +++- .../mate/tool/browser/BrowserProperties.java | 57 ++++++ .../mate/tool/browser/UrlSafetyChecker.java | 88 ++++++++- .../vip/mate/tool/builtin/BrowserUseTool.java | 157 +++++++++++----- .../tool/browser/BrowserPropertiesTest.java | 92 ++++++++++ .../tool/browser/UrlSafetyCheckerTest.java | 170 ++++++++++++++++++ 8 files changed, 582 insertions(+), 53 deletions(-) create mode 100644 mateclaw-server/src/test/java/vip/mate/tool/browser/BrowserPropertiesTest.java diff --git a/.env.example b/.env.example index f392a592..1213459f 100644 --- a/.env.example +++ b/.env.example @@ -60,6 +60,20 @@ MATECLAW_BROWSER_CDP_URL= MATECLAW_BROWSER_CHROME_PATH= MATECLAW_BROWSER_CHANNEL= +# ==================== 局域网 部署放开（可选，默认 false 严格模式） ==================== +# 浏览器 SSRF 防护：放行本地回环和私有 IP（127.0.0.1 / 10.x / 192.168.x / +# 172.16-31.x / IPv6 fc00::/7 等），公网部署务必保持 false，否则 SSRF 防护失效 +PLAYWRIGHT_ALLOW_PRIVATE_NETWORK=false +# 浏览器忽略 HTTPS 证书错误（自签证书 / IP 直连 HTTPS 场景） +# 公网部署务必保持 false，否则中间人攻击可绕过证书校验 +PLAYWRIGHT_IGNORE_HTTPS_ERRORS=false +# Playwright 单次操作超时（秒），慢链路 / 大页面可调高 +PLAYWRIGHT_DEFAULT_TIMEOUT_SECONDS=30 +# Playwright 导航超时（秒），慢网络可调高 +PLAYWRIGHT_NAVIGATION_TIMEOUT_SECONDS=30 +# snapshot 文本截断长度，超出会返回 truncated:true 提示 LLM 用 selector 缩小范围 +PLAYWRIGHT_SNAPSHOT_MAX_LENGTH=20000 + # ==================== OpenAI OAuth（Docker，可选） ==================== # # OpenAI ChatGPT OAuth 使用 Codex CLI 的 public client + PKCE / device code， diff --git a/docker-compose.yml b/docker-compose.yml index 36fe8052..66fbab91 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -92,6 +92,30 @@ services: MATECLAW_BROWSER_CDP_URL: ${MATECLAW_BROWSER_CDP_URL:-} MATECLAW_BROWSER_CHROME_PATH: ${MATECLAW_BROWSER_CHROME_PATH:-} MATECLAW_BROWSER_CHANNEL: ${MATECLAW_BROWSER_CHANNEL:-} + # SSRF / TLS relaxations for isolated LAN / on-prem deployments. + # Both default to false (strict mode, public-internet safe). + # The .env file uses the PLAYWRIGHT_* prefix (component-oriented naming, + # not product-oriented) — here we translate to the MATECLAW_BROWSER_* + # container env that Spring Boot relaxed-binding maps to BrowserProperties. + # - PLAYWRIGHT_ALLOW_PRIVATE_NETWORK=true: allow loopback / private / link-local + # addresses through the browser SSRF guard. Cloud-metadata endpoints stay + # blocked. Turn on when the agent must drive http://192.168.x.x:port style + # internal services and has no path to the public internet. + # - PLAYWRIGHT_IGNORE_HTTPS_ERRORS=true: ignore HTTPS certificate errors. + # Auto-enables --ignore-certificate-errors at the Chromium command line + # when ALLOW_PRIVATE_NETWORK is also true (so CDP-attached external + # browsers benefit too). Leave false on internet-facing deployments. + MATECLAW_BROWSER_ALLOW_PRIVATE_NETWORK: ${PLAYWRIGHT_ALLOW_PRIVATE_NETWORK:-false} + MATECLAW_BROWSER_IGNORE_HTTPS_ERRORS: ${PLAYWRIGHT_IGNORE_HTTPS_ERRORS:-false} + # Playwright action / navigation timeouts (seconds). Increase for slow + # LAN or large-page scenarios. Defaults match Playwright's own (30s). + MATECLAW_BROWSER_DEFAULT_TIMEOUT_SECONDS: ${PLAYWRIGHT_DEFAULT_TIMEOUT_SECONDS:-30} + MATECLAW_BROWSER_DEFAULT_NAVIGATION_TIMEOUT_SECONDS: ${PLAYWRIGHT_NAVIGATION_TIMEOUT_SECONDS:-30} + # Hard cap on the textual snapshot returned by action=snapshot. Content + # beyond this length is dropped with a truncated:true flag and a hint to + # retry with selector. Results > framework spill threshold (~8000 chars) + # are further spilt to disk by ToolResultStorage. + MATECLAW_BROWSER_SNAPSHOT_MAX_LENGTH: ${PLAYWRIGHT_SNAPSHOT_MAX_LENGTH:-20000} # OAuth 模式默认保持 auto：localhost 访问走 LOCAL，IP/域名访问走 DEVICE_CODE。 # 本机 Docker 若要强制使用 localhost:1455 回调，可在 .env 显式设为 local。 MATECLAW_OAUTH_OPENAI_DEPLOYMENT_MODE: ${MATECLAW_OAUTH_OPENAI_DEPLOYMENT_MODE:-} diff --git a/mateclaw-server/src/main/java/vip/mate/tool/browser/BrowserLauncher.java b/mateclaw-server/src/main/java/vip/mate/tool/browser/BrowserLauncher.java index ec63b592..6383b622 100644 --- a/mateclaw-server/src/main/java/vip/mate/tool/browser/BrowserLauncher.java +++ b/mateclaw-server/src/main/java/vip/mate/tool/browser/BrowserLauncher.java @@ -126,6 +126,7 @@ public class BrowserLauncher { context = browser.newContext(); page = context.newPage(); } + applyContextDefaults(context); long elapsed = System.currentTimeMillis() - t0; trace.add(Attempt.ok(strategy, "connectOverCDP(" + normalized + ")", elapsed)); return Result.success(browser, context, page, true, normalized, strategy, trace); @@ -282,6 +283,7 @@ public class BrowserLauncher { ? browser.newContext() : browser.contexts().get(0); Page page = context.pages().isEmpty() ? context.newPage() : context.pages().get(0); + applyContextDefaults(context); long elapsed = System.currentTimeMillis() - t0; trace.add(Attempt.ok(Strategy.EXTERNAL_CDP, browserBin + " + connectOverCDP(" + cdpBase + ")", elapsed)); // Hand ownership of `proc` and `userDataDir` to the caller — the session that @@ -336,6 +338,17 @@ public class BrowserLauncher { private BrowserType.LaunchOptions baseLaunchOptions(boolean headed) { BrowserType.LaunchOptions opts = new BrowserType.LaunchOptions().setHeadless(!headed); List args = chromiumLaunchArgs(); + // When the deployment is LAN-isolated AND the operator opted into ignoring + // HTTPS errors, push the flag to the Chromium command line as well. This + // covers the EXTERNAL_CDP path where the browser is spawned by us but its + // contexts are created without NewContextOptions (so setIgnoreHTTPSErrors + // would not apply), and is also a stronger guarantee than the per-context + // flag for self-signed LAN CAs. Gated on allowPrivateNetwork so internet- + // facing deployments cannot accidentally disable cert validation globally. + if (props.isIgnoreHttpsErrors() && props.isAllowPrivateNetwork()) { + args.add("--ignore-certificate-errors"); + args.add("--allow-running-insecure-content"); + } if (!args.isEmpty()) { opts.setArgs(args); } @@ -344,14 +357,30 @@ public class BrowserLauncher { private Result wrapLocalBrowser(Browser browser, Strategy strategy, String desc, long elapsedMs, List trace) { - BrowserContext context = browser.newContext(new Browser.NewContextOptions() + Browser.NewContextOptions opts = new Browser.NewContextOptions() .setViewportSize(props.getViewportWidth(), props.getViewportHeight()) - .setLocale("zh-CN")); + .setLocale("zh-CN"); + if (props.isIgnoreHttpsErrors()) { + opts.setIgnoreHTTPSErrors(true); + } + BrowserContext context = browser.newContext(opts); + applyContextDefaults(context); Page page = context.newPage(); trace.add(Attempt.ok(strategy, desc, elapsedMs)); return Result.success(browser, context, page, false, null, strategy, trace); } + /** + * Apply configurable Playwright timeouts to a freshly acquired context. + * Called on every context creation path (wrapLocalBrowser, tryCdp, + * tryExternalCdpLaunch) so {@code page.click / page.fill / page.navigate} + * inherit the configured limits without per-call boilerplate. + */ + private void applyContextDefaults(BrowserContext context) { + context.setDefaultTimeout(props.getDefaultTimeoutSeconds() * 1000L); + context.setDefaultNavigationTimeout(props.getDefaultNavigationTimeoutSeconds() * 1000L); + } + public static List chromiumLaunchArgs() { List args = new ArrayList<>(); boolean inContainer = isRunningInContainer(); diff --git a/mateclaw-server/src/main/java/vip/mate/tool/browser/BrowserProperties.java b/mateclaw-server/src/main/java/vip/mate/tool/browser/BrowserProperties.java index 1c83f7bb..6836fd19 100644 --- a/mateclaw-server/src/main/java/vip/mate/tool/browser/BrowserProperties.java +++ b/mateclaw-server/src/main/java/vip/mate/tool/browser/BrowserProperties.java @@ -51,13 +51,70 @@ public class BrowserProperties { /** * Block navigations to loopback, private, link-local and cloud-metadata hosts. * Narrow exceptions are configured via {@code mateclaw.security.ssrf-allowlist}. + * Only takes effect when {@link #allowPrivateNetwork} is {@code false}. */ private boolean ssrfCheckEnabled = true; + /** + * Permit the browser to reach loopback / private / link-local addresses + * (127.0.0.1, 10.x, 192.168.x, 172.16-31.x, fc00::/7, ::1, …). Cloud-metadata + * endpoints (169.254.169.254, fd00:ec2::254, …) stay blocked in every mode. + * + *

Scope: browser tool only — webhook / image-download SSRF guards still + * enforce strict mode. Turn on for isolated LAN / on-prem deployments where + * the agent must drive internal services (e.g. {@code http://192.168.x.x:port}) + * and has no path to the public internet. Leave off for internet-facing + * deployments; the {@code ssrf-allowlist} is the narrower escape hatch there. + */ + private boolean allowPrivateNetwork = false; + + /** + * Whether Playwright should ignore HTTPS certificate errors when creating a + * browser context. Useful for LAN deployments where internal services use + * self-signed certificates. Defaults to {@code false} so the strict CA + * validation chain is preserved on internet-facing deployments. + * + *

Effect: + *

    + *
  • Sets {@code Browser.NewContextOptions.ignoreHTTPSErrors = true} for + * contexts created by {@link BrowserLauncher} via Playwright launch.
    • + *
  • When {@link #allowPrivateNetwork} is also {@code true}, additionally + * passes {@code --ignore-certificate-errors} / {@code --allow-running-insecure-content} + * to the Chromium command line — this covers CDP-attached external browsers + * whose existing contexts cannot be re-configured at the NewContext layer.
    • + *
+ * No effect on contexts pre-existing on a user-managed Chrome (action=connect_cdp + * when Chrome already has tabs open) — those keep the Chrome process's own setting. + */ + private boolean ignoreHttpsErrors = false; + /** Viewport width (px) for launched browsers. */ private int viewportWidth = 1280; /** Viewport height (px) for launched browsers. */ private int viewportHeight = 800; + + /** + * Default Playwright action timeout in seconds. Applies to every + * {@code page.click / page.fill / page.waitForLoadState} call after the + * browser context is created. Increase for slow LAN / large-page scenarios. + */ + private int defaultTimeoutSeconds = 30; + + /** + * Default Playwright navigation timeout in seconds. Applies to + * {@code page.navigate} and load-state waits. Increase for slow networks. + */ + private int defaultNavigationTimeoutSeconds = 30; + + /** + * Hard cap on the textual snapshot returned by {@code action=snapshot}. + * Content beyond this length is dropped with a {@code truncated:true} flag + * and a hint suggesting the {@code selector} parameter. Note: results + * larger than the framework spill threshold (~8000 chars) are further + * spilt to disk by {@code ToolResultStorage}; keep this value reasonable + * to avoid forcing every snapshot through the spill-and-preview path. + */ + private int snapshotMaxLength = 20_000; } diff --git a/mateclaw-server/src/main/java/vip/mate/tool/browser/UrlSafetyChecker.java b/mateclaw-server/src/main/java/vip/mate/tool/browser/UrlSafetyChecker.java index 1ddf302b..6d783e56 100644 --- a/mateclaw-server/src/main/java/vip/mate/tool/browser/UrlSafetyChecker.java +++ b/mateclaw-server/src/main/java/vip/mate/tool/browser/UrlSafetyChecker.java @@ -2,6 +2,7 @@ package vip.mate.tool.browser; import vip.mate.common.net.SsrfAllowlist; +import java.net.Inet6Address; import java.net.InetAddress; import java.net.URI; import java.util.Collection; @@ -15,6 +16,25 @@ import java.util.Set; *

Call this before passing any user-controlled URL to the browser or to an * outbound HTTP client. An optional allowlist lets administrators reach specific * internal hosts/IPs/CIDR blocks while every other restricted address stays blocked. + * + *

Modes: + *

    + *
  • Strict (default, {@code allowPrivateNetwork=false}) — blocks + * loopback, any-local, link-local, site-local (private), multicast, and + * all known cloud-metadata endpoints. Use when the agent may reach the + * public internet, where SSRF protection is required.
    • + *
  • Private-network allow ({@code allowPrivateNetwork=true}) — for + * isolated LAN / on-prem deployments. Loopback, private, and link-local + * addresses are allowed through; only cloud-metadata endpoints remain + * blocked so a misconfigured agent cannot exfiltrate cloud credentials.
    • + *
+ * + *

Known limitation: DNS rebinding (TOCTOU). The check resolves the hostname + * once and validates every returned address, but the browser may re-resolve + * the same hostname later and obtain a different IP. Fully closing this requires + * hooking the browser's DNS layer, which Playwright does not expose; the + * private-network-allow mode makes this a non-issue because every private + * address is permitted anyway. */ public final class UrlSafetyChecker { @@ -36,18 +56,43 @@ public final class UrlSafetyChecker { /** * Throw {@link SecurityException} if the URL is unsafe. Accepts http:// and https:// only. + * Equivalent to {@code check(url, List.of(), false)} (strict mode, no allowlist). */ public static void check(String url) { - check(url, List.of()); + check(url, List.of(), false); } /** - * Throw {@link SecurityException} if the URL is unsafe. + * Throw {@link SecurityException} if the URL is unsafe (strict mode). * * @param allowlist hostnames, literal IPs, or IPv4 CIDR blocks that are permitted even * when they would otherwise be blocked (loopback, private, metadata, …). */ public static void check(String url, Collection allowlist) { + check(url, allowlist, false); + } + + /** + * Throw {@link SecurityException} if the URL is unsafe (no allowlist). + * + * @param allowPrivateNetwork {@code true} permits loopback / private / link-local + * addresses; cloud-metadata endpoints stay blocked. + */ + public static void check(String url, boolean allowPrivateNetwork) { + check(url, List.of(), allowPrivateNetwork); + } + + /** + * Throw {@link SecurityException} if the URL is unsafe. + * + * @param allowlist hostnames, literal IPs, or IPv4 CIDR blocks that are + * permitted even when they would otherwise be blocked. + * @param allowPrivateNetwork {@code true} permits loopback / private / link-local + * addresses; cloud-metadata endpoints stay blocked. + * Use only for isolated LAN / on-prem deployments where + * the agent has no path to the public internet. + */ + public static void check(String url, Collection allowlist, boolean allowPrivateNetwork) { if (url == null || url.isBlank()) { throw new SecurityException("URL is required"); } @@ -80,9 +125,19 @@ public final class UrlSafetyChecker { if (SsrfAllowlist.matchesAddress(addr, allowlist)) { continue; } + // Cloud-metadata endpoints are blocked in every mode — never exfiltrate + // cloud credentials via the browser tool, even in private-network-allow mode. + if (isMetadataIp(addr)) { + throw new SecurityException("SSRF blocked: " + hostname + + " resolves to cloud-metadata endpoint " + addr.getHostAddress()); + } + if (allowPrivateNetwork) { + // Skip loopback / any-local / link-local / site-local / multicast checks. + continue; + } if (addr.isLoopbackAddress() || addr.isAnyLocalAddress() || addr.isLinkLocalAddress() || addr.isSiteLocalAddress() - || addr.isMulticastAddress() || isMetadataIp(addr)) { + || addr.isMulticastAddress()) { throw new SecurityException("SSRF blocked: " + hostname + " resolves to restricted address " + addr.getHostAddress()); } @@ -91,14 +146,35 @@ public final class UrlSafetyChecker { throw e; } catch (Exception e) { // DNS resolution failure — let the caller deal with it (browser will show its own error). + // Known limitation: a deliberately slow/timeout DNS server can use this to bypass the + // guard. Not fixable here without a hard fail policy; document as accepted risk. } } + /** + * Identify cloud instance-metadata endpoints by IP. Covers IPv4 literals used by + * AWS / Azure / GCP / Alibaba, and the AWS IPv6 IMDS prefix {@code fd00:ec2::/64} + * (the only documented IPv6 metadata range). IPv6 addresses outside this prefix + * but inside the broader ULA range {@code fc00::/7} are NOT treated as metadata. + */ private static boolean isMetadataIp(InetAddress addr) { String ip = addr.getHostAddress(); - return "169.254.169.254".equals(ip) + // IPv4 literals — kept as string compares for clarity and zero allocation on the hot path. + if ("169.254.169.254".equals(ip) || "100.100.100.200".equals(ip) - || "192.0.0.192".equals(ip) - || "fd00:ec2::254".equalsIgnoreCase(ip); + || "192.0.0.192".equals(ip)) { + return true; + } + // AWS IPv6 IMDS lives in fd00:ec2::/64 — match by 64-bit prefix to cover + // fd00:ec2::254 and any future variant under the same prefix. + if (addr instanceof Inet6Address) { + byte[] b = addr.getAddress(); + // 16 bytes; first 8 must equal fd 00 0e c2 00 00 00 00 + return b.length == 16 + && b[0] == (byte) 0xfd && b[1] == 0x00 + && b[2] == 0x0e && b[3] == (byte) 0xc2 + && b[4] == 0 && b[5] == 0 && b[6] == 0 && b[7] == 0; + } + return false; } } diff --git a/mateclaw-server/src/main/java/vip/mate/tool/builtin/BrowserUseTool.java b/mateclaw-server/src/main/java/vip/mate/tool/builtin/BrowserUseTool.java index e9586c6b..6f80d092 100644 --- a/mateclaw-server/src/main/java/vip/mate/tool/builtin/BrowserUseTool.java +++ b/mateclaw-server/src/main/java/vip/mate/tool/builtin/BrowserUseTool.java @@ -6,6 +6,7 @@ import cn.hutool.json.JSONObject; import cn.hutool.json.JSONUtil; import com.microsoft.playwright.Browser; import com.microsoft.playwright.BrowserContext; +import com.microsoft.playwright.ElementHandle; import com.microsoft.playwright.Page; import com.microsoft.playwright.Playwright; import com.microsoft.playwright.PlaywrightException; @@ -39,10 +40,75 @@ import java.util.regex.Pattern; public class BrowserUseTool { private static final long IDLE_TIMEOUT_MINUTES = 30; - private static final int MAX_SNAPSHOT_LENGTH = 20_000; private static final int CDP_SCAN_PORT_MIN = 9000; private static final int CDP_SCAN_PORT_MAX = 10000; + /** + * Snapshot extractor — runs as an ElementHandle.evaluate so {@code this} + * is the scoped root (document.body when no selector is passed). Uses a + * budget object so truncation stops at element boundaries rather than + * mid-TEXT_NODE, and surfaces a {@code truncated:true} flag to the caller + * so the LLM can be told to retry with a narrower selector. + */ + private static final String SNAPSHOT_JS = """ + (maxLen) => { + const budget = { remaining: maxLen, truncated: false }; + function getVisibleText(node, depth) { + if (depth > 10 || budget.remaining <= 0) return ''; + const results = []; + if (node.nodeType === Node.TEXT_NODE) { + const text = node.textContent.trim(); + if (text) { + if (text.length > budget.remaining) { + const slice = text.substring(0, budget.remaining); + const lastSpace = slice.lastIndexOf(' '); + results.push(lastSpace > budget.remaining * 0.5 + ? slice.substring(0, lastSpace) : slice); + budget.remaining = 0; + budget.truncated = true; + } else { + results.push(text); + budget.remaining -= text.length; + } + } + } else if (node.nodeType === Node.ELEMENT_NODE) { + const el = node; + const style = window.getComputedStyle(el); + if (style.display === 'none' || style.visibility === 'hidden') return ''; + const tag = el.tagName.toLowerCase(); + if (['a', 'button', 'input', 'select', 'textarea'].includes(tag)) { + const id = el.id ? '#' + el.id : ''; + const cls = el.className && typeof el.className === 'string' + ? '.' + el.className.trim().split(/\\s+/).slice(0, 2).join('.') + : ''; + const text = el.textContent ? el.textContent.trim().substring(0, 80) : ''; + const href = el.getAttribute('href') || ''; + const placeholder = el.getAttribute('placeholder') || ''; + const desc = '[' + tag + id + cls + ']' + + (text ? ' "' + text + '"' : '') + + (href ? ' href=' + href : '') + + (placeholder ? ' placeholder=' + placeholder : ''); + if (desc.length > budget.remaining) { + budget.remaining = 0; + budget.truncated = true; + return results.join('\\n'); + } + results.push(desc); + budget.remaining -= desc.length; + } + for (const child of el.childNodes) { + if (budget.remaining <= 0) break; + const childText = getVisibleText(child, depth + 1); + if (childText) results.push(childText); + } + } + return results.join('\\n'); + } + const text = getVisibleText(this, 0); + return JSON.stringify({ text: text, truncated: budget.truncated }); + } + """; + /** SSE broadcaster for pushing browser actions to the frontend in real time. */ private final vip.mate.channel.web.ChatStreamTracker streamTracker; private final BrowserLauncher launcher; @@ -99,7 +165,10 @@ public class BrowserUseTool { - start: Launch a new browser (tries system Chrome, system Edge, then Playwright bundled). Optional headed=true. - stop: Close browser. If connected via CDP, only disconnects (Chrome keeps running). - open: Navigate to a URL. Requires url parameter. Auto-starts browser if not running. - - snapshot: Get page text content, interactive elements, and title. + - snapshot: Get page text content, interactive elements, and title. Optional `selector` + scopes to a subtree — USE IT when the page is large (big tables, long lists) to avoid + truncation. Without selector, content is capped and a `truncated:true` flag is returned + with a hint to retry using selector. - screenshot: Take a screenshot. Optional path to save file; returns base64 if no path. - click: Click an element. Requires selector (CSS selector). - type: Type text into an element. Requires selector and text. @@ -112,7 +181,7 @@ public class BrowserUseTool { public String browser_use( @ToolParam(description = "Action: start|stop|open|snapshot|screenshot|click|type|eval|connect_cdp|list_cdp_targets|navigate_back|diagnose") String action, @ToolParam(description = "URL to navigate to (for open), or CDP base URL (for connect_cdp, e.g. http://localhost:9222)", required = false) String url, - @ToolParam(description = "CSS selector for target element (for click/type)", required = false) String selector, + @ToolParam(description = "CSS selector. REQUIRED for click/type. OPTIONAL for snapshot: pass to scope to a subtree when previous snapshot returned truncated:true.", required = false) String selector, @ToolParam(description = "Text to type (for action=type)", required = false) String text, @ToolParam(description = "JavaScript code to execute (for action=eval). Top-level await is allowed; add `return` to return a value when the snippet uses await.", required = false) String code, @ToolParam(description = "File path to save screenshot (for action=screenshot)", required = false) String path, @@ -138,7 +207,7 @@ public class BrowserUseTool { case "start" -> doStart(sessionKey, Boolean.TRUE.equals(headed)); case "stop" -> doStop(sessionKey); case "open" -> doOpen(sessionKey, url); - case "snapshot" -> doSnapshot(sessionKey); + case "snapshot" -> doSnapshot(sessionKey, selector); case "screenshot" -> doScreenshot(sessionKey, path); case "click" -> doClick(sessionKey, selector); case "type" -> doType(sessionKey, selector, text); @@ -432,7 +501,9 @@ public class BrowserUseTool { if (launcher.properties().isSsrfCheckEnabled()) { try { - UrlSafetyChecker.check(normalizedUrl, ssrfProperties.getSsrfAllowlist()); + UrlSafetyChecker.check(normalizedUrl, + ssrfProperties.getSsrfAllowlist(), + launcher.properties().isAllowPrivateNetwork()); } catch (SecurityException se) { log.warn("[BrowserUse] SSRF check rejected url={}: {}", normalizedUrl, se.getMessage()); return error(se.getMessage()); @@ -490,7 +561,7 @@ public class BrowserUseTool { return JSONUtil.toJsonPrettyStr(result); } - private String doSnapshot(String sessionKey) { + private String doSnapshot(String sessionKey, String selector) { BrowserSession session = requireSession(sessionKey); if (session == null) { return error("No browser running. Use action=start first."); @@ -502,50 +573,46 @@ public class BrowserUseTool { String title = page.title(); String url = page.url(); - String textContent = page.evaluate(""" - (() => { - function getVisibleText(node, depth) { - if (depth > 10) return ''; - const results = []; - if (node.nodeType === Node.TEXT_NODE) { - const text = node.textContent.trim(); - if (text) results.push(text); - } else if (node.nodeType === Node.ELEMENT_NODE) { - const el = node; - const style = window.getComputedStyle(el); - if (style.display === 'none' || style.visibility === 'hidden') return ''; - const tag = el.tagName.toLowerCase(); - if (['a', 'button', 'input', 'select', 'textarea'].includes(tag)) { - const id = el.id ? '#' + el.id : ''; - const cls = el.className && typeof el.className === 'string' - ? '.' + el.className.trim().split(/\\s+/).slice(0, 2).join('.') - : ''; - const text = el.textContent ? el.textContent.trim().substring(0, 80) : ''; - const href = el.getAttribute('href') || ''; - const placeholder = el.getAttribute('placeholder') || ''; - const selector = tag + id + cls; - let desc = '[' + selector + ']'; - if (text) desc += ' "' + text + '"'; - if (href) desc += ' href=' + href; - if (placeholder) desc += ' placeholder=' + placeholder; - results.push(desc); - } - for (const child of el.childNodes) { - const childText = getVisibleText(child, depth + 1); - if (childText) results.push(childText); - } - } - return results.join('\\n'); - } - const text = getVisibleText(document.body, 0); - return text.substring(0, %d); - })() - """.formatted(MAX_SNAPSHOT_LENGTH)).toString(); + // Resolve root element via ElementHandle — safer than string-concatenating + // the selector into JS (avoids selector-injection via crafted selectors). + // Falls back to body when no selector is provided. + ElementHandle root; + if (selector != null && !selector.isBlank()) { + root = page.querySelector(selector); + if (root == null) { + return error("Snapshot root not found for selector: " + selector); + } + } else { + root = page.querySelector("body"); + if (root == null) { + return error("Snapshot failed: document.body not available"); + } + } + + int maxLen = launcher.properties().getSnapshotMaxLength(); + String jsResult = (String) root.evaluate(SNAPSHOT_JS, maxLen); + + // JS returns { text: "...", truncated: true/false } + JSONObject parsed = JSONUtil.parseObj(jsResult); + String textContent = parsed.getStr("text"); + boolean truncated = parsed.getBool("truncated", false); JSONObject result = new JSONObject(); result.set("ok", true); result.set("title", title); result.set("url", url); + // IMPORTANT: truncated + hint MUST come before content. The framework's + // spill-preview keeps only the head ~800 chars of the JSON, so placing + // these flags first ensures the LLM still sees them after a spill. + result.set("truncated", truncated); + if (truncated) { + result.set("hint", "Content truncated. Re-call browser_use with action=snapshot" + + " and selector= to scope to a subtree (e.g. selector='#main'," + + " selector='table tbody tr')."); + } + if (selector != null && !selector.isBlank()) { + result.set("scopedTo", selector); + } result.set("content", textContent); return JSONUtil.toJsonPrettyStr(result); } diff --git a/mateclaw-server/src/test/java/vip/mate/tool/browser/BrowserPropertiesTest.java b/mateclaw-server/src/test/java/vip/mate/tool/browser/BrowserPropertiesTest.java new file mode 100644 index 00000000..6070b25e --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/tool/browser/BrowserPropertiesTest.java @@ -0,0 +1,92 @@ +package vip.mate.tool.browser; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Unit tests for the configurable {@link BrowserProperties} fields: SSRF / + * TLS relaxation toggles, Playwright timeouts, and snapshot length cap. + * + *

Defaults must keep deployments unchanged from before this feature: + * strict SSRF, strict TLS, 30s timeouts, 20000-char snapshot cap. + */ +class BrowserPropertiesTest { + + @Test + @DisplayName("Defaults: allowPrivateNetwork=false, ignoreHttpsErrors=false, ssrfCheckEnabled=true") + void defaultsAreStrict() { + BrowserProperties props = new BrowserProperties(); + assertFalse(props.isAllowPrivateNetwork(), + "allowPrivateNetwork must default to false (strict SSRF mode)"); + assertFalse(props.isIgnoreHttpsErrors(), + "ignoreHttpsErrors must default to false (strict TLS validation)"); + assertTrue(props.isSsrfCheckEnabled(), + "ssrfCheckEnabled must remain true (untouched by this feature)"); + } + + @Test + @DisplayName("Defaults: timeouts=30s, snapshotMaxLength=20000") + void defaultsForTimeoutsAndSnapshot() { + BrowserProperties props = new BrowserProperties(); + assertEquals(30, props.getDefaultTimeoutSeconds(), + "defaultTimeoutSeconds must default to 30 (Playwright default)"); + assertEquals(30, props.getDefaultNavigationTimeoutSeconds(), + "defaultNavigationTimeoutSeconds must default to 30 (Playwright default)"); + assertEquals(20_000, props.getSnapshotMaxLength(), + "snapshotMaxLength must default to 20000 (legacy MAX_SNAPSHOT_LENGTH)"); + } + + @Test + @DisplayName("Setter round-trip: allowPrivateNetwork") + void setterAllowPrivateNetwork() { + BrowserProperties props = new BrowserProperties(); + props.setAllowPrivateNetwork(true); + assertTrue(props.isAllowPrivateNetwork()); + props.setAllowPrivateNetwork(false); + assertFalse(props.isAllowPrivateNetwork()); + } + + @Test + @DisplayName("Setter round-trip: ignoreHttpsErrors") + void setterIgnoreHttpsErrors() { + BrowserProperties props = new BrowserProperties(); + props.setIgnoreHttpsErrors(true); + assertTrue(props.isIgnoreHttpsErrors()); + props.setIgnoreHttpsErrors(false); + assertFalse(props.isIgnoreHttpsErrors()); + } + + @Test + @DisplayName("Setter round-trip: defaultTimeoutSeconds") + void setterDefaultTimeoutSeconds() { + BrowserProperties props = new BrowserProperties(); + props.setDefaultTimeoutSeconds(120); + assertEquals(120, props.getDefaultTimeoutSeconds()); + props.setDefaultTimeoutSeconds(5); + assertEquals(5, props.getDefaultTimeoutSeconds()); + } + + @Test + @DisplayName("Setter round-trip: defaultNavigationTimeoutSeconds") + void setterDefaultNavigationTimeoutSeconds() { + BrowserProperties props = new BrowserProperties(); + props.setDefaultNavigationTimeoutSeconds(60); + assertEquals(60, props.getDefaultNavigationTimeoutSeconds()); + props.setDefaultNavigationTimeoutSeconds(15); + assertEquals(15, props.getDefaultNavigationTimeoutSeconds()); + } + + @Test + @DisplayName("Setter round-trip: snapshotMaxLength") + void setterSnapshotMaxLength() { + BrowserProperties props = new BrowserProperties(); + props.setSnapshotMaxLength(5_000); + assertEquals(5_000, props.getSnapshotMaxLength()); + props.setSnapshotMaxLength(100_000); + assertEquals(100_000, props.getSnapshotMaxLength()); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/tool/browser/UrlSafetyCheckerTest.java b/mateclaw-server/src/test/java/vip/mate/tool/browser/UrlSafetyCheckerTest.java index a0c486c1..6246a95c 100644 --- a/mateclaw-server/src/test/java/vip/mate/tool/browser/UrlSafetyCheckerTest.java +++ b/mateclaw-server/src/test/java/vip/mate/tool/browser/UrlSafetyCheckerTest.java @@ -1,15 +1,33 @@ package vip.mate.tool.browser; import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; import java.util.List; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; +/** + * Unit tests for {@link UrlSafetyChecker}. + * + *

Covers: + *

    + *
  • The three check overloads ({@code check(url)}, {@code check(url, allowlist)}, + * {@code check(url, boolean)}, {@code check(url, allowlist, boolean)}).
    • + *
  • Strict mode (default) — blocks loopback / private / link-local / multicast / metadata.
    • + *
  • Private-network-allow mode — permits loopback / private / link-local but still + * blocks cloud-metadata endpoints (IPv4 literals and the AWS IPv6 IMDS prefix + * {@code fd00:ec2::/64}).
    • + *
  • Allowlist short-circuit, scheme/host validation, IPv6 AWS IMDS prefix matching.
    • + *
+ */ class UrlSafetyCheckerTest { + // ==================== Strict mode (default) ==================== + @Test @DisplayName("Rejects private, loopback and metadata addresses by default") void blocksRestrictedAddressesWithoutAllowlist() { @@ -60,4 +78,156 @@ class UrlSafetyCheckerTest { assertDoesNotThrow(() -> UrlSafetyChecker.check("http://8.8.8.8/")); assertDoesNotThrow(() -> UrlSafetyChecker.check("https://1.1.1.1/")); } + + // ==================== AWS IPv6 IMDS prefix (new) ==================== + + @Test + @DisplayName("Blocks AWS IPv6 IMDS literal fd00:ec2::254 in strict mode") + void blocksAwsIpv6ImdsLiteral() { + // Before the prefix match, only the exact string "fd00:ec2::254" was blocked. + // Now any address in fd00:ec2::/64 is blocked — this test guards the literal too. + assertThrows(SecurityException.class, () -> + UrlSafetyChecker.check("http://[fd00:ec2::254]/latest/meta-data/")); + } + + @Test + @DisplayName("Blocks any address in AWS IPv6 IMDS prefix fd00:ec2::/64 (strict mode)") + void blocksAwsIpv6ImdsPrefix() { + // These were NOT blocked before the prefix match was added — they would slip through + // because InetAddress.isSiteLocalAddress() returns false for IPv6. The new + // isMetadataIp byte-prefix check closes that gap. + assertThrows(SecurityException.class, () -> + UrlSafetyChecker.check("http://[fd00:ec2::1]/")); + assertThrows(SecurityException.class, () -> + UrlSafetyChecker.check("http://[fd00:ec2::ffff]/")); + assertThrows(SecurityException.class, () -> + UrlSafetyChecker.check("http://[fd00:ec2:0:0:0:0:0:1]/")); + } + + // ==================== Private-network-allow mode (new) ==================== + + @Nested + @DisplayName("Private-network-allow mode (allowPrivateNetwork=true)") + class PrivateNetworkAllowMode { + + @Test + @DisplayName("Permits loopback IPv4") + void permitsLoopbackIpv4() { + assertDoesNotThrow(() -> + UrlSafetyChecker.check("http://127.0.0.1:18080/", true)); + } + + @Test + @DisplayName("Permits private IPv4 ranges (10.x / 172.16-31.x / 192.168.x)") + void permitsPrivateIpv4Ranges() { + assertDoesNotThrow(() -> UrlSafetyChecker.check("http://10.0.0.5/", true)); + assertDoesNotThrow(() -> UrlSafetyChecker.check("http://192.168.1.1/", true)); + assertDoesNotThrow(() -> UrlSafetyChecker.check("http://172.16.0.1/", true)); + assertDoesNotThrow(() -> UrlSafetyChecker.check("http://172.31.255.255/", true)); + } + + @Test + @DisplayName("Permits link-local IPv4 (169.254.0.0/16) except cloud metadata") + void permitsLinkLocalIpv4() { + // 169.254.x.x is link-local — typically used for LAN service discovery. + // The metadata endpoint 169.254.169.254 is excluded separately below. + assertDoesNotThrow(() -> UrlSafetyChecker.check("http://169.254.1.1/", true)); + } + + @Test + @DisplayName("Still blocks cloud metadata IPv4 endpoints") + void stillBlocksMetadataIpv4() { + // Cloud metadata endpoints must remain blocked in every mode. + assertThrows(SecurityException.class, () -> + UrlSafetyChecker.check("http://169.254.169.254/latest/meta-data/", true)); + assertThrows(SecurityException.class, () -> + UrlSafetyChecker.check("http://100.100.100.200/", true)); + assertThrows(SecurityException.class, () -> + UrlSafetyChecker.check("http://192.0.0.192/", true)); + } + + @Test + @DisplayName("Still blocks AWS IPv6 IMDS endpoints (literal and prefix)") + void stillBlocksAwsIpv6Imds() { + assertThrows(SecurityException.class, () -> + UrlSafetyChecker.check("http://[fd00:ec2::254]/", true)); + assertThrows(SecurityException.class, () -> + UrlSafetyChecker.check("http://[fd00:ec2::1]/", true)); + } + + @Test + @DisplayName("Still blocks hard-coded blocked hostnames (localhost, ::1)") + void stillBlocksHardcodedHostnames() { + // BLOCKED_HOSTNAMES is a hard blacklist that allowPrivateNetwork cannot bypass. + // Operators who need these hosts must use the explicit allowlist instead. + assertThrows(SecurityException.class, () -> + UrlSafetyChecker.check("http://localhost/", true)); + assertThrows(SecurityException.class, () -> + UrlSafetyChecker.check("http://[::1]/", true)); + } + + @Test + @DisplayName("Permits public IPv4 addresses (no regression)") + void permitsPublicIpv4() { + assertDoesNotThrow(() -> UrlSafetyChecker.check("http://8.8.8.8/", true)); + assertDoesNotThrow(() -> UrlSafetyChecker.check("https://1.1.1.1/", true)); + } + + @Test + @DisplayName("Three-arg overload combines allowlist + allowPrivateNetwork") + void combinesAllowlistAndAllowPrivateNetwork() { + // Allowlist short-circuits even in strict mode — but here allowPrivateNetwork=true + // already permits the private IP, so the allowlist is redundant. Verify both + // paths reach the same outcome. + assertDoesNotThrow(() -> + UrlSafetyChecker.check("http://192.168.100.100/", List.of(), true)); + // Allowlist can still unblock a blocked hostname in private-network mode. + assertDoesNotThrow(() -> + UrlSafetyChecker.check("http://localhost/", List.of("localhost"), true)); + } + + @Test + @DisplayName("Boolean overload equals three-arg overload with empty allowlist") + void booleanOverloadMatchesThreeArg() { + // Sanity: check(url, true) behaves identically to check(url, List.of(), true). + assertDoesNotThrow(() -> UrlSafetyChecker.check("http://127.0.0.1/", true)); + assertDoesNotThrow(() -> UrlSafetyChecker.check("http://127.0.0.1/", List.of(), true)); + assertThrows(SecurityException.class, () -> UrlSafetyChecker.check("http://169.254.169.254/", true)); + assertThrows(SecurityException.class, () -> UrlSafetyChecker.check("http://169.254.169.254/", List.of(), true)); + } + } + + // ==================== Input validation ==================== + + @Test + @DisplayName("Rejects null or blank URL") + void rejectsNullOrlBlankUrl() { + SecurityException nullEx = assertThrows(SecurityException.class, () -> UrlSafetyChecker.check(null)); + assertEquals("URL is required", nullEx.getMessage()); + SecurityException blankEx = assertThrows(SecurityException.class, () -> UrlSafetyChecker.check(" ")); + assertEquals("URL is required", blankEx.getMessage()); + } + + @Test + @DisplayName("Rejects malformed URL") + void rejectsMalformedUrl() { + // URI.create rejects strings that are not valid URIs. + assertThrows(SecurityException.class, () -> UrlSafetyChecker.check("http://[invalid")); + } + + @Test + @DisplayName("Rejects URL without a host") + void rejectsUrlWithoutHost() { + assertThrows(SecurityException.class, () -> UrlSafetyChecker.check("http://")); + assertThrows(SecurityException.class, () -> UrlSafetyChecker.check("https:///path")); + } + + @Test + @DisplayName("Rejects blocked hostname metadata.google.internal") + void blocksMetadataHostname() { + assertThrows(SecurityException.class, () -> + UrlSafetyChecker.check("http://metadata.google.internal/computeMetadata/v1/")); + assertThrows(SecurityException.class, () -> + UrlSafetyChecker.check("http://metadata.google.internal/", true)); + } }