diff --git a/mateclaw-server/src/main/java/vip/mate/tool/browser/BrowserPrivacyGuard.java b/mateclaw-server/src/main/java/vip/mate/tool/browser/BrowserPrivacyGuard.java new file mode 100644 index 00000000..5255e2e9 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/browser/BrowserPrivacyGuard.java @@ -0,0 +1,142 @@ +package vip.mate.tool.browser; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import vip.mate.tool.guard.model.GuardDecision; +import vip.mate.tool.guard.model.GuardSeverity; +import vip.mate.tool.guard.model.ToolGuardAuditLogEntity; +import vip.mate.tool.guard.repository.ToolGuardAuditLogMapper; + +import java.net.URI; +import java.util.List; + +/** + * Privacy guard for browser sessions attached to a user's own logged-in browser. + * + *

When the browser tool connects to a Chrome the user is running themselves + * (via the DevTools Protocol, process not spawned by us), that browser may have + * banking, webmail or internal-admin tabs open with live sessions. Reading such + * a page in full — screenshot, arbitrary JS eval, or a text/accessibility dump — + * would funnel private content into the model and possibly into persisted + * transcripts. This guard classifies the current page and refuses those + * content-reading actions on pages that look sensitive, while leaving plain + * navigation untouched. It never applies to headless or self-spawned browsers. + * + *

Classification precedence: configured trusted hosts (always safe) → + * configured sensitive hosts → a built-in keyword heuristic over host + path. + * Every block is overridable by adding the host to + * {@code mateclaw.browser.privacy.trusted-hosts}. + */ +@Slf4j +@Component +public class BrowserPrivacyGuard { + + /** Keyword signals that a page handles money, identity, or privileged access. */ + private static final List SENSITIVE_SIGNALS = List.of( + "bank", "banking", "pay", "payment", "wallet", "checkout", "billing", "invoice", + "mail", "webmail", "signin", "login", "logon", "account", "admin", "console", + "secure", "oauth", "authorize", "password", "passport", "identity"); + + private final BrowserProperties properties; + private final ToolGuardAuditLogMapper auditMapper; + + public BrowserPrivacyGuard(BrowserProperties properties, ToolGuardAuditLogMapper auditMapper) { + this.properties = properties; + this.auditMapper = auditMapper; + } + + /** + * Decide whether a content-reading action must be refused. Returns a + * human-readable block reason, or {@code null} when the action may proceed. + * + * @param userManagedBrowser true only when attached to a Chrome the user runs themselves + * @param url the current page URL + * @param action the action being attempted (screenshot / eval / snapshot) + */ + public String blockReason(boolean userManagedBrowser, String url, String action) { + if (!properties.getPrivacy().isEnabled() || !userManagedBrowser) { + return null; + } + if (!isSensitive(url)) { + return null; + } + return "Refusing action=" + action + " on what looks like a sensitive page (" + safeHost(url) + + ") inside your own logged-in browser, to avoid exposing private content to the" + + " model. Plain navigation is still allowed. If this page is safe to read, add its" + + " host to mateclaw.browser.privacy.trusted-hosts."; + } + + /** True when the URL matches a configured/heuristic sensitive signal and is not trusted. */ + public boolean isSensitive(String url) { + if (url == null || url.isBlank()) { + return false; + } + String host; + String path; + try { + URI uri = URI.create(url); + host = uri.getHost() == null ? "" : uri.getHost().toLowerCase(); + path = uri.getPath() == null ? "" : uri.getPath().toLowerCase(); + } catch (IllegalArgumentException e) { + return false; + } + if (host.isEmpty()) { + return false; + } + if (hostMatches(host, properties.getPrivacy().getTrustedHosts())) { + return false; + } + if (hostMatches(host, properties.getPrivacy().getSensitiveHosts())) { + return true; + } + String hostAndPath = host + path; + for (String sig : SENSITIVE_SIGNALS) { + if (hostAndPath.contains(sig)) { + return true; + } + } + return false; + } + + /** Record a blocked read into the shared tool-guard audit log so it shows up in the audit panel. */ + public void audit(String conversationId, String action, String url, String reason) { + try { + ToolGuardAuditLogEntity entity = new ToolGuardAuditLogEntity(); + entity.setConversationId(conversationId); + entity.setToolName("browser_use"); + entity.setDecision(GuardDecision.BLOCK.name()); + entity.setMaxSeverity(GuardSeverity.HIGH.name()); + entity.setToolParamsJson("{\"action\":\"" + action + "\",\"host\":\"" + safeHost(url) + "\"}"); + entity.setFindingsJson("[{\"type\":\"sensitive-page\",\"reason\":\"" + + reason.replace("\"", "'") + "\"}]"); + auditMapper.insert(entity); + } catch (Exception e) { + log.warn("[BrowserPrivacyGuard] Failed to record audit entry: {}", e.getMessage()); + } + } + + private static boolean hostMatches(String host, List patterns) { + if (patterns == null) { + return false; + } + for (String p : patterns) { + if (p == null || p.isBlank()) { + continue; + } + String pat = p.trim().toLowerCase(); + if (host.equals(pat) || host.endsWith("." + pat)) { + return true; + } + } + return false; + } + + private static String safeHost(String url) { + try { + String h = URI.create(url).getHost(); + return h == null ? "unknown" : h; + } catch (Exception e) { + return "unknown"; + } + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/tool/browser/BrowserProperties.java b/mateclaw-server/src/main/java/vip/mate/tool/browser/BrowserProperties.java index 6836fd19..5d110a46 100644 --- a/mateclaw-server/src/main/java/vip/mate/tool/browser/BrowserProperties.java +++ b/mateclaw-server/src/main/java/vip/mate/tool/browser/BrowserProperties.java @@ -116,5 +116,69 @@ public class BrowserProperties { * to avoid forcing every snapshot through the spill-and-preview path. */ private int snapshotMaxLength = 20_000; + + /** + * Whether {@code action=snapshot} includes non-interactive structural nodes + * (headings, list items, navigation, images) in the accessibility tree. + * Interactive elements always get a reference handle; structural nodes are + * emitted without one, purely to give the model page context. Turn off to + * produce a terser tree of only actionable elements. + */ + private boolean snapshotIncludeNonInteractive = true; + + /** Privacy guard for sessions attached to a user's own logged-in browser (action=connect_cdp). */ + private Privacy privacy = new Privacy(); + + /** Raw DevTools Protocol escape hatch (action=cdp) configuration. */ + private Cdp cdp = new Cdp(); + + /** + * Controls {@code action=cdp}, which forwards a raw Chrome DevTools Protocol + * command. Constrained by a method allowlist; content-reading methods are + * additionally subject to {@link Privacy} on user-managed browsers. + */ + @Data + public static class Cdp { + /** Master switch for action=cdp. */ + private boolean enabled = true; + + /** + * Allowed CDP methods. An entry is either an exact method + * ({@code "Page.navigate"}) or a domain wildcard ({@code "Input.*"}). + * Defaults to safe actuation methods; extend for advanced automation. + * Content-reading methods stay guarded by {@link Privacy} even if added. + */ + private java.util.List allowedMethods = new java.util.ArrayList<>(java.util.List.of( + "Input.*", + "Page.navigate", "Page.reload", "Page.bringToFront", + "Page.getNavigationHistory", "Page.navigateToHistoryEntry")); + } + + /** + * When the browser tool is attached to a user-managed Chrome (connected via + * CDP, process not spawned by us), that Chrome may have banking / email / + * internal-admin tabs open. This guard refuses content-reading actions + * (screenshot / eval / full snapshot) on pages that look sensitive, so + * private content is not funnelled into the model / persisted. It never + * affects headless or self-spawned browsers. + */ + @Data + public static class Privacy { + /** Master switch. When false, no sensitive-page blocking happens. */ + private boolean enabled = true; + + /** + * Extra hosts to always treat as sensitive (exact host or any subdomain), + * on top of the built-in heuristic. E.g. {@code intranet.corp.example}. + */ + private java.util.List sensitiveHosts = new java.util.ArrayList<>(); + + /** + * Hosts to always treat as safe (exact host or any subdomain). Overrides + * both the heuristic and {@link #sensitiveHosts}. Use to un-block a page + * the heuristic flagged that you know is fine to read. + */ + private java.util.List trustedHosts = new java.util.ArrayList<>(); + } } diff --git a/mateclaw-server/src/main/java/vip/mate/tool/browser/PageSnapshotScript.java b/mateclaw-server/src/main/java/vip/mate/tool/browser/PageSnapshotScript.java new file mode 100644 index 00000000..fbf871a1 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/browser/PageSnapshotScript.java @@ -0,0 +1,230 @@ +package vip.mate.tool.browser; + +import cn.hutool.json.JSONArray; +import cn.hutool.json.JSONObject; +import cn.hutool.json.JSONUtil; + +import java.util.ArrayList; +import java.util.List; + +/** + * Accessibility-tree page snapshot for browser automation. + * + *

Replaces the older visible-text dump with a compact accessibility tree in + * which every interactive element is tagged with a stable reference handle + * ({@code @e1}, {@code @e2}, ...). The reference is materialised as a + * {@code data-mate-ref} attribute on the live DOM node, so a follow-up + * click/type action can address the element by {@code [data-mate-ref='eN']} + * instead of forcing the model to guess a brittle CSS selector. + * + *

Why an injected attribute rather than a framework-native snapshot: the + * attribute approach is independent of the browser-driver version, survives + * driver upgrades, and produces a selector that plugs straight into the + * existing click/type plumbing. References stay valid only for the snapshot + * that produced them — a navigation wipes the attributes, so a stale reference + * naturally resolves to "not found" and the caller is told to re-snapshot. + */ +public final class PageSnapshotScript { + + private PageSnapshotScript() { + } + + /** + * Injected snapshot function. Runs as {@code root.evaluate(SNAPSHOT_JS, opts)}. + * Playwright's {@code ElementHandle.evaluate} invokes the function as + * {@code fn(element, arg)} — the scoped root element is the FIRST positional + * parameter (NOT {@code this}, which Playwright never binds to the element), + * and the caller's {@code opts} object is the second. {@code opts} is + * {@code {maxLen, includeNonInteractive}}. + * + *

Returns a JSON string {@code {tree, truncated, refs}} where: + *

+ */ + public static final String SNAPSHOT_JS = """ + (rootEl, opts) => { + const maxLen = opts.maxLen; + const includeNon = opts.includeNonInteractive; + const budget = { remaining: maxLen, truncated: false }; + let counter = 0; + const refs = []; + + // Wipe references from a prior snapshot so ids never collide + // across generations and a navigated-away page leaves nothing behind. + document.querySelectorAll('[data-mate-ref]').forEach(function (n) { + n.removeAttribute('data-mate-ref'); + }); + + function isVisible(el) { + const style = window.getComputedStyle(el); + if (style.display === 'none' || style.visibility === 'hidden' || style.opacity === '0') return false; + return el.offsetWidth > 0 || el.offsetHeight > 0 || el.getClientRects().length > 0; + } + + function isInteractive(el) { + const tag = el.tagName.toLowerCase(); + if (['a', 'button', 'input', 'select', 'textarea', 'summary'].includes(tag)) return true; + const role = el.getAttribute('role'); + if (role && ['button', 'link', 'checkbox', 'radio', 'tab', 'menuitem', + 'switch', 'textbox', 'combobox', 'option', 'searchbox', 'slider'].includes(role)) return true; + if (el.hasAttribute('onclick')) return true; + if (el.isContentEditable) return true; + const ti = el.getAttribute('tabindex'); + if (ti !== null && ti !== '-1') return true; + return false; + } + + function roleOf(el) { + const explicit = el.getAttribute('role'); + if (explicit) return explicit; + const tag = el.tagName.toLowerCase(); + switch (tag) { + case 'a': return el.hasAttribute('href') ? 'link' : 'generic'; + case 'button': return 'button'; + case 'select': return 'combobox'; + case 'textarea': return 'textbox'; + case 'summary': return 'button'; + case 'input': { + const t = (el.getAttribute('type') || 'text').toLowerCase(); + if (t === 'checkbox') return 'checkbox'; + if (t === 'radio') return 'radio'; + if (t === 'submit' || t === 'button' || t === 'reset') return 'button'; + if (t === 'search') return 'searchbox'; + if (t === 'hidden') return null; + return 'textbox'; + } + case 'h1': case 'h2': case 'h3': case 'h4': case 'h5': case 'h6': return 'heading'; + case 'li': return 'listitem'; + case 'ul': case 'ol': return 'list'; + case 'nav': return 'navigation'; + case 'img': return 'img'; + default: return null; + } + } + + function nameOf(el) { + const aria = el.getAttribute('aria-label'); + if (aria) return aria.trim(); + const labelledby = el.getAttribute('aria-labelledby'); + if (labelledby) { + const target = document.getElementById(labelledby); + if (target) return (target.textContent || '').trim(); + } + const tag = el.tagName.toLowerCase(); + if (tag === 'input' || tag === 'textarea') { + const ph = el.getAttribute('placeholder'); + if (ph) return ph.trim(); + if (el.value) return String(el.value).trim(); + if (el.id) { + const lab = document.querySelector('label[for="' + (window.CSS ? CSS.escape(el.id) : el.id) + '"]'); + if (lab) return (lab.textContent || '').trim(); + } + // Wrapping label: — common + // and has no for= link, so climb to the nearest label ancestor. + const wrap = el.closest('label'); + if (wrap) { + const wt = (wrap.textContent || '').trim().replace(/\\s+/g, ' '); + if (wt) return wt; + } + return ''; + } + if (tag === 'img') { + const alt = el.getAttribute('alt'); + if (alt) return alt.trim(); + } + const title = el.getAttribute('title'); + if (title) return title.trim(); + const txt = el.textContent ? el.textContent.trim().replace(/\\s+/g, ' ') : ''; + return txt; + } + + function clip(s, n) { + if (!s) return ''; + return s.length > n ? s.substring(0, n) + '…' : s; + } + + const lines = []; + + function emit(text) { + if (budget.remaining <= 0) { budget.truncated = true; return false; } + if (text.length + 1 > budget.remaining) { + budget.truncated = true; + budget.remaining = 0; + return false; + } + lines.push(text); + budget.remaining -= (text.length + 1); + return true; + } + + function walk(el, depth) { + if (depth > 20 || budget.remaining <= 0) return; + if (!isVisible(el)) return; + + const role = roleOf(el); + const interactive = isInteractive(el); + let line = null; + + if (interactive && role !== 'generic' && role !== null) { + counter += 1; + const ref = 'e' + counter; + el.setAttribute('data-mate-ref', ref); + refs.push(ref); + const nm = clip(nameOf(el), 100); + line = role + (nm ? ' "' + nm + '"' : '') + ' @' + ref; + } else if (includeNon && role && role !== 'generic') { + const nm = clip(nameOf(el), 100); + if (nm || role === 'list' || role === 'navigation') { + let extra = ''; + if (role === 'heading') { + const lvl = el.getAttribute('aria-level') + || (el.tagName.length === 2 ? el.tagName.charAt(1) : ''); + if (lvl) extra = ' [level=' + lvl + ']'; + } + line = role + (nm ? ' "' + nm + '"' : '') + extra; + } + } + + if (line !== null) { + if (!emit(' '.repeat(Math.min(depth, 10)) + '- ' + line)) return; + } + + const childDepth = line !== null ? depth + 1 : depth; + for (const child of el.children) { + if (budget.remaining <= 0) break; + walk(child, childDepth); + } + } + + walk(rootEl, 0); + return JSON.stringify({ tree: lines.join('\\n'), truncated: budget.truncated, refs: refs }); + } + """; + + /** Parsed result of a snapshot evaluation. */ + public record Result(String tree, boolean truncated, List refs) { + public static Result fromJson(String json) { + JSONObject obj = JSONUtil.parseObj(json); + String tree = obj.getStr("tree", ""); + boolean truncated = obj.getBool("truncated", false); + List refs = new ArrayList<>(); + JSONArray arr = obj.getJSONArray("refs"); + if (arr != null) { + for (Object o : arr) { + if (o != null) { + refs.add(o.toString()); + } + } + } + return new Result(tree, truncated, refs); + } + } + + /** Build the deterministic attribute selector for a reference id. */ + public static String selectorForRef(String ref) { + return "[data-mate-ref='" + ref + "']"; + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/tool/builtin/BrowserUseTool.java b/mateclaw-server/src/main/java/vip/mate/tool/builtin/BrowserUseTool.java index 6f80d092..3f03255d 100644 --- a/mateclaw-server/src/main/java/vip/mate/tool/builtin/BrowserUseTool.java +++ b/mateclaw-server/src/main/java/vip/mate/tool/builtin/BrowserUseTool.java @@ -4,8 +4,11 @@ import cn.hutool.http.HttpUtil; import cn.hutool.json.JSONArray; import cn.hutool.json.JSONObject; import cn.hutool.json.JSONUtil; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; import com.microsoft.playwright.Browser; import com.microsoft.playwright.BrowserContext; +import com.microsoft.playwright.CDPSession; import com.microsoft.playwright.ElementHandle; import com.microsoft.playwright.Page; import com.microsoft.playwright.Playwright; @@ -20,7 +23,9 @@ import org.springframework.lang.Nullable; import org.springframework.stereotype.Component; import vip.mate.tool.browser.BrowserDiagnosticsService; import vip.mate.tool.browser.BrowserLauncher; +import vip.mate.tool.browser.BrowserPrivacyGuard; import vip.mate.common.net.SsrfProperties; +import vip.mate.tool.browser.PageSnapshotScript; import vip.mate.tool.browser.UrlSafetyChecker; import java.net.Socket; @@ -44,14 +49,16 @@ public class BrowserUseTool { private static final int CDP_SCAN_PORT_MAX = 10000; /** - * Snapshot extractor — runs as an ElementHandle.evaluate so {@code this} - * is the scoped root (document.body when no selector is passed). Uses a - * budget object so truncation stops at element boundaries rather than - * mid-TEXT_NODE, and surfaces a {@code truncated:true} flag to the caller - * so the LLM can be told to retry with a narrower selector. + * Legacy visible-text extractor kept as a fallback: {@link #doSnapshot} + * prefers the accessibility-tree snapshot ({@link PageSnapshotScript}) and + * only falls back to this plain-text dump if the tree script throws on an + * unusual page. Runs as an ElementHandle.evaluate so {@code this} is the + * scoped root (document.body when no selector is passed). Uses a budget + * object so truncation stops at element boundaries rather than + * mid-TEXT_NODE, and surfaces a {@code truncated:true} flag. */ - private static final String SNAPSHOT_JS = """ - (maxLen) => { + private static final String TEXT_SNAPSHOT_JS_FALLBACK = """ + (rootEl, maxLen) => { const budget = { remaining: maxLen, truncated: false }; function getVisibleText(node, depth) { if (depth > 10 || budget.remaining <= 0) return ''; @@ -104,7 +111,7 @@ public class BrowserUseTool { } return results.join('\\n'); } - const text = getVisibleText(this, 0); + const text = getVisibleText(rootEl, 0); return JSON.stringify({ text: text, truncated: budget.truncated }); } """; @@ -114,15 +121,35 @@ public class BrowserUseTool { private final BrowserLauncher launcher; private final BrowserDiagnosticsService diagnostics; private final SsrfProperties ssrfProperties; + private final BrowserPrivacyGuard privacyGuard; public BrowserUseTool(vip.mate.channel.web.ChatStreamTracker streamTracker, BrowserLauncher launcher, BrowserDiagnosticsService diagnostics, - SsrfProperties ssrfProperties) { + SsrfProperties ssrfProperties, + BrowserPrivacyGuard privacyGuard) { this.streamTracker = streamTracker; this.launcher = launcher; this.diagnostics = diagnostics; this.ssrfProperties = ssrfProperties; + this.privacyGuard = privacyGuard; + } + + /** + * Enforce the privacy guard for a content-reading action. Returns an error + * JSON string to return to the caller when the action is refused on a + * sensitive page of a user-managed browser, or {@code null} to proceed. + */ + private String guardReadOrNull(BrowserSession session, String action) { + String reason = privacyGuard.blockReason(session.isUserManagedBrowser(), + session.page.url(), action); + if (reason == null) { + return null; + } + String conversationId = ToolExecutionContext.conversationId(currentToolContext); + privacyGuard.audit(conversationId, action, session.page.url(), reason); + log.info("[BrowserUse] Privacy guard blocked action={} on {}", action, session.page.url()); + return error(reason); } /** @@ -165,25 +192,34 @@ public class BrowserUseTool { - start: Launch a new browser (tries system Chrome, system Edge, then Playwright bundled). Optional headed=true. - stop: Close browser. If connected via CDP, only disconnects (Chrome keeps running). - open: Navigate to a URL. Requires url parameter. Auto-starts browser if not running. - - snapshot: Get page text content, interactive elements, and title. Optional `selector` - scopes to a subtree — USE IT when the page is large (big tables, long lists) to avoid - truncation. Without selector, content is capped and a `truncated:true` flag is returned - with a hint to retry using selector. + - snapshot: Get the page as an accessibility tree. Every interactive element (link, button, + input, ...) is tagged with a stable reference like `@e1`, `@e2`. Read the tree, then act on + an element by passing ref= to action=click/type — no CSS selector guessing needed. + References belong to the returned `generation`; re-snapshot if the page changes. Optional + `selector` scopes to a subtree — USE IT when the page is large to avoid truncation + (`truncated:true` is flagged with a hint). - screenshot: Take a screenshot. Optional path to save file; returns base64 if no path. - - click: Click an element. Requires selector (CSS selector). - - type: Type text into an element. Requires selector and text. + - click: Click an element. Pass ref= from a snapshot (preferred) or a CSS selector. + - type: Type text into an element. Pass ref= (preferred) or selector, plus text. + - hover: Hover over an element (reveals menus/tooltips). Pass ref= or selector. + - select: Choose an option in a dropdown. Pass ref= or selector, plus value. - eval: Execute JavaScript on the page. Requires code parameter. Top-level await is supported; use `return` to surface a value. + - cdp: Send a raw Chrome DevTools Protocol command. Requires method (e.g. 'Page.navigate'), optional params (JSON). Constrained by an allowlist; use only when the higher-level actions cannot express what you need. - connect_cdp: Connect to an existing Chrome via CDP. Requires url (e.g. "http://localhost:9222"). - list_cdp_targets: Scan local ports (9000-10000) for CDP endpoints. Optional cdpPort for single port. - navigate_back: Go back in browser history. - diagnose: Run a self-check — reports which launch strategies are available and what to install if none are. """) public String browser_use( - @ToolParam(description = "Action: start|stop|open|snapshot|screenshot|click|type|eval|connect_cdp|list_cdp_targets|navigate_back|diagnose") String action, + @ToolParam(description = "Action: start|stop|open|snapshot|screenshot|click|type|hover|select|eval|connect_cdp|list_cdp_targets|navigate_back|diagnose") String action, @ToolParam(description = "URL to navigate to (for open), or CDP base URL (for connect_cdp, e.g. http://localhost:9222)", required = false) String url, - @ToolParam(description = "CSS selector. REQUIRED for click/type. OPTIONAL for snapshot: pass to scope to a subtree when previous snapshot returned truncated:true.", required = false) String selector, + @ToolParam(description = "CSS selector. Alternative to ref for click/type/hover/select. OPTIONAL for snapshot: pass to scope to a subtree when previous snapshot returned truncated:true.", required = false) String selector, + @ToolParam(description = "Element reference from a snapshot (e.g. 'e4'). PREFERRED for click/type/hover/select — takes priority over selector. Re-snapshot if it reports stale.", required = false) String ref, @ToolParam(description = "Text to type (for action=type)", required = false) String text, + @ToolParam(description = "Option value or visible label to choose (for action=select)", required = false) String value, @ToolParam(description = "JavaScript code to execute (for action=eval). Top-level await is allowed; add `return` to return a value when the snippet uses await.", required = false) String code, + @ToolParam(description = "CDP method for action=cdp (e.g. 'Page.navigate', 'Input.dispatchMouseEvent'). Must be in the allowlist.", required = false) String method, + @ToolParam(description = "JSON object of params for action=cdp (e.g. {\"url\":\"https://example.com\"})", required = false) String params, @ToolParam(description = "File path to save screenshot (for action=screenshot)", required = false) String path, @ToolParam(description = "Launch visible browser window (for action=start, default false)", required = false) Boolean headed, @ToolParam(description = "Single CDP port to scan (for action=list_cdp_targets)", required = false) Integer cdpPort, @@ -199,8 +235,14 @@ public class BrowserUseTool { return error("action is required"); } - String sessionKey = "default"; - log.info("[BrowserUse] action={}, url={}, selector={}, headed={}, cdpPort={}", action, url, selector, headed, cdpPort); + // Isolate browser state per conversation so concurrent chats don't drive + // (and navigate) each other's page. Falls back to a shared key when no + // conversation context is present (e.g. internal/system invocations). + String conversationId = ToolExecutionContext.conversationId(ctx); + String sessionKey = (conversationId != null && !conversationId.isBlank()) + ? conversationId : "default"; + log.info("[BrowserUse] action={}, session={}, url={}, selector={}, headed={}, cdpPort={}", + action, sessionKey, url, selector, headed, cdpPort); try { return switch (action.toLowerCase().trim()) { @@ -209,14 +251,17 @@ public class BrowserUseTool { case "open" -> doOpen(sessionKey, url); case "snapshot" -> doSnapshot(sessionKey, selector); case "screenshot" -> doScreenshot(sessionKey, path); - case "click" -> doClick(sessionKey, selector); - case "type" -> doType(sessionKey, selector, text); + case "click" -> doClick(sessionKey, ref, selector); + case "type" -> doType(sessionKey, ref, selector, text); + case "hover" -> doHover(sessionKey, ref, selector); + case "select" -> doSelect(sessionKey, ref, selector, value); case "eval" -> doEval(sessionKey, code); + case "cdp" -> doCdp(sessionKey, method, params); case "connect_cdp" -> doConnectCdp(sessionKey, url); case "list_cdp_targets" -> doListCdpTargets(cdpPort); case "navigate_back" -> doNavigateBack(sessionKey); case "diagnose" -> doDiagnose(); - default -> error("Unknown action: " + action + ". Supported: start, stop, open, snapshot, screenshot, click, type, eval, connect_cdp, list_cdp_targets, navigate_back, diagnose"); + default -> error("Unknown action: " + action + ". Supported: start, stop, open, snapshot, screenshot, click, type, hover, select, eval, cdp, connect_cdp, list_cdp_targets, navigate_back, diagnose"); }; } catch (PlaywrightException e) { log.error("[BrowserUse] Playwright error: {}", e.getMessage()); @@ -520,6 +565,7 @@ public class BrowserUseTool { } session.touch(); + session.invalidateRefs(); Page page = session.page; page.navigate(normalizedUrl); @@ -546,6 +592,7 @@ public class BrowserUseTool { } session.touch(); + session.invalidateRefs(); session.page.goBack(); String title = session.page.title(); @@ -567,6 +614,11 @@ public class BrowserUseTool { return error("No browser running. Use action=start first."); } + String blocked = guardReadOrNull(session, "snapshot"); + if (blocked != null) { + return blocked; + } + session.touch(); Page page = session.page; @@ -590,31 +642,70 @@ public class BrowserUseTool { } int maxLen = launcher.properties().getSnapshotMaxLength(); - String jsResult = (String) root.evaluate(SNAPSHOT_JS, maxLen); - - // JS returns { text: "...", truncated: true/false } - JSONObject parsed = JSONUtil.parseObj(jsResult); - String textContent = parsed.getStr("text"); - boolean truncated = parsed.getBool("truncated", false); + boolean includeNon = launcher.properties().isSnapshotIncludeNonInteractive(); JSONObject result = new JSONObject(); result.set("ok", true); result.set("title", title); result.set("url", url); - // IMPORTANT: truncated + hint MUST come before content. The framework's - // spill-preview keeps only the head ~800 chars of the JSON, so placing - // these flags first ensures the LLM still sees them after a spill. - result.set("truncated", truncated); - if (truncated) { - result.set("hint", "Content truncated. Re-call browser_use with action=snapshot" - + " and selector= to scope to a subtree (e.g. selector='#main'," - + " selector='table tbody tr')."); + + try { + // Accessibility-tree snapshot: assigns stable @eN references to + // interactive elements (materialised as data-mate-ref attributes) + // so click/type can address them precisely instead of guessing a + // CSS selector. Bumps the session generation so a later action on a + // stale reference (page changed / navigated) resolves to not-found. + JSONObject opts = new JSONObject(); + opts.set("maxLen", maxLen); + opts.set("includeNonInteractive", includeNon); + String jsResult = (String) root.evaluate(PageSnapshotScript.SNAPSHOT_JS, opts); + PageSnapshotScript.Result snap = PageSnapshotScript.Result.fromJson(jsResult); + + int generation = session.nextSnapshotGeneration(snap.refs()); + + result.set("snapshotMode", "accessibility-tree"); + result.set("generation", generation); + // IMPORTANT: flags/hints MUST precede the (large) tree. The framework's + // spill-preview keeps only the head of the JSON, so ordering these + // first ensures the LLM still sees them after a spill. + result.set("truncated", snap.truncated()); + result.set("hint", "Interactive elements are tagged @eN. To act on one, call" + + " action=click or action=type with ref= (e.g. ref='e4') — no CSS" + + " selector needed. References are valid only for generation " + + generation + "; re-snapshot if the page changes." + + (snap.truncated() ? " Content truncated — pass selector= to scope" + + " to a subtree (e.g. selector='#main')." : "")); + if (selector != null && !selector.isBlank()) { + result.set("scopedTo", selector); + } + result.set("refCount", snap.refs().size()); + result.set("content", snap.tree()); + return JSONUtil.toJsonPrettyStr(result); + } catch (PlaywrightException e) { + // Rare pages break the tree walk (exotic custom elements, CSP on + // attribute writes). Fall back to the plain visible-text dump so the + // model still gets *something* readable, flagged so it knows refs + // are unavailable and it must use CSS selectors for this page. + log.warn("[BrowserUse] Accessibility snapshot failed, falling back to text dump: {}", e.getMessage()); + // The tree script wipes data-mate-ref attributes before it walks, so a + // mid-walk failure leaves the DOM with no refs. Drop currentRefs too, + // otherwise a later ref action would pass the stale-check but resolve + // to nothing (bare timeout) instead of a clean re-snapshot prompt. + session.invalidateRefs(); + String jsResult = (String) root.evaluate(TEXT_SNAPSHOT_JS_FALLBACK, maxLen); + JSONObject parsed = JSONUtil.parseObj(jsResult); + boolean truncated = parsed.getBool("truncated", false); + result.set("snapshotMode", "text-fallback"); + result.set("truncated", truncated); + result.set("hint", "Accessibility tree unavailable on this page — no @eN refs." + + " Use action=click/type with selector=." + + (truncated ? " Content truncated — pass selector= to scope." : "")); + if (selector != null && !selector.isBlank()) { + result.set("scopedTo", selector); + } + result.set("content", parsed.getStr("text")); + return JSONUtil.toJsonPrettyStr(result); } - if (selector != null && !selector.isBlank()) { - result.set("scopedTo", selector); - } - result.set("content", textContent); - return JSONUtil.toJsonPrettyStr(result); } private String doScreenshot(String sessionKey, String path) { @@ -623,6 +714,11 @@ public class BrowserUseTool { return error("No browser running. Use action=start first."); } + String blocked = guardReadOrNull(session, "screenshot"); + if (blocked != null) { + return blocked; + } + session.touch(); Page page = session.page; @@ -654,63 +750,159 @@ public class BrowserUseTool { } } - private String doClick(String sessionKey, String selector) { - if (selector == null || selector.isBlank()) { - return error("selector is required for action=click"); + /** Result of resolving a click/type/hover/select target: a selector, or an error to return. */ + private record TargetResolution(String selector, String label, String error) { + static TargetResolution ok(String selector, String label) { + return new TargetResolution(selector, label, null); } + static TargetResolution fail(String error) { + return new TargetResolution(null, null, error); + } + } + /** + * Resolve an action target to a CSS selector. A snapshot {@code ref} takes + * priority over an explicit CSS {@code selector}; a ref that is not part of + * the current snapshot is reported as stale so the caller re-snapshots + * instead of getting a bare element-not-found timeout. + */ + private TargetResolution resolveTarget(BrowserSession session, String ref, String selector) { + if (ref != null && !ref.isBlank()) { + String r = ref.trim(); + if (!session.currentRefs.contains(r)) { + return TargetResolution.fail("ref '" + r + "' is not part of the current snapshot" + + " (generation " + session.snapshotGeneration + "). The page likely changed," + + " or you have not snapshotted since it did. Call action=snapshot first," + + " then use a ref from that fresh result."); + } + return TargetResolution.ok(PageSnapshotScript.selectorForRef(r), r); + } + if (selector != null && !selector.isBlank()) { + return TargetResolution.ok(selector, selector); + } + return TargetResolution.fail("Either ref (from a snapshot, e.g. ref='e4') or a CSS selector is required."); + } + + private String doClick(String sessionKey, String ref, String selector) { BrowserSession session = requireSession(sessionKey); if (session == null) { return error("No browser running. Use action=start first."); } + TargetResolution t = resolveTarget(session, ref, selector); + if (t.error() != null) { + return error(t.error()); + } session.touch(); Page page = session.page; - page.click(selector); + String before = page.url(); + page.click(t.selector()); page.waitForLoadState(LoadState.DOMCONTENTLOADED); String title = page.title(); String url = page.url(); + // A navigation invalidates the snapshot references; a same-page click + // (toggle, expand) keeps them so the model can act on more refs. + if (!url.equals(before)) { + session.invalidateRefs(); + } - log.info("[BrowserUse] Clicked: {} (page now: {})", selector, url); + log.info("[BrowserUse] Clicked: {} (page now: {})", t.label(), url); broadcastBrowserEvent("click", true, url, title, null, 0); JSONObject result = new JSONObject(); result.set("ok", true); - result.set("selector", selector); + result.set("target", t.label()); result.set("currentUrl", url); result.set("currentTitle", title); - result.set("message", "Clicked element: " + selector); + if (!url.equals(before)) { + result.set("navigated", true); + result.set("hint", "The page navigated — previous @eN refs are stale. Re-snapshot before acting."); + } + result.set("message", "Clicked element: " + t.label()); return JSONUtil.toJsonPrettyStr(result); } - private String doType(String sessionKey, String selector, String text) { - if (selector == null || selector.isBlank()) { - return error("selector is required for action=type"); - } + private String doType(String sessionKey, String ref, String selector, String text) { if (text == null) { return error("text is required for action=type"); } - BrowserSession session = requireSession(sessionKey); if (session == null) { return error("No browser running. Use action=start first."); } + TargetResolution t = resolveTarget(session, ref, selector); + if (t.error() != null) { + return error(t.error()); + } session.touch(); - Page page = session.page; + session.page.fill(t.selector(), text); - page.fill(selector, text); - - log.info("[BrowserUse] Typed into: {} ({} chars)", selector, text.length()); + log.info("[BrowserUse] Typed into: {} ({} chars)", t.label(), text.length()); broadcastBrowserEvent("type", true, null, null, null, 0); JSONObject result = new JSONObject(); result.set("ok", true); - result.set("selector", selector); + result.set("target", t.label()); result.set("textLength", text.length()); - result.set("message", "Typed " + text.length() + " characters into " + selector); + result.set("message", "Typed " + text.length() + " characters into " + t.label()); + return JSONUtil.toJsonPrettyStr(result); + } + + private String doHover(String sessionKey, String ref, String selector) { + BrowserSession session = requireSession(sessionKey); + if (session == null) { + return error("No browser running. Use action=start first."); + } + TargetResolution t = resolveTarget(session, ref, selector); + if (t.error() != null) { + return error(t.error()); + } + + session.touch(); + session.page.hover(t.selector()); + + log.info("[BrowserUse] Hovered: {}", t.label()); + broadcastBrowserEvent("hover", true, null, null, null, 0); + + JSONObject result = new JSONObject(); + result.set("ok", true); + result.set("target", t.label()); + result.set("message", "Hovered element: " + t.label() + + ". Re-snapshot to capture any menu/tooltip it revealed."); + return JSONUtil.toJsonPrettyStr(result); + } + + private String doSelect(String sessionKey, String ref, String selector, String value) { + if (value == null || value.isBlank()) { + return error("value is required for action=select"); + } + BrowserSession session = requireSession(sessionKey); + if (session == null) { + return error("No browser running. Use action=start first."); + } + TargetResolution t = resolveTarget(session, ref, selector); + if (t.error() != null) { + return error(t.error()); + } + + session.touch(); + // Playwright matches by option value, label, or visible text, so a + // human-readable value from the model works without extra hints. + List chosen = session.page.selectOption(t.selector(), value); + + log.info("[BrowserUse] Selected {} in {} -> {}", value, t.label(), chosen); + broadcastBrowserEvent("select", true, null, null, null, 0); + + JSONObject result = new JSONObject(); + result.set("ok", true); + result.set("target", t.label()); + result.set("selected", chosen); + result.set("message", chosen.isEmpty() + ? "No option matched '" + value + "'. Re-snapshot and check the option labels." + : "Selected '" + value + "' in " + t.label()); return JSONUtil.toJsonPrettyStr(result); } @@ -737,6 +929,11 @@ public class BrowserUseTool { return error("No browser running. Use action=start first."); } + String blocked = guardReadOrNull(session, "eval"); + if (blocked != null) { + return blocked; + } + session.touch(); Page page = session.page; @@ -777,6 +974,110 @@ public class BrowserUseTool { return JSONUtil.toJsonPrettyStr(result); } + /** + * CDP methods that read page/network/storage content. Even when allowlisted, + * these are refused by the privacy guard on a sensitive page of a + * user-managed browser. Entries ending in {@code .} match a whole domain. + */ + private static final List CDP_CONTENT_READ = List.of( + "Network.getResponseBody", "Network.getResponseBodyForInterception", + "Network.getRequestPostData", "Network.getAllCookies", "Network.getCookies", + "Storage.", "DOMStorage.", "IndexedDB.", "CacheStorage.", + "Page.captureScreenshot", "Page.captureSnapshot", "Page.printToPDF", + "Page.getResourceContent", "Page.getResourceTree", + "DOM.getOuterHTML", "DOM.getDocument", "Runtime.evaluate", "Runtime.getProperties"); + + private boolean isCdpMethodAllowed(String method) { + for (String entry : launcher.properties().getCdp().getAllowedMethods()) { + if (entry == null || entry.isBlank()) { + continue; + } + String e = entry.trim(); + if (e.endsWith(".*")) { + if (method.startsWith(e.substring(0, e.length() - 1))) { // "Input." prefix + return true; + } + } else if (e.equals(method)) { + return true; + } + } + return false; + } + + private static boolean isCdpContentReading(String method) { + for (String p : CDP_CONTENT_READ) { + if (p.endsWith(".") ? method.startsWith(p) : method.equals(p)) { + return true; + } + } + return false; + } + + private String doCdp(String sessionKey, String method, String paramsJson) { + if (!launcher.properties().getCdp().isEnabled()) { + return error("action=cdp is disabled (mateclaw.browser.cdp.enabled=false)."); + } + if (method == null || method.isBlank()) { + return error("method is required for action=cdp (e.g. 'Page.navigate')."); + } + BrowserSession session = requireSession(sessionKey); + if (session == null) { + return error("No browser running. Use action=start first."); + } + String m = method.trim(); + if (!isCdpMethodAllowed(m)) { + return error("CDP method '" + m + "' is not allowed. Allowlist: " + + launcher.properties().getCdp().getAllowedMethods() + + ". Add it to mateclaw.browser.cdp.allowed-methods if you trust it."); + } + if (isCdpContentReading(m)) { + String reason = privacyGuard.blockReason(session.isUserManagedBrowser(), + session.page.url(), "cdp:" + m); + if (reason != null) { + privacyGuard.audit(ToolExecutionContext.conversationId(currentToolContext), + "cdp:" + m, session.page.url(), reason); + return error(reason); + } + } + + JsonObject parsed = null; + if (paramsJson != null && !paramsJson.isBlank()) { + try { + parsed = JsonParser.parseString(paramsJson).getAsJsonObject(); + } catch (RuntimeException e) { + return error("params must be a JSON object: " + e.getMessage()); + } + } + + session.touch(); + // A fresh CDP session per call keeps the escape hatch stateless and avoids + // leaking listeners; navigation-triggering methods invalidate references. + CDPSession cdp = session.context.newCDPSession(session.page); + try { + JsonObject res = parsed != null ? cdp.send(m, parsed) : cdp.send(m); + if (m.startsWith("Page.navigate") || m.equals("Page.navigateToHistoryEntry")) { + session.invalidateRefs(); + } + String out = res != null ? res.toString() : "{}"; + if (out.length() > 10_000) { + out = out.substring(0, 10_000) + "\n... [truncated]"; + } + log.info("[BrowserUse] CDP {} -> {} chars", m, out.length()); + broadcastBrowserEvent("cdp", true, session.page.url(), null, null, 0); + JSONObject result = new JSONObject(); + result.set("ok", true); + result.set("method", m); + result.set("result", out); + return JSONUtil.toJsonPrettyStr(result); + } finally { + try { + cdp.detach(); + } catch (Exception ignored) { + // best-effort; the session is discarded either way + } + } + } + // ==================== CDP Helpers ==================== private boolean isPortOpen(int port) { @@ -917,6 +1218,35 @@ public class BrowserUseTool { /** 空闲看门狗定时任务（stop 时取消，避免泄漏） */ volatile ScheduledFuture idleWatchdog; + /** + * Monotonic snapshot generation. Each {@code action=snapshot} bumps it and + * replaces {@link #currentRefs} with the references assigned that pass. + * A click/type by a reference not in {@link #currentRefs} is reported as + * stale, prompting the caller to re-snapshot. Safe as plain volatile + * because tool calls are serialized per executor. + */ + volatile int snapshotGeneration; + volatile java.util.Set currentRefs = java.util.Set.of(); + + int nextSnapshotGeneration(java.util.List refs) { + this.currentRefs = java.util.Set.copyOf(refs); + return ++this.snapshotGeneration; + } + + /** Drop all references — the DOM they pointed at is gone (navigation). */ + void invalidateRefs() { + this.currentRefs = java.util.Set.of(); + } + + /** + * True when this session is attached to a Chrome the user runs themselves: + * connected over CDP and NOT spawned by us. Only these carry the user's + * live logins, so the privacy guard applies only here. + */ + boolean isUserManagedBrowser() { + return connectedViaCdp && ownedProcess == null; + } + BrowserSession(Browser browser, BrowserContext context, Page page, boolean headed, boolean connectedViaCdp, String cdpUrl, java.nio.file.Path userDataDir, Process ownedProcess) { diff --git a/mateclaw-server/src/test/java/vip/mate/tool/browser/BrowserPrivacyGuardTest.java b/mateclaw-server/src/test/java/vip/mate/tool/browser/BrowserPrivacyGuardTest.java new file mode 100644 index 00000000..65a31c04 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/tool/browser/BrowserPrivacyGuardTest.java @@ -0,0 +1,86 @@ +package vip.mate.tool.browser; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Unit tests for {@link BrowserPrivacyGuard} classification and blocking. + * The audit mapper is never touched by {@code isSensitive}/{@code blockReason}, + * so a null mapper is fine here. + */ +class BrowserPrivacyGuardTest { + + private BrowserPrivacyGuard guardWith(List sensitive, List trusted) { + BrowserProperties props = new BrowserProperties(); + props.getPrivacy().setSensitiveHosts(sensitive); + props.getPrivacy().setTrustedHosts(trusted); + return new BrowserPrivacyGuard(props, null); + } + + @Test + @DisplayName("Heuristic flags banking / webmail / login / admin pages") + void heuristicFlagsSensitivePages() { + BrowserPrivacyGuard g = guardWith(List.of(), List.of()); + assertTrue(g.isSensitive("https://www.chase.com/banking")); + assertTrue(g.isSensitive("https://mail.google.com/mail/u/0")); + assertTrue(g.isSensitive("https://github.com/login")); + assertTrue(g.isSensitive("https://example.com/account/settings")); + assertTrue(g.isSensitive("https://admin.example.com/")); + } + + @Test + @DisplayName("Ordinary content pages are not flagged") + void ordinaryPagesNotFlagged() { + BrowserPrivacyGuard g = guardWith(List.of(), List.of()); + assertFalse(g.isSensitive("https://example.com/products/42")); + assertFalse(g.isSensitive("https://news.example.com/article/hello-world")); + assertFalse(g.isSensitive("")); + assertFalse(g.isSensitive(null)); + } + + @Test + @DisplayName("trustedHosts overrides both the heuristic and sensitiveHosts") + void trustedOverridesEverything() { + BrowserPrivacyGuard g = guardWith(List.of("internal.corp"), List.of("mail.google.com")); + assertFalse(g.isSensitive("https://mail.google.com/mail")); + // subdomain of a trusted host is also trusted + BrowserPrivacyGuard g2 = guardWith(List.of(), List.of("example.com")); + assertFalse(g2.isSensitive("https://admin.example.com/login")); + } + + @Test + @DisplayName("sensitiveHosts adds hosts the heuristic would miss") + void sensitiveHostsExtendHeuristic() { + BrowserPrivacyGuard g = guardWith(List.of("intranet.corp"), List.of()); + assertTrue(g.isSensitive("https://intranet.corp/dashboard")); + assertTrue(g.isSensitive("https://hr.intranet.corp/")); + } + + @Test + @DisplayName("blockReason only fires for a user-managed browser on a sensitive page") + void blockReasonScope() { + BrowserPrivacyGuard g = guardWith(List.of(), List.of()); + // sensitive page but self-spawned browser → allowed + assertNull(g.blockReason(false, "https://github.com/login", "screenshot")); + // user-managed browser but ordinary page → allowed + assertNull(g.blockReason(true, "https://example.com/products", "eval")); + // user-managed browser on a sensitive page → blocked + assertNotNull(g.blockReason(true, "https://github.com/login", "eval")); + } + + @Test + @DisplayName("Disabling the guard allows everything") + void disabledGuardAllows() { + BrowserProperties props = new BrowserProperties(); + props.getPrivacy().setEnabled(false); + BrowserPrivacyGuard g = new BrowserPrivacyGuard(props, null); + assertNull(g.blockReason(true, "https://www.chase.com/banking", "screenshot")); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/tool/browser/PageSnapshotProbe.java b/mateclaw-server/src/test/java/vip/mate/tool/browser/PageSnapshotProbe.java new file mode 100644 index 00000000..4fed6a64 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/tool/browser/PageSnapshotProbe.java @@ -0,0 +1,116 @@ +package vip.mate.tool.browser; + +import cn.hutool.json.JSONObject; +import com.microsoft.playwright.Browser; +import com.microsoft.playwright.ElementHandle; +import com.microsoft.playwright.Page; +import com.microsoft.playwright.Playwright; +import vip.mate.common.net.SsrfProperties; + +import java.util.List; + +/** + * Manual end-to-end probe for {@link PageSnapshotScript}. Not a JUnit test — run via + * {@code mvn -q test-compile exec:java -Dexec.mainClass=vip.mate.tool.browser.PageSnapshotProbe + * -Dexec.classpathScope=test} (use {@code test-compile}, not {@code compile}, so this + * test-scoped class is built). + * + *

Launches a real browser and drives the EXACT path the tool uses — + * {@code page.querySelector("body").evaluate(PageSnapshotScript.SNAPSHOT_JS, opts)} — + * against a known page. It exists to lock the most subtle failure mode found in + * review: Playwright's {@code ElementHandle.evaluate} passes the element as the + * FIRST POSITIONAL ARGUMENT (never as {@code this}). If the snapshot function is + * ever reverted to read {@code this}, the walk starts from the wrong root and the + * tree comes back empty (or throws) — this probe then fails loudly instead of the + * regression shipping silently, the way the original {@code this}-based version did. + * + *

Asserts, on a fixed HTML page: + *

    + *
  • references are assigned (non-empty) and materialised as {@code data-mate-ref};
    • + *
  • interactive elements appear with the right role + accessible name + * (incl. wrapping-label resolution and the {@code display:none} filter);
    • + *
  • {@code [data-mate-ref='e1']} resolves back to the expected element.
    • + *
+ * Exits non-zero on any failed assertion. + */ +public final class PageSnapshotProbe { + + private static final String HTML = """ + +

Probe Form

+ + Learn more + + +
+ + """; + + private static int failures = 0; + + public static void main(String[] args) { + System.out.println("=== PageSnapshotScript probe ==="); + BrowserProperties props = new BrowserProperties(); + BrowserLauncher launcher = new BrowserLauncher(props, new SsrfProperties()); + + int exit = 0; + try (Playwright pw = Playwright.create()) { + BrowserLauncher.Result r = launcher.launch(pw, false); + if (!r.isSuccess()) { + System.err.println("FAIL: could not launch a browser: " + r.getFailureSummary()); + System.exit(1); + } + try (Browser browser = r.getBrowser()) { + Page page = r.getPage(); + page.setContent(HTML); + + // EXACT tool path: element handle + Hutool JSONObject opts. + ElementHandle root = page.querySelector("body"); + JSONObject opts = new JSONObject(); + opts.set("maxLen", 20_000); + opts.set("includeNonInteractive", true); + String json = (String) root.evaluate(PageSnapshotScript.SNAPSHOT_JS, opts); + PageSnapshotScript.Result snap = PageSnapshotScript.Result.fromJson(json); + + System.out.println("tree:\n" + snap.tree()); + System.out.println("refs: " + snap.refs()); + + List refs = snap.refs(); + String tree = snap.tree(); + + // The regression guard: the this-vs-first-arg bug makes this empty. + check(!refs.isEmpty(), "references assigned (element passed as first arg, not `this`)"); + check(refs.size() >= 4, "at least 4 interactive refs (input, link, select, button), got " + refs.size()); + + check(tree.contains("textbox \"Customer name:\""), "input resolves wrapping-label name"); + check(tree.contains("link \"Learn more\""), "anchor rendered as link with text"); + check(tree.contains("combobox"), "select rendered as combobox"); + check(tree.contains("button \"Submit\""), "button rendered with text"); + check(tree.contains("heading \"Probe Form\" [level=1]"), "h1 rendered as heading level 1"); + check(!tree.contains("Hidden Button"), "display:none subtree is filtered out"); + + boolean e1Resolves = page.querySelector("[data-mate-ref='e1']") != null; + check(e1Resolves, "[data-mate-ref='e1'] resolves back to a live element"); + + if (failures == 0) { + System.out.println("\nALL CHECKS PASSED (" + refs.size() + " refs) via " + r.getStrategy()); + } else { + System.err.println("\n" + failures + " CHECK(S) FAILED"); + exit = 1; + } + } + } catch (Exception e) { + System.err.println("FAIL: probe threw: " + e); + e.printStackTrace(); + exit = 1; + } + System.exit(exit); + } + + private static void check(boolean ok, String what) { + System.out.println((ok ? " [PASS] " : " [FAIL] ") + what); + if (!ok) { + failures++; + } + } +}