feat(browser): harden browser automation reliability

This commit is contained in:
matevip 2026-08-12 06:02:36 -04:00
parent 0ae8552f5a
commit ba0e4506b7
19 changed files with 1086 additions and 85 deletions

View File

@ -78,7 +78,7 @@ RUN mvn -pl mateclaw-server -am package -Dmaven.test.skip=true -q ${MAVEN_FLAGS}
# bump the Java dependency, bump this tag in lockstep — Microsoft rebuilds each
# tag with the matching driver, so mismatched versions cause the java driver to
# re-download browsers at runtime (defeating the whole point of this image).
FROM mcr.microsoft.com/playwright:v1.59.0-noble
FROM mcr.microsoft.com/playwright:v1.62.0-noble
WORKDIR /app
# JDK 21 is NOT part of the base image (it ships Node for the JS driver).

View File

@ -1776,7 +1776,7 @@ public class AgentGraphBuilder {
## ProgressLedger Discipline (mandatory)
The `## 当前任务进度` block injected near the top of every turn is the **authoritative record** of what you have done and what remains. Treat it as ground truth, not as a scratchpad you may ignore.
- **On starting any multi-step task** (3 tool calls expected), call `progress_update` in a parallel tool_calls batch to register every pending step BEFORE doing the work. Do not wait until "later" context compression can trim earlier turns and you will lose track.
- **On starting any multi-step task** (3 tool calls expected), call `progress_update` in parallel batches of at most 16 calls to register every pending step BEFORE doing the work. Split larger ledgers across turns so the executor cap never drops entries.
- **After each completed sub-step**, immediately call `progress_update` to flip its status to `done`. "Immediately" means in the same tool_calls batch that returns the result, not after the next reasoning turn.
- **Never re-execute a step the ledger shows as `done`** unless you can articulate why the prior result is stale.
- **🔒 固定约束 entries** (pinned from skill manifests) are non-negotiable. They survive context compression for a reason re-read them every turn and make sure your planned action still satisfies them.

View File

@ -257,8 +257,8 @@ public class ReasoningNode implements NodeAction {
+ "\"调研 10 个模型\"\"逐节起草报告\"\"批量生成 N 份文档\"\n"
+ "\"依次调用 N 个 API\"\"对每个文件执行同一操作\"等。\n\n"
+ "**必须做的事**\n"
+ "1. **第一轮回复就用并行 tool_calls 批量注册全部子目标为 `pending`**\n"
+ " 一条回复里 N 个 `progress_update` 同时发出(不要串行)\n"
+ "1. **第一轮回复就用并行 tool_calls 批量注册子目标为 `pending`**\n"
+ " 每批最多 16 个 `progress_update`;超过 16 个时分批注册,避免超过执行器上限\n"
+ " 例:要调研 10 个模型,第一轮就发 10 个 `progress_update(stepKey=\"model_xxx\", status=\"pending\")`。\n"
+ "2. **每开始一个子目标**前发 `progress_update(同 stepKey, status=\"in_progress\")`。\n"
+ "3. **每完成一个子目标**后立即发 `progress_update(同 stepKey, status=\"done\")`。\n"

View File

@ -0,0 +1,49 @@
package vip.mate.tool.browser;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import java.util.Collection;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
* Shared URL safety checks for browser navigation surfaces beyond action=open.
*/
public final class BrowserNavigationGuard {
private static final Pattern URL_LITERAL =
Pattern.compile("['\"](https?://[^'\"\\s)]+)['\"]", Pattern.CASE_INSENSITIVE);
private static final Pattern EVAL_NAVIGATION_INTENT = Pattern.compile(
"\\b(location(?:\\.href|\\.assign|\\.replace)?|window\\.open|fetch|XMLHttpRequest)\\b",
Pattern.CASE_INSENSITIVE);
private BrowserNavigationGuard() {
}
public static void checkCdp(String method, JsonObject params, Collection<String> allowlist,
boolean allowPrivateNetwork) {
if (!"Page.navigate".equals(method) || params == null || !params.has("url")) {
return;
}
JsonElement el = params.get("url");
if (el == null || !el.isJsonPrimitive()) {
return;
}
UrlSafetyChecker.check(el.getAsString(), allowlist, allowPrivateNetwork);
}
public static void checkEval(String code, Collection<String> allowlist, boolean allowPrivateNetwork) {
if (code == null || code.isBlank()) {
return;
}
if (!EVAL_NAVIGATION_INTENT.matcher(code).find()) {
return;
}
Matcher matcher = URL_LITERAL.matcher(code);
while (matcher.find()) {
UrlSafetyChecker.check(matcher.group(1), allowlist, allowPrivateNetwork);
}
}
}

View File

@ -0,0 +1,108 @@
package vip.mate.tool.browser;
import java.util.List;
import java.util.Map;
import java.util.Set;
/**
* Thread-safe lifecycle state for snapshot references in one browser session.
*/
public final class BrowserRefState {
public enum Status {
NONE,
VALID,
INVALIDATED
}
private int snapshotGeneration;
private long navigationEpoch;
private long snapshotNavigationEpoch = -1;
private String currentUrl = "";
private String snapshotUrl = "";
private Status status = Status.NONE;
private Set<String> refs = Set.of();
private Map<String, PageSnapshotScript.RefFingerprint> refInfos = Map.of();
public synchronized int recordSnapshot(
String url,
List<String> newRefs,
Map<String, PageSnapshotScript.RefFingerprint> newRefInfos) {
currentUrl = normalizeUrl(url);
snapshotUrl = currentUrl;
snapshotNavigationEpoch = navigationEpoch;
refs = Set.copyOf(newRefs);
refInfos = Map.copyOf(newRefInfos);
status = Status.VALID;
return ++snapshotGeneration;
}
public synchronized void onMainFrameNavigated(String url) {
currentUrl = normalizeUrl(url);
navigationEpoch++;
invalidateSnapshot();
}
public synchronized void reconcileUrl(String url) {
String normalized = normalizeUrl(url);
if (!currentUrl.isEmpty() && !currentUrl.equals(normalized)) {
onMainFrameNavigated(normalized);
return;
}
currentUrl = normalized;
}
public synchronized void invalidate() {
invalidateSnapshot();
}
private void invalidateSnapshot() {
refs = Set.of();
refInfos = Map.of();
status = snapshotGeneration == 0 ? Status.NONE : Status.INVALIDATED;
}
private static String normalizeUrl(String url) {
return url == null ? "" : url;
}
public synchronized Status status() {
return status;
}
public synchronized boolean refsValid() {
return status == Status.VALID && snapshotNavigationEpoch == navigationEpoch;
}
public synchronized boolean contains(String ref) {
return refs.contains(ref);
}
public synchronized PageSnapshotScript.RefFingerprint fingerprint(String ref) {
return refInfos.get(ref);
}
public synchronized int refCount() {
return refs.size();
}
public synchronized int snapshotGeneration() {
return snapshotGeneration;
}
public synchronized long navigationEpoch() {
return navigationEpoch;
}
public synchronized long snapshotNavigationEpoch() {
return snapshotNavigationEpoch;
}
public synchronized String currentUrl() {
return currentUrl;
}
public synchronized String snapshotUrl() {
return snapshotUrl;
}
}

View File

@ -0,0 +1,34 @@
package vip.mate.tool.browser;
import java.util.concurrent.locks.ReentrantLock;
/**
* Fixed-size striped lock gate for serializing operations on one browser session.
*/
public final class BrowserSessionGate {
private final ReentrantLock[] locks;
public BrowserSessionGate(int stripes) {
if (stripes <= 0) {
throw new IllegalArgumentException("stripes must be positive");
}
locks = new ReentrantLock[stripes];
for (int i = 0; i < stripes; i++) {
locks[i] = new ReentrantLock(true);
}
}
public Lease enter(String sessionKey) {
int index = Math.floorMod(sessionKey.hashCode(), locks.length);
ReentrantLock lock = locks[index];
lock.lock();
return lock::unlock;
}
@FunctionalInterface
public interface Lease extends AutoCloseable {
@Override
void close();
}
}

View File

@ -0,0 +1,52 @@
package vip.mate.tool.browser;
/**
* Parsed, bounded wait request for browser_use action=wait_for.
*/
public record BrowserWaitCondition(Kind kind, String target, int timeoutMillis) {
public enum Kind {
SELECTOR,
TEXT,
URL,
LOAD_STATE
}
public static BrowserWaitCondition parse(String condition, String selector, String text, String value,
Integer timeoutSeconds, int maxTimeoutSeconds) {
if (condition == null || condition.isBlank()) {
throw new IllegalArgumentException("condition is required for action=wait_for");
}
Kind kind = switch (condition.trim().toLowerCase()) {
case "selector" -> Kind.SELECTOR;
case "text" -> Kind.TEXT;
case "url" -> Kind.URL;
case "load_state", "loadstate", "state" -> Kind.LOAD_STATE;
default -> throw new IllegalArgumentException("Unknown wait_for condition: " + condition
+ ". Supported: selector, text, url, load_state");
};
String target = switch (kind) {
case SELECTOR -> firstNonBlank(selector);
case TEXT -> firstNonBlank(text, value);
case URL -> firstNonBlank(value, text);
case LOAD_STATE -> firstNonBlank(value, text);
};
if (target == null) {
throw new IllegalArgumentException("Target is required for wait_for condition=" + condition);
}
int max = Math.max(1, maxTimeoutSeconds);
int requested = timeoutSeconds == null ? max : Math.max(1, timeoutSeconds);
return new BrowserWaitCondition(kind, target, Math.min(requested, max) * 1000);
}
private static String firstNonBlank(String... values) {
for (String value : values) {
if (value != null && !value.isBlank()) {
return value.trim();
}
}
return null;
}
}

View File

@ -5,7 +5,10 @@ import cn.hutool.json.JSONObject;
import cn.hutool.json.JSONUtil;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
/**
* Accessibility-tree page snapshot for browser automation.
@ -51,6 +54,7 @@ public final class PageSnapshotScript {
const budget = { remaining: maxLen, truncated: false };
let counter = 0;
const refs = [];
const refInfos = [];
// Wipe references from a prior snapshot so ids never collide
// across generations and a navigated-away page leaves nothing behind.
@ -141,9 +145,30 @@ public final class PageSnapshotScript {
return txt;
}
function clip(s, n) {
function normalizeName(s) {
if (!s) return '';
return s.length > n ? s.substring(0, n) + '…' : s;
return s.length > 100 ? s.substring(0, 100) + '…' : s;
}
function stateOf(el, ref, role, name) {
const tag = el.tagName.toLowerCase();
const info = {
ref: ref,
role: role || '',
name: name || '',
tag: tag,
type: el.getAttribute('type') || '',
href: el.getAttribute('href') || '',
value: '',
checked: !!el.checked,
selected: !!el.selected,
disabled: !!el.disabled || el.getAttribute('aria-disabled') === 'true',
expanded: el.getAttribute('aria-expanded') === null ? null : el.getAttribute('aria-expanded') === 'true'
};
if (tag === 'input' || tag === 'textarea' || tag === 'select') {
info.value = String(el.value || '');
}
return info;
}
const lines = [];
@ -173,10 +198,11 @@ public final class PageSnapshotScript {
const ref = 'e' + counter;
el.setAttribute('data-mate-ref', ref);
refs.push(ref);
const nm = clip(nameOf(el), 100);
const nm = normalizeName(nameOf(el));
refInfos.push(stateOf(el, ref, role, nm));
line = role + (nm ? ' "' + nm + '"' : '') + ' @' + ref;
} else if (includeNon && role && role !== 'generic') {
const nm = clip(nameOf(el), 100);
const nm = normalizeName(nameOf(el));
if (nm || role === 'list' || role === 'navigation') {
let extra = '';
if (role === 'heading') {
@ -200,12 +226,91 @@ public final class PageSnapshotScript {
}
walk(rootEl, 0);
return JSON.stringify({ tree: lines.join('\\n'), truncated: budget.truncated, refs: refs });
return JSON.stringify({ tree: lines.join('\\n'), truncated: budget.truncated, refs: refs, refInfos: refInfos });
}
""";
/** Evaluate on an element handle to capture the same core fingerprint used by snapshots. */
public static final String REF_FINGERPRINT_JS = """
(el, ref) => {
function nameOf(el) {
const aria = el.getAttribute('aria-label');
if (aria) return aria.trim();
const labelledby = el.getAttribute('aria-labelledby');
if (labelledby) {
const target = document.getElementById(labelledby);
if (target) return (target.textContent || '').trim();
}
const tag = el.tagName.toLowerCase();
if (tag === 'input' || tag === 'textarea') {
const ph = el.getAttribute('placeholder');
if (ph) return ph.trim();
if (el.value) return String(el.value).trim();
if (el.id) {
const lab = document.querySelector('label[for="' + (window.CSS ? CSS.escape(el.id) : el.id) + '"]');
if (lab) return (lab.textContent || '').trim();
}
const wrap = el.closest('label');
if (wrap) {
const wt = (wrap.textContent || '').trim().replace(/\\s+/g, ' ');
if (wt) return wt;
}
return '';
}
if (tag === 'img') {
const alt = el.getAttribute('alt');
if (alt) return alt.trim();
}
const title = el.getAttribute('title');
if (title) return title.trim();
return el.textContent ? el.textContent.trim().replace(/\\s+/g, ' ').substring(0, 100) : '';
}
function roleOf(el) {
const explicit = el.getAttribute('role');
if (explicit) return explicit;
const tag = el.tagName.toLowerCase();
switch (tag) {
case 'a': return el.hasAttribute('href') ? 'link' : 'generic';
case 'button': return 'button';
case 'select': return 'combobox';
case 'textarea': return 'textbox';
case 'summary': return 'button';
case 'input': {
const t = (el.getAttribute('type') || 'text').toLowerCase();
if (t === 'checkbox') return 'checkbox';
if (t === 'radio') return 'radio';
if (t === 'submit' || t === 'button' || t === 'reset') return 'button';
if (t === 'search') return 'searchbox';
if (t === 'hidden') return null;
return 'textbox';
}
default: return null;
}
}
function normalizeName(s) {
if (!s) return '';
return s.length > 100 ? s.substring(0, 100) + '…' : s;
}
const tag = el.tagName.toLowerCase();
return JSON.stringify({
ref: ref || el.getAttribute('data-mate-ref') || '',
role: roleOf(el) || '',
name: normalizeName(nameOf(el)),
tag: tag,
type: el.getAttribute('type') || '',
href: el.getAttribute('href') || '',
value: (tag === 'input' || tag === 'textarea' || tag === 'select') ? String(el.value || '') : '',
checked: !!el.checked,
selected: !!el.selected,
disabled: !!el.disabled || el.getAttribute('aria-disabled') === 'true',
expanded: el.getAttribute('aria-expanded') === null ? null : el.getAttribute('aria-expanded') === 'true'
});
}
""";
/** Parsed result of a snapshot evaluation. */
public record Result(String tree, boolean truncated, List<String> refs) {
public record Result(String tree, boolean truncated, List<String> refs,
Map<String, RefFingerprint> refInfos) {
public static Result fromJson(String json) {
JSONObject obj = JSONUtil.parseObj(json);
String tree = obj.getStr("tree", "");
@ -219,7 +324,52 @@ public final class PageSnapshotScript {
}
}
}
return new Result(tree, truncated, refs);
Map<String, RefFingerprint> refInfos = new LinkedHashMap<>();
JSONArray infoArr = obj.getJSONArray("refInfos");
if (infoArr != null) {
for (Object o : infoArr) {
if (o instanceof JSONObject info) {
RefFingerprint fp = RefFingerprint.fromJson(info);
if (fp.ref() != null && !fp.ref().isBlank()) {
refInfos.put(fp.ref(), fp);
}
}
}
}
return new Result(tree, truncated, refs, Map.copyOf(refInfos));
}
}
public record RefFingerprint(String ref, String role, String name, String tag, String type,
String href, String value, boolean checked, boolean selected,
boolean disabled, Boolean expanded) {
public static RefFingerprint fromJson(JSONObject obj) {
return new RefFingerprint(
obj.getStr("ref", ""),
obj.getStr("role", ""),
obj.getStr("name", ""),
obj.getStr("tag", ""),
obj.getStr("type", ""),
obj.getStr("href", ""),
obj.getStr("value", ""),
obj.getBool("checked", false),
obj.getBool("selected", false),
obj.getBool("disabled", false),
obj.get("expanded") == null ? null : obj.getBool("expanded", false));
}
public boolean sameCoreIdentity(RefFingerprint other) {
if (other == null) {
return false;
}
return Objects.equals(normalize(role), normalize(other.role))
&& Objects.equals(normalize(name), normalize(other.name))
&& Objects.equals(normalize(tag), normalize(other.tag))
&& Objects.equals(normalize(type), normalize(other.type));
}
private static String normalize(String value) {
return value == null ? "" : value.trim().replaceAll("\\s+", " ").toLowerCase();
}
}

View File

@ -5,15 +5,18 @@ import cn.hutool.json.JSONArray;
import cn.hutool.json.JSONObject;
import cn.hutool.json.JSONUtil;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import com.google.gson.Gson;
import com.microsoft.playwright.Browser;
import com.microsoft.playwright.BrowserContext;
import com.microsoft.playwright.CDPSession;
import com.microsoft.playwright.ConsoleMessage;
import com.microsoft.playwright.ElementHandle;
import com.microsoft.playwright.Locator;
import com.microsoft.playwright.Page;
import com.microsoft.playwright.Playwright;
import com.microsoft.playwright.PlaywrightException;
import com.microsoft.playwright.options.LoadState;
import com.microsoft.playwright.options.WaitForSelectorState;
import jakarta.annotation.PreDestroy;
import lombok.extern.slf4j.Slf4j;
import org.springframework.ai.chat.model.ToolContext;
@ -23,7 +26,11 @@ import org.springframework.lang.Nullable;
import org.springframework.stereotype.Component;
import vip.mate.tool.browser.BrowserDiagnosticsService;
import vip.mate.tool.browser.BrowserLauncher;
import vip.mate.tool.browser.BrowserNavigationGuard;
import vip.mate.tool.browser.BrowserPrivacyGuard;
import vip.mate.tool.browser.BrowserRefState;
import vip.mate.tool.browser.BrowserSessionGate;
import vip.mate.tool.browser.BrowserWaitCondition;
import vip.mate.common.net.SsrfProperties;
import vip.mate.tool.browser.PageSnapshotScript;
import vip.mate.tool.browser.UrlSafetyChecker;
@ -32,6 +39,7 @@ import java.net.Socket;
import java.nio.file.Paths;
import java.util.Base64;
import java.util.List;
import java.util.Map;
import java.util.concurrent.*;
import java.util.regex.Pattern;
@ -47,6 +55,7 @@ public class BrowserUseTool {
private static final long IDLE_TIMEOUT_MINUTES = 30;
private static final int CDP_SCAN_PORT_MIN = 9000;
private static final int CDP_SCAN_PORT_MAX = 10000;
private static final Gson GSON = new Gson();
/**
* Legacy visible-text extractor kept as a fallback: {@link #doSnapshot}
@ -146,7 +155,7 @@ public class BrowserUseTool {
if (reason == null) {
return null;
}
String conversationId = ToolExecutionContext.conversationId(currentToolContext);
String conversationId = ToolExecutionContext.conversationId(currentToolContext.get());
privacyGuard.audit(conversationId, action, session.page.url(), reason);
log.info("[BrowserUse] Privacy guard blocked action={} on {}", action, session.page.url());
return error(reason);
@ -163,13 +172,12 @@ public class BrowserUseTool {
private final ConcurrentHashMap<String, BrowserSession> sessions = new ConcurrentHashMap<>();
/**
* RFC-063r §2.5 transition: ToolContext for the current invocation, set
* at the @Tool entry point and read by {@link #broadcastBrowserEvent}.
* Tool calls are serialized per ToolExecutionExecutor instance so this
* volatile field is safe; the field is read-only inside the action
* handlers.
* Invocation context is thread-local. The shared Playwright driver is
* guarded globally because Playwright Java permits multi-threaded callers
* only when no two threads invoke its objects at the same time.
*/
private volatile ToolContext currentToolContext;
private final ThreadLocal<ToolContext> currentToolContext = new ThreadLocal<>();
private final BrowserSessionGate sessionGate = new BrowserSessionGate(1);
private final ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(r -> {
Thread t = new Thread(r, "browser-idle-watchdog");
t.setDaemon(true);
@ -179,7 +187,7 @@ public class BrowserUseTool {
@Tool(description = """
Control a browser (Playwright with multi-strategy launch: system Chrome/Edge channel, explicit path, bundled, or external CDP).
Default is headless. Use headed=true with action=start for a visible window.
Typical flow: start open(url) snapshot click/type stop.
Typical flow: start open(url) snapshot click/type current_surface or wait_for stop.
If start fails, run action=diagnose for a full report of what's missing and how to fix it.
SCOPE use this tool ONLY for tasks that require driving a real browser:
@ -199,6 +207,8 @@ public class BrowserUseTool {
`selector` scopes to a subtree USE IT when the page is large to avoid truncation
(`truncated:true` is flagged with a hint).
- screenshot: Take a screenshot. Optional path to save file; returns base64 if no path.
- current_surface: Return current URL/title/document readiness, recent console/page errors, and ref validity.
- wait_for: Wait for condition=selector|text|url|load_state. Use selector/text/value plus optional timeoutSeconds.
- click: Click an element. Pass ref=<eN> from a snapshot (preferred) or a CSS selector.
- type: Type text into an element. Pass ref=<eN> (preferred) or selector, plus text.
- hover: Hover over an element (reveals menus/tooltips). Pass ref=<eN> or selector.
@ -211,26 +221,23 @@ public class BrowserUseTool {
- diagnose: Run a self-check reports which launch strategies are available and what to install if none are.
""")
public String browser_use(
@ToolParam(description = "Action: start|stop|open|snapshot|screenshot|click|type|hover|select|eval|connect_cdp|list_cdp_targets|navigate_back|diagnose") String action,
@ToolParam(description = "Action: start|stop|open|snapshot|screenshot|current_surface|wait_for|click|type|hover|select|eval|connect_cdp|list_cdp_targets|navigate_back|diagnose") String action,
@ToolParam(description = "URL to navigate to (for open), or CDP base URL (for connect_cdp, e.g. http://localhost:9222)", required = false) String url,
@ToolParam(description = "CSS selector. Alternative to ref for click/type/hover/select. OPTIONAL for snapshot: pass to scope to a subtree when previous snapshot returned truncated:true.", required = false) String selector,
@ToolParam(description = "Element reference from a snapshot (e.g. 'e4'). PREFERRED for click/type/hover/select — takes priority over selector. Re-snapshot if it reports stale.", required = false) String ref,
@ToolParam(description = "Text to type (for action=type)", required = false) String text,
@ToolParam(description = "Option value or visible label to choose (for action=select)", required = false) String value,
@ToolParam(description = "Wait condition for action=wait_for: selector|text|url|load_state", required = false) String condition,
@ToolParam(description = "Timeout seconds for action=wait_for; capped by mateclaw.browser.default-timeout-seconds", required = false) Integer timeoutSeconds,
@ToolParam(description = "JavaScript code to execute (for action=eval). Top-level await is allowed; add `return` to return a value when the snippet uses await.", required = false) String code,
@ToolParam(description = "CDP method for action=cdp (e.g. 'Page.navigate', 'Input.dispatchMouseEvent'). Must be in the allowlist.", required = false) String method,
@ToolParam(description = "JSON object of params for action=cdp (e.g. {\"url\":\"https://example.com\"})", required = false) String params,
@ToolParam(description = "Structured params object for action=cdp (e.g. {\"url\":\"https://example.com\"})", required = false) Map<String, Object> params,
@ToolParam(description = "File path to save screenshot (for action=screenshot)", required = false) String path,
@ToolParam(description = "Launch visible browser window (for action=start, default false)", required = false) Boolean headed,
@ToolParam(description = "Single CDP port to scan (for action=list_cdp_targets)", required = false) Integer cdpPort,
// RFC-063r §2.5: hidden from LLM by JsonSchemaGenerator.
@Nullable ToolContext ctx
) {
// The conversationId resolution lives in broadcastBrowserEvent below;
// capture the ctx into a field so the helper can read it without
// passing it down every action handler. Race-free because tool calls
// are serialized per executor.
this.currentToolContext = ctx;
if (action == null || action.isBlank()) {
return error("action is required");
}
@ -244,31 +251,38 @@ public class BrowserUseTool {
log.info("[BrowserUse] action={}, session={}, url={}, selector={}, headed={}, cdpPort={}",
action, sessionKey, url, selector, headed, cdpPort);
try {
return switch (action.toLowerCase().trim()) {
case "start" -> doStart(sessionKey, Boolean.TRUE.equals(headed));
case "stop" -> doStop(sessionKey);
case "open" -> doOpen(sessionKey, url);
case "snapshot" -> doSnapshot(sessionKey, selector);
case "screenshot" -> doScreenshot(sessionKey, path);
case "click" -> doClick(sessionKey, ref, selector);
case "type" -> doType(sessionKey, ref, selector, text);
case "hover" -> doHover(sessionKey, ref, selector);
case "select" -> doSelect(sessionKey, ref, selector, value);
case "eval" -> doEval(sessionKey, code);
case "cdp" -> doCdp(sessionKey, method, params);
case "connect_cdp" -> doConnectCdp(sessionKey, url);
case "list_cdp_targets" -> doListCdpTargets(cdpPort);
case "navigate_back" -> doNavigateBack(sessionKey);
case "diagnose" -> doDiagnose();
default -> error("Unknown action: " + action + ". Supported: start, stop, open, snapshot, screenshot, click, type, hover, select, eval, cdp, connect_cdp, list_cdp_targets, navigate_back, diagnose");
};
} catch (PlaywrightException e) {
log.error("[BrowserUse] Playwright error: {}", e.getMessage());
return error("Browser error: " + e.getMessage());
} catch (Exception e) {
log.error("[BrowserUse] Unexpected error: {}", e.getMessage(), e);
return error("Unexpected error: " + e.getMessage());
try (BrowserSessionGate.Lease ignored = sessionGate.enter(sessionKey)) {
currentToolContext.set(ctx);
try {
return switch (action.toLowerCase().trim()) {
case "start" -> doStart(sessionKey, Boolean.TRUE.equals(headed));
case "stop" -> doStop(sessionKey);
case "open" -> doOpen(sessionKey, url);
case "snapshot" -> doSnapshot(sessionKey, selector);
case "screenshot" -> doScreenshot(sessionKey, path);
case "current_surface" -> doCurrentSurface(sessionKey);
case "wait_for" -> doWaitFor(sessionKey, condition, selector, text, value, timeoutSeconds);
case "click" -> doClick(sessionKey, ref, selector);
case "type" -> doType(sessionKey, ref, selector, text);
case "hover" -> doHover(sessionKey, ref, selector);
case "select" -> doSelect(sessionKey, ref, selector, value);
case "eval" -> doEval(sessionKey, code);
case "cdp" -> doCdp(sessionKey, method, params);
case "connect_cdp" -> doConnectCdp(sessionKey, url);
case "list_cdp_targets" -> doListCdpTargets(cdpPort);
case "navigate_back" -> doNavigateBack(sessionKey);
case "diagnose" -> doDiagnose();
default -> error("Unknown action: " + action + ". Supported: start, stop, open, snapshot, screenshot, current_surface, wait_for, click, type, hover, select, eval, cdp, connect_cdp, list_cdp_targets, navigate_back, diagnose");
};
} catch (PlaywrightException e) {
log.error("[BrowserUse] Playwright error: {}", e.getMessage());
return error("Browser error: " + e.getMessage());
} catch (Exception e) {
log.error("[BrowserUse] Unexpected error: {}", e.getMessage(), e);
return error("Unexpected error: " + e.getMessage());
} finally {
currentToolContext.remove();
}
}
}
@ -320,7 +334,7 @@ public class BrowserUseTool {
*/
private void broadcastBrowserEvent(String action, boolean success, String url, String title,
String screenshot, long durationMs) {
String conversationId = ToolExecutionContext.conversationId(currentToolContext);
String conversationId = ToolExecutionContext.conversationId(currentToolContext.get());
if (conversationId == null || streamTracker == null) {
return;
}
@ -597,6 +611,7 @@ public class BrowserUseTool {
String title = session.page.title();
String url = session.page.url();
session.refState.reconcileUrl(url);
log.info("[BrowserUse] Navigated back to: {} ({})", url, title);
@ -661,7 +676,9 @@ public class BrowserUseTool {
String jsResult = (String) root.evaluate(PageSnapshotScript.SNAPSHOT_JS, opts);
PageSnapshotScript.Result snap = PageSnapshotScript.Result.fromJson(jsResult);
int generation = session.nextSnapshotGeneration(snap.refs());
String snapshotUrl = page.url();
int generation = session.nextSnapshotGeneration(snapshotUrl, snap.refs(), snap.refInfos());
result.set("url", snapshotUrl);
result.set("snapshotMode", "accessibility-tree");
result.set("generation", generation);
@ -679,6 +696,7 @@ public class BrowserUseTool {
result.set("scopedTo", selector);
}
result.set("refCount", snap.refs().size());
result.set("nativeAriaSnapshot", clippedNativeAriaSnapshot(page));
result.set("content", snap.tree());
return JSONUtil.toJsonPrettyStr(result);
} catch (PlaywrightException e) {
@ -750,6 +768,136 @@ public class BrowserUseTool {
}
}
private String doCurrentSurface(String sessionKey) {
BrowserSession session = requireSession(sessionKey);
if (session == null) {
return error("No browser running. Use action=start first.");
}
String blocked = guardReadOrNull(session, "current_surface");
if (blocked != null) {
return blocked;
}
session.touch();
return JSONUtil.toJsonPrettyStr(surfaceResult(session));
}
private String doWaitFor(String sessionKey, String condition, String selector, String text,
String value, Integer timeoutSeconds) {
BrowserSession session = requireSession(sessionKey);
if (session == null) {
return error("No browser running. Use action=start first.");
}
String blocked = guardReadOrNull(session, "wait_for");
if (blocked != null) {
return blocked;
}
BrowserWaitCondition wait;
try {
wait = BrowserWaitCondition.parse(condition, selector, text, value, timeoutSeconds,
launcher.properties().getDefaultTimeoutSeconds());
} catch (IllegalArgumentException e) {
return error(e.getMessage());
}
session.touch();
Page page = session.page;
switch (wait.kind()) {
case SELECTOR -> page.waitForSelector(wait.target(),
new Page.WaitForSelectorOptions()
.setState(WaitForSelectorState.VISIBLE)
.setStrict(false)
.setTimeout(wait.timeoutMillis()));
case TEXT -> page.getByText(wait.target()).first().waitFor(
new Locator.WaitForOptions()
.setState(WaitForSelectorState.VISIBLE)
.setTimeout(wait.timeoutMillis()));
case URL -> page.waitForURL(wait.target(),
new Page.WaitForURLOptions().setTimeout(wait.timeoutMillis()));
case LOAD_STATE -> page.waitForLoadState(loadState(wait.target()),
new Page.WaitForLoadStateOptions().setTimeout(wait.timeoutMillis()));
}
JSONObject result = surfaceResult(session);
result.set("waitedFor", wait.kind().name().toLowerCase());
result.set("waitTarget", wait.target());
result.set("timeoutMillis", wait.timeoutMillis());
return JSONUtil.toJsonPrettyStr(result);
}
private JSONObject surfaceResult(BrowserSession session) {
Page page = session.page;
session.refState.reconcileUrl(page.url());
JSONObject result = new JSONObject();
result.set("ok", true);
result.set("currentUrl", page.url());
result.set("currentTitle", page.title());
result.set("readyState", safeReadyState(page));
result.set("snapshotGeneration", session.refState.snapshotGeneration());
result.set("refStatus", session.refState.status().name().toLowerCase());
result.set("refsValid", session.refState.refsValid());
result.set("refCount", session.refState.refCount());
result.set("navigationEpoch", session.refState.navigationEpoch());
result.set("snapshotNavigationEpoch", session.refState.snapshotNavigationEpoch());
result.set("snapshotUrl", session.refState.snapshotUrl());
JSONArray console = new JSONArray();
try {
List<ConsoleMessage> messages = page.consoleMessages();
int start = Math.max(0, messages.size() - 5);
for (ConsoleMessage message : messages.subList(start, messages.size())) {
JSONObject item = new JSONObject();
item.set("type", message.type());
item.set("text", message.text());
item.set("timestamp", message.timestamp());
console.add(item);
}
} catch (Exception e) {
log.debug("[BrowserUse] consoleMessages unavailable: {}", e.getMessage());
}
result.set("recentConsoleMessages", console);
JSONArray errors = new JSONArray();
try {
List<String> pageErrors = page.pageErrors();
int start = Math.max(0, pageErrors.size() - 5);
for (String pageError : pageErrors.subList(start, pageErrors.size())) {
errors.add(pageError);
}
} catch (Exception e) {
log.debug("[BrowserUse] pageErrors unavailable: {}", e.getMessage());
}
result.set("recentPageErrors", errors);
return result;
}
private static String safeReadyState(Page page) {
try {
Object ready = page.evaluate("document.readyState");
return ready != null ? ready.toString() : "unknown";
} catch (Exception e) {
return "unknown";
}
}
private static String clippedNativeAriaSnapshot(Page page) {
try {
String snapshot = page.ariaSnapshot();
if (snapshot == null) {
return "";
}
return snapshot.length() > 4000 ? snapshot.substring(0, 4000) + "\n... [truncated]" : snapshot;
} catch (Exception e) {
return "";
}
}
private static LoadState loadState(String state) {
return switch (state.trim().toLowerCase()) {
case "load" -> LoadState.LOAD;
case "domcontentloaded", "dom_content_loaded" -> LoadState.DOMCONTENTLOADED;
case "networkidle", "network_idle" -> LoadState.NETWORKIDLE;
default -> throw new IllegalArgumentException("Unknown load state: " + state
+ ". Supported: load, domcontentloaded, networkidle");
};
}
/** Result of resolving a click/type/hover/select target: a selector, or an error to return. */
private record TargetResolution(String selector, String label, String error) {
static TargetResolution ok(String selector, String label) {
@ -769,13 +917,30 @@ public class BrowserUseTool {
private TargetResolution resolveTarget(BrowserSession session, String ref, String selector) {
if (ref != null && !ref.isBlank()) {
String r = ref.trim();
if (!session.currentRefs.contains(r)) {
if (!session.refState.contains(r)) {
return TargetResolution.fail("ref '" + r + "' is not part of the current snapshot"
+ " (generation " + session.snapshotGeneration + "). The page likely changed,"
+ " (generation " + session.refState.snapshotGeneration() + "). The page likely changed,"
+ " or you have not snapshotted since it did. Call action=snapshot first,"
+ " then use a ref from that fresh result.");
}
return TargetResolution.ok(PageSnapshotScript.selectorForRef(r), r);
String refSelector = PageSnapshotScript.selectorForRef(r);
PageSnapshotScript.RefFingerprint expected = session.refState.fingerprint(r);
if (expected != null) {
ElementHandle element = session.page.querySelector(refSelector);
if (element == null) {
session.invalidateRefs();
return TargetResolution.fail("ref '" + r + "' no longer exists on the page."
+ " Call action=snapshot again before acting.");
}
PageSnapshotScript.RefFingerprint actual = liveFingerprint(element, r);
if (!expected.sameCoreIdentity(actual)) {
session.invalidateRefs();
return TargetResolution.fail("ref '" + r + "' now points to a different element."
+ " Expected " + expected.role() + " '" + expected.name() + "', got "
+ actual.role() + " '" + actual.name() + "'. Call action=snapshot again.");
}
}
return TargetResolution.ok(refSelector, r);
}
if (selector != null && !selector.isBlank()) {
return TargetResolution.ok(selector, selector);
@ -783,6 +948,11 @@ public class BrowserUseTool {
return TargetResolution.fail("Either ref (from a snapshot, e.g. ref='e4') or a CSS selector is required.");
}
private static PageSnapshotScript.RefFingerprint liveFingerprint(ElementHandle element, String ref) {
String json = (String) element.evaluate(PageSnapshotScript.REF_FINGERPRINT_JS, ref);
return PageSnapshotScript.RefFingerprint.fromJson(JSONUtil.parseObj(json));
}
private String doClick(String sessionKey, String ref, String selector) {
BrowserSession session = requireSession(sessionKey);
if (session == null) {
@ -797,6 +967,7 @@ public class BrowserUseTool {
Page page = session.page;
String before = page.url();
String beforeTitle = page.title();
page.click(t.selector());
page.waitForLoadState(LoadState.DOMCONTENTLOADED);
@ -804,9 +975,7 @@ public class BrowserUseTool {
String url = page.url();
// A navigation invalidates the snapshot references; a same-page click
// (toggle, expand) keeps them so the model can act on more refs.
if (!url.equals(before)) {
session.invalidateRefs();
}
session.refState.reconcileUrl(url);
log.info("[BrowserUse] Clicked: {} (page now: {})", t.label(), url);
broadcastBrowserEvent("click", true, url, title, null, 0);
@ -816,6 +985,9 @@ public class BrowserUseTool {
result.set("target", t.label());
result.set("currentUrl", url);
result.set("currentTitle", title);
result.set("urlChanged", !url.equals(before));
result.set("titleChanged", !title.equals(beforeTitle));
result.set("refsValid", session.refState.refsValid());
if (!url.equals(before)) {
result.set("navigated", true);
result.set("hint", "The page navigated — previous @eN refs are stale. Re-snapshot before acting.");
@ -838,14 +1010,24 @@ public class BrowserUseTool {
}
session.touch();
String before = session.page.url();
String beforeTitle = session.page.title();
session.page.fill(t.selector(), text);
String url = session.page.url();
String title = session.page.title();
session.refState.reconcileUrl(url);
log.info("[BrowserUse] Typed into: {} ({} chars)", t.label(), text.length());
broadcastBrowserEvent("type", true, null, null, null, 0);
broadcastBrowserEvent("type", true, url, title, null, 0);
JSONObject result = new JSONObject();
result.set("ok", true);
result.set("target", t.label());
result.set("currentUrl", url);
result.set("currentTitle", title);
result.set("urlChanged", !url.equals(before));
result.set("titleChanged", !title.equals(beforeTitle));
result.set("refsValid", session.refState.refsValid());
result.set("textLength", text.length());
result.set("message", "Typed " + text.length() + " characters into " + t.label());
return JSONUtil.toJsonPrettyStr(result);
@ -862,14 +1044,24 @@ public class BrowserUseTool {
}
session.touch();
String before = session.page.url();
String beforeTitle = session.page.title();
session.page.hover(t.selector());
String url = session.page.url();
String title = session.page.title();
session.refState.reconcileUrl(url);
log.info("[BrowserUse] Hovered: {}", t.label());
broadcastBrowserEvent("hover", true, null, null, null, 0);
broadcastBrowserEvent("hover", true, url, title, null, 0);
JSONObject result = new JSONObject();
result.set("ok", true);
result.set("target", t.label());
result.set("currentUrl", url);
result.set("currentTitle", title);
result.set("urlChanged", !url.equals(before));
result.set("titleChanged", !title.equals(beforeTitle));
result.set("refsValid", session.refState.refsValid());
result.set("message", "Hovered element: " + t.label()
+ ". Re-snapshot to capture any menu/tooltip it revealed.");
return JSONUtil.toJsonPrettyStr(result);
@ -889,16 +1081,26 @@ public class BrowserUseTool {
}
session.touch();
String before = session.page.url();
String beforeTitle = session.page.title();
// Playwright matches by option value, label, or visible text, so a
// human-readable value from the model works without extra hints.
List<String> chosen = session.page.selectOption(t.selector(), value);
String url = session.page.url();
String title = session.page.title();
session.refState.reconcileUrl(url);
log.info("[BrowserUse] Selected {} in {} -> {}", value, t.label(), chosen);
broadcastBrowserEvent("select", true, null, null, null, 0);
broadcastBrowserEvent("select", true, url, title, null, 0);
JSONObject result = new JSONObject();
result.set("ok", true);
result.set("target", t.label());
result.set("currentUrl", url);
result.set("currentTitle", title);
result.set("urlChanged", !url.equals(before));
result.set("titleChanged", !title.equals(beforeTitle));
result.set("refsValid", session.refState.refsValid());
result.set("selected", chosen);
result.set("message", chosen.isEmpty()
? "No option matched '" + value + "'. Re-snapshot and check the option labels."
@ -933,6 +1135,16 @@ public class BrowserUseTool {
if (blocked != null) {
return blocked;
}
if (launcher.properties().isSsrfCheckEnabled()) {
try {
BrowserNavigationGuard.checkEval(code,
ssrfProperties.getSsrfAllowlist(),
launcher.properties().isAllowPrivateNetwork());
} catch (SecurityException se) {
log.warn("[BrowserUse] Eval navigation guard rejected script: {}", se.getMessage());
return error(se.getMessage());
}
}
session.touch();
Page page = session.page;
@ -961,6 +1173,7 @@ public class BrowserUseTool {
}
}
String resultStr = evalResult != null ? evalResult.toString() : "null";
session.refState.reconcileUrl(page.url());
if (resultStr.length() > 10_000) {
resultStr = resultStr.substring(0, 10_000) + "\n... [truncated]";
@ -971,6 +1184,8 @@ public class BrowserUseTool {
JSONObject result = new JSONObject();
result.set("ok", true);
result.set("result", resultStr);
result.set("currentUrl", page.url());
result.set("refsValid", session.refState.refsValid());
return JSONUtil.toJsonPrettyStr(result);
}
@ -1013,7 +1228,7 @@ public class BrowserUseTool {
return false;
}
private String doCdp(String sessionKey, String method, String paramsJson) {
private String doCdp(String sessionKey, String method, Map<String, Object> params) {
if (!launcher.properties().getCdp().isEnabled()) {
return error("action=cdp is disabled (mateclaw.browser.cdp.enabled=false).");
}
@ -1034,18 +1249,22 @@ public class BrowserUseTool {
String reason = privacyGuard.blockReason(session.isUserManagedBrowser(),
session.page.url(), "cdp:" + m);
if (reason != null) {
privacyGuard.audit(ToolExecutionContext.conversationId(currentToolContext),
privacyGuard.audit(ToolExecutionContext.conversationId(currentToolContext.get()),
"cdp:" + m, session.page.url(), reason);
return error(reason);
}
}
JsonObject parsed = null;
if (paramsJson != null && !paramsJson.isBlank()) {
JsonObject parsed = params == null ? null : GSON.toJsonTree(params).getAsJsonObject();
if (launcher.properties().isSsrfCheckEnabled()) {
try {
parsed = JsonParser.parseString(paramsJson).getAsJsonObject();
} catch (RuntimeException e) {
return error("params must be a JSON object: " + e.getMessage());
BrowserNavigationGuard.checkCdp(m, parsed,
ssrfProperties.getSsrfAllowlist(),
launcher.properties().isAllowPrivateNetwork());
} catch (SecurityException se) {
log.warn("[BrowserUse] CDP navigation guard rejected method={} params={}: {}",
m, params, se.getMessage());
return error(se.getMessage());
}
}
@ -1058,6 +1277,7 @@ public class BrowserUseTool {
if (m.startsWith("Page.navigate") || m.equals("Page.navigateToHistoryEntry")) {
session.invalidateRefs();
}
session.refState.reconcileUrl(session.page.url());
String out = res != null ? res.toString() : "{}";
if (out.length() > 10_000) {
out = out.substring(0, 10_000) + "\n... [truncated]";
@ -1140,7 +1360,13 @@ public class BrowserUseTool {
long idleMinutes = (System.currentTimeMillis() - s.lastActivity) / 60_000;
if (idleMinutes >= IDLE_TIMEOUT_MINUTES) {
log.info("[BrowserUse] Idle timeout ({}min), stopping session: {}", idleMinutes, sessionKey);
doStop(sessionKey);
try (BrowserSessionGate.Lease ignored = sessionGate.enter(sessionKey)) {
BrowserSession current = sessions.get(sessionKey);
if (current != null && System.currentTimeMillis() - current.lastActivity
>= TimeUnit.MINUTES.toMillis(IDLE_TIMEOUT_MINUTES)) {
doStop(sessionKey);
}
}
}
}, IDLE_TIMEOUT_MINUTES, 5, TimeUnit.MINUTES);
@ -1219,23 +1445,19 @@ public class BrowserUseTool {
volatile ScheduledFuture<?> idleWatchdog;
/**
* Monotonic snapshot generation. Each {@code action=snapshot} bumps it and
* replaces {@link #currentRefs} with the references assigned that pass.
* A click/type by a reference not in {@link #currentRefs} is reported as
* stale, prompting the caller to re-snapshot. Safe as plain volatile
* because tool calls are serialized per executor.
* Snapshot reference lifecycle, including navigation epochs and the
* fingerprints assigned by the latest snapshot.
*/
volatile int snapshotGeneration;
volatile java.util.Set<String> currentRefs = java.util.Set.of();
final BrowserRefState refState = new BrowserRefState();
int nextSnapshotGeneration(java.util.List<String> refs) {
this.currentRefs = java.util.Set.copyOf(refs);
return ++this.snapshotGeneration;
int nextSnapshotGeneration(String url, java.util.List<String> refs,
Map<String, PageSnapshotScript.RefFingerprint> refInfos) {
return refState.recordSnapshot(url, refs, refInfos);
}
/** Drop all references — the DOM they pointed at is gone (navigation). */
void invalidateRefs() {
this.currentRefs = java.util.Set.of();
refState.invalidate();
}
/**
@ -1259,6 +1481,12 @@ public class BrowserUseTool {
this.userDataDir = userDataDir;
this.ownedProcess = ownedProcess;
this.lastActivity = System.currentTimeMillis();
this.refState.reconcileUrl(page.url());
page.onFrameNavigated(frame -> {
if (frame == page.mainFrame()) {
refState.onMainFrameNavigated(frame.url());
}
});
}
void touch() {

View File

@ -98,7 +98,7 @@ Then comment out the `searxng` service block in `docker-compose.yml`. Make sure
### What the image actually contains
The backend runtime stage (`mateclaw-server/Dockerfile` stage 3) is based on `mcr.microsoft.com/playwright:v1.52.0-noble` (Ubuntu Noble 24.04, glibc) and installs on top of it:
The backend runtime stage (`mateclaw-server/Dockerfile` stage 3) is based on `mcr.microsoft.com/playwright:v1.62.0-noble` (Ubuntu Noble 24.04, glibc) and installs on top of it:
- `openjdk-21-jre-headless` — runs the Spring Boot JAR
- `fonts-noto-cjk` — Chinese/Japanese/Korean rendering in screenshots

View File

@ -98,7 +98,7 @@ SEARXNG_BASE_URL=https://your-searxng.example.com
### 镜像里到底装了什么
后端镜像以 `mcr.microsoft.com/playwright:v1.52.0-noble` 为基础Ubuntu Noble 24.04glibc`mateclaw-server/Dockerfile` 的第三阶段拉起,额外装:
后端镜像以 `mcr.microsoft.com/playwright:v1.62.0-noble` 为基础Ubuntu Noble 24.04glibc`mateclaw-server/Dockerfile` 的第三阶段拉起,额外装:
- `openjdk-21-jre-headless` —— 跑 Spring Boot JAR
- `fonts-noto-cjk` —— 中文页面截图不出豆腐块

View File

@ -0,0 +1,22 @@
package vip.mate.agent.graph.node;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Test;
import java.lang.reflect.Field;
import static org.assertj.core.api.Assertions.assertThat;
class ReasoningNodeProgressPromptTest {
@Test
@DisplayName("progress registration guidance respects the executor batch cap")
void progressPromptCapsRegistrationBatches() throws Exception {
Field field = ReasoningNode.class.getDeclaredField("TOOL_USE_ENFORCEMENT");
field.setAccessible(true);
String prompt = (String) field.get(null);
assertThat(prompt).contains("每批最多 16 个");
assertThat(prompt).doesNotContain("一条回复里 N 个 `progress_update` 同时发出");
}
}

View File

@ -0,0 +1,60 @@
package vip.mate.tool.browser;
import com.google.gson.JsonObject;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Test;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertThrows;
class BrowserNavigationGuardTest {
@Test
@DisplayName("blocks unsafe CDP Page.navigate URLs before dispatch")
void blocksUnsafeCdpNavigate() {
JsonObject params = new JsonObject();
params.addProperty("url", "http://169.254.169.254/latest/meta-data");
assertThrows(SecurityException.class,
() -> BrowserNavigationGuard.checkCdp("Page.navigate", params, List.of(), false));
}
@Test
@DisplayName("ignores non-navigation CDP methods")
void ignoresNonNavigationCdpMethod() {
JsonObject params = new JsonObject();
params.addProperty("url", "http://169.254.169.254/latest/meta-data");
assertDoesNotThrow(
() -> BrowserNavigationGuard.checkCdp("Runtime.evaluate", params, List.of(), false));
}
@Test
@DisplayName("blocks obvious JavaScript navigation to unsafe URL")
void blocksUnsafeEvalNavigation() {
assertThrows(SecurityException.class,
() -> BrowserNavigationGuard.checkEval(
"window.location.href = 'http://169.254.169.254/latest/meta-data'",
List.of(), false));
}
@Test
@DisplayName("blocks obvious JavaScript network calls to unsafe URL")
void blocksUnsafeEvalFetch() {
assertThrows(SecurityException.class,
() -> BrowserNavigationGuard.checkEval(
"return await fetch(\"http://169.254.169.254/latest/meta-data\")",
List.of(), false));
}
@Test
@DisplayName("allows inert URL strings without navigation intent")
void allowsInertUrlString() {
assertDoesNotThrow(
() -> BrowserNavigationGuard.checkEval(
"const note = 'http://169.254.169.254/latest/meta-data'; return note.length;",
List.of(), false));
}
}

View File

@ -0,0 +1,69 @@
package vip.mate.tool.browser;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Test;
import java.util.List;
import java.util.Map;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
class BrowserRefStateTest {
@Test
@DisplayName("a snapshot with zero interactive refs is still a valid snapshot")
void emptySnapshotIsValid() {
BrowserRefState state = new BrowserRefState();
int generation = state.recordSnapshot("https://example.com/empty", List.of(), Map.of());
assertEquals(1, generation);
assertEquals(BrowserRefState.Status.VALID, state.status());
assertTrue(state.refsValid());
assertEquals(0, state.refCount());
assertEquals("https://example.com/empty", state.snapshotUrl());
}
@Test
@DisplayName("main-frame navigation invalidates refs and advances navigation epoch")
void navigationInvalidatesSnapshot() {
BrowserRefState state = new BrowserRefState();
state.recordSnapshot("https://example.com/", List.of("e1"), Map.of());
state.onMainFrameNavigated("https://www.iana.org/help/example-domains");
assertEquals(BrowserRefState.Status.INVALIDATED, state.status());
assertFalse(state.refsValid());
assertEquals(0, state.refCount());
assertEquals(1L, state.navigationEpoch());
assertEquals("https://www.iana.org/help/example-domains", state.currentUrl());
}
@Test
@DisplayName("surface URL reconciliation catches SPA URL changes without frame navigation")
void reconcileUrlInvalidatesSnapshot() {
BrowserRefState state = new BrowserRefState();
state.recordSnapshot("https://example.com/page/1", List.of("e1"), Map.of());
state.reconcileUrl("https://example.com/page/2");
assertEquals(BrowserRefState.Status.INVALIDATED, state.status());
assertFalse(state.refsValid());
assertEquals(1L, state.navigationEpoch());
}
@Test
@DisplayName("navigation epoch keeps advancing after refs are already invalidated")
void repeatedUrlChangesAdvanceEpoch() {
BrowserRefState state = new BrowserRefState();
state.recordSnapshot("https://example.com/page/1", List.of("e1"), Map.of());
state.reconcileUrl("https://example.com/page/2");
state.reconcileUrl("https://example.com/page/3");
assertEquals(2L, state.navigationEpoch());
assertEquals("https://example.com/page/3", state.currentUrl());
}
}

View File

@ -0,0 +1,84 @@
package vip.mate.tool.browser;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Test;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
class BrowserSessionGateTest {
@Test
@DisplayName("operations for one browser session never overlap")
void serializesSameSession() throws Exception {
BrowserSessionGate gate = new BrowserSessionGate(16);
AtomicInteger active = new AtomicInteger();
AtomicInteger maxActive = new AtomicInteger();
CountDownLatch firstEntered = new CountDownLatch(1);
CountDownLatch releaseFirst = new CountDownLatch(1);
try (ExecutorService executor = Executors.newVirtualThreadPerTaskExecutor()) {
var first = executor.submit(() -> {
try (BrowserSessionGate.Lease ignored = gate.enter("conversation-1")) {
int now = active.incrementAndGet();
maxActive.accumulateAndGet(now, Math::max);
firstEntered.countDown();
assertTrue(releaseFirst.await(2, TimeUnit.SECONDS));
active.decrementAndGet();
return "first";
}
});
assertTrue(firstEntered.await(2, TimeUnit.SECONDS));
var second = executor.submit(() -> {
try (BrowserSessionGate.Lease ignored = gate.enter("conversation-1")) {
int now = active.incrementAndGet();
maxActive.accumulateAndGet(now, Math::max);
active.decrementAndGet();
return "second";
}
});
Thread.sleep(50);
releaseFirst.countDown();
assertEquals("first", first.get(2, TimeUnit.SECONDS));
assertEquals("second", second.get(2, TimeUnit.SECONDS));
}
assertEquals(1, maxActive.get());
}
@Test
@DisplayName("one stripe serializes different sessions sharing a Playwright driver")
void singleStripeSerializesDifferentSessions() throws Exception {
BrowserSessionGate gate = new BrowserSessionGate(1);
AtomicInteger active = new AtomicInteger();
AtomicInteger maxActive = new AtomicInteger();
try (ExecutorService executor = Executors.newVirtualThreadPerTaskExecutor()) {
var first = executor.submit(() -> runMeasured(gate, "conversation-1", active, maxActive));
var second = executor.submit(() -> runMeasured(gate, "conversation-2", active, maxActive));
first.get(2, TimeUnit.SECONDS);
second.get(2, TimeUnit.SECONDS);
}
assertEquals(1, maxActive.get());
}
private static String runMeasured(BrowserSessionGate gate, String key,
AtomicInteger active, AtomicInteger maxActive) throws Exception {
try (BrowserSessionGate.Lease ignored = gate.enter(key)) {
int now = active.incrementAndGet();
maxActive.accumulateAndGet(now, Math::max);
Thread.sleep(25);
active.decrementAndGet();
return key;
}
}
}

View File

@ -0,0 +1,54 @@
package vip.mate.tool.browser;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
class BrowserWaitConditionTest {
@Test
@DisplayName("parses selector wait and caps timeout")
void parsesSelectorAndCapsTimeout() {
BrowserWaitCondition parsed = BrowserWaitCondition.parse("selector", "#ready", null, null, 120, 30);
assertEquals(BrowserWaitCondition.Kind.SELECTOR, parsed.kind());
assertEquals("#ready", parsed.target());
assertEquals(30_000, parsed.timeoutMillis());
}
@Test
@DisplayName("parses text wait from text parameter")
void parsesTextFromTextParameter() {
BrowserWaitCondition parsed = BrowserWaitCondition.parse("text", null, "Saved", null, null, 30);
assertEquals(BrowserWaitCondition.Kind.TEXT, parsed.kind());
assertEquals("Saved", parsed.target());
assertEquals(30_000, parsed.timeoutMillis());
}
@Test
@DisplayName("parses load state aliases")
void parsesLoadStateAlias() {
BrowserWaitCondition parsed = BrowserWaitCondition.parse("load_state", null, null, "domcontentloaded", 5, 30);
assertEquals(BrowserWaitCondition.Kind.LOAD_STATE, parsed.kind());
assertEquals("domcontentloaded", parsed.target());
assertEquals(5_000, parsed.timeoutMillis());
}
@Test
@DisplayName("requires target for selector wait")
void requiresSelectorTarget() {
assertThrows(IllegalArgumentException.class,
() -> BrowserWaitCondition.parse("selector", " ", null, null, 5, 30));
}
@Test
@DisplayName("rejects unknown conditions")
void rejectsUnknownCondition() {
assertThrows(IllegalArgumentException.class,
() -> BrowserWaitCondition.parse("sleep", null, null, null, 5, 30));
}
}

View File

@ -0,0 +1,55 @@
package vip.mate.tool.browser;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
class PageSnapshotScriptResultTest {
@Test
@DisplayName("parses ref metadata and keeps refs backward-compatible")
void parsesRefMetadata() {
PageSnapshotScript.Result result = PageSnapshotScript.Result.fromJson("""
{
"tree":"- button \\"Save\\" @e1",
"truncated":false,
"refs":["e1"],
"refInfos":[
{"ref":"e1","role":"button","name":"Save","tag":"button","type":"","href":"","value":"","checked":false,"selected":false,"disabled":false,"expanded":true}
]
}
""");
assertEquals("e1", result.refs().getFirst());
assertEquals(1, result.refInfos().size());
PageSnapshotScript.RefFingerprint ref = result.refInfos().get("e1");
assertEquals("button", ref.role());
assertEquals("Save", ref.name());
assertEquals("button", ref.tag());
assertTrue(ref.expanded());
}
@Test
@DisplayName("detects core ref identity changes")
void detectsCoreIdentityChanges() {
PageSnapshotScript.RefFingerprint before = new PageSnapshotScript.RefFingerprint(
"e1", "button", "Save", "button", "", "", "", false, false, false, null);
PageSnapshotScript.RefFingerprint same = new PageSnapshotScript.RefFingerprint(
"e1", "button", "Save", "button", "", "", "new value", false, false, false, null);
PageSnapshotScript.RefFingerprint changed = new PageSnapshotScript.RefFingerprint(
"e1", "link", "Save", "a", "", "/save", "", false, false, false, null);
assertTrue(before.sameCoreIdentity(same));
assertFalse(before.sameCoreIdentity(changed));
}
@Test
@DisplayName("snapshot and live fingerprint scripts normalize long names identically")
void fingerprintScriptsShareNameNormalization() {
assertTrue(PageSnapshotScript.SNAPSHOT_JS.contains("normalizeName(nameOf(el))"));
assertTrue(PageSnapshotScript.REF_FINGERPRINT_JS.contains("normalizeName(nameOf(el))"));
}
}

View File

@ -0,0 +1,36 @@
package vip.mate.tool.builtin;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Test;
import org.springframework.ai.chat.model.ToolContext;
import org.springframework.ai.support.ToolCallbacks;
import java.lang.reflect.Method;
import java.util.Map;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
class BrowserUseToolContractTest {
@Test
@DisplayName("CDP params are exposed to the model as a structured object")
void cdpParamsUseStructuredMapSchema() throws NoSuchMethodException {
Method method = BrowserUseTool.class.getMethod("browser_use",
String.class, String.class, String.class, String.class, String.class,
String.class, String.class, Integer.class, String.class, String.class,
Map.class, String.class, Boolean.class, Integer.class, ToolContext.class);
assertEquals(Map.class, method.getParameterTypes()[10]);
}
@Test
@DisplayName("Spring AI publishes CDP params as a JSON object schema")
void generatedToolSchemaUsesObjectParams() {
BrowserUseTool tool = new BrowserUseTool(null, null, null, null, null);
String schema = ToolCallbacks.from(tool)[0].getToolDefinition().inputSchema();
assertTrue(schema.matches("(?s).*\\\"params\\\"\\s*:\\s*\\{.*?\\\"type\\\"\\s*:\\s*\\\"object\\\".*"), schema);
}
}

View File

@ -51,7 +51,7 @@
<!-- Browser automation and WebSocket runtime -->
<zxing.version>3.5.4</zxing.version>
<playwright.version>1.59.0</playwright.version>
<playwright.version>1.62.0</playwright.version>
<tyrus.version>2.2.2</tyrus.version>
<!-- Document, wiki, and rendering toolchain -->