From ba0e4506b7c1eb13c8b1748fde5774b67e5ed478 Mon Sep 17 00:00:00 2001 From: matevip Date: Wed, 12 Aug 2026 06:02:36 -0400 Subject: [PATCH] feat(browser): harden browser automation reliability --- mateclaw-server/Dockerfile | 2 +- .../vip/mate/agent/AgentGraphBuilder.java | 2 +- .../mate/agent/graph/node/ReasoningNode.java | 4 +- .../tool/browser/BrowserNavigationGuard.java | 49 +++ .../mate/tool/browser/BrowserRefState.java | 108 +++++ .../mate/tool/browser/BrowserSessionGate.java | 34 ++ .../tool/browser/BrowserWaitCondition.java | 52 +++ .../mate/tool/browser/PageSnapshotScript.java | 164 +++++++- .../vip/mate/tool/builtin/BrowserUseTool.java | 370 ++++++++++++++---- .../main/resources/docs/en/docker-deploy.md | 2 +- .../main/resources/docs/zh/docker-deploy.md | 2 +- .../node/ReasoningNodeProgressPromptTest.java | 22 ++ .../browser/BrowserNavigationGuardTest.java | 60 +++ .../tool/browser/BrowserRefStateTest.java | 69 ++++ .../tool/browser/BrowserSessionGateTest.java | 84 ++++ .../browser/BrowserWaitConditionTest.java | 54 +++ .../browser/PageSnapshotScriptResultTest.java | 55 +++ .../builtin/BrowserUseToolContractTest.java | 36 ++ pom.xml | 2 +- 19 files changed, 1086 insertions(+), 85 deletions(-) create mode 100644 mateclaw-server/src/main/java/vip/mate/tool/browser/BrowserNavigationGuard.java create mode 100644 mateclaw-server/src/main/java/vip/mate/tool/browser/BrowserRefState.java create mode 100644 mateclaw-server/src/main/java/vip/mate/tool/browser/BrowserSessionGate.java create mode 100644 mateclaw-server/src/main/java/vip/mate/tool/browser/BrowserWaitCondition.java create mode 100644 mateclaw-server/src/test/java/vip/mate/agent/graph/node/ReasoningNodeProgressPromptTest.java create mode 100644 mateclaw-server/src/test/java/vip/mate/tool/browser/BrowserNavigationGuardTest.java create mode 100644 mateclaw-server/src/test/java/vip/mate/tool/browser/BrowserRefStateTest.java create mode 100644 mateclaw-server/src/test/java/vip/mate/tool/browser/BrowserSessionGateTest.java create mode 100644 mateclaw-server/src/test/java/vip/mate/tool/browser/BrowserWaitConditionTest.java create mode 100644 mateclaw-server/src/test/java/vip/mate/tool/browser/PageSnapshotScriptResultTest.java create mode 100644 mateclaw-server/src/test/java/vip/mate/tool/builtin/BrowserUseToolContractTest.java diff --git a/mateclaw-server/Dockerfile b/mateclaw-server/Dockerfile index 8875b50c..6c5b672b 100644 --- a/mateclaw-server/Dockerfile +++ b/mateclaw-server/Dockerfile @@ -78,7 +78,7 @@ RUN mvn -pl mateclaw-server -am package -Dmaven.test.skip=true -q ${MAVEN_FLAGS} # bump the Java dependency, bump this tag in lockstep — Microsoft rebuilds each # tag with the matching driver, so mismatched versions cause the java driver to # re-download browsers at runtime (defeating the whole point of this image). -FROM mcr.microsoft.com/playwright:v1.59.0-noble +FROM mcr.microsoft.com/playwright:v1.62.0-noble WORKDIR /app # JDK 21 is NOT part of the base image (it ships Node for the JS driver). diff --git a/mateclaw-server/src/main/java/vip/mate/agent/AgentGraphBuilder.java b/mateclaw-server/src/main/java/vip/mate/agent/AgentGraphBuilder.java index 001c5ca4..0ae5680c 100644 --- a/mateclaw-server/src/main/java/vip/mate/agent/AgentGraphBuilder.java +++ b/mateclaw-server/src/main/java/vip/mate/agent/AgentGraphBuilder.java @@ -1776,7 +1776,7 @@ public class AgentGraphBuilder { ## ProgressLedger Discipline (mandatory) The `## 当前任务进度` block injected near the top of every turn is the **authoritative record** of what you have done and what remains. Treat it as ground truth, not as a scratchpad you may ignore. - - **On starting any multi-step task** (≥3 tool calls expected), call `progress_update` in a parallel tool_calls batch to register every pending step BEFORE doing the work. Do not wait until "later" — context compression can trim earlier turns and you will lose track. + - **On starting any multi-step task** (≥3 tool calls expected), call `progress_update` in parallel batches of at most 16 calls to register every pending step BEFORE doing the work. Split larger ledgers across turns so the executor cap never drops entries. - **After each completed sub-step**, immediately call `progress_update` to flip its status to `done`. "Immediately" means in the same tool_calls batch that returns the result, not after the next reasoning turn. - **Never re-execute a step the ledger shows as `done`** unless you can articulate why the prior result is stale. - **🔒 固定约束 entries** (pinned from skill manifests) are non-negotiable. They survive context compression for a reason — re-read them every turn and make sure your planned action still satisfies them. diff --git a/mateclaw-server/src/main/java/vip/mate/agent/graph/node/ReasoningNode.java b/mateclaw-server/src/main/java/vip/mate/agent/graph/node/ReasoningNode.java index 92b4e514..bd7ffd1b 100644 --- a/mateclaw-server/src/main/java/vip/mate/agent/graph/node/ReasoningNode.java +++ b/mateclaw-server/src/main/java/vip/mate/agent/graph/node/ReasoningNode.java @@ -257,8 +257,8 @@ public class ReasoningNode implements NodeAction { + "\"调研 10 个模型\"、\"逐节起草报告\"、\"批量生成 N 份文档\"、\n" + "\"依次调用 N 个 API\"、\"对每个文件执行同一操作\"等。\n\n" + "**必须做的事**：\n" - + "1. **第一轮回复就用并行 tool_calls 批量注册全部子目标为 `pending`**\n" - + " 一条回复里 N 个 `progress_update` 同时发出（不要串行）。\n" + + "1. **第一轮回复就用并行 tool_calls 批量注册子目标为 `pending`**\n" + + " 每批最多 16 个 `progress_update`；超过 16 个时分批注册，避免超过执行器上限。\n" + " 例：要调研 10 个模型，第一轮就发 10 个 `progress_update(stepKey=\"model_xxx\", status=\"pending\")`。\n" + "2. **每开始一个子目标**前发 `progress_update(同 stepKey, status=\"in_progress\")`。\n" + "3. **每完成一个子目标**后立即发 `progress_update(同 stepKey, status=\"done\")`。\n" diff --git a/mateclaw-server/src/main/java/vip/mate/tool/browser/BrowserNavigationGuard.java b/mateclaw-server/src/main/java/vip/mate/tool/browser/BrowserNavigationGuard.java new file mode 100644 index 00000000..781a65da --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/browser/BrowserNavigationGuard.java @@ -0,0 +1,49 @@ +package vip.mate.tool.browser; + +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; + +import java.util.Collection; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Shared URL safety checks for browser navigation surfaces beyond action=open. + */ +public final class BrowserNavigationGuard { + + private static final Pattern URL_LITERAL = + Pattern.compile("['\"](https?://[^'\"\\s)]+)['\"]", Pattern.CASE_INSENSITIVE); + + private static final Pattern EVAL_NAVIGATION_INTENT = Pattern.compile( + "\\b(location(?:\\.href|\\.assign|\\.replace)?|window\\.open|fetch|XMLHttpRequest)\\b", + Pattern.CASE_INSENSITIVE); + + private BrowserNavigationGuard() { + } + + public static void checkCdp(String method, JsonObject params, Collection allowlist, + boolean allowPrivateNetwork) { + if (!"Page.navigate".equals(method) || params == null || !params.has("url")) { + return; + } + JsonElement el = params.get("url"); + if (el == null || !el.isJsonPrimitive()) { + return; + } + UrlSafetyChecker.check(el.getAsString(), allowlist, allowPrivateNetwork); + } + + public static void checkEval(String code, Collection allowlist, boolean allowPrivateNetwork) { + if (code == null || code.isBlank()) { + return; + } + if (!EVAL_NAVIGATION_INTENT.matcher(code).find()) { + return; + } + Matcher matcher = URL_LITERAL.matcher(code); + while (matcher.find()) { + UrlSafetyChecker.check(matcher.group(1), allowlist, allowPrivateNetwork); + } + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/tool/browser/BrowserRefState.java b/mateclaw-server/src/main/java/vip/mate/tool/browser/BrowserRefState.java new file mode 100644 index 00000000..ba4f5d4b --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/browser/BrowserRefState.java @@ -0,0 +1,108 @@ +package vip.mate.tool.browser; + +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Thread-safe lifecycle state for snapshot references in one browser session. + */ +public final class BrowserRefState { + + public enum Status { + NONE, + VALID, + INVALIDATED + } + + private int snapshotGeneration; + private long navigationEpoch; + private long snapshotNavigationEpoch = -1; + private String currentUrl = ""; + private String snapshotUrl = ""; + private Status status = Status.NONE; + private Set refs = Set.of(); + private Map refInfos = Map.of(); + + public synchronized int recordSnapshot( + String url, + List newRefs, + Map newRefInfos) { + currentUrl = normalizeUrl(url); + snapshotUrl = currentUrl; + snapshotNavigationEpoch = navigationEpoch; + refs = Set.copyOf(newRefs); + refInfos = Map.copyOf(newRefInfos); + status = Status.VALID; + return ++snapshotGeneration; + } + + public synchronized void onMainFrameNavigated(String url) { + currentUrl = normalizeUrl(url); + navigationEpoch++; + invalidateSnapshot(); + } + + public synchronized void reconcileUrl(String url) { + String normalized = normalizeUrl(url); + if (!currentUrl.isEmpty() && !currentUrl.equals(normalized)) { + onMainFrameNavigated(normalized); + return; + } + currentUrl = normalized; + } + + public synchronized void invalidate() { + invalidateSnapshot(); + } + + private void invalidateSnapshot() { + refs = Set.of(); + refInfos = Map.of(); + status = snapshotGeneration == 0 ? Status.NONE : Status.INVALIDATED; + } + + private static String normalizeUrl(String url) { + return url == null ? "" : url; + } + + public synchronized Status status() { + return status; + } + + public synchronized boolean refsValid() { + return status == Status.VALID && snapshotNavigationEpoch == navigationEpoch; + } + + public synchronized boolean contains(String ref) { + return refs.contains(ref); + } + + public synchronized PageSnapshotScript.RefFingerprint fingerprint(String ref) { + return refInfos.get(ref); + } + + public synchronized int refCount() { + return refs.size(); + } + + public synchronized int snapshotGeneration() { + return snapshotGeneration; + } + + public synchronized long navigationEpoch() { + return navigationEpoch; + } + + public synchronized long snapshotNavigationEpoch() { + return snapshotNavigationEpoch; + } + + public synchronized String currentUrl() { + return currentUrl; + } + + public synchronized String snapshotUrl() { + return snapshotUrl; + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/tool/browser/BrowserSessionGate.java b/mateclaw-server/src/main/java/vip/mate/tool/browser/BrowserSessionGate.java new file mode 100644 index 00000000..a32142da --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/browser/BrowserSessionGate.java @@ -0,0 +1,34 @@ +package vip.mate.tool.browser; + +import java.util.concurrent.locks.ReentrantLock; + +/** + * Fixed-size striped lock gate for serializing operations on one browser session. + */ +public final class BrowserSessionGate { + + private final ReentrantLock[] locks; + + public BrowserSessionGate(int stripes) { + if (stripes <= 0) { + throw new IllegalArgumentException("stripes must be positive"); + } + locks = new ReentrantLock[stripes]; + for (int i = 0; i < stripes; i++) { + locks[i] = new ReentrantLock(true); + } + } + + public Lease enter(String sessionKey) { + int index = Math.floorMod(sessionKey.hashCode(), locks.length); + ReentrantLock lock = locks[index]; + lock.lock(); + return lock::unlock; + } + + @FunctionalInterface + public interface Lease extends AutoCloseable { + @Override + void close(); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/tool/browser/BrowserWaitCondition.java b/mateclaw-server/src/main/java/vip/mate/tool/browser/BrowserWaitCondition.java new file mode 100644 index 00000000..a19c4672 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/browser/BrowserWaitCondition.java @@ -0,0 +1,52 @@ +package vip.mate.tool.browser; + +/** + * Parsed, bounded wait request for browser_use action=wait_for. + */ +public record BrowserWaitCondition(Kind kind, String target, int timeoutMillis) { + + public enum Kind { + SELECTOR, + TEXT, + URL, + LOAD_STATE + } + + public static BrowserWaitCondition parse(String condition, String selector, String text, String value, + Integer timeoutSeconds, int maxTimeoutSeconds) { + if (condition == null || condition.isBlank()) { + throw new IllegalArgumentException("condition is required for action=wait_for"); + } + Kind kind = switch (condition.trim().toLowerCase()) { + case "selector" -> Kind.SELECTOR; + case "text" -> Kind.TEXT; + case "url" -> Kind.URL; + case "load_state", "loadstate", "state" -> Kind.LOAD_STATE; + default -> throw new IllegalArgumentException("Unknown wait_for condition: " + condition + + ". Supported: selector, text, url, load_state"); + }; + + String target = switch (kind) { + case SELECTOR -> firstNonBlank(selector); + case TEXT -> firstNonBlank(text, value); + case URL -> firstNonBlank(value, text); + case LOAD_STATE -> firstNonBlank(value, text); + }; + if (target == null) { + throw new IllegalArgumentException("Target is required for wait_for condition=" + condition); + } + + int max = Math.max(1, maxTimeoutSeconds); + int requested = timeoutSeconds == null ? max : Math.max(1, timeoutSeconds); + return new BrowserWaitCondition(kind, target, Math.min(requested, max) * 1000); + } + + private static String firstNonBlank(String... values) { + for (String value : values) { + if (value != null && !value.isBlank()) { + return value.trim(); + } + } + return null; + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/tool/browser/PageSnapshotScript.java b/mateclaw-server/src/main/java/vip/mate/tool/browser/PageSnapshotScript.java index fbf871a1..26a0bb3a 100644 --- a/mateclaw-server/src/main/java/vip/mate/tool/browser/PageSnapshotScript.java +++ b/mateclaw-server/src/main/java/vip/mate/tool/browser/PageSnapshotScript.java @@ -5,7 +5,10 @@ import cn.hutool.json.JSONObject; import cn.hutool.json.JSONUtil; import java.util.ArrayList; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; +import java.util.Objects; /** * Accessibility-tree page snapshot for browser automation. @@ -51,6 +54,7 @@ public final class PageSnapshotScript { const budget = { remaining: maxLen, truncated: false }; let counter = 0; const refs = []; + const refInfos = []; // Wipe references from a prior snapshot so ids never collide // across generations and a navigated-away page leaves nothing behind. @@ -141,9 +145,30 @@ public final class PageSnapshotScript { return txt; } - function clip(s, n) { + function normalizeName(s) { if (!s) return ''; - return s.length > n ? s.substring(0, n) + '…' : s; + return s.length > 100 ? s.substring(0, 100) + '…' : s; + } + + function stateOf(el, ref, role, name) { + const tag = el.tagName.toLowerCase(); + const info = { + ref: ref, + role: role || '', + name: name || '', + tag: tag, + type: el.getAttribute('type') || '', + href: el.getAttribute('href') || '', + value: '', + checked: !!el.checked, + selected: !!el.selected, + disabled: !!el.disabled || el.getAttribute('aria-disabled') === 'true', + expanded: el.getAttribute('aria-expanded') === null ? null : el.getAttribute('aria-expanded') === 'true' + }; + if (tag === 'input' || tag === 'textarea' || tag === 'select') { + info.value = String(el.value || ''); + } + return info; } const lines = []; @@ -173,10 +198,11 @@ public final class PageSnapshotScript { const ref = 'e' + counter; el.setAttribute('data-mate-ref', ref); refs.push(ref); - const nm = clip(nameOf(el), 100); + const nm = normalizeName(nameOf(el)); + refInfos.push(stateOf(el, ref, role, nm)); line = role + (nm ? ' "' + nm + '"' : '') + ' @' + ref; } else if (includeNon && role && role !== 'generic') { - const nm = clip(nameOf(el), 100); + const nm = normalizeName(nameOf(el)); if (nm || role === 'list' || role === 'navigation') { let extra = ''; if (role === 'heading') { @@ -200,12 +226,91 @@ public final class PageSnapshotScript { } walk(rootEl, 0); - return JSON.stringify({ tree: lines.join('\\n'), truncated: budget.truncated, refs: refs }); + return JSON.stringify({ tree: lines.join('\\n'), truncated: budget.truncated, refs: refs, refInfos: refInfos }); + } + """; + + /** Evaluate on an element handle to capture the same core fingerprint used by snapshots. */ + public static final String REF_FINGERPRINT_JS = """ + (el, ref) => { + function nameOf(el) { + const aria = el.getAttribute('aria-label'); + if (aria) return aria.trim(); + const labelledby = el.getAttribute('aria-labelledby'); + if (labelledby) { + const target = document.getElementById(labelledby); + if (target) return (target.textContent || '').trim(); + } + const tag = el.tagName.toLowerCase(); + if (tag === 'input' || tag === 'textarea') { + const ph = el.getAttribute('placeholder'); + if (ph) return ph.trim(); + if (el.value) return String(el.value).trim(); + if (el.id) { + const lab = document.querySelector('label[for="' + (window.CSS ? CSS.escape(el.id) : el.id) + '"]'); + if (lab) return (lab.textContent || '').trim(); + } + const wrap = el.closest('label'); + if (wrap) { + const wt = (wrap.textContent || '').trim().replace(/\\s+/g, ' '); + if (wt) return wt; + } + return ''; + } + if (tag === 'img') { + const alt = el.getAttribute('alt'); + if (alt) return alt.trim(); + } + const title = el.getAttribute('title'); + if (title) return title.trim(); + return el.textContent ? el.textContent.trim().replace(/\\s+/g, ' ').substring(0, 100) : ''; + } + function roleOf(el) { + const explicit = el.getAttribute('role'); + if (explicit) return explicit; + const tag = el.tagName.toLowerCase(); + switch (tag) { + case 'a': return el.hasAttribute('href') ? 'link' : 'generic'; + case 'button': return 'button'; + case 'select': return 'combobox'; + case 'textarea': return 'textbox'; + case 'summary': return 'button'; + case 'input': { + const t = (el.getAttribute('type') || 'text').toLowerCase(); + if (t === 'checkbox') return 'checkbox'; + if (t === 'radio') return 'radio'; + if (t === 'submit' || t === 'button' || t === 'reset') return 'button'; + if (t === 'search') return 'searchbox'; + if (t === 'hidden') return null; + return 'textbox'; + } + default: return null; + } + } + function normalizeName(s) { + if (!s) return ''; + return s.length > 100 ? s.substring(0, 100) + '…' : s; + } + const tag = el.tagName.toLowerCase(); + return JSON.stringify({ + ref: ref || el.getAttribute('data-mate-ref') || '', + role: roleOf(el) || '', + name: normalizeName(nameOf(el)), + tag: tag, + type: el.getAttribute('type') || '', + href: el.getAttribute('href') || '', + value: (tag === 'input' || tag === 'textarea' || tag === 'select') ? String(el.value || '') : '', + checked: !!el.checked, + selected: !!el.selected, + disabled: !!el.disabled || el.getAttribute('aria-disabled') === 'true', + expanded: el.getAttribute('aria-expanded') === null ? null : el.getAttribute('aria-expanded') === 'true' + }); } """; /** Parsed result of a snapshot evaluation. */ - public record Result(String tree, boolean truncated, List refs) { + public record Result(String tree, boolean truncated, List refs, + Map refInfos) { public static Result fromJson(String json) { JSONObject obj = JSONUtil.parseObj(json); String tree = obj.getStr("tree", ""); @@ -219,7 +324,52 @@ public final class PageSnapshotScript { } } } - return new Result(tree, truncated, refs); + Map refInfos = new LinkedHashMap<>(); + JSONArray infoArr = obj.getJSONArray("refInfos"); + if (infoArr != null) { + for (Object o : infoArr) { + if (o instanceof JSONObject info) { + RefFingerprint fp = RefFingerprint.fromJson(info); + if (fp.ref() != null && !fp.ref().isBlank()) { + refInfos.put(fp.ref(), fp); + } + } + } + } + return new Result(tree, truncated, refs, Map.copyOf(refInfos)); + } + } + + public record RefFingerprint(String ref, String role, String name, String tag, String type, + String href, String value, boolean checked, boolean selected, + boolean disabled, Boolean expanded) { + public static RefFingerprint fromJson(JSONObject obj) { + return new RefFingerprint( + obj.getStr("ref", ""), + obj.getStr("role", ""), + obj.getStr("name", ""), + obj.getStr("tag", ""), + obj.getStr("type", ""), + obj.getStr("href", ""), + obj.getStr("value", ""), + obj.getBool("checked", false), + obj.getBool("selected", false), + obj.getBool("disabled", false), + obj.get("expanded") == null ? null : obj.getBool("expanded", false)); + } + + public boolean sameCoreIdentity(RefFingerprint other) { + if (other == null) { + return false; + } + return Objects.equals(normalize(role), normalize(other.role)) + && Objects.equals(normalize(name), normalize(other.name)) + && Objects.equals(normalize(tag), normalize(other.tag)) + && Objects.equals(normalize(type), normalize(other.type)); + } + + private static String normalize(String value) { + return value == null ? "" : value.trim().replaceAll("\\s+", " ").toLowerCase(); } } diff --git a/mateclaw-server/src/main/java/vip/mate/tool/builtin/BrowserUseTool.java b/mateclaw-server/src/main/java/vip/mate/tool/builtin/BrowserUseTool.java index 3f03255d..13095b5a 100644 --- a/mateclaw-server/src/main/java/vip/mate/tool/builtin/BrowserUseTool.java +++ b/mateclaw-server/src/main/java/vip/mate/tool/builtin/BrowserUseTool.java @@ -5,15 +5,18 @@ import cn.hutool.json.JSONArray; import cn.hutool.json.JSONObject; import cn.hutool.json.JSONUtil; import com.google.gson.JsonObject; -import com.google.gson.JsonParser; +import com.google.gson.Gson; import com.microsoft.playwright.Browser; import com.microsoft.playwright.BrowserContext; import com.microsoft.playwright.CDPSession; +import com.microsoft.playwright.ConsoleMessage; import com.microsoft.playwright.ElementHandle; +import com.microsoft.playwright.Locator; import com.microsoft.playwright.Page; import com.microsoft.playwright.Playwright; import com.microsoft.playwright.PlaywrightException; import com.microsoft.playwright.options.LoadState; +import com.microsoft.playwright.options.WaitForSelectorState; import jakarta.annotation.PreDestroy; import lombok.extern.slf4j.Slf4j; import org.springframework.ai.chat.model.ToolContext; @@ -23,7 +26,11 @@ import org.springframework.lang.Nullable; import org.springframework.stereotype.Component; import vip.mate.tool.browser.BrowserDiagnosticsService; import vip.mate.tool.browser.BrowserLauncher; +import vip.mate.tool.browser.BrowserNavigationGuard; import vip.mate.tool.browser.BrowserPrivacyGuard; +import vip.mate.tool.browser.BrowserRefState; +import vip.mate.tool.browser.BrowserSessionGate; +import vip.mate.tool.browser.BrowserWaitCondition; import vip.mate.common.net.SsrfProperties; import vip.mate.tool.browser.PageSnapshotScript; import vip.mate.tool.browser.UrlSafetyChecker; @@ -32,6 +39,7 @@ import java.net.Socket; import java.nio.file.Paths; import java.util.Base64; import java.util.List; +import java.util.Map; import java.util.concurrent.*; import java.util.regex.Pattern; @@ -47,6 +55,7 @@ public class BrowserUseTool { private static final long IDLE_TIMEOUT_MINUTES = 30; private static final int CDP_SCAN_PORT_MIN = 9000; private static final int CDP_SCAN_PORT_MAX = 10000; + private static final Gson GSON = new Gson(); /** * Legacy visible-text extractor kept as a fallback: {@link #doSnapshot} @@ -146,7 +155,7 @@ public class BrowserUseTool { if (reason == null) { return null; } - String conversationId = ToolExecutionContext.conversationId(currentToolContext); + String conversationId = ToolExecutionContext.conversationId(currentToolContext.get()); privacyGuard.audit(conversationId, action, session.page.url(), reason); log.info("[BrowserUse] Privacy guard blocked action={} on {}", action, session.page.url()); return error(reason); @@ -163,13 +172,12 @@ public class BrowserUseTool { private final ConcurrentHashMap sessions = new ConcurrentHashMap<>(); /** - * RFC-063r §2.5 transition: ToolContext for the current invocation, set - * at the @Tool entry point and read by {@link #broadcastBrowserEvent}. - * Tool calls are serialized per ToolExecutionExecutor instance so this - * volatile field is safe; the field is read-only inside the action - * handlers. + * Invocation context is thread-local. The shared Playwright driver is + * guarded globally because Playwright Java permits multi-threaded callers + * only when no two threads invoke its objects at the same time. */ - private volatile ToolContext currentToolContext; + private final ThreadLocal currentToolContext = new ThreadLocal<>(); + private final BrowserSessionGate sessionGate = new BrowserSessionGate(1); private final ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(r -> { Thread t = new Thread(r, "browser-idle-watchdog"); t.setDaemon(true); @@ -179,7 +187,7 @@ public class BrowserUseTool { @Tool(description = """ Control a browser (Playwright with multi-strategy launch: system Chrome/Edge channel, explicit path, bundled, or external CDP). Default is headless. Use headed=true with action=start for a visible window. - Typical flow: start → open(url) → snapshot → click/type → stop. + Typical flow: start → open(url) → snapshot → click/type → current_surface or wait_for → stop. If start fails, run action=diagnose for a full report of what's missing and how to fix it. SCOPE — use this tool ONLY for tasks that require driving a real browser: @@ -199,6 +207,8 @@ public class BrowserUseTool { `selector` scopes to a subtree — USE IT when the page is large to avoid truncation (`truncated:true` is flagged with a hint). - screenshot: Take a screenshot. Optional path to save file; returns base64 if no path. + - current_surface: Return current URL/title/document readiness, recent console/page errors, and ref validity. + - wait_for: Wait for condition=selector|text|url|load_state. Use selector/text/value plus optional timeoutSeconds. - click: Click an element. Pass ref= from a snapshot (preferred) or a CSS selector. - type: Type text into an element. Pass ref= (preferred) or selector, plus text. - hover: Hover over an element (reveals menus/tooltips). Pass ref= or selector. @@ -211,26 +221,23 @@ public class BrowserUseTool { - diagnose: Run a self-check — reports which launch strategies are available and what to install if none are. """) public String browser_use( - @ToolParam(description = "Action: start|stop|open|snapshot|screenshot|click|type|hover|select|eval|connect_cdp|list_cdp_targets|navigate_back|diagnose") String action, + @ToolParam(description = "Action: start|stop|open|snapshot|screenshot|current_surface|wait_for|click|type|hover|select|eval|connect_cdp|list_cdp_targets|navigate_back|diagnose") String action, @ToolParam(description = "URL to navigate to (for open), or CDP base URL (for connect_cdp, e.g. http://localhost:9222)", required = false) String url, @ToolParam(description = "CSS selector. Alternative to ref for click/type/hover/select. OPTIONAL for snapshot: pass to scope to a subtree when previous snapshot returned truncated:true.", required = false) String selector, @ToolParam(description = "Element reference from a snapshot (e.g. 'e4'). PREFERRED for click/type/hover/select — takes priority over selector. Re-snapshot if it reports stale.", required = false) String ref, @ToolParam(description = "Text to type (for action=type)", required = false) String text, @ToolParam(description = "Option value or visible label to choose (for action=select)", required = false) String value, + @ToolParam(description = "Wait condition for action=wait_for: selector|text|url|load_state", required = false) String condition, + @ToolParam(description = "Timeout seconds for action=wait_for; capped by mateclaw.browser.default-timeout-seconds", required = false) Integer timeoutSeconds, @ToolParam(description = "JavaScript code to execute (for action=eval). Top-level await is allowed; add `return` to return a value when the snippet uses await.", required = false) String code, @ToolParam(description = "CDP method for action=cdp (e.g. 'Page.navigate', 'Input.dispatchMouseEvent'). Must be in the allowlist.", required = false) String method, - @ToolParam(description = "JSON object of params for action=cdp (e.g. {\"url\":\"https://example.com\"})", required = false) String params, + @ToolParam(description = "Structured params object for action=cdp (e.g. {\"url\":\"https://example.com\"})", required = false) Map params, @ToolParam(description = "File path to save screenshot (for action=screenshot)", required = false) String path, @ToolParam(description = "Launch visible browser window (for action=start, default false)", required = false) Boolean headed, @ToolParam(description = "Single CDP port to scan (for action=list_cdp_targets)", required = false) Integer cdpPort, // RFC-063r §2.5: hidden from LLM by JsonSchemaGenerator. @Nullable ToolContext ctx ) { - // The conversationId resolution lives in broadcastBrowserEvent below; - // capture the ctx into a field so the helper can read it without - // passing it down every action handler. Race-free because tool calls - // are serialized per executor. - this.currentToolContext = ctx; if (action == null || action.isBlank()) { return error("action is required"); } @@ -244,31 +251,38 @@ public class BrowserUseTool { log.info("[BrowserUse] action={}, session={}, url={}, selector={}, headed={}, cdpPort={}", action, sessionKey, url, selector, headed, cdpPort); - try { - return switch (action.toLowerCase().trim()) { - case "start" -> doStart(sessionKey, Boolean.TRUE.equals(headed)); - case "stop" -> doStop(sessionKey); - case "open" -> doOpen(sessionKey, url); - case "snapshot" -> doSnapshot(sessionKey, selector); - case "screenshot" -> doScreenshot(sessionKey, path); - case "click" -> doClick(sessionKey, ref, selector); - case "type" -> doType(sessionKey, ref, selector, text); - case "hover" -> doHover(sessionKey, ref, selector); - case "select" -> doSelect(sessionKey, ref, selector, value); - case "eval" -> doEval(sessionKey, code); - case "cdp" -> doCdp(sessionKey, method, params); - case "connect_cdp" -> doConnectCdp(sessionKey, url); - case "list_cdp_targets" -> doListCdpTargets(cdpPort); - case "navigate_back" -> doNavigateBack(sessionKey); - case "diagnose" -> doDiagnose(); - default -> error("Unknown action: " + action + ". Supported: start, stop, open, snapshot, screenshot, click, type, hover, select, eval, cdp, connect_cdp, list_cdp_targets, navigate_back, diagnose"); - }; - } catch (PlaywrightException e) { - log.error("[BrowserUse] Playwright error: {}", e.getMessage()); - return error("Browser error: " + e.getMessage()); - } catch (Exception e) { - log.error("[BrowserUse] Unexpected error: {}", e.getMessage(), e); - return error("Unexpected error: " + e.getMessage()); + try (BrowserSessionGate.Lease ignored = sessionGate.enter(sessionKey)) { + currentToolContext.set(ctx); + try { + return switch (action.toLowerCase().trim()) { + case "start" -> doStart(sessionKey, Boolean.TRUE.equals(headed)); + case "stop" -> doStop(sessionKey); + case "open" -> doOpen(sessionKey, url); + case "snapshot" -> doSnapshot(sessionKey, selector); + case "screenshot" -> doScreenshot(sessionKey, path); + case "current_surface" -> doCurrentSurface(sessionKey); + case "wait_for" -> doWaitFor(sessionKey, condition, selector, text, value, timeoutSeconds); + case "click" -> doClick(sessionKey, ref, selector); + case "type" -> doType(sessionKey, ref, selector, text); + case "hover" -> doHover(sessionKey, ref, selector); + case "select" -> doSelect(sessionKey, ref, selector, value); + case "eval" -> doEval(sessionKey, code); + case "cdp" -> doCdp(sessionKey, method, params); + case "connect_cdp" -> doConnectCdp(sessionKey, url); + case "list_cdp_targets" -> doListCdpTargets(cdpPort); + case "navigate_back" -> doNavigateBack(sessionKey); + case "diagnose" -> doDiagnose(); + default -> error("Unknown action: " + action + ". Supported: start, stop, open, snapshot, screenshot, current_surface, wait_for, click, type, hover, select, eval, cdp, connect_cdp, list_cdp_targets, navigate_back, diagnose"); + }; + } catch (PlaywrightException e) { + log.error("[BrowserUse] Playwright error: {}", e.getMessage()); + return error("Browser error: " + e.getMessage()); + } catch (Exception e) { + log.error("[BrowserUse] Unexpected error: {}", e.getMessage(), e); + return error("Unexpected error: " + e.getMessage()); + } finally { + currentToolContext.remove(); + } } } @@ -320,7 +334,7 @@ public class BrowserUseTool { */ private void broadcastBrowserEvent(String action, boolean success, String url, String title, String screenshot, long durationMs) { - String conversationId = ToolExecutionContext.conversationId(currentToolContext); + String conversationId = ToolExecutionContext.conversationId(currentToolContext.get()); if (conversationId == null || streamTracker == null) { return; } @@ -597,6 +611,7 @@ public class BrowserUseTool { String title = session.page.title(); String url = session.page.url(); + session.refState.reconcileUrl(url); log.info("[BrowserUse] Navigated back to: {} ({})", url, title); @@ -661,7 +676,9 @@ public class BrowserUseTool { String jsResult = (String) root.evaluate(PageSnapshotScript.SNAPSHOT_JS, opts); PageSnapshotScript.Result snap = PageSnapshotScript.Result.fromJson(jsResult); - int generation = session.nextSnapshotGeneration(snap.refs()); + String snapshotUrl = page.url(); + int generation = session.nextSnapshotGeneration(snapshotUrl, snap.refs(), snap.refInfos()); + result.set("url", snapshotUrl); result.set("snapshotMode", "accessibility-tree"); result.set("generation", generation); @@ -679,6 +696,7 @@ public class BrowserUseTool { result.set("scopedTo", selector); } result.set("refCount", snap.refs().size()); + result.set("nativeAriaSnapshot", clippedNativeAriaSnapshot(page)); result.set("content", snap.tree()); return JSONUtil.toJsonPrettyStr(result); } catch (PlaywrightException e) { @@ -750,6 +768,136 @@ public class BrowserUseTool { } } + private String doCurrentSurface(String sessionKey) { + BrowserSession session = requireSession(sessionKey); + if (session == null) { + return error("No browser running. Use action=start first."); + } + String blocked = guardReadOrNull(session, "current_surface"); + if (blocked != null) { + return blocked; + } + session.touch(); + return JSONUtil.toJsonPrettyStr(surfaceResult(session)); + } + + private String doWaitFor(String sessionKey, String condition, String selector, String text, + String value, Integer timeoutSeconds) { + BrowserSession session = requireSession(sessionKey); + if (session == null) { + return error("No browser running. Use action=start first."); + } + String blocked = guardReadOrNull(session, "wait_for"); + if (blocked != null) { + return blocked; + } + BrowserWaitCondition wait; + try { + wait = BrowserWaitCondition.parse(condition, selector, text, value, timeoutSeconds, + launcher.properties().getDefaultTimeoutSeconds()); + } catch (IllegalArgumentException e) { + return error(e.getMessage()); + } + + session.touch(); + Page page = session.page; + switch (wait.kind()) { + case SELECTOR -> page.waitForSelector(wait.target(), + new Page.WaitForSelectorOptions() + .setState(WaitForSelectorState.VISIBLE) + .setStrict(false) + .setTimeout(wait.timeoutMillis())); + case TEXT -> page.getByText(wait.target()).first().waitFor( + new Locator.WaitForOptions() + .setState(WaitForSelectorState.VISIBLE) + .setTimeout(wait.timeoutMillis())); + case URL -> page.waitForURL(wait.target(), + new Page.WaitForURLOptions().setTimeout(wait.timeoutMillis())); + case LOAD_STATE -> page.waitForLoadState(loadState(wait.target()), + new Page.WaitForLoadStateOptions().setTimeout(wait.timeoutMillis())); + } + JSONObject result = surfaceResult(session); + result.set("waitedFor", wait.kind().name().toLowerCase()); + result.set("waitTarget", wait.target()); + result.set("timeoutMillis", wait.timeoutMillis()); + return JSONUtil.toJsonPrettyStr(result); + } + + private JSONObject surfaceResult(BrowserSession session) { + Page page = session.page; + session.refState.reconcileUrl(page.url()); + JSONObject result = new JSONObject(); + result.set("ok", true); + result.set("currentUrl", page.url()); + result.set("currentTitle", page.title()); + result.set("readyState", safeReadyState(page)); + result.set("snapshotGeneration", session.refState.snapshotGeneration()); + result.set("refStatus", session.refState.status().name().toLowerCase()); + result.set("refsValid", session.refState.refsValid()); + result.set("refCount", session.refState.refCount()); + result.set("navigationEpoch", session.refState.navigationEpoch()); + result.set("snapshotNavigationEpoch", session.refState.snapshotNavigationEpoch()); + result.set("snapshotUrl", session.refState.snapshotUrl()); + JSONArray console = new JSONArray(); + try { + List messages = page.consoleMessages(); + int start = Math.max(0, messages.size() - 5); + for (ConsoleMessage message : messages.subList(start, messages.size())) { + JSONObject item = new JSONObject(); + item.set("type", message.type()); + item.set("text", message.text()); + item.set("timestamp", message.timestamp()); + console.add(item); + } + } catch (Exception e) { + log.debug("[BrowserUse] consoleMessages unavailable: {}", e.getMessage()); + } + result.set("recentConsoleMessages", console); + JSONArray errors = new JSONArray(); + try { + List pageErrors = page.pageErrors(); + int start = Math.max(0, pageErrors.size() - 5); + for (String pageError : pageErrors.subList(start, pageErrors.size())) { + errors.add(pageError); + } + } catch (Exception e) { + log.debug("[BrowserUse] pageErrors unavailable: {}", e.getMessage()); + } + result.set("recentPageErrors", errors); + return result; + } + + private static String safeReadyState(Page page) { + try { + Object ready = page.evaluate("document.readyState"); + return ready != null ? ready.toString() : "unknown"; + } catch (Exception e) { + return "unknown"; + } + } + + private static String clippedNativeAriaSnapshot(Page page) { + try { + String snapshot = page.ariaSnapshot(); + if (snapshot == null) { + return ""; + } + return snapshot.length() > 4000 ? snapshot.substring(0, 4000) + "\n... [truncated]" : snapshot; + } catch (Exception e) { + return ""; + } + } + + private static LoadState loadState(String state) { + return switch (state.trim().toLowerCase()) { + case "load" -> LoadState.LOAD; + case "domcontentloaded", "dom_content_loaded" -> LoadState.DOMCONTENTLOADED; + case "networkidle", "network_idle" -> LoadState.NETWORKIDLE; + default -> throw new IllegalArgumentException("Unknown load state: " + state + + ". Supported: load, domcontentloaded, networkidle"); + }; + } + /** Result of resolving a click/type/hover/select target: a selector, or an error to return. */ private record TargetResolution(String selector, String label, String error) { static TargetResolution ok(String selector, String label) { @@ -769,13 +917,30 @@ public class BrowserUseTool { private TargetResolution resolveTarget(BrowserSession session, String ref, String selector) { if (ref != null && !ref.isBlank()) { String r = ref.trim(); - if (!session.currentRefs.contains(r)) { + if (!session.refState.contains(r)) { return TargetResolution.fail("ref '" + r + "' is not part of the current snapshot" - + " (generation " + session.snapshotGeneration + "). The page likely changed," + + " (generation " + session.refState.snapshotGeneration() + "). The page likely changed," + " or you have not snapshotted since it did. Call action=snapshot first," + " then use a ref from that fresh result."); } - return TargetResolution.ok(PageSnapshotScript.selectorForRef(r), r); + String refSelector = PageSnapshotScript.selectorForRef(r); + PageSnapshotScript.RefFingerprint expected = session.refState.fingerprint(r); + if (expected != null) { + ElementHandle element = session.page.querySelector(refSelector); + if (element == null) { + session.invalidateRefs(); + return TargetResolution.fail("ref '" + r + "' no longer exists on the page." + + " Call action=snapshot again before acting."); + } + PageSnapshotScript.RefFingerprint actual = liveFingerprint(element, r); + if (!expected.sameCoreIdentity(actual)) { + session.invalidateRefs(); + return TargetResolution.fail("ref '" + r + "' now points to a different element." + + " Expected " + expected.role() + " '" + expected.name() + "', got " + + actual.role() + " '" + actual.name() + "'. Call action=snapshot again."); + } + } + return TargetResolution.ok(refSelector, r); } if (selector != null && !selector.isBlank()) { return TargetResolution.ok(selector, selector); @@ -783,6 +948,11 @@ public class BrowserUseTool { return TargetResolution.fail("Either ref (from a snapshot, e.g. ref='e4') or a CSS selector is required."); } + private static PageSnapshotScript.RefFingerprint liveFingerprint(ElementHandle element, String ref) { + String json = (String) element.evaluate(PageSnapshotScript.REF_FINGERPRINT_JS, ref); + return PageSnapshotScript.RefFingerprint.fromJson(JSONUtil.parseObj(json)); + } + private String doClick(String sessionKey, String ref, String selector) { BrowserSession session = requireSession(sessionKey); if (session == null) { @@ -797,6 +967,7 @@ public class BrowserUseTool { Page page = session.page; String before = page.url(); + String beforeTitle = page.title(); page.click(t.selector()); page.waitForLoadState(LoadState.DOMCONTENTLOADED); @@ -804,9 +975,7 @@ public class BrowserUseTool { String url = page.url(); // A navigation invalidates the snapshot references; a same-page click // (toggle, expand) keeps them so the model can act on more refs. - if (!url.equals(before)) { - session.invalidateRefs(); - } + session.refState.reconcileUrl(url); log.info("[BrowserUse] Clicked: {} (page now: {})", t.label(), url); broadcastBrowserEvent("click", true, url, title, null, 0); @@ -816,6 +985,9 @@ public class BrowserUseTool { result.set("target", t.label()); result.set("currentUrl", url); result.set("currentTitle", title); + result.set("urlChanged", !url.equals(before)); + result.set("titleChanged", !title.equals(beforeTitle)); + result.set("refsValid", session.refState.refsValid()); if (!url.equals(before)) { result.set("navigated", true); result.set("hint", "The page navigated — previous @eN refs are stale. Re-snapshot before acting."); @@ -838,14 +1010,24 @@ public class BrowserUseTool { } session.touch(); + String before = session.page.url(); + String beforeTitle = session.page.title(); session.page.fill(t.selector(), text); + String url = session.page.url(); + String title = session.page.title(); + session.refState.reconcileUrl(url); log.info("[BrowserUse] Typed into: {} ({} chars)", t.label(), text.length()); - broadcastBrowserEvent("type", true, null, null, null, 0); + broadcastBrowserEvent("type", true, url, title, null, 0); JSONObject result = new JSONObject(); result.set("ok", true); result.set("target", t.label()); + result.set("currentUrl", url); + result.set("currentTitle", title); + result.set("urlChanged", !url.equals(before)); + result.set("titleChanged", !title.equals(beforeTitle)); + result.set("refsValid", session.refState.refsValid()); result.set("textLength", text.length()); result.set("message", "Typed " + text.length() + " characters into " + t.label()); return JSONUtil.toJsonPrettyStr(result); @@ -862,14 +1044,24 @@ public class BrowserUseTool { } session.touch(); + String before = session.page.url(); + String beforeTitle = session.page.title(); session.page.hover(t.selector()); + String url = session.page.url(); + String title = session.page.title(); + session.refState.reconcileUrl(url); log.info("[BrowserUse] Hovered: {}", t.label()); - broadcastBrowserEvent("hover", true, null, null, null, 0); + broadcastBrowserEvent("hover", true, url, title, null, 0); JSONObject result = new JSONObject(); result.set("ok", true); result.set("target", t.label()); + result.set("currentUrl", url); + result.set("currentTitle", title); + result.set("urlChanged", !url.equals(before)); + result.set("titleChanged", !title.equals(beforeTitle)); + result.set("refsValid", session.refState.refsValid()); result.set("message", "Hovered element: " + t.label() + ". Re-snapshot to capture any menu/tooltip it revealed."); return JSONUtil.toJsonPrettyStr(result); @@ -889,16 +1081,26 @@ public class BrowserUseTool { } session.touch(); + String before = session.page.url(); + String beforeTitle = session.page.title(); // Playwright matches by option value, label, or visible text, so a // human-readable value from the model works without extra hints. List chosen = session.page.selectOption(t.selector(), value); + String url = session.page.url(); + String title = session.page.title(); + session.refState.reconcileUrl(url); log.info("[BrowserUse] Selected {} in {} -> {}", value, t.label(), chosen); - broadcastBrowserEvent("select", true, null, null, null, 0); + broadcastBrowserEvent("select", true, url, title, null, 0); JSONObject result = new JSONObject(); result.set("ok", true); result.set("target", t.label()); + result.set("currentUrl", url); + result.set("currentTitle", title); + result.set("urlChanged", !url.equals(before)); + result.set("titleChanged", !title.equals(beforeTitle)); + result.set("refsValid", session.refState.refsValid()); result.set("selected", chosen); result.set("message", chosen.isEmpty() ? "No option matched '" + value + "'. Re-snapshot and check the option labels." @@ -933,6 +1135,16 @@ public class BrowserUseTool { if (blocked != null) { return blocked; } + if (launcher.properties().isSsrfCheckEnabled()) { + try { + BrowserNavigationGuard.checkEval(code, + ssrfProperties.getSsrfAllowlist(), + launcher.properties().isAllowPrivateNetwork()); + } catch (SecurityException se) { + log.warn("[BrowserUse] Eval navigation guard rejected script: {}", se.getMessage()); + return error(se.getMessage()); + } + } session.touch(); Page page = session.page; @@ -961,6 +1173,7 @@ public class BrowserUseTool { } } String resultStr = evalResult != null ? evalResult.toString() : "null"; + session.refState.reconcileUrl(page.url()); if (resultStr.length() > 10_000) { resultStr = resultStr.substring(0, 10_000) + "\n... [truncated]"; @@ -971,6 +1184,8 @@ public class BrowserUseTool { JSONObject result = new JSONObject(); result.set("ok", true); result.set("result", resultStr); + result.set("currentUrl", page.url()); + result.set("refsValid", session.refState.refsValid()); return JSONUtil.toJsonPrettyStr(result); } @@ -1013,7 +1228,7 @@ public class BrowserUseTool { return false; } - private String doCdp(String sessionKey, String method, String paramsJson) { + private String doCdp(String sessionKey, String method, Map params) { if (!launcher.properties().getCdp().isEnabled()) { return error("action=cdp is disabled (mateclaw.browser.cdp.enabled=false)."); } @@ -1034,18 +1249,22 @@ public class BrowserUseTool { String reason = privacyGuard.blockReason(session.isUserManagedBrowser(), session.page.url(), "cdp:" + m); if (reason != null) { - privacyGuard.audit(ToolExecutionContext.conversationId(currentToolContext), + privacyGuard.audit(ToolExecutionContext.conversationId(currentToolContext.get()), "cdp:" + m, session.page.url(), reason); return error(reason); } } - JsonObject parsed = null; - if (paramsJson != null && !paramsJson.isBlank()) { + JsonObject parsed = params == null ? null : GSON.toJsonTree(params).getAsJsonObject(); + if (launcher.properties().isSsrfCheckEnabled()) { try { - parsed = JsonParser.parseString(paramsJson).getAsJsonObject(); - } catch (RuntimeException e) { - return error("params must be a JSON object: " + e.getMessage()); + BrowserNavigationGuard.checkCdp(m, parsed, + ssrfProperties.getSsrfAllowlist(), + launcher.properties().isAllowPrivateNetwork()); + } catch (SecurityException se) { + log.warn("[BrowserUse] CDP navigation guard rejected method={} params={}: {}", + m, params, se.getMessage()); + return error(se.getMessage()); } } @@ -1058,6 +1277,7 @@ public class BrowserUseTool { if (m.startsWith("Page.navigate") || m.equals("Page.navigateToHistoryEntry")) { session.invalidateRefs(); } + session.refState.reconcileUrl(session.page.url()); String out = res != null ? res.toString() : "{}"; if (out.length() > 10_000) { out = out.substring(0, 10_000) + "\n... [truncated]"; @@ -1140,7 +1360,13 @@ public class BrowserUseTool { long idleMinutes = (System.currentTimeMillis() - s.lastActivity) / 60_000; if (idleMinutes >= IDLE_TIMEOUT_MINUTES) { log.info("[BrowserUse] Idle timeout ({}min), stopping session: {}", idleMinutes, sessionKey); - doStop(sessionKey); + try (BrowserSessionGate.Lease ignored = sessionGate.enter(sessionKey)) { + BrowserSession current = sessions.get(sessionKey); + if (current != null && System.currentTimeMillis() - current.lastActivity + >= TimeUnit.MINUTES.toMillis(IDLE_TIMEOUT_MINUTES)) { + doStop(sessionKey); + } + } } }, IDLE_TIMEOUT_MINUTES, 5, TimeUnit.MINUTES); @@ -1219,23 +1445,19 @@ public class BrowserUseTool { volatile ScheduledFuture idleWatchdog; /** - * Monotonic snapshot generation. Each {@code action=snapshot} bumps it and - * replaces {@link #currentRefs} with the references assigned that pass. - * A click/type by a reference not in {@link #currentRefs} is reported as - * stale, prompting the caller to re-snapshot. Safe as plain volatile - * because tool calls are serialized per executor. + * Snapshot reference lifecycle, including navigation epochs and the + * fingerprints assigned by the latest snapshot. */ - volatile int snapshotGeneration; - volatile java.util.Set currentRefs = java.util.Set.of(); + final BrowserRefState refState = new BrowserRefState(); - int nextSnapshotGeneration(java.util.List refs) { - this.currentRefs = java.util.Set.copyOf(refs); - return ++this.snapshotGeneration; + int nextSnapshotGeneration(String url, java.util.List refs, + Map refInfos) { + return refState.recordSnapshot(url, refs, refInfos); } /** Drop all references — the DOM they pointed at is gone (navigation). */ void invalidateRefs() { - this.currentRefs = java.util.Set.of(); + refState.invalidate(); } /** @@ -1259,6 +1481,12 @@ public class BrowserUseTool { this.userDataDir = userDataDir; this.ownedProcess = ownedProcess; this.lastActivity = System.currentTimeMillis(); + this.refState.reconcileUrl(page.url()); + page.onFrameNavigated(frame -> { + if (frame == page.mainFrame()) { + refState.onMainFrameNavigated(frame.url()); + } + }); } void touch() { diff --git a/mateclaw-server/src/main/resources/docs/en/docker-deploy.md b/mateclaw-server/src/main/resources/docs/en/docker-deploy.md index 6a15e732..695cdfc2 100644 --- a/mateclaw-server/src/main/resources/docs/en/docker-deploy.md +++ b/mateclaw-server/src/main/resources/docs/en/docker-deploy.md @@ -98,7 +98,7 @@ Then comment out the `searxng` service block in `docker-compose.yml`. Make sure ### What the image actually contains -The backend runtime stage (`mateclaw-server/Dockerfile` stage 3) is based on `mcr.microsoft.com/playwright:v1.52.0-noble` (Ubuntu Noble 24.04, glibc) and installs on top of it: +The backend runtime stage (`mateclaw-server/Dockerfile` stage 3) is based on `mcr.microsoft.com/playwright:v1.62.0-noble` (Ubuntu Noble 24.04, glibc) and installs on top of it: - `openjdk-21-jre-headless` — runs the Spring Boot JAR - `fonts-noto-cjk` — Chinese/Japanese/Korean rendering in screenshots diff --git a/mateclaw-server/src/main/resources/docs/zh/docker-deploy.md b/mateclaw-server/src/main/resources/docs/zh/docker-deploy.md index 2f424235..34f1bb6c 100644 --- a/mateclaw-server/src/main/resources/docs/zh/docker-deploy.md +++ b/mateclaw-server/src/main/resources/docs/zh/docker-deploy.md @@ -98,7 +98,7 @@ SEARXNG_BASE_URL=https://your-searxng.example.com ### 镜像里到底装了什么 -后端镜像以 `mcr.microsoft.com/playwright:v1.52.0-noble` 为基础（Ubuntu Noble 24.04，glibc），由 `mateclaw-server/Dockerfile` 的第三阶段拉起，额外装： +后端镜像以 `mcr.microsoft.com/playwright:v1.62.0-noble` 为基础（Ubuntu Noble 24.04，glibc），由 `mateclaw-server/Dockerfile` 的第三阶段拉起，额外装： - `openjdk-21-jre-headless` —— 跑 Spring Boot JAR - `fonts-noto-cjk` —— 中文页面截图不出豆腐块 diff --git a/mateclaw-server/src/test/java/vip/mate/agent/graph/node/ReasoningNodeProgressPromptTest.java b/mateclaw-server/src/test/java/vip/mate/agent/graph/node/ReasoningNodeProgressPromptTest.java new file mode 100644 index 00000000..bfaacab6 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/agent/graph/node/ReasoningNodeProgressPromptTest.java @@ -0,0 +1,22 @@ +package vip.mate.agent.graph.node; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import java.lang.reflect.Field; + +import static org.assertj.core.api.Assertions.assertThat; + +class ReasoningNodeProgressPromptTest { + + @Test + @DisplayName("progress registration guidance respects the executor batch cap") + void progressPromptCapsRegistrationBatches() throws Exception { + Field field = ReasoningNode.class.getDeclaredField("TOOL_USE_ENFORCEMENT"); + field.setAccessible(true); + String prompt = (String) field.get(null); + + assertThat(prompt).contains("每批最多 16 个"); + assertThat(prompt).doesNotContain("一条回复里 N 个 `progress_update` 同时发出"); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/tool/browser/BrowserNavigationGuardTest.java b/mateclaw-server/src/test/java/vip/mate/tool/browser/BrowserNavigationGuardTest.java new file mode 100644 index 00000000..630a3be6 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/tool/browser/BrowserNavigationGuardTest.java @@ -0,0 +1,60 @@ +package vip.mate.tool.browser; + +import com.google.gson.JsonObject; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertThrows; + +class BrowserNavigationGuardTest { + + @Test + @DisplayName("blocks unsafe CDP Page.navigate URLs before dispatch") + void blocksUnsafeCdpNavigate() { + JsonObject params = new JsonObject(); + params.addProperty("url", "http://169.254.169.254/latest/meta-data"); + + assertThrows(SecurityException.class, + () -> BrowserNavigationGuard.checkCdp("Page.navigate", params, List.of(), false)); + } + + @Test + @DisplayName("ignores non-navigation CDP methods") + void ignoresNonNavigationCdpMethod() { + JsonObject params = new JsonObject(); + params.addProperty("url", "http://169.254.169.254/latest/meta-data"); + + assertDoesNotThrow( + () -> BrowserNavigationGuard.checkCdp("Runtime.evaluate", params, List.of(), false)); + } + + @Test + @DisplayName("blocks obvious JavaScript navigation to unsafe URL") + void blocksUnsafeEvalNavigation() { + assertThrows(SecurityException.class, + () -> BrowserNavigationGuard.checkEval( + "window.location.href = 'http://169.254.169.254/latest/meta-data'", + List.of(), false)); + } + + @Test + @DisplayName("blocks obvious JavaScript network calls to unsafe URL") + void blocksUnsafeEvalFetch() { + assertThrows(SecurityException.class, + () -> BrowserNavigationGuard.checkEval( + "return await fetch(\"http://169.254.169.254/latest/meta-data\")", + List.of(), false)); + } + + @Test + @DisplayName("allows inert URL strings without navigation intent") + void allowsInertUrlString() { + assertDoesNotThrow( + () -> BrowserNavigationGuard.checkEval( + "const note = 'http://169.254.169.254/latest/meta-data'; return note.length;", + List.of(), false)); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/tool/browser/BrowserRefStateTest.java b/mateclaw-server/src/test/java/vip/mate/tool/browser/BrowserRefStateTest.java new file mode 100644 index 00000000..bed4b81a --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/tool/browser/BrowserRefStateTest.java @@ -0,0 +1,69 @@ +package vip.mate.tool.browser; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class BrowserRefStateTest { + + @Test + @DisplayName("a snapshot with zero interactive refs is still a valid snapshot") + void emptySnapshotIsValid() { + BrowserRefState state = new BrowserRefState(); + + int generation = state.recordSnapshot("https://example.com/empty", List.of(), Map.of()); + + assertEquals(1, generation); + assertEquals(BrowserRefState.Status.VALID, state.status()); + assertTrue(state.refsValid()); + assertEquals(0, state.refCount()); + assertEquals("https://example.com/empty", state.snapshotUrl()); + } + + @Test + @DisplayName("main-frame navigation invalidates refs and advances navigation epoch") + void navigationInvalidatesSnapshot() { + BrowserRefState state = new BrowserRefState(); + state.recordSnapshot("https://example.com/", List.of("e1"), Map.of()); + + state.onMainFrameNavigated("https://www.iana.org/help/example-domains"); + + assertEquals(BrowserRefState.Status.INVALIDATED, state.status()); + assertFalse(state.refsValid()); + assertEquals(0, state.refCount()); + assertEquals(1L, state.navigationEpoch()); + assertEquals("https://www.iana.org/help/example-domains", state.currentUrl()); + } + + @Test + @DisplayName("surface URL reconciliation catches SPA URL changes without frame navigation") + void reconcileUrlInvalidatesSnapshot() { + BrowserRefState state = new BrowserRefState(); + state.recordSnapshot("https://example.com/page/1", List.of("e1"), Map.of()); + + state.reconcileUrl("https://example.com/page/2"); + + assertEquals(BrowserRefState.Status.INVALIDATED, state.status()); + assertFalse(state.refsValid()); + assertEquals(1L, state.navigationEpoch()); + } + + @Test + @DisplayName("navigation epoch keeps advancing after refs are already invalidated") + void repeatedUrlChangesAdvanceEpoch() { + BrowserRefState state = new BrowserRefState(); + state.recordSnapshot("https://example.com/page/1", List.of("e1"), Map.of()); + + state.reconcileUrl("https://example.com/page/2"); + state.reconcileUrl("https://example.com/page/3"); + + assertEquals(2L, state.navigationEpoch()); + assertEquals("https://example.com/page/3", state.currentUrl()); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/tool/browser/BrowserSessionGateTest.java b/mateclaw-server/src/test/java/vip/mate/tool/browser/BrowserSessionGateTest.java new file mode 100644 index 00000000..d549c640 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/tool/browser/BrowserSessionGateTest.java @@ -0,0 +1,84 @@ +package vip.mate.tool.browser; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class BrowserSessionGateTest { + + @Test + @DisplayName("operations for one browser session never overlap") + void serializesSameSession() throws Exception { + BrowserSessionGate gate = new BrowserSessionGate(16); + AtomicInteger active = new AtomicInteger(); + AtomicInteger maxActive = new AtomicInteger(); + CountDownLatch firstEntered = new CountDownLatch(1); + CountDownLatch releaseFirst = new CountDownLatch(1); + + try (ExecutorService executor = Executors.newVirtualThreadPerTaskExecutor()) { + var first = executor.submit(() -> { + try (BrowserSessionGate.Lease ignored = gate.enter("conversation-1")) { + int now = active.incrementAndGet(); + maxActive.accumulateAndGet(now, Math::max); + firstEntered.countDown(); + assertTrue(releaseFirst.await(2, TimeUnit.SECONDS)); + active.decrementAndGet(); + return "first"; + } + }); + assertTrue(firstEntered.await(2, TimeUnit.SECONDS)); + + var second = executor.submit(() -> { + try (BrowserSessionGate.Lease ignored = gate.enter("conversation-1")) { + int now = active.incrementAndGet(); + maxActive.accumulateAndGet(now, Math::max); + active.decrementAndGet(); + return "second"; + } + }); + + Thread.sleep(50); + releaseFirst.countDown(); + assertEquals("first", first.get(2, TimeUnit.SECONDS)); + assertEquals("second", second.get(2, TimeUnit.SECONDS)); + } + + assertEquals(1, maxActive.get()); + } + + @Test + @DisplayName("one stripe serializes different sessions sharing a Playwright driver") + void singleStripeSerializesDifferentSessions() throws Exception { + BrowserSessionGate gate = new BrowserSessionGate(1); + AtomicInteger active = new AtomicInteger(); + AtomicInteger maxActive = new AtomicInteger(); + + try (ExecutorService executor = Executors.newVirtualThreadPerTaskExecutor()) { + var first = executor.submit(() -> runMeasured(gate, "conversation-1", active, maxActive)); + var second = executor.submit(() -> runMeasured(gate, "conversation-2", active, maxActive)); + first.get(2, TimeUnit.SECONDS); + second.get(2, TimeUnit.SECONDS); + } + + assertEquals(1, maxActive.get()); + } + + private static String runMeasured(BrowserSessionGate gate, String key, + AtomicInteger active, AtomicInteger maxActive) throws Exception { + try (BrowserSessionGate.Lease ignored = gate.enter(key)) { + int now = active.incrementAndGet(); + maxActive.accumulateAndGet(now, Math::max); + Thread.sleep(25); + active.decrementAndGet(); + return key; + } + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/tool/browser/BrowserWaitConditionTest.java b/mateclaw-server/src/test/java/vip/mate/tool/browser/BrowserWaitConditionTest.java new file mode 100644 index 00000000..b3c6229b --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/tool/browser/BrowserWaitConditionTest.java @@ -0,0 +1,54 @@ +package vip.mate.tool.browser; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +class BrowserWaitConditionTest { + + @Test + @DisplayName("parses selector wait and caps timeout") + void parsesSelectorAndCapsTimeout() { + BrowserWaitCondition parsed = BrowserWaitCondition.parse("selector", "#ready", null, null, 120, 30); + + assertEquals(BrowserWaitCondition.Kind.SELECTOR, parsed.kind()); + assertEquals("#ready", parsed.target()); + assertEquals(30_000, parsed.timeoutMillis()); + } + + @Test + @DisplayName("parses text wait from text parameter") + void parsesTextFromTextParameter() { + BrowserWaitCondition parsed = BrowserWaitCondition.parse("text", null, "Saved", null, null, 30); + + assertEquals(BrowserWaitCondition.Kind.TEXT, parsed.kind()); + assertEquals("Saved", parsed.target()); + assertEquals(30_000, parsed.timeoutMillis()); + } + + @Test + @DisplayName("parses load state aliases") + void parsesLoadStateAlias() { + BrowserWaitCondition parsed = BrowserWaitCondition.parse("load_state", null, null, "domcontentloaded", 5, 30); + + assertEquals(BrowserWaitCondition.Kind.LOAD_STATE, parsed.kind()); + assertEquals("domcontentloaded", parsed.target()); + assertEquals(5_000, parsed.timeoutMillis()); + } + + @Test + @DisplayName("requires target for selector wait") + void requiresSelectorTarget() { + assertThrows(IllegalArgumentException.class, + () -> BrowserWaitCondition.parse("selector", " ", null, null, 5, 30)); + } + + @Test + @DisplayName("rejects unknown conditions") + void rejectsUnknownCondition() { + assertThrows(IllegalArgumentException.class, + () -> BrowserWaitCondition.parse("sleep", null, null, null, 5, 30)); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/tool/browser/PageSnapshotScriptResultTest.java b/mateclaw-server/src/test/java/vip/mate/tool/browser/PageSnapshotScriptResultTest.java new file mode 100644 index 00000000..acd242a8 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/tool/browser/PageSnapshotScriptResultTest.java @@ -0,0 +1,55 @@ +package vip.mate.tool.browser; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class PageSnapshotScriptResultTest { + + @Test + @DisplayName("parses ref metadata and keeps refs backward-compatible") + void parsesRefMetadata() { + PageSnapshotScript.Result result = PageSnapshotScript.Result.fromJson(""" + { + "tree":"- button \\"Save\\" @e1", + "truncated":false, + "refs":["e1"], + "refInfos":[ + {"ref":"e1","role":"button","name":"Save","tag":"button","type":"","href":"","value":"","checked":false,"selected":false,"disabled":false,"expanded":true} + ] + } + """); + + assertEquals("e1", result.refs().getFirst()); + assertEquals(1, result.refInfos().size()); + PageSnapshotScript.RefFingerprint ref = result.refInfos().get("e1"); + assertEquals("button", ref.role()); + assertEquals("Save", ref.name()); + assertEquals("button", ref.tag()); + assertTrue(ref.expanded()); + } + + @Test + @DisplayName("detects core ref identity changes") + void detectsCoreIdentityChanges() { + PageSnapshotScript.RefFingerprint before = new PageSnapshotScript.RefFingerprint( + "e1", "button", "Save", "button", "", "", "", false, false, false, null); + PageSnapshotScript.RefFingerprint same = new PageSnapshotScript.RefFingerprint( + "e1", "button", "Save", "button", "", "", "new value", false, false, false, null); + PageSnapshotScript.RefFingerprint changed = new PageSnapshotScript.RefFingerprint( + "e1", "link", "Save", "a", "", "/save", "", false, false, false, null); + + assertTrue(before.sameCoreIdentity(same)); + assertFalse(before.sameCoreIdentity(changed)); + } + + @Test + @DisplayName("snapshot and live fingerprint scripts normalize long names identically") + void fingerprintScriptsShareNameNormalization() { + assertTrue(PageSnapshotScript.SNAPSHOT_JS.contains("normalizeName(nameOf(el))")); + assertTrue(PageSnapshotScript.REF_FINGERPRINT_JS.contains("normalizeName(nameOf(el))")); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/tool/builtin/BrowserUseToolContractTest.java b/mateclaw-server/src/test/java/vip/mate/tool/builtin/BrowserUseToolContractTest.java new file mode 100644 index 00000000..dd439ce7 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/tool/builtin/BrowserUseToolContractTest.java @@ -0,0 +1,36 @@ +package vip.mate.tool.builtin; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.springframework.ai.chat.model.ToolContext; +import org.springframework.ai.support.ToolCallbacks; + +import java.lang.reflect.Method; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class BrowserUseToolContractTest { + + @Test + @DisplayName("CDP params are exposed to the model as a structured object") + void cdpParamsUseStructuredMapSchema() throws NoSuchMethodException { + Method method = BrowserUseTool.class.getMethod("browser_use", + String.class, String.class, String.class, String.class, String.class, + String.class, String.class, Integer.class, String.class, String.class, + Map.class, String.class, Boolean.class, Integer.class, ToolContext.class); + + assertEquals(Map.class, method.getParameterTypes()[10]); + } + + @Test + @DisplayName("Spring AI publishes CDP params as a JSON object schema") + void generatedToolSchemaUsesObjectParams() { + BrowserUseTool tool = new BrowserUseTool(null, null, null, null, null); + + String schema = ToolCallbacks.from(tool)[0].getToolDefinition().inputSchema(); + + assertTrue(schema.matches("(?s).*\\\"params\\\"\\s*:\\s*\\{.*?\\\"type\\\"\\s*:\\s*\\\"object\\\".*"), schema); + } +} diff --git a/pom.xml b/pom.xml index 0c2545e4..52aa93eb 100644 --- a/pom.xml +++ b/pom.xml @@ -51,7 +51,7 @@ 3.5.4 - 1.59.0 + 1.62.0 2.2.2