mirror of
https://gitee.com/mateos/mateclaw.git
synced 2026-09-13 03:13:41 +08:00
feat(browser): harden browser automation reliability
This commit is contained in:
parent
0ae8552f5a
commit
ba0e4506b7
@ -78,7 +78,7 @@ RUN mvn -pl mateclaw-server -am package -Dmaven.test.skip=true -q ${MAVEN_FLAGS}
|
||||
# bump the Java dependency, bump this tag in lockstep — Microsoft rebuilds each
|
||||
# tag with the matching driver, so mismatched versions cause the java driver to
|
||||
# re-download browsers at runtime (defeating the whole point of this image).
|
||||
FROM mcr.microsoft.com/playwright:v1.59.0-noble
|
||||
FROM mcr.microsoft.com/playwright:v1.62.0-noble
|
||||
WORKDIR /app
|
||||
|
||||
# JDK 21 is NOT part of the base image (it ships Node for the JS driver).
|
||||
|
||||
@ -1776,7 +1776,7 @@ public class AgentGraphBuilder {
|
||||
|
||||
## ProgressLedger Discipline (mandatory)
|
||||
The `## 当前任务进度` block injected near the top of every turn is the **authoritative record** of what you have done and what remains. Treat it as ground truth, not as a scratchpad you may ignore.
|
||||
- **On starting any multi-step task** (≥3 tool calls expected), call `progress_update` in a parallel tool_calls batch to register every pending step BEFORE doing the work. Do not wait until "later" — context compression can trim earlier turns and you will lose track.
|
||||
- **On starting any multi-step task** (≥3 tool calls expected), call `progress_update` in parallel batches of at most 16 calls to register every pending step BEFORE doing the work. Split larger ledgers across turns so the executor cap never drops entries.
|
||||
- **After each completed sub-step**, immediately call `progress_update` to flip its status to `done`. "Immediately" means in the same tool_calls batch that returns the result, not after the next reasoning turn.
|
||||
- **Never re-execute a step the ledger shows as `done`** unless you can articulate why the prior result is stale.
|
||||
- **🔒 固定约束 entries** (pinned from skill manifests) are non-negotiable. They survive context compression for a reason — re-read them every turn and make sure your planned action still satisfies them.
|
||||
|
||||
@ -257,8 +257,8 @@ public class ReasoningNode implements NodeAction {
|
||||
+ "\"调研 10 个模型\"、\"逐节起草报告\"、\"批量生成 N 份文档\"、\n"
|
||||
+ "\"依次调用 N 个 API\"、\"对每个文件执行同一操作\"等。\n\n"
|
||||
+ "**必须做的事**:\n"
|
||||
+ "1. **第一轮回复就用并行 tool_calls 批量注册全部子目标为 `pending`**\n"
|
||||
+ " 一条回复里 N 个 `progress_update` 同时发出(不要串行)。\n"
|
||||
+ "1. **第一轮回复就用并行 tool_calls 批量注册子目标为 `pending`**\n"
|
||||
+ " 每批最多 16 个 `progress_update`;超过 16 个时分批注册,避免超过执行器上限。\n"
|
||||
+ " 例:要调研 10 个模型,第一轮就发 10 个 `progress_update(stepKey=\"model_xxx\", status=\"pending\")`。\n"
|
||||
+ "2. **每开始一个子目标**前发 `progress_update(同 stepKey, status=\"in_progress\")`。\n"
|
||||
+ "3. **每完成一个子目标**后立即发 `progress_update(同 stepKey, status=\"done\")`。\n"
|
||||
|
||||
@ -0,0 +1,49 @@
|
||||
package vip.mate.tool.browser;
|
||||
|
||||
import com.google.gson.JsonElement;
|
||||
import com.google.gson.JsonObject;
|
||||
|
||||
import java.util.Collection;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
/**
|
||||
* Shared URL safety checks for browser navigation surfaces beyond action=open.
|
||||
*/
|
||||
public final class BrowserNavigationGuard {
|
||||
|
||||
private static final Pattern URL_LITERAL =
|
||||
Pattern.compile("['\"](https?://[^'\"\\s)]+)['\"]", Pattern.CASE_INSENSITIVE);
|
||||
|
||||
private static final Pattern EVAL_NAVIGATION_INTENT = Pattern.compile(
|
||||
"\\b(location(?:\\.href|\\.assign|\\.replace)?|window\\.open|fetch|XMLHttpRequest)\\b",
|
||||
Pattern.CASE_INSENSITIVE);
|
||||
|
||||
private BrowserNavigationGuard() {
|
||||
}
|
||||
|
||||
public static void checkCdp(String method, JsonObject params, Collection<String> allowlist,
|
||||
boolean allowPrivateNetwork) {
|
||||
if (!"Page.navigate".equals(method) || params == null || !params.has("url")) {
|
||||
return;
|
||||
}
|
||||
JsonElement el = params.get("url");
|
||||
if (el == null || !el.isJsonPrimitive()) {
|
||||
return;
|
||||
}
|
||||
UrlSafetyChecker.check(el.getAsString(), allowlist, allowPrivateNetwork);
|
||||
}
|
||||
|
||||
public static void checkEval(String code, Collection<String> allowlist, boolean allowPrivateNetwork) {
|
||||
if (code == null || code.isBlank()) {
|
||||
return;
|
||||
}
|
||||
if (!EVAL_NAVIGATION_INTENT.matcher(code).find()) {
|
||||
return;
|
||||
}
|
||||
Matcher matcher = URL_LITERAL.matcher(code);
|
||||
while (matcher.find()) {
|
||||
UrlSafetyChecker.check(matcher.group(1), allowlist, allowPrivateNetwork);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,108 @@
|
||||
package vip.mate.tool.browser;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
/**
|
||||
* Thread-safe lifecycle state for snapshot references in one browser session.
|
||||
*/
|
||||
public final class BrowserRefState {
|
||||
|
||||
public enum Status {
|
||||
NONE,
|
||||
VALID,
|
||||
INVALIDATED
|
||||
}
|
||||
|
||||
private int snapshotGeneration;
|
||||
private long navigationEpoch;
|
||||
private long snapshotNavigationEpoch = -1;
|
||||
private String currentUrl = "";
|
||||
private String snapshotUrl = "";
|
||||
private Status status = Status.NONE;
|
||||
private Set<String> refs = Set.of();
|
||||
private Map<String, PageSnapshotScript.RefFingerprint> refInfos = Map.of();
|
||||
|
||||
public synchronized int recordSnapshot(
|
||||
String url,
|
||||
List<String> newRefs,
|
||||
Map<String, PageSnapshotScript.RefFingerprint> newRefInfos) {
|
||||
currentUrl = normalizeUrl(url);
|
||||
snapshotUrl = currentUrl;
|
||||
snapshotNavigationEpoch = navigationEpoch;
|
||||
refs = Set.copyOf(newRefs);
|
||||
refInfos = Map.copyOf(newRefInfos);
|
||||
status = Status.VALID;
|
||||
return ++snapshotGeneration;
|
||||
}
|
||||
|
||||
public synchronized void onMainFrameNavigated(String url) {
|
||||
currentUrl = normalizeUrl(url);
|
||||
navigationEpoch++;
|
||||
invalidateSnapshot();
|
||||
}
|
||||
|
||||
public synchronized void reconcileUrl(String url) {
|
||||
String normalized = normalizeUrl(url);
|
||||
if (!currentUrl.isEmpty() && !currentUrl.equals(normalized)) {
|
||||
onMainFrameNavigated(normalized);
|
||||
return;
|
||||
}
|
||||
currentUrl = normalized;
|
||||
}
|
||||
|
||||
public synchronized void invalidate() {
|
||||
invalidateSnapshot();
|
||||
}
|
||||
|
||||
private void invalidateSnapshot() {
|
||||
refs = Set.of();
|
||||
refInfos = Map.of();
|
||||
status = snapshotGeneration == 0 ? Status.NONE : Status.INVALIDATED;
|
||||
}
|
||||
|
||||
private static String normalizeUrl(String url) {
|
||||
return url == null ? "" : url;
|
||||
}
|
||||
|
||||
public synchronized Status status() {
|
||||
return status;
|
||||
}
|
||||
|
||||
public synchronized boolean refsValid() {
|
||||
return status == Status.VALID && snapshotNavigationEpoch == navigationEpoch;
|
||||
}
|
||||
|
||||
public synchronized boolean contains(String ref) {
|
||||
return refs.contains(ref);
|
||||
}
|
||||
|
||||
public synchronized PageSnapshotScript.RefFingerprint fingerprint(String ref) {
|
||||
return refInfos.get(ref);
|
||||
}
|
||||
|
||||
public synchronized int refCount() {
|
||||
return refs.size();
|
||||
}
|
||||
|
||||
public synchronized int snapshotGeneration() {
|
||||
return snapshotGeneration;
|
||||
}
|
||||
|
||||
public synchronized long navigationEpoch() {
|
||||
return navigationEpoch;
|
||||
}
|
||||
|
||||
public synchronized long snapshotNavigationEpoch() {
|
||||
return snapshotNavigationEpoch;
|
||||
}
|
||||
|
||||
public synchronized String currentUrl() {
|
||||
return currentUrl;
|
||||
}
|
||||
|
||||
public synchronized String snapshotUrl() {
|
||||
return snapshotUrl;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,34 @@
|
||||
package vip.mate.tool.browser;
|
||||
|
||||
import java.util.concurrent.locks.ReentrantLock;
|
||||
|
||||
/**
|
||||
* Fixed-size striped lock gate for serializing operations on one browser session.
|
||||
*/
|
||||
public final class BrowserSessionGate {
|
||||
|
||||
private final ReentrantLock[] locks;
|
||||
|
||||
public BrowserSessionGate(int stripes) {
|
||||
if (stripes <= 0) {
|
||||
throw new IllegalArgumentException("stripes must be positive");
|
||||
}
|
||||
locks = new ReentrantLock[stripes];
|
||||
for (int i = 0; i < stripes; i++) {
|
||||
locks[i] = new ReentrantLock(true);
|
||||
}
|
||||
}
|
||||
|
||||
public Lease enter(String sessionKey) {
|
||||
int index = Math.floorMod(sessionKey.hashCode(), locks.length);
|
||||
ReentrantLock lock = locks[index];
|
||||
lock.lock();
|
||||
return lock::unlock;
|
||||
}
|
||||
|
||||
@FunctionalInterface
|
||||
public interface Lease extends AutoCloseable {
|
||||
@Override
|
||||
void close();
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,52 @@
|
||||
package vip.mate.tool.browser;
|
||||
|
||||
/**
|
||||
* Parsed, bounded wait request for browser_use action=wait_for.
|
||||
*/
|
||||
public record BrowserWaitCondition(Kind kind, String target, int timeoutMillis) {
|
||||
|
||||
public enum Kind {
|
||||
SELECTOR,
|
||||
TEXT,
|
||||
URL,
|
||||
LOAD_STATE
|
||||
}
|
||||
|
||||
public static BrowserWaitCondition parse(String condition, String selector, String text, String value,
|
||||
Integer timeoutSeconds, int maxTimeoutSeconds) {
|
||||
if (condition == null || condition.isBlank()) {
|
||||
throw new IllegalArgumentException("condition is required for action=wait_for");
|
||||
}
|
||||
Kind kind = switch (condition.trim().toLowerCase()) {
|
||||
case "selector" -> Kind.SELECTOR;
|
||||
case "text" -> Kind.TEXT;
|
||||
case "url" -> Kind.URL;
|
||||
case "load_state", "loadstate", "state" -> Kind.LOAD_STATE;
|
||||
default -> throw new IllegalArgumentException("Unknown wait_for condition: " + condition
|
||||
+ ". Supported: selector, text, url, load_state");
|
||||
};
|
||||
|
||||
String target = switch (kind) {
|
||||
case SELECTOR -> firstNonBlank(selector);
|
||||
case TEXT -> firstNonBlank(text, value);
|
||||
case URL -> firstNonBlank(value, text);
|
||||
case LOAD_STATE -> firstNonBlank(value, text);
|
||||
};
|
||||
if (target == null) {
|
||||
throw new IllegalArgumentException("Target is required for wait_for condition=" + condition);
|
||||
}
|
||||
|
||||
int max = Math.max(1, maxTimeoutSeconds);
|
||||
int requested = timeoutSeconds == null ? max : Math.max(1, timeoutSeconds);
|
||||
return new BrowserWaitCondition(kind, target, Math.min(requested, max) * 1000);
|
||||
}
|
||||
|
||||
private static String firstNonBlank(String... values) {
|
||||
for (String value : values) {
|
||||
if (value != null && !value.isBlank()) {
|
||||
return value.trim();
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@ -5,7 +5,10 @@ import cn.hutool.json.JSONObject;
|
||||
import cn.hutool.json.JSONUtil;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
|
||||
/**
|
||||
* Accessibility-tree page snapshot for browser automation.
|
||||
@ -51,6 +54,7 @@ public final class PageSnapshotScript {
|
||||
const budget = { remaining: maxLen, truncated: false };
|
||||
let counter = 0;
|
||||
const refs = [];
|
||||
const refInfos = [];
|
||||
|
||||
// Wipe references from a prior snapshot so ids never collide
|
||||
// across generations and a navigated-away page leaves nothing behind.
|
||||
@ -141,9 +145,30 @@ public final class PageSnapshotScript {
|
||||
return txt;
|
||||
}
|
||||
|
||||
function clip(s, n) {
|
||||
function normalizeName(s) {
|
||||
if (!s) return '';
|
||||
return s.length > n ? s.substring(0, n) + '…' : s;
|
||||
return s.length > 100 ? s.substring(0, 100) + '…' : s;
|
||||
}
|
||||
|
||||
function stateOf(el, ref, role, name) {
|
||||
const tag = el.tagName.toLowerCase();
|
||||
const info = {
|
||||
ref: ref,
|
||||
role: role || '',
|
||||
name: name || '',
|
||||
tag: tag,
|
||||
type: el.getAttribute('type') || '',
|
||||
href: el.getAttribute('href') || '',
|
||||
value: '',
|
||||
checked: !!el.checked,
|
||||
selected: !!el.selected,
|
||||
disabled: !!el.disabled || el.getAttribute('aria-disabled') === 'true',
|
||||
expanded: el.getAttribute('aria-expanded') === null ? null : el.getAttribute('aria-expanded') === 'true'
|
||||
};
|
||||
if (tag === 'input' || tag === 'textarea' || tag === 'select') {
|
||||
info.value = String(el.value || '');
|
||||
}
|
||||
return info;
|
||||
}
|
||||
|
||||
const lines = [];
|
||||
@ -173,10 +198,11 @@ public final class PageSnapshotScript {
|
||||
const ref = 'e' + counter;
|
||||
el.setAttribute('data-mate-ref', ref);
|
||||
refs.push(ref);
|
||||
const nm = clip(nameOf(el), 100);
|
||||
const nm = normalizeName(nameOf(el));
|
||||
refInfos.push(stateOf(el, ref, role, nm));
|
||||
line = role + (nm ? ' "' + nm + '"' : '') + ' @' + ref;
|
||||
} else if (includeNon && role && role !== 'generic') {
|
||||
const nm = clip(nameOf(el), 100);
|
||||
const nm = normalizeName(nameOf(el));
|
||||
if (nm || role === 'list' || role === 'navigation') {
|
||||
let extra = '';
|
||||
if (role === 'heading') {
|
||||
@ -200,12 +226,91 @@ public final class PageSnapshotScript {
|
||||
}
|
||||
|
||||
walk(rootEl, 0);
|
||||
return JSON.stringify({ tree: lines.join('\\n'), truncated: budget.truncated, refs: refs });
|
||||
return JSON.stringify({ tree: lines.join('\\n'), truncated: budget.truncated, refs: refs, refInfos: refInfos });
|
||||
}
|
||||
""";
|
||||
|
||||
/** Evaluate on an element handle to capture the same core fingerprint used by snapshots. */
|
||||
public static final String REF_FINGERPRINT_JS = """
|
||||
(el, ref) => {
|
||||
function nameOf(el) {
|
||||
const aria = el.getAttribute('aria-label');
|
||||
if (aria) return aria.trim();
|
||||
const labelledby = el.getAttribute('aria-labelledby');
|
||||
if (labelledby) {
|
||||
const target = document.getElementById(labelledby);
|
||||
if (target) return (target.textContent || '').trim();
|
||||
}
|
||||
const tag = el.tagName.toLowerCase();
|
||||
if (tag === 'input' || tag === 'textarea') {
|
||||
const ph = el.getAttribute('placeholder');
|
||||
if (ph) return ph.trim();
|
||||
if (el.value) return String(el.value).trim();
|
||||
if (el.id) {
|
||||
const lab = document.querySelector('label[for="' + (window.CSS ? CSS.escape(el.id) : el.id) + '"]');
|
||||
if (lab) return (lab.textContent || '').trim();
|
||||
}
|
||||
const wrap = el.closest('label');
|
||||
if (wrap) {
|
||||
const wt = (wrap.textContent || '').trim().replace(/\\s+/g, ' ');
|
||||
if (wt) return wt;
|
||||
}
|
||||
return '';
|
||||
}
|
||||
if (tag === 'img') {
|
||||
const alt = el.getAttribute('alt');
|
||||
if (alt) return alt.trim();
|
||||
}
|
||||
const title = el.getAttribute('title');
|
||||
if (title) return title.trim();
|
||||
return el.textContent ? el.textContent.trim().replace(/\\s+/g, ' ').substring(0, 100) : '';
|
||||
}
|
||||
function roleOf(el) {
|
||||
const explicit = el.getAttribute('role');
|
||||
if (explicit) return explicit;
|
||||
const tag = el.tagName.toLowerCase();
|
||||
switch (tag) {
|
||||
case 'a': return el.hasAttribute('href') ? 'link' : 'generic';
|
||||
case 'button': return 'button';
|
||||
case 'select': return 'combobox';
|
||||
case 'textarea': return 'textbox';
|
||||
case 'summary': return 'button';
|
||||
case 'input': {
|
||||
const t = (el.getAttribute('type') || 'text').toLowerCase();
|
||||
if (t === 'checkbox') return 'checkbox';
|
||||
if (t === 'radio') return 'radio';
|
||||
if (t === 'submit' || t === 'button' || t === 'reset') return 'button';
|
||||
if (t === 'search') return 'searchbox';
|
||||
if (t === 'hidden') return null;
|
||||
return 'textbox';
|
||||
}
|
||||
default: return null;
|
||||
}
|
||||
}
|
||||
function normalizeName(s) {
|
||||
if (!s) return '';
|
||||
return s.length > 100 ? s.substring(0, 100) + '…' : s;
|
||||
}
|
||||
const tag = el.tagName.toLowerCase();
|
||||
return JSON.stringify({
|
||||
ref: ref || el.getAttribute('data-mate-ref') || '',
|
||||
role: roleOf(el) || '',
|
||||
name: normalizeName(nameOf(el)),
|
||||
tag: tag,
|
||||
type: el.getAttribute('type') || '',
|
||||
href: el.getAttribute('href') || '',
|
||||
value: (tag === 'input' || tag === 'textarea' || tag === 'select') ? String(el.value || '') : '',
|
||||
checked: !!el.checked,
|
||||
selected: !!el.selected,
|
||||
disabled: !!el.disabled || el.getAttribute('aria-disabled') === 'true',
|
||||
expanded: el.getAttribute('aria-expanded') === null ? null : el.getAttribute('aria-expanded') === 'true'
|
||||
});
|
||||
}
|
||||
""";
|
||||
|
||||
/** Parsed result of a snapshot evaluation. */
|
||||
public record Result(String tree, boolean truncated, List<String> refs) {
|
||||
public record Result(String tree, boolean truncated, List<String> refs,
|
||||
Map<String, RefFingerprint> refInfos) {
|
||||
public static Result fromJson(String json) {
|
||||
JSONObject obj = JSONUtil.parseObj(json);
|
||||
String tree = obj.getStr("tree", "");
|
||||
@ -219,7 +324,52 @@ public final class PageSnapshotScript {
|
||||
}
|
||||
}
|
||||
}
|
||||
return new Result(tree, truncated, refs);
|
||||
Map<String, RefFingerprint> refInfos = new LinkedHashMap<>();
|
||||
JSONArray infoArr = obj.getJSONArray("refInfos");
|
||||
if (infoArr != null) {
|
||||
for (Object o : infoArr) {
|
||||
if (o instanceof JSONObject info) {
|
||||
RefFingerprint fp = RefFingerprint.fromJson(info);
|
||||
if (fp.ref() != null && !fp.ref().isBlank()) {
|
||||
refInfos.put(fp.ref(), fp);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return new Result(tree, truncated, refs, Map.copyOf(refInfos));
|
||||
}
|
||||
}
|
||||
|
||||
public record RefFingerprint(String ref, String role, String name, String tag, String type,
|
||||
String href, String value, boolean checked, boolean selected,
|
||||
boolean disabled, Boolean expanded) {
|
||||
public static RefFingerprint fromJson(JSONObject obj) {
|
||||
return new RefFingerprint(
|
||||
obj.getStr("ref", ""),
|
||||
obj.getStr("role", ""),
|
||||
obj.getStr("name", ""),
|
||||
obj.getStr("tag", ""),
|
||||
obj.getStr("type", ""),
|
||||
obj.getStr("href", ""),
|
||||
obj.getStr("value", ""),
|
||||
obj.getBool("checked", false),
|
||||
obj.getBool("selected", false),
|
||||
obj.getBool("disabled", false),
|
||||
obj.get("expanded") == null ? null : obj.getBool("expanded", false));
|
||||
}
|
||||
|
||||
public boolean sameCoreIdentity(RefFingerprint other) {
|
||||
if (other == null) {
|
||||
return false;
|
||||
}
|
||||
return Objects.equals(normalize(role), normalize(other.role))
|
||||
&& Objects.equals(normalize(name), normalize(other.name))
|
||||
&& Objects.equals(normalize(tag), normalize(other.tag))
|
||||
&& Objects.equals(normalize(type), normalize(other.type));
|
||||
}
|
||||
|
||||
private static String normalize(String value) {
|
||||
return value == null ? "" : value.trim().replaceAll("\\s+", " ").toLowerCase();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -5,15 +5,18 @@ import cn.hutool.json.JSONArray;
|
||||
import cn.hutool.json.JSONObject;
|
||||
import cn.hutool.json.JSONUtil;
|
||||
import com.google.gson.JsonObject;
|
||||
import com.google.gson.JsonParser;
|
||||
import com.google.gson.Gson;
|
||||
import com.microsoft.playwright.Browser;
|
||||
import com.microsoft.playwright.BrowserContext;
|
||||
import com.microsoft.playwright.CDPSession;
|
||||
import com.microsoft.playwright.ConsoleMessage;
|
||||
import com.microsoft.playwright.ElementHandle;
|
||||
import com.microsoft.playwright.Locator;
|
||||
import com.microsoft.playwright.Page;
|
||||
import com.microsoft.playwright.Playwright;
|
||||
import com.microsoft.playwright.PlaywrightException;
|
||||
import com.microsoft.playwright.options.LoadState;
|
||||
import com.microsoft.playwright.options.WaitForSelectorState;
|
||||
import jakarta.annotation.PreDestroy;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.ai.chat.model.ToolContext;
|
||||
@ -23,7 +26,11 @@ import org.springframework.lang.Nullable;
|
||||
import org.springframework.stereotype.Component;
|
||||
import vip.mate.tool.browser.BrowserDiagnosticsService;
|
||||
import vip.mate.tool.browser.BrowserLauncher;
|
||||
import vip.mate.tool.browser.BrowserNavigationGuard;
|
||||
import vip.mate.tool.browser.BrowserPrivacyGuard;
|
||||
import vip.mate.tool.browser.BrowserRefState;
|
||||
import vip.mate.tool.browser.BrowserSessionGate;
|
||||
import vip.mate.tool.browser.BrowserWaitCondition;
|
||||
import vip.mate.common.net.SsrfProperties;
|
||||
import vip.mate.tool.browser.PageSnapshotScript;
|
||||
import vip.mate.tool.browser.UrlSafetyChecker;
|
||||
@ -32,6 +39,7 @@ import java.net.Socket;
|
||||
import java.nio.file.Paths;
|
||||
import java.util.Base64;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.*;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
@ -47,6 +55,7 @@ public class BrowserUseTool {
|
||||
private static final long IDLE_TIMEOUT_MINUTES = 30;
|
||||
private static final int CDP_SCAN_PORT_MIN = 9000;
|
||||
private static final int CDP_SCAN_PORT_MAX = 10000;
|
||||
private static final Gson GSON = new Gson();
|
||||
|
||||
/**
|
||||
* Legacy visible-text extractor kept as a fallback: {@link #doSnapshot}
|
||||
@ -146,7 +155,7 @@ public class BrowserUseTool {
|
||||
if (reason == null) {
|
||||
return null;
|
||||
}
|
||||
String conversationId = ToolExecutionContext.conversationId(currentToolContext);
|
||||
String conversationId = ToolExecutionContext.conversationId(currentToolContext.get());
|
||||
privacyGuard.audit(conversationId, action, session.page.url(), reason);
|
||||
log.info("[BrowserUse] Privacy guard blocked action={} on {}", action, session.page.url());
|
||||
return error(reason);
|
||||
@ -163,13 +172,12 @@ public class BrowserUseTool {
|
||||
private final ConcurrentHashMap<String, BrowserSession> sessions = new ConcurrentHashMap<>();
|
||||
|
||||
/**
|
||||
* RFC-063r §2.5 transition: ToolContext for the current invocation, set
|
||||
* at the @Tool entry point and read by {@link #broadcastBrowserEvent}.
|
||||
* Tool calls are serialized per ToolExecutionExecutor instance so this
|
||||
* volatile field is safe; the field is read-only inside the action
|
||||
* handlers.
|
||||
* Invocation context is thread-local. The shared Playwright driver is
|
||||
* guarded globally because Playwright Java permits multi-threaded callers
|
||||
* only when no two threads invoke its objects at the same time.
|
||||
*/
|
||||
private volatile ToolContext currentToolContext;
|
||||
private final ThreadLocal<ToolContext> currentToolContext = new ThreadLocal<>();
|
||||
private final BrowserSessionGate sessionGate = new BrowserSessionGate(1);
|
||||
private final ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(r -> {
|
||||
Thread t = new Thread(r, "browser-idle-watchdog");
|
||||
t.setDaemon(true);
|
||||
@ -179,7 +187,7 @@ public class BrowserUseTool {
|
||||
@Tool(description = """
|
||||
Control a browser (Playwright with multi-strategy launch: system Chrome/Edge channel, explicit path, bundled, or external CDP).
|
||||
Default is headless. Use headed=true with action=start for a visible window.
|
||||
Typical flow: start → open(url) → snapshot → click/type → stop.
|
||||
Typical flow: start → open(url) → snapshot → click/type → current_surface or wait_for → stop.
|
||||
If start fails, run action=diagnose for a full report of what's missing and how to fix it.
|
||||
|
||||
SCOPE — use this tool ONLY for tasks that require driving a real browser:
|
||||
@ -199,6 +207,8 @@ public class BrowserUseTool {
|
||||
`selector` scopes to a subtree — USE IT when the page is large to avoid truncation
|
||||
(`truncated:true` is flagged with a hint).
|
||||
- screenshot: Take a screenshot. Optional path to save file; returns base64 if no path.
|
||||
- current_surface: Return current URL/title/document readiness, recent console/page errors, and ref validity.
|
||||
- wait_for: Wait for condition=selector|text|url|load_state. Use selector/text/value plus optional timeoutSeconds.
|
||||
- click: Click an element. Pass ref=<eN> from a snapshot (preferred) or a CSS selector.
|
||||
- type: Type text into an element. Pass ref=<eN> (preferred) or selector, plus text.
|
||||
- hover: Hover over an element (reveals menus/tooltips). Pass ref=<eN> or selector.
|
||||
@ -211,26 +221,23 @@ public class BrowserUseTool {
|
||||
- diagnose: Run a self-check — reports which launch strategies are available and what to install if none are.
|
||||
""")
|
||||
public String browser_use(
|
||||
@ToolParam(description = "Action: start|stop|open|snapshot|screenshot|click|type|hover|select|eval|connect_cdp|list_cdp_targets|navigate_back|diagnose") String action,
|
||||
@ToolParam(description = "Action: start|stop|open|snapshot|screenshot|current_surface|wait_for|click|type|hover|select|eval|connect_cdp|list_cdp_targets|navigate_back|diagnose") String action,
|
||||
@ToolParam(description = "URL to navigate to (for open), or CDP base URL (for connect_cdp, e.g. http://localhost:9222)", required = false) String url,
|
||||
@ToolParam(description = "CSS selector. Alternative to ref for click/type/hover/select. OPTIONAL for snapshot: pass to scope to a subtree when previous snapshot returned truncated:true.", required = false) String selector,
|
||||
@ToolParam(description = "Element reference from a snapshot (e.g. 'e4'). PREFERRED for click/type/hover/select — takes priority over selector. Re-snapshot if it reports stale.", required = false) String ref,
|
||||
@ToolParam(description = "Text to type (for action=type)", required = false) String text,
|
||||
@ToolParam(description = "Option value or visible label to choose (for action=select)", required = false) String value,
|
||||
@ToolParam(description = "Wait condition for action=wait_for: selector|text|url|load_state", required = false) String condition,
|
||||
@ToolParam(description = "Timeout seconds for action=wait_for; capped by mateclaw.browser.default-timeout-seconds", required = false) Integer timeoutSeconds,
|
||||
@ToolParam(description = "JavaScript code to execute (for action=eval). Top-level await is allowed; add `return` to return a value when the snippet uses await.", required = false) String code,
|
||||
@ToolParam(description = "CDP method for action=cdp (e.g. 'Page.navigate', 'Input.dispatchMouseEvent'). Must be in the allowlist.", required = false) String method,
|
||||
@ToolParam(description = "JSON object of params for action=cdp (e.g. {\"url\":\"https://example.com\"})", required = false) String params,
|
||||
@ToolParam(description = "Structured params object for action=cdp (e.g. {\"url\":\"https://example.com\"})", required = false) Map<String, Object> params,
|
||||
@ToolParam(description = "File path to save screenshot (for action=screenshot)", required = false) String path,
|
||||
@ToolParam(description = "Launch visible browser window (for action=start, default false)", required = false) Boolean headed,
|
||||
@ToolParam(description = "Single CDP port to scan (for action=list_cdp_targets)", required = false) Integer cdpPort,
|
||||
// RFC-063r §2.5: hidden from LLM by JsonSchemaGenerator.
|
||||
@Nullable ToolContext ctx
|
||||
) {
|
||||
// The conversationId resolution lives in broadcastBrowserEvent below;
|
||||
// capture the ctx into a field so the helper can read it without
|
||||
// passing it down every action handler. Race-free because tool calls
|
||||
// are serialized per executor.
|
||||
this.currentToolContext = ctx;
|
||||
if (action == null || action.isBlank()) {
|
||||
return error("action is required");
|
||||
}
|
||||
@ -244,31 +251,38 @@ public class BrowserUseTool {
|
||||
log.info("[BrowserUse] action={}, session={}, url={}, selector={}, headed={}, cdpPort={}",
|
||||
action, sessionKey, url, selector, headed, cdpPort);
|
||||
|
||||
try {
|
||||
return switch (action.toLowerCase().trim()) {
|
||||
case "start" -> doStart(sessionKey, Boolean.TRUE.equals(headed));
|
||||
case "stop" -> doStop(sessionKey);
|
||||
case "open" -> doOpen(sessionKey, url);
|
||||
case "snapshot" -> doSnapshot(sessionKey, selector);
|
||||
case "screenshot" -> doScreenshot(sessionKey, path);
|
||||
case "click" -> doClick(sessionKey, ref, selector);
|
||||
case "type" -> doType(sessionKey, ref, selector, text);
|
||||
case "hover" -> doHover(sessionKey, ref, selector);
|
||||
case "select" -> doSelect(sessionKey, ref, selector, value);
|
||||
case "eval" -> doEval(sessionKey, code);
|
||||
case "cdp" -> doCdp(sessionKey, method, params);
|
||||
case "connect_cdp" -> doConnectCdp(sessionKey, url);
|
||||
case "list_cdp_targets" -> doListCdpTargets(cdpPort);
|
||||
case "navigate_back" -> doNavigateBack(sessionKey);
|
||||
case "diagnose" -> doDiagnose();
|
||||
default -> error("Unknown action: " + action + ". Supported: start, stop, open, snapshot, screenshot, click, type, hover, select, eval, cdp, connect_cdp, list_cdp_targets, navigate_back, diagnose");
|
||||
};
|
||||
} catch (PlaywrightException e) {
|
||||
log.error("[BrowserUse] Playwright error: {}", e.getMessage());
|
||||
return error("Browser error: " + e.getMessage());
|
||||
} catch (Exception e) {
|
||||
log.error("[BrowserUse] Unexpected error: {}", e.getMessage(), e);
|
||||
return error("Unexpected error: " + e.getMessage());
|
||||
try (BrowserSessionGate.Lease ignored = sessionGate.enter(sessionKey)) {
|
||||
currentToolContext.set(ctx);
|
||||
try {
|
||||
return switch (action.toLowerCase().trim()) {
|
||||
case "start" -> doStart(sessionKey, Boolean.TRUE.equals(headed));
|
||||
case "stop" -> doStop(sessionKey);
|
||||
case "open" -> doOpen(sessionKey, url);
|
||||
case "snapshot" -> doSnapshot(sessionKey, selector);
|
||||
case "screenshot" -> doScreenshot(sessionKey, path);
|
||||
case "current_surface" -> doCurrentSurface(sessionKey);
|
||||
case "wait_for" -> doWaitFor(sessionKey, condition, selector, text, value, timeoutSeconds);
|
||||
case "click" -> doClick(sessionKey, ref, selector);
|
||||
case "type" -> doType(sessionKey, ref, selector, text);
|
||||
case "hover" -> doHover(sessionKey, ref, selector);
|
||||
case "select" -> doSelect(sessionKey, ref, selector, value);
|
||||
case "eval" -> doEval(sessionKey, code);
|
||||
case "cdp" -> doCdp(sessionKey, method, params);
|
||||
case "connect_cdp" -> doConnectCdp(sessionKey, url);
|
||||
case "list_cdp_targets" -> doListCdpTargets(cdpPort);
|
||||
case "navigate_back" -> doNavigateBack(sessionKey);
|
||||
case "diagnose" -> doDiagnose();
|
||||
default -> error("Unknown action: " + action + ". Supported: start, stop, open, snapshot, screenshot, current_surface, wait_for, click, type, hover, select, eval, cdp, connect_cdp, list_cdp_targets, navigate_back, diagnose");
|
||||
};
|
||||
} catch (PlaywrightException e) {
|
||||
log.error("[BrowserUse] Playwright error: {}", e.getMessage());
|
||||
return error("Browser error: " + e.getMessage());
|
||||
} catch (Exception e) {
|
||||
log.error("[BrowserUse] Unexpected error: {}", e.getMessage(), e);
|
||||
return error("Unexpected error: " + e.getMessage());
|
||||
} finally {
|
||||
currentToolContext.remove();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -320,7 +334,7 @@ public class BrowserUseTool {
|
||||
*/
|
||||
private void broadcastBrowserEvent(String action, boolean success, String url, String title,
|
||||
String screenshot, long durationMs) {
|
||||
String conversationId = ToolExecutionContext.conversationId(currentToolContext);
|
||||
String conversationId = ToolExecutionContext.conversationId(currentToolContext.get());
|
||||
if (conversationId == null || streamTracker == null) {
|
||||
return;
|
||||
}
|
||||
@ -597,6 +611,7 @@ public class BrowserUseTool {
|
||||
|
||||
String title = session.page.title();
|
||||
String url = session.page.url();
|
||||
session.refState.reconcileUrl(url);
|
||||
|
||||
log.info("[BrowserUse] Navigated back to: {} ({})", url, title);
|
||||
|
||||
@ -661,7 +676,9 @@ public class BrowserUseTool {
|
||||
String jsResult = (String) root.evaluate(PageSnapshotScript.SNAPSHOT_JS, opts);
|
||||
PageSnapshotScript.Result snap = PageSnapshotScript.Result.fromJson(jsResult);
|
||||
|
||||
int generation = session.nextSnapshotGeneration(snap.refs());
|
||||
String snapshotUrl = page.url();
|
||||
int generation = session.nextSnapshotGeneration(snapshotUrl, snap.refs(), snap.refInfos());
|
||||
result.set("url", snapshotUrl);
|
||||
|
||||
result.set("snapshotMode", "accessibility-tree");
|
||||
result.set("generation", generation);
|
||||
@ -679,6 +696,7 @@ public class BrowserUseTool {
|
||||
result.set("scopedTo", selector);
|
||||
}
|
||||
result.set("refCount", snap.refs().size());
|
||||
result.set("nativeAriaSnapshot", clippedNativeAriaSnapshot(page));
|
||||
result.set("content", snap.tree());
|
||||
return JSONUtil.toJsonPrettyStr(result);
|
||||
} catch (PlaywrightException e) {
|
||||
@ -750,6 +768,136 @@ public class BrowserUseTool {
|
||||
}
|
||||
}
|
||||
|
||||
private String doCurrentSurface(String sessionKey) {
|
||||
BrowserSession session = requireSession(sessionKey);
|
||||
if (session == null) {
|
||||
return error("No browser running. Use action=start first.");
|
||||
}
|
||||
String blocked = guardReadOrNull(session, "current_surface");
|
||||
if (blocked != null) {
|
||||
return blocked;
|
||||
}
|
||||
session.touch();
|
||||
return JSONUtil.toJsonPrettyStr(surfaceResult(session));
|
||||
}
|
||||
|
||||
private String doWaitFor(String sessionKey, String condition, String selector, String text,
|
||||
String value, Integer timeoutSeconds) {
|
||||
BrowserSession session = requireSession(sessionKey);
|
||||
if (session == null) {
|
||||
return error("No browser running. Use action=start first.");
|
||||
}
|
||||
String blocked = guardReadOrNull(session, "wait_for");
|
||||
if (blocked != null) {
|
||||
return blocked;
|
||||
}
|
||||
BrowserWaitCondition wait;
|
||||
try {
|
||||
wait = BrowserWaitCondition.parse(condition, selector, text, value, timeoutSeconds,
|
||||
launcher.properties().getDefaultTimeoutSeconds());
|
||||
} catch (IllegalArgumentException e) {
|
||||
return error(e.getMessage());
|
||||
}
|
||||
|
||||
session.touch();
|
||||
Page page = session.page;
|
||||
switch (wait.kind()) {
|
||||
case SELECTOR -> page.waitForSelector(wait.target(),
|
||||
new Page.WaitForSelectorOptions()
|
||||
.setState(WaitForSelectorState.VISIBLE)
|
||||
.setStrict(false)
|
||||
.setTimeout(wait.timeoutMillis()));
|
||||
case TEXT -> page.getByText(wait.target()).first().waitFor(
|
||||
new Locator.WaitForOptions()
|
||||
.setState(WaitForSelectorState.VISIBLE)
|
||||
.setTimeout(wait.timeoutMillis()));
|
||||
case URL -> page.waitForURL(wait.target(),
|
||||
new Page.WaitForURLOptions().setTimeout(wait.timeoutMillis()));
|
||||
case LOAD_STATE -> page.waitForLoadState(loadState(wait.target()),
|
||||
new Page.WaitForLoadStateOptions().setTimeout(wait.timeoutMillis()));
|
||||
}
|
||||
JSONObject result = surfaceResult(session);
|
||||
result.set("waitedFor", wait.kind().name().toLowerCase());
|
||||
result.set("waitTarget", wait.target());
|
||||
result.set("timeoutMillis", wait.timeoutMillis());
|
||||
return JSONUtil.toJsonPrettyStr(result);
|
||||
}
|
||||
|
||||
private JSONObject surfaceResult(BrowserSession session) {
|
||||
Page page = session.page;
|
||||
session.refState.reconcileUrl(page.url());
|
||||
JSONObject result = new JSONObject();
|
||||
result.set("ok", true);
|
||||
result.set("currentUrl", page.url());
|
||||
result.set("currentTitle", page.title());
|
||||
result.set("readyState", safeReadyState(page));
|
||||
result.set("snapshotGeneration", session.refState.snapshotGeneration());
|
||||
result.set("refStatus", session.refState.status().name().toLowerCase());
|
||||
result.set("refsValid", session.refState.refsValid());
|
||||
result.set("refCount", session.refState.refCount());
|
||||
result.set("navigationEpoch", session.refState.navigationEpoch());
|
||||
result.set("snapshotNavigationEpoch", session.refState.snapshotNavigationEpoch());
|
||||
result.set("snapshotUrl", session.refState.snapshotUrl());
|
||||
JSONArray console = new JSONArray();
|
||||
try {
|
||||
List<ConsoleMessage> messages = page.consoleMessages();
|
||||
int start = Math.max(0, messages.size() - 5);
|
||||
for (ConsoleMessage message : messages.subList(start, messages.size())) {
|
||||
JSONObject item = new JSONObject();
|
||||
item.set("type", message.type());
|
||||
item.set("text", message.text());
|
||||
item.set("timestamp", message.timestamp());
|
||||
console.add(item);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.debug("[BrowserUse] consoleMessages unavailable: {}", e.getMessage());
|
||||
}
|
||||
result.set("recentConsoleMessages", console);
|
||||
JSONArray errors = new JSONArray();
|
||||
try {
|
||||
List<String> pageErrors = page.pageErrors();
|
||||
int start = Math.max(0, pageErrors.size() - 5);
|
||||
for (String pageError : pageErrors.subList(start, pageErrors.size())) {
|
||||
errors.add(pageError);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.debug("[BrowserUse] pageErrors unavailable: {}", e.getMessage());
|
||||
}
|
||||
result.set("recentPageErrors", errors);
|
||||
return result;
|
||||
}
|
||||
|
||||
private static String safeReadyState(Page page) {
|
||||
try {
|
||||
Object ready = page.evaluate("document.readyState");
|
||||
return ready != null ? ready.toString() : "unknown";
|
||||
} catch (Exception e) {
|
||||
return "unknown";
|
||||
}
|
||||
}
|
||||
|
||||
private static String clippedNativeAriaSnapshot(Page page) {
|
||||
try {
|
||||
String snapshot = page.ariaSnapshot();
|
||||
if (snapshot == null) {
|
||||
return "";
|
||||
}
|
||||
return snapshot.length() > 4000 ? snapshot.substring(0, 4000) + "\n... [truncated]" : snapshot;
|
||||
} catch (Exception e) {
|
||||
return "";
|
||||
}
|
||||
}
|
||||
|
||||
private static LoadState loadState(String state) {
|
||||
return switch (state.trim().toLowerCase()) {
|
||||
case "load" -> LoadState.LOAD;
|
||||
case "domcontentloaded", "dom_content_loaded" -> LoadState.DOMCONTENTLOADED;
|
||||
case "networkidle", "network_idle" -> LoadState.NETWORKIDLE;
|
||||
default -> throw new IllegalArgumentException("Unknown load state: " + state
|
||||
+ ". Supported: load, domcontentloaded, networkidle");
|
||||
};
|
||||
}
|
||||
|
||||
/** Result of resolving a click/type/hover/select target: a selector, or an error to return. */
|
||||
private record TargetResolution(String selector, String label, String error) {
|
||||
static TargetResolution ok(String selector, String label) {
|
||||
@ -769,13 +917,30 @@ public class BrowserUseTool {
|
||||
private TargetResolution resolveTarget(BrowserSession session, String ref, String selector) {
|
||||
if (ref != null && !ref.isBlank()) {
|
||||
String r = ref.trim();
|
||||
if (!session.currentRefs.contains(r)) {
|
||||
if (!session.refState.contains(r)) {
|
||||
return TargetResolution.fail("ref '" + r + "' is not part of the current snapshot"
|
||||
+ " (generation " + session.snapshotGeneration + "). The page likely changed,"
|
||||
+ " (generation " + session.refState.snapshotGeneration() + "). The page likely changed,"
|
||||
+ " or you have not snapshotted since it did. Call action=snapshot first,"
|
||||
+ " then use a ref from that fresh result.");
|
||||
}
|
||||
return TargetResolution.ok(PageSnapshotScript.selectorForRef(r), r);
|
||||
String refSelector = PageSnapshotScript.selectorForRef(r);
|
||||
PageSnapshotScript.RefFingerprint expected = session.refState.fingerprint(r);
|
||||
if (expected != null) {
|
||||
ElementHandle element = session.page.querySelector(refSelector);
|
||||
if (element == null) {
|
||||
session.invalidateRefs();
|
||||
return TargetResolution.fail("ref '" + r + "' no longer exists on the page."
|
||||
+ " Call action=snapshot again before acting.");
|
||||
}
|
||||
PageSnapshotScript.RefFingerprint actual = liveFingerprint(element, r);
|
||||
if (!expected.sameCoreIdentity(actual)) {
|
||||
session.invalidateRefs();
|
||||
return TargetResolution.fail("ref '" + r + "' now points to a different element."
|
||||
+ " Expected " + expected.role() + " '" + expected.name() + "', got "
|
||||
+ actual.role() + " '" + actual.name() + "'. Call action=snapshot again.");
|
||||
}
|
||||
}
|
||||
return TargetResolution.ok(refSelector, r);
|
||||
}
|
||||
if (selector != null && !selector.isBlank()) {
|
||||
return TargetResolution.ok(selector, selector);
|
||||
@ -783,6 +948,11 @@ public class BrowserUseTool {
|
||||
return TargetResolution.fail("Either ref (from a snapshot, e.g. ref='e4') or a CSS selector is required.");
|
||||
}
|
||||
|
||||
private static PageSnapshotScript.RefFingerprint liveFingerprint(ElementHandle element, String ref) {
|
||||
String json = (String) element.evaluate(PageSnapshotScript.REF_FINGERPRINT_JS, ref);
|
||||
return PageSnapshotScript.RefFingerprint.fromJson(JSONUtil.parseObj(json));
|
||||
}
|
||||
|
||||
private String doClick(String sessionKey, String ref, String selector) {
|
||||
BrowserSession session = requireSession(sessionKey);
|
||||
if (session == null) {
|
||||
@ -797,6 +967,7 @@ public class BrowserUseTool {
|
||||
Page page = session.page;
|
||||
|
||||
String before = page.url();
|
||||
String beforeTitle = page.title();
|
||||
page.click(t.selector());
|
||||
page.waitForLoadState(LoadState.DOMCONTENTLOADED);
|
||||
|
||||
@ -804,9 +975,7 @@ public class BrowserUseTool {
|
||||
String url = page.url();
|
||||
// A navigation invalidates the snapshot references; a same-page click
|
||||
// (toggle, expand) keeps them so the model can act on more refs.
|
||||
if (!url.equals(before)) {
|
||||
session.invalidateRefs();
|
||||
}
|
||||
session.refState.reconcileUrl(url);
|
||||
|
||||
log.info("[BrowserUse] Clicked: {} (page now: {})", t.label(), url);
|
||||
broadcastBrowserEvent("click", true, url, title, null, 0);
|
||||
@ -816,6 +985,9 @@ public class BrowserUseTool {
|
||||
result.set("target", t.label());
|
||||
result.set("currentUrl", url);
|
||||
result.set("currentTitle", title);
|
||||
result.set("urlChanged", !url.equals(before));
|
||||
result.set("titleChanged", !title.equals(beforeTitle));
|
||||
result.set("refsValid", session.refState.refsValid());
|
||||
if (!url.equals(before)) {
|
||||
result.set("navigated", true);
|
||||
result.set("hint", "The page navigated — previous @eN refs are stale. Re-snapshot before acting.");
|
||||
@ -838,14 +1010,24 @@ public class BrowserUseTool {
|
||||
}
|
||||
|
||||
session.touch();
|
||||
String before = session.page.url();
|
||||
String beforeTitle = session.page.title();
|
||||
session.page.fill(t.selector(), text);
|
||||
String url = session.page.url();
|
||||
String title = session.page.title();
|
||||
session.refState.reconcileUrl(url);
|
||||
|
||||
log.info("[BrowserUse] Typed into: {} ({} chars)", t.label(), text.length());
|
||||
broadcastBrowserEvent("type", true, null, null, null, 0);
|
||||
broadcastBrowserEvent("type", true, url, title, null, 0);
|
||||
|
||||
JSONObject result = new JSONObject();
|
||||
result.set("ok", true);
|
||||
result.set("target", t.label());
|
||||
result.set("currentUrl", url);
|
||||
result.set("currentTitle", title);
|
||||
result.set("urlChanged", !url.equals(before));
|
||||
result.set("titleChanged", !title.equals(beforeTitle));
|
||||
result.set("refsValid", session.refState.refsValid());
|
||||
result.set("textLength", text.length());
|
||||
result.set("message", "Typed " + text.length() + " characters into " + t.label());
|
||||
return JSONUtil.toJsonPrettyStr(result);
|
||||
@ -862,14 +1044,24 @@ public class BrowserUseTool {
|
||||
}
|
||||
|
||||
session.touch();
|
||||
String before = session.page.url();
|
||||
String beforeTitle = session.page.title();
|
||||
session.page.hover(t.selector());
|
||||
String url = session.page.url();
|
||||
String title = session.page.title();
|
||||
session.refState.reconcileUrl(url);
|
||||
|
||||
log.info("[BrowserUse] Hovered: {}", t.label());
|
||||
broadcastBrowserEvent("hover", true, null, null, null, 0);
|
||||
broadcastBrowserEvent("hover", true, url, title, null, 0);
|
||||
|
||||
JSONObject result = new JSONObject();
|
||||
result.set("ok", true);
|
||||
result.set("target", t.label());
|
||||
result.set("currentUrl", url);
|
||||
result.set("currentTitle", title);
|
||||
result.set("urlChanged", !url.equals(before));
|
||||
result.set("titleChanged", !title.equals(beforeTitle));
|
||||
result.set("refsValid", session.refState.refsValid());
|
||||
result.set("message", "Hovered element: " + t.label()
|
||||
+ ". Re-snapshot to capture any menu/tooltip it revealed.");
|
||||
return JSONUtil.toJsonPrettyStr(result);
|
||||
@ -889,16 +1081,26 @@ public class BrowserUseTool {
|
||||
}
|
||||
|
||||
session.touch();
|
||||
String before = session.page.url();
|
||||
String beforeTitle = session.page.title();
|
||||
// Playwright matches by option value, label, or visible text, so a
|
||||
// human-readable value from the model works without extra hints.
|
||||
List<String> chosen = session.page.selectOption(t.selector(), value);
|
||||
String url = session.page.url();
|
||||
String title = session.page.title();
|
||||
session.refState.reconcileUrl(url);
|
||||
|
||||
log.info("[BrowserUse] Selected {} in {} -> {}", value, t.label(), chosen);
|
||||
broadcastBrowserEvent("select", true, null, null, null, 0);
|
||||
broadcastBrowserEvent("select", true, url, title, null, 0);
|
||||
|
||||
JSONObject result = new JSONObject();
|
||||
result.set("ok", true);
|
||||
result.set("target", t.label());
|
||||
result.set("currentUrl", url);
|
||||
result.set("currentTitle", title);
|
||||
result.set("urlChanged", !url.equals(before));
|
||||
result.set("titleChanged", !title.equals(beforeTitle));
|
||||
result.set("refsValid", session.refState.refsValid());
|
||||
result.set("selected", chosen);
|
||||
result.set("message", chosen.isEmpty()
|
||||
? "No option matched '" + value + "'. Re-snapshot and check the option labels."
|
||||
@ -933,6 +1135,16 @@ public class BrowserUseTool {
|
||||
if (blocked != null) {
|
||||
return blocked;
|
||||
}
|
||||
if (launcher.properties().isSsrfCheckEnabled()) {
|
||||
try {
|
||||
BrowserNavigationGuard.checkEval(code,
|
||||
ssrfProperties.getSsrfAllowlist(),
|
||||
launcher.properties().isAllowPrivateNetwork());
|
||||
} catch (SecurityException se) {
|
||||
log.warn("[BrowserUse] Eval navigation guard rejected script: {}", se.getMessage());
|
||||
return error(se.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
session.touch();
|
||||
Page page = session.page;
|
||||
@ -961,6 +1173,7 @@ public class BrowserUseTool {
|
||||
}
|
||||
}
|
||||
String resultStr = evalResult != null ? evalResult.toString() : "null";
|
||||
session.refState.reconcileUrl(page.url());
|
||||
|
||||
if (resultStr.length() > 10_000) {
|
||||
resultStr = resultStr.substring(0, 10_000) + "\n... [truncated]";
|
||||
@ -971,6 +1184,8 @@ public class BrowserUseTool {
|
||||
JSONObject result = new JSONObject();
|
||||
result.set("ok", true);
|
||||
result.set("result", resultStr);
|
||||
result.set("currentUrl", page.url());
|
||||
result.set("refsValid", session.refState.refsValid());
|
||||
return JSONUtil.toJsonPrettyStr(result);
|
||||
}
|
||||
|
||||
@ -1013,7 +1228,7 @@ public class BrowserUseTool {
|
||||
return false;
|
||||
}
|
||||
|
||||
private String doCdp(String sessionKey, String method, String paramsJson) {
|
||||
private String doCdp(String sessionKey, String method, Map<String, Object> params) {
|
||||
if (!launcher.properties().getCdp().isEnabled()) {
|
||||
return error("action=cdp is disabled (mateclaw.browser.cdp.enabled=false).");
|
||||
}
|
||||
@ -1034,18 +1249,22 @@ public class BrowserUseTool {
|
||||
String reason = privacyGuard.blockReason(session.isUserManagedBrowser(),
|
||||
session.page.url(), "cdp:" + m);
|
||||
if (reason != null) {
|
||||
privacyGuard.audit(ToolExecutionContext.conversationId(currentToolContext),
|
||||
privacyGuard.audit(ToolExecutionContext.conversationId(currentToolContext.get()),
|
||||
"cdp:" + m, session.page.url(), reason);
|
||||
return error(reason);
|
||||
}
|
||||
}
|
||||
|
||||
JsonObject parsed = null;
|
||||
if (paramsJson != null && !paramsJson.isBlank()) {
|
||||
JsonObject parsed = params == null ? null : GSON.toJsonTree(params).getAsJsonObject();
|
||||
if (launcher.properties().isSsrfCheckEnabled()) {
|
||||
try {
|
||||
parsed = JsonParser.parseString(paramsJson).getAsJsonObject();
|
||||
} catch (RuntimeException e) {
|
||||
return error("params must be a JSON object: " + e.getMessage());
|
||||
BrowserNavigationGuard.checkCdp(m, parsed,
|
||||
ssrfProperties.getSsrfAllowlist(),
|
||||
launcher.properties().isAllowPrivateNetwork());
|
||||
} catch (SecurityException se) {
|
||||
log.warn("[BrowserUse] CDP navigation guard rejected method={} params={}: {}",
|
||||
m, params, se.getMessage());
|
||||
return error(se.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
@ -1058,6 +1277,7 @@ public class BrowserUseTool {
|
||||
if (m.startsWith("Page.navigate") || m.equals("Page.navigateToHistoryEntry")) {
|
||||
session.invalidateRefs();
|
||||
}
|
||||
session.refState.reconcileUrl(session.page.url());
|
||||
String out = res != null ? res.toString() : "{}";
|
||||
if (out.length() > 10_000) {
|
||||
out = out.substring(0, 10_000) + "\n... [truncated]";
|
||||
@ -1140,7 +1360,13 @@ public class BrowserUseTool {
|
||||
long idleMinutes = (System.currentTimeMillis() - s.lastActivity) / 60_000;
|
||||
if (idleMinutes >= IDLE_TIMEOUT_MINUTES) {
|
||||
log.info("[BrowserUse] Idle timeout ({}min), stopping session: {}", idleMinutes, sessionKey);
|
||||
doStop(sessionKey);
|
||||
try (BrowserSessionGate.Lease ignored = sessionGate.enter(sessionKey)) {
|
||||
BrowserSession current = sessions.get(sessionKey);
|
||||
if (current != null && System.currentTimeMillis() - current.lastActivity
|
||||
>= TimeUnit.MINUTES.toMillis(IDLE_TIMEOUT_MINUTES)) {
|
||||
doStop(sessionKey);
|
||||
}
|
||||
}
|
||||
}
|
||||
}, IDLE_TIMEOUT_MINUTES, 5, TimeUnit.MINUTES);
|
||||
|
||||
@ -1219,23 +1445,19 @@ public class BrowserUseTool {
|
||||
volatile ScheduledFuture<?> idleWatchdog;
|
||||
|
||||
/**
|
||||
* Monotonic snapshot generation. Each {@code action=snapshot} bumps it and
|
||||
* replaces {@link #currentRefs} with the references assigned that pass.
|
||||
* A click/type by a reference not in {@link #currentRefs} is reported as
|
||||
* stale, prompting the caller to re-snapshot. Safe as plain volatile
|
||||
* because tool calls are serialized per executor.
|
||||
* Snapshot reference lifecycle, including navigation epochs and the
|
||||
* fingerprints assigned by the latest snapshot.
|
||||
*/
|
||||
volatile int snapshotGeneration;
|
||||
volatile java.util.Set<String> currentRefs = java.util.Set.of();
|
||||
final BrowserRefState refState = new BrowserRefState();
|
||||
|
||||
int nextSnapshotGeneration(java.util.List<String> refs) {
|
||||
this.currentRefs = java.util.Set.copyOf(refs);
|
||||
return ++this.snapshotGeneration;
|
||||
int nextSnapshotGeneration(String url, java.util.List<String> refs,
|
||||
Map<String, PageSnapshotScript.RefFingerprint> refInfos) {
|
||||
return refState.recordSnapshot(url, refs, refInfos);
|
||||
}
|
||||
|
||||
/** Drop all references — the DOM they pointed at is gone (navigation). */
|
||||
void invalidateRefs() {
|
||||
this.currentRefs = java.util.Set.of();
|
||||
refState.invalidate();
|
||||
}
|
||||
|
||||
/**
|
||||
@ -1259,6 +1481,12 @@ public class BrowserUseTool {
|
||||
this.userDataDir = userDataDir;
|
||||
this.ownedProcess = ownedProcess;
|
||||
this.lastActivity = System.currentTimeMillis();
|
||||
this.refState.reconcileUrl(page.url());
|
||||
page.onFrameNavigated(frame -> {
|
||||
if (frame == page.mainFrame()) {
|
||||
refState.onMainFrameNavigated(frame.url());
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
void touch() {
|
||||
|
||||
@ -98,7 +98,7 @@ Then comment out the `searxng` service block in `docker-compose.yml`. Make sure
|
||||
|
||||
### What the image actually contains
|
||||
|
||||
The backend runtime stage (`mateclaw-server/Dockerfile` stage 3) is based on `mcr.microsoft.com/playwright:v1.52.0-noble` (Ubuntu Noble 24.04, glibc) and installs on top of it:
|
||||
The backend runtime stage (`mateclaw-server/Dockerfile` stage 3) is based on `mcr.microsoft.com/playwright:v1.62.0-noble` (Ubuntu Noble 24.04, glibc) and installs on top of it:
|
||||
|
||||
- `openjdk-21-jre-headless` — runs the Spring Boot JAR
|
||||
- `fonts-noto-cjk` — Chinese/Japanese/Korean rendering in screenshots
|
||||
|
||||
@ -98,7 +98,7 @@ SEARXNG_BASE_URL=https://your-searxng.example.com
|
||||
|
||||
### 镜像里到底装了什么
|
||||
|
||||
后端镜像以 `mcr.microsoft.com/playwright:v1.52.0-noble` 为基础(Ubuntu Noble 24.04,glibc),由 `mateclaw-server/Dockerfile` 的第三阶段拉起,额外装:
|
||||
后端镜像以 `mcr.microsoft.com/playwright:v1.62.0-noble` 为基础(Ubuntu Noble 24.04,glibc),由 `mateclaw-server/Dockerfile` 的第三阶段拉起,额外装:
|
||||
|
||||
- `openjdk-21-jre-headless` —— 跑 Spring Boot JAR
|
||||
- `fonts-noto-cjk` —— 中文页面截图不出豆腐块
|
||||
|
||||
@ -0,0 +1,22 @@
|
||||
package vip.mate.agent.graph.node;
|
||||
|
||||
import org.junit.jupiter.api.DisplayName;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.lang.reflect.Field;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
class ReasoningNodeProgressPromptTest {
|
||||
|
||||
@Test
|
||||
@DisplayName("progress registration guidance respects the executor batch cap")
|
||||
void progressPromptCapsRegistrationBatches() throws Exception {
|
||||
Field field = ReasoningNode.class.getDeclaredField("TOOL_USE_ENFORCEMENT");
|
||||
field.setAccessible(true);
|
||||
String prompt = (String) field.get(null);
|
||||
|
||||
assertThat(prompt).contains("每批最多 16 个");
|
||||
assertThat(prompt).doesNotContain("一条回复里 N 个 `progress_update` 同时发出");
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,60 @@
|
||||
package vip.mate.tool.browser;
|
||||
|
||||
import com.google.gson.JsonObject;
|
||||
import org.junit.jupiter.api.DisplayName;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
|
||||
class BrowserNavigationGuardTest {
|
||||
|
||||
@Test
|
||||
@DisplayName("blocks unsafe CDP Page.navigate URLs before dispatch")
|
||||
void blocksUnsafeCdpNavigate() {
|
||||
JsonObject params = new JsonObject();
|
||||
params.addProperty("url", "http://169.254.169.254/latest/meta-data");
|
||||
|
||||
assertThrows(SecurityException.class,
|
||||
() -> BrowserNavigationGuard.checkCdp("Page.navigate", params, List.of(), false));
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("ignores non-navigation CDP methods")
|
||||
void ignoresNonNavigationCdpMethod() {
|
||||
JsonObject params = new JsonObject();
|
||||
params.addProperty("url", "http://169.254.169.254/latest/meta-data");
|
||||
|
||||
assertDoesNotThrow(
|
||||
() -> BrowserNavigationGuard.checkCdp("Runtime.evaluate", params, List.of(), false));
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("blocks obvious JavaScript navigation to unsafe URL")
|
||||
void blocksUnsafeEvalNavigation() {
|
||||
assertThrows(SecurityException.class,
|
||||
() -> BrowserNavigationGuard.checkEval(
|
||||
"window.location.href = 'http://169.254.169.254/latest/meta-data'",
|
||||
List.of(), false));
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("blocks obvious JavaScript network calls to unsafe URL")
|
||||
void blocksUnsafeEvalFetch() {
|
||||
assertThrows(SecurityException.class,
|
||||
() -> BrowserNavigationGuard.checkEval(
|
||||
"return await fetch(\"http://169.254.169.254/latest/meta-data\")",
|
||||
List.of(), false));
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("allows inert URL strings without navigation intent")
|
||||
void allowsInertUrlString() {
|
||||
assertDoesNotThrow(
|
||||
() -> BrowserNavigationGuard.checkEval(
|
||||
"const note = 'http://169.254.169.254/latest/meta-data'; return note.length;",
|
||||
List.of(), false));
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,69 @@
|
||||
package vip.mate.tool.browser;
|
||||
|
||||
import org.junit.jupiter.api.DisplayName;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
class BrowserRefStateTest {
|
||||
|
||||
@Test
|
||||
@DisplayName("a snapshot with zero interactive refs is still a valid snapshot")
|
||||
void emptySnapshotIsValid() {
|
||||
BrowserRefState state = new BrowserRefState();
|
||||
|
||||
int generation = state.recordSnapshot("https://example.com/empty", List.of(), Map.of());
|
||||
|
||||
assertEquals(1, generation);
|
||||
assertEquals(BrowserRefState.Status.VALID, state.status());
|
||||
assertTrue(state.refsValid());
|
||||
assertEquals(0, state.refCount());
|
||||
assertEquals("https://example.com/empty", state.snapshotUrl());
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("main-frame navigation invalidates refs and advances navigation epoch")
|
||||
void navigationInvalidatesSnapshot() {
|
||||
BrowserRefState state = new BrowserRefState();
|
||||
state.recordSnapshot("https://example.com/", List.of("e1"), Map.of());
|
||||
|
||||
state.onMainFrameNavigated("https://www.iana.org/help/example-domains");
|
||||
|
||||
assertEquals(BrowserRefState.Status.INVALIDATED, state.status());
|
||||
assertFalse(state.refsValid());
|
||||
assertEquals(0, state.refCount());
|
||||
assertEquals(1L, state.navigationEpoch());
|
||||
assertEquals("https://www.iana.org/help/example-domains", state.currentUrl());
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("surface URL reconciliation catches SPA URL changes without frame navigation")
|
||||
void reconcileUrlInvalidatesSnapshot() {
|
||||
BrowserRefState state = new BrowserRefState();
|
||||
state.recordSnapshot("https://example.com/page/1", List.of("e1"), Map.of());
|
||||
|
||||
state.reconcileUrl("https://example.com/page/2");
|
||||
|
||||
assertEquals(BrowserRefState.Status.INVALIDATED, state.status());
|
||||
assertFalse(state.refsValid());
|
||||
assertEquals(1L, state.navigationEpoch());
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("navigation epoch keeps advancing after refs are already invalidated")
|
||||
void repeatedUrlChangesAdvanceEpoch() {
|
||||
BrowserRefState state = new BrowserRefState();
|
||||
state.recordSnapshot("https://example.com/page/1", List.of("e1"), Map.of());
|
||||
|
||||
state.reconcileUrl("https://example.com/page/2");
|
||||
state.reconcileUrl("https://example.com/page/3");
|
||||
|
||||
assertEquals(2L, state.navigationEpoch());
|
||||
assertEquals("https://example.com/page/3", state.currentUrl());
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,84 @@
|
||||
package vip.mate.tool.browser;
|
||||
|
||||
import org.junit.jupiter.api.DisplayName;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
class BrowserSessionGateTest {
|
||||
|
||||
@Test
|
||||
@DisplayName("operations for one browser session never overlap")
|
||||
void serializesSameSession() throws Exception {
|
||||
BrowserSessionGate gate = new BrowserSessionGate(16);
|
||||
AtomicInteger active = new AtomicInteger();
|
||||
AtomicInteger maxActive = new AtomicInteger();
|
||||
CountDownLatch firstEntered = new CountDownLatch(1);
|
||||
CountDownLatch releaseFirst = new CountDownLatch(1);
|
||||
|
||||
try (ExecutorService executor = Executors.newVirtualThreadPerTaskExecutor()) {
|
||||
var first = executor.submit(() -> {
|
||||
try (BrowserSessionGate.Lease ignored = gate.enter("conversation-1")) {
|
||||
int now = active.incrementAndGet();
|
||||
maxActive.accumulateAndGet(now, Math::max);
|
||||
firstEntered.countDown();
|
||||
assertTrue(releaseFirst.await(2, TimeUnit.SECONDS));
|
||||
active.decrementAndGet();
|
||||
return "first";
|
||||
}
|
||||
});
|
||||
assertTrue(firstEntered.await(2, TimeUnit.SECONDS));
|
||||
|
||||
var second = executor.submit(() -> {
|
||||
try (BrowserSessionGate.Lease ignored = gate.enter("conversation-1")) {
|
||||
int now = active.incrementAndGet();
|
||||
maxActive.accumulateAndGet(now, Math::max);
|
||||
active.decrementAndGet();
|
||||
return "second";
|
||||
}
|
||||
});
|
||||
|
||||
Thread.sleep(50);
|
||||
releaseFirst.countDown();
|
||||
assertEquals("first", first.get(2, TimeUnit.SECONDS));
|
||||
assertEquals("second", second.get(2, TimeUnit.SECONDS));
|
||||
}
|
||||
|
||||
assertEquals(1, maxActive.get());
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("one stripe serializes different sessions sharing a Playwright driver")
|
||||
void singleStripeSerializesDifferentSessions() throws Exception {
|
||||
BrowserSessionGate gate = new BrowserSessionGate(1);
|
||||
AtomicInteger active = new AtomicInteger();
|
||||
AtomicInteger maxActive = new AtomicInteger();
|
||||
|
||||
try (ExecutorService executor = Executors.newVirtualThreadPerTaskExecutor()) {
|
||||
var first = executor.submit(() -> runMeasured(gate, "conversation-1", active, maxActive));
|
||||
var second = executor.submit(() -> runMeasured(gate, "conversation-2", active, maxActive));
|
||||
first.get(2, TimeUnit.SECONDS);
|
||||
second.get(2, TimeUnit.SECONDS);
|
||||
}
|
||||
|
||||
assertEquals(1, maxActive.get());
|
||||
}
|
||||
|
||||
private static String runMeasured(BrowserSessionGate gate, String key,
|
||||
AtomicInteger active, AtomicInteger maxActive) throws Exception {
|
||||
try (BrowserSessionGate.Lease ignored = gate.enter(key)) {
|
||||
int now = active.incrementAndGet();
|
||||
maxActive.accumulateAndGet(now, Math::max);
|
||||
Thread.sleep(25);
|
||||
active.decrementAndGet();
|
||||
return key;
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,54 @@
|
||||
package vip.mate.tool.browser;
|
||||
|
||||
import org.junit.jupiter.api.DisplayName;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
|
||||
class BrowserWaitConditionTest {
|
||||
|
||||
@Test
|
||||
@DisplayName("parses selector wait and caps timeout")
|
||||
void parsesSelectorAndCapsTimeout() {
|
||||
BrowserWaitCondition parsed = BrowserWaitCondition.parse("selector", "#ready", null, null, 120, 30);
|
||||
|
||||
assertEquals(BrowserWaitCondition.Kind.SELECTOR, parsed.kind());
|
||||
assertEquals("#ready", parsed.target());
|
||||
assertEquals(30_000, parsed.timeoutMillis());
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("parses text wait from text parameter")
|
||||
void parsesTextFromTextParameter() {
|
||||
BrowserWaitCondition parsed = BrowserWaitCondition.parse("text", null, "Saved", null, null, 30);
|
||||
|
||||
assertEquals(BrowserWaitCondition.Kind.TEXT, parsed.kind());
|
||||
assertEquals("Saved", parsed.target());
|
||||
assertEquals(30_000, parsed.timeoutMillis());
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("parses load state aliases")
|
||||
void parsesLoadStateAlias() {
|
||||
BrowserWaitCondition parsed = BrowserWaitCondition.parse("load_state", null, null, "domcontentloaded", 5, 30);
|
||||
|
||||
assertEquals(BrowserWaitCondition.Kind.LOAD_STATE, parsed.kind());
|
||||
assertEquals("domcontentloaded", parsed.target());
|
||||
assertEquals(5_000, parsed.timeoutMillis());
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("requires target for selector wait")
|
||||
void requiresSelectorTarget() {
|
||||
assertThrows(IllegalArgumentException.class,
|
||||
() -> BrowserWaitCondition.parse("selector", " ", null, null, 5, 30));
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("rejects unknown conditions")
|
||||
void rejectsUnknownCondition() {
|
||||
assertThrows(IllegalArgumentException.class,
|
||||
() -> BrowserWaitCondition.parse("sleep", null, null, null, 5, 30));
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,55 @@
|
||||
package vip.mate.tool.browser;
|
||||
|
||||
import org.junit.jupiter.api.DisplayName;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
class PageSnapshotScriptResultTest {
|
||||
|
||||
@Test
|
||||
@DisplayName("parses ref metadata and keeps refs backward-compatible")
|
||||
void parsesRefMetadata() {
|
||||
PageSnapshotScript.Result result = PageSnapshotScript.Result.fromJson("""
|
||||
{
|
||||
"tree":"- button \\"Save\\" @e1",
|
||||
"truncated":false,
|
||||
"refs":["e1"],
|
||||
"refInfos":[
|
||||
{"ref":"e1","role":"button","name":"Save","tag":"button","type":"","href":"","value":"","checked":false,"selected":false,"disabled":false,"expanded":true}
|
||||
]
|
||||
}
|
||||
""");
|
||||
|
||||
assertEquals("e1", result.refs().getFirst());
|
||||
assertEquals(1, result.refInfos().size());
|
||||
PageSnapshotScript.RefFingerprint ref = result.refInfos().get("e1");
|
||||
assertEquals("button", ref.role());
|
||||
assertEquals("Save", ref.name());
|
||||
assertEquals("button", ref.tag());
|
||||
assertTrue(ref.expanded());
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("detects core ref identity changes")
|
||||
void detectsCoreIdentityChanges() {
|
||||
PageSnapshotScript.RefFingerprint before = new PageSnapshotScript.RefFingerprint(
|
||||
"e1", "button", "Save", "button", "", "", "", false, false, false, null);
|
||||
PageSnapshotScript.RefFingerprint same = new PageSnapshotScript.RefFingerprint(
|
||||
"e1", "button", "Save", "button", "", "", "new value", false, false, false, null);
|
||||
PageSnapshotScript.RefFingerprint changed = new PageSnapshotScript.RefFingerprint(
|
||||
"e1", "link", "Save", "a", "", "/save", "", false, false, false, null);
|
||||
|
||||
assertTrue(before.sameCoreIdentity(same));
|
||||
assertFalse(before.sameCoreIdentity(changed));
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("snapshot and live fingerprint scripts normalize long names identically")
|
||||
void fingerprintScriptsShareNameNormalization() {
|
||||
assertTrue(PageSnapshotScript.SNAPSHOT_JS.contains("normalizeName(nameOf(el))"));
|
||||
assertTrue(PageSnapshotScript.REF_FINGERPRINT_JS.contains("normalizeName(nameOf(el))"));
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,36 @@
|
||||
package vip.mate.tool.builtin;
|
||||
|
||||
import org.junit.jupiter.api.DisplayName;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.ai.chat.model.ToolContext;
|
||||
import org.springframework.ai.support.ToolCallbacks;
|
||||
|
||||
import java.lang.reflect.Method;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
class BrowserUseToolContractTest {
|
||||
|
||||
@Test
|
||||
@DisplayName("CDP params are exposed to the model as a structured object")
|
||||
void cdpParamsUseStructuredMapSchema() throws NoSuchMethodException {
|
||||
Method method = BrowserUseTool.class.getMethod("browser_use",
|
||||
String.class, String.class, String.class, String.class, String.class,
|
||||
String.class, String.class, Integer.class, String.class, String.class,
|
||||
Map.class, String.class, Boolean.class, Integer.class, ToolContext.class);
|
||||
|
||||
assertEquals(Map.class, method.getParameterTypes()[10]);
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("Spring AI publishes CDP params as a JSON object schema")
|
||||
void generatedToolSchemaUsesObjectParams() {
|
||||
BrowserUseTool tool = new BrowserUseTool(null, null, null, null, null);
|
||||
|
||||
String schema = ToolCallbacks.from(tool)[0].getToolDefinition().inputSchema();
|
||||
|
||||
assertTrue(schema.matches("(?s).*\\\"params\\\"\\s*:\\s*\\{.*?\\\"type\\\"\\s*:\\s*\\\"object\\\".*"), schema);
|
||||
}
|
||||
}
|
||||
2
pom.xml
2
pom.xml
@ -51,7 +51,7 @@
|
||||
|
||||
<!-- Browser automation and WebSocket runtime -->
|
||||
<zxing.version>3.5.4</zxing.version>
|
||||
<playwright.version>1.59.0</playwright.version>
|
||||
<playwright.version>1.62.0</playwright.version>
|
||||
<tyrus.version>2.2.2</tyrus.version>
|
||||
|
||||
<!-- Document, wiki, and rendering toolchain -->
|
||||
|
||||
Loading…
Reference in New Issue
Block a user