mirror of
https://gitee.com/mateos/mateclaw.git
synced 2026-09-13 19:23:42 +08:00
* feat(browser): 放开内网服务访问限制,新增局域网部署模式开关 ### 背景 局域网部署时,Agent 用浏览器工具访问 http://192.168.x.x:port 等内网服务会被默认 SSRF 严格模式拦截。 ### 方案 新增两个 .env 开关(默认 false,行为与改动前完全一致): - PLAYWRIGHT_ALLOW_PRIVATE_NETWORK=true :放行本地回环和私有 IP,云元数据端点仍拦截 - PLAYWRIGHT_IGNORE_HTTPS_ERRORS=true :忽略 HTTPS 证书错误(自签证书场景) 顺带修复 IPv6 AWS IMDS 网段 fd00:ec2::/64 在严格模式下漏网的问题。 ### 验证 24 个单元测试全通过(UrlSafetyChecker 21 + BrowserProperties 3),覆盖严格/豁免两模式 + 4 个 check 重载 + IPv6 IMDS 网段。 ### 风险 开关仅作用于浏览器工具;公网部署务必保持 false。 * feat(browser): Playwright 超时可配 + snapshot 智能截断与 selector 作用域 ### 背景 Agent 用浏览器工具访问慢链路或大页面(超大表格)时遇到两类问题: 1. Playwright 默认 30s 超时不够用,且无法配置 2. snapshot 全量抓取页面文本,硬截断在 20000 字符处会切在元素中间,LLM 拿到残缺数据且不知道有截断 ### 方案 新增三个 .env 开关(默认值与改动前完全一致): 开关 作用 PLAYWRIGHT_DEFAULT_TIMEOUT_SECONDS 单次操作超时(click / fill / waitForLoadState) PLAYWRIGHT_NAVIGATION_TIMEOUT_SECONDS 导航超时(page.navigate / load-state) PLAYWRIGHT_SNAPSHOT_MAX_LENGTH snapshot 文本截断长度 snapshot 改造: - 支持 selector 参数作用域到子树(之前只用于 click/type) - budget 机制按元素边界智能截断,不再切在 <td> 中间 - 返回 truncated:true + hint 引导 LLM 用 selector 重抓 - JSON 字段顺序优化:truncated/hint 放 content 前,确保框架 spill preview(head 800 chars)能切到 ### 验证 - 28 个单元测试全通过(BrowserPropertiesTest 7 + UrlSafetyCheckerTest 21) - IDE 诊断 0 错误 - 覆盖:默认值不变 + setter 往返 + 严格/豁免两模式 ### 兼容性 - 默认值保持 30s / 30s / 20000,行为与改动前完全一致 - selector 参数本就是 @ToolParam(required=false) ,LLM schema 无变化,只是描述更新引导 snapshot 场景也能用 - 不影响 webhook / image download 等其他 SSRF 守卫 ## 改动汇总 文件 改动 BrowserProperties.java +3 字段: defaultTimeoutSeconds / defaultNavigationTimeoutSeconds / snapshotMaxLength (默认 30/30/20000) BrowserLauncher.java 抽 applyContextDefaults(context) 在三处 context 创建点调用;补全 setIgnoreHTTPSErrors 在 wrapLocalBrowser 落地 BrowserUseTool.java 工具描述 + selector 描述引导 LLM 在 snapshot 场景用 selector;doSnapshot 改造支持 selector 参数 + budget 智能截断 + JSON 字段顺序(truncated/hint 放 content 前确保 spill preview 能切到) docker-compose.yml +3 开关: PLAYWRIGHT_DEFAULT_TIMEOUT_SECONDS / PLAYWRIGHT_NAVIGATION_TIMEOUT_SECONDS / PLAYWRIGHT_SNAPSHOT_MAX_LENGTH .env.example +3 开关,简短说明 BrowserPropertiesTest.java +4 测试覆盖新字段默认值和 setter ## 测试结果 ## 数据流论证的关键决策 决策 依据 truncated:true 和 hint 放在 JSON content 字段之前 ToolResultStorage.buildPreview 会把 >8000 chars 的结果截到 head 800 chars,放在前面确保 LLM 看到 selector 参数描述从 "for click/type" 改为明确说 "OPTIONAL for snapshot" LLM 读 JSON schema 时按描述判断参数用途,原描述误导 LLM 不在 snapshot 用 selector 截断从硬 substring(0, N) 改为 JS budget 机制递归累计 避免切在 <td>订单号 ABC 中间,截断发生在 TEXT_NODE 完整段或下一个子元素开始之前 用 ElementHandle.querySelector + evaluate 替代字符串拼接 selector 进 JS 防止 selector 注入(selector 含特殊字符如引号、反斜杠) 三处 context 创建点统一调 applyContextDefaults 确保 CDP / external-CDP / 本地 launch 三条路径都应用配置的 timeout ## 临时改动还原 文件 改动 还原状态 mateclaw-server/pom.xml 临时加 maven-compiler-plugin + Lombok annotation processor ✅ 已删除,恢复原始状态 ## 未测项 项 原因 doSnapshot 的 JS budget 逻辑 需启动真 Playwright + 大页面,单元测试范围外 BrowserLauncher.applyContextDefaults 是否真的影响 page.click 行为 同上,集成测试范围 LLM 是否真的会按 hint 用 selector 重调 取决于 LLM 推理能力,需端到端测试
152 lines
7.6 KiB
YAML
152 lines
7.6 KiB
YAML
version: '3.8'
|
||
|
||
services:
|
||
# MySQL 数据库
|
||
#
|
||
# ⚠️ 密码通过环境变量传入,必须从 .env 文件提供。首次部署前:
|
||
# 1. cp .env.example .env
|
||
# 2. 编辑 .env 把 DB_ROOT_PASSWORD / DB_PASSWORD 改成强密码
|
||
# 未设置会直接在 `docker compose up` 时报错,避免把默认密码带到生产环境。
|
||
mysql:
|
||
image: mysql:8.0
|
||
container_name: mateclaw-mysql
|
||
restart: unless-stopped
|
||
environment:
|
||
MYSQL_ROOT_PASSWORD: ${DB_ROOT_PASSWORD:?DB_ROOT_PASSWORD is required in .env}
|
||
MYSQL_DATABASE: ${DB_NAME:-mateclaw}
|
||
MYSQL_USER: ${DB_USERNAME:-mateclaw}
|
||
MYSQL_PASSWORD: ${DB_PASSWORD:?DB_PASSWORD is required in .env}
|
||
TZ: Asia/Shanghai
|
||
ports:
|
||
- "3306:3306"
|
||
volumes:
|
||
- mysql_data:/var/lib/mysql
|
||
# Schema and seed data are managed by Flyway on application startup.
|
||
# Do NOT mount legacy schema.sql / data.sql here — Flyway creates all
|
||
# tables from V1 baseline and applies incremental migrations automatically.
|
||
command: --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci
|
||
healthcheck:
|
||
test: ["CMD", "mysqladmin", "ping", "-h", "localhost"]
|
||
interval: 10s
|
||
timeout: 5s
|
||
retries: 5
|
||
|
||
# SearXNG 搜索引擎(keyless 搜索 provider)
|
||
#
|
||
# The custom image at docker/searxng/ bakes in settings.yml so the sidecar
|
||
# works out of the box (upstream image ships JSON disabled + Limiter enabled,
|
||
# both of which silently break mateclaw's SearXNGSearchProvider).
|
||
# No host bind-mount — edit docker/searxng/settings.yml and rebuild.
|
||
searxng:
|
||
build:
|
||
context: ./docker/searxng
|
||
container_name: mateclaw-searxng
|
||
restart: unless-stopped
|
||
environment:
|
||
- SEARXNG_BASE_URL=http://searxng:8080
|
||
- SEARXNG_SECRET=${SEARXNG_SECRET:-mateclaw-dev-searxng-secret-change-me}
|
||
- UWSGI_WORKERS=2
|
||
- UWSGI_THREADS=4
|
||
ports:
|
||
- "8088:8080"
|
||
healthcheck:
|
||
# Healthz needs json format, so this also doubles as an integration check.
|
||
test: ["CMD", "wget", "--spider", "-q", "http://localhost:8080/healthz"]
|
||
interval: 30s
|
||
timeout: 5s
|
||
retries: 3
|
||
|
||
# MateClaw 后端服务
|
||
mateclaw-server:
|
||
build:
|
||
context: .
|
||
dockerfile: mateclaw-server/Dockerfile
|
||
args:
|
||
MAVEN_FLAGS: ${MAVEN_FLAGS:-}
|
||
container_name: mateclaw-server
|
||
restart: unless-stopped
|
||
depends_on:
|
||
mysql:
|
||
condition: service_healthy
|
||
searxng:
|
||
condition: service_healthy
|
||
environment:
|
||
SPRING_PROFILES_ACTIVE: mysql
|
||
DB_HOST: mysql
|
||
DB_PORT: 3306
|
||
DB_NAME: ${DB_NAME:-mateclaw}
|
||
DB_USERNAME: ${DB_USERNAME:-mateclaw}
|
||
DB_PASSWORD: ${DB_PASSWORD:?DB_PASSWORD is required in .env}
|
||
# LLM provider keys (DashScope / OpenAI / Anthropic / DeepSeek / Kimi / …) are
|
||
# NOT configured via env vars. After startup, add providers in the admin UI:
|
||
# Settings → Models → Add Provider
|
||
# Keys are stored in mate_model_provider and hot-reloaded.
|
||
SERPER_API_KEY: ${SERPER_API_KEY:-}
|
||
JWT_SECRET: ${JWT_SECRET:-}
|
||
MATECLAW_CORS_ALLOWED_ORIGINS: ${MATECLAW_CORS_ALLOWED_ORIGINS:-}
|
||
# SearXNG: tell the app where to reach the sidecar container
|
||
SEARXNG_BASE_URL: ${SEARXNG_BASE_URL:-http://searxng:8080}
|
||
# Browser automation: the runtime image (mcr.microsoft.com/playwright:*)
|
||
# bakes Chromium + system libs + fonts in, so the tool works out of the box.
|
||
# Override these if you want to attach to an external Chrome (CDP sidecar):
|
||
MATECLAW_BROWSER_CDP_URL: ${MATECLAW_BROWSER_CDP_URL:-}
|
||
MATECLAW_BROWSER_CHROME_PATH: ${MATECLAW_BROWSER_CHROME_PATH:-}
|
||
MATECLAW_BROWSER_CHANNEL: ${MATECLAW_BROWSER_CHANNEL:-}
|
||
# SSRF / TLS relaxations for isolated LAN / on-prem deployments.
|
||
# Both default to false (strict mode, public-internet safe).
|
||
# The .env file uses the PLAYWRIGHT_* prefix (component-oriented naming,
|
||
# not product-oriented) — here we translate to the MATECLAW_BROWSER_*
|
||
# container env that Spring Boot relaxed-binding maps to BrowserProperties.
|
||
# - PLAYWRIGHT_ALLOW_PRIVATE_NETWORK=true: allow loopback / private / link-local
|
||
# addresses through the browser SSRF guard. Cloud-metadata endpoints stay
|
||
# blocked. Turn on when the agent must drive http://192.168.x.x:port style
|
||
# internal services and has no path to the public internet.
|
||
# - PLAYWRIGHT_IGNORE_HTTPS_ERRORS=true: ignore HTTPS certificate errors.
|
||
# Auto-enables --ignore-certificate-errors at the Chromium command line
|
||
# when ALLOW_PRIVATE_NETWORK is also true (so CDP-attached external
|
||
# browsers benefit too). Leave false on internet-facing deployments.
|
||
MATECLAW_BROWSER_ALLOW_PRIVATE_NETWORK: ${PLAYWRIGHT_ALLOW_PRIVATE_NETWORK:-false}
|
||
MATECLAW_BROWSER_IGNORE_HTTPS_ERRORS: ${PLAYWRIGHT_IGNORE_HTTPS_ERRORS:-false}
|
||
# Playwright action / navigation timeouts (seconds). Increase for slow
|
||
# LAN or large-page scenarios. Defaults match Playwright's own (30s).
|
||
MATECLAW_BROWSER_DEFAULT_TIMEOUT_SECONDS: ${PLAYWRIGHT_DEFAULT_TIMEOUT_SECONDS:-30}
|
||
MATECLAW_BROWSER_DEFAULT_NAVIGATION_TIMEOUT_SECONDS: ${PLAYWRIGHT_NAVIGATION_TIMEOUT_SECONDS:-30}
|
||
# Hard cap on the textual snapshot returned by action=snapshot. Content
|
||
# beyond this length is dropped with a truncated:true flag and a hint to
|
||
# retry with selector. Results > framework spill threshold (~8000 chars)
|
||
# are further spilt to disk by ToolResultStorage.
|
||
MATECLAW_BROWSER_SNAPSHOT_MAX_LENGTH: ${PLAYWRIGHT_SNAPSHOT_MAX_LENGTH:-20000}
|
||
# OAuth 模式默认保持 auto:localhost 访问走 LOCAL,IP/域名访问走 DEVICE_CODE。
|
||
# 本机 Docker 若要强制使用 localhost:1455 回调,可在 .env 显式设为 local。
|
||
MATECLAW_OAUTH_OPENAI_DEPLOYMENT_MODE: ${MATECLAW_OAUTH_OPENAI_DEPLOYMENT_MODE:-}
|
||
MATECLAW_OAUTH_OPENAI_CALLBACK_BIND_HOST: ${MATECLAW_OAUTH_OPENAI_CALLBACK_BIND_HOST:-0.0.0.0}
|
||
# Wiki 知识库目录扫描白名单(逗号分隔,留空则禁止所有目录扫描)。
|
||
# 示例:MATE_WIKI_ALLOWED_SOURCE_ROOTS=/data/wiki,/opt/docs
|
||
# 记得同步在 volumes 里把宿主机路径挂进容器。
|
||
MATE_WIKI_ALLOWED_SOURCE_ROOTS: ${MATE_WIKI_ALLOWED_SOURCE_ROOTS:-}
|
||
# Wiki 知识源自动同步总开关(运维总闸,默认关)。AND 语义:全局开关与
|
||
# 每个知识库自己的「自动同步」开关都开,该库才会被定时扫描。
|
||
# 间隔单位毫秒,默认 5 分钟。
|
||
MATE_WIKI_WATCHER_ENABLED: ${MATE_WIKI_WATCHER_ENABLED:-false}
|
||
MATE_WIKI_WATCHER_INTERVAL_MS: ${MATE_WIKI_WATCHER_INTERVAL_MS:-300000}
|
||
# Skill 工作区根目录。放在 /app/data 下,让现有的 server_data 卷一并持久化
|
||
# 已安装的 skill、运行时积累的 LESSONS.md 以及 skill 运行产物,容器重启不丢。
|
||
# 内置 skill 仍由 JAR classpath 每次启动现场释放,空卷不会丢内置文件。
|
||
MATECLAW_SKILL_WORKSPACE_ROOT: ${MATECLAW_SKILL_WORKSPACE_ROOT:-/app/data/skills}
|
||
# Chromium needs a real /dev/shm. Docker defaults to 64MB which causes
|
||
# SIGBUS / "Target page closed" errors under load. 2GB is the usual
|
||
# recommendation for Playwright / headless chrome.
|
||
shm_size: 2gb
|
||
ports:
|
||
- "18080:18088" # host:container — app listens on 18088 inside the container
|
||
- "1455:1455"
|
||
volumes:
|
||
# server_data covers /app/data — H2 DB, wiki-uploads, AND the skill
|
||
# workspace (MATECLAW_SKILL_WORKSPACE_ROOT=/app/data/skills above), so a
|
||
# single volume persists everything. No separate skills volume needed.
|
||
- server_data:/app/data
|
||
|
||
volumes:
|
||
mysql_data:
|
||
server_data:
|