From 09d9fba1ffb1e0ef655b01bc15884b302c88c739 Mon Sep 17 00:00:00 2001 From: matevip Date: Mon, 6 Jul 2026 14:21:53 +0800 Subject: [PATCH] chore(deploy): switch public Docker stack to PostgreSQL 16 (#491) --- .env.example | 28 +++++- docker-compose.yml | 93 +++++++++++++------ docker/postgres/init/10-app-role.sh | 43 +++++++++ .../main/resources/docs/en/docker-deploy.md | 12 ++- .../main/resources/docs/zh/docker-deploy.md | 14 +-- 5 files changed, 147 insertions(+), 43 deletions(-) create mode 100755 docker/postgres/init/10-app-role.sh diff --git a/.env.example b/.env.example index 1213459f..a198de4d 100644 --- a/.env.example +++ b/.env.example @@ -6,18 +6,33 @@ # ⚠️ 所有标注「必填」的项若没配置，`docker compose up` 会直接失败退出，避免把默认/示例值带到生产环境。 # ==================== 数据库（Docker 模式必填） ==================== +# +# ⚠️ Docker 栈已切换到 PostgreSQL 16（此前为 MySQL）。老部署升级前请先读 +# docker-compose.yml 顶部的迁移说明：旧 mysql_data 卷不会被读取，需要先 +# mysqldump 再用 pgloader 等工具导入，或钉在切换前的 tag 上继续用 MySQL。 DB_HOST=localhost -DB_PORT=3306 +DB_PORT=5432 DB_NAME=mateclaw + +# 应用连接账号（最小权限角色，由 docker/postgres/init/10-app-role.sh 首次 +# 初始化时自动创建，仅拥有 mateclaw schema，不是超级用户）。 DB_USERNAME=mateclaw # ⚠️ 必填且请改成强密码（至少 16 位，含大小写+数字+符号）。 # docker-compose.yml 会通过 ${DB_PASSWORD:?} 强制要求此项。 DB_PASSWORD=change-me-strong-user-password -# ⚠️ MySQL root 账号密码，仅用于容器内初始化。请改成与 DB_PASSWORD 不同的强密码。 -DB_ROOT_PASSWORD=change-me-strong-root-password +# ⚠️ PostgreSQL 引导超级账号，仅用于容器内初始化和运维。 +# 请改成与 DB_PASSWORD 不同的强密码。 +DB_ADMIN_USERNAME=mateclaw_admin +DB_ADMIN_PASSWORD=change-me-strong-admin-password + +# ==================== 搜索（可选） ==================== + +# WebSearch 工具的云端搜索 API（可选，二选一或都不配；不配可用 SearXNG sidecar）。 +SERPER_API_KEY= +TAVILY_API_KEY= # ==================== 安全（强烈建议覆盖） ==================== @@ -123,6 +138,13 @@ MATE_WIKI_WATCHER_INTERVAL_MS=300000 # 并记得在 docker-compose.yml 的 volumes 里把对应宿主机目录挂进容器。 MATECLAW_SKILL_WORKSPACE_ROOT= +# ── Skill ZIP 上传大小上限（MB，可选）──────────────────────────── +# 技能包上传/市场安装的应用层上限，默认单文件 1MB、整包 50MB。 +# 解包时整包缓存在内存里，整包上限调多大，单次安装峰值内存就可能吃多大。 +# 同时注意 Spring 层 spring.servlet.multipart 的上限（默认 100MB/200MB）。 +MATECLAW_SKILL_UPLOAD_MAX_ENTRY_SIZE_MB= +MATECLAW_SKILL_UPLOAD_MAX_TOTAL_SIZE_MB= + # ── Maven 镜像（国内加速）───────────────────────────────────────── # 在中国大陆构建时取消注释，将 Aliyun 仓库优先级提前，大幅提速 mvn 拉包。 # 空值（默认）使用 US Maven Central → Google CDN → Aliyun 的顺序。 diff --git a/docker-compose.yml b/docker-compose.yml index 66fbab91..6668f530 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,35 +1,57 @@ -version: '3.8' - +# ============================================================================ +# ⚠️ DATABASE ENGINE: PostgreSQL (was MySQL before) +# +# This stack now runs on PostgreSQL 16. Switching the DB engine is a BREAKING +# change for existing deployments: `docker compose up -d` on a host that +# previously ran the MySQL stack starts a FRESH, EMPTY PostgreSQL volume +# (postgres_data). The old `mysql_data` volume is NOT read and the app +# re-seeds default data — existing data is not lost, but it is also NOT +# visible to the new stack. +# +# Treat this as FRESH-INSTALL-ONLY. To carry data across from a MySQL +# deployment, dump the OLD stack BEFORE pulling this change, e.g.: +# docker compose exec mysql mysqldump -u"$DB_USERNAME" -p"$DB_PASSWORD" "$DB_NAME" > mateclaw-mysql.sql +# then load into PostgreSQL with a cross-engine tool such as pgloader — +# there is no automatic MySQL → PostgreSQL migration here. Alternatively, +# keep running MySQL by pinning your checkout to a pre-switch tag and setting +# SPRING_PROFILES_ACTIVE=mysql (the mysql Spring profile remains supported). +# ============================================================================ services: - # MySQL 数据库 + # PostgreSQL 数据库 # # ⚠️ 密码通过环境变量传入，必须从 .env 文件提供。首次部署前： # 1. cp .env.example .env - # 2. 编辑 .env 把 DB_ROOT_PASSWORD / DB_PASSWORD 改成强密码 + # 2. 编辑 .env 把 DB_ADMIN_PASSWORD / DB_PASSWORD 改成强密码 # 未设置会直接在 `docker compose up` 时报错，避免把默认密码带到生产环境。 - mysql: - image: mysql:8.0 - container_name: mateclaw-mysql + postgres: + image: postgres:16 + container_name: mateclaw-postgres restart: unless-stopped environment: - MYSQL_ROOT_PASSWORD: ${DB_ROOT_PASSWORD:?DB_ROOT_PASSWORD is required in .env} - MYSQL_DATABASE: ${DB_NAME:-mateclaw} - MYSQL_USER: ${DB_USERNAME:-mateclaw} - MYSQL_PASSWORD: ${DB_PASSWORD:?DB_PASSWORD is required in .env} + # Bootstrap/superuser — owns the cluster, used only for init + admin tasks. + POSTGRES_DB: ${DB_NAME:-mateclaw} + POSTGRES_USER: ${DB_ADMIN_USERNAME:-mateclaw_admin} + POSTGRES_PASSWORD: ${DB_ADMIN_PASSWORD:?DB_ADMIN_PASSWORD is required in .env} + # Least-privilege application role created by the init script below; this + # is the account the server connects with (NOT a superuser). + APP_DB_USERNAME: ${DB_USERNAME:-mateclaw} + APP_DB_PASSWORD: ${DB_PASSWORD:?DB_PASSWORD is required in .env} TZ: Asia/Shanghai - ports: - - "3306:3306" + # No host port on purpose — the DB is only reachable from the compose + # network. For ad-hoc inspection use `docker compose exec postgres psql`, + # or temporarily add: ports: ["127.0.0.1:5432:5432"] volumes: - - mysql_data:/var/lib/mysql - # Schema and seed data are managed by Flyway on application startup. - # Do NOT mount legacy schema.sql / data.sql here — Flyway creates all - # tables from V1 baseline and applies incremental migrations automatically. - command: --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci + - postgres_data:/var/lib/postgresql/data + # Runs once on first init (empty data dir): creates the restricted app + # role and the mateclaw schema it owns. See docker/postgres/init/. + - ./docker/postgres/init:/docker-entrypoint-initdb.d:ro healthcheck: - test: ["CMD", "mysqladmin", "ping", "-h", "localhost"] + test: ["CMD-SHELL", "pg_isready -U ${DB_ADMIN_USERNAME:-mateclaw_admin} -d ${DB_NAME:-mateclaw}"] interval: 10s timeout: 5s retries: 5 + networks: + - mateclaw-net # SearXNG 搜索引擎（keyless 搜索 provider） # @@ -37,6 +59,8 @@ services: # works out of the box (upstream image ships JSON disabled + Limiter enabled, # both of which silently break mateclaw's SearXNGSearchProvider). # No host bind-mount — edit docker/searxng/settings.yml and rebuild. + # Internal-only: the app reaches it via the compose network. To debug from + # the host, temporarily add: ports: ["127.0.0.1:8088:8080"] searxng: build: context: ./docker/searxng @@ -47,14 +71,14 @@ services: - SEARXNG_SECRET=${SEARXNG_SECRET:-mateclaw-dev-searxng-secret-change-me} - UWSGI_WORKERS=2 - UWSGI_THREADS=4 - ports: - - "8088:8080" healthcheck: # Healthz needs json format, so this also doubles as an integration check. test: ["CMD", "wget", "--spider", "-q", "http://localhost:8080/healthz"] interval: 30s timeout: 5s retries: 3 + networks: + - mateclaw-net # MateClaw 后端服务 mateclaw-server: @@ -66,14 +90,14 @@ services: container_name: mateclaw-server restart: unless-stopped depends_on: - mysql: + postgres: condition: service_healthy searxng: condition: service_healthy environment: - SPRING_PROFILES_ACTIVE: mysql - DB_HOST: mysql - DB_PORT: 3306 + SPRING_PROFILES_ACTIVE: postgres + DB_HOST: postgres + DB_PORT: 5432 DB_NAME: ${DB_NAME:-mateclaw} DB_USERNAME: ${DB_USERNAME:-mateclaw} DB_PASSWORD: ${DB_PASSWORD:?DB_PASSWORD is required in .env} @@ -82,6 +106,7 @@ services: # Settings → Models → Add Provider # Keys are stored in mate_model_provider and hot-reloaded. SERPER_API_KEY: ${SERPER_API_KEY:-} + TAVILY_API_KEY: ${TAVILY_API_KEY:-} JWT_SECRET: ${JWT_SECRET:-} MATECLAW_CORS_ALLOWED_ORIGINS: ${MATECLAW_CORS_ALLOWED_ORIGINS:-} # SearXNG: tell the app where to reach the sidecar container @@ -133,6 +158,10 @@ services: # 已安装的 skill、运行时积累的 LESSONS.md 以及 skill 运行产物，容器重启不丢。 # 内置 skill 仍由 JAR classpath 每次启动现场释放，空卷不会丢内置文件。 MATECLAW_SKILL_WORKSPACE_ROOT: ${MATECLAW_SKILL_WORKSPACE_ROOT:-/app/data/skills} + # Skill ZIP 上传/安装大小上限（MB）。解包过程整包缓存在内存里， + # max-total 调多大，单次安装的峰值内存就可能吃多大。 + MATECLAW_SKILL_UPLOAD_MAX_ENTRY_SIZE_MB: ${MATECLAW_SKILL_UPLOAD_MAX_ENTRY_SIZE_MB:-1} + MATECLAW_SKILL_UPLOAD_MAX_TOTAL_SIZE_MB: ${MATECLAW_SKILL_UPLOAD_MAX_TOTAL_SIZE_MB:-50} # Chromium needs a real /dev/shm. Docker defaults to 64MB which causes # SIGBUS / "Target page closed" errors under load. 2GB is the usual # recommendation for Playwright / headless chrome. @@ -141,11 +170,17 @@ services: - "18080:18088" # host:container — app listens on 18088 inside the container - "1455:1455" volumes: - # server_data covers /app/data — H2 DB, wiki-uploads, AND the skill - # workspace (MATECLAW_SKILL_WORKSPACE_ROOT=/app/data/skills above), so a - # single volume persists everything. No separate skills volume needed. + # server_data covers /app/data — wiki-uploads AND the skill workspace + # (MATECLAW_SKILL_WORKSPACE_ROOT=/app/data/skills above), so a single + # volume persists everything. No separate skills volume needed. - server_data:/app/data + networks: + - mateclaw-net volumes: - mysql_data: + postgres_data: server_data: + +networks: + mateclaw-net: + driver: bridge diff --git a/docker/postgres/init/10-app-role.sh b/docker/postgres/init/10-app-role.sh new file mode 100755 index 00000000..388bd9c3 --- /dev/null +++ b/docker/postgres/init/10-app-role.sh @@ -0,0 +1,43 @@ +#!/bin/sh +# ============================================================================ +# Create a least-privilege application role for the MateClaw server. +# +# Runs once, during first container init (empty data dir), as the bootstrap +# superuser (POSTGRES_USER) against POSTGRES_DB. The app role: +# - can log in and CONNECT to the database, +# - owns the `mateclaw` schema (so Flyway can create/alter tables in it), +# - is NOT a superuser and cannot touch other databases/roles. +# +# The server connects as APP_DB_USERNAME / APP_DB_PASSWORD. +# ============================================================================ +set -e + +# Pass credentials as psql variables (-v) rather than interpolating them into +# the SQL text. The quoted heredoc (<<'EOSQL') keeps the body literal, and psql +# does the quoting: :'var' -> safe string literal, :"var" -> safe identifier. +# CREATE ROLE is generated via format(%I, %L) + \gexec so a password containing +# a quote (or an exotic role name) can't break or inject into the statement. +psql -v ON_ERROR_STOP=1 \ + --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" \ + -v app_user="$APP_DB_USERNAME" \ + -v app_pw="$APP_DB_PASSWORD" \ + -v db="$POSTGRES_DB" <<'EOSQL' + SELECT format('CREATE ROLE %I LOGIN PASSWORD %L', :'app_user', :'app_pw') + WHERE NOT EXISTS (SELECT FROM pg_roles WHERE rolname = :'app_user') + \gexec + + -- CONNECT to use the database; CREATE so the role can create schemas in it. + -- CREATE is required because Flyway's init-sql runs CREATE SCHEMA IF NOT + -- EXISTS, and PostgreSQL checks the database-level CREATE privilege *before* + -- the IF NOT EXISTS short-circuit — so even a pre-existing schema is denied + -- without it. Still scoped to this one database; not a cluster superuser. + GRANT CONNECT, CREATE ON DATABASE :"db" TO :"app_user"; + + -- The app owns its schema so Flyway DDL works, without cluster superuser rights. + CREATE SCHEMA IF NOT EXISTS mateclaw AUTHORIZATION :"app_user"; + + -- Default to the app schema on every connection from this role. + ALTER ROLE :"app_user" SET search_path TO mateclaw, public; +EOSQL + +echo "[init] application role '${APP_DB_USERNAME}' and schema 'mateclaw' ready" diff --git a/mateclaw-server/src/main/resources/docs/en/docker-deploy.md b/mateclaw-server/src/main/resources/docs/en/docker-deploy.md index d70703a2..6a15e732 100644 --- a/mateclaw-server/src/main/resources/docs/en/docker-deploy.md +++ b/mateclaw-server/src/main/resources/docs/en/docker-deploy.md @@ -1,6 +1,8 @@ # Docker Deployment -The only recommended production deployment outside the desktop app. One `docker compose up -d` brings up three containers: MySQL, SearXNG, and mateclaw-server. +The only recommended production deployment outside the desktop app. One `docker compose up -d` brings up three containers: PostgreSQL, SearXNG, and mateclaw-server. + +> **⚠️ Upgrading from the MySQL stack**: the Docker stack switched from MySQL to PostgreSQL 16. Pulling this change and running `up -d` on an existing host starts a **fresh, empty PostgreSQL volume** — the old `mysql_data` volume is not read (data is not lost, but the new stack cannot see it). To carry data across, `mysqldump` from the old stack first and import into PostgreSQL with a cross-engine tool such as pgloader; or stay on a pre-switch tag to keep running MySQL (the `mysql` Spring profile remains supported). This page covers **requirements, steps, verification, and common gotchas**. For the full environment variable reference, see [Configuration](./config). @@ -13,7 +15,7 @@ This page covers **requirements, steps, verification, and common gotchas**. For | Docker Engine | 24.0+ | latest stable | `docker --version` | | Docker Compose | v2.20+ | v2.30+ | `docker compose version` (the v2 plugin, not the legacy `docker-compose`) | | Host RAM | 4 GB | 8 GB+ | Chromium consumes 1-2 GB when the browser tool is active | -| Disk | 6 GB | 20 GB+ | ~2 GB image + MySQL data + workspace files | +| Disk | 6 GB | 20 GB+ | ~2 GB image + PostgreSQL data + workspace files | | /dev/shm | default | compose sets 2 GB automatically | Chromium uses shared memory for rendering; the 64 MB default causes SIGBUS | | Network | outbound | — | for pulling images and calling LLM APIs | @@ -25,7 +27,7 @@ This page covers **requirements, steps, verification, and common gotchas**. For | Service | Image | Role | Exposed port | |---|---|---|---| -| `mysql` | `mysql:8.0` | Business data | `3306` | +| `postgres` | `postgres:16` | Business data | compose network only (no host port by default) | | `searxng` | Built from `./docker/searxng/` | Keyless search fallback | `8088` | | `mateclaw-server` | Built from `mateclaw-server/Dockerfile` | Spring Boot backend + embedded browser | `18080` | @@ -179,7 +181,7 @@ vi .env # see table below | Variable | Notes | |---|---| | `DB_PASSWORD` | App DB password — 16+ chars, mixed case, digits, symbols | -| `DB_ROOT_PASSWORD` | MySQL root password — **must differ from the above** | +| `DB_ADMIN_PASSWORD` | PostgreSQL bootstrap superuser password (init + admin only) — **must differ from the above** | **Strongly recommended** (not enforced, but startup logs WARN if missing): @@ -299,7 +301,7 @@ docker compose build mateclaw-server # only rebuild the backend docker compose up -d mateclaw-server ``` -The `mysql_data` volume persists across rebuilds. Flyway runs incremental migrations automatically and self-heals checksum changes on restart. **Version is pinned in `mateclaw-server/pom.xml` and the git tag** — prefer pinning to a tag in production, not tracking `dev`. +The `postgres_data` volume persists across rebuilds. Flyway runs incremental migrations automatically and self-heals checksum changes on restart. **Version is pinned in `mateclaw-server/pom.xml` and the git tag** — prefer pinning to a tag in production, not tracking `dev`. --- diff --git a/mateclaw-server/src/main/resources/docs/zh/docker-deploy.md b/mateclaw-server/src/main/resources/docs/zh/docker-deploy.md index 8ed18af9..2f424235 100644 --- a/mateclaw-server/src/main/resources/docs/zh/docker-deploy.md +++ b/mateclaw-server/src/main/resources/docs/zh/docker-deploy.md @@ -1,6 +1,8 @@ # Docker 部署 -桌面端之外的唯一推荐生产部署方式。一条 `docker compose up -d` 起三个容器：MySQL、SearXNG、mateclaw-server。 +桌面端之外的唯一推荐生产部署方式。一条 `docker compose up -d` 起三个容器：PostgreSQL、SearXNG、mateclaw-server。 + +> **⚠️ 从 MySQL 栈升级**：Docker 栈已从 MySQL 切换到 PostgreSQL 16。老部署直接 `git pull` 后 `up -d` 会启动一个**全新的空 PostgreSQL 卷**——旧 `mysql_data` 卷不会被读取（数据不丢，但新栈看不到）。要携带数据请先在旧栈上 `mysqldump`，再用 pgloader 等工具导入 PostgreSQL；或钉在切换前的 tag 上继续用 MySQL（`mysql` Spring profile 仍然支持）。 这一页覆盖**要求、步骤、验证、常见坑**。配置变量明细请看 [配置说明](./config)。 @@ -13,7 +15,7 @@ | Docker Engine | 24.0+ | 最新稳定 | `docker --version` 确认 | | Docker Compose | v2.20+ | v2.30+ | `docker compose version`（注意是 `compose` 不是 `compose`） | | 宿主 RAM | 4 GB | 8 GB+ | 浏览器工具启动时 Chromium 会吃 1-2 GB | -| 磁盘空间 | 6 GB | 20 GB+ | 镜像约 2 GB + MySQL 数据 + 工作空间文件 | +| 磁盘空间 | 6 GB | 20 GB+ | 镜像约 2 GB + PostgreSQL 数据 + 工作空间文件 | | /dev/shm | 默认 | compose 已自动设 2 GB | Chromium 用共享内存做渲染，默认 64 MB 会 SIGBUS | | 网络 | 出公网 | — | 拉镜像 + 调 LLM API | @@ -25,7 +27,7 @@ | 服务 | 镜像 | 作用 | 暴露端口 | |---|---|---|---| -| `mysql` | `mysql:8.0` | 业务数据存储 | `3306` | +| `postgres` | `postgres:16` | 业务数据存储 | 仅容器网络内（默认不映射宿主端口） | | `searxng` | 本地构建 `./docker/searxng/` | 无 API Key 搜索兜底 | `8088` | | `mateclaw-server` | 本地构建 `mateclaw-server/Dockerfile` | Spring Boot 后端 + 内置浏览器 | `18080` | @@ -178,8 +180,8 @@ vi .env # 见下方必填表 | 变量 | 说明 | |---|---| -| `DB_PASSWORD` | 业务库账号密码，建议 16+ 位 + 大小写 + 数字 + 符号 | -| `DB_ROOT_PASSWORD` | MySQL root 密码，**与上面不同** | +| `DB_PASSWORD` | 应用账号密码（最小权限角色，仅拥有 `mateclaw` schema），建议 16+ 位 + 大小写 + 数字 + 符号 | +| `DB_ADMIN_PASSWORD` | PostgreSQL 引导超级账号密码，仅用于首次初始化与运维，**与上面不同** | **强烈建议**（不填不会报错，启动日志里 WARN）： @@ -299,7 +301,7 @@ docker compose build mateclaw-server # 只重建后端 docker compose up -d mateclaw-server ``` -MySQL 数据卷（`mysql_data`）不会动，Flyway 自动跑增量迁移 + 自愈 checksum 变化。**版本号写在 `mateclaw-server/pom.xml` 和 git tag**，生产环境建议钉 tag 而不是 `dev` 分支。 +PostgreSQL 数据卷（`postgres_data`）不会动，Flyway 自动跑增量迁移 + 自愈 checksum 变化。**版本号写在 `mateclaw-server/pom.xml` 和 git tag**，生产环境建议钉 tag 而不是 `dev` 分支。 ---