From 0394ba2803690f96a768d95ed75e36162f1920b5 Mon Sep 17 00:00:00 2001 From: RobinZhiBin Date: Fri, 9 Oct 2026 15:52:30 +0800 Subject: [PATCH] fix(db): require explicit opt-in for Flyway history repair (#657) Contributed by RobinZhiBin (@RobinZhiBin). Preserve migration history validation by default and document controlled recovery. Reviewed with the related community fixes; 71 backend regression tests passed. --- .../vip/mate/config/FlywayRepairConfig.java | 23 ++- .../main/resources/application-kingbase.yml | 4 +- .../main/resources/application-postgres.yml | 4 +- .../main/resources/docs/en/architecture.md | 2 +- .../src/main/resources/docs/en/config.md | 13 +- .../main/resources/docs/zh/architecture.md | 2 +- .../src/main/resources/docs/zh/config.md | 13 +- .../mate/config/FlywayRepairConfigTest.java | 174 ++++++++++++++++++ 8 files changed, 215 insertions(+), 20 deletions(-) create mode 100644 mateclaw-server/src/test/java/vip/mate/config/FlywayRepairConfigTest.java diff --git a/mateclaw-server/src/main/java/vip/mate/config/FlywayRepairConfig.java b/mateclaw-server/src/main/java/vip/mate/config/FlywayRepairConfig.java index 75b85d46..886de74c 100644 --- a/mateclaw-server/src/main/java/vip/mate/config/FlywayRepairConfig.java +++ b/mateclaw-server/src/main/java/vip/mate/config/FlywayRepairConfig.java @@ -2,30 +2,29 @@ package vip.mate.config; import lombok.extern.slf4j.Slf4j; import org.flywaydb.core.Flyway; +import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.flyway.FlywayMigrationInitializer; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; /** - * Flyway auto-repair configuration. - *

- * Replaces the default {@link FlywayMigrationInitializer} with one that - * calls {@code flyway.repair()} before {@code flyway.migrate()}. - * This handles failed migrations and checksum mismatches transparently - * during version upgrades — especially important for Desktop app users - * who cannot manually run CLI commands. - * - * @author MateClaw Team + * Strict migration startup by default. Repair is an explicit operator opt-in: + * changing applied checksums must never silently rewrite migration history. */ @Slf4j @Configuration public class FlywayRepairConfig { @Bean - public FlywayMigrationInitializer flywayInitializer(Flyway flyway) { + public FlywayMigrationInitializer flywayInitializer(Flyway flyway, + @Value("${mateclaw.flyway.auto-repair:false}") boolean autoRepair) { return new FlywayMigrationInitializer(flyway, f -> { - log.info("[Flyway] Running repair before migrate (auto-fix failed/changed migrations)..."); - f.repair(); + if (autoRepair) { + log.warn("[Flyway] Explicit auto-repair is enabled; applied migration history may be changed"); + f.repair(); + } + // Let Flyway validate before migration. In particular, checksum drift + // must fail startup rather than be hidden by an automatic repair. f.migrate(); }); } diff --git a/mateclaw-server/src/main/resources/application-kingbase.yml b/mateclaw-server/src/main/resources/application-kingbase.yml index c570cba2..d983b56c 100644 --- a/mateclaw-server/src/main/resources/application-kingbase.yml +++ b/mateclaw-server/src/main/resources/application-kingbase.yml @@ -43,8 +43,8 @@ spring: # 迁移前确保目标 schema 存在（Kingbase 不会自动创建非 public schema） init-sqls: - CREATE SCHEMA IF NOT EXISTS mateclaw - # 首次迁移或脚本变更后跳过 checksum 校验，避免因脚本转换导致的校验失败卡住 - validate-on-migrate: false + # Inherit strict checksum validation from application.yml. + # Use explicit, reviewed history repair when recovering an older database. h2: console: diff --git a/mateclaw-server/src/main/resources/application-postgres.yml b/mateclaw-server/src/main/resources/application-postgres.yml index 38a3bc21..070f36f6 100644 --- a/mateclaw-server/src/main/resources/application-postgres.yml +++ b/mateclaw-server/src/main/resources/application-postgres.yml @@ -40,8 +40,8 @@ spring: # auto-create a non-public schema). init-sqls: - CREATE SCHEMA IF NOT EXISTS mateclaw - # Skip checksum validation so an in-place migration edit doesn't block startup. - validate-on-migrate: false + # Inherit strict checksum validation from application.yml. + # Use explicit, reviewed history repair when recovering an older database. h2: console: diff --git a/mateclaw-server/src/main/resources/docs/en/architecture.md b/mateclaw-server/src/main/resources/docs/en/architecture.md index 319f94ec..11cc4a87 100644 --- a/mateclaw-server/src/main/resources/docs/en/architecture.md +++ b/mateclaw-server/src/main/resources/docs/en/architecture.md @@ -266,7 +266,7 @@ MateClaw uses **MyBatis Plus** (not JPA) for database access. Conventions: - `snake_case` columns, `camelCase` Java fields, auto-mapped - Every table has `create_time`, `update_time`, `deleted` (logical delete) - **Flyway** manages schema migrations — `db/migration/h2/` and `db/migration/mysql/` hold dialect-specific scripts, auto-selected on startup -- `FlywayRepairConfig` runs `repair()` before `migrate()` on every boot, self-healing checksum drift and partially-failed migrations +- `FlywayRepairConfig` runs `migrate()` with validation enabled by default; history repair requires an explicit operator opt-in. See [database recovery](./config#database-schema-init) before enabling it - Seed data loaded by `DatabaseBootstrapRunner` from `db/data-*.sql`, idempotent ### Table groups diff --git a/mateclaw-server/src/main/resources/docs/en/config.md b/mateclaw-server/src/main/resources/docs/en/config.md index 96118286..ad6993de 100644 --- a/mateclaw-server/src/main/resources/docs/en/config.md +++ b/mateclaw-server/src/main/resources/docs/en/config.md @@ -267,7 +267,18 @@ MateClaw uses **Flyway** for schema migrations: 2. `db/migration/mysql/V*__*.sql` — MySQL-dialect migration scripts 3. After migrations, seed data is loaded from `db/data-*.sql` — idempotent -Flyway auto-selects the correct dialect path based on the active Spring profile. Every startup runs a `repair` before `migrate`, self-healing checksum drift and partially-failed migrations (especially important for desktop users upgrading offline). +Flyway selects the migration dialect from the active Spring profile. Startup runs `migrate` with checksum validation enabled by default in all bundled profiles. Applied migration files must not be edited: checksum drift or a failed migration stops startup rather than silently changing the schema history. + +**Desktop upgrade compatibility:** an existing desktop database whose applied scripts differ from the new release can now fail to start. A new database and an unchanged migration history need no repair. Keep a backup that can be opened with the previous application version; do not delete the database to clear a validation error. + +If validation fails: + +1. Stop the application and back up the database. Keep the exact previous and target release artifacts, and inspect the failing migration and schema history. +2. Compare the old and new SQL with the actual schema/data. Prefer restoring the expected migration file or a maintainer-provided forward migration. `repair` changes migration metadata; it does **not** execute changed DDL, undo partial DDL/data changes, or prove the schema is correct. Resolve any partial migration effects before proceeding. +3. Only after confirming that history repair is appropriate, set `mateclaw.flyway.auto-repair=true` (environment variable `MATECLAW_FLYWAY_AUTOREPAIR=true`) for one controlled startup. Repair uses the configured migration locations and is followed by normal migration. Keep validation enabled. +4. Remove the opt-in immediately afterwards, restart normally, and verify validation and application behavior before allowing writes. If recovery fails, restore the backup with its matching application version. + +The opt-in defaults to `false`, including Desktop. It is not an unattended upgrade policy. `spring.flyway.validate-on-migrate` remains externally configurable, but setting it to `false` bypasses the validation guard and is not a repair procedure. ### Table conventions diff --git a/mateclaw-server/src/main/resources/docs/zh/architecture.md b/mateclaw-server/src/main/resources/docs/zh/architecture.md index 977f4fd8..2cf74b96 100644 --- a/mateclaw-server/src/main/resources/docs/zh/architecture.md +++ b/mateclaw-server/src/main/resources/docs/zh/architecture.md @@ -266,7 +266,7 @@ MateClaw 用 **MyBatis Plus**（不是 JPA）做数据库访问。约定： - `snake_case` 列、`camelCase` Java 字段、自动映射 - 每张表有 `create_time`、`update_time`、`deleted`（逻辑删除） - **Flyway** 管理 schema 迁移——`db/migration/h2/` 和 `db/migration/mysql/` 各有一套方言脚本，启动时自动选择 -- `FlywayRepairConfig` 在每次启动时先 `repair()` 再 `migrate()`，checksum 变更和部分失败的迁移自动修复 +- `FlywayRepairConfig` 默认执行开启校验的 `migrate()`；修复历史记录必须显式开启，操作前参阅[数据库恢复说明](./config#数据库-schema-初始化) - 种子数据由 `DatabaseBootstrapRunner` 从 `db/data-*.sql` 加载，幂等执行 ### 表分组 diff --git a/mateclaw-server/src/main/resources/docs/zh/config.md b/mateclaw-server/src/main/resources/docs/zh/config.md index 11dcacc3..fe7c38b7 100644 --- a/mateclaw-server/src/main/resources/docs/zh/config.md +++ b/mateclaw-server/src/main/resources/docs/zh/config.md @@ -267,7 +267,18 @@ MateClaw 用 **Flyway** 管理 schema 迁移： 2. `db/migration/mysql/V*__*.sql`——MySQL 方言的迁移脚本 3. 迁移完成后加载种子数据（`db/data-*.sql`），幂等执行 -启动时 Flyway 根据 active profile 自动选择正确的方言路径。每次启动时先做一次 `repair`，再 `migrate`——checksum 变更和部分失败的迁移自动修复（对桌面端离线升级用户尤其重要）。 +Flyway 根据 active profile 选择迁移方言。启动时执行 `migrate`，所有内置 profile 默认开启 checksum 校验。已执行的迁移脚本不应修改；校验不匹配或迁移失败会阻止启动，不会自动改写历史记录。 + +**桌面版升级兼容性：** 如果旧数据库记录的脚本与新版本不同，升级后可能无法启动。新数据库和迁移历史一致的数据库无需 repair。保留可用旧版应用打开的数据库备份，不要通过删除数据库来消除校验错误。 + +校验失败时： + +1. 停止应用并备份数据库。保留准确的旧版、新版安装包，检查出错的迁移及 schema history。 +2. 对比新旧 SQL 与实际表结构、数据，优先恢复正确的迁移文件或使用维护者提供的向前迁移。`repair` 只修正迁移元数据，**不会**执行被修改的 DDL、撤销部分执行的 DDL/数据变更，也不能证明表结构正确。应先处理部分迁移造成的影响。 +3. 只有确认适合修复历史记录后，才为一次受控启动设置 `mateclaw.flyway.auto-repair=true`（环境变量 `MATECLAW_FLYWAY_AUTOREPAIR=true`）。repair 使用当前配置的迁移路径，随后正常迁移；保持校验开启。 +4. 立即移除该开关，正常重启，确认校验及业务功能通过后再允许写入。恢复失败时，使用匹配的旧版应用和数据库备份一起回退。 + +该开关默认 `false`，桌面版也一样，不应作为无人值守升级策略。仍可通过外部配置覆盖 `spring.flyway.validate-on-migrate`，但设为 `false` 会绕过校验保护，不能代替修复。 ### 表约定 diff --git a/mateclaw-server/src/test/java/vip/mate/config/FlywayRepairConfigTest.java b/mateclaw-server/src/test/java/vip/mate/config/FlywayRepairConfigTest.java new file mode 100644 index 00000000..d8b7356e --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/config/FlywayRepairConfigTest.java @@ -0,0 +1,174 @@ +package vip.mate.config; + +import org.flywaydb.core.Flyway; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.springframework.boot.test.context.ConfigDataApplicationContextInitializer; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.core.env.Environment; +import org.springframework.core.env.SystemEnvironmentPropertySource; +import org.springframework.boot.test.context.runner.ApplicationContextRunner; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.sql.DriverManager; +import java.util.UUID; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; + +class FlywayRepairConfigTest { + @TempDir Path migrations; + + private String databaseUrl() { + return "jdbc:h2:mem:flyway-guard-" + UUID.randomUUID() + ";DB_CLOSE_DELAY=-1"; + } + + private Flyway flyway(String url) { + return Flyway.configure().dataSource(url, "sa", "") + .locations("filesystem:" + migrations.toAbsolutePath()) + .validateOnMigrate(true).load(); + } + + private ApplicationContextRunner context(String url) { + return new ApplicationContextRunner().withUserConfiguration(FlywayRepairConfig.class) + .withBean(Flyway.class, () -> flyway(url)); + } + + private int historyChecksum(String url) throws Exception { + try (var connection = DriverManager.getConnection(url, "sa", ""); + var statement = connection.createStatement(); + var rows = statement.executeQuery("SELECT \"checksum\" FROM \"flyway_schema_history\" WHERE \"version\" = '1'")) { + assertThat(rows.next()).isTrue(); + return rows.getInt(1); + } + } + + @Test + void defaultInitializerMigratesFreshDatabaseAndRepeatedStartupIsIdempotent() throws Exception { + Files.writeString(migrations.resolve("V1__fixture.sql"), "CREATE TABLE upgrade_fixture(id INT PRIMARY KEY);\n"); + String url = databaseUrl(); + context(url).run(started -> assertThat(started).hasNotFailed()); + int appliedChecksum = historyChecksum(url); + context(url).run(started -> assertThat(started).hasNotFailed()); + assertThat(historyChecksum(url)).isEqualTo(appliedChecksum); + assertThat(flyway(url).info().applied()).hasSize(1); + assertThat(flyway(url).migrate().migrationsExecuted).isZero(); + } + + @Test + void defaultInitializerRejectsChangedAppliedSqlWithoutRepairingHistory() throws Exception { + Path script = migrations.resolve("V1__fixture.sql"); + Files.writeString(script, "CREATE TABLE upgrade_fixture(id INT PRIMARY KEY);\n"); + String url = databaseUrl(); + context(url).run(started -> assertThat(started).hasNotFailed()); + int appliedChecksum = historyChecksum(url); + Files.writeString(script, "CREATE TABLE upgrade_fixture(id INT PRIMARY KEY, changed_column INT);\n"); + Files.writeString(migrations.resolve("V2__must_not_run.sql"), "CREATE TABLE should_not_exist(id INT);\n"); + + context(url).run(started -> { + assertThat(started).hasFailed(); + assertThat(started.getStartupFailure()).hasStackTraceContaining("Migration checksum mismatch for migration version 1"); + }); + assertThat(historyChecksum(url)).isEqualTo(appliedChecksum); + try (var connection = DriverManager.getConnection(url, "sa", ""); + var statement = connection.createStatement(); + var rows = statement.executeQuery("SELECT COUNT(*) FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 'SHOULD_NOT_EXIST'")) { + assertThat(rows.next()).isTrue(); + assertThat(rows.getInt(1)).isZero(); + } + } + + @Test + void explicitRepairOptInIsRequiredToChangeHistory() throws Exception { + Path script = migrations.resolve("V1__fixture.sql"); + Files.writeString(script, "CREATE TABLE upgrade_fixture(id INT PRIMARY KEY);\n"); + String url = databaseUrl(); + context(url).run(started -> assertThat(started).hasNotFailed()); + int original = historyChecksum(url); + Files.writeString(script, "-- audited comment change\nCREATE TABLE upgrade_fixture(id INT PRIMARY KEY);\n"); + context(url).withPropertyValues("mateclaw.flyway.auto-repair=true") + .run(started -> assertThat(started).hasNotFailed()); + assertThat(historyChecksum(url)).isNotEqualTo(original); + assertThat(flyway(url).validateWithResult().validationSuccessful).isTrue(); + } + + private ApplicationContextRunner profileContext(String url, String profile) { + return new ApplicationContextRunner() + .withInitializer(new ConfigDataApplicationContextInitializer()) + .withUserConfiguration(FlywayRepairConfig.class, TestDatabaseConfiguration.class) + .withPropertyValues("spring.profiles.active=" + profile, + "test.database-url=" + url, + "test.migration-location=filesystem:" + migrations.toAbsolutePath()); + } + + @Configuration(proxyBeanMethods = false) + static class TestDatabaseConfiguration { + @Bean + Flyway flyway(Environment environment) { + // Use real application/profile configuration for validation, but an + // isolated H2 fixture: no profile's external datasource is opened. + return Flyway.configure() + .dataSource(environment.getRequiredProperty("test.database-url"), "sa", "") + .locations(environment.getRequiredProperty("test.migration-location")) + .validateOnMigrate(environment.getRequiredProperty( + "spring.flyway.validate-on-migrate", Boolean.class)) + .load(); + } + } + + @ParameterizedTest + @ValueSource(strings = {"default", "mysql", "postgres", "kingbase"}) + void allShippedProfilesRejectChecksumDrift(String profile) throws Exception { + Path script = migrations.resolve("V1__fixture.sql"); + Files.writeString(script, "CREATE TABLE profile_fixture(id INT PRIMARY KEY);\n"); + String url = databaseUrl(); + profileContext(url, profile).run(started -> { + assertThat(started).hasNotFailed(); + assertThat(started.getBean(Flyway.class).getConfiguration().isValidateOnMigrate()).isTrue(); + }); + int original = historyChecksum(url); + Files.writeString(script, "CREATE TABLE profile_fixture(id INT PRIMARY KEY, changed_column INT);\n"); + profileContext(url, profile).run(started -> { + assertThat(started).hasFailed(); + assertThat(started.getStartupFailure()).hasStackTraceContaining( + "Migration checksum mismatch for migration version 1"); + }); + assertThat(historyChecksum(url)).isEqualTo(original); + } + + @ParameterizedTest + @ValueSource(strings = {"default", "mysql", "postgres", "kingbase"}) + void explicitValidationOverrideRemainsAvailable(String profile) throws Exception { + Files.writeString(migrations.resolve("V1__fixture.sql"), + "CREATE TABLE profile_fixture(id INT PRIMARY KEY);\n"); + profileContext(databaseUrl(), profile) + .withPropertyValues("spring.flyway.validate-on-migrate=false") + .run(started -> { + assertThat(started).hasNotFailed(); + assertThat(started.getBean(Flyway.class).getConfiguration().isValidateOnMigrate()).isFalse(); + }); + } + + + @Test + void documentedEnvironmentOptInRepairsReviewedHistoryChange() throws Exception { + Path script = migrations.resolve("V1__fixture.sql"); + Files.writeString(script, "CREATE TABLE profile_fixture(id INT PRIMARY KEY);\n"); + String url = databaseUrl(); + profileContext(url, "default").run(started -> assertThat(started).hasNotFailed()); + int original = historyChecksum(url); + Files.writeString(script, "-- reviewed comment change\nCREATE TABLE profile_fixture(id INT PRIMARY KEY);\n"); + profileContext(url, "default") + .withInitializer(context -> context.getEnvironment().getPropertySources().addFirst( + new SystemEnvironmentPropertySource("systemEnvironment", + Map.of("MATECLAW_FLYWAY_AUTOREPAIR", "true")))) + .run(started -> assertThat(started).hasNotFailed()); + assertThat(historyChecksum(url)).isNotEqualTo(original); + assertThat(flyway(url).validateWithResult().validationSuccessful).isTrue(); + } + +}