From 3f86c863b86e648c15b987cd6c2bcae353a6b612 Mon Sep 17 00:00:00 2001 From: longbingljw Date: Mon, 10 Nov 2025 17:21:00 +0800 Subject: [PATCH] fix:ci test (#52) * fix * tmp ci * fix * fix * dify_config test fix * add mysql test * add mysql migration test * fix * fix config default value * test * fix * fix --- .github/workflows/api-tests.yml | 2 +- .github/workflows/db-migration-test.yml | 55 +++++++- api/.env.example | 2 +- api/configs/middleware/__init__.py | 24 ++-- .../middleware/vdb/oceanbase_config.py | 36 +++--- .../tools/workflow_tools_manage_service.py | 2 - .../tools/test_tools_transform_service.py | 3 - .../unit_tests/configs/test_dify_config.py | 122 +++++++++++++----- docker/.env.example | 2 +- docker/middleware.env.example | 2 +- 10 files changed, 179 insertions(+), 71 deletions(-) diff --git a/.github/workflows/api-tests.yml b/.github/workflows/api-tests.yml index 37d351627b..557d747a8c 100644 --- a/.github/workflows/api-tests.yml +++ b/.github/workflows/api-tests.yml @@ -62,7 +62,7 @@ jobs: compose-file: | docker/docker-compose.middleware.yaml services: | - db + db_postgres redis sandbox ssrf_proxy diff --git a/.github/workflows/db-migration-test.yml b/.github/workflows/db-migration-test.yml index b9961a4714..04812ca305 100644 --- a/.github/workflows/db-migration-test.yml +++ b/.github/workflows/db-migration-test.yml @@ -8,7 +8,7 @@ concurrency: cancel-in-progress: true jobs: - db-migration-test: + db-migration-test-postgres: runs-on: ubuntu-latest steps: @@ -45,7 +45,7 @@ jobs: compose-file: | docker/docker-compose.middleware.yaml services: | - db + db_postgres redis - name: Prepare configs @@ -57,3 +57,54 @@ jobs: env: DEBUG: true run: uv run --directory api flask upgrade-db + + db-migration-test-mysql: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Setup UV and Python + uses: astral-sh/setup-uv@v6 + with: + enable-cache: true + python-version: "3.12" + cache-dependency-glob: api/uv.lock + + - name: Install dependencies + run: uv sync --project api + - name: Ensure Offline migration are supported + run: | + # upgrade + uv run --directory api flask db upgrade 'base:head' --sql + # downgrade + uv run --directory api flask db downgrade 'head:base' --sql + + - name: Prepare middleware env + run: | + cd docker + cp middleware.env.example middleware.env + + - name: Set up Middlewares + uses: hoverkraft-tech/compose-action@v2.0.2 + with: + compose-file: | + docker/docker-compose.middleware.yaml + services: | + db_mysql + redis + + - name: Prepare configs for MySQL + run: | + cd api + cp .env.example .env + sed -i 's/DB_TYPE=postgresql/DB_TYPE=mysql/' .env + + - name: Run DB Migration + env: + DEBUG: true + run: uv run --directory api flask upgrade-db diff --git a/api/.env.example b/api/.env.example index 8b82b76bde..ebc126dbb3 100644 --- a/api/.env.example +++ b/api/.env.example @@ -81,7 +81,7 @@ POSTGRES_PORT=5432 POSTGRES_DATABASE=dify # MySQL configuration -MYSQL_USER=root +MYSQL_USER=mysql MYSQL_PASSWORD=difyai123456 MYSQL_HOST=localhost MYSQL_PORT=3306 diff --git a/api/configs/middleware/__init__.py b/api/configs/middleware/__init__.py index a8a9a1953a..6a44ef4202 100644 --- a/api/configs/middleware/__init__.py +++ b/api/configs/middleware/__init__.py @@ -163,7 +163,7 @@ class DatabaseConfig(BaseSettings): default="dify", ) - # OceanBase configuration (MySQL-compatible) + # OceanBase configuration(MySQL-compatible) OCEANBASE_HOST: str = Field( description="OceanBase hostname or IP address.", default="localhost", @@ -195,50 +195,50 @@ class DatabaseConfig(BaseSettings): def DB_HOST(self) -> str: if self.DB_TYPE == "postgresql": return self.POSTGRES_HOST + elif self.DB_TYPE == "mysql": + return self.MYSQL_HOST elif self.DB_TYPE == "oceanbase": return self.OCEANBASE_HOST - else: - return self.MYSQL_HOST - + @computed_field # type: ignore[prop-decorator] @property def DB_PORT(self) -> int: if self.DB_TYPE == "postgresql": return self.POSTGRES_PORT + elif self.DB_TYPE == "mysql": + return self.MYSQL_PORT elif self.DB_TYPE == "oceanbase": return self.OCEANBASE_PORT - else: - return self.MYSQL_PORT @computed_field # type: ignore[prop-decorator] @property def DB_USERNAME(self) -> str: if self.DB_TYPE == "postgresql": return self.POSTGRES_USER + elif self.DB_TYPE == "mysql": + return self.MYSQL_USER elif self.DB_TYPE == "oceanbase": return self.OCEANBASE_USER - else: - return self.MYSQL_USER @computed_field # type: ignore[prop-decorator] @property def DB_PASSWORD(self) -> str: if self.DB_TYPE == "postgresql": return self.POSTGRES_PASSWORD + elif self.DB_TYPE == "mysql": + return self.MYSQL_PASSWORD elif self.DB_TYPE == "oceanbase": return self.OCEANBASE_PASSWORD - else: - return self.MYSQL_PASSWORD @computed_field # type: ignore[prop-decorator] @property def DB_DATABASE(self) -> str: if self.DB_TYPE == "postgresql": return self.POSTGRES_DATABASE + elif self.DB_TYPE == "mysql": + return self.MYSQL_DATABASE elif self.DB_TYPE == "oceanbase": return self.OCEANBASE_DATABASE - else: - return self.MYSQL_DATABASE DB_CHARSET: str = Field( description="Character set for database connection.", diff --git a/api/configs/middleware/vdb/oceanbase_config.py b/api/configs/middleware/vdb/oceanbase_config.py index 1bb94b0a78..76d05fb5a1 100644 --- a/api/configs/middleware/vdb/oceanbase_config.py +++ b/api/configs/middleware/vdb/oceanbase_config.py @@ -7,29 +7,29 @@ class OceanBaseVectorConfig(BaseSettings): Configuration settings for OceanBase Vector database """ - OCEANBASE_HOST: str | None = Field( - description="Hostname or IP address of the OceanBase Vector server (e.g. 'localhost')", - default=None, + OCEANBASE_HOST: str = Field( + description="OceanBase hostname or IP address.", + default="localhost", ) - - OCEANBASE_PORT: PositiveInt | None = Field( - description="Port number on which the OceanBase Vector server is listening (default is 2881)", + + OCEANBASE_PORT: PositiveInt = Field( + description="OceanBase port number.", default=2881, ) - - OCEANBASE_USER: str | None = Field( - description="Username for authenticating with the OceanBase Vector database", - default=None, + + OCEANBASE_USER: str = Field( + description="OceanBase username.", + default="root@test", ) - - OCEANBASE_PASSWORD: str | None = Field( - description="Password for authenticating with the OceanBase Vector database", - default=None, + + OCEANBASE_PASSWORD: str = Field( + description="OceanBase password.", + default="difyai123456", ) - - OCEANBASE_DATABASE: str | None = Field( - description="Name of the OceanBase Vector database to connect to", - default=None, + + OCEANBASE_DATABASE: str = Field( + description="OceanBase database name.", + default="test", ) OCEANBASE_ENABLE_HYBRID_SEARCH: bool = Field( diff --git a/api/services/tools/workflow_tools_manage_service.py b/api/services/tools/workflow_tools_manage_service.py index b1cc963681..5413725798 100644 --- a/api/services/tools/workflow_tools_manage_service.py +++ b/api/services/tools/workflow_tools_manage_service.py @@ -14,7 +14,6 @@ from core.tools.utils.workflow_configuration_sync import WorkflowToolConfigurati from core.tools.workflow_as_tool.provider import WorkflowToolProviderController from core.tools.workflow_as_tool.tool import WorkflowTool from extensions.ext_database import db -from libs.uuid_utils import uuidv7 from models.model import App from models.tools import WorkflowToolProvider from models.workflow import Workflow @@ -67,7 +66,6 @@ class WorkflowToolManageService: with Session(db.engine, expire_on_commit=False) as session, session.begin(): workflow_tool_provider = WorkflowToolProvider( - id=str(uuidv7()), tenant_id=tenant_id, user_id=user_id, app_id=workflow_app_id, diff --git a/api/tests/test_containers_integration_tests/services/tools/test_tools_transform_service.py b/api/tests/test_containers_integration_tests/services/tools/test_tools_transform_service.py index e2c616420f..ae0c7b7a6b 100644 --- a/api/tests/test_containers_integration_tests/services/tools/test_tools_transform_service.py +++ b/api/tests/test_containers_integration_tests/services/tools/test_tools_transform_service.py @@ -6,7 +6,6 @@ from faker import Faker from core.tools.entities.api_entities import ToolProviderApiEntity from core.tools.entities.common_entities import I18nObject from core.tools.entities.tool_entities import ToolProviderType -from libs.uuid_utils import uuidv7 from models.tools import ApiToolProvider, BuiltinToolProvider, MCPToolProvider, WorkflowToolProvider from services.tools.tools_transform_service import ToolTransformService @@ -67,7 +66,6 @@ class TestToolTransformService: ) elif provider_type == "workflow": provider = WorkflowToolProvider( - id=str(uuidv7()), name=fake.company(), description=fake.text(max_nb_chars=100), icon='{"background": "#FF6B6B", "content": "🔧"}', @@ -760,7 +758,6 @@ class TestToolTransformService: # Create workflow tool provider provider = WorkflowToolProvider( - id=str(uuidv7()), name=fake.company(), description=fake.text(max_nb_chars=100), icon='{"background": "#FF6B6B", "content": "🔧"}', diff --git a/api/tests/unit_tests/configs/test_dify_config.py b/api/tests/unit_tests/configs/test_dify_config.py index 209b6bf59b..4d44fd8e56 100644 --- a/api/tests/unit_tests/configs/test_dify_config.py +++ b/api/tests/unit_tests/configs/test_dify_config.py @@ -16,11 +16,11 @@ def test_dify_config(monkeypatch: pytest.MonkeyPatch): monkeypatch.setenv("CONSOLE_API_URL", "https://example.com") monkeypatch.setenv("CONSOLE_WEB_URL", "https://example.com") monkeypatch.setenv("HTTP_REQUEST_MAX_WRITE_TIMEOUT", "30") # Custom value for testing - monkeypatch.setenv("DB_USERNAME", "postgres") - monkeypatch.setenv("DB_PASSWORD", "postgres") - monkeypatch.setenv("DB_HOST", "localhost") - monkeypatch.setenv("DB_PORT", "5432") - monkeypatch.setenv("DB_DATABASE", "dify") + monkeypatch.setenv("POSTGRES_USER", "postgres") + monkeypatch.setenv("POSTGRES_PASSWORD", "postgres") + monkeypatch.setenv("POSTGRES_HOST", "localhost") + monkeypatch.setenv("POSTGRES_PORT", "5432") + monkeypatch.setenv("POSTGRES_DATABASE", "dify") monkeypatch.setenv("HTTP_REQUEST_MAX_READ_TIMEOUT", "300") # Custom value for testing # load dotenv file with pydantic-settings @@ -51,11 +51,11 @@ def test_http_timeout_defaults(monkeypatch: pytest.MonkeyPatch): os.environ.clear() # Set minimal required env vars - monkeypatch.setenv("DB_USERNAME", "postgres") - monkeypatch.setenv("DB_PASSWORD", "postgres") - monkeypatch.setenv("DB_HOST", "localhost") - monkeypatch.setenv("DB_PORT", "5432") - monkeypatch.setenv("DB_DATABASE", "dify") + monkeypatch.setenv("POSTGRES_USER", "postgres") + monkeypatch.setenv("POSTGRES_PASSWORD", "postgres") + monkeypatch.setenv("POSTGRES_HOST", "localhost") + monkeypatch.setenv("POSTGRES_PORT", "5432") + monkeypatch.setenv("POSTGRES_DATABASE", "dify") config = DifyConfig() @@ -75,11 +75,11 @@ def test_flask_configs(monkeypatch: pytest.MonkeyPatch): # Set environment variables using monkeypatch monkeypatch.setenv("CONSOLE_API_URL", "https://example.com") monkeypatch.setenv("CONSOLE_WEB_URL", "https://example.com") - monkeypatch.setenv("DB_USERNAME", "postgres") - monkeypatch.setenv("DB_PASSWORD", "postgres") - monkeypatch.setenv("DB_HOST", "localhost") - monkeypatch.setenv("DB_PORT", "5432") - monkeypatch.setenv("DB_DATABASE", "dify") + monkeypatch.setenv("POSTGRES_USER", "postgres") + monkeypatch.setenv("POSTGRES_PASSWORD", "postgres") + monkeypatch.setenv("POSTGRES_HOST", "localhost") + monkeypatch.setenv("POSTGRES_PORT", "5432") + monkeypatch.setenv("POSTGRES_DATABASE", "dify") monkeypatch.setenv("WEB_API_CORS_ALLOW_ORIGINS", "http://127.0.0.1:3000,*") monkeypatch.setenv("CODE_EXECUTION_ENDPOINT", "http://127.0.0.1:8194/") @@ -120,15 +120,77 @@ def test_flask_configs(monkeypatch: pytest.MonkeyPatch): assert str(URL(str(config["CODE_EXECUTION_ENDPOINT"])) / "v1") == "http://127.0.0.1:8194/v1" +def test_flask_configs_mysql(monkeypatch: pytest.MonkeyPatch): + """Test Flask configuration with MySQL database type""" + flask_app = Flask("app") + # clear system environment variables + os.environ.clear() + + # Set environment variables using monkeypatch for MySQL + monkeypatch.setenv("CONSOLE_API_URL", "https://example.com") + monkeypatch.setenv("CONSOLE_WEB_URL", "https://example.com") + monkeypatch.setenv("DB_TYPE", "mysql") + monkeypatch.setenv("MYSQL_USER", "root") + monkeypatch.setenv("MYSQL_PASSWORD", "mysql123") + monkeypatch.setenv("MYSQL_HOST", "mysql-host") + monkeypatch.setenv("MYSQL_PORT", "3306") + monkeypatch.setenv("MYSQL_DATABASE", "dify_mysql") + monkeypatch.setenv("WEB_API_CORS_ALLOW_ORIGINS", "http://127.0.0.1:3000,*") + monkeypatch.setenv("CODE_EXECUTION_ENDPOINT", "http://127.0.0.1:8194/") + + flask_app.config.from_mapping(DifyConfig().model_dump()) # pyright: ignore + config = flask_app.config + + # configs read from pydantic-settings + assert config["LOG_LEVEL"] == "INFO" + assert config["COMMIT_SHA"] == "" + assert config["EDITION"] == "SELF_HOSTED" + assert config["API_COMPRESSION_ENABLED"] is False + assert config["SENTRY_TRACES_SAMPLE_RATE"] == 1.0 + + # value from env file + assert config["CONSOLE_API_URL"] == "https://example.com" + # fallback to alias choices value as CONSOLE_API_URL + assert config["FILES_URL"] == "https://example.com" + + # Test MySQL database configuration + assert config["DB_TYPE"] == "mysql" + assert config["SQLALCHEMY_DATABASE_URI"] == "mysql+pymysql://root:mysql123@mysql-host:3306/dify_mysql" + assert config["SQLALCHEMY_DATABASE_URI_SCHEME"] == "mysql+pymysql" + assert config["SQLALCHEMY_ENGINE_OPTIONS"] == { + "connect_args": {}, # MySQL doesn't have PostgreSQL-specific options + "max_overflow": 10, + "pool_pre_ping": False, + "pool_recycle": 3600, + "pool_size": 30, + "pool_use_lifo": False, + "pool_reset_on_return": None, + "pool_timeout": 30, + } + + # Test computed fields for MySQL + assert config["DB_HOST"] == "mysql-host" + assert config["DB_PORT"] == 3306 + assert config["DB_USERNAME"] == "root" + assert config["DB_PASSWORD"] == "mysql123" + assert config["DB_DATABASE"] == "dify_mysql" + + assert config["CONSOLE_WEB_URL"] == "https://example.com" + assert config["CONSOLE_CORS_ALLOW_ORIGINS"] == ["https://example.com"] + assert config["WEB_API_CORS_ALLOW_ORIGINS"] == ["http://127.0.0.1:3000", "*"] + + assert str(config["CODE_EXECUTION_ENDPOINT"]) == "http://127.0.0.1:8194/" + assert str(URL(str(config["CODE_EXECUTION_ENDPOINT"])) / "v1") == "http://127.0.0.1:8194/v1" + def test_inner_api_config_exist(monkeypatch: pytest.MonkeyPatch): # Set environment variables using monkeypatch monkeypatch.setenv("CONSOLE_API_URL", "https://example.com") monkeypatch.setenv("CONSOLE_WEB_URL", "https://example.com") - monkeypatch.setenv("DB_USERNAME", "postgres") - monkeypatch.setenv("DB_PASSWORD", "postgres") - monkeypatch.setenv("DB_HOST", "localhost") - monkeypatch.setenv("DB_PORT", "5432") - monkeypatch.setenv("DB_DATABASE", "dify") + monkeypatch.setenv("POSTGRES_USER", "postgres") + monkeypatch.setenv("POSTGRES_PASSWORD", "postgres") + monkeypatch.setenv("POSTGRES_HOST", "localhost") + monkeypatch.setenv("POSTGRES_PORT", "5432") + monkeypatch.setenv("POSTGRES_DATABASE", "dify") monkeypatch.setenv("INNER_API_KEY", "test-inner-api-key") config = DifyConfig() @@ -140,11 +202,11 @@ def test_inner_api_config_exist(monkeypatch: pytest.MonkeyPatch): def test_db_extras_options_merging(monkeypatch: pytest.MonkeyPatch): """Test that DB_EXTRAS options are properly merged with default timezone setting""" # Set environment variables - monkeypatch.setenv("DB_USERNAME", "postgres") - monkeypatch.setenv("DB_PASSWORD", "postgres") - monkeypatch.setenv("DB_HOST", "localhost") - monkeypatch.setenv("DB_PORT", "5432") - monkeypatch.setenv("DB_DATABASE", "dify") + monkeypatch.setenv("POSTGRES_USER", "postgres") + monkeypatch.setenv("POSTGRES_PASSWORD", "postgres") + monkeypatch.setenv("POSTGRES_HOST", "localhost") + monkeypatch.setenv("POSTGRES_PORT", "5432") + monkeypatch.setenv("POSTGRES_DATABASE", "dify") monkeypatch.setenv("DB_EXTRAS", "options=-c search_path=myschema") # Create config @@ -199,11 +261,11 @@ def test_celery_broker_url_with_special_chars_password( # Set up basic required environment variables (following existing pattern) monkeypatch.setenv("CONSOLE_API_URL", "https://example.com") monkeypatch.setenv("CONSOLE_WEB_URL", "https://example.com") - monkeypatch.setenv("DB_USERNAME", "postgres") - monkeypatch.setenv("DB_PASSWORD", "postgres") - monkeypatch.setenv("DB_HOST", "localhost") - monkeypatch.setenv("DB_PORT", "5432") - monkeypatch.setenv("DB_DATABASE", "dify") + monkeypatch.setenv("POSTGRES_USER", "postgres") + monkeypatch.setenv("POSTGRES_PASSWORD", "postgres") + monkeypatch.setenv("POSTGRES_HOST", "localhost") + monkeypatch.setenv("POSTGRES_PORT", "5432") + monkeypatch.setenv("POSTGRES_DATABASE", "dify") # Set the CELERY_BROKER_URL to test monkeypatch.setenv("CELERY_BROKER_URL", broker_url) diff --git a/docker/.env.example b/docker/.env.example index 4c3be634a7..3665b624e4 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -282,7 +282,7 @@ POSTGRES_IDLE_IN_TRANSACTION_SESSION_TIMEOUT=0 # MySQL Default Configuration -MYSQL_USER=root +MYSQL_USER=mysql MYSQL_PASSWORD=difyai123456 MYSQL_HOST=db_mysql MYSQL_PORT=3306 diff --git a/docker/middleware.env.example b/docker/middleware.env.example index 9c64e8babc..c465c410a0 100644 --- a/docker/middleware.env.example +++ b/docker/middleware.env.example @@ -19,7 +19,7 @@ PGDATA=/var/lib/postgresql/data/pgdata PGDATA_HOST_VOLUME=./volumes/db/data # MySQL Configuration -MYSQL_USER=root +MYSQL_USER=mysql # MySQL password MYSQL_PASSWORD=difyai123456 # MySQL database name