test(api): manage backend pytest services natively (#36235)

2026-06-07 16:32:01 +08:00 · 2026-05-19 15:52:15 +08:00 · 2026-05-19 15:52:15 +08:00 · 34a89416f7
commit 34a89416f7
parent a13ab76002
40 changed files with 791 additions and 355 deletions
--- a/.coveragerc
+++ b/.coveragerc
@ -1,5 +1,6 @@
 [run]
 omit =
+    api/conftest.py
    api/tests/*
    api/migrations/*
    api/core/rag/datasource/vdb/*
--- a/.github/workflows/api-tests.yml
+++ b/.github/workflows/api-tests.yml
@ -48,10 +48,23 @@ jobs:
        run: uv sync --project api --dev

      - name: Run dify config tests
-        run: uv run --project api dev/pytest/pytest_config_tests.py
+        run: uv run --project api pytest api/tests/unit_tests/configs/test_env_consistency.py

      - name: Run Unit Tests
-        run: uv run --project api bash dev/pytest/pytest_unit_tests.sh
+        run: |
+          uv run --project api pytest \
+            -p no:benchmark \
+            --timeout "${PYTEST_TIMEOUT:-20}" \
+            -n auto \
+            api/tests/unit_tests \
+            api/providers/vdb/*/tests/unit_tests \
+            api/providers/trace/*/tests/unit_tests \
+            --ignore=api/tests/unit_tests/controllers
+          # Controller tests register Flask routes at import time, so keep them out of xdist.
+          uv run --project api pytest \
+            --timeout "${PYTEST_TIMEOUT:-20}" \
+            --cov-append \
+            api/tests/unit_tests/controllers

      - name: Upload unit coverage data
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
@ -96,32 +109,11 @@ jobs:
      - name: Install dependencies
        run: uv sync --project api --dev

-      - name: Set up dotenvs
-        run: |
-          cp docker/.env.example docker/.env
-          cp docker/envs/middleware.env.example docker/middleware.env
-
-      - name: Expose Service Ports
-        run: sh .github/workflows/expose_service_ports.sh
-
-      - name: Set up Sandbox
-        uses: hoverkraft-tech/compose-action@d2bee4f07e8ca410d6b196d00f90c12e7d48c33a # v2.6.0
-        with:
-          compose-file: |
-            docker/docker-compose.middleware.yaml
-          services: |
-            db_postgres
-            redis
-            sandbox
-            ssrf_proxy
-
-      - name: setup test config
-        run: |
-          cp api/tests/integration_tests/.env.example api/tests/integration_tests/.env
-
      - name: Run Integration Tests
        run: |
          uv run --project api pytest \
+            -p no:benchmark \
+            --start-middleware \
            -n auto \
            --timeout "${PYTEST_TIMEOUT:-180}" \
            api/tests/integration_tests/workflow \
--- a/.github/workflows/expose_service_ports.sh
+++ b/.github/workflows/expose_service_ports.sh
@ -1,17 +0,0 @@
-#!/bin/bash
-
-yq eval '.services.weaviate.ports += ["8080:8080"]' -i docker/docker-compose.yaml
-yq eval '.services.weaviate.ports += ["50051:50051"]' -i docker/docker-compose.yaml
-yq eval '.services.qdrant.ports += ["6333:6333"]' -i docker/docker-compose.yaml
-yq eval '.services.chroma.ports += ["8000:8000"]' -i docker/docker-compose.yaml
-yq eval '.services["milvus-standalone"].ports += ["19530:19530"]' -i docker/docker-compose.yaml
-yq eval '.services.pgvector.ports += ["5433:5432"]' -i docker/docker-compose.yaml
-yq eval '.services["pgvecto-rs"].ports += ["5431:5432"]' -i docker/docker-compose.yaml
-yq eval '.services["elasticsearch"].ports += ["9200:9200"]' -i docker/docker-compose.yaml
-yq eval '.services.couchbase-server.ports += ["8091-8096:8091-8096"]' -i docker/docker-compose.yaml
-yq eval '.services.couchbase-server.ports += ["11210:11210"]' -i docker/docker-compose.yaml
-yq eval '.services.tidb.ports += ["4000:4000"]' -i docker/tidb/docker-compose.yaml
-yq eval '.services.oceanbase.ports += ["2881:2881"]' -i docker/docker-compose.yaml
-yq eval '.services.opengauss.ports += ["6600:6600"]' -i docker/docker-compose.yaml
-
-echo "Ports exposed for sandbox, weaviate (HTTP 8080, gRPC 50051), tidb, qdrant, chroma, milvus, pgvector, pgvecto-rs, elasticsearch, couchbase, opengauss"
--- a/.github/workflows/main-ci.yml
+++ b/.github/workflows/main-ci.yml
@ -55,7 +55,6 @@ jobs:
            api:
              - 'api/**'
              - '.github/workflows/api-tests.yml'
-              - '.github/workflows/expose_service_ports.sh'
              - 'docker/.env.example'
              - 'docker/envs/middleware.env.example'
              - 'docker/docker-compose.middleware.yaml'
@ -90,11 +89,13 @@ jobs:
            vdb:
              - 'api/core/rag/datasource/**'
              - 'api/tests/integration_tests/vdb/**'
+              - 'api/conftest.py'
+              - 'api/tests/pytest_dify.py'
              - 'api/providers/vdb/*/tests/**'
              - '.github/workflows/vdb-tests.yml'
-              - '.github/workflows/expose_service_ports.sh'
              - 'docker/.env.example'
              - 'docker/envs/middleware.env.example'
+              - 'docker/docker-compose.pytest.ports.yaml'
              - 'docker/docker-compose.yaml'
              - 'docker/docker-compose-template.yaml'
              - 'docker/generate_docker_compose'
@ -114,7 +115,6 @@ jobs:
              - 'api/migrations/**'
              - 'api/.env.example'
              - '.github/workflows/db-migration-test.yml'
-              - '.github/workflows/expose_service_ports.sh'
              - 'docker/.env.example'
              - 'docker/envs/middleware.env.example'
              - 'docker/docker-compose.middleware.yaml'
--- a/.github/workflows/vdb-tests-full.yml
+++ b/.github/workflows/vdb-tests-full.yml
@ -48,14 +48,6 @@ jobs:
      - name: Install dependencies
        run: uv sync --project api --dev

-      - name: Set up dotenvs
-        run: |
-          cp docker/.env.example docker/.env
-          cp docker/envs/middleware.env.example docker/middleware.env
-
-      - name: Expose Service Ports
-        run: sh .github/workflows/expose_service_ports.sh
-
 #      - name: Set up Vector Store (TiDB)
 #        uses: hoverkraft-tech/compose-action@v2.0.2
 #        with:
@ -64,32 +56,13 @@ jobs:
 #            tidb
 #            tiflash

-      - name: Set up Full Vector Store Matrix
-        uses: hoverkraft-tech/compose-action@d2bee4f07e8ca410d6b196d00f90c12e7d48c33a # v2.6.0
-        with:
-          compose-file: |
-            docker/docker-compose.yaml
-          services: |
-            weaviate
-            qdrant
-            couchbase-server
-            etcd
-            minio
-            milvus-standalone
-            pgvecto-rs
-            pgvector
-            chroma
-            elasticsearch
-            oceanbase
-
-      - name: setup test config
-        run: |
-          echo $(pwd)
-          ls -lah .
-          cp api/tests/integration_tests/.env.example api/tests/integration_tests/.env
-
 #      - name: Check VDB Ready (TiDB)
 #        run: uv run --project api python api/providers/vdb/tidb-vector/tests/integration_tests/check_tiflash_ready.py

      - name: Test Vector Stores
-        run: uv run --project api bash dev/pytest/pytest_vdb.sh
+        run: |
+          uv run --project api pytest \
+            --start-vdb \
+            --vdb-services "weaviate,qdrant,couchbase-server,etcd,minio,milvus-standalone,pgvecto-rs,pgvector,chroma,elasticsearch,oceanbase" \
+            --timeout "${PYTEST_TIMEOUT:-180}" \
+            api/providers/vdb/*/tests/integration_tests
--- a/.github/workflows/vdb-tests.yml
+++ b/.github/workflows/vdb-tests.yml
@ -45,14 +45,6 @@ jobs:
      - name: Install dependencies
        run: uv sync --project api --dev

-      - name: Set up dotenvs
-        run: |
-          cp docker/.env.example docker/.env
-          cp docker/envs/middleware.env.example docker/middleware.env
-
-      - name: Expose Service Ports
-        run: sh .github/workflows/expose_service_ports.sh
-
 #      - name: Set up Vector Store (TiDB)
 #        uses: hoverkraft-tech/compose-action@v2.0.2
 #        with:
@ -61,31 +53,14 @@ jobs:
 #            tidb
 #            tiflash

-      - name: Set up Vector Stores for Smoke Coverage
-        uses: hoverkraft-tech/compose-action@d2bee4f07e8ca410d6b196d00f90c12e7d48c33a # v2.6.0
-        with:
-          compose-file: |
-            docker/docker-compose.yaml
-          services: |
-            db_postgres
-            redis
-            weaviate
-            qdrant
-            pgvector
-            chroma
-
-      - name: setup test config
-        run: |
-          echo $(pwd)
-          ls -lah .
-          cp api/tests/integration_tests/.env.example api/tests/integration_tests/.env
-
 #      - name: Check VDB Ready (TiDB)
 #        run: uv run --project api python api/providers/vdb/tidb-vector/tests/integration_tests/check_tiflash_ready.py

      - name: Test Vector Stores
        run: |
-          uv run --project api pytest --timeout "${PYTEST_TIMEOUT:-180}" \
+          uv run --project api pytest \
+            --start-vdb \
+            --timeout "${PYTEST_TIMEOUT:-180}" \
            api/providers/vdb/vdb-chroma/tests/integration_tests \
            api/providers/vdb/vdb-pgvector/tests/integration_tests \
            api/providers/vdb/vdb-qdrant/tests/integration_tests \
--- a/48
+++ b/48
@ -85,13 +85,13 @@ lint:
 type-check:
 	@echo "📝 Running type checks (pyrefly + mypy)..."
 	@./dev/pyrefly-check-local $(PATH_TO_CHECK)
-	@uv --directory api run mypy --exclude-gitignore --exclude 'tests/' --exclude 'migrations/' --exclude 'dev/generate_swagger_specs.py' --exclude 'dev/generate_fastopenapi_specs.py' --check-untyped-defs --disable-error-code=import-untyped .
+	@uv --directory api run mypy --exclude-gitignore --exclude '(^|/)conftest\.py$$' --exclude 'tests/' --exclude 'migrations/' --exclude 'dev/generate_swagger_specs.py' --exclude 'dev/generate_fastopenapi_specs.py' --check-untyped-defs --disable-error-code=import-untyped .
 	@echo "✅ Type checks complete"

 type-check-core:
 	@echo "📝 Running core type checks (pyrefly + mypy)..."
 	@./dev/pyrefly-check-local $(PATH_TO_CHECK)
-	@uv --directory api run mypy --exclude-gitignore --exclude 'tests/' --exclude 'migrations/' --exclude 'dev/generate_swagger_specs.py' --exclude 'dev/generate_fastopenapi_specs.py' --check-untyped-defs --disable-error-code=import-untyped .
+	@uv --directory api run mypy --exclude-gitignore --exclude '(^|/)conftest\.py$$' --exclude 'tests/' --exclude 'migrations/' --exclude 'dev/generate_swagger_specs.py' --exclude 'dev/generate_fastopenapi_specs.py' --check-untyped-defs --disable-error-code=import-untyped .
 	@echo "✅ Core type checks complete"

 test:
@ -100,7 +100,46 @@ test:
 		echo "Target: $(TARGET_TESTS)"; \
 		uv run --project api --dev pytest $(TARGET_TESTS); \
 	else \
-		PYTEST_XDIST_ARGS="-n auto" uv run --project api --dev dev/pytest/pytest_unit_tests.sh; \
+		echo "Running backend unit tests"; \
+		uv run --project api --dev pytest -p no:benchmark --timeout "$${PYTEST_TIMEOUT:-20}" -n auto \
+			api/tests/unit_tests \
+			api/providers/vdb/*/tests/unit_tests \
+			api/providers/trace/*/tests/unit_tests \
+			--ignore=api/tests/unit_tests/controllers; \
+		uv run --project api --dev pytest --timeout "$${PYTEST_TIMEOUT:-20}" --cov-append \
+			api/tests/unit_tests/controllers; \
+	fi
+	@echo "✅ Unit tests complete"
+
+test-all:
+	@echo "🧪 Running full backend test suite..."
+	@if [ -n "$(TARGET_TESTS)" ]; then \
+		echo "Target: $(TARGET_TESTS)"; \
+		uv run --project api --dev pytest $(TARGET_TESTS); \
+	else \
+		echo "Running backend unit tests"; \
+		uv run --project api --dev pytest -p no:benchmark --timeout "$${PYTEST_TIMEOUT:-20}" -n auto \
+			api/tests/unit_tests \
+			api/providers/vdb/*/tests/unit_tests \
+			api/providers/trace/*/tests/unit_tests \
+			--ignore=api/tests/unit_tests/controllers; \
+		uv run --project api --dev pytest --timeout "$${PYTEST_TIMEOUT:-20}" --cov-append \
+			api/tests/unit_tests/controllers; \
+		echo "Running backend integration tests"; \
+		uv run --project api --dev pytest -p no:benchmark --start-middleware -n auto \
+			--timeout "$${PYTEST_TIMEOUT:-180}" \
+			--cov-append \
+			api/tests/integration_tests/workflow \
+			api/tests/integration_tests/tools \
+			api/tests/test_containers_integration_tests; \
+		echo "Running VDB smoke tests"; \
+		uv run --project api --dev pytest --start-vdb \
+			--timeout "$${PYTEST_TIMEOUT:-180}" \
+			--cov-append \
+			api/providers/vdb/vdb-chroma/tests/integration_tests \
+			api/providers/vdb/vdb-pgvector/tests/integration_tests \
+			api/providers/vdb/vdb-qdrant/tests/integration_tests \
+			api/providers/vdb/vdb-weaviate/tests/integration_tests; \
 	fi
 	@echo "✅ Tests complete"

@ -155,6 +194,7 @@ help:
 	@echo "  make type-check     - Run type checks (pyrefly, mypy)"
 	@echo "  make type-check-core - Run core type checks (pyrefly, mypy)"
 	@echo "  make test           - Run backend unit tests (or TARGET_TESTS=./api/tests/<target_tests>)"
+	@echo "  make test-all       - Run full backend tests, including Docker-backed suites"
 	@echo ""
 	@echo "Docker Build Targets:"
 	@echo "  make build-web      - Build web Docker image"
@ -164,4 +204,4 @@ help:
 	@echo "  make build-push-all - Build and push all Docker images"

 # Phony targets
-.PHONY: build-web build-api push-web push-api build-all push-all build-push-all dev-setup prepare-docker prepare-web prepare-api dev-clean help format check lint type-check test
+.PHONY: build-web build-api push-web push-api build-all push-all build-push-all dev-setup prepare-docker prepare-web prepare-api dev-clean help format check lint type-check test test-all
--- a/api/AGENTS.md
+++ b/api/AGENTS.md
@ -180,6 +180,8 @@ Quick checks while iterating:
 - Format: `make format`
 - Lint (includes auto-fix): `make lint`
 - Type check: `make type-check`
+- Unit tests: `make test`
+- Full backend tests, including Docker-backed suites: `make test-all`
 - Targeted tests: `make test TARGET_TESTS=./api/tests/<target_tests>`

 Before opening a PR / submitting:
--- a/api/conftest.py
+++ b/api/conftest.py
@ -0,0 +1,91 @@
+"""Global pytest hooks for Dify backend tests.
+
+This root conftest is loaded before package-specific conftests, which lets tests opt
+into Docker-backed middleware before application modules read environment config.
+It intentionally lives at the API root because pytest applies conftest.py files to
+tests below their directory, and this setup is shared by api/tests and api/providers.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import pytest
+
+from tests.pytest_dify import (
+    DEFAULT_MIDDLEWARE_SERVICES,
+    DEFAULT_VDB_SERVICES,
+    DockerComposeStack,
+    build_middleware_stack,
+    build_vdb_stack,
+    ensure_backend_test_environment,
+    ensure_compose_env_files,
+    parse_services,
+)
+
+_REPO_ROOT = Path(__file__).resolve().parent.parent
+_DIFY_COMPOSE_STACKS_KEY = pytest.StashKey[list[DockerComposeStack]]()
+
+# This must run at import time because package-specific conftests can import the
+# Flask app before pytest_configure hooks from this file are called.
+ensure_backend_test_environment(_REPO_ROOT)
+
+
+def pytest_addoption(parser: pytest.Parser) -> None:
+    group = parser.getgroup("dify")
+    group.addoption(
+        "--start-middleware",
+        action="store_true",
+        default=False,
+        help="Start the Docker middleware services needed by API integration tests.",
+    )
+    group.addoption(
+        "--middleware-services",
+        default=",".join(DEFAULT_MIDDLEWARE_SERVICES),
+        help="Comma-separated services from docker/docker-compose.middleware.yaml to start.",
+    )
+    group.addoption(
+        "--start-vdb",
+        action="store_true",
+        default=False,
+        help="Start vector-store Docker services for VDB integration tests.",
+    )
+    group.addoption(
+        "--vdb-services",
+        default=",".join(DEFAULT_VDB_SERVICES),
+        help="Comma-separated services from docker/docker-compose.yaml to start for VDB tests.",
+    )
+
+
+def pytest_configure(config: pytest.Config) -> None:
+    config.stash[_DIFY_COMPOSE_STACKS_KEY] = []
+
+
+def pytest_sessionstart(session: pytest.Session) -> None:
+    config = session.config
+    if hasattr(config, "workerinput"):
+        return
+
+    stacks: list[DockerComposeStack] = []
+    if config.getoption("start_middleware"):
+        ensure_compose_env_files(_REPO_ROOT)
+        stack = build_middleware_stack(_REPO_ROOT, parse_services(config.getoption("middleware_services")))
+        stack.up()
+        stacks.append(stack)
+
+    if config.getoption("start_vdb"):
+        ensure_compose_env_files(_REPO_ROOT)
+        stack = build_vdb_stack(_REPO_ROOT, parse_services(config.getoption("vdb_services")))
+        stack.up()
+        stacks.append(stack)
+
+    config.stash[_DIFY_COMPOSE_STACKS_KEY] = stacks
+
+
+def pytest_unconfigure(config: pytest.Config) -> None:
+    if hasattr(config, "workerinput"):
+        return
+
+    stacks = config.stash.get(_DIFY_COMPOSE_STACKS_KEY, [])
+    for stack in reversed(stacks):
+        stack.down()
--- a/api/events/init.py
+++ b/api/events/init.py
@ -0,0 +1,67 @@
+"""Dify event package.
+
+The package name intentionally stays as ``events`` for existing Dify imports. Some
+third-party clients also import ``Events`` from a top-level ``events`` package, so
+we expose a small compatible implementation to avoid import shadowing failures.
+"""
+
+from collections.abc import Callable, Iterator
+from typing import Any
+
+
+class EventsError(Exception):
+    """Raised for invalid event slot operations."""
+
+
+EventsException = EventsError
+
+
+class _EventSlot:
+    """A dynamically-created event slot supporting ``+=`` and call dispatch."""
+
+    targets: list[Callable[..., Any]]
+    __name__: str
+
+    def __init__(self, name: str) -> None:
+        self.targets = []
+        self.__name__ = name
+
+    def __call__(self, *args: Any, **kwargs: Any) -> None:
+        for target in tuple(self.targets):
+            target(*args, **kwargs)
+
+    def __iadd__(self, target: Callable[..., Any]) -> "_EventSlot":
+        self.targets.append(target)
+        return self
+
+    def __isub__(self, target: Callable[..., Any]) -> "_EventSlot":
+        while target in self.targets:
+            self.targets.remove(target)
+        return self
+
+    def __iter__(self) -> Iterator[Callable[..., Any]]:
+        return iter(self.targets)
+
+    def __len__(self) -> int:
+        return len(self.targets)
+
+
+class Events:
+    """A minimal C#-style event container compatible with the external Events package."""
+
+    _slots: dict[str, _EventSlot]
+
+    def __init__(self, *event_names: str) -> None:
+        self._slots = {}
+        for event_name in event_names:
+            self._slots[event_name] = _EventSlot(event_name)
+
+    def __getattr__(self, name: str) -> _EventSlot:
+        if name.startswith("_"):
+            raise AttributeError(name)
+        slot = _EventSlot(name)
+        self._slots[name] = slot
+        return slot
+
+
+__all__ = ["Events", "EventsError", "EventsException"]
--- a/api/providers/vdb/vdb-chroma/tests/integration_tests/test_chroma.py
+++ b/api/providers/vdb/vdb-chroma/tests/integration_tests/test_chroma.py
@ -13,7 +13,7 @@ class ChromaVectorTest(AbstractVectorTest):
        self.vector = ChromaVector(
            collection_name=self.collection_name,
            config=ChromaConfig(
-                host="localhost",
+                host="127.0.0.1",
                port=8000,
                tenant=chromadb.DEFAULT_TENANT,
                database=chromadb.DEFAULT_DATABASE,
--- a/api/providers/vdb/vdb-qdrant/tests/integration_tests/test_qdrant.py
+++ b/api/providers/vdb/vdb-qdrant/tests/integration_tests/test_qdrant.py
@ -16,7 +16,7 @@ class QdrantVectorTest(AbstractVectorTest):
            collection_name=self.collection_name,
            group_id=self.dataset_id,
            config=QdrantConfig(
-                endpoint="http://localhost:6333",
+                endpoint="http://127.0.0.1:6333",
                api_key="difyai123456",
            ),
        )
--- a/api/tests/integration_tests/conftest.py
+++ b/api/tests/integration_tests/conftest.py
@ -26,20 +26,24 @@ _logger = logging.getLogger(__name__)
 # Loading the .env file if it exists
 def _load_env():
    current_file_path = pathlib.Path(__file__).absolute()
-    # Items later in the list have higher precedence.
    env_file_paths = [
-        os.getenv("DIFY_TEST_ENV_FILE", str(current_file_path.parent / _DEFUALT_TEST_ENV)),
-        os.getenv("DIFY_VDB_TEST_ENV_FILE", str(current_file_path.parent / _DEFAULT_VDB_TEST_ENV)),
+        pathlib.Path(os.getenv("DIFY_TEST_ENV_FILE", str(current_file_path.parent / _DEFUALT_TEST_ENV))),
    ]
+    vdb_env_path = pathlib.Path(
+        os.getenv("DIFY_VDB_TEST_ENV_FILE", str(current_file_path.parent / _DEFAULT_VDB_TEST_ENV))
+    )
+    if vdb_env_path.exists() or "DIFY_VDB_TEST_ENV_FILE" in os.environ:
+        env_file_paths.append(vdb_env_path)

-    for env_path_str in env_file_paths:
-        if not pathlib.Path(env_path_str).exists():
-            _logger.warning("specified configuration file %s not exist", env_path_str)
+    for env_path in env_file_paths:
+        if not env_path.exists():
+            _logger.warning("specified configuration file %s not exist", env_path)
+            continue

        from dotenv import load_dotenv

        # Set `override=True` to ensure values from `vdb.env` take priority over values from `.env`
-        load_dotenv(str(env_path_str), override=True)
+        load_dotenv(str(env_path), override=True)


 _load_env()
--- a/api/tests/pytest_dify.py
+++ b/api/tests/pytest_dify.py
@ -0,0 +1,198 @@
+"""Pytest support helpers for Dify backend test environment setup.
+
+The helpers in this module keep Docker and environment preparation behind explicit
+pytest options so ordinary unit-test runs do not start external services.
+"""
+
+from __future__ import annotations
+
+import os
+import shutil
+import subprocess
+import time
+import urllib.error
+import urllib.request
+from dataclasses import dataclass
+from pathlib import Path
+
+DEFAULT_LOG_FORMAT = "%(asctime)s,%(msecs)d %(levelname)-2s [%(filename)s:%(lineno)d] %(req_id)s %(message)s"
+DEFAULT_MIDDLEWARE_SERVICES = ("db_postgres", "redis", "sandbox", "ssrf_proxy")
+DEFAULT_VDB_SERVICES = ("db_postgres", "redis", "weaviate", "qdrant", "pgvector", "chroma")
+VDB_SERVICE_PROFILES = {
+    "db_postgres": "postgresql",
+    "weaviate": "weaviate",
+    "qdrant": "qdrant",
+    "couchbase-server": "couchbase",
+    "etcd": "milvus",
+    "minio": "milvus",
+    "milvus-standalone": "milvus",
+    "pgvecto-rs": "pgvecto-rs",
+    "pgvector": "pgvector",
+    "chroma": "chroma",
+    "elasticsearch": "elasticsearch",
+    "oceanbase": "oceanbase",
+}
+
+
+def parse_services(value: str) -> list[str]:
+    """Parse a comma-separated service list from a pytest option."""
+    return [service.strip() for service in value.split(",") if service.strip()]
+
+
+def ensure_backend_test_environment(repo_root: Path) -> None:
+    """Set deterministic defaults needed before test conftests import application config."""
+    integration_tests_dir = repo_root / "api" / "tests" / "integration_tests"
+    test_env_file = integration_tests_dir / ".env"
+    test_env_example_file = integration_tests_dir / ".env.example"
+    vdb_env_file = integration_tests_dir / "vdb.env"
+
+    if "DIFY_TEST_ENV_FILE" not in os.environ:
+        os.environ["DIFY_TEST_ENV_FILE"] = str(test_env_file if test_env_file.exists() else test_env_example_file)
+
+    if "DIFY_VDB_TEST_ENV_FILE" not in os.environ and vdb_env_file.exists():
+        os.environ["DIFY_VDB_TEST_ENV_FILE"] = str(vdb_env_file)
+
+    os.environ["LOG_OUTPUT_FORMAT"] = "text"
+    os.environ["LOG_FORMAT"] = DEFAULT_LOG_FORMAT
+    os.environ.setdefault("STORAGE_TYPE", "opendal")
+    os.environ.setdefault("OPENDAL_SCHEME", "fs")
+    os.environ.setdefault("OPENDAL_FS_ROOT", "/tmp/dify-storage")
+    Path(os.environ["OPENDAL_FS_ROOT"]).mkdir(parents=True, exist_ok=True)
+
+
+def ensure_compose_env_files(repo_root: Path) -> None:
+    """Create ignored Docker env files from examples when Docker-backed tests request compose."""
+    docker_dir = repo_root / "docker"
+    env_file = docker_dir / ".env"
+    env_example_file = docker_dir / ".env.example"
+    middleware_env_file = docker_dir / "middleware.env"
+    middleware_env_example_file = docker_dir / "envs" / "middleware.env.example"
+
+    if not env_file.exists():
+        shutil.copyfile(env_example_file, env_file)
+    if not middleware_env_file.exists():
+        shutil.copyfile(middleware_env_example_file, middleware_env_file)
+
+
+@dataclass(frozen=True)
+class DockerComposeStack:
+    """A docker compose project that pytest can start before collection and stop at shutdown."""
+
+    name: str
+    project_name: str
+    repo_root: Path
+    compose_files: tuple[Path, ...]
+    env_file: Path
+    services: tuple[str, ...]
+    profiles: tuple[str, ...] = ()
+    ready_delay_seconds: float = 0.0
+    warmup_urls: tuple[str, ...] = ()
+
+    def _compose_command(self) -> list[str]:
+        command = [
+            "docker",
+            "compose",
+            "--project-name",
+            self.project_name,
+            "--env-file",
+            str(self.env_file),
+        ]
+        for profile in self.profiles:
+            command.extend(("--profile", profile))
+        for compose_file in self.compose_files:
+            command.extend(("-f", str(compose_file)))
+        return command
+
+    def up(self) -> None:
+        """Start the configured services and wait for compose healthchecks when supported."""
+        wait_command = self._compose_command() + [
+            "up",
+            "-d",
+            "--wait",
+            "--wait-timeout",
+            "180",
+            *self.services,
+        ]
+        completed = subprocess.run(wait_command, cwd=self.repo_root, text=True, capture_output=True)
+        if completed.returncode == 0:
+            if self.ready_delay_seconds > 0:
+                time.sleep(self.ready_delay_seconds)
+            self._warm_up()
+            return
+
+        combined_output = f"{completed.stdout}\n{completed.stderr}"
+        if "unknown flag: --wait" in combined_output or "unknown flag: wait-timeout" in combined_output:
+            subprocess.run(self._compose_command() + ["up", "-d", *self.services], cwd=self.repo_root, check=True)
+            time.sleep(5)
+            self._warm_up()
+            return
+
+        raise subprocess.CalledProcessError(
+            returncode=completed.returncode,
+            cmd=wait_command,
+            output=completed.stdout,
+            stderr=completed.stderr,
+        )
+
+    def down(self) -> None:
+        """Stop services started for this pytest run."""
+        subprocess.run(self._compose_command() + ["down"], cwd=self.repo_root, check=True)
+
+    def _warm_up(self) -> None:
+        for url in self.warmup_urls:
+            deadline = time.monotonic() + 30.0
+            last_error: Exception | None = None
+            while time.monotonic() < deadline:
+                try:
+                    with urllib.request.urlopen(url, timeout=5) as response:
+                        if 200 <= response.status < 300:
+                            break
+                except urllib.error.HTTPError as error:
+                    if error.code < 500:
+                        break
+                    last_error = error
+                except (OSError, urllib.error.URLError) as error:
+                    last_error = error
+                time.sleep(1)
+            else:
+                raise RuntimeError(f"Timed out waiting for {self.name} warmup URL {url}") from last_error
+
+
+def build_middleware_stack(repo_root: Path, services: list[str]) -> DockerComposeStack:
+    """Build the middleware compose stack used by API integration tests."""
+    return DockerComposeStack(
+        name="middleware",
+        project_name="dify-pytest-middleware",
+        repo_root=repo_root,
+        compose_files=(repo_root / "docker" / "docker-compose.middleware.yaml",),
+        env_file=repo_root / "docker" / "middleware.env",
+        services=tuple(services),
+        ready_delay_seconds=5.0,
+        warmup_urls=("http://127.0.0.1:8194/health",),
+    )
+
+
+def build_vdb_stack(repo_root: Path, services: list[str]) -> DockerComposeStack:
+    """Build the vector-store compose stack used by VDB integration tests."""
+    profiles = tuple(
+        dict.fromkeys(profile for service in services if (profile := VDB_SERVICE_PROFILES.get(service)) is not None)
+    )
+    service_names = set(services)
+    warmup_urls = []
+    if "qdrant" in service_names:
+        warmup_urls.append("http://127.0.0.1:6333/collections")
+    if "chroma" in service_names:
+        warmup_urls.append("http://127.0.0.1:8000/api/v2/auth/identity")
+    return DockerComposeStack(
+        name="vdb",
+        project_name="dify-pytest-vdb",
+        repo_root=repo_root,
+        compose_files=(
+            repo_root / "docker" / "docker-compose.yaml",
+            repo_root / "docker" / "docker-compose.pytest.ports.yaml",
+        ),
+        env_file=repo_root / "docker" / ".env",
+        services=tuple(services),
+        profiles=profiles,
+        warmup_urls=tuple(warmup_urls),
+    )
--- a/api/tests/test_containers_integration_tests/libs/broadcast_channel/redis/test_channel.py
+++ b/api/tests/test_containers_integration_tests/libs/broadcast_channel/redis/test_channel.py
@ -252,6 +252,9 @@ class TestRedisBroadcastChannelIntegration:
        def consumer_thread() -> set[bytes]:
            received_msgs: set[bytes] = set()
            with subscription:
+                # Prime the subscription before producers publish. Redis Pub/Sub does not
+                # replay messages sent before the subscribe command is active.
+                subscription.receive(timeout=0.1)
                consumer_ready.set()
                while True:
                    try:
@ -278,8 +281,10 @@ class TestRedisBroadcastChannelIntegration:
            for future in as_completed(producer_futures, timeout=30.0):
                sent_msgs.update(future.result())

-            subscription.close()
-            consumer_received_msgs = consumer_future.result(timeout=30.0)
+            try:
+                consumer_received_msgs = consumer_future.result(timeout=30.0)
+            finally:
+                subscription.close()

        # Verify message content
        assert sent_msgs == consumer_received_msgs
--- a/api/tests/test_containers_integration_tests/services/test_app_generate_service.py
+++ b/api/tests/test_containers_integration_tests/services/test_app_generate_service.py
@ -37,9 +37,9 @@ class TestAppGenerateService:
                "services.app_generate_service.MessageBasedAppGenerator", autospec=True
            ) as mock_message_based_generator,
            patch("services.account_service.FeatureService", autospec=True) as mock_account_feature_service,
-            patch("services.app_generate_service.dify_config", autospec=True) as mock_dify_config,
-            patch("services.quota_service.dify_config", autospec=True) as mock_quota_dify_config,
-            patch("configs.dify_config", autospec=True) as mock_global_dify_config,
+            patch("services.app_generate_service.dify_config") as mock_dify_config,
+            patch("services.quota_service.dify_config") as mock_quota_dify_config,
+            patch("configs.dify_config") as mock_global_dify_config,
        ):
            # Setup default mock returns for billing service
            mock_billing_service.quota_reserve.return_value = {
--- a/api/tests/test_containers_integration_tests/services/test_recommended_app_service.py
+++ b/api/tests/test_containers_integration_tests/services/test_recommended_app_service.py
@ -84,7 +84,7 @@ def _mock_factory_for_apps(

 class TestRecommendedAppServiceGetApps:
    @patch("services.recommended_app_service.RecommendAppRetrievalFactory", autospec=True)
-    @patch("services.recommended_app_service.dify_config", autospec=True)
+    @patch("services.recommended_app_service.dify_config")
    def test_success_with_apps(self, mock_config, mock_factory_class):
        mock_config.HOSTED_FETCH_APP_TEMPLATES_MODE = "remote"
        expected = _apps_response()
@ -103,7 +103,7 @@ class TestRecommendedAppServiceGetApps:
        mock_instance.get_recommended_apps_and_categories.assert_called_once_with("en-US")

    @patch("services.recommended_app_service.RecommendAppRetrievalFactory", autospec=True)
-    @patch("services.recommended_app_service.dify_config", autospec=True)
+    @patch("services.recommended_app_service.dify_config")
    def test_fallback_to_builtin_when_empty(self, mock_config, mock_factory_class):
        mock_config.HOSTED_FETCH_APP_TEMPLATES_MODE = "remote"
        empty_response = {"recommended_apps": [], "categories": []}
@ -126,7 +126,7 @@ class TestRecommendedAppServiceGetApps:
        mock_builtin_instance.fetch_recommended_apps_from_builtin.assert_called_once_with("en-US")

    @patch("services.recommended_app_service.RecommendAppRetrievalFactory", autospec=True)
-    @patch("services.recommended_app_service.dify_config", autospec=True)
+    @patch("services.recommended_app_service.dify_config")
    def test_fallback_when_none_recommended_apps(self, mock_config, mock_factory_class):
        mock_config.HOSTED_FETCH_APP_TEMPLATES_MODE = "db"
        none_response = {"recommended_apps": None, "categories": ["test"]}
@ -146,7 +146,7 @@ class TestRecommendedAppServiceGetApps:
        mock_builtin_instance.fetch_recommended_apps_from_builtin.assert_called_once()

    @patch("services.recommended_app_service.RecommendAppRetrievalFactory", autospec=True)
-    @patch("services.recommended_app_service.dify_config", autospec=True)
+    @patch("services.recommended_app_service.dify_config")
    def test_different_languages(self, mock_config, mock_factory_class):
        mock_config.HOSTED_FETCH_APP_TEMPLATES_MODE = "builtin"

@ -164,7 +164,7 @@ class TestRecommendedAppServiceGetApps:
            mock_instance.get_recommended_apps_and_categories.assert_called_with(language)

    @patch("services.recommended_app_service.RecommendAppRetrievalFactory", autospec=True)
-    @patch("services.recommended_app_service.dify_config", autospec=True)
+    @patch("services.recommended_app_service.dify_config")
    def test_uses_correct_factory_mode(self, mock_config, mock_factory_class):
        for mode in ["remote", "builtin", "db"]:
            mock_config.HOSTED_FETCH_APP_TEMPLATES_MODE = mode
@ -183,7 +183,7 @@ class TestRecommendedAppServiceGetApps:

 class TestRecommendedAppServiceGetDetail:
    @patch("services.recommended_app_service.RecommendAppRetrievalFactory", autospec=True)
-    @patch("services.recommended_app_service.dify_config", autospec=True)
+    @patch("services.recommended_app_service.dify_config")
    def test_success(self, mock_config, mock_factory_class):
        mock_config.HOSTED_FETCH_APP_TEMPLATES_MODE = "remote"
        expected = _app_detail(app_id="app-123", name="Productivity App", description="A great app")
@ -199,7 +199,7 @@ class TestRecommendedAppServiceGetDetail:
        mock_instance.get_recommend_app_detail.assert_called_once_with("app-123")

    @patch("services.recommended_app_service.RecommendAppRetrievalFactory", autospec=True)
-    @patch("services.recommended_app_service.dify_config", autospec=True)
+    @patch("services.recommended_app_service.dify_config")
    def test_different_modes(self, mock_config, mock_factory_class):
        for mode in ["remote", "builtin", "db"]:
            mock_config.HOSTED_FETCH_APP_TEMPLATES_MODE = mode
@ -214,7 +214,7 @@ class TestRecommendedAppServiceGetDetail:
            mock_factory_class.get_recommend_app_factory.assert_called_with(mode)

    @patch("services.recommended_app_service.RecommendAppRetrievalFactory", autospec=True)
-    @patch("services.recommended_app_service.dify_config", autospec=True)
+    @patch("services.recommended_app_service.dify_config")
    def test_returns_none_when_not_found(self, mock_config, mock_factory_class):
        mock_config.HOSTED_FETCH_APP_TEMPLATES_MODE = "remote"
        mock_instance = MagicMock()
@ -227,7 +227,7 @@ class TestRecommendedAppServiceGetDetail:
        mock_instance.get_recommend_app_detail.assert_called_once_with("nonexistent")

    @patch("services.recommended_app_service.RecommendAppRetrievalFactory", autospec=True)
-    @patch("services.recommended_app_service.dify_config", autospec=True)
+    @patch("services.recommended_app_service.dify_config")
    def test_returns_empty_dict(self, mock_config, mock_factory_class):
        mock_config.HOSTED_FETCH_APP_TEMPLATES_MODE = "builtin"
        mock_instance = MagicMock()
@ -239,7 +239,7 @@ class TestRecommendedAppServiceGetDetail:
        assert result == {}

    @patch("services.recommended_app_service.RecommendAppRetrievalFactory", autospec=True)
-    @patch("services.recommended_app_service.dify_config", autospec=True)
+    @patch("services.recommended_app_service.dify_config")
    def test_complex_model_config(self, mock_config, mock_factory_class):
        mock_config.HOSTED_FETCH_APP_TEMPLATES_MODE = "remote"
        complex_config = {
--- a/api/tests/test_containers_integration_tests/tasks/test_mail_invite_member_task.py
+++ b/api/tests/test_containers_integration_tests/tasks/test_mail_invite_member_task.py
@ -60,7 +60,7 @@ class TestMailInviteMemberTask:
        with (
            patch("tasks.mail_invite_member_task.mail", autospec=True) as mock_mail,
            patch("tasks.mail_invite_member_task.get_email_i18n_service", autospec=True) as mock_email_service,
-            patch("tasks.mail_invite_member_task.dify_config", autospec=True) as mock_config,
+            patch("tasks.mail_invite_member_task.dify_config") as mock_config,
        ):
            # Setup mail service mock
            mock_mail.is_inited.return_value = True
--- a/api/tests/test_containers_integration_tests/tasks/test_mail_register_task.py
+++ b/api/tests/test_containers_integration_tests/tasks/test_mail_register_task.py
@ -90,7 +90,7 @@ class TestMailRegisterTask:
        to_email = fake.email()
        account_name = fake.name()

-        with patch("tasks.mail_register_task.dify_config", autospec=True) as mock_config:
+        with patch("tasks.mail_register_task.dify_config") as mock_config:
            mock_config.CONSOLE_WEB_URL = "https://console.dify.ai"

            send_email_register_mail_task_when_account_exist(language=language, to=to_email, account_name=account_name)
--- a/api/tests/unit_tests/configs/test_env_consistency.py
+++ b/api/tests/unit_tests/configs/test_env_consistency.py
@ -1,6 +1,5 @@
 from pathlib import Path

-import yaml  # type: ignore
 from dotenv import dotenv_values

 BASE_API_AND_DOCKER_CONFIG_SET_DIFF: frozenset[str] = frozenset(
@ -91,34 +90,29 @@ BASE_API_AND_DOCKER_COMPOSE_CONFIG_SET_DIFF: frozenset[str] = frozenset(
    )
 )

-API_CONFIG_SET = set(dotenv_values(Path("api") / Path(".env.example")).keys())
-DOCKER_CONFIG_SET = set(dotenv_values(Path("docker") / Path(".env.example")).keys())
-DOCKER_COMPOSE_CONFIG_SET = set(DOCKER_CONFIG_SET)
-
-# Read environment variables from the split env files used by docker-compose
-# Walk through all .env.example files in subdirectories (per-module structure)
-envs_dir = Path("docker") / Path("envs")
-if envs_dir.exists():
-    for env_file_path in envs_dir.rglob("*.env.example"):
-        env_keys = set(dotenv_values(env_file_path).keys())
-        DOCKER_CONFIG_SET.update(env_keys)
-        DOCKER_COMPOSE_CONFIG_SET.update(env_keys)
+REPO_ROOT = Path(__file__).resolve().parents[4]


-def test_yaml_config():
-    # python set == operator is used to compare two sets
-    DIFF_API_WITH_DOCKER = API_CONFIG_SET - DOCKER_CONFIG_SET - BASE_API_AND_DOCKER_CONFIG_SET_DIFF
-    if DIFF_API_WITH_DOCKER:
-        print(f"API and Docker config sets are different with key: {DIFF_API_WITH_DOCKER}")
-        raise Exception("API and Docker config sets are different")
-    DIFF_API_WITH_DOCKER_COMPOSE = (
-        API_CONFIG_SET - DOCKER_COMPOSE_CONFIG_SET - BASE_API_AND_DOCKER_COMPOSE_CONFIG_SET_DIFF
-    )
-    if DIFF_API_WITH_DOCKER_COMPOSE:
-        print(f"API and Docker Compose config sets are different with key: {DIFF_API_WITH_DOCKER_COMPOSE}")
-        raise Exception("API and Docker Compose config sets are different")
-    print("All tests passed!")
+def _api_config_set() -> set[str]:
+    return set(dotenv_values(REPO_ROOT / "api" / ".env.example").keys())


-if __name__ == "__main__":
-    test_yaml_config()
+def _docker_config_set() -> set[str]:
+    docker_config_set = set(dotenv_values(REPO_ROOT / "docker" / ".env.example").keys())
+    envs_dir = REPO_ROOT / "docker" / "envs"
+    if envs_dir.exists():
+        for env_file_path in envs_dir.rglob("*.env.example"):
+            docker_config_set.update(dotenv_values(env_file_path).keys())
+    return docker_config_set
+
+
+def test_api_env_keys_exist_in_docker_env_examples():
+    diff = _api_config_set() - _docker_config_set() - BASE_API_AND_DOCKER_CONFIG_SET_DIFF
+
+    assert not diff, f"API and Docker config sets are different with keys: {sorted(diff)}"
+
+
+def test_api_env_keys_exist_in_docker_compose_env_examples():
+    diff = _api_config_set() - _docker_config_set() - BASE_API_AND_DOCKER_COMPOSE_CONFIG_SET_DIFF
+
+    assert not diff, f"API and Docker Compose config sets are different with keys: {sorted(diff)}"
--- a/api/tests/unit_tests/controllers/service_api/app/test_app.py
+++ b/api/tests/unit_tests/controllers/service_api/app/test_app.py
@ -15,6 +15,11 @@ from models.model import App, AppMode
 from tests.unit_tests.conftest import setup_mock_tenant_owner_execute_result


+def _configure_current_app_mock(mock_current_app):
+    mock_current_app.login_manager = Mock()
+    mock_current_app._get_current_object = Mock(return_value=Mock())
+
+
 class TestAppParameterApi:
    """Test suite for AppParameterApi"""

@ -45,7 +50,7 @@ class TestAppParameterApi:
    ):
        """Test retrieving parameters for a chat app."""
        # Arrange
-        mock_current_app.login_manager = Mock()
+        _configure_current_app_mock(mock_current_app)

        mock_config = Mock()
        mock_config.id = str(uuid.uuid4())
@ -95,7 +100,7 @@ class TestAppParameterApi:
    ):
        """Test retrieving parameters for a workflow app."""
        # Arrange
-        mock_current_app.login_manager = Mock()
+        _configure_current_app_mock(mock_current_app)

        mock_app_model.mode = AppMode.WORKFLOW
        mock_workflow = Mock()
@ -140,7 +145,7 @@ class TestAppParameterApi:
    ):
        """Test that AppUnavailableError is raised when chat app has no config."""
        # Arrange
-        mock_current_app.login_manager = Mock()
+        _configure_current_app_mock(mock_current_app)

        mock_app_model.app_model_config = None
        mock_app_model.workflow = None
@ -178,7 +183,7 @@ class TestAppParameterApi:
    ):
        """Test that AppUnavailableError is raised when workflow app has no workflow."""
        # Arrange
-        mock_current_app.login_manager = Mock()
+        _configure_current_app_mock(mock_current_app)

        mock_app_model.mode = AppMode.WORKFLOW
        mock_app_model.workflow = None
@ -245,7 +250,7 @@ class TestAppMetaApi:
    ):
        """Test retrieving app metadata via AppService."""
        # Arrange
-        mock_current_app.login_manager = Mock()
+        _configure_current_app_mock(mock_current_app)

        mock_service_instance = Mock()
        mock_service_instance.get_app_meta.return_value = {
@ -320,7 +325,7 @@ class TestAppInfoApi:
        self, mock_db, mock_validate_token, mock_current_app, mock_user_logged_in, app: Flask, mock_app_model
    ):
        """Test retrieving basic app information."""
-        mock_current_app.login_manager = Mock()
+        _configure_current_app_mock(mock_current_app)

        # Mock authentication
        mock_api_token = Mock()
@ -361,7 +366,7 @@ class TestAppInfoApi:
    ):
        """Test retrieving app info with multiple tags."""
        # Arrange
-        mock_current_app.login_manager = Mock()
+        _configure_current_app_mock(mock_current_app)

        mock_app = Mock(spec=App)
        mock_app.id = str(uuid.uuid4())
@ -414,7 +419,7 @@ class TestAppInfoApi:
    ):
        """Test retrieving app info when app has no tags."""
        # Arrange
-        mock_current_app.login_manager = Mock()
+        _configure_current_app_mock(mock_current_app)

        mock_app = Mock(spec=App)
        mock_app.id = str(uuid.uuid4())
@ -466,7 +471,7 @@ class TestAppInfoApi:
    ):
        """Test that all app modes are correctly returned."""
        # Arrange
-        mock_current_app.login_manager = Mock()
+        _configure_current_app_mock(mock_current_app)

        mock_app = Mock(spec=App)
        mock_app.id = str(uuid.uuid4())
--- a/api/tests/unit_tests/controllers/service_api/test_index.py
+++ b/api/tests/unit_tests/controllers/service_api/test_index.py
@ -2,31 +2,29 @@
 Unit tests for Service API Index endpoint
 """

-from unittest.mock import MagicMock, patch
+from unittest.mock import patch

 import pytest
 from flask import Flask

+from controllers.service_api import index as index_module
 from controllers.service_api.index import IndexApi


+def _get_index_response(app: Flask, version: str) -> dict[str, str]:
+    with patch.object(index_module.dify_config.project, "version", version):
+        with app.test_request_context("/", method="GET"):
+            index_api = IndexApi()
+            return index_api.get()
+
+
 class TestIndexApi:
    """Test suite for IndexApi resource."""

-    @patch("controllers.service_api.index.dify_config", autospec=True)
-    def test_get_returns_api_info(self, mock_config, app: Flask):
+    def test_get_returns_api_info(self, app: Flask):
        """Test that GET returns API metadata with correct structure."""
-        # Arrange
-        mock_config.project.version = "1.0.0-test"
-
        # Act
-        with app.test_request_context("/", method="GET"):
-            index_api = IndexApi()
-            response = index_api.get()
-        with patch("controllers.service_api.index.dify_config", mock_config):
-            with app.test_request_context("/", method="GET"):
-                index_api = IndexApi()
-                response = index_api.get()
+        response = _get_index_response(app, "1.0.0-test")

        # Assert
        assert response["welcome"] == "Dify OpenAPI"
@ -35,15 +33,8 @@ class TestIndexApi:

    def test_get_response_has_required_fields(self, app: Flask):
        """Test that response contains all required fields."""
-        # Arrange
-        mock_config = MagicMock()
-        mock_config.project.version = "1.11.4"
-
        # Act
-        with patch("controllers.service_api.index.dify_config", mock_config):
-            with app.test_request_context("/", method="GET"):
-                index_api = IndexApi()
-                response = index_api.get()
+        response = _get_index_response(app, "1.11.4")

        # Assert
        assert "welcome" in response
@ -56,15 +47,8 @@ class TestIndexApi:
    @pytest.mark.parametrize("version", ["0.0.1", "1.0.0", "2.0.0-beta", "1.11.4"])
    def test_get_returns_correct_version(self, app: Flask, version):
        """Test that server_version matches config version."""
-        # Arrange
-        mock_config = MagicMock()
-        mock_config.project.version = version
-
        # Act
-        with patch("controllers.service_api.index.dify_config", mock_config):
-            with app.test_request_context("/", method="GET"):
-                index_api = IndexApi()
-                response = index_api.get()
+        response = _get_index_response(app, version)

        # Assert
        assert response["server_version"] == version
--- a/api/tests/unit_tests/controllers/service_api/test_wraps.py
+++ b/api/tests/unit_tests/controllers/service_api/test_wraps.py
@ -29,6 +29,11 @@ from tests.unit_tests.conftest import (
 )


+def _configure_current_app_mock(mock_current_app):
+    mock_current_app.login_manager = Mock()
+    mock_current_app._get_current_object = Mock(return_value=Mock())
+
+
 class TestValidateAndGetApiToken:
    """Test suite for validate_and_get_api_token function"""

@ -120,8 +125,7 @@ class TestValidateAppToken:
    ):
        """Test that valid app token allows access to decorated view."""
        # Arrange
-        # Use standard Mock for login_manager to avoid AsyncMockMixin warnings
-        mock_current_app.login_manager = Mock()
+        _configure_current_app_mock(mock_current_app)

        mock_api_token = Mock()
        mock_api_token.app_id = str(uuid.uuid4())
@ -448,8 +452,7 @@ class TestValidateDatasetToken:
    def test_valid_dataset_token(self, mock_current_app, mock_validate_token, mock_db, mock_user_logged_in, app: Flask):
        """Test that valid dataset token allows access."""
        # Arrange
-        # Use standard Mock for login_manager
-        mock_current_app.login_manager = Mock()
+        _configure_current_app_mock(mock_current_app)

        tenant_id = str(uuid.uuid4())
        mock_api_token = Mock()
--- a/api/tests/unit_tests/core/mcp/test_utils.py
+++ b/api/tests/unit_tests/core/mcp/test_utils.py
@ -32,7 +32,7 @@ class TestConstants:
 class TestCreateSSRFProxyMCPHTTPClient:
    """Test create_ssrf_proxy_mcp_http_client function."""

-    @patch("core.mcp.utils.dify_config", autospec=True)
+    @patch("core.mcp.utils.dify_config")
    def test_create_client_with_all_url_proxy(self, mock_config):
        """Test client creation with SSRF_PROXY_ALL_URL configured."""
        mock_config.SSRF_PROXY_ALL_URL = "http://proxy.example.com:8080"
@ -50,7 +50,7 @@ class TestCreateSSRFProxyMCPHTTPClient:
        # Clean up
        client.close()

-    @patch("core.mcp.utils.dify_config", autospec=True)
+    @patch("core.mcp.utils.dify_config")
    def test_create_client_with_http_https_proxies(self, mock_config):
        """Test client creation with separate HTTP/HTTPS proxies."""
        mock_config.SSRF_PROXY_ALL_URL = None
@ -66,7 +66,7 @@ class TestCreateSSRFProxyMCPHTTPClient:
        # Clean up
        client.close()

-    @patch("core.mcp.utils.dify_config", autospec=True)
+    @patch("core.mcp.utils.dify_config")
    def test_create_client_without_proxy(self, mock_config):
        """Test client creation without proxy configuration."""
        mock_config.SSRF_PROXY_ALL_URL = None
@ -88,7 +88,7 @@ class TestCreateSSRFProxyMCPHTTPClient:
        # Clean up
        client.close()

-    @patch("core.mcp.utils.dify_config", autospec=True)
+    @patch("core.mcp.utils.dify_config")
    def test_create_client_default_params(self, mock_config):
        """Test client creation with default parameters."""
        mock_config.SSRF_PROXY_ALL_URL = None
@ -140,7 +140,7 @@ class TestSSRFProxySSEConnect:

    @patch("core.mcp.utils.connect_sse", autospec=True)
    @patch("core.mcp.utils.create_ssrf_proxy_mcp_http_client", autospec=True)
-    @patch("core.mcp.utils.dify_config", autospec=True)
+    @patch("core.mcp.utils.dify_config")
    def test_sse_connect_without_client(self, mock_config, mock_create_client, mock_connect_sse):
        """Test SSE connection without pre-configured client."""
        # Setup config
--- a/api/tests/unit_tests/core/moderation/test_output_moderation.py
+++ b/api/tests/unit_tests/core/moderation/test_output_moderation.py
@ -88,7 +88,7 @@ class TestOutputModeration:
    def test_start_thread(self, output_moderation):
        mock_app = MagicMock(spec=Flask)
        with patch("core.moderation.output_moderation.current_app") as mock_current_app:
-            mock_current_app._get_current_object.return_value = mock_app
+            mock_current_app._get_current_object = MagicMock(return_value=mock_app)
            with patch("threading.Thread") as mock_thread_class:
                mock_thread_instance = MagicMock()
                mock_thread_class.return_value = mock_thread_instance
--- a/api/tests/unit_tests/core/rag/indexing/processor/test_qa_index_processor.py
+++ b/api/tests/unit_tests/core/rag/indexing/processor/test_qa_index_processor.py
@ -106,7 +106,7 @@ class TestQAIndexProcessor:
            patch.object(processor, "_format_qa_document", side_effect=_append_document) as mock_format,
            patch("core.rag.index_processor.processor.qa_index_processor.current_app") as mock_current_app,
        ):
-            mock_current_app._get_current_object.return_value = fake_flask_app
+            mock_current_app._get_current_object = Mock(return_value=fake_flask_app)
            result = processor.transform(
                [document],
                process_rule=process_rule,
@ -155,7 +155,7 @@ class TestQAIndexProcessor:
                "core.rag.index_processor.processor.qa_index_processor.threading.Thread", side_effect=_ImmediateThread
            ),
        ):
-            mock_current_app._get_current_object.return_value = fake_flask_app
+            mock_current_app._get_current_object = Mock(return_value=fake_flask_app)
            result = processor.transform(documents, process_rule=process_rule, preview=False, tenant_id="tenant-1")

        assert len(result) == 2
--- a/api/tests/unit_tests/core/rag/indexing/test_indexing_runner.py
+++ b/api/tests/unit_tests/core/rag/indexing/test_indexing_runner.py
@ -594,6 +594,7 @@ class TestIndexingRunnerLoad:
            patch("core.indexing_runner.threading.Thread") as mock_thread,
            patch("core.indexing_runner.concurrent.futures.ThreadPoolExecutor") as mock_executor,
        ):
+            mock_app._get_current_object = Mock(return_value=Mock())
            yield {
                "db": mock_db,
                "model_manager": mock_model_manager,
--- a/api/tests/unit_tests/core/repositories/test_factory.py
+++ b/api/tests/unit_tests/core/repositories/test_factory.py
@ -51,7 +51,7 @@ class TestRepositoryFactory:
            import_string("invalidpath")
        assert "doesn't look like a module path" in str(exc_info.value)

-    @patch("core.repositories.factory.dify_config", autospec=True)
+    @patch("core.repositories.factory.dify_config")
    def test_create_workflow_execution_repository_success(self, mock_config):
        """Test successful WorkflowExecutionRepository creation."""
        # Setup mock configuration
@ -86,7 +86,7 @@ class TestRepositoryFactory:
            )
            assert result is mock_repository_instance

-    @patch("core.repositories.factory.dify_config", autospec=True)
+    @patch("core.repositories.factory.dify_config")
    def test_create_workflow_execution_repository_import_error(self, mock_config):
        """Test WorkflowExecutionRepository creation with import error."""
        # Setup mock configuration with invalid class path
@ -104,7 +104,7 @@ class TestRepositoryFactory:
            )
        assert "Failed to create WorkflowExecutionRepository" in str(exc_info.value)

-    @patch("core.repositories.factory.dify_config", autospec=True)
+    @patch("core.repositories.factory.dify_config")
    def test_create_workflow_execution_repository_instantiation_error(self, mock_config):
        """Test WorkflowExecutionRepository creation with instantiation error."""
        # Setup mock configuration
@ -128,7 +128,7 @@ class TestRepositoryFactory:
                )
            assert "Failed to create WorkflowExecutionRepository" in str(exc_info.value)

-    @patch("core.repositories.factory.dify_config", autospec=True)
+    @patch("core.repositories.factory.dify_config")
    def test_create_workflow_node_execution_repository_success(self, mock_config):
        """Test successful WorkflowNodeExecutionRepository creation."""
        # Setup mock configuration
@ -163,7 +163,7 @@ class TestRepositoryFactory:
            )
            assert result is mock_repository_instance

-    @patch("core.repositories.factory.dify_config", autospec=True)
+    @patch("core.repositories.factory.dify_config")
    def test_create_workflow_node_execution_repository_import_error(self, mock_config):
        """Test WorkflowNodeExecutionRepository creation with import error."""
        # Setup mock configuration with invalid class path
@ -181,7 +181,7 @@ class TestRepositoryFactory:
            )
        assert "Failed to create WorkflowNodeExecutionRepository" in str(exc_info.value)

-    @patch("core.repositories.factory.dify_config", autospec=True)
+    @patch("core.repositories.factory.dify_config")
    def test_create_workflow_node_execution_repository_instantiation_error(self, mock_config):
        """Test WorkflowNodeExecutionRepository creation with instantiation error."""
        # Setup mock configuration
@ -211,7 +211,7 @@ class TestRepositoryFactory:
        error = RepositoryImportError(error_message)
        assert str(error) == error_message

-    @patch("core.repositories.factory.dify_config", autospec=True)
+    @patch("core.repositories.factory.dify_config")
    def test_create_with_engine_instead_of_sessionmaker(self, mock_config):
        """Test repository creation with Engine instead of sessionmaker."""
        # Setup mock configuration
--- a/api/tests/unit_tests/core/schemas/test_resolver.py
+++ b/api/tests/unit_tests/core/schemas/test_resolver.py
@ -1,3 +1,4 @@
+import copy
 import time
 from concurrent.futures import ThreadPoolExecutor
 from unittest.mock import MagicMock, patch
@ -472,7 +473,7 @@ class TestSchemaResolverClass:
        assert resolved[2]["title"] == "Q&A Structure"

    def test_cache_performance(self):
-        """Test that caching improves performance"""
+        """Test that repeated references share cached schema lookups."""
        SchemaResolver.clear_cache()

        # Create a schema with many references to the same schema
@ -484,36 +485,16 @@ class TestSchemaResolverClass:
            },
        }

-        # First run (no cache) - run multiple times to warm up
-        results1 = []
-        for _ in range(3):
-            SchemaResolver.clear_cache()
-            start = time.perf_counter()
-            result1 = resolve_dify_schema_refs(schema)
-            time_no_cache = time.perf_counter() - start
-            results1.append(time_no_cache)
+        registry = SchemaRegistry.default_registry()
+        file_schema = registry.get_schema("https://dify.ai/schemas/v1/file.json")
+        assert file_schema is not None

-        avg_time_no_cache = sum(results1) / len(results1)
+        with patch.object(registry, "get_schema", wraps=registry.get_schema) as mock_get:
+            result1 = resolve_dify_schema_refs(copy.deepcopy(schema), registry=registry)
+            result2 = resolve_dify_schema_refs(copy.deepcopy(schema), registry=registry)

-        # Second run (with cache) - run multiple times
-        # Warm up cache first
-        resolve_dify_schema_refs(schema)
-
-        results2 = []
-        for _ in range(3):
-            start = time.perf_counter()
-            result2 = resolve_dify_schema_refs(schema)
-            time_with_cache = time.perf_counter() - start
-            results2.append(time_with_cache)
-
-        avg_time_with_cache = sum(results2) / len(results2)
-
-        # Cache should make it faster (more lenient check)
        assert result1 == result2
-        # Cache should provide some performance benefit (allow for measurement variance)
-        # We expect cache to be faster, but allow for small timing variations
-        performance_ratio = avg_time_with_cache / avg_time_no_cache if avg_time_no_cache > 0 else 1.0
-        assert performance_ratio <= 2.0, f"Cache performance degraded too much: {performance_ratio}"
+        mock_get.assert_called_once_with("https://dify.ai/schemas/v1/file.json")

    def test_fast_path_performance_no_refs(self):
        """Test that schemas without $refs use fast path and avoid deep copying"""
--- a/api/tests/unit_tests/events/test_events_package_compat.py
+++ b/api/tests/unit_tests/events/test_events_package_compat.py
@ -0,0 +1,42 @@
+import pytest
+
+from events import Events, EventsError, EventsException
+
+
+def test_events_package_exposes_opensearchpy_compatible_events_class():
+    calls: list[str] = []
+    events = Events()
+
+    events.request_start += lambda: calls.append("start")
+    events.request_end += lambda: calls.append("end")
+
+    events.request_start()
+    events.request_end()
+
+    assert calls == ["start", "end"]
+
+
+def test_events_package_supports_named_slots_iteration_removal_and_private_attrs():
+    calls: list[str] = []
+
+    def handler() -> None:
+        calls.append("handled")
+
+    events = Events("request_start")
+
+    events.request_start += handler
+    events.request_start += handler
+
+    assert len(events.request_start) == 2
+    assert list(events.request_start) == [handler, handler]
+
+    events.request_start -= handler
+
+    assert len(events.request_start) == 0
+    events.request_start()
+    assert calls == []
+
+    with pytest.raises(AttributeError):
+        _ = events._private  # type: ignore[attr-defined]
+
+    assert EventsException is EventsError
--- a/api/tests/unit_tests/extensions/storage/test_supabase_storage.py
+++ b/api/tests/unit_tests/extensions/storage/test_supabase_storage.py
@ -11,7 +11,7 @@ class TestSupabaseStorage:

    def test_init_success_with_all_config(self):
        """Test successful initialization when all required config is provided."""
-        with patch("extensions.storage.supabase_storage.dify_config", autospec=True) as mock_config:
+        with patch("extensions.storage.supabase_storage.dify_config") as mock_config:
            mock_config.SUPABASE_URL = "https://test.supabase.co"
            mock_config.SUPABASE_API_KEY = "test-api-key"
            mock_config.SUPABASE_BUCKET_NAME = "test-bucket"
@ -31,7 +31,7 @@ class TestSupabaseStorage:

    def test_init_raises_error_when_url_missing(self):
        """Test initialization raises ValueError when SUPABASE_URL is None."""
-        with patch("extensions.storage.supabase_storage.dify_config", autospec=True) as mock_config:
+        with patch("extensions.storage.supabase_storage.dify_config") as mock_config:
            mock_config.SUPABASE_URL = None
            mock_config.SUPABASE_API_KEY = "test-api-key"
            mock_config.SUPABASE_BUCKET_NAME = "test-bucket"
@ -41,7 +41,7 @@ class TestSupabaseStorage:

    def test_init_raises_error_when_api_key_missing(self):
        """Test initialization raises ValueError when SUPABASE_API_KEY is None."""
-        with patch("extensions.storage.supabase_storage.dify_config", autospec=True) as mock_config:
+        with patch("extensions.storage.supabase_storage.dify_config") as mock_config:
            mock_config.SUPABASE_URL = "https://test.supabase.co"
            mock_config.SUPABASE_API_KEY = None
            mock_config.SUPABASE_BUCKET_NAME = "test-bucket"
@ -51,7 +51,7 @@ class TestSupabaseStorage:

    def test_init_raises_error_when_bucket_name_missing(self):
        """Test initialization raises ValueError when SUPABASE_BUCKET_NAME is None."""
-        with patch("extensions.storage.supabase_storage.dify_config", autospec=True) as mock_config:
+        with patch("extensions.storage.supabase_storage.dify_config") as mock_config:
            mock_config.SUPABASE_URL = "https://test.supabase.co"
            mock_config.SUPABASE_API_KEY = "test-api-key"
            mock_config.SUPABASE_BUCKET_NAME = None
@ -61,7 +61,7 @@ class TestSupabaseStorage:

    def test_create_bucket_when_not_exists(self):
        """Test create_bucket creates bucket when it doesn't exist."""
-        with patch("extensions.storage.supabase_storage.dify_config", autospec=True) as mock_config:
+        with patch("extensions.storage.supabase_storage.dify_config") as mock_config:
            mock_config.SUPABASE_URL = "https://test.supabase.co"
            mock_config.SUPABASE_API_KEY = "test-api-key"
            mock_config.SUPABASE_BUCKET_NAME = "test-bucket"
@ -77,7 +77,7 @@ class TestSupabaseStorage:

    def test_create_bucket_when_exists(self):
        """Test create_bucket does not create bucket when it already exists."""
-        with patch("extensions.storage.supabase_storage.dify_config", autospec=True) as mock_config:
+        with patch("extensions.storage.supabase_storage.dify_config") as mock_config:
            mock_config.SUPABASE_URL = "https://test.supabase.co"
            mock_config.SUPABASE_API_KEY = "test-api-key"
            mock_config.SUPABASE_BUCKET_NAME = "test-bucket"
@ -94,7 +94,7 @@ class TestSupabaseStorage:
    @pytest.fixture
    def storage_with_mock_client(self):
        """Fixture providing SupabaseStorage with mocked client."""
-        with patch("extensions.storage.supabase_storage.dify_config", autospec=True) as mock_config:
+        with patch("extensions.storage.supabase_storage.dify_config") as mock_config:
            mock_config.SUPABASE_URL = "https://test.supabase.co"
            mock_config.SUPABASE_API_KEY = "test-api-key"
            mock_config.SUPABASE_BUCKET_NAME = "test-bucket"
@ -209,7 +209,7 @@ class TestSupabaseStorage:

    def test_bucket_exists_returns_true_when_bucket_found(self):
        """Test bucket_exists returns True when bucket is found in list."""
-        with patch("extensions.storage.supabase_storage.dify_config", autospec=True) as mock_config:
+        with patch("extensions.storage.supabase_storage.dify_config") as mock_config:
            mock_config.SUPABASE_URL = "https://test.supabase.co"
            mock_config.SUPABASE_API_KEY = "test-api-key"
            mock_config.SUPABASE_BUCKET_NAME = "test-bucket"
@ -229,7 +229,7 @@ class TestSupabaseStorage:

    def test_bucket_exists_returns_false_when_bucket_not_found(self):
        """Test bucket_exists returns False when bucket is not found in list."""
-        with patch("extensions.storage.supabase_storage.dify_config", autospec=True) as mock_config:
+        with patch("extensions.storage.supabase_storage.dify_config") as mock_config:
            mock_config.SUPABASE_URL = "https://test.supabase.co"
            mock_config.SUPABASE_API_KEY = "test-api-key"
            mock_config.SUPABASE_BUCKET_NAME = "test-bucket"
@ -252,7 +252,7 @@ class TestSupabaseStorage:

    def test_bucket_exists_returns_false_when_no_buckets(self):
        """Test bucket_exists returns False when no buckets exist."""
-        with patch("extensions.storage.supabase_storage.dify_config", autospec=True) as mock_config:
+        with patch("extensions.storage.supabase_storage.dify_config") as mock_config:
            mock_config.SUPABASE_URL = "https://test.supabase.co"
            mock_config.SUPABASE_API_KEY = "test-api-key"
            mock_config.SUPABASE_BUCKET_NAME = "test-bucket"
--- a/api/tests/unit_tests/services/test_archive_workflow_run_logs.py
+++ b/api/tests/unit_tests/services/test_archive_workflow_run_logs.py
@ -15,7 +15,7 @@ from services.retention.workflow_run.constants import ARCHIVE_BUNDLE_NAME
 class TestWorkflowRunArchiver:
    """Tests for the WorkflowRunArchiver class."""

-    @patch("services.retention.workflow_run.archive_paid_plan_workflow_run.dify_config", autospec=True)
+    @patch("services.retention.workflow_run.archive_paid_plan_workflow_run.dify_config")
    @patch("services.retention.workflow_run.archive_paid_plan_workflow_run.get_archive_storage", autospec=True)
    def test_archiver_initialization(self, mock_get_storage, mock_config):
        """Test archiver can be initialized with various options."""
--- a/api/tests/unit_tests/services/test_messages_clean_service.py
+++ b/api/tests/unit_tests/services/test_messages_clean_service.py
@ -403,7 +403,7 @@ class TestBillingDisabledPolicyFilterMessageIds:
 class TestCreateMessageCleanPolicy:
    """Unit tests for create_message_clean_policy factory function."""

-    @patch("services.retention.conversation.messages_clean_policy.dify_config", autospec=True)
+    @patch("services.retention.conversation.messages_clean_policy.dify_config")
    def test_billing_disabled_returns_billing_disabled_policy(self, mock_config):
        """Test that BILLING_ENABLED=False returns BillingDisabledPolicy."""
        # Arrange
@ -416,7 +416,7 @@ class TestCreateMessageCleanPolicy:
        assert isinstance(policy, BillingDisabledPolicy)

    @patch("services.retention.conversation.messages_clean_policy.BillingService", autospec=True)
-    @patch("services.retention.conversation.messages_clean_policy.dify_config", autospec=True)
+    @patch("services.retention.conversation.messages_clean_policy.dify_config")
    def test_billing_enabled_policy_has_correct_internals(self, mock_config, mock_billing_service):
        """Test that BillingSandboxPolicy is created with correct internal values."""
        # Arrange
--- a/api/tests/unit_tests/services/test_recommended_app_service.py
+++ b/api/tests/unit_tests/services/test_recommended_app_service.py
@ -13,7 +13,7 @@ from services.recommended_app_service import RecommendedAppService
 class TestGetRecommendAppDetailNullCheck:
    @patch("services.recommended_app_service.FeatureService", autospec=True)
    @patch("services.recommended_app_service.RecommendAppRetrievalFactory", autospec=True)
-    @patch("services.recommended_app_service.dify_config", autospec=True)
+    @patch("services.recommended_app_service.dify_config")
    def test_returns_none_when_retrieval_returns_none_and_trial_disabled(
        self, mock_config, mock_factory_class, mock_feature_service
    ):
@ -29,7 +29,7 @@ class TestGetRecommendAppDetailNullCheck:

    @patch("services.recommended_app_service.FeatureService", autospec=True)
    @patch("services.recommended_app_service.RecommendAppRetrievalFactory", autospec=True)
-    @patch("services.recommended_app_service.dify_config", autospec=True)
+    @patch("services.recommended_app_service.dify_config")
    def test_returns_none_when_retrieval_returns_none_and_trial_enabled(
        self, mock_config, mock_factory_class, mock_feature_service
    ):
--- a/api/tests/unit_tests/test_makefile_backend_tests.py
+++ b/api/tests/unit_tests/test_makefile_backend_tests.py
@ -0,0 +1,41 @@
+import subprocess
+from pathlib import Path
+
+
+def test_default_make_test_runs_backend_unit_suites():
+    repo_root = Path(__file__).resolve().parents[3]
+
+    completed = subprocess.run(["make", "-n", "test"], cwd=repo_root, check=True, capture_output=True, text=True)
+    dry_run_output = completed.stdout
+
+    assert "api/tests/unit_tests" in dry_run_output
+    assert "api/providers/vdb/*/tests/unit_tests" in dry_run_output
+    assert "api/providers/trace/*/tests/unit_tests" in dry_run_output
+    assert "-p no:benchmark" in dry_run_output
+    assert "api/tests/unit_tests/controllers" in dry_run_output
+    assert "--start-middleware" not in dry_run_output
+    assert "api/tests/integration_tests/workflow" not in dry_run_output
+    assert "api/tests/test_containers_integration_tests" not in dry_run_output
+    assert "--start-vdb" not in dry_run_output
+    assert "api/providers/vdb/vdb-chroma/tests/integration_tests" not in dry_run_output
+
+
+def test_make_test_all_runs_backend_pytest_suites():
+    repo_root = Path(__file__).resolve().parents[3]
+
+    completed = subprocess.run(["make", "-n", "test-all"], cwd=repo_root, check=True, capture_output=True, text=True)
+    dry_run_output = completed.stdout
+
+    assert "api/tests/unit_tests" in dry_run_output
+    assert "api/providers/vdb/*/tests/unit_tests" in dry_run_output
+    assert "api/providers/trace/*/tests/unit_tests" in dry_run_output
+    assert "-p no:benchmark" in dry_run_output
+    assert "--start-middleware" in dry_run_output
+    assert "api/tests/integration_tests/workflow" in dry_run_output
+    assert "api/tests/integration_tests/tools" in dry_run_output
+    assert "api/tests/test_containers_integration_tests" in dry_run_output
+    assert "--start-vdb" in dry_run_output
+    assert "api/providers/vdb/vdb-chroma/tests/integration_tests" in dry_run_output
+    assert "api/providers/vdb/vdb-pgvector/tests/integration_tests" in dry_run_output
+    assert "api/providers/vdb/vdb-qdrant/tests/integration_tests" in dry_run_output
+    assert "api/providers/vdb/vdb-weaviate/tests/integration_tests" in dry_run_output
--- a/api/tests/unit_tests/test_pytest_dify.py
+++ b/api/tests/unit_tests/test_pytest_dify.py
@ -0,0 +1,115 @@
+import os
+import subprocess
+from pathlib import Path
+
+from tests.pytest_dify import (
+    DEFAULT_LOG_FORMAT,
+    DockerComposeStack,
+    build_middleware_stack,
+    build_vdb_stack,
+    ensure_backend_test_environment,
+    ensure_compose_env_files,
+    parse_services,
+)
+
+
+def test_ensure_backend_test_environment_uses_example_env_and_stable_logging(
+    tmp_path: Path,
+    monkeypatch,
+):
+    repo_root = tmp_path
+    integration_tests_dir = repo_root / "api" / "tests" / "integration_tests"
+    integration_tests_dir.mkdir(parents=True)
+    env_example = integration_tests_dir / ".env.example"
+    env_example.write_text("LOG_LEVEL=INFO\n")
+    storage_root = repo_root / "storage"
+
+    monkeypatch.setenv("LOG_FORMAT", "json")
+    monkeypatch.delenv("LOG_OUTPUT_FORMAT", raising=False)
+    monkeypatch.delenv("DIFY_TEST_ENV_FILE", raising=False)
+    monkeypatch.delenv("DIFY_VDB_TEST_ENV_FILE", raising=False)
+    monkeypatch.setenv("OPENDAL_FS_ROOT", str(storage_root))
+
+    ensure_backend_test_environment(repo_root)
+
+    assert os.environ["DIFY_TEST_ENV_FILE"] == str(env_example)
+    assert "DIFY_VDB_TEST_ENV_FILE" not in os.environ
+    assert os.environ["LOG_OUTPUT_FORMAT"] == "text"
+    assert os.environ["LOG_FORMAT"] == DEFAULT_LOG_FORMAT
+    assert os.environ["STORAGE_TYPE"] == "opendal"
+    assert os.environ["OPENDAL_SCHEME"] == "fs"
+    assert storage_root.is_dir()
+
+
+def test_ensure_compose_env_files_copies_missing_env_files(tmp_path: Path):
+    docker_dir = tmp_path / "docker"
+    envs_dir = docker_dir / "envs"
+    envs_dir.mkdir(parents=True)
+    (docker_dir / ".env.example").write_text("APP_WEB_URL=http://localhost\n")
+    (envs_dir / "middleware.env.example").write_text("DB_PASSWORD=difyai123456\n")
+
+    ensure_compose_env_files(tmp_path)
+
+    assert (docker_dir / ".env").read_text() == "APP_WEB_URL=http://localhost\n"
+    assert (docker_dir / "middleware.env").read_text() == "DB_PASSWORD=difyai123456\n"
+
+
+def test_parse_services_discards_empty_items():
+    assert parse_services(" db_postgres, redis,, sandbox ") == ["db_postgres", "redis", "sandbox"]
+
+
+def test_stack_up_uses_waiting_compose_command(monkeypatch, tmp_path: Path):
+    calls: list[list[str]] = []
+
+    def fake_run(args, **kwargs):
+        calls.append(args)
+        return subprocess.CompletedProcess(args=args, returncode=0)
+
+    monkeypatch.setattr(subprocess, "run", fake_run)
+    monkeypatch.setattr("time.sleep", lambda _: None)
+
+    stack = DockerComposeStack(
+        name="middleware",
+        project_name="dify-pytest-middleware",
+        repo_root=tmp_path,
+        compose_files=(tmp_path / "docker-compose.yaml",),
+        env_file=tmp_path / "middleware.env",
+        services=("db_postgres", "redis"),
+    )
+
+    stack.up()
+
+    assert calls == [
+        [
+            "docker",
+            "compose",
+            "--project-name",
+            "dify-pytest-middleware",
+            "--env-file",
+            str(tmp_path / "middleware.env"),
+            "-f",
+            str(tmp_path / "docker-compose.yaml"),
+            "up",
+            "-d",
+            "--wait",
+            "--wait-timeout",
+            "180",
+            "db_postgres",
+            "redis",
+        ]
+    ]
+
+
+def test_builders_use_expected_compose_files(tmp_path: Path):
+    middleware = build_middleware_stack(tmp_path, ["db_postgres"])
+    vdb = build_vdb_stack(tmp_path, ["weaviate", "qdrant"])
+
+    assert middleware.compose_files == (tmp_path / "docker" / "docker-compose.middleware.yaml",)
+    assert middleware.env_file == tmp_path / "docker" / "middleware.env"
+    assert middleware.ready_delay_seconds == 5.0
+    assert vdb.compose_files == (
+        tmp_path / "docker" / "docker-compose.yaml",
+        tmp_path / "docker" / "docker-compose.pytest.ports.yaml",
+    )
+    assert vdb.env_file == tmp_path / "docker" / ".env"
+    assert vdb.profiles == ("weaviate", "qdrant")
--- a/dev/pytest/pytest_full.sh
+++ b/dev/pytest/pytest_full.sh
@ -1,58 +0,0 @@
-#!/bin/bash
-set -euo pipefail
-set -ex
-
-SCRIPT_DIR="$(dirname "$(realpath "$0")")"
-cd "$SCRIPT_DIR/../.."
-
-PYTEST_TIMEOUT="${PYTEST_TIMEOUT:-180}"
-
-# Ensure OpenDAL local storage works even if .env isn't loaded
-export STORAGE_TYPE=${STORAGE_TYPE:-opendal}
-export OPENDAL_SCHEME=${OPENDAL_SCHEME:-fs}
-export OPENDAL_FS_ROOT=${OPENDAL_FS_ROOT:-/tmp/dify-storage}
-mkdir -p "${OPENDAL_FS_ROOT}"
-
-# Prepare env files like CI
-cp -n docker/.env.example docker/.env || true
-cp -n docker/envs/middleware.env.example docker/middleware.env || true
-cp -n api/tests/integration_tests/.env.example api/tests/integration_tests/.env || true
-
-# Expose service ports (same as CI) without leaving the repo dirty
-EXPOSE_BACKUPS=()
-for f in docker/docker-compose.yaml docker/tidb/docker-compose.yaml; do
-  if [[ -f "$f" ]]; then
-    cp "$f" "$f.ci.bak"
-    EXPOSE_BACKUPS+=("$f")
-  fi
-done
-if command -v yq >/dev/null 2>&1; then
-  sh .github/workflows/expose_service_ports.sh || true
-else
-  echo "skip expose_service_ports (yq not installed)" >&2
-fi
-
-# Optionally start middleware stack (db, redis, sandbox, ssrf proxy) to mirror CI
-STARTED_MIDDLEWARE=0
-if [[ "${SKIP_MIDDLEWARE:-0}" != "1" ]]; then
-  docker compose -f docker/docker-compose.middleware.yaml --env-file docker/middleware.env up -d db_postgres redis sandbox ssrf_proxy
-  STARTED_MIDDLEWARE=1
-  # Give services a moment to come up
-  sleep 5
-fi
-
-cleanup() {
-  if [[ $STARTED_MIDDLEWARE -eq 1 ]]; then
-    docker compose -f docker/docker-compose.middleware.yaml --env-file docker/middleware.env down
-  fi
-  for f in "${EXPOSE_BACKUPS[@]}"; do
-    mv "$f.ci.bak" "$f"
-  done
-}
-trap cleanup EXIT
-
-pytest --timeout "${PYTEST_TIMEOUT}" \
-  api/tests/integration_tests/workflow \
-  api/tests/integration_tests/tools \
-  api/tests/test_containers_integration_tests \
-  api/tests/unit_tests
--- a/dev/pytest/pytest_unit_tests.sh
+++ b/dev/pytest/pytest_unit_tests.sh
@ -1,21 +0,0 @@
-#!/bin/bash
-set -euxo pipefail
-
-SCRIPT_DIR="$(dirname "$(realpath "$0")")"
-cd "$SCRIPT_DIR/../.."
-
-PYTEST_TIMEOUT="${PYTEST_TIMEOUT:-20}"
-PYTEST_XDIST_ARGS="${PYTEST_XDIST_ARGS:--n auto}"
-
-# Run most tests in parallel (excluding controllers which have import conflicts with xdist)
-# Controller tests have module-level side effects (Flask route registration) that cause
-# race conditions when imported concurrently by multiple pytest-xdist workers.
-pytest --timeout "${PYTEST_TIMEOUT}" ${PYTEST_XDIST_ARGS} \
-  api/tests/unit_tests \
-  api/providers/vdb/*/tests/unit_tests \
-  api/providers/trace/*/tests/unit_tests \
-  --ignore=api/tests/unit_tests/controllers
-
-# Run controller tests sequentially to avoid import race conditions
-pytest --timeout "${PYTEST_TIMEOUT}" --cov-append api/tests/unit_tests/controllers
-
--- a/dev/pytest/pytest_vdb.sh
+++ b/dev/pytest/pytest_vdb.sh
@ -1,12 +0,0 @@
-#!/bin/bash
-set -x
-
-SCRIPT_DIR="$(dirname "$(realpath "$0")")"
-cd "$SCRIPT_DIR/../.."
-
-PYTEST_TIMEOUT="${PYTEST_TIMEOUT:-180}"
-
-uv sync --project api --group dev 
-
-uv run --project api pytest --timeout "${PYTEST_TIMEOUT}" \
-  api/providers/vdb/*/tests/integration_tests \
--- a/docker/docker-compose.pytest.ports.yaml
+++ b/docker/docker-compose.pytest.ports.yaml
@ -0,0 +1,30 @@
+services:
+  weaviate:
+    ports:
+      - "${EXPOSE_WEAVIATE_PORT:-8080}:8080"
+      - "${EXPOSE_WEAVIATE_GRPC_PORT:-50051}:50051"
+
+  qdrant:
+    ports:
+      - "${EXPOSE_QDRANT_PORT:-6333}:6333"
+
+  pgvector:
+    ports:
+      - "${EXPOSE_PGVECTOR_PORT:-5433}:5432"
+
+  pgvecto-rs:
+    ports:
+      - "${EXPOSE_PGVECTO_RS_PORT:-5431}:5432"
+
+  chroma:
+    ports:
+      - "${EXPOSE_CHROMA_PORT:-8000}:8000"
+
+  couchbase-server:
+    ports:
+      - "${EXPOSE_COUCHBASE_WEB_PORT_RANGE:-8091-8096}:8091-8096"
+      - "${EXPOSE_COUCHBASE_DATA_PORT:-11210}:11210"
+
+  opensearch:
+    ports:
+      - "${EXPOSE_OPENSEARCH_PORT:-9200}:9200"