ci: parallelize change detection and gate style jobs (#41678)

Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>

.github/workflows/api-tests.yml

@@ -71,10 +71,10 @@ jobs:
           retention-days: 1
 
   api-integration:
-    name: API Integration Tests
-    runs-on: depot-ubuntu-24.04
+    name: API Integration Tests (${{ matrix.python-version }}, shard ${{ matrix.shardIndex }}/${{ matrix.shardTotal }})
+    runs-on: depot-ubuntu-24.04-4
     env:
-      COVERAGE_FILE: coverage-integration
+      COVERAGE_FILE: coverage-integration-${{ matrix.python-version }}-${{ matrix.shardIndex }}
       STORAGE_TYPE: opendal
       OPENDAL_SCHEME: fs
       OPENDAL_FS_ROOT: /tmp/dify-storage
@@ -82,9 +82,17 @@ jobs:
       run:
         shell: bash
     strategy:
+      fail-fast: false
       matrix:
         python-version:
           - '3.12'
+        shardIndex:
+          - 1
+          - 2
+          - 3
+          - 4
+        shardTotal:
+          - 4
 
     steps:
       - name: Checkout code
@@ -112,6 +120,8 @@ jobs:
             -p no:benchmark \
             --start-middleware \
             -n auto \
+            --shard-index "${{ matrix.shardIndex }}" \
+            --shard-total "${{ matrix.shardTotal }}" \
             --timeout "${PYTEST_TIMEOUT:-180}" \
             api/tests/integration_tests/workflow \
             api/tests/integration_tests/tools \
@@ -120,8 +130,9 @@ jobs:
       - name: Upload integration coverage data
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
-          name: api-coverage-integration
-          path: coverage-integration
+          name: api-coverage-integration-${{ matrix.python-version }}-${{ matrix.shardIndex }}
+          path: ${{ env.COVERAGE_FILE }}
+          if-no-files-found: error
           retention-days: 1
 
   api-coverage:
@@ -171,12 +182,15 @@ jobs:
           echo "" >> "$GITHUB_STEP_SUMMARY"
 
           unit_coverage="$(find coverage-data -type f -name coverage-unit -print -quit)"
-          integration_coverage="$(find coverage-data -type f -name coverage-integration -print -quit)"
+          mapfile -t integration_coverage < <(find coverage-data -type f -name 'coverage-integration-*' -print | sort)
           : "${unit_coverage:?coverage-unit artifact not found}"
-          : "${integration_coverage:?coverage-integration artifact not found}"
+          if [[ "${#integration_coverage[@]}" -ne 4 ]]; then
+            echo "expected 4 integration coverage artifacts, found ${#integration_coverage[@]}" >&2
+            exit 1
+          fi
 
           report_file="$(mktemp)"
-          uv run --project api coverage combine "$unit_coverage" "$integration_coverage"
+          uv run --project api coverage combine "$unit_coverage" "${integration_coverage[@]}"
           uv run --project api coverage report --show-missing | tee "$report_file"
           echo "Summary: \`$(tail -n 1 "$report_file")\`" >> "$GITHUB_STEP_SUMMARY"
           {

.github/workflows/main-ci.yml

@@ -37,8 +37,6 @@ jobs:
   # Check which paths were changed to determine which tests to run
   check-changes:
     name: Check Changed Files
-    needs: pre_job
-    if: needs.pre_job.outputs.should_skip != 'true'
     runs-on: depot-ubuntu-24.04
     outputs:
       api-changed: ${{ steps.changes.outputs.api }}
@@ -49,6 +47,11 @@ jobs:
       migration-changed: ${{ steps.changes.outputs.migration }}
       sandbox-runtime-changed: ${{ steps.changes.outputs.sandbox-runtime }}
       dify-agent-changed: ${{ steps.changes.outputs.dify-agent }}
+      python-style-changed: ${{ steps.changes.outputs.python-style }}
+      dify-agent-style-changed: ${{ steps.changes.outputs.dify-agent-style }}
+      web-style-changed: ${{ steps.changes.outputs.web-style }}
+      ts-common-style-changed: ${{ steps.changes.outputs.ts-common-style }}
+      superlinter-changed: ${{ steps.changes.outputs.superlinter }}
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
       - uses: dorny/paths-filter@ceb8a2b8f2d89434be7ff52d3de7ec3738c5cc9d # v4.0.3
@@ -150,6 +153,58 @@ jobs:
               - 'docker/generate_docker_compose'
               - 'docker/ssrf_proxy/**'
               - 'docker/volumes/sandbox/conf/**'
+            python-style:
+              - 'api/**'
+              - 'scripts/ast_grep_guard.py'
+              - 'scripts/check_no_new_getattr.py'
+              - 'scripts/check_no_new_controller_sqlalchemy.py'
+              - 'scripts/lint_controller_sqlalchemy.py'
+              - 'scripts/ast_grep_rules/no_new_getattr.yml'
+              - 'scripts/ast_grep_rules/no_new_controller_sqlalchemy.yml'
+              - '.github/workflows/style.yml'
+              - '.github/workflows/main-ci.yml'
+            dify-agent-style:
+              - 'dify-agent/**'
+              - '.github/workflows/style.yml'
+              - '.github/workflows/main-ci.yml'
+            web-style:
+              - 'web/**'
+              - 'e2e/**'
+              - 'sdks/nodejs-client/**'
+              - 'packages/**'
+              - 'package.json'
+              - 'pnpm-lock.yaml'
+              - 'pnpm-workspace.yaml'
+              - 'knip.config.ts'
+              - 'scripts/check-web-production-unused-after-knip-fix.mjs'
+              - '.github/workflows/style.yml'
+              - '.github/actions/setup-web/**'
+            ts-common-style:
+              - 'web/**'
+              - 'cli/**'
+              - 'e2e/**'
+              - 'sdks/nodejs-client/**'
+              - 'packages/**'
+              - 'package.json'
+              - 'pnpm-lock.yaml'
+              - 'pnpm-workspace.yaml'
+              - 'vite.config.ts'
+              - 'lint.config.ts'
+              - 'eslint.config.mjs'
+              - 'knip.config.ts'
+              - 'scripts/check-web-production-unused-after-knip-fix.mjs'
+              - 'oxlint-suppressions.json'
+              - 'eslint-suppressions.json'
+              - '.vscode/**'
+              - '.github/**'
+            superlinter:
+              - '**.sh'
+              - '**.yaml'
+              - '**.yml'
+              - '**Dockerfile'
+              - 'dev/**'
+              - '.editorconfig'
+              - '.vite-hooks/**'
 
   # Run tests in parallel while always emitting stable required checks.
   api-tests-run:
@@ -396,10 +451,19 @@ jobs:
 
   style-check:
     name: Style Check
-    needs: pre_job
+    if: ${{ always() }}
+    needs:
+      - pre_job
+      - check-changes
     uses: ./.github/workflows/style.yml
     with:
       base-rev: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha }}
+      classification-succeeded: ${{ needs.check-changes.result == 'success' }}
+      run-python-style: ${{ needs.check-changes.outputs.python-style-changed == 'true' }}
+      run-dify-agent-style: ${{ needs.check-changes.outputs.dify-agent-style-changed == 'true' }}
+      run-web-style: ${{ needs.check-changes.outputs.web-style-changed == 'true' }}
+      run-ts-common-style: ${{ needs.check-changes.outputs.ts-common-style-changed == 'true' }}
+      run-superlinter: ${{ needs.check-changes.outputs.superlinter-changed == 'true' }}
       skip-checks: ${{ needs.pre_job.outputs.should_skip == 'true' }}
 
   vdb-tests-run:

.github/workflows/style.yml

@@ -6,6 +6,30 @@ on:
       base-rev:
         required: true
         type: string
+      classification-succeeded:
+        description: Whether the caller successfully classified the changed files.
+        required: true
+        type: boolean
+      run-python-style:
+        description: Run Python style checks for the changed files.
+        required: true
+        type: boolean
+      run-dify-agent-style:
+        description: Run Dify Agent style checks for the changed files.
+        required: true
+        type: boolean
+      run-web-style:
+        description: Run Web style checks for the changed files.
+        required: true
+        type: boolean
+      run-ts-common-style:
+        description: Run shared TypeScript style checks for the changed files.
+        required: true
+        type: boolean
+      run-superlinter:
+        description: Run SuperLinter for the changed files.
+        required: true
+        type: boolean
       skip-checks:
         description: Create the required check runs without repeating previously successful work.
         required: false
@@ -20,42 +44,23 @@ permissions:
 jobs:
   python-style:
     name: Python Style
-    if: ${{ !inputs.skip-checks }}
+    if: ${{ !inputs.skip-checks && (!inputs.classification-succeeded || inputs.run-python-style || inputs.run-dify-agent-style) }}
     runs-on: depot-ubuntu-24.04
 
     steps:
+      - name: Verify changed-file classification
+        if: ${{ !inputs.classification-succeeded }}
+        run: |
+          echo "Changed-file classification failed; refusing to skip Python style checks." >&2
+          exit 1
+
       - name: Checkout code
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           persist-credentials: false
           fetch-depth: 0
 
-      - name: Check changed files
-        id: changed-files
-        uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
-        with:
-          files: |
-            api/**
-            scripts/ast_grep_guard.py
-            scripts/check_no_new_getattr.py
-            scripts/check_no_new_controller_sqlalchemy.py
-            scripts/lint_controller_sqlalchemy.py
-            scripts/ast_grep_rules/no_new_getattr.yml
-            scripts/ast_grep_rules/no_new_controller_sqlalchemy.yml
-            .github/workflows/style.yml
-            .github/workflows/main-ci.yml
-
-      - name: Check changed files (dify-agent)
-        id: changed-files-dify-agent
-        uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
-        with:
-          files: |
-            dify-agent/**
-            .github/workflows/style.yml
-            .github/workflows/main-ci.yml
-
       - name: Setup UV and Python
-        if: steps.changed-files.outputs.any_changed == 'true' || steps.changed-files-dify-agent.outputs.any_changed == 'true'
         uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
         with:
           enable-cache: false
@@ -65,46 +70,46 @@ jobs:
             dify-agent/uv.lock
 
       - name: Install dependencies
-        if: steps.changed-files.outputs.any_changed == 'true'
+        if: inputs.run-python-style
         run: uv sync --project api --dev
 
       - name: Run Import Linter
-        if: steps.changed-files.outputs.any_changed == 'true'
+        if: inputs.run-python-style
         run: uv run --directory api --dev lint-imports
 
       - name: Run Response Contract Linter
-        if: steps.changed-files.outputs.any_changed == 'true'
+        if: inputs.run-python-style
         run: uv run --project api --dev python api/dev/lint_response_contracts.py --fail-on-mismatch
 
       - name: Run No New Getattr Guard
-        if: steps.changed-files.outputs.any_changed == 'true'
+        if: inputs.run-python-style
         run: uv run --project api python scripts/check_no_new_getattr.py --base-rev "${{ inputs.base-rev }}"
 
       - name: Run No New Controller SQLAlchemy Guard
-        if: steps.changed-files.outputs.any_changed == 'true'
+        if: inputs.run-python-style
         run: uv run --project api python scripts/check_no_new_controller_sqlalchemy.py --base-rev "${{ inputs.base-rev }}"
 
       - name: Run Type Checks
-        if: steps.changed-files.outputs.any_changed == 'true'
+        if: inputs.run-python-style
         env:
           PYREFLY_OUTPUT_FORMAT: github
         run: make type-check-core
 
       - name: Run Dify Agent Ruff
-        if: steps.changed-files-dify-agent.outputs.any_changed == 'true'
+        if: inputs.run-dify-agent-style
         run: make -C dify-agent check
 
       - name: Run Dify Agent Type Checks
-        if: steps.changed-files-dify-agent.outputs.any_changed == 'true'
+        if: inputs.run-dify-agent-style
         run: make -C dify-agent typecheck
 
       - name: Dotenv check
-        if: steps.changed-files.outputs.any_changed == 'true'
+        if: inputs.run-python-style
         run: uv run --project api dotenv-linter ./api/.env.example ./web/.env.example
 
   web-style:
     name: Web Style
-    if: ${{ !inputs.skip-checks }}
+    if: ${{ !inputs.skip-checks && (!inputs.classification-succeeded || inputs.run-web-style) }}
     runs-on: depot-ubuntu-24.04
     defaults:
       run:
@@ -114,126 +119,84 @@ jobs:
       pull-requests: read
 
     steps:
+      - name: Verify changed-file classification
+        if: ${{ !inputs.classification-succeeded }}
+        working-directory: .
+        run: |
+          echo "Changed-file classification failed; refusing to skip Web style checks." >&2
+          exit 1
+
       - name: Checkout code
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           persist-credentials: false
 
-      - name: Check changed files
-        id: changed-files
-        uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
-        with:
-          files: |
-            web/**
-            e2e/**
-            sdks/nodejs-client/**
-            packages/**
-            package.json
-            pnpm-lock.yaml
-            pnpm-workspace.yaml
-            knip.config.ts
-            scripts/check-web-production-unused-after-knip-fix.mjs
-            .github/workflows/style.yml
-            .github/actions/setup-web/**
-
       - name: Setup web environment
-        if: steps.changed-files.outputs.any_changed == 'true'
         uses: ./.github/actions/setup-web
 
       - name: Web tsslint
-        if: steps.changed-files.outputs.any_changed == 'true'
         env:
           NODE_OPTIONS: --max-old-space-size=4096
         run: vp run lint:tss
 
       - name: Web dead code check
-        if: steps.changed-files.outputs.any_changed == 'true'
         working-directory: .
         run: vp run knip
 
       - name: Web dead code check production
-        if: steps.changed-files.outputs.any_changed == 'true'
         working-directory: .
         run: vp run knip:production
 
       - name: Web production unused declarations check
-        if: steps.changed-files.outputs.any_changed == 'true'
         working-directory: .
         run: vp run knip:production-unused-check
 
   ts-common-style:
     name: TS Common
-    if: ${{ !inputs.skip-checks }}
+    if: ${{ !inputs.skip-checks && (!inputs.classification-succeeded || inputs.run-ts-common-style) }}
     runs-on: depot-ubuntu-24.04-4
     permissions:
       checks: write
       pull-requests: read
 
     steps:
+      - name: Verify changed-file classification
+        if: ${{ !inputs.classification-succeeded }}
+        run: |
+          echo "Changed-file classification failed; refusing to skip TypeScript style checks." >&2
+          exit 1
+
       - name: Checkout code
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           persist-credentials: false
 
-      - name: Check changed files
-        id: changed-files
-        uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
-        with:
-          files: |
-            web/**
-            cli/**
-            e2e/**
-            sdks/nodejs-client/**
-            packages/**
-            package.json
-            pnpm-lock.yaml
-            pnpm-workspace.yaml
-            vite.config.ts
-            lint.config.ts
-            eslint.config.mjs
-            knip.config.ts
-            scripts/check-web-production-unused-after-knip-fix.mjs
-            oxlint-suppressions.json
-            eslint-suppressions.json
-            .vscode/**
-            .github/**
-
       - name: Setup web environment
-        if: steps.changed-files.outputs.any_changed == 'true'
         uses: ./.github/actions/setup-web
 
       - name: Static check
-        if: steps.changed-files.outputs.any_changed == 'true'
         run: pnpm -w check
 
   superlinter:
     name: SuperLinter
-    if: ${{ !inputs.skip-checks }}
+    if: ${{ !inputs.skip-checks && (!inputs.classification-succeeded || inputs.run-superlinter) }}
     runs-on: depot-ubuntu-24.04
 
     steps:
+      - name: Verify changed-file classification
+        if: ${{ !inputs.classification-succeeded }}
+        run: |
+          echo "Changed-file classification failed; refusing to skip SuperLinter." >&2
+          exit 1
+
       - name: Checkout code
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           fetch-depth: 0
           persist-credentials: false
 
-      - name: Check changed files
-        id: changed-files
-        uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
-        with:
-          files: |
-            **.sh
-            **.yaml
-            **.yml
-            **Dockerfile
-            dev/**
-            .editorconfig
-            .vite-hooks/**
-
       - name: Super-linter
         uses: super-linter/super-linter/slim@4ce20838b8ab83717e78138c5b3a1407148e0918 # v8.7.0
-        if: steps.changed-files.outputs.any_changed == 'true'
         env:
           BASH_SEVERITY: warning
           DEFAULT_BRANCH: origin/main

api/conftest.py

@@ -33,6 +33,18 @@ ensure_backend_test_environment(_REPO_ROOT)
 
 def pytest_addoption(parser: pytest.Parser) -> None:
     group = parser.getgroup("dify")
+    group.addoption(
+        "--shard-index",
+        type=int,
+        default=1,
+        help="One-based index of the test shard to run.",
+    )
+    group.addoption(
+        "--shard-total",
+        type=int,
+        default=1,
+        help="Total number of test shards.",
+    )
     group.addoption(
         "--start-middleware",
         action="store_true",
@@ -58,9 +70,33 @@ def pytest_addoption(parser: pytest.Parser) -> None:
 
 
 def pytest_configure(config: pytest.Config) -> None:
+    shard_index = config.getoption("shard_index")
+    shard_total = config.getoption("shard_total")
+    if shard_total < 1:
+        raise pytest.UsageError("--shard-total must be at least 1")
+    if not 1 <= shard_index <= shard_total:
+        raise pytest.UsageError("--shard-index must be between 1 and --shard-total")
+
     config.stash[_DIFY_COMPOSE_STACKS_KEY] = []
 
 
+def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None:
+    """Select a deterministic, evenly sized slice of collected tests for this shard."""
+    shard_index = config.getoption("shard_index")
+    shard_total = config.getoption("shard_total")
+    if shard_total == 1:
+        return
+
+    selected: list[pytest.Item] = []
+    deselected: list[pytest.Item] = []
+    for item_index, item in enumerate(items):
+        target = selected if item_index % shard_total == shard_index - 1 else deselected
+        target.append(item)
+
+    config.hook.pytest_deselected(items=deselected)
+    items[:] = selected
+
+
 def pytest_sessionstart(session: pytest.Session) -> None:
     config = session.config
     if hasattr(config, "workerinput"):

api/tests/unit_tests/commands/test_check_no_new_getattr.py

@@ -228,19 +228,6 @@ def test_style_workflow_wires_no_new_getattr_guard() -> None:
     assert checkout_step is not None
     assert "fetch-depth: 0" in checkout_step.group("step")
 
-    changed_files_step = re.search(
-        r"(?ms)^      - name: Check changed files\n.*?^          files: \|\n(?P<files>(?:^            \S[^\n]*\n)+)",
-        job_text,
-    )
-    assert changed_files_step is not None
-
-    files_block = changed_files_step.group("files")
-    assert "api/**\n" in files_block
-    assert "scripts/check_no_new_getattr.py\n" in files_block
-    assert "scripts/ast_grep_rules/no_new_getattr.yml\n" in files_block
-    assert ".github/workflows/style.yml\n" in files_block
-    assert ".github/workflows/main-ci.yml\n" in files_block
-
     guard_command = 'scripts/check_no_new_getattr.py --base-rev "${{ inputs.base-rev }}"'
     assert guard_command in job_text
 
@@ -251,6 +238,7 @@ def test_style_workflow_wires_no_new_getattr_guard() -> None:
     )
     assert guard_step is not None
 
+    assert "if: inputs.run-python-style" in guard_step.group("step")
     assert "GITHUB_BASE_SHA" not in guard_step.group("step")
 
 
@@ -266,17 +254,22 @@ def test_main_ci_passes_style_base_rev_input() -> None:
         "base-rev: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha }}"
         in style_job.group("job")
     )
-
-    api_filter = re.search(
-        r"(?ms)^            api:\n(?P<filter>(?:^              - '[^']+'\n)+)",
-        workflow,
+    assert "run-python-style: ${{ needs.check-changes.outputs.python-style-changed == 'true' }}" in style_job.group(
+        "job"
     )
-    assert api_filter is not None
-    filter_text = api_filter.group("filter")
-    assert "scripts/check_no_new_getattr.py" in filter_text
-    assert "scripts/ast_grep_rules/no_new_getattr.yml" in filter_text
-    assert ".github/workflows/style.yml" in filter_text
-    assert ".github/workflows/main-ci.yml" in filter_text
+
+    for filter_name in ("api", "python-style"):
+        path_filter = re.search(
+            rf"(?ms)^            {re.escape(filter_name)}:\n(?P<filter>(?:^              - '[^']+'\n)+)",
+            workflow,
+        )
+        assert path_filter is not None, filter_name
+        filter_text = path_filter.group("filter")
+        assert "api/**" in filter_text
+        assert "scripts/check_no_new_getattr.py" in filter_text
+        assert "scripts/ast_grep_rules/no_new_getattr.yml" in filter_text
+        assert ".github/workflows/style.yml" in filter_text
+        assert ".github/workflows/main-ci.yml" in filter_text
 
 
 def test_base_rev_mode_passes_when_only_legacy_getattr_exists(tmp_path: Path) -> None: