diff --git a/.github/workflows/api-tests.yml b/.github/workflows/api-tests.yml index 1b1afe96529..4e2abe4c433 100644 --- a/.github/workflows/api-tests.yml +++ b/.github/workflows/api-tests.yml @@ -71,10 +71,10 @@ jobs: retention-days: 1 api-integration: - name: API Integration Tests - runs-on: depot-ubuntu-24.04 + name: API Integration Tests (${{ matrix.python-version }}, shard ${{ matrix.shardIndex }}/${{ matrix.shardTotal }}) + runs-on: depot-ubuntu-24.04-4 env: - COVERAGE_FILE: coverage-integration + COVERAGE_FILE: coverage-integration-${{ matrix.python-version }}-${{ matrix.shardIndex }} STORAGE_TYPE: opendal OPENDAL_SCHEME: fs OPENDAL_FS_ROOT: /tmp/dify-storage @@ -82,9 +82,17 @@ jobs: run: shell: bash strategy: + fail-fast: false matrix: python-version: - '3.12' + shardIndex: + - 1 + - 2 + - 3 + - 4 + shardTotal: + - 4 steps: - name: Checkout code @@ -112,6 +120,8 @@ jobs: -p no:benchmark \ --start-middleware \ -n auto \ + --shard-index "${{ matrix.shardIndex }}" \ + --shard-total "${{ matrix.shardTotal }}" \ --timeout "${PYTEST_TIMEOUT:-180}" \ api/tests/integration_tests/workflow \ api/tests/integration_tests/tools \ @@ -120,8 +130,9 @@ jobs: - name: Upload integration coverage data uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: api-coverage-integration - path: coverage-integration + name: api-coverage-integration-${{ matrix.python-version }}-${{ matrix.shardIndex }} + path: ${{ env.COVERAGE_FILE }} + if-no-files-found: error retention-days: 1 api-coverage: @@ -171,12 +182,15 @@ jobs: echo "" >> "$GITHUB_STEP_SUMMARY" unit_coverage="$(find coverage-data -type f -name coverage-unit -print -quit)" - integration_coverage="$(find coverage-data -type f -name coverage-integration -print -quit)" + mapfile -t integration_coverage < <(find coverage-data -type f -name 'coverage-integration-*' -print | sort) : "${unit_coverage:?coverage-unit artifact not found}" - : "${integration_coverage:?coverage-integration artifact not found}" + if [[ "${#integration_coverage[@]}" -ne 4 ]]; then + echo "expected 4 integration coverage artifacts, found ${#integration_coverage[@]}" >&2 + exit 1 + fi report_file="$(mktemp)" - uv run --project api coverage combine "$unit_coverage" "$integration_coverage" + uv run --project api coverage combine "$unit_coverage" "${integration_coverage[@]}" uv run --project api coverage report --show-missing | tee "$report_file" echo "Summary: \`$(tail -n 1 "$report_file")\`" >> "$GITHUB_STEP_SUMMARY" { diff --git a/.github/workflows/main-ci.yml b/.github/workflows/main-ci.yml index 6023aa7fae2..b924fd18936 100644 --- a/.github/workflows/main-ci.yml +++ b/.github/workflows/main-ci.yml @@ -37,8 +37,6 @@ jobs: # Check which paths were changed to determine which tests to run check-changes: name: Check Changed Files - needs: pre_job - if: needs.pre_job.outputs.should_skip != 'true' runs-on: depot-ubuntu-24.04 outputs: api-changed: ${{ steps.changes.outputs.api }} @@ -49,6 +47,11 @@ jobs: migration-changed: ${{ steps.changes.outputs.migration }} sandbox-runtime-changed: ${{ steps.changes.outputs.sandbox-runtime }} dify-agent-changed: ${{ steps.changes.outputs.dify-agent }} + python-style-changed: ${{ steps.changes.outputs.python-style }} + dify-agent-style-changed: ${{ steps.changes.outputs.dify-agent-style }} + web-style-changed: ${{ steps.changes.outputs.web-style }} + ts-common-style-changed: ${{ steps.changes.outputs.ts-common-style }} + superlinter-changed: ${{ steps.changes.outputs.superlinter }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - uses: dorny/paths-filter@ceb8a2b8f2d89434be7ff52d3de7ec3738c5cc9d # v4.0.3 @@ -150,6 +153,58 @@ jobs: - 'docker/generate_docker_compose' - 'docker/ssrf_proxy/**' - 'docker/volumes/sandbox/conf/**' + python-style: + - 'api/**' + - 'scripts/ast_grep_guard.py' + - 'scripts/check_no_new_getattr.py' + - 'scripts/check_no_new_controller_sqlalchemy.py' + - 'scripts/lint_controller_sqlalchemy.py' + - 'scripts/ast_grep_rules/no_new_getattr.yml' + - 'scripts/ast_grep_rules/no_new_controller_sqlalchemy.yml' + - '.github/workflows/style.yml' + - '.github/workflows/main-ci.yml' + dify-agent-style: + - 'dify-agent/**' + - '.github/workflows/style.yml' + - '.github/workflows/main-ci.yml' + web-style: + - 'web/**' + - 'e2e/**' + - 'sdks/nodejs-client/**' + - 'packages/**' + - 'package.json' + - 'pnpm-lock.yaml' + - 'pnpm-workspace.yaml' + - 'knip.config.ts' + - 'scripts/check-web-production-unused-after-knip-fix.mjs' + - '.github/workflows/style.yml' + - '.github/actions/setup-web/**' + ts-common-style: + - 'web/**' + - 'cli/**' + - 'e2e/**' + - 'sdks/nodejs-client/**' + - 'packages/**' + - 'package.json' + - 'pnpm-lock.yaml' + - 'pnpm-workspace.yaml' + - 'vite.config.ts' + - 'lint.config.ts' + - 'eslint.config.mjs' + - 'knip.config.ts' + - 'scripts/check-web-production-unused-after-knip-fix.mjs' + - 'oxlint-suppressions.json' + - 'eslint-suppressions.json' + - '.vscode/**' + - '.github/**' + superlinter: + - '**.sh' + - '**.yaml' + - '**.yml' + - '**Dockerfile' + - 'dev/**' + - '.editorconfig' + - '.vite-hooks/**' # Run tests in parallel while always emitting stable required checks. api-tests-run: @@ -396,10 +451,19 @@ jobs: style-check: name: Style Check - needs: pre_job + if: ${{ always() }} + needs: + - pre_job + - check-changes uses: ./.github/workflows/style.yml with: base-rev: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha }} + classification-succeeded: ${{ needs.check-changes.result == 'success' }} + run-python-style: ${{ needs.check-changes.outputs.python-style-changed == 'true' }} + run-dify-agent-style: ${{ needs.check-changes.outputs.dify-agent-style-changed == 'true' }} + run-web-style: ${{ needs.check-changes.outputs.web-style-changed == 'true' }} + run-ts-common-style: ${{ needs.check-changes.outputs.ts-common-style-changed == 'true' }} + run-superlinter: ${{ needs.check-changes.outputs.superlinter-changed == 'true' }} skip-checks: ${{ needs.pre_job.outputs.should_skip == 'true' }} vdb-tests-run: diff --git a/.github/workflows/style.yml b/.github/workflows/style.yml index d809f1e208b..11fb97b6963 100644 --- a/.github/workflows/style.yml +++ b/.github/workflows/style.yml @@ -6,6 +6,30 @@ on: base-rev: required: true type: string + classification-succeeded: + description: Whether the caller successfully classified the changed files. + required: true + type: boolean + run-python-style: + description: Run Python style checks for the changed files. + required: true + type: boolean + run-dify-agent-style: + description: Run Dify Agent style checks for the changed files. + required: true + type: boolean + run-web-style: + description: Run Web style checks for the changed files. + required: true + type: boolean + run-ts-common-style: + description: Run shared TypeScript style checks for the changed files. + required: true + type: boolean + run-superlinter: + description: Run SuperLinter for the changed files. + required: true + type: boolean skip-checks: description: Create the required check runs without repeating previously successful work. required: false @@ -20,42 +44,23 @@ permissions: jobs: python-style: name: Python Style - if: ${{ !inputs.skip-checks }} + if: ${{ !inputs.skip-checks && (!inputs.classification-succeeded || inputs.run-python-style || inputs.run-dify-agent-style) }} runs-on: depot-ubuntu-24.04 steps: + - name: Verify changed-file classification + if: ${{ !inputs.classification-succeeded }} + run: | + echo "Changed-file classification failed; refusing to skip Python style checks." >&2 + exit 1 + - name: Checkout code uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false fetch-depth: 0 - - name: Check changed files - id: changed-files - uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6 - with: - files: | - api/** - scripts/ast_grep_guard.py - scripts/check_no_new_getattr.py - scripts/check_no_new_controller_sqlalchemy.py - scripts/lint_controller_sqlalchemy.py - scripts/ast_grep_rules/no_new_getattr.yml - scripts/ast_grep_rules/no_new_controller_sqlalchemy.yml - .github/workflows/style.yml - .github/workflows/main-ci.yml - - - name: Check changed files (dify-agent) - id: changed-files-dify-agent - uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6 - with: - files: | - dify-agent/** - .github/workflows/style.yml - .github/workflows/main-ci.yml - - name: Setup UV and Python - if: steps.changed-files.outputs.any_changed == 'true' || steps.changed-files-dify-agent.outputs.any_changed == 'true' uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 with: enable-cache: false @@ -65,46 +70,46 @@ jobs: dify-agent/uv.lock - name: Install dependencies - if: steps.changed-files.outputs.any_changed == 'true' + if: inputs.run-python-style run: uv sync --project api --dev - name: Run Import Linter - if: steps.changed-files.outputs.any_changed == 'true' + if: inputs.run-python-style run: uv run --directory api --dev lint-imports - name: Run Response Contract Linter - if: steps.changed-files.outputs.any_changed == 'true' + if: inputs.run-python-style run: uv run --project api --dev python api/dev/lint_response_contracts.py --fail-on-mismatch - name: Run No New Getattr Guard - if: steps.changed-files.outputs.any_changed == 'true' + if: inputs.run-python-style run: uv run --project api python scripts/check_no_new_getattr.py --base-rev "${{ inputs.base-rev }}" - name: Run No New Controller SQLAlchemy Guard - if: steps.changed-files.outputs.any_changed == 'true' + if: inputs.run-python-style run: uv run --project api python scripts/check_no_new_controller_sqlalchemy.py --base-rev "${{ inputs.base-rev }}" - name: Run Type Checks - if: steps.changed-files.outputs.any_changed == 'true' + if: inputs.run-python-style env: PYREFLY_OUTPUT_FORMAT: github run: make type-check-core - name: Run Dify Agent Ruff - if: steps.changed-files-dify-agent.outputs.any_changed == 'true' + if: inputs.run-dify-agent-style run: make -C dify-agent check - name: Run Dify Agent Type Checks - if: steps.changed-files-dify-agent.outputs.any_changed == 'true' + if: inputs.run-dify-agent-style run: make -C dify-agent typecheck - name: Dotenv check - if: steps.changed-files.outputs.any_changed == 'true' + if: inputs.run-python-style run: uv run --project api dotenv-linter ./api/.env.example ./web/.env.example web-style: name: Web Style - if: ${{ !inputs.skip-checks }} + if: ${{ !inputs.skip-checks && (!inputs.classification-succeeded || inputs.run-web-style) }} runs-on: depot-ubuntu-24.04 defaults: run: @@ -114,126 +119,84 @@ jobs: pull-requests: read steps: + - name: Verify changed-file classification + if: ${{ !inputs.classification-succeeded }} + working-directory: . + run: | + echo "Changed-file classification failed; refusing to skip Web style checks." >&2 + exit 1 + - name: Checkout code uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - name: Check changed files - id: changed-files - uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6 - with: - files: | - web/** - e2e/** - sdks/nodejs-client/** - packages/** - package.json - pnpm-lock.yaml - pnpm-workspace.yaml - knip.config.ts - scripts/check-web-production-unused-after-knip-fix.mjs - .github/workflows/style.yml - .github/actions/setup-web/** - - name: Setup web environment - if: steps.changed-files.outputs.any_changed == 'true' uses: ./.github/actions/setup-web - name: Web tsslint - if: steps.changed-files.outputs.any_changed == 'true' env: NODE_OPTIONS: --max-old-space-size=4096 run: vp run lint:tss - name: Web dead code check - if: steps.changed-files.outputs.any_changed == 'true' working-directory: . run: vp run knip - name: Web dead code check production - if: steps.changed-files.outputs.any_changed == 'true' working-directory: . run: vp run knip:production - name: Web production unused declarations check - if: steps.changed-files.outputs.any_changed == 'true' working-directory: . run: vp run knip:production-unused-check ts-common-style: name: TS Common - if: ${{ !inputs.skip-checks }} + if: ${{ !inputs.skip-checks && (!inputs.classification-succeeded || inputs.run-ts-common-style) }} runs-on: depot-ubuntu-24.04-4 permissions: checks: write pull-requests: read steps: + - name: Verify changed-file classification + if: ${{ !inputs.classification-succeeded }} + run: | + echo "Changed-file classification failed; refusing to skip TypeScript style checks." >&2 + exit 1 + - name: Checkout code uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - name: Check changed files - id: changed-files - uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6 - with: - files: | - web/** - cli/** - e2e/** - sdks/nodejs-client/** - packages/** - package.json - pnpm-lock.yaml - pnpm-workspace.yaml - vite.config.ts - lint.config.ts - eslint.config.mjs - knip.config.ts - scripts/check-web-production-unused-after-knip-fix.mjs - oxlint-suppressions.json - eslint-suppressions.json - .vscode/** - .github/** - - name: Setup web environment - if: steps.changed-files.outputs.any_changed == 'true' uses: ./.github/actions/setup-web - name: Static check - if: steps.changed-files.outputs.any_changed == 'true' run: pnpm -w check superlinter: name: SuperLinter - if: ${{ !inputs.skip-checks }} + if: ${{ !inputs.skip-checks && (!inputs.classification-succeeded || inputs.run-superlinter) }} runs-on: depot-ubuntu-24.04 steps: + - name: Verify changed-file classification + if: ${{ !inputs.classification-succeeded }} + run: | + echo "Changed-file classification failed; refusing to skip SuperLinter." >&2 + exit 1 + - name: Checkout code uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: fetch-depth: 0 persist-credentials: false - - name: Check changed files - id: changed-files - uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6 - with: - files: | - **.sh - **.yaml - **.yml - **Dockerfile - dev/** - .editorconfig - .vite-hooks/** - - name: Super-linter uses: super-linter/super-linter/slim@4ce20838b8ab83717e78138c5b3a1407148e0918 # v8.7.0 - if: steps.changed-files.outputs.any_changed == 'true' env: BASH_SEVERITY: warning DEFAULT_BRANCH: origin/main diff --git a/api/conftest.py b/api/conftest.py index 350b0306016..e967a723d2a 100644 --- a/api/conftest.py +++ b/api/conftest.py @@ -33,6 +33,18 @@ ensure_backend_test_environment(_REPO_ROOT) def pytest_addoption(parser: pytest.Parser) -> None: group = parser.getgroup("dify") + group.addoption( + "--shard-index", + type=int, + default=1, + help="One-based index of the test shard to run.", + ) + group.addoption( + "--shard-total", + type=int, + default=1, + help="Total number of test shards.", + ) group.addoption( "--start-middleware", action="store_true", @@ -58,9 +70,33 @@ def pytest_addoption(parser: pytest.Parser) -> None: def pytest_configure(config: pytest.Config) -> None: + shard_index = config.getoption("shard_index") + shard_total = config.getoption("shard_total") + if shard_total < 1: + raise pytest.UsageError("--shard-total must be at least 1") + if not 1 <= shard_index <= shard_total: + raise pytest.UsageError("--shard-index must be between 1 and --shard-total") + config.stash[_DIFY_COMPOSE_STACKS_KEY] = [] +def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None: + """Select a deterministic, evenly sized slice of collected tests for this shard.""" + shard_index = config.getoption("shard_index") + shard_total = config.getoption("shard_total") + if shard_total == 1: + return + + selected: list[pytest.Item] = [] + deselected: list[pytest.Item] = [] + for item_index, item in enumerate(items): + target = selected if item_index % shard_total == shard_index - 1 else deselected + target.append(item) + + config.hook.pytest_deselected(items=deselected) + items[:] = selected + + def pytest_sessionstart(session: pytest.Session) -> None: config = session.config if hasattr(config, "workerinput"): diff --git a/api/tests/unit_tests/commands/test_check_no_new_getattr.py b/api/tests/unit_tests/commands/test_check_no_new_getattr.py index a63569c706f..a846955a770 100644 --- a/api/tests/unit_tests/commands/test_check_no_new_getattr.py +++ b/api/tests/unit_tests/commands/test_check_no_new_getattr.py @@ -228,19 +228,6 @@ def test_style_workflow_wires_no_new_getattr_guard() -> None: assert checkout_step is not None assert "fetch-depth: 0" in checkout_step.group("step") - changed_files_step = re.search( - r"(?ms)^ - name: Check changed files\n.*?^ files: \|\n(?P(?:^ \S[^\n]*\n)+)", - job_text, - ) - assert changed_files_step is not None - - files_block = changed_files_step.group("files") - assert "api/**\n" in files_block - assert "scripts/check_no_new_getattr.py\n" in files_block - assert "scripts/ast_grep_rules/no_new_getattr.yml\n" in files_block - assert ".github/workflows/style.yml\n" in files_block - assert ".github/workflows/main-ci.yml\n" in files_block - guard_command = 'scripts/check_no_new_getattr.py --base-rev "${{ inputs.base-rev }}"' assert guard_command in job_text @@ -251,6 +238,7 @@ def test_style_workflow_wires_no_new_getattr_guard() -> None: ) assert guard_step is not None + assert "if: inputs.run-python-style" in guard_step.group("step") assert "GITHUB_BASE_SHA" not in guard_step.group("step") @@ -266,17 +254,22 @@ def test_main_ci_passes_style_base_rev_input() -> None: "base-rev: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha }}" in style_job.group("job") ) - - api_filter = re.search( - r"(?ms)^ api:\n(?P(?:^ - '[^']+'\n)+)", - workflow, + assert "run-python-style: ${{ needs.check-changes.outputs.python-style-changed == 'true' }}" in style_job.group( + "job" ) - assert api_filter is not None - filter_text = api_filter.group("filter") - assert "scripts/check_no_new_getattr.py" in filter_text - assert "scripts/ast_grep_rules/no_new_getattr.yml" in filter_text - assert ".github/workflows/style.yml" in filter_text - assert ".github/workflows/main-ci.yml" in filter_text + + for filter_name in ("api", "python-style"): + path_filter = re.search( + rf"(?ms)^ {re.escape(filter_name)}:\n(?P(?:^ - '[^']+'\n)+)", + workflow, + ) + assert path_filter is not None, filter_name + filter_text = path_filter.group("filter") + assert "api/**" in filter_text + assert "scripts/check_no_new_getattr.py" in filter_text + assert "scripts/ast_grep_rules/no_new_getattr.yml" in filter_text + assert ".github/workflows/style.yml" in filter_text + assert ".github/workflows/main-ci.yml" in filter_text def test_base_rev_mode_passes_when_only_legacy_getattr_exists(tmp_path: Path) -> None: