From 762ba27036e717aeceeb1b43f3090f0e9a049869 Mon Sep 17 00:00:00 2001 From: Tom Scogland Date: Wed, 7 Sep 2022 11:12:57 -0700 Subject: Make GHA tests parallel by using xdist (#32361) * Add two no-op jobs named "all-prechecks" and "all" These are a suggestion from @tgamblin, they are stable named markers we can use from gitlab and possibly for required checks to make CI more resilient to refactors changing the names of specific checks. * Enable parallel testing using xdist for unit testing in CI * Normalize tmp paths to deal with macos * add -u flag compatibility to spack python As of now, it is accepted and ignored. The usage with xdist, where it is invoked specifically by `python -u spack python` which is then passed `-u` by xdist is the entire reason for doing this. It should never be used without explicitly passing -u to the executing python interpreter. * use spack python in xdist to support python 2 When running on python2, spack has many import cycles unless started through main. To allow that, this uses `spack python` as the interpreter, leveraging the `-u` support so xdist doesn't error out when it unconditionally requests unbuffered binary IO. * Use shutil.move to account for tmpdir being in a separate filesystem sometimes --- .github/workflows/audit.yaml | 44 ++++++++++++++++++++++++++++++++++ .github/workflows/bootstrap.yml | 2 +- .github/workflows/build-containers.yml | 2 +- .github/workflows/ci.yaml | 25 ++++++++++++++++--- .github/workflows/unit_tests.yaml | 33 ++++++++++++++++++------- .github/workflows/valid-style.yml | 34 +++++--------------------- .github/workflows/windows_python.yml | 2 +- 7 files changed, 100 insertions(+), 42 deletions(-) create mode 100644 .github/workflows/audit.yaml (limited to '.github') diff --git a/.github/workflows/audit.yaml b/.github/workflows/audit.yaml new file mode 100644 index 0000000000..2b8c989518 --- /dev/null +++ b/.github/workflows/audit.yaml @@ -0,0 +1,44 @@ +name: audit + +on: + workflow_call: + inputs: + with_coverage: + required: true + type: string + python_version: + required: true + type: string + +concurrency: + group: audit-${{inputs.python_version}}-${{github.ref}}-${{github.event.pull_request.number || github.run_number}} + cancel-in-progress: true + +jobs: + # Run audits on all the packages in the built-in repository + package-audits: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # @v2 + - uses: actions/setup-python@b55428b1882923874294fa556849718a1d7f2ca5 # @v2 + with: + python-version: ${{inputs.python_version}} + - name: Install Python packages + run: | + pip install --upgrade pip six setuptools pytest codecov 'coverage[toml]<=6.2' + - name: Package audits (with coverage) + if: ${{ inputs.with_coverage == 'true' }} + run: | + . share/spack/setup-env.sh + coverage run $(which spack) audit packages + coverage combine + coverage xml + - name: Package audits (without coverage) + if: ${{ inputs.with_coverage == 'false' }} + run: | + . share/spack/setup-env.sh + $(which spack) audit packages + - uses: codecov/codecov-action@81cd2dc8148241f03f5839d295e000b8f761e378 # @v2.1.0 + if: ${{ inputs.with_coverage == 'true' }} + with: + flags: unittests,linux,audits diff --git a/.github/workflows/bootstrap.yml b/.github/workflows/bootstrap.yml index 138a9c7382..e649de2fb4 100644 --- a/.github/workflows/bootstrap.yml +++ b/.github/workflows/bootstrap.yml @@ -9,7 +9,7 @@ on: - cron: '16 2 * * *' concurrency: - group: bootstrap-${{ github.workflow }}-${{ github.event.pull_request.number || github.run_number }} + group: bootstrap-${{github.ref}}-${{github.event.pull_request.number || github.run_number}} cancel-in-progress: true jobs: diff --git a/.github/workflows/build-containers.yml b/.github/workflows/build-containers.yml index f849747995..30765cfc86 100644 --- a/.github/workflows/build-containers.yml +++ b/.github/workflows/build-containers.yml @@ -20,7 +20,7 @@ on: types: [published] concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_number }} + group: build_containers-${{github.ref}}-${{github.event.pull_request.number || github.run_number}} cancel-in-progress: true jobs: diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 62f2be080f..f4943789d6 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -11,7 +11,7 @@ on: - releases/** concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_number }} + group: ci-${{github.ref}}-${{github.event.pull_request.number || github.run_number}} cancel-in-progress: true jobs: @@ -20,6 +20,18 @@ jobs: uses: ./.github/workflows/valid-style.yml with: with_coverage: ${{ needs.changes.outputs.with_coverage }} + audit-ancient-python: + uses: ./.github/workflows/audit.yaml + needs: [ changes ] + with: + with_coverage: ${{ needs.changes.outputs.with_coverage }} + python_version: 2.7 + all-prechecks: + needs: [ prechecks ] + runs-on: ubuntu-latest + steps: + - name: Success + run: "true" # Check which files have been updated by the PR changes: runs-on: ubuntu-latest @@ -53,6 +65,7 @@ jobs: - 'share/spack/**' - '.github/workflows/bootstrap.yml' core: + - '!lib/spack/docs/**' - './!(var/**)/**' packages: - 'var/**' @@ -79,7 +92,7 @@ jobs: needs: [ prechecks, changes ] uses: ./.github/workflows/bootstrap.yml unit-tests: - if: ${{ github.repository == 'spack/spack' }} + if: ${{ github.repository == 'spack/spack' && needs.changes.outputs.core == 'true' }} needs: [ prechecks, changes ] uses: ./.github/workflows/unit_tests.yaml with: @@ -87,7 +100,13 @@ jobs: packages: ${{ needs.changes.outputs.packages }} with_coverage: ${{ needs.changes.outputs.with_coverage }} windows: - if: ${{ github.repository == 'spack/spack' }} + if: ${{ github.repository == 'spack/spack' && needs.changes.outputs.core == 'true' }} needs: [ prechecks ] uses: ./.github/workflows/windows_python.yml + all: + needs: [ windows, unit-tests, bootstrap, audit-ancient-python ] + runs-on: ubuntu-latest + steps: + - name: Success + run: "true" diff --git a/.github/workflows/unit_tests.yaml b/.github/workflows/unit_tests.yaml index 3856f1039f..6b1411c61b 100644 --- a/.github/workflows/unit_tests.yaml +++ b/.github/workflows/unit_tests.yaml @@ -1,6 +1,7 @@ name: unit tests on: + workflow_dispatch: workflow_call: inputs: core: @@ -14,7 +15,7 @@ on: type: string concurrency: - group: unit_tests-${{ github.workflow }}-${{ github.event.pull_request.number || github.run_number }} + group: unit_tests-${{github.ref}}-${{github.event.pull_request.number || github.run_number}} cancel-in-progress: true jobs: @@ -25,11 +26,26 @@ jobs: matrix: python-version: ['2.7', '3.6', '3.7', '3.8', '3.9', '3.10'] concretizer: ['clingo'] + on_develop: + - ${{ github.ref == 'refs/heads/develop' }} include: - python-version: 2.7 concretizer: original - - python-version: 3.9 + on_develop: false + - python-version: '3.10' concretizer: original + on_develop: false + exclude: + - python-version: '3.7' + concretizer: 'clingo' + on_develop: false + - python-version: '3.8' + concretizer: 'clingo' + on_develop: false + - python-version: '3.9' + concretizer: 'clingo' + on_develop: false + steps: - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # @v2 with: @@ -46,7 +62,7 @@ jobs: patchelf cmake bison libbison-dev kcov - name: Install Python packages run: | - pip install --upgrade pip six setuptools pytest codecov "coverage[toml]<=6.2" + pip install --upgrade pip six setuptools pytest codecov "coverage[toml]<=6.2" pytest-xdist # ensure style checks are not skipped in unit tests for python >= 3.6 # note that true/false (i.e., 1/0) are opposite in conditions in python and bash if python -c 'import sys; sys.exit(not sys.version_info >= (3, 6))'; then @@ -108,7 +124,7 @@ jobs: sudo apt-get install -y coreutils kcov csh zsh tcsh fish dash bash - name: Install Python packages run: | - pip install --upgrade pip six setuptools pytest codecov coverage[toml]==6.2 + pip install --upgrade pip six setuptools pytest codecov coverage[toml]==6.2 pytest-xdist - name: Setup git configuration run: | # Need this for the git tests to succeed. @@ -174,7 +190,7 @@ jobs: patchelf kcov - name: Install Python packages run: | - pip install --upgrade pip six setuptools pytest codecov coverage[toml]==6.2 clingo + pip install --upgrade pip six setuptools pytest codecov coverage[toml]==6.2 clingo pytest-xdist - name: Setup git configuration run: | # Need this for the git tests to succeed. @@ -216,7 +232,7 @@ jobs: - name: Install Python packages run: | pip install --upgrade pip six setuptools - pip install --upgrade pytest codecov coverage[toml]==6.2 + pip install --upgrade pytest codecov coverage[toml]==6.2 pytest-xdist - name: Setup Homebrew packages run: | brew install dash fish gcc gnupg2 kcov @@ -229,9 +245,10 @@ jobs: . share/spack/setup-env.sh $(which spack) bootstrap untrust spack-install $(which spack) solve zlib + common_args=(--dist loadfile --tx '4*popen//python=./bin/spack-tmpconfig python -u ./bin/spack python' -x) if [ "${{ inputs.with_coverage }}" == "true" ] then - coverage run $(which spack) unit-test -x + coverage run $(which spack) unit-test "${common_args[@]}" coverage combine coverage xml # Delete the symlink going from ./lib/spack/docs/_spack_root back to @@ -239,7 +256,7 @@ jobs: rm lib/spack/docs/_spack_root else echo "ONLY PACKAGE RECIPES CHANGED [skipping coverage]" - $(which spack) unit-test -x -m "not maybeslow" -k "test_all_virtual_packages_have_default_providers" + $(which spack) unit-test "${common_args[@]}" -m "not maybeslow" -k "test_all_virtual_packages_have_default_providers" fi - uses: codecov/codecov-action@81cd2dc8148241f03f5839d295e000b8f761e378 # @v2.1.0 if: ${{ inputs.with_coverage == 'true' }} diff --git a/.github/workflows/valid-style.yml b/.github/workflows/valid-style.yml index 73c8dc76f1..2519fd77a0 100644 --- a/.github/workflows/valid-style.yml +++ b/.github/workflows/valid-style.yml @@ -8,7 +8,7 @@ on: type: string concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_number }} + group: style-${{github.ref}}-${{github.event.pull_request.number || github.run_number}} cancel-in-progress: true @@ -53,30 +53,8 @@ jobs: - name: Run style tests run: | share/spack/qa/run-style-tests - # Run audits on all the packages in the built-in repository - package-audits: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # @v2 - - uses: actions/setup-python@b55428b1882923874294fa556849718a1d7f2ca5 # @v2 - with: - python-version: '3.10' - - name: Install Python packages - run: | - pip install --upgrade pip six setuptools pytest codecov coverage[toml]==6.2 - - name: Package audits (with coverage) - if: ${{ inputs.with_coverage == 'true' }} - run: | - . share/spack/setup-env.sh - coverage run $(which spack) audit packages - coverage combine - coverage xml - - name: Package audits (without coverage) - if: ${{ inputs.with_coverage == 'false' }} - run: | - . share/spack/setup-env.sh - $(which spack) audit packages - - uses: codecov/codecov-action@81cd2dc8148241f03f5839d295e000b8f761e378 # @v2.1.0 - if: ${{ inputs.with_coverage == 'true' }} - with: - flags: unittests,linux,audits + audit: + uses: ./.github/workflows/audit.yaml + with: + with_coverage: ${{ inputs.with_coverage }} + python_version: '3.10' diff --git a/.github/workflows/windows_python.yml b/.github/workflows/windows_python.yml index 5bd009df44..82e48370ac 100644 --- a/.github/workflows/windows_python.yml +++ b/.github/workflows/windows_python.yml @@ -4,7 +4,7 @@ on: workflow_call: concurrency: - group: windows-${{ github.workflow }}-${{ github.event.pull_request.number || github.run_number }} + group: windows-${{github.ref}}-${{github.event.pull_request.number || github.run_number}} cancel-in-progress: true defaults: -- cgit v1.2.3-60-g2f50