Reorder workflow execution in GHA (#32183)

This patchset refactors our GitHub actions into a single top-level ci workflow that
invokes a series of reusable actions.  The main goal of this is to be able to easily
control which tests run and in what order based on the success or failure of top-level
prechecks.  Our previous workflows ran in three sets:

* nix tests: style and verification first, then linux and macos tests if successful
* windows tests: style and verification first, then linux and macos tests if successful
* bootstrap tests

As a result, the bootstrap tests ran even if the style failed, and style and verification
had to run on two different platforms despite running identical checks.  I'm relatively
sure that's because of the limitation on dependencies between steps in the jobs.
Reusable workflows allow us to run the style, verification and now audit checks once,
then depending on the results, and the files changed, run the appropriate nix, windows
and bootstrap tests.  While it saves only a few minutes by itself, this makes it easier to
refactor checks to subset tests without having to replicate tests or other workflow
components in the future.

Co-authored-by: Massimiliano Culpo <massimiliano.culpo@gmail.com>

8 files changed, 367 insertions, 245 deletions

diff --git a/.github/workflows/bootstrap-test.sh b/.github/workflows/bootstrap-test.sh
new file mode 100755
index 0000000000..fc8f93d68b
--- /dev/null
+++ b/.github/workflows/bootstrap-test.sh
@@ -0,0 +1,7 @@
+#!/bin/bash
+set -ex
+source share/spack/setup-env.sh
+$PYTHON bin/spack bootstrap untrust spack-install
+$PYTHON bin/spack -d solve zlib
+tree $BOOTSTRAP/store
+exit 0
diff --git a/.github/workflows/bootstrap.yml b/.github/workflows/bootstrap.yml
index 9dd051d0ae..138a9c7382 100644
--- a/.github/workflows/bootstrap.yml
+++ b/.github/workflows/bootstrap.yml
@@ -3,33 +3,19 @@ name: Bootstrapping
 on:
   # This Workflow can be triggered manually
   workflow_dispatch:
-  pull_request:
-    branches:
-      - develop
-      - releases/**
-    paths-ignore:
-      # Don't run if we only modified packages in the
-      # built-in repository or documentation
-      - 'var/spack/repos/builtin/**'
-      - '!var/spack/repos/builtin/packages/clingo-bootstrap/**'
-      - '!var/spack/repos/builtin/packages/clingo/**'
-      - '!var/spack/repos/builtin/packages/python/**'
-      - '!var/spack/repos/builtin/packages/re2c/**'
-      - 'lib/spack/docs/**'
+  workflow_call:
   schedule:
     # nightly at 2:16 AM
     - cron: '16 2 * * *'
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_number }}
+  group: bootstrap-${{ github.workflow }}-${{ github.event.pull_request.number || github.run_number }}
   cancel-in-progress: true
 
 jobs:
-
   fedora-clingo-sources:
     runs-on: ubuntu-latest
     container: "fedora:latest"
-    if: github.repository == 'spack/spack'
     steps:
       - name: Install dependencies
         run: |
@@ -39,6 +25,8 @@ jobs:
               cmake bison bison-devel libstdc++-static
       - name: Checkout
         uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b
+        with:
+          fetch-depth: 0
       - name: Setup non-root user
         run: |
           # See [1] below
@@ -49,7 +37,6 @@ jobs:
         shell: runuser -u spack-test -- bash {0}
         run: |
           git --version
-          git fetch --unshallow
           . .github/workflows/setup_git.sh
       - name: Bootstrap clingo
         shell: runuser -u spack-test -- bash {0}
@@ -63,7 +50,6 @@ jobs:
   ubuntu-clingo-sources:
     runs-on: ubuntu-latest
     container: "ubuntu:latest"
-    if: github.repository == 'spack/spack'
     steps:
       - name: Install dependencies
         env:
@@ -76,6 +62,8 @@ jobs:
               cmake bison
       - name: Checkout
         uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b
+        with:
+          fetch-depth: 0
       - name: Setup non-root user
         run: |
           # See [1] below
@@ -86,7 +74,6 @@ jobs:
         shell: runuser -u spack-test -- bash {0}
         run: |
           git --version
-          git fetch --unshallow
           . .github/workflows/setup_git.sh
       - name: Bootstrap clingo
         shell: runuser -u spack-test -- bash {0}
@@ -100,7 +87,6 @@ jobs:
   ubuntu-clingo-binaries-and-patchelf:
     runs-on: ubuntu-latest
     container: "ubuntu:latest"
-    if: github.repository == 'spack/spack'
     steps:
       - name: Install dependencies
         env:
@@ -112,6 +98,8 @@ jobs:
               make patch unzip xz-utils python3 python3-dev tree
       - name: Checkout
         uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b
+        with:
+          fetch-depth: 0
       - name: Setup non-root user
         run: |
           # See [1] below
@@ -122,7 +110,6 @@ jobs:
         shell: runuser -u spack-test -- bash {0}
         run: |
           git --version
-          git fetch --unshallow
           . .github/workflows/setup_git.sh
       - name: Bootstrap clingo
         shell: runuser -u spack-test -- bash {0}
@@ -134,7 +121,6 @@ jobs:
   opensuse-clingo-sources:
     runs-on: ubuntu-latest
     container: "opensuse/leap:latest"
-    if: github.repository == 'spack/spack'
     steps:
       - name: Install dependencies
         run: |
@@ -146,12 +132,13 @@ jobs:
               cmake bison
       - name: Checkout
         uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b
+        with:
+          fetch-depth: 0
       - name: Setup repo
         run: |
           # See [1] below
           git config --global --add safe.directory /__w/spack/spack
           git --version
-          git fetch --unshallow
           . .github/workflows/setup_git.sh
       - name: Bootstrap clingo
         run: |
@@ -163,7 +150,6 @@ jobs:
 
   macos-clingo-sources:
     runs-on: macos-latest
-    if: github.repository == 'spack/spack'
     steps:
       - name: Install dependencies
         run: |
@@ -183,53 +169,70 @@ jobs:
     runs-on: ${{ matrix.macos-version }}
     strategy:
       matrix:
-        python-version: ['3.6', '3.7', '3.8', '3.9', '3.10']
         macos-version: ['macos-11', 'macos-12']
-    if: github.repository == 'spack/spack'
     steps:
       - name: Install dependencies
         run: |
           brew install tree
       - name: Checkout
         uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b
-      - uses: actions/setup-python@b55428b1882923874294fa556849718a1d7f2ca5
-        with:
-          python-version: ${{ matrix.python-version }}
       - name: Bootstrap clingo
         run: |
-          source share/spack/setup-env.sh
-          spack bootstrap untrust spack-install
-          spack -d solve zlib
-          tree ~/.spack/bootstrap/store/
+          set -ex
+          for ver in '3.6' '3.7' '3.8' '3.9' '3.10' ; do
+            not_found=1
+            ver_dir="$(find $RUNNER_TOOL_CACHE/Python -wholename "*/${ver}.*/*/bin" | grep . || true)"
+            echo "Testing $ver_dir"
+            if [[ -d "$ver_dir" ]] ; then
+              if $ver_dir/python --version ; then
+                export PYTHON="$ver_dir/python"
+                not_found=0
+                old_path="$PATH"
+                export PATH="$ver_dir:$PATH"
+                ./bin/spack-tmpconfig -b ./.github/workflows/bootstrap-test.sh
+                export PATH="$old_path"
+              fi
+            fi
+            # NOTE: test all pythons that exist, not all do on 12
+          done
 
   ubuntu-clingo-binaries:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version: ['2.7', '3.6', '3.7', '3.8', '3.9', '3.10']
-    if: github.repository == 'spack/spack'
+    runs-on: ubuntu-20.04
     steps:
       - name: Checkout
         uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b
-      - uses: actions/setup-python@b55428b1882923874294fa556849718a1d7f2ca5
         with:
-          python-version: ${{ matrix.python-version }}
+          fetch-depth: 0
       - name: Setup repo
         run: |
           git --version
-          git fetch --unshallow
           . .github/workflows/setup_git.sh
       - name: Bootstrap clingo
         run: |
-          source share/spack/setup-env.sh
-          spack bootstrap untrust spack-install
-          spack -d solve zlib
-          tree ~/.spack/bootstrap/store/
+          set -ex
+          for ver in '2.7' '3.6' '3.7' '3.8' '3.9' '3.10' ; do
+            not_found=1
+            ver_dir="$(find $RUNNER_TOOL_CACHE/Python -wholename "*/${ver}.*/*/bin" | grep . || true)"
+            echo "Testing $ver_dir"
+            if [[ -d "$ver_dir" ]] ; then
+              if $ver_dir/python --version ; then
+                export PYTHON="$ver_dir/python"
+                not_found=0
+                old_path="$PATH"
+                export PATH="$ver_dir:$PATH"
+                ./bin/spack-tmpconfig -b ./.github/workflows/bootstrap-test.sh
+                export PATH="$old_path"
+              fi
+            fi
+            if (($not_found)) ; then
+              echo Required python version $ver not found in runner!
+              exit 1
+            fi
+          done
 
   ubuntu-gnupg-binaries:
     runs-on: ubuntu-latest
     container: "ubuntu:latest"
-    if: github.repository == 'spack/spack'
     steps:
       - name: Install dependencies
         env:
@@ -241,6 +244,8 @@ jobs:
               make patch unzip xz-utils python3 python3-dev tree
       - name: Checkout
         uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b
+        with:
+          fetch-depth: 0
       - name: Setup non-root user
         run: |
           # See [1] below
@@ -251,7 +256,6 @@ jobs:
         shell: runuser -u spack-test -- bash {0}
         run: |
           git --version
-          git fetch --unshallow
           . .github/workflows/setup_git.sh
       - name: Bootstrap GnuPG
         shell: runuser -u spack-test -- bash {0}
@@ -264,7 +268,6 @@ jobs:
   ubuntu-gnupg-sources:
     runs-on: ubuntu-latest
     container: "ubuntu:latest"
-    if: github.repository == 'spack/spack'
     steps:
       - name: Install dependencies
         env:
@@ -277,6 +280,8 @@ jobs:
               gawk
       - name: Checkout
         uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b
+        with:
+          fetch-depth: 0
       - name: Setup non-root user
         run: |
           # See [1] below
@@ -287,7 +292,6 @@ jobs:
         shell: runuser -u spack-test -- bash {0}
         run: |
           git --version
-          git fetch --unshallow
           . .github/workflows/setup_git.sh
       - name: Bootstrap GnuPG
         shell: runuser -u spack-test -- bash {0}
@@ -300,7 +304,6 @@ jobs:
 
   macos-gnupg-binaries:
     runs-on: macos-latest
-    if: github.repository == 'spack/spack'
     steps:
       - name: Install dependencies
         run: |
@@ -318,7 +321,6 @@ jobs:
 
   macos-gnupg-sources:
     runs-on: macos-latest
-    if: github.repository == 'spack/spack'
     steps:
       - name: Install dependencies
         run: |
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
new file mode 100644
index 0000000000..62f2be080f
--- /dev/null
+++ b/.github/workflows/ci.yaml
@@ -0,0 +1,93 @@
+name: ci
+
+on:
+  push:
+    branches:
+      - develop
+      - releases/**
+  pull_request:
+    branches:
+      - develop
+      - releases/**
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_number }}
+  cancel-in-progress: true
+
+jobs:
+  prechecks:
+    needs: [ changes ]
+    uses: ./.github/workflows/valid-style.yml
+    with:
+      with_coverage: ${{ needs.changes.outputs.with_coverage }}
+  # Check which files have been updated by the PR
+  changes:
+    runs-on: ubuntu-latest
+      # Set job outputs to values from filter step
+    outputs:
+      bootstrap: ${{ steps.filter.outputs.bootstrap }}
+      core: ${{ steps.filter.outputs.core }}
+      packages: ${{ steps.filter.outputs.packages }}
+      with_coverage: ${{ steps.coverage.outputs.with_coverage }}
+    steps:
+      - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # @v2
+        if: ${{ github.event_name == 'push' }}
+        with:
+          fetch-depth: 0
+            # For pull requests it's not necessary to checkout the code
+      - uses: dorny/paths-filter@b2feaf19c27470162a626bd6fa8438ae5b263721
+        id: filter
+        with:
+          # See https://github.com/dorny/paths-filter/issues/56 for the syntax used below
+          # Don't run if we only modified packages in the
+          # built-in repository or documentation
+          filters: |
+            bootstrap:
+            - '!var/spack/repos/builtin/**'
+            - 'var/spack/repos/builtin/packages/clingo-bootstrap/**'
+            - 'var/spack/repos/builtin/packages/clingo/**'
+            - 'var/spack/repos/builtin/packages/python/**'
+            - 'var/spack/repos/builtin/packages/re2c/**'
+            - '!lib/spack/docs/**'
+            - 'lib/spack/**'
+            - 'share/spack/**'
+            - '.github/workflows/bootstrap.yml'
+            core:
+            - './!(var/**)/**'
+            packages:
+            - 'var/**'
+      # Some links for easier reference:
+      #
+      # "github" context: https://docs.github.com/en/actions/reference/context-and-expression-syntax-for-github-actions#github-context
+      # job outputs: https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#jobsjob_idoutputs
+      # setting environment variables from earlier steps: https://docs.github.com/en/actions/reference/workflow-commands-for-github-actions#setting-an-environment-variable
+      #
+      - id: coverage
+        # Run the subsequent jobs with coverage if core has been modified,
+        # regardless of whether this is a pull request or a push to a branch
+        run: |
+          echo Core changes: ${{ steps.filter.outputs.core }}
+          echo Event name: ${{ github.event_name }}
+          if [ "${{ steps.filter.outputs.core }}" == "true" ]
+          then
+            echo "::set-output name=with_coverage::true"
+          else
+            echo "::set-output name=with_coverage::false"
+          fi
+  bootstrap:
+    if: ${{ github.repository == 'spack/spack' && needs.changes.outputs.bootstrap == 'true' }}
+    needs: [ prechecks, changes ]
+    uses: ./.github/workflows/bootstrap.yml
+  unit-tests:
+    if: ${{ github.repository == 'spack/spack' }}
+    needs: [ prechecks, changes ]
+    uses: ./.github/workflows/unit_tests.yaml
+    with:
+      core: ${{ needs.changes.outputs.core }}
+      packages: ${{ needs.changes.outputs.packages }}
+      with_coverage: ${{ needs.changes.outputs.with_coverage }}
+  windows:
+    if: ${{ github.repository == 'spack/spack' }}
+    needs: [ prechecks ]
+    uses: ./.github/workflows/windows_python.yml
+
diff --git a/.github/workflows/unit_tests.yaml b/.github/workflows/unit_tests.yaml
index b20d969a43..3856f1039f 100644
--- a/.github/workflows/unit_tests.yaml
+++ b/.github/workflows/unit_tests.yaml
@@ -1,103 +1,25 @@
-name: linux tests
+name: unit tests
 
 on:
-  push:
-    branches:
-      - develop
-      - releases/**
-  pull_request:
-    branches:
-      - develop
-      - releases/**
+  workflow_call:
+    inputs:
+      core:
+        required: true
+        type: string
+      packages:
+        required: true
+        type: string
+      with_coverage:
+        required: true
+        type: string
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_number }}
+  group: unit_tests-${{ github.workflow }}-${{ github.event.pull_request.number || github.run_number }}
   cancel-in-progress: true
 
 jobs:
-  # Validate that the code can be run on all the Python versions
-  # supported by Spack
-  validate:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # @v2
-    - uses: actions/setup-python@b55428b1882923874294fa556849718a1d7f2ca5 # @v2
-      with:
-        python-version: '3.10'
-    - name: Install Python Packages
-      run: |
-        pip install --upgrade pip
-        pip install --upgrade vermin
-    - name: vermin (Spack's Core)
-      run: vermin --backport argparse --violations --backport typing -t=2.7- -t=3.6- -vvv lib/spack/spack/ lib/spack/llnl/ bin/
-    - name: vermin (Repositories)
-      run: vermin --backport argparse --violations --backport typing -t=2.7- -t=3.6- -vvv var/spack/repos
-  # Run style checks on the files that have been changed
-  style:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # @v2
-      with:
-        fetch-depth: 0
-    - uses: actions/setup-python@b55428b1882923874294fa556849718a1d7f2ca5 # @v2
-      with:
-        python-version: '3.10'
-    - name: Install Python packages
-      run: |
-        pip install --upgrade pip six setuptools types-six
-    - name: Setup git configuration
-      run: |
-        # Need this for the git tests to succeed.
-        git --version
-        . .github/workflows/setup_git.sh
-    - name: Run style tests
-      run: |
-          share/spack/qa/run-style-tests
-  # Check which files have been updated by the PR
-  changes:
-    runs-on: ubuntu-latest
-    # Set job outputs to values from filter step
-    outputs:
-      core: ${{ steps.filter.outputs.core }}
-      packages: ${{ steps.filter.outputs.packages }}
-      with_coverage: ${{ steps.coverage.outputs.with_coverage }}
-    steps:
-    - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # @v2
-      if: ${{ github.event_name == 'push' }}
-      with:
-        fetch-depth: 0
-    # For pull requests it's not necessary to checkout the code
-    - uses: dorny/paths-filter@b2feaf19c27470162a626bd6fa8438ae5b263721
-      id: filter
-      with:
-        # See https://github.com/dorny/paths-filter/issues/56 for the syntax used below
-        filters: |
-          core:
-          - './!(var/**)/**'
-          packages:
-          - 'var/**'
-    # Some links for easier reference:
-    #
-    # "github" context: https://docs.github.com/en/actions/reference/context-and-expression-syntax-for-github-actions#github-context
-    # job outputs: https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#jobsjob_idoutputs
-    # setting environment variables from earlier steps: https://docs.github.com/en/actions/reference/workflow-commands-for-github-actions#setting-an-environment-variable
-    #
-    - id: coverage
-      # Run the subsequent jobs with coverage if core has been modified,
-      # regardless of whether this is a pull request or a push to a branch
-      run: |
-        echo Core changes: ${{ steps.filter.outputs.core }}
-        echo Event name: ${{ github.event_name }}
-        if [ "${{ steps.filter.outputs.core }}" == "true" ]
-        then
-          echo "::set-output name=with_coverage::true"
-        else
-          echo "::set-output name=with_coverage::false"
-        fi
-
   # Run unit tests with different configurations on linux
-  unittests:
-    needs: [ validate, style, changes ]
+  ubuntu:
     runs-on: ubuntu-latest
     strategy:
       matrix:
@@ -148,7 +70,7 @@ jobs:
           spack bootstrap untrust spack-install
           spack -v solve zlib
     - name: Run unit tests (full suite with coverage)
-      if: ${{ needs.changes.outputs.with_coverage == 'true' }}
+      if: ${{ inputs.with_coverage == 'true' }}
       env:
           SPACK_PYTHON: python
           COVERAGE: true
@@ -158,7 +80,7 @@ jobs:
           coverage combine
           coverage xml
     - name: Run unit tests (reduced suite without coverage)
-      if: ${{ needs.changes.outputs.with_coverage == 'false' }}
+      if: ${{ inputs.with_coverage == 'false' }}
       env:
           SPACK_PYTHON: python
           ONLY_PACKAGES: true
@@ -166,12 +88,11 @@ jobs:
       run: |
           share/spack/qa/run-unit-tests
     - uses: codecov/codecov-action@81cd2dc8148241f03f5839d295e000b8f761e378 # @v2.1.0
-      if: ${{ needs.changes.outputs.with_coverage == 'true' }}
+      if: ${{ inputs.with_coverage == 'true' }}
       with:
         flags: unittests,linux,${{ matrix.concretizer }}
   # Test shell integration
   shell:
-    needs: [ validate, style, changes ]
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # @v2
@@ -194,26 +115,25 @@ jobs:
           git --version
           . .github/workflows/setup_git.sh
     - name: Run shell tests (without coverage)
-      if: ${{ needs.changes.outputs.with_coverage == 'false' }}
+      if: ${{ inputs.with_coverage == 'false' }}
       run: |
           share/spack/qa/run-shell-tests
     - name: Run shell tests (with coverage)
-      if: ${{ needs.changes.outputs.with_coverage == 'true' }}
+      if: ${{ inputs.with_coverage == 'true' }}
       env:
           COVERAGE: true
       run: |
           share/spack/qa/run-shell-tests
     - uses: codecov/codecov-action@81cd2dc8148241f03f5839d295e000b8f761e378 # @v2.1.0
-      if: ${{ needs.changes.outputs.with_coverage == 'true' }}
+      if: ${{ inputs.with_coverage == 'true' }}
       with:
         flags: shelltests,linux
 
   # Test RHEL8 UBI with platform Python. This job is run
   # only on PRs modifying core Spack
   rhel8-platform-python:
-    needs: [ validate, style, changes ]
     runs-on: ubuntu-latest
-    if: ${{ needs.changes.outputs.with_coverage == 'true' }}
+    if: ${{ inputs.with_coverage == 'true' }}
     container: registry.access.redhat.com/ubi8/ubi
     steps:
     - name: Install dependencies
@@ -237,7 +157,6 @@ jobs:
           spack unit-test -k 'not cvs and not svn and not hg' -x --verbose
   # Test for the clingo based solver (using clingo-cffi)
   clingo-cffi:
-    needs: [ validate, style, changes ]
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # @v2
@@ -262,7 +181,7 @@ jobs:
           git --version
           . .github/workflows/setup_git.sh
     - name: Run unit tests (full suite with coverage)
-      if: ${{ needs.changes.outputs.with_coverage == 'true' }}
+      if: ${{ inputs.with_coverage == 'true' }}
       env:
           COVERAGE: true
           SPACK_TEST_SOLVER: clingo
@@ -271,19 +190,18 @@ jobs:
           coverage combine
           coverage xml
     - name: Run unit tests (reduced suite without coverage)
-      if: ${{ needs.changes.outputs.with_coverage == 'false' }}
+      if: ${{ inputs.with_coverage == 'false' }}
       env:
           ONLY_PACKAGES: true
           SPACK_TEST_SOLVER: clingo
       run: |
           share/spack/qa/run-unit-tests
     - uses: codecov/codecov-action@81cd2dc8148241f03f5839d295e000b8f761e378 # @v2.1.0
-      if: ${{ needs.changes.outputs.with_coverage == 'true' }}
+      if: ${{ inputs.with_coverage == 'true' }}
       with:
         flags: unittests,linux,clingo
   # Run unit tests on MacOS
-  build:
-    needs: [ validate, style, changes ]
+  macos:
     runs-on: macos-latest
     strategy:
       matrix:
@@ -311,7 +229,7 @@ jobs:
         . share/spack/setup-env.sh
         $(which spack) bootstrap untrust spack-install
         $(which spack) solve zlib
-        if [ "${{ needs.changes.outputs.with_coverage }}" == "true" ]
+        if [ "${{ inputs.with_coverage }}" == "true" ]
         then
           coverage run $(which spack) unit-test -x
           coverage combine
@@ -324,36 +242,8 @@ jobs:
           $(which spack) unit-test -x -m "not maybeslow" -k "test_all_virtual_packages_have_default_providers"
         fi
     - uses: codecov/codecov-action@81cd2dc8148241f03f5839d295e000b8f761e378 # @v2.1.0
-      if: ${{ needs.changes.outputs.with_coverage == 'true' }}
+      if: ${{ inputs.with_coverage == 'true' }}
       with:
         files: ./coverage.xml
         flags: unittests,macos
 
-  # Run audits on all the packages in the built-in repository
-  package-audits:
-    needs: [ validate, style, changes ]
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # @v2
-    - uses: actions/setup-python@b55428b1882923874294fa556849718a1d7f2ca5 # @v2
-      with:
-        python-version: '3.10'
-    - name: Install Python packages
-      run: |
-        pip install --upgrade pip six setuptools pytest codecov coverage[toml]==6.2
-    - name: Package audits (with coverage)
-      if: ${{ needs.changes.outputs.with_coverage == 'true' }}
-      run: |
-          . share/spack/setup-env.sh
-          coverage run $(which spack) audit packages
-          coverage combine
-          coverage xml
-    - name: Package audits (without coverage)
-      if: ${{ needs.changes.outputs.with_coverage == 'false' }}
-      run: |
-          . share/spack/setup-env.sh
-          $(which spack) audit packages
-    - uses: codecov/codecov-action@81cd2dc8148241f03f5839d295e000b8f761e378 # @v2.1.0
-      if: ${{ needs.changes.outputs.with_coverage == 'true' }}
-      with:
-        flags: unittests,linux,audits
diff --git a/.github/workflows/valid-style.yml b/.github/workflows/valid-style.yml
new file mode 100644
index 0000000000..73c8dc76f1
--- /dev/null
+++ b/.github/workflows/valid-style.yml
@@ -0,0 +1,82 @@
+name: style
+
+on:
+  workflow_call:
+    inputs:
+      with_coverage:
+        required: true
+        type: string
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_number }}
+  cancel-in-progress: true
+
+
+jobs:
+  # Validate that the code can be run on all the Python versions
+  # supported by Spack
+  validate:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # @v2
+    - uses: actions/setup-python@b55428b1882923874294fa556849718a1d7f2ca5 # @v2
+      with:
+        python-version: '3.10'
+        cache: 'pip'
+    - name: Install Python Packages
+      run: |
+        pip install --upgrade pip
+        pip install --upgrade vermin
+    - name: vermin (Spack's Core)
+      run: vermin --backport argparse --violations --backport typing -t=2.7- -t=3.6- -vvv lib/spack/spack/ lib/spack/llnl/ bin/
+    - name: vermin (Repositories)
+      run: vermin --backport argparse --violations --backport typing -t=2.7- -t=3.6- -vvv var/spack/repos
+  # Run style checks on the files that have been changed
+  style:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # @v2
+      with:
+        fetch-depth: 0
+    - uses: actions/setup-python@b55428b1882923874294fa556849718a1d7f2ca5 # @v2
+      with:
+        python-version: '3.10'
+        cache: 'pip'
+    - name: Install Python packages
+      run: |
+        python3 -m pip install --upgrade pip six setuptools types-six click==8.0.2 'black==21.12b0' mypy isort clingo flake8
+    - name: Setup git configuration
+      run: |
+        # Need this for the git tests to succeed.
+        git --version
+        . .github/workflows/setup_git.sh
+    - name: Run style tests
+      run: |
+          share/spack/qa/run-style-tests
+  # Run audits on all the packages in the built-in repository
+  package-audits:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # @v2
+    - uses: actions/setup-python@b55428b1882923874294fa556849718a1d7f2ca5 # @v2
+      with:
+        python-version: '3.10'
+    - name: Install Python packages
+      run: |
+        pip install --upgrade pip six setuptools pytest codecov coverage[toml]==6.2
+    - name: Package audits (with coverage)
+      if: ${{ inputs.with_coverage == 'true' }}
+      run: |
+          . share/spack/setup-env.sh
+          coverage run $(which spack) audit packages
+          coverage combine
+          coverage xml
+    - name: Package audits (without coverage)
+      if: ${{ inputs.with_coverage == 'false' }}
+      run: |
+          . share/spack/setup-env.sh
+          $(which spack) audit packages
+    - uses: codecov/codecov-action@81cd2dc8148241f03f5839d295e000b8f761e378 # @v2.1.0
+      if: ${{ inputs.with_coverage == 'true' }}
+      with:
+        flags: unittests,linux,audits
diff --git a/.github/workflows/windows_python.yml b/.github/workflows/windows_python.yml
index e45fdb2bc8..5bd009df44 100644
--- a/.github/workflows/windows_python.yml
+++ b/.github/workflows/windows_python.yml
@@ -1,17 +1,10 @@
-name: windows tests
+name: windows
 
 on:
-  push:
-    branches:
-      - develop
-      - releases/**
-  pull_request:
-    branches:
-      - develop
-      - releases/**
+  workflow_call:
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_number }}
+  group: windows-${{ github.workflow }}-${{ github.event.pull_request.number || github.run_number }}
   cancel-in-progress: true
 
 defaults:
@@ -19,45 +12,7 @@ defaults:
     shell:
      powershell Invoke-Expression -Command ".\share\spack\qa\windows_test_setup.ps1"; {0}
 jobs:
-  validate:
-    runs-on: windows-latest
-    steps:
-    - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b
-    - uses: actions/setup-python@b55428b1882923874294fa556849718a1d7f2ca5
-      with:
-        python-version: 3.9
-    - name: Install Python Packages
-      run: |
-        python -m pip install --upgrade pip
-        python -m pip install --upgrade vermin
-    - name: vermin (Spack's Core)
-      run: vermin --backport argparse --backport typing -t='2.7-' -t='3.6-' -v spack/lib/spack/spack/ spack/lib/spack/llnl/ spack/bin/
-    - name: vermin (Repositories)
-      run: vermin --backport argparse --backport typing -t='2.7-' -t='3.6-' -v spack/var/spack/repos
-  # Run style checks on the files that have been changed
-  style:
-    runs-on: windows-latest
-    steps:
-    - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b
-      with:
-        fetch-depth: 0
-    - uses: actions/setup-python@b55428b1882923874294fa556849718a1d7f2ca5
-      with:
-        python-version: 3.9
-    - name: Install Python packages
-      run: |
-        python -m pip install --upgrade pip six setuptools flake8 "isort>=4.3.5" "mypy>=0.800" "click==8.0.4" "black<=21.12b0" pywin32 types-python-dateutil
-    - name: Create local develop
-      run: |
-        .\spack\.github\workflows\setup_git.ps1
-    - name: Run style tests
-      run: |
-        spack style
-    - name: Verify license headers
-      run: |
-        python spack\bin\spack license verify
-  unittest:
-    needs: [ validate, style ]
+  unit-tests:
     runs-on: windows-latest
     steps:
     - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b
@@ -76,8 +31,7 @@ jobs:
       run: |
         echo F|xcopy .\spack\share\spack\qa\configuration\windows_config.yaml $env:USERPROFILE\.spack\windows\config.yaml
         spack unit-test --verbose --ignore=lib/spack/spack/test/cmd
-  unittest-cmd:
-    needs: [ validate, style ]
+  unit-tests-cmd:
     runs-on: windows-latest
     steps:
     - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b
@@ -96,8 +50,7 @@ jobs:
       run: |
         echo F|xcopy .\spack\share\spack\qa\configuration\windows_config.yaml $env:USERPROFILE\.spack\windows\config.yaml
         spack unit-test lib/spack/spack/test/cmd --verbose
-  buildtest:
-    needs: [ validate, style ]
+  build-abseil:
     runs-on: windows-latest
     steps:
     - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b
@@ -116,8 +69,7 @@ jobs:
         spack external find cmake
         spack external find ninja
         spack install abseil-cpp
-  generate-installer-test:
-    needs: [ validate, style ]
+  make-installer:
     runs-on: windows-latest
     steps:
     - name: Disable Windows Symlinks
@@ -153,7 +105,7 @@ jobs:
         name: Windows Spack Installer
         path: ${{ env.installer_root}}\pkg\Spack.msi
   execute-installer:
-    needs: generate-installer-test
+    needs: make-installer
     runs-on: windows-latest
     defaults:
       run:
diff --git a/bin/spack-tmpconfig b/bin/spack-tmpconfig
new file mode 100755
index 0000000000..b9cca15cc0
--- /dev/null
+++ b/bin/spack-tmpconfig
@@ -0,0 +1,96 @@
+#!/bin/bash
+set -euo pipefail
+[[ -n "${TMPCONFIG_DEBUG:=}" ]] && set -x
+DIR="$(cd -P "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+mkdir -p "${XDG_RUNTIME_DIR:=/tmp}/spack-tests"
+export TMPDIR="${XDG_RUNTIME_DIR}"
+export TMP_DIR="$(mktemp -d -t spack-test-XXXXX)"
+clean_up() {
+    [[ -n "$TMPCONFIG_DEBUG" ]] && printf "cleaning up: $TMP_DIR\n"
+    [[ -n "$TMPCONFIG_DEBUG" ]] && tree "$TMP_DIR"
+    rm -rf "$TMP_DIR"
+}
+trap clean_up EXIT
+trap clean_up ERR
+
+[[ -n "$TMPCONFIG_DEBUG" ]] && printf "Redirecting TMP_DIR and spack directories to $TMP_DIR\n"
+
+export BOOTSTRAP="${SPACK_USER_CACHE_PATH:=$HOME/.spack}/bootstrap"
+export SPACK_USER_CACHE_PATH="$TMP_DIR/user_cache"
+mkdir -p "$SPACK_USER_CACHE_PATH"
+
+private_bootstrap="$SPACK_USER_CACHE_PATH/bootstrap"
+use_spack=''
+use_bwrap=''
+# argument handling
+while (($# >= 1)) ; do
+    case "$1" in
+        -b) # privatize bootstrap too, useful for CI but not always cheap
+            shift
+            export BOOTSTRAP="$private_bootstrap"
+            ;;
+        -B) # use specified bootstrap dir
+            export BOOTSTRAP="$2"
+            shift 2
+            ;;
+        -s) # run spack directly with remaining args
+            shift
+            use_spack=1
+            ;;
+        --contain=bwrap)
+            if bwrap --help 2>&1 > /dev/null ; then
+                use_bwrap=1
+            else
+                echo Bubblewrap containment requested, but no bwrap command found
+                exit 1
+            fi
+            shift
+            ;;
+        --)
+            shift
+            break
+            ;;
+        *)
+            break
+            ;;
+    esac
+done
+typeset -a CMD
+if [[ -n "$use_spack" ]] ; then
+    CMD=("$DIR/spack" "$@")
+else
+    CMD=("$@")
+fi
+
+mkdir -p "$BOOTSTRAP"
+
+export SPACK_SYSTEM_CONFIG_PATH="$TMP_DIR/sys_conf"
+export SPACK_USER_CONFIG_PATH="$TMP_DIR/user_conf"
+mkdir -p "$SPACK_USER_CONFIG_PATH"
+cat >"$SPACK_USER_CONFIG_PATH/config.yaml" <<EOF
+config:
+  install_tree:
+    root: $TMP_DIR/install
+  misc_cache: $$user_cache_path/cache
+  source_cache: $$user_cache_path/source
+EOF
+cat >"$SPACK_USER_CONFIG_PATH/bootstrap.yaml" <<EOF
+bootstrap:
+  root: $BOOTSTRAP
+EOF
+
+if [[ -n "$use_bwrap" ]] ; then
+    CMD=(
+        bwrap
+        --dev-bind / /
+        --ro-bind "$DIR/.." "$DIR/.." # do not touch spack root
+        --ro-bind $HOME/.spack $HOME/.spack # do not touch user config/cache dir
+        --bind "$TMP_DIR" "$TMP_DIR"
+        --bind "$BOOTSTRAP" "$BOOTSTRAP"
+        --die-with-parent
+        "${CMD[@]}"
+    )
+fi
+
+(( ${TMPCONFIG_DEBUG:=0} > 1)) && echo "Running: ${CMD[@]}"
+"${CMD[@]}"
diff --git a/lib/spack/spack/test/cmd/env.py b/lib/spack/spack/test/cmd/env.py
index e465060392..ceefd3323b 100644
--- a/lib/spack/spack/test/cmd/env.py
+++ b/lib/spack/spack/test/cmd/env.py
@@ -788,7 +788,7 @@ def test_env_with_included_config_file(packages_file):
     e = ev.read("test")
 
     included_path = os.path.join(e.path, include_filename)
-    fs.rename(packages_file.strpath, included_path)
+    shutil.move(packages_file.strpath, included_path)
 
     with e:
         e.concretize()