Make GHA tests parallel by using xdist (#32361)

* Add two no-op jobs named "all-prechecks" and "all"

These are a suggestion from @tgamblin, they are stable named markers we
can use from gitlab and possibly for required checks to make CI more
resilient to refactors changing the names of specific checks.

* Enable parallel testing using xdist for unit testing in CI

* Normalize tmp paths to deal with macos

* add -u flag compatibility to spack python

As of now, it is accepted and ignored.  The usage with xdist, where it
is invoked specifically by `python -u spack python` which is then passed
`-u` by xdist is the entire reason for doing this.  It should never be
used without explicitly passing -u to the executing python interpreter.

* use spack python in xdist to support python 2

When running on python2, spack has many import cycles unless started
through main.  To allow that, this uses `spack python` as the
interpreter, leveraging the `-u` support so xdist doesn't error out when
it unconditionally requests unbuffered binary IO.

* Use shutil.move to account for tmpdir being in a separate filesystem sometimes

7 files changed, 100 insertions, 42 deletions

diff --git a/.github/workflows/audit.yaml b/.github/workflows/audit.yaml
new file mode 100644
index 0000000000..2b8c989518
--- /dev/null
+++ b/.github/workflows/audit.yaml
@@ -0,0 +1,44 @@
+name: audit
+
+on:
+  workflow_call:
+    inputs:
+      with_coverage:
+        required: true
+        type: string
+      python_version:
+        required: true
+        type: string
+
+concurrency:
+  group: audit-${{inputs.python_version}}-${{github.ref}}-${{github.event.pull_request.number || github.run_number}}
+  cancel-in-progress: true
+
+jobs:
+  # Run audits on all the packages in the built-in repository
+  package-audits:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # @v2
+    - uses: actions/setup-python@b55428b1882923874294fa556849718a1d7f2ca5 # @v2
+      with:
+        python-version: ${{inputs.python_version}}
+    - name: Install Python packages
+      run: |
+        pip install --upgrade pip six setuptools pytest codecov 'coverage[toml]<=6.2'
+    - name: Package audits (with coverage)
+      if: ${{ inputs.with_coverage == 'true' }}
+      run: |
+          . share/spack/setup-env.sh
+          coverage run $(which spack) audit packages
+          coverage combine
+          coverage xml
+    - name: Package audits (without coverage)
+      if: ${{ inputs.with_coverage == 'false' }}
+      run: |
+          . share/spack/setup-env.sh
+          $(which spack) audit packages
+    - uses: codecov/codecov-action@81cd2dc8148241f03f5839d295e000b8f761e378 # @v2.1.0
+      if: ${{ inputs.with_coverage == 'true' }}
+      with:
+        flags: unittests,linux,audits
diff --git a/.github/workflows/bootstrap.yml b/.github/workflows/bootstrap.yml
index 138a9c7382..e649de2fb4 100644
--- a/.github/workflows/bootstrap.yml
+++ b/.github/workflows/bootstrap.yml
@@ -9,7 +9,7 @@ on:
     - cron: '16 2 * * *'
 
 concurrency:
-  group: bootstrap-${{ github.workflow }}-${{ github.event.pull_request.number || github.run_number }}
+  group: bootstrap-${{github.ref}}-${{github.event.pull_request.number || github.run_number}}
   cancel-in-progress: true
 
 jobs:
diff --git a/.github/workflows/build-containers.yml b/.github/workflows/build-containers.yml
index f849747995..30765cfc86 100644
--- a/.github/workflows/build-containers.yml
+++ b/.github/workflows/build-containers.yml
@@ -20,7 +20,7 @@ on:
     types: [published]
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_number }}
+  group: build_containers-${{github.ref}}-${{github.event.pull_request.number || github.run_number}}
   cancel-in-progress: true
 
 jobs:
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 62f2be080f..f4943789d6 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -11,7 +11,7 @@ on:
       - releases/**
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_number }}
+  group: ci-${{github.ref}}-${{github.event.pull_request.number || github.run_number}}
   cancel-in-progress: true
 
 jobs:
@@ -20,6 +20,18 @@ jobs:
     uses: ./.github/workflows/valid-style.yml
     with:
       with_coverage: ${{ needs.changes.outputs.with_coverage }}
+  audit-ancient-python:
+    uses: ./.github/workflows/audit.yaml
+    needs: [ changes ]
+    with:
+      with_coverage: ${{ needs.changes.outputs.with_coverage }}
+      python_version: 2.7
+  all-prechecks:
+    needs: [ prechecks ]
+    runs-on: ubuntu-latest
+    steps:
+    - name: Success
+      run: "true"
   # Check which files have been updated by the PR
   changes:
     runs-on: ubuntu-latest
@@ -53,6 +65,7 @@ jobs:
             - 'share/spack/**'
             - '.github/workflows/bootstrap.yml'
             core:
+            - '!lib/spack/docs/**'
             - './!(var/**)/**'
             packages:
             - 'var/**'
@@ -79,7 +92,7 @@ jobs:
     needs: [ prechecks, changes ]
     uses: ./.github/workflows/bootstrap.yml
   unit-tests:
-    if: ${{ github.repository == 'spack/spack' }}
+    if: ${{ github.repository == 'spack/spack' && needs.changes.outputs.core == 'true' }}
     needs: [ prechecks, changes ]
     uses: ./.github/workflows/unit_tests.yaml
     with:
@@ -87,7 +100,13 @@ jobs:
       packages: ${{ needs.changes.outputs.packages }}
       with_coverage: ${{ needs.changes.outputs.with_coverage }}
   windows:
-    if: ${{ github.repository == 'spack/spack' }}
+    if: ${{ github.repository == 'spack/spack' && needs.changes.outputs.core == 'true' }}
     needs: [ prechecks ]
     uses: ./.github/workflows/windows_python.yml
+  all:
+    needs: [ windows, unit-tests, bootstrap, audit-ancient-python ]
+    runs-on: ubuntu-latest
+    steps:
+    - name: Success
+      run: "true"
 
diff --git a/.github/workflows/unit_tests.yaml b/.github/workflows/unit_tests.yaml
index 3856f1039f..6b1411c61b 100644
--- a/.github/workflows/unit_tests.yaml
+++ b/.github/workflows/unit_tests.yaml
@@ -1,6 +1,7 @@
 name: unit tests
 
 on:
+  workflow_dispatch:
   workflow_call:
     inputs:
       core:
@@ -14,7 +15,7 @@ on:
         type: string
 
 concurrency:
-  group: unit_tests-${{ github.workflow }}-${{ github.event.pull_request.number || github.run_number }}
+  group: unit_tests-${{github.ref}}-${{github.event.pull_request.number || github.run_number}}
   cancel-in-progress: true
 
 jobs:
@@ -25,11 +26,26 @@ jobs:
       matrix:
         python-version: ['2.7', '3.6', '3.7', '3.8', '3.9', '3.10']
         concretizer: ['clingo']
+        on_develop:
+        - ${{ github.ref == 'refs/heads/develop' }}
         include:
         - python-version: 2.7
           concretizer: original
-        - python-version: 3.9
+          on_develop: false
+        - python-version: '3.10'
           concretizer: original
+          on_develop: false
+        exclude:
+        - python-version: '3.7'
+          concretizer: 'clingo'
+          on_develop: false
+        - python-version: '3.8'
+          concretizer: 'clingo'
+          on_develop: false
+        - python-version: '3.9'
+          concretizer: 'clingo'
+          on_develop: false
+
     steps:
     - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # @v2
       with:
@@ -46,7 +62,7 @@ jobs:
               patchelf cmake bison libbison-dev kcov
     - name: Install Python packages
       run: |
-          pip install --upgrade pip six setuptools pytest codecov "coverage[toml]<=6.2"
+          pip install --upgrade pip six setuptools pytest codecov "coverage[toml]<=6.2" pytest-xdist
           # ensure style checks are not skipped in unit tests for python >= 3.6
           # note that true/false (i.e., 1/0) are opposite in conditions in python and bash
           if python -c 'import sys; sys.exit(not sys.version_info >= (3, 6))'; then
@@ -108,7 +124,7 @@ jobs:
           sudo apt-get install -y coreutils kcov csh zsh tcsh fish dash bash
     - name: Install Python packages
       run: |
-          pip install --upgrade pip six setuptools pytest codecov coverage[toml]==6.2
+          pip install --upgrade pip six setuptools pytest codecov coverage[toml]==6.2 pytest-xdist
     - name: Setup git configuration
       run: |
           # Need this for the git tests to succeed.
@@ -174,7 +190,7 @@ jobs:
               patchelf kcov
     - name: Install Python packages
       run: |
-          pip install --upgrade pip six setuptools pytest codecov coverage[toml]==6.2 clingo
+          pip install --upgrade pip six setuptools pytest codecov coverage[toml]==6.2 clingo pytest-xdist
     - name: Setup git configuration
       run: |
           # Need this for the git tests to succeed.
@@ -216,7 +232,7 @@ jobs:
     - name: Install Python packages
       run: |
           pip install --upgrade pip six setuptools
-          pip install --upgrade pytest codecov coverage[toml]==6.2
+          pip install --upgrade pytest codecov coverage[toml]==6.2 pytest-xdist
     - name: Setup Homebrew packages
       run: |
         brew install dash fish gcc gnupg2 kcov
@@ -229,9 +245,10 @@ jobs:
         . share/spack/setup-env.sh
         $(which spack) bootstrap untrust spack-install
         $(which spack) solve zlib
+        common_args=(--dist loadfile --tx '4*popen//python=./bin/spack-tmpconfig python -u ./bin/spack python' -x)
         if [ "${{ inputs.with_coverage }}" == "true" ]
         then
-          coverage run $(which spack) unit-test -x
+          coverage run $(which spack) unit-test "${common_args[@]}"
           coverage combine
           coverage xml
           # Delete the symlink going from ./lib/spack/docs/_spack_root back to
@@ -239,7 +256,7 @@ jobs:
           rm lib/spack/docs/_spack_root
         else
           echo "ONLY PACKAGE RECIPES CHANGED [skipping coverage]"
-          $(which spack) unit-test -x -m "not maybeslow" -k "test_all_virtual_packages_have_default_providers"
+          $(which spack) unit-test "${common_args[@]}" -m "not maybeslow" -k "test_all_virtual_packages_have_default_providers"
         fi
     - uses: codecov/codecov-action@81cd2dc8148241f03f5839d295e000b8f761e378 # @v2.1.0
       if: ${{ inputs.with_coverage == 'true' }}
diff --git a/.github/workflows/valid-style.yml b/.github/workflows/valid-style.yml
index 73c8dc76f1..2519fd77a0 100644
--- a/.github/workflows/valid-style.yml
+++ b/.github/workflows/valid-style.yml
@@ -8,7 +8,7 @@ on:
         type: string
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_number }}
+  group: style-${{github.ref}}-${{github.event.pull_request.number || github.run_number}}
   cancel-in-progress: true
 
 
@@ -53,30 +53,8 @@ jobs:
     - name: Run style tests
       run: |
           share/spack/qa/run-style-tests
-  # Run audits on all the packages in the built-in repository
-  package-audits:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # @v2
-    - uses: actions/setup-python@b55428b1882923874294fa556849718a1d7f2ca5 # @v2
-      with:
-        python-version: '3.10'
-    - name: Install Python packages
-      run: |
-        pip install --upgrade pip six setuptools pytest codecov coverage[toml]==6.2
-    - name: Package audits (with coverage)
-      if: ${{ inputs.with_coverage == 'true' }}
-      run: |
-          . share/spack/setup-env.sh
-          coverage run $(which spack) audit packages
-          coverage combine
-          coverage xml
-    - name: Package audits (without coverage)
-      if: ${{ inputs.with_coverage == 'false' }}
-      run: |
-          . share/spack/setup-env.sh
-          $(which spack) audit packages
-    - uses: codecov/codecov-action@81cd2dc8148241f03f5839d295e000b8f761e378 # @v2.1.0
-      if: ${{ inputs.with_coverage == 'true' }}
-      with:
-        flags: unittests,linux,audits
+  audit:
+    uses: ./.github/workflows/audit.yaml
+    with:
+      with_coverage: ${{ inputs.with_coverage }}
+      python_version: '3.10'
diff --git a/.github/workflows/windows_python.yml b/.github/workflows/windows_python.yml
index 5bd009df44..82e48370ac 100644
--- a/.github/workflows/windows_python.yml
+++ b/.github/workflows/windows_python.yml
@@ -4,7 +4,7 @@ on:
   workflow_call:
 
 concurrency:
-  group: windows-${{ github.workflow }}-${{ github.event.pull_request.number || github.run_number }}
+  group: windows-${{github.ref}}-${{github.event.pull_request.number || github.run_number}}
   cancel-in-progress: true
 
 defaults: