Reorder workflow execution in GHA (#32183)

This patchset refactors our GitHub actions into a single top-level ci workflow that
invokes a series of reusable actions.  The main goal of this is to be able to easily
control which tests run and in what order based on the success or failure of top-level
prechecks.  Our previous workflows ran in three sets:

* nix tests: style and verification first, then linux and macos tests if successful
* windows tests: style and verification first, then linux and macos tests if successful
* bootstrap tests

As a result, the bootstrap tests ran even if the style failed, and style and verification
had to run on two different platforms despite running identical checks.  I'm relatively
sure that's because of the limitation on dependencies between steps in the jobs.
Reusable workflows allow us to run the style, verification and now audit checks once,
then depending on the results, and the files changed, run the appropriate nix, windows
and bootstrap tests.  While it saves only a few minutes by itself, this makes it easier to
refactor checks to subset tests without having to replicate tests or other workflow
components in the future.

Co-authored-by: Massimiliano Culpo <massimiliano.culpo@gmail.com>

1 files changed, 53 insertions, 51 deletions

diff --git a/.github/workflows/bootstrap.yml b/.github/workflows/bootstrap.yml
index 9dd051d0ae..138a9c7382 100644
--- a/.github/workflows/bootstrap.yml
+++ b/.github/workflows/bootstrap.yml
@@ -3,33 +3,19 @@ name: Bootstrapping
 on:
   # This Workflow can be triggered manually
   workflow_dispatch:
-  pull_request:
-    branches:
-      - develop
-      - releases/**
-    paths-ignore:
-      # Don't run if we only modified packages in the
-      # built-in repository or documentation
-      - 'var/spack/repos/builtin/**'
-      - '!var/spack/repos/builtin/packages/clingo-bootstrap/**'
-      - '!var/spack/repos/builtin/packages/clingo/**'
-      - '!var/spack/repos/builtin/packages/python/**'
-      - '!var/spack/repos/builtin/packages/re2c/**'
-      - 'lib/spack/docs/**'
+  workflow_call:
   schedule:
     # nightly at 2:16 AM
     - cron: '16 2 * * *'
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_number }}
+  group: bootstrap-${{ github.workflow }}-${{ github.event.pull_request.number || github.run_number }}
   cancel-in-progress: true
 
 jobs:
-
   fedora-clingo-sources:
     runs-on: ubuntu-latest
     container: "fedora:latest"
-    if: github.repository == 'spack/spack'
     steps:
       - name: Install dependencies
         run: |
@@ -39,6 +25,8 @@ jobs:
               cmake bison bison-devel libstdc++-static
       - name: Checkout
         uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b
+        with:
+          fetch-depth: 0
       - name: Setup non-root user
         run: |
           # See [1] below
@@ -49,7 +37,6 @@ jobs:
         shell: runuser -u spack-test -- bash {0}
         run: |
           git --version
-          git fetch --unshallow
           . .github/workflows/setup_git.sh
       - name: Bootstrap clingo
         shell: runuser -u spack-test -- bash {0}
@@ -63,7 +50,6 @@ jobs:
   ubuntu-clingo-sources:
     runs-on: ubuntu-latest
     container: "ubuntu:latest"
-    if: github.repository == 'spack/spack'
     steps:
       - name: Install dependencies
         env:
@@ -76,6 +62,8 @@ jobs:
               cmake bison
       - name: Checkout
         uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b
+        with:
+          fetch-depth: 0
       - name: Setup non-root user
         run: |
           # See [1] below
@@ -86,7 +74,6 @@ jobs:
         shell: runuser -u spack-test -- bash {0}
         run: |
           git --version
-          git fetch --unshallow
           . .github/workflows/setup_git.sh
       - name: Bootstrap clingo
         shell: runuser -u spack-test -- bash {0}
@@ -100,7 +87,6 @@ jobs:
   ubuntu-clingo-binaries-and-patchelf:
     runs-on: ubuntu-latest
     container: "ubuntu:latest"
-    if: github.repository == 'spack/spack'
     steps:
       - name: Install dependencies
         env:
@@ -112,6 +98,8 @@ jobs:
               make patch unzip xz-utils python3 python3-dev tree
       - name: Checkout
         uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b
+        with:
+          fetch-depth: 0
       - name: Setup non-root user
         run: |
           # See [1] below
@@ -122,7 +110,6 @@ jobs:
         shell: runuser -u spack-test -- bash {0}
         run: |
           git --version
-          git fetch --unshallow
           . .github/workflows/setup_git.sh
       - name: Bootstrap clingo
         shell: runuser -u spack-test -- bash {0}
@@ -134,7 +121,6 @@ jobs:
   opensuse-clingo-sources:
     runs-on: ubuntu-latest
     container: "opensuse/leap:latest"
-    if: github.repository == 'spack/spack'
     steps:
       - name: Install dependencies
         run: |
@@ -146,12 +132,13 @@ jobs:
               cmake bison
       - name: Checkout
         uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b
+        with:
+          fetch-depth: 0
       - name: Setup repo
         run: |
           # See [1] below
           git config --global --add safe.directory /__w/spack/spack
           git --version
-          git fetch --unshallow
           . .github/workflows/setup_git.sh
       - name: Bootstrap clingo
         run: |
@@ -163,7 +150,6 @@ jobs:
 
   macos-clingo-sources:
     runs-on: macos-latest
-    if: github.repository == 'spack/spack'
     steps:
       - name: Install dependencies
         run: |
@@ -183,53 +169,70 @@ jobs:
     runs-on: ${{ matrix.macos-version }}
     strategy:
       matrix:
-        python-version: ['3.6', '3.7', '3.8', '3.9', '3.10']
         macos-version: ['macos-11', 'macos-12']
-    if: github.repository == 'spack/spack'
     steps:
       - name: Install dependencies
         run: |
           brew install tree
       - name: Checkout
         uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b
-      - uses: actions/setup-python@b55428b1882923874294fa556849718a1d7f2ca5
-        with:
-          python-version: ${{ matrix.python-version }}
       - name: Bootstrap clingo
         run: |
-          source share/spack/setup-env.sh
-          spack bootstrap untrust spack-install
-          spack -d solve zlib
-          tree ~/.spack/bootstrap/store/
+          set -ex
+          for ver in '3.6' '3.7' '3.8' '3.9' '3.10' ; do
+            not_found=1
+            ver_dir="$(find $RUNNER_TOOL_CACHE/Python -wholename "*/${ver}.*/*/bin" | grep . || true)"
+            echo "Testing $ver_dir"
+            if [[ -d "$ver_dir" ]] ; then
+              if $ver_dir/python --version ; then
+                export PYTHON="$ver_dir/python"
+                not_found=0
+                old_path="$PATH"
+                export PATH="$ver_dir:$PATH"
+                ./bin/spack-tmpconfig -b ./.github/workflows/bootstrap-test.sh
+                export PATH="$old_path"
+              fi
+            fi
+            # NOTE: test all pythons that exist, not all do on 12
+          done
 
   ubuntu-clingo-binaries:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version: ['2.7', '3.6', '3.7', '3.8', '3.9', '3.10']
-    if: github.repository == 'spack/spack'
+    runs-on: ubuntu-20.04
     steps:
       - name: Checkout
         uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b
-      - uses: actions/setup-python@b55428b1882923874294fa556849718a1d7f2ca5
         with:
-          python-version: ${{ matrix.python-version }}
+          fetch-depth: 0
       - name: Setup repo
         run: |
           git --version
-          git fetch --unshallow
           . .github/workflows/setup_git.sh
       - name: Bootstrap clingo
         run: |
-          source share/spack/setup-env.sh
-          spack bootstrap untrust spack-install
-          spack -d solve zlib
-          tree ~/.spack/bootstrap/store/
+          set -ex
+          for ver in '2.7' '3.6' '3.7' '3.8' '3.9' '3.10' ; do
+            not_found=1
+            ver_dir="$(find $RUNNER_TOOL_CACHE/Python -wholename "*/${ver}.*/*/bin" | grep . || true)"
+            echo "Testing $ver_dir"
+            if [[ -d "$ver_dir" ]] ; then
+              if $ver_dir/python --version ; then
+                export PYTHON="$ver_dir/python"
+                not_found=0
+                old_path="$PATH"
+                export PATH="$ver_dir:$PATH"
+                ./bin/spack-tmpconfig -b ./.github/workflows/bootstrap-test.sh
+                export PATH="$old_path"
+              fi
+            fi
+            if (($not_found)) ; then
+              echo Required python version $ver not found in runner!
+              exit 1
+            fi
+          done
 
   ubuntu-gnupg-binaries:
     runs-on: ubuntu-latest
     container: "ubuntu:latest"
-    if: github.repository == 'spack/spack'
     steps:
       - name: Install dependencies
         env:
@@ -241,6 +244,8 @@ jobs:
               make patch unzip xz-utils python3 python3-dev tree
       - name: Checkout
         uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b
+        with:
+          fetch-depth: 0
       - name: Setup non-root user
         run: |
           # See [1] below
@@ -251,7 +256,6 @@ jobs:
         shell: runuser -u spack-test -- bash {0}
         run: |
           git --version
-          git fetch --unshallow
           . .github/workflows/setup_git.sh
       - name: Bootstrap GnuPG
         shell: runuser -u spack-test -- bash {0}
@@ -264,7 +268,6 @@ jobs:
   ubuntu-gnupg-sources:
     runs-on: ubuntu-latest
     container: "ubuntu:latest"
-    if: github.repository == 'spack/spack'
     steps:
       - name: Install dependencies
         env:
@@ -277,6 +280,8 @@ jobs:
               gawk
       - name: Checkout
         uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b
+        with:
+          fetch-depth: 0
       - name: Setup non-root user
         run: |
           # See [1] below
@@ -287,7 +292,6 @@ jobs:
         shell: runuser -u spack-test -- bash {0}
         run: |
           git --version
-          git fetch --unshallow
           . .github/workflows/setup_git.sh
       - name: Bootstrap GnuPG
         shell: runuser -u spack-test -- bash {0}
@@ -300,7 +304,6 @@ jobs:
 
   macos-gnupg-binaries:
     runs-on: macos-latest
-    if: github.repository == 'spack/spack'
     steps:
       - name: Install dependencies
         run: |
@@ -318,7 +321,6 @@ jobs:
 
   macos-gnupg-sources:
     runs-on: macos-latest
-    if: github.repository == 'spack/spack'
     steps:
       - name: Install dependencies
         run: |