From 69f7a8f4d142cf5b7142bf8f20925b214e56316a Mon Sep 17 00:00:00 2001 From: Tom Scogland Date: Fri, 2 Sep 2022 14:09:23 -0700 Subject: Reorder workflow execution in GHA (#32183) This patchset refactors our GitHub actions into a single top-level ci workflow that invokes a series of reusable actions. The main goal of this is to be able to easily control which tests run and in what order based on the success or failure of top-level prechecks. Our previous workflows ran in three sets: * nix tests: style and verification first, then linux and macos tests if successful * windows tests: style and verification first, then linux and macos tests if successful * bootstrap tests As a result, the bootstrap tests ran even if the style failed, and style and verification had to run on two different platforms despite running identical checks. I'm relatively sure that's because of the limitation on dependencies between steps in the jobs. Reusable workflows allow us to run the style, verification and now audit checks once, then depending on the results, and the files changed, run the appropriate nix, windows and bootstrap tests. While it saves only a few minutes by itself, this makes it easier to refactor checks to subset tests without having to replicate tests or other workflow components in the future. Co-authored-by: Massimiliano Culpo --- .github/workflows/bootstrap.yml | 104 ++++++++++++++++++++-------------------- 1 file changed, 53 insertions(+), 51 deletions(-) (limited to '.github/workflows/bootstrap.yml') diff --git a/.github/workflows/bootstrap.yml b/.github/workflows/bootstrap.yml index 9dd051d0ae..138a9c7382 100644 --- a/.github/workflows/bootstrap.yml +++ b/.github/workflows/bootstrap.yml @@ -3,33 +3,19 @@ name: Bootstrapping on: # This Workflow can be triggered manually workflow_dispatch: - pull_request: - branches: - - develop - - releases/** - paths-ignore: - # Don't run if we only modified packages in the - # built-in repository or documentation - - 'var/spack/repos/builtin/**' - - '!var/spack/repos/builtin/packages/clingo-bootstrap/**' - - '!var/spack/repos/builtin/packages/clingo/**' - - '!var/spack/repos/builtin/packages/python/**' - - '!var/spack/repos/builtin/packages/re2c/**' - - 'lib/spack/docs/**' + workflow_call: schedule: # nightly at 2:16 AM - cron: '16 2 * * *' concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_number }} + group: bootstrap-${{ github.workflow }}-${{ github.event.pull_request.number || github.run_number }} cancel-in-progress: true jobs: - fedora-clingo-sources: runs-on: ubuntu-latest container: "fedora:latest" - if: github.repository == 'spack/spack' steps: - name: Install dependencies run: | @@ -39,6 +25,8 @@ jobs: cmake bison bison-devel libstdc++-static - name: Checkout uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b + with: + fetch-depth: 0 - name: Setup non-root user run: | # See [1] below @@ -49,7 +37,6 @@ jobs: shell: runuser -u spack-test -- bash {0} run: | git --version - git fetch --unshallow . .github/workflows/setup_git.sh - name: Bootstrap clingo shell: runuser -u spack-test -- bash {0} @@ -63,7 +50,6 @@ jobs: ubuntu-clingo-sources: runs-on: ubuntu-latest container: "ubuntu:latest" - if: github.repository == 'spack/spack' steps: - name: Install dependencies env: @@ -76,6 +62,8 @@ jobs: cmake bison - name: Checkout uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b + with: + fetch-depth: 0 - name: Setup non-root user run: | # See [1] below @@ -86,7 +74,6 @@ jobs: shell: runuser -u spack-test -- bash {0} run: | git --version - git fetch --unshallow . .github/workflows/setup_git.sh - name: Bootstrap clingo shell: runuser -u spack-test -- bash {0} @@ -100,7 +87,6 @@ jobs: ubuntu-clingo-binaries-and-patchelf: runs-on: ubuntu-latest container: "ubuntu:latest" - if: github.repository == 'spack/spack' steps: - name: Install dependencies env: @@ -112,6 +98,8 @@ jobs: make patch unzip xz-utils python3 python3-dev tree - name: Checkout uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b + with: + fetch-depth: 0 - name: Setup non-root user run: | # See [1] below @@ -122,7 +110,6 @@ jobs: shell: runuser -u spack-test -- bash {0} run: | git --version - git fetch --unshallow . .github/workflows/setup_git.sh - name: Bootstrap clingo shell: runuser -u spack-test -- bash {0} @@ -134,7 +121,6 @@ jobs: opensuse-clingo-sources: runs-on: ubuntu-latest container: "opensuse/leap:latest" - if: github.repository == 'spack/spack' steps: - name: Install dependencies run: | @@ -146,12 +132,13 @@ jobs: cmake bison - name: Checkout uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b + with: + fetch-depth: 0 - name: Setup repo run: | # See [1] below git config --global --add safe.directory /__w/spack/spack git --version - git fetch --unshallow . .github/workflows/setup_git.sh - name: Bootstrap clingo run: | @@ -163,7 +150,6 @@ jobs: macos-clingo-sources: runs-on: macos-latest - if: github.repository == 'spack/spack' steps: - name: Install dependencies run: | @@ -183,53 +169,70 @@ jobs: runs-on: ${{ matrix.macos-version }} strategy: matrix: - python-version: ['3.6', '3.7', '3.8', '3.9', '3.10'] macos-version: ['macos-11', 'macos-12'] - if: github.repository == 'spack/spack' steps: - name: Install dependencies run: | brew install tree - name: Checkout uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b - - uses: actions/setup-python@b55428b1882923874294fa556849718a1d7f2ca5 - with: - python-version: ${{ matrix.python-version }} - name: Bootstrap clingo run: | - source share/spack/setup-env.sh - spack bootstrap untrust spack-install - spack -d solve zlib - tree ~/.spack/bootstrap/store/ + set -ex + for ver in '3.6' '3.7' '3.8' '3.9' '3.10' ; do + not_found=1 + ver_dir="$(find $RUNNER_TOOL_CACHE/Python -wholename "*/${ver}.*/*/bin" | grep . || true)" + echo "Testing $ver_dir" + if [[ -d "$ver_dir" ]] ; then + if $ver_dir/python --version ; then + export PYTHON="$ver_dir/python" + not_found=0 + old_path="$PATH" + export PATH="$ver_dir:$PATH" + ./bin/spack-tmpconfig -b ./.github/workflows/bootstrap-test.sh + export PATH="$old_path" + fi + fi + # NOTE: test all pythons that exist, not all do on 12 + done ubuntu-clingo-binaries: - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ['2.7', '3.6', '3.7', '3.8', '3.9', '3.10'] - if: github.repository == 'spack/spack' + runs-on: ubuntu-20.04 steps: - name: Checkout uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b - - uses: actions/setup-python@b55428b1882923874294fa556849718a1d7f2ca5 with: - python-version: ${{ matrix.python-version }} + fetch-depth: 0 - name: Setup repo run: | git --version - git fetch --unshallow . .github/workflows/setup_git.sh - name: Bootstrap clingo run: | - source share/spack/setup-env.sh - spack bootstrap untrust spack-install - spack -d solve zlib - tree ~/.spack/bootstrap/store/ + set -ex + for ver in '2.7' '3.6' '3.7' '3.8' '3.9' '3.10' ; do + not_found=1 + ver_dir="$(find $RUNNER_TOOL_CACHE/Python -wholename "*/${ver}.*/*/bin" | grep . || true)" + echo "Testing $ver_dir" + if [[ -d "$ver_dir" ]] ; then + if $ver_dir/python --version ; then + export PYTHON="$ver_dir/python" + not_found=0 + old_path="$PATH" + export PATH="$ver_dir:$PATH" + ./bin/spack-tmpconfig -b ./.github/workflows/bootstrap-test.sh + export PATH="$old_path" + fi + fi + if (($not_found)) ; then + echo Required python version $ver not found in runner! + exit 1 + fi + done ubuntu-gnupg-binaries: runs-on: ubuntu-latest container: "ubuntu:latest" - if: github.repository == 'spack/spack' steps: - name: Install dependencies env: @@ -241,6 +244,8 @@ jobs: make patch unzip xz-utils python3 python3-dev tree - name: Checkout uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b + with: + fetch-depth: 0 - name: Setup non-root user run: | # See [1] below @@ -251,7 +256,6 @@ jobs: shell: runuser -u spack-test -- bash {0} run: | git --version - git fetch --unshallow . .github/workflows/setup_git.sh - name: Bootstrap GnuPG shell: runuser -u spack-test -- bash {0} @@ -264,7 +268,6 @@ jobs: ubuntu-gnupg-sources: runs-on: ubuntu-latest container: "ubuntu:latest" - if: github.repository == 'spack/spack' steps: - name: Install dependencies env: @@ -277,6 +280,8 @@ jobs: gawk - name: Checkout uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b + with: + fetch-depth: 0 - name: Setup non-root user run: | # See [1] below @@ -287,7 +292,6 @@ jobs: shell: runuser -u spack-test -- bash {0} run: | git --version - git fetch --unshallow . .github/workflows/setup_git.sh - name: Bootstrap GnuPG shell: runuser -u spack-test -- bash {0} @@ -300,7 +304,6 @@ jobs: macos-gnupg-binaries: runs-on: macos-latest - if: github.repository == 'spack/spack' steps: - name: Install dependencies run: | @@ -318,7 +321,6 @@ jobs: macos-gnupg-sources: runs-on: macos-latest - if: github.repository == 'spack/spack' steps: - name: Install dependencies run: | -- cgit v1.2.3-70-g09d2