From 7c5fbee327f7735e1ac5a7cf1a55a292428f0690 Mon Sep 17 00:00:00 2001 From: Alec Scott Date: Tue, 9 Jul 2024 04:46:09 +0200 Subject: Improve organization of CI workflow scripts and pip requirements (#45037) --- .github/dependabot.yml | 3 ++- .github/workflows/bin/bootstrap-test.sh | 8 ++++++++ .github/workflows/bin/execute_installer.ps1 | 7 +++++++ .github/workflows/bin/generate_spack_yaml_containerize.sh | 9 +++++++++ .github/workflows/bin/setup_git.ps1 | 10 ++++++++++ .github/workflows/bin/setup_git.sh | 8 ++++++++ .github/workflows/bin/system_shortcut_check.ps1 | 4 ++++ .github/workflows/bootstrap-test.sh | 8 -------- .github/workflows/bootstrap.yml | 3 +-- .github/workflows/build-containers.yml | 2 +- .github/workflows/execute_installer.ps1 | 7 ------- .github/workflows/generate_spack_yaml_containerize.sh | 9 --------- .github/workflows/install_spack.sh | 8 -------- .github/workflows/requirements/style/requirements.txt | 7 +++++++ .github/workflows/setup_git.ps1 | 10 ---------- .github/workflows/setup_git.sh | 8 -------- .github/workflows/style/requirements.txt | 7 ------- .github/workflows/system_shortcut_check.ps1 | 4 ---- .github/workflows/unit_tests.yaml | 12 ++++++------ .github/workflows/valid-style.yml | 8 ++++---- 20 files changed, 67 insertions(+), 75 deletions(-) create mode 100755 .github/workflows/bin/bootstrap-test.sh create mode 100644 .github/workflows/bin/execute_installer.ps1 create mode 100755 .github/workflows/bin/generate_spack_yaml_containerize.sh create mode 100644 .github/workflows/bin/setup_git.ps1 create mode 100755 .github/workflows/bin/setup_git.sh create mode 100644 .github/workflows/bin/system_shortcut_check.ps1 delete mode 100755 .github/workflows/bootstrap-test.sh delete mode 100644 .github/workflows/execute_installer.ps1 delete mode 100755 .github/workflows/generate_spack_yaml_containerize.sh delete mode 100755 .github/workflows/install_spack.sh create mode 100644 .github/workflows/requirements/style/requirements.txt delete mode 100644 .github/workflows/setup_git.ps1 delete mode 100755 .github/workflows/setup_git.sh delete mode 100644 .github/workflows/style/requirements.txt delete mode 100644 .github/workflows/system_shortcut_check.ps1 diff --git a/.github/dependabot.yml b/.github/dependabot.yml index a5b42ffcf7..b19c9381ec 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -12,6 +12,7 @@ updates: interval: "daily" # Requirements to run style checks - package-ecosystem: "pip" - directory: "/.github/workflows/style" + directories: + - "/.github/workflows/requirements/*" schedule: interval: "daily" diff --git a/.github/workflows/bin/bootstrap-test.sh b/.github/workflows/bin/bootstrap-test.sh new file mode 100755 index 0000000000..563eb28643 --- /dev/null +++ b/.github/workflows/bin/bootstrap-test.sh @@ -0,0 +1,8 @@ +#!/bin/bash +set -e +source share/spack/setup-env.sh +$PYTHON bin/spack bootstrap disable github-actions-v0.4 +$PYTHON bin/spack bootstrap disable spack-install +$PYTHON bin/spack $SPACK_FLAGS solve zlib +tree $BOOTSTRAP/store +exit 0 diff --git a/.github/workflows/bin/execute_installer.ps1 b/.github/workflows/bin/execute_installer.ps1 new file mode 100644 index 0000000000..9d9f5cfbeb --- /dev/null +++ b/.github/workflows/bin/execute_installer.ps1 @@ -0,0 +1,7 @@ +$ proc = Start-Process ${{ env.spack_installer }}\spack.exe "/install /quiet" -Passthru +$handle = $proc.Handle # cache proc.Handle +$proc.WaitForExit(); + +if ($proc.ExitCode -ne 0) { + Write-Warning "$_ exited with status code $($proc.ExitCode)" +} diff --git a/.github/workflows/bin/generate_spack_yaml_containerize.sh b/.github/workflows/bin/generate_spack_yaml_containerize.sh new file mode 100755 index 0000000000..2a180b7dd1 --- /dev/null +++ b/.github/workflows/bin/generate_spack_yaml_containerize.sh @@ -0,0 +1,9 @@ +#!/bin/bash + (echo "spack:" \ +&& echo " specs: []" \ +&& echo " container:" \ +&& echo " format: docker" \ +&& echo " images:" \ +&& echo " os: \"${SPACK_YAML_OS}\"" \ +&& echo " spack:" \ +&& echo " ref: ${GITHUB_REF}") > spack.yaml diff --git a/.github/workflows/bin/setup_git.ps1 b/.github/workflows/bin/setup_git.ps1 new file mode 100644 index 0000000000..836b7f8a2c --- /dev/null +++ b/.github/workflows/bin/setup_git.ps1 @@ -0,0 +1,10 @@ +# (c) 2022 Lawrence Livermore National Laboratory + +git config --global user.email "spack@example.com" +git config --global user.name "Test User" +git config --global core.longpaths true + +if ($(git branch --show-current) -ne "develop") +{ + git branch develop origin/develop +} diff --git a/.github/workflows/bin/setup_git.sh b/.github/workflows/bin/setup_git.sh new file mode 100755 index 0000000000..4eb416720b --- /dev/null +++ b/.github/workflows/bin/setup_git.sh @@ -0,0 +1,8 @@ +#!/bin/bash -e +git config --global user.email "spack@example.com" +git config --global user.name "Test User" + +# create a local pr base branch +if [[ -n $GITHUB_BASE_REF ]]; then + git fetch origin "${GITHUB_BASE_REF}:${GITHUB_BASE_REF}" +fi diff --git a/.github/workflows/bin/system_shortcut_check.ps1 b/.github/workflows/bin/system_shortcut_check.ps1 new file mode 100644 index 0000000000..03b3a236f1 --- /dev/null +++ b/.github/workflows/bin/system_shortcut_check.ps1 @@ -0,0 +1,4 @@ +param ($systemFolder, $shortcut) + +$start = [System.Environment]::GetFolderPath("$systemFolder") +Invoke-Item "$start\Programs\Spack\$shortcut" diff --git a/.github/workflows/bootstrap-test.sh b/.github/workflows/bootstrap-test.sh deleted file mode 100755 index 563eb28643..0000000000 --- a/.github/workflows/bootstrap-test.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash -set -e -source share/spack/setup-env.sh -$PYTHON bin/spack bootstrap disable github-actions-v0.4 -$PYTHON bin/spack bootstrap disable spack-install -$PYTHON bin/spack $SPACK_FLAGS solve zlib -tree $BOOTSTRAP/store -exit 0 diff --git a/.github/workflows/bootstrap.yml b/.github/workflows/bootstrap.yml index 25e74d38bc..4002b1b9ea 100644 --- a/.github/workflows/bootstrap.yml +++ b/.github/workflows/bootstrap.yml @@ -152,7 +152,7 @@ jobs: not_found=0 old_path="$PATH" export PATH="$ver_dir:$PATH" - ./bin/spack-tmpconfig -b ./.github/workflows/bootstrap-test.sh + ./bin/spack-tmpconfig -b ./.github/workflows/bin/bootstrap-test.sh export PATH="$old_path" fi fi @@ -166,4 +166,3 @@ jobs: source share/spack/setup-env.sh spack -d gpg list tree ~/.spack/bootstrap/store/ - diff --git a/.github/workflows/build-containers.yml b/.github/workflows/build-containers.yml index 05b8097809..a3016f5f38 100644 --- a/.github/workflows/build-containers.yml +++ b/.github/workflows/build-containers.yml @@ -76,7 +76,7 @@ jobs: env: SPACK_YAML_OS: "${{ matrix.dockerfile[2] }}" run: | - .github/workflows/generate_spack_yaml_containerize.sh + .github/workflows/bin/generate_spack_yaml_containerize.sh . share/spack/setup-env.sh mkdir -p dockerfiles/${{ matrix.dockerfile[0] }} spack containerize --last-stage=bootstrap | tee dockerfiles/${{ matrix.dockerfile[0] }}/Dockerfile diff --git a/.github/workflows/execute_installer.ps1 b/.github/workflows/execute_installer.ps1 deleted file mode 100644 index 9d9f5cfbeb..0000000000 --- a/.github/workflows/execute_installer.ps1 +++ /dev/null @@ -1,7 +0,0 @@ -$ proc = Start-Process ${{ env.spack_installer }}\spack.exe "/install /quiet" -Passthru -$handle = $proc.Handle # cache proc.Handle -$proc.WaitForExit(); - -if ($proc.ExitCode -ne 0) { - Write-Warning "$_ exited with status code $($proc.ExitCode)" -} diff --git a/.github/workflows/generate_spack_yaml_containerize.sh b/.github/workflows/generate_spack_yaml_containerize.sh deleted file mode 100755 index 2a180b7dd1..0000000000 --- a/.github/workflows/generate_spack_yaml_containerize.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - (echo "spack:" \ -&& echo " specs: []" \ -&& echo " container:" \ -&& echo " format: docker" \ -&& echo " images:" \ -&& echo " os: \"${SPACK_YAML_OS}\"" \ -&& echo " spack:" \ -&& echo " ref: ${GITHUB_REF}") > spack.yaml diff --git a/.github/workflows/install_spack.sh b/.github/workflows/install_spack.sh deleted file mode 100755 index be8ec8af21..0000000000 --- a/.github/workflows/install_spack.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env sh -. share/spack/setup-env.sh -echo -e "config:\n build_jobs: 2" > etc/spack/config.yaml -spack config add "packages:all:target:[x86_64]" -spack compiler find -spack compiler info apple-clang -spack debug report -spack solve zlib diff --git a/.github/workflows/requirements/style/requirements.txt b/.github/workflows/requirements/style/requirements.txt new file mode 100644 index 0000000000..a6ff4bdc84 --- /dev/null +++ b/.github/workflows/requirements/style/requirements.txt @@ -0,0 +1,7 @@ +black==24.4.2 +clingo==5.7.1 +flake8==7.1.0 +isort==5.13.2 +mypy==1.8.0 +types-six==1.16.21.20240513 +vermin==1.6.0 diff --git a/.github/workflows/setup_git.ps1 b/.github/workflows/setup_git.ps1 deleted file mode 100644 index 836b7f8a2c..0000000000 --- a/.github/workflows/setup_git.ps1 +++ /dev/null @@ -1,10 +0,0 @@ -# (c) 2022 Lawrence Livermore National Laboratory - -git config --global user.email "spack@example.com" -git config --global user.name "Test User" -git config --global core.longpaths true - -if ($(git branch --show-current) -ne "develop") -{ - git branch develop origin/develop -} diff --git a/.github/workflows/setup_git.sh b/.github/workflows/setup_git.sh deleted file mode 100755 index 4eb416720b..0000000000 --- a/.github/workflows/setup_git.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash -e -git config --global user.email "spack@example.com" -git config --global user.name "Test User" - -# create a local pr base branch -if [[ -n $GITHUB_BASE_REF ]]; then - git fetch origin "${GITHUB_BASE_REF}:${GITHUB_BASE_REF}" -fi diff --git a/.github/workflows/style/requirements.txt b/.github/workflows/style/requirements.txt deleted file mode 100644 index a6ff4bdc84..0000000000 --- a/.github/workflows/style/requirements.txt +++ /dev/null @@ -1,7 +0,0 @@ -black==24.4.2 -clingo==5.7.1 -flake8==7.1.0 -isort==5.13.2 -mypy==1.8.0 -types-six==1.16.21.20240513 -vermin==1.6.0 diff --git a/.github/workflows/system_shortcut_check.ps1 b/.github/workflows/system_shortcut_check.ps1 deleted file mode 100644 index 03b3a236f1..0000000000 --- a/.github/workflows/system_shortcut_check.ps1 +++ /dev/null @@ -1,4 +0,0 @@ -param ($systemFolder, $shortcut) - -$start = [System.Environment]::GetFolderPath("$systemFolder") -Invoke-Item "$start\Programs\Spack\$shortcut" diff --git a/.github/workflows/unit_tests.yaml b/.github/workflows/unit_tests.yaml index d1b385c9e1..f474b8fcd2 100644 --- a/.github/workflows/unit_tests.yaml +++ b/.github/workflows/unit_tests.yaml @@ -72,7 +72,7 @@ jobs: run: | # Need this for the git tests to succeed. git --version - . .github/workflows/setup_git.sh + . .github/workflows/bin/setup_git.sh - name: Bootstrap clingo if: ${{ matrix.concretizer == 'clingo' }} env: @@ -118,7 +118,7 @@ jobs: run: | # Need this for the git tests to succeed. git --version - . .github/workflows/setup_git.sh + . .github/workflows/bin/setup_git.sh - name: Run shell tests env: COVERAGE: true @@ -147,7 +147,7 @@ jobs: git --version git config --global --add safe.directory /__w/spack/spack git fetch --unshallow - . .github/workflows/setup_git.sh + . .github/workflows/bin/setup_git.sh useradd spack-test chown -R spack-test . - name: Run unit tests @@ -178,7 +178,7 @@ jobs: run: | # Need this for the git tests to succeed. git --version - . .github/workflows/setup_git.sh + . .github/workflows/bin/setup_git.sh - name: Run unit tests (full suite with coverage) env: COVERAGE: true @@ -217,7 +217,7 @@ jobs: SPACK_TEST_PARALLEL: 4 run: | git --version - . .github/workflows/setup_git.sh + . .github/workflows/bin/setup_git.sh . share/spack/setup-env.sh $(which spack) bootstrap disable spack-install $(which spack) solve zlib @@ -247,7 +247,7 @@ jobs: python -m pip install --upgrade pip pywin32 setuptools pytest-cov clingo - name: Create local develop run: | - ./.github/workflows/setup_git.ps1 + ./.github/workflows/bin/setup_git.ps1 - name: Unit Test run: | spack unit-test -x --verbose --cov --cov-config=pyproject.toml diff --git a/.github/workflows/valid-style.yml b/.github/workflows/valid-style.yml index d4c600ac2f..05faab0a8d 100644 --- a/.github/workflows/valid-style.yml +++ b/.github/workflows/valid-style.yml @@ -26,7 +26,7 @@ jobs: - name: Install Python Packages run: | pip install --upgrade pip setuptools - pip install -r .github/workflows/style/requirements.txt + pip install -r .github/workflows/requirements/style/requirements.txt - name: vermin (Spack's Core) run: vermin --backport importlib --backport argparse --violations --backport typing -t=3.6- -vvv lib/spack/spack/ lib/spack/llnl/ bin/ - name: vermin (Repositories) @@ -45,12 +45,12 @@ jobs: - name: Install Python packages run: | pip install --upgrade pip setuptools - pip install -r .github/workflows/style/requirements.txt + pip install -r .github/workflows/requirements/style/requirements.txt - name: Setup git configuration run: | # Need this for the git tests to succeed. git --version - . .github/workflows/setup_git.sh + . .github/workflows/bin/setup_git.sh - name: Run style tests run: | share/spack/qa/run-style-tests @@ -76,7 +76,7 @@ jobs: git --version git config --global --add safe.directory /__w/spack/spack git fetch --unshallow - . .github/workflows/setup_git.sh + . .github/workflows/bin/setup_git.sh useradd spack-test chown -R spack-test . - name: Bootstrap Spack development environment -- cgit v1.2.3-70-g09d2