diff options
| -rwxr-xr-x | .github/workflows/bootstrap-test.sh | 7 | ||||
| -rw-r--r-- | .github/workflows/bootstrap.yml | 104 | ||||
| -rw-r--r-- | .github/workflows/ci.yaml | 93 | ||||
| -rw-r--r-- | .github/workflows/unit_tests.yaml | 164 | ||||
| -rw-r--r-- | .github/workflows/valid-style.yml | 82 | ||||
| -rw-r--r-- | .github/workflows/windows_python.yml | 64 | ||||
| -rwxr-xr-x | bin/spack-tmpconfig | 96 | ||||
| -rw-r--r-- | lib/spack/spack/test/cmd/env.py | 2 |
8 files changed, 367 insertions, 245 deletions
diff --git a/.github/workflows/bootstrap-test.sh b/.github/workflows/bootstrap-test.sh new file mode 100755 index 0000000000..fc8f93d68b --- /dev/null +++ b/.github/workflows/bootstrap-test.sh @@ -0,0 +1,7 @@ +#!/bin/bash +set -ex +source share/spack/setup-env.sh +$PYTHON bin/spack bootstrap untrust spack-install +$PYTHON bin/spack -d solve zlib +tree $BOOTSTRAP/store +exit 0 diff --git a/.github/workflows/bootstrap.yml b/.github/workflows/bootstrap.yml index 9dd051d0ae..138a9c7382 100644 --- a/.github/workflows/bootstrap.yml +++ b/.github/workflows/bootstrap.yml @@ -3,33 +3,19 @@ name: Bootstrapping on: # This Workflow can be triggered manually workflow_dispatch: - pull_request: - branches: - - develop - - releases/** - paths-ignore: - # Don't run if we only modified packages in the - # built-in repository or documentation - - 'var/spack/repos/builtin/**' - - '!var/spack/repos/builtin/packages/clingo-bootstrap/**' - - '!var/spack/repos/builtin/packages/clingo/**' - - '!var/spack/repos/builtin/packages/python/**' - - '!var/spack/repos/builtin/packages/re2c/**' - - 'lib/spack/docs/**' + workflow_call: schedule: # nightly at 2:16 AM - cron: '16 2 * * *' concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_number }} + group: bootstrap-${{ github.workflow }}-${{ github.event.pull_request.number || github.run_number }} cancel-in-progress: true jobs: - fedora-clingo-sources: runs-on: ubuntu-latest container: "fedora:latest" - if: github.repository == 'spack/spack' steps: - name: Install dependencies run: | @@ -39,6 +25,8 @@ jobs: cmake bison bison-devel libstdc++-static - name: Checkout uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b + with: + fetch-depth: 0 - name: Setup non-root user run: | # See [1] below @@ -49,7 +37,6 @@ jobs: shell: runuser -u spack-test -- bash {0} run: | git --version - git fetch --unshallow . .github/workflows/setup_git.sh - name: Bootstrap clingo shell: runuser -u spack-test -- bash {0} @@ -63,7 +50,6 @@ jobs: ubuntu-clingo-sources: runs-on: ubuntu-latest container: "ubuntu:latest" - if: github.repository == 'spack/spack' steps: - name: Install dependencies env: @@ -76,6 +62,8 @@ jobs: cmake bison - name: Checkout uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b + with: + fetch-depth: 0 - name: Setup non-root user run: | # See [1] below @@ -86,7 +74,6 @@ jobs: shell: runuser -u spack-test -- bash {0} run: | git --version - git fetch --unshallow . .github/workflows/setup_git.sh - name: Bootstrap clingo shell: runuser -u spack-test -- bash {0} @@ -100,7 +87,6 @@ jobs: ubuntu-clingo-binaries-and-patchelf: runs-on: ubuntu-latest container: "ubuntu:latest" - if: github.repository == 'spack/spack' steps: - name: Install dependencies env: @@ -112,6 +98,8 @@ jobs: make patch unzip xz-utils python3 python3-dev tree - name: Checkout uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b + with: + fetch-depth: 0 - name: Setup non-root user run: | # See [1] below @@ -122,7 +110,6 @@ jobs: shell: runuser -u spack-test -- bash {0} run: | git --version - git fetch --unshallow . .github/workflows/setup_git.sh - name: Bootstrap clingo shell: runuser -u spack-test -- bash {0} @@ -134,7 +121,6 @@ jobs: opensuse-clingo-sources: runs-on: ubuntu-latest container: "opensuse/leap:latest" - if: github.repository == 'spack/spack' steps: - name: Install dependencies run: | @@ -146,12 +132,13 @@ jobs: cmake bison - name: Checkout uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b + with: + fetch-depth: 0 - name: Setup repo run: | # See [1] below git config --global --add safe.directory /__w/spack/spack git --version - git fetch --unshallow . .github/workflows/setup_git.sh - name: Bootstrap clingo run: | @@ -163,7 +150,6 @@ jobs: macos-clingo-sources: runs-on: macos-latest - if: github.repository == 'spack/spack' steps: - name: Install dependencies run: | @@ -183,53 +169,70 @@ jobs: runs-on: ${{ matrix.macos-version }} strategy: matrix: - python-version: ['3.6', '3.7', '3.8', '3.9', '3.10'] macos-version: ['macos-11', 'macos-12'] - if: github.repository == 'spack/spack' steps: - name: Install dependencies run: | brew install tree - name: Checkout uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b - - uses: actions/setup-python@b55428b1882923874294fa556849718a1d7f2ca5 - with: - python-version: ${{ matrix.python-version }} - name: Bootstrap clingo run: | - source share/spack/setup-env.sh - spack bootstrap untrust spack-install - spack -d solve zlib - tree ~/.spack/bootstrap/store/ + set -ex + for ver in '3.6' '3.7' '3.8' '3.9' '3.10' ; do + not_found=1 + ver_dir="$(find $RUNNER_TOOL_CACHE/Python -wholename "*/${ver}.*/*/bin" | grep . || true)" + echo "Testing $ver_dir" + if [[ -d "$ver_dir" ]] ; then + if $ver_dir/python --version ; then + export PYTHON="$ver_dir/python" + not_found=0 + old_path="$PATH" + export PATH="$ver_dir:$PATH" + ./bin/spack-tmpconfig -b ./.github/workflows/bootstrap-test.sh + export PATH="$old_path" + fi + fi + # NOTE: test all pythons that exist, not all do on 12 + done ubuntu-clingo-binaries: - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ['2.7', '3.6', '3.7', '3.8', '3.9', '3.10'] - if: github.repository == 'spack/spack' + runs-on: ubuntu-20.04 steps: - name: Checkout uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b - - uses: actions/setup-python@b55428b1882923874294fa556849718a1d7f2ca5 with: - python-version: ${{ matrix.python-version }} + fetch-depth: 0 - name: Setup repo run: | git --version - git fetch --unshallow . .github/workflows/setup_git.sh - name: Bootstrap clingo run: | - source share/spack/setup-env.sh - spack bootstrap untrust spack-install - spack -d solve zlib - tree ~/.spack/bootstrap/store/ + set -ex + for ver in '2.7' '3.6' '3.7' '3.8' '3.9' '3.10' ; do + not_found=1 + ver_dir="$(find $RUNNER_TOOL_CACHE/Python -wholename "*/${ver}.*/*/bin" | grep . || true)" + echo "Testing $ver_dir" + if [[ -d "$ver_dir" ]] ; then + if $ver_dir/python --version ; then + export PYTHON="$ver_dir/python" + not_found=0 + old_path="$PATH" + export PATH="$ver_dir:$PATH" + ./bin/spack-tmpconfig -b ./.github/workflows/bootstrap-test.sh + export PATH="$old_path" + fi + fi + if (($not_found)) ; then + echo Required python version $ver not found in runner! + exit 1 + fi + done ubuntu-gnupg-binaries: runs-on: ubuntu-latest container: "ubuntu:latest" - if: github.repository == 'spack/spack' steps: - name: Install dependencies env: @@ -241,6 +244,8 @@ jobs: make patch unzip xz-utils python3 python3-dev tree - name: Checkout uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b + with: + fetch-depth: 0 - name: Setup non-root user run: | # See [1] below @@ -251,7 +256,6 @@ jobs: shell: runuser -u spack-test -- bash {0} run: | git --version - git fetch --unshallow . .github/workflows/setup_git.sh - name: Bootstrap GnuPG shell: runuser -u spack-test -- bash {0} @@ -264,7 +268,6 @@ jobs: ubuntu-gnupg-sources: runs-on: ubuntu-latest container: "ubuntu:latest" - if: github.repository == 'spack/spack' steps: - name: Install dependencies env: @@ -277,6 +280,8 @@ jobs: gawk - name: Checkout uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b + with: + fetch-depth: 0 - name: Setup non-root user run: | # See [1] below @@ -287,7 +292,6 @@ jobs: shell: runuser -u spack-test -- bash {0} run: | git --version - git fetch --unshallow . .github/workflows/setup_git.sh - name: Bootstrap GnuPG shell: runuser -u spack-test -- bash {0} @@ -300,7 +304,6 @@ jobs: macos-gnupg-binaries: runs-on: macos-latest - if: github.repository == 'spack/spack' steps: - name: Install dependencies run: | @@ -318,7 +321,6 @@ jobs: macos-gnupg-sources: runs-on: macos-latest - if: github.repository == 'spack/spack' steps: - name: Install dependencies run: | diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 0000000000..62f2be080f --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,93 @@ +name: ci + +on: + push: + branches: + - develop + - releases/** + pull_request: + branches: + - develop + - releases/** + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_number }} + cancel-in-progress: true + +jobs: + prechecks: + needs: [ changes ] + uses: ./.github/workflows/valid-style.yml + with: + with_coverage: ${{ needs.changes.outputs.with_coverage }} + # Check which files have been updated by the PR + changes: + runs-on: ubuntu-latest + # Set job outputs to values from filter step + outputs: + bootstrap: ${{ steps.filter.outputs.bootstrap }} + core: ${{ steps.filter.outputs.core }} + packages: ${{ steps.filter.outputs.packages }} + with_coverage: ${{ steps.coverage.outputs.with_coverage }} + steps: + - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # @v2 + if: ${{ github.event_name == 'push' }} + with: + fetch-depth: 0 + # For pull requests it's not necessary to checkout the code + - uses: dorny/paths-filter@b2feaf19c27470162a626bd6fa8438ae5b263721 + id: filter + with: + # See https://github.com/dorny/paths-filter/issues/56 for the syntax used below + # Don't run if we only modified packages in the + # built-in repository or documentation + filters: | + bootstrap: + - '!var/spack/repos/builtin/**' + - 'var/spack/repos/builtin/packages/clingo-bootstrap/**' + - 'var/spack/repos/builtin/packages/clingo/**' + - 'var/spack/repos/builtin/packages/python/**' + - 'var/spack/repos/builtin/packages/re2c/**' + - '!lib/spack/docs/**' + - 'lib/spack/**' + - 'share/spack/**' + - '.github/workflows/bootstrap.yml' + core: + - './!(var/**)/**' + packages: + - 'var/**' + # Some links for easier reference: + # + # "github" context: https://docs.github.com/en/actions/reference/context-and-expression-syntax-for-github-actions#github-context + # job outputs: https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#jobsjob_idoutputs + # setting environment variables from earlier steps: https://docs.github.com/en/actions/reference/workflow-commands-for-github-actions#setting-an-environment-variable + # + - id: coverage + # Run the subsequent jobs with coverage if core has been modified, + # regardless of whether this is a pull request or a push to a branch + run: | + echo Core changes: ${{ steps.filter.outputs.core }} + echo Event name: ${{ github.event_name }} + if [ "${{ steps.filter.outputs.core }}" == "true" ] + then + echo "::set-output name=with_coverage::true" + else + echo "::set-output name=with_coverage::false" + fi + bootstrap: + if: ${{ github.repository == 'spack/spack' && needs.changes.outputs.bootstrap == 'true' }} + needs: [ prechecks, changes ] + uses: ./.github/workflows/bootstrap.yml + unit-tests: + if: ${{ github.repository == 'spack/spack' }} + needs: [ prechecks, changes ] + uses: ./.github/workflows/unit_tests.yaml + with: + core: ${{ needs.changes.outputs.core }} + packages: ${{ needs.changes.outputs.packages }} + with_coverage: ${{ needs.changes.outputs.with_coverage }} + windows: + if: ${{ github.repository == 'spack/spack' }} + needs: [ prechecks ] + uses: ./.github/workflows/windows_python.yml + diff --git a/.github/workflows/unit_tests.yaml b/.github/workflows/unit_tests.yaml index b20d969a43..3856f1039f 100644 --- a/.github/workflows/unit_tests.yaml +++ b/.github/workflows/unit_tests.yaml @@ -1,103 +1,25 @@ -name: linux tests +name: unit tests on: - push: - branches: - - develop - - releases/** - pull_request: - branches: - - develop - - releases/** + workflow_call: + inputs: + core: + required: true + type: string + packages: + required: true + type: string + with_coverage: + required: true + type: string concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_number }} + group: unit_tests-${{ github.workflow }}-${{ github.event.pull_request.number || github.run_number }} cancel-in-progress: true jobs: - # Validate that the code can be run on all the Python versions - # supported by Spack - validate: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # @v2 - - uses: actions/setup-python@b55428b1882923874294fa556849718a1d7f2ca5 # @v2 - with: - python-version: '3.10' - - name: Install Python Packages - run: | - pip install --upgrade pip - pip install --upgrade vermin - - name: vermin (Spack's Core) - run: vermin --backport argparse --violations --backport typing -t=2.7- -t=3.6- -vvv lib/spack/spack/ lib/spack/llnl/ bin/ - - name: vermin (Repositories) - run: vermin --backport argparse --violations --backport typing -t=2.7- -t=3.6- -vvv var/spack/repos - # Run style checks on the files that have been changed - style: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # @v2 - with: - fetch-depth: 0 - - uses: actions/setup-python@b55428b1882923874294fa556849718a1d7f2ca5 # @v2 - with: - python-version: '3.10' - - name: Install Python packages - run: | - pip install --upgrade pip six setuptools types-six - - name: Setup git configuration - run: | - # Need this for the git tests to succeed. - git --version - . .github/workflows/setup_git.sh - - name: Run style tests - run: | - share/spack/qa/run-style-tests - # Check which files have been updated by the PR - changes: - runs-on: ubuntu-latest - # Set job outputs to values from filter step - outputs: - core: ${{ steps.filter.outputs.core }} - packages: ${{ steps.filter.outputs.packages }} - with_coverage: ${{ steps.coverage.outputs.with_coverage }} - steps: - - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # @v2 - if: ${{ github.event_name == 'push' }} - with: - fetch-depth: 0 - # For pull requests it's not necessary to checkout the code - - uses: dorny/paths-filter@b2feaf19c27470162a626bd6fa8438ae5b263721 - id: filter - with: - # See https://github.com/dorny/paths-filter/issues/56 for the syntax used below - filters: | - core: - - './!(var/**)/**' - packages: - - 'var/**' - # Some links for easier reference: - # - # "github" context: https://docs.github.com/en/actions/reference/context-and-expression-syntax-for-github-actions#github-context - # job outputs: https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#jobsjob_idoutputs - # setting environment variables from earlier steps: https://docs.github.com/en/actions/reference/workflow-commands-for-github-actions#setting-an-environment-variable - # - - id: coverage - # Run the subsequent jobs with coverage if core has been modified, - # regardless of whether this is a pull request or a push to a branch - run: | - echo Core changes: ${{ steps.filter.outputs.core }} - echo Event name: ${{ github.event_name }} - if [ "${{ steps.filter.outputs.core }}" == "true" ] - then - echo "::set-output name=with_coverage::true" - else - echo "::set-output name=with_coverage::false" - fi - # Run unit tests with different configurations on linux - unittests: - needs: [ validate, style, changes ] + ubuntu: runs-on: ubuntu-latest strategy: matrix: @@ -148,7 +70,7 @@ jobs: spack bootstrap untrust spack-install spack -v solve zlib - name: Run unit tests (full suite with coverage) - if: ${{ needs.changes.outputs.with_coverage == 'true' }} + if: ${{ inputs.with_coverage == 'true' }} env: SPACK_PYTHON: python COVERAGE: true @@ -158,7 +80,7 @@ jobs: coverage combine coverage xml - name: Run unit tests (reduced suite without coverage) - if: ${{ needs.changes.outputs.with_coverage == 'false' }} + if: ${{ inputs.with_coverage == 'false' }} env: SPACK_PYTHON: python ONLY_PACKAGES: true @@ -166,12 +88,11 @@ jobs: run: | share/spack/qa/run-unit-tests - uses: codecov/codecov-action@81cd2dc8148241f03f5839d295e000b8f761e378 # @v2.1.0 - if: ${{ needs.changes.outputs.with_coverage == 'true' }} + if: ${{ inputs.with_coverage == 'true' }} with: flags: unittests,linux,${{ matrix.concretizer }} # Test shell integration shell: - needs: [ validate, style, changes ] runs-on: ubuntu-latest steps: - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # @v2 @@ -194,26 +115,25 @@ jobs: git --version . .github/workflows/setup_git.sh - name: Run shell tests (without coverage) - if: ${{ needs.changes.outputs.with_coverage == 'false' }} + if: ${{ inputs.with_coverage == 'false' }} run: | share/spack/qa/run-shell-tests - name: Run shell tests (with coverage) - if: ${{ needs.changes.outputs.with_coverage == 'true' }} + if: ${{ inputs.with_coverage == 'true' }} env: COVERAGE: true run: | share/spack/qa/run-shell-tests - uses: codecov/codecov-action@81cd2dc8148241f03f5839d295e000b8f761e378 # @v2.1.0 - if: ${{ needs.changes.outputs.with_coverage == 'true' }} + if: ${{ inputs.with_coverage == 'true' }} with: flags: shelltests,linux # Test RHEL8 UBI with platform Python. This job is run # only on PRs modifying core Spack rhel8-platform-python: - needs: [ validate, style, changes ] runs-on: ubuntu-latest - if: ${{ needs.changes.outputs.with_coverage == 'true' }} + if: ${{ inputs.with_coverage == 'true' }} container: registry.access.redhat.com/ubi8/ubi steps: - name: Install dependencies @@ -237,7 +157,6 @@ jobs: spack unit-test -k 'not cvs and not svn and not hg' -x --verbose # Test for the clingo based solver (using clingo-cffi) clingo-cffi: - needs: [ validate, style, changes ] runs-on: ubuntu-latest steps: - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # @v2 @@ -262,7 +181,7 @@ jobs: git --version . .github/workflows/setup_git.sh - name: Run unit tests (full suite with coverage) - if: ${{ needs.changes.outputs.with_coverage == 'true' }} + if: ${{ inputs.with_coverage == 'true' }} env: COVERAGE: true SPACK_TEST_SOLVER: clingo @@ -271,19 +190,18 @@ jobs: coverage combine coverage xml - name: Run unit tests (reduced suite without coverage) - if: ${{ needs.changes.outputs.with_coverage == 'false' }} + if: ${{ inputs.with_coverage == 'false' }} env: ONLY_PACKAGES: true SPACK_TEST_SOLVER: clingo run: | share/spack/qa/run-unit-tests - uses: codecov/codecov-action@81cd2dc8148241f03f5839d295e000b8f761e378 # @v2.1.0 - if: ${{ needs.changes.outputs.with_coverage == 'true' }} + if: ${{ inputs.with_coverage == 'true' }} with: flags: unittests,linux,clingo # Run unit tests on MacOS - build: - needs: [ validate, style, changes ] + macos: runs-on: macos-latest strategy: matrix: @@ -311,7 +229,7 @@ jobs: . share/spack/setup-env.sh $(which spack) bootstrap untrust spack-install $(which spack) solve zlib - if [ "${{ needs.changes.outputs.with_coverage }}" == "true" ] + if [ "${{ inputs.with_coverage }}" == "true" ] then coverage run $(which spack) unit-test -x coverage combine @@ -324,36 +242,8 @@ jobs: $(which spack) unit-test -x -m "not maybeslow" -k "test_all_virtual_packages_have_default_providers" fi - uses: codecov/codecov-action@81cd2dc8148241f03f5839d295e000b8f761e378 # @v2.1.0 - if: ${{ needs.changes.outputs.with_coverage == 'true' }} + if: ${{ inputs.with_coverage == 'true' }} with: files: ./coverage.xml flags: unittests,macos - # Run audits on all the packages in the built-in repository - package-audits: - needs: [ validate, style, changes ] - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # @v2 - - uses: actions/setup-python@b55428b1882923874294fa556849718a1d7f2ca5 # @v2 - with: - python-version: '3.10' - - name: Install Python packages - run: | - pip install --upgrade pip six setuptools pytest codecov coverage[toml]==6.2 - - name: Package audits (with coverage) - if: ${{ needs.changes.outputs.with_coverage == 'true' }} - run: | - . share/spack/setup-env.sh - coverage run $(which spack) audit packages - coverage combine - coverage xml - - name: Package audits (without coverage) - if: ${{ needs.changes.outputs.with_coverage == 'false' }} - run: | - . share/spack/setup-env.sh - $(which spack) audit packages - - uses: codecov/codecov-action@81cd2dc8148241f03f5839d295e000b8f761e378 # @v2.1.0 - if: ${{ needs.changes.outputs.with_coverage == 'true' }} - with: - flags: unittests,linux,audits diff --git a/.github/workflows/valid-style.yml b/.github/workflows/valid-style.yml new file mode 100644 index 0000000000..73c8dc76f1 --- /dev/null +++ b/.github/workflows/valid-style.yml @@ -0,0 +1,82 @@ +name: style + +on: + workflow_call: + inputs: + with_coverage: + required: true + type: string + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_number }} + cancel-in-progress: true + + +jobs: + # Validate that the code can be run on all the Python versions + # supported by Spack + validate: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # @v2 + - uses: actions/setup-python@b55428b1882923874294fa556849718a1d7f2ca5 # @v2 + with: + python-version: '3.10' + cache: 'pip' + - name: Install Python Packages + run: | + pip install --upgrade pip + pip install --upgrade vermin + - name: vermin (Spack's Core) + run: vermin --backport argparse --violations --backport typing -t=2.7- -t=3.6- -vvv lib/spack/spack/ lib/spack/llnl/ bin/ + - name: vermin (Repositories) + run: vermin --backport argparse --violations --backport typing -t=2.7- -t=3.6- -vvv var/spack/repos + # Run style checks on the files that have been changed + style: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # @v2 + with: + fetch-depth: 0 + - uses: actions/setup-python@b55428b1882923874294fa556849718a1d7f2ca5 # @v2 + with: + python-version: '3.10' + cache: 'pip' + - name: Install Python packages + run: | + python3 -m pip install --upgrade pip six setuptools types-six click==8.0.2 'black==21.12b0' mypy isort clingo flake8 + - name: Setup git configuration + run: | + # Need this for the git tests to succeed. + git --version + . .github/workflows/setup_git.sh + - name: Run style tests + run: | + share/spack/qa/run-style-tests + # Run audits on all the packages in the built-in repository + package-audits: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # @v2 + - uses: actions/setup-python@b55428b1882923874294fa556849718a1d7f2ca5 # @v2 + with: + python-version: '3.10' + - name: Install Python packages + run: | + pip install --upgrade pip six setuptools pytest codecov coverage[toml]==6.2 + - name: Package audits (with coverage) + if: ${{ inputs.with_coverage == 'true' }} + run: | + . share/spack/setup-env.sh + coverage run $(which spack) audit packages + coverage combine + coverage xml + - name: Package audits (without coverage) + if: ${{ inputs.with_coverage == 'false' }} + run: | + . share/spack/setup-env.sh + $(which spack) audit packages + - uses: codecov/codecov-action@81cd2dc8148241f03f5839d295e000b8f761e378 # @v2.1.0 + if: ${{ inputs.with_coverage == 'true' }} + with: + flags: unittests,linux,audits diff --git a/.github/workflows/windows_python.yml b/.github/workflows/windows_python.yml index e45fdb2bc8..5bd009df44 100644 --- a/.github/workflows/windows_python.yml +++ b/.github/workflows/windows_python.yml @@ -1,17 +1,10 @@ -name: windows tests +name: windows on: - push: - branches: - - develop - - releases/** - pull_request: - branches: - - develop - - releases/** + workflow_call: concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_number }} + group: windows-${{ github.workflow }}-${{ github.event.pull_request.number || github.run_number }} cancel-in-progress: true defaults: @@ -19,45 +12,7 @@ defaults: shell: powershell Invoke-Expression -Command ".\share\spack\qa\windows_test_setup.ps1"; {0} jobs: - validate: - runs-on: windows-latest - steps: - - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b - - uses: actions/setup-python@b55428b1882923874294fa556849718a1d7f2ca5 - with: - python-version: 3.9 - - name: Install Python Packages - run: | - python -m pip install --upgrade pip - python -m pip install --upgrade vermin - - name: vermin (Spack's Core) - run: vermin --backport argparse --backport typing -t='2.7-' -t='3.6-' -v spack/lib/spack/spack/ spack/lib/spack/llnl/ spack/bin/ - - name: vermin (Repositories) - run: vermin --backport argparse --backport typing -t='2.7-' -t='3.6-' -v spack/var/spack/repos - # Run style checks on the files that have been changed - style: - runs-on: windows-latest - steps: - - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b - with: - fetch-depth: 0 - - uses: actions/setup-python@b55428b1882923874294fa556849718a1d7f2ca5 - with: - python-version: 3.9 - - name: Install Python packages - run: | - python -m pip install --upgrade pip six setuptools flake8 "isort>=4.3.5" "mypy>=0.800" "click==8.0.4" "black<=21.12b0" pywin32 types-python-dateutil - - name: Create local develop - run: | - .\spack\.github\workflows\setup_git.ps1 - - name: Run style tests - run: | - spack style - - name: Verify license headers - run: | - python spack\bin\spack license verify - unittest: - needs: [ validate, style ] + unit-tests: runs-on: windows-latest steps: - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b @@ -76,8 +31,7 @@ jobs: run: | echo F|xcopy .\spack\share\spack\qa\configuration\windows_config.yaml $env:USERPROFILE\.spack\windows\config.yaml spack unit-test --verbose --ignore=lib/spack/spack/test/cmd - unittest-cmd: - needs: [ validate, style ] + unit-tests-cmd: runs-on: windows-latest steps: - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b @@ -96,8 +50,7 @@ jobs: run: | echo F|xcopy .\spack\share\spack\qa\configuration\windows_config.yaml $env:USERPROFILE\.spack\windows\config.yaml spack unit-test lib/spack/spack/test/cmd --verbose - buildtest: - needs: [ validate, style ] + build-abseil: runs-on: windows-latest steps: - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b @@ -116,8 +69,7 @@ jobs: spack external find cmake spack external find ninja spack install abseil-cpp - generate-installer-test: - needs: [ validate, style ] + make-installer: runs-on: windows-latest steps: - name: Disable Windows Symlinks @@ -153,7 +105,7 @@ jobs: name: Windows Spack Installer path: ${{ env.installer_root}}\pkg\Spack.msi execute-installer: - needs: generate-installer-test + needs: make-installer runs-on: windows-latest defaults: run: diff --git a/bin/spack-tmpconfig b/bin/spack-tmpconfig new file mode 100755 index 0000000000..b9cca15cc0 --- /dev/null +++ b/bin/spack-tmpconfig @@ -0,0 +1,96 @@ +#!/bin/bash +set -euo pipefail +[[ -n "${TMPCONFIG_DEBUG:=}" ]] && set -x +DIR="$(cd -P "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +mkdir -p "${XDG_RUNTIME_DIR:=/tmp}/spack-tests" +export TMPDIR="${XDG_RUNTIME_DIR}" +export TMP_DIR="$(mktemp -d -t spack-test-XXXXX)" +clean_up() { + [[ -n "$TMPCONFIG_DEBUG" ]] && printf "cleaning up: $TMP_DIR\n" + [[ -n "$TMPCONFIG_DEBUG" ]] && tree "$TMP_DIR" + rm -rf "$TMP_DIR" +} +trap clean_up EXIT +trap clean_up ERR + +[[ -n "$TMPCONFIG_DEBUG" ]] && printf "Redirecting TMP_DIR and spack directories to $TMP_DIR\n" + +export BOOTSTRAP="${SPACK_USER_CACHE_PATH:=$HOME/.spack}/bootstrap" +export SPACK_USER_CACHE_PATH="$TMP_DIR/user_cache" +mkdir -p "$SPACK_USER_CACHE_PATH" + +private_bootstrap="$SPACK_USER_CACHE_PATH/bootstrap" +use_spack='' +use_bwrap='' +# argument handling +while (($# >= 1)) ; do + case "$1" in + -b) # privatize bootstrap too, useful for CI but not always cheap + shift + export BOOTSTRAP="$private_bootstrap" + ;; + -B) # use specified bootstrap dir + export BOOTSTRAP="$2" + shift 2 + ;; + -s) # run spack directly with remaining args + shift + use_spack=1 + ;; + --contain=bwrap) + if bwrap --help 2>&1 > /dev/null ; then + use_bwrap=1 + else + echo Bubblewrap containment requested, but no bwrap command found + exit 1 + fi + shift + ;; + --) + shift + break + ;; + *) + break + ;; + esac +done +typeset -a CMD +if [[ -n "$use_spack" ]] ; then + CMD=("$DIR/spack" "$@") +else + CMD=("$@") +fi + +mkdir -p "$BOOTSTRAP" + +export SPACK_SYSTEM_CONFIG_PATH="$TMP_DIR/sys_conf" +export SPACK_USER_CONFIG_PATH="$TMP_DIR/user_conf" +mkdir -p "$SPACK_USER_CONFIG_PATH" +cat >"$SPACK_USER_CONFIG_PATH/config.yaml" <<EOF +config: + install_tree: + root: $TMP_DIR/install + misc_cache: $$user_cache_path/cache + source_cache: $$user_cache_path/source +EOF +cat >"$SPACK_USER_CONFIG_PATH/bootstrap.yaml" <<EOF +bootstrap: + root: $BOOTSTRAP +EOF + +if [[ -n "$use_bwrap" ]] ; then + CMD=( + bwrap + --dev-bind / / + --ro-bind "$DIR/.." "$DIR/.." # do not touch spack root + --ro-bind $HOME/.spack $HOME/.spack # do not touch user config/cache dir + --bind "$TMP_DIR" "$TMP_DIR" + --bind "$BOOTSTRAP" "$BOOTSTRAP" + --die-with-parent + "${CMD[@]}" + ) +fi + +(( ${TMPCONFIG_DEBUG:=0} > 1)) && echo "Running: ${CMD[@]}" +"${CMD[@]}" diff --git a/lib/spack/spack/test/cmd/env.py b/lib/spack/spack/test/cmd/env.py index e465060392..ceefd3323b 100644 --- a/lib/spack/spack/test/cmd/env.py +++ b/lib/spack/spack/test/cmd/env.py @@ -788,7 +788,7 @@ def test_env_with_included_config_file(packages_file): e = ev.read("test") included_path = os.path.join(e.path, include_filename) - fs.rename(packages_file.strpath, included_path) + shutil.move(packages_file.strpath, included_path) with e: e.concretize() |