summaryrefslogtreecommitdiff
path: root/.github/workflows
diff options
context:
space:
mode:
authorTodd Gamblin <tgamblin@llnl.gov>2024-09-16 19:18:21 -0700
committerGitHub <noreply@github.com>2024-09-16 19:18:21 -0700
commit930e71177144c7f7c1b7ff6013a298db8a5eafad (patch)
tree0646e6c361a27ca39222e443ec526fd76cb00ff8 /.github/workflows
parentb1905186a621d8f2b2afe13921d30fd056b19aeb (diff)
downloadspack-930e71177144c7f7c1b7ff6013a298db8a5eafad.tar.gz
spack-930e71177144c7f7c1b7ff6013a298db8a5eafad.tar.bz2
spack-930e71177144c7f7c1b7ff6013a298db8a5eafad.tar.xz
spack-930e71177144c7f7c1b7ff6013a298db8a5eafad.zip
coverage: only upload to codecov once (#46385)
Historically, every PR, push, etc. to Spack generates a bunch of jobs, each of which uploads its coverage report to codecov independently. This means that we get annoying partial coverage numbers when only a few of the jobs have finished, and frequently codecov is bad at understanding when to merge reports for a given PR. The numbers of the site can be weird as a result. This restructures our coverage handling so that we do all the merging ourselves and upload exactly one report per GitHub actions workflow. In practice, that means that every push to every PR will get exactly one coverage report and exactly one coverage number reported. I think this will at least partially restore peoples' faith in what codecov is telling them, and it might even make codecov handle Spack a bit better, since this reduces the report burden by ~7x. - [x] test and audit jobs now upload artifacts for coverage - [x] add a new job that downloads artifacts and merges coverage reports together - [x] set `paths` section of `pyproject.toml` so that cross-platform clone locations are merged - [x] upload to codecov once, at the end of the workflow Signed-off-by: Todd Gamblin <tgamblin@llnl.gov>
Diffstat (limited to '.github/workflows')
-rw-r--r--.github/workflows/audit.yaml17
-rw-r--r--.github/workflows/ci.yaml24
-rw-r--r--.github/workflows/unit_tests.yaml47
3 files changed, 57 insertions, 31 deletions
diff --git a/.github/workflows/audit.yaml b/.github/workflows/audit.yaml
index 69f1781f53..caf0096d57 100644
--- a/.github/workflows/audit.yaml
+++ b/.github/workflows/audit.yaml
@@ -40,6 +40,8 @@ jobs:
run: |
python -m pip install --upgrade pywin32
- name: Package audits (with coverage)
+ env:
+ COVERAGE_FILE: coverage/.coverage-audits-${{ matrix.system.os }}
if: ${{ inputs.with_coverage == 'true' && runner.os != 'Windows' }}
run: |
. share/spack/setup-env.sh
@@ -47,27 +49,26 @@ jobs:
coverage run $(which spack) audit configs
coverage run $(which spack) -d audit externals
coverage combine
- coverage xml
- name: Package audits (without coverage)
if: ${{ inputs.with_coverage == 'false' && runner.os != 'Windows' }}
run: |
- . share/spack/setup-env.sh
+ . share/spack/setup-env.sh
spack -d audit packages
spack -d audit configs
spack -d audit externals
- name: Package audits (without coverage)
if: ${{ runner.os == 'Windows' }}
run: |
- . share/spack/setup-env.sh
+ . share/spack/setup-env.sh
spack -d audit packages
./share/spack/qa/validate_last_exit.ps1
spack -d audit configs
./share/spack/qa/validate_last_exit.ps1
spack -d audit externals
./share/spack/qa/validate_last_exit.ps1
- - uses: codecov/codecov-action@e28ff129e5465c2c0dcc6f003fc735cb6ae0c673
- if: ${{ inputs.with_coverage == 'true' }}
+ - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874
+ if: ${{ inputs.with_coverage == 'true' && runner.os != 'Windows' }}
with:
- flags: unittests,audits
- token: ${{ secrets.CODECOV_TOKEN }}
- verbose: true
+ name: coverage-audits-${{ matrix.system.os }}
+ path: coverage
+ include-hidden-files: true
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 96fb38eb64..20f004cf41 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -84,8 +84,30 @@ jobs:
needs: [ prechecks, changes ]
uses: ./.github/workflows/unit_tests.yaml
secrets: inherit
+ upload-coverage:
+ needs: [ unit-tests, prechecks ]
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
+ with:
+ fetch-depth: 0
+ - name: Download coverage files
+ uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16
+ with:
+ pattern: coverage-*
+ path: coverage
+ merge-multiple: true
+ - run: pip install --upgrade coverage
+ - run: ls -la coverage
+ - run: coverage combine -a coverage/.coverage*
+ - run: coverage xml
+ - name: "Upload coverage"
+ uses: codecov/codecov-action@e28ff129e5465c2c0dcc6f003fc735cb6ae0c673
+ with:
+ token: ${{ secrets.CODECOV_TOKEN }}
+ verbose: true
all:
- needs: [ unit-tests, bootstrap ]
+ needs: [ upload-coverage, bootstrap ]
runs-on: ubuntu-latest
steps:
- name: Success
diff --git a/.github/workflows/unit_tests.yaml b/.github/workflows/unit_tests.yaml
index 719dcdc447..93f7de36d2 100644
--- a/.github/workflows/unit_tests.yaml
+++ b/.github/workflows/unit_tests.yaml
@@ -76,14 +76,15 @@ jobs:
SPACK_PYTHON: python
SPACK_TEST_PARALLEL: 2
COVERAGE: true
+ COVERAGE_FILE: coverage/.coverage-${{ matrix.os }}-python${{ matrix.python-version }}
UNIT_TEST_COVERAGE: ${{ matrix.python-version == '3.11' }}
run: |
share/spack/qa/run-unit-tests
- - uses: codecov/codecov-action@e28ff129e5465c2c0dcc6f003fc735cb6ae0c673
+ - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874
with:
- flags: unittests,linux,${{ matrix.concretizer }}
- token: ${{ secrets.CODECOV_TOKEN }}
- verbose: true
+ name: coverage-${{ matrix.os }}-python${{ matrix.python-version }}
+ path: coverage
+ include-hidden-files: true
# Test shell integration
shell:
runs-on: ubuntu-latest
@@ -112,11 +113,11 @@ jobs:
COVERAGE: true
run: |
share/spack/qa/run-shell-tests
- - uses: codecov/codecov-action@e28ff129e5465c2c0dcc6f003fc735cb6ae0c673
+ - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874
with:
- flags: shelltests,linux
- token: ${{ secrets.CODECOV_TOKEN }}
- verbose: true
+ name: coverage-shell
+ path: coverage
+ include-hidden-files: true
# Test RHEL8 UBI with platform Python. This job is run
# only on PRs modifying core Spack
@@ -170,13 +171,14 @@ jobs:
- name: Run unit tests (full suite with coverage)
env:
COVERAGE: true
+ COVERAGE_FILE: coverage/.coverage-clingo-cffi
run: |
share/spack/qa/run-unit-tests
- - uses: codecov/codecov-action@e28ff129e5465c2c0dcc6f003fc735cb6ae0c673
+ - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874
with:
- flags: unittests,linux,clingo
- token: ${{ secrets.CODECOV_TOKEN }}
- verbose: true
+ name: coverage-clingo-cffi
+ path: coverage
+ include-hidden-files: true
# Run unit tests on MacOS
macos:
runs-on: ${{ matrix.os }}
@@ -201,6 +203,7 @@ jobs:
- name: Run unit tests
env:
SPACK_TEST_PARALLEL: 4
+ COVERAGE_FILE: coverage/.coverage-${{ matrix.os }}-python${{ matrix.python-version }}
run: |
git --version
. .github/workflows/bin/setup_git.sh
@@ -209,11 +212,11 @@ jobs:
$(which spack) solve zlib
common_args=(--dist loadfile --tx '4*popen//python=./bin/spack-tmpconfig python -u ./bin/spack python' -x)
$(which spack) unit-test --verbose --cov --cov-config=pyproject.toml --cov-report=xml:coverage.xml "${common_args[@]}"
- - uses: codecov/codecov-action@e28ff129e5465c2c0dcc6f003fc735cb6ae0c673
+ - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874
with:
- flags: unittests,macos
- token: ${{ secrets.CODECOV_TOKEN }}
- verbose: true
+ name: coverage-${{ matrix.os }}-python${{ matrix.python-version }}
+ path: coverage
+ include-hidden-files: true
# Run unit tests on Windows
windows:
defaults:
@@ -235,13 +238,13 @@ jobs:
run: |
./.github/workflows/bin/setup_git.ps1
- name: Unit Test
+ env:
+ COVERAGE_FILE: coverage/.coverage-windows
run: |
spack unit-test -x --verbose --cov --cov-config=pyproject.toml
./share/spack/qa/validate_last_exit.ps1
- coverage combine -a
- coverage xml
- - uses: codecov/codecov-action@e28ff129e5465c2c0dcc6f003fc735cb6ae0c673
+ - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874
with:
- flags: unittests,windows
- token: ${{ secrets.CODECOV_TOKEN }}
- verbose: true
+ name: coverage-windows
+ path: coverage
+ include-hidden-files: true