From 1edfc82123d2185908feb601be3ac7adf5a6da1d Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Tue, 9 Aug 2016 11:09:17 -0500 Subject: Run flake8 checks on changed uncommitted files --- share/spack/qa/run-flake8 | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) (limited to 'share') diff --git a/share/spack/qa/run-flake8 b/share/spack/qa/run-flake8 index c59bfc9490..2c01800ee9 100755 --- a/share/spack/qa/run-flake8 +++ b/share/spack/qa/run-flake8 @@ -17,11 +17,18 @@ if [[ ! $flake8 ]]; then exit 1 fi -# Check if changed files are flake8 conformant [framework] -changed=$(git diff --name-only --find-renames develop... | grep '.py$') +# Add changed files that have been committed since branching off of develop +changed=($(git diff --name-only --find-renames develop... -- '*.py')) +# Add changed files that have been staged but not yet committed +changed+=($(git diff --name-only --find-renames --cached -- '*.py')) +# Add changed files that are unstaged +changed+=($(git diff --name-only --find-renames -- '*.py')) + +# Ensure that each file in the array is unique +changed=($(printf '%s\n' "${changed[@]}" | sort -u)) # Add approved style exemptions to the changed packages. -for file in $changed; do +for file in "${changed[@]}"; do # Make a backup to restore later cp "$file" "$file.sbak~" @@ -48,14 +55,14 @@ for file in $changed; do done return_code=0 -if [[ $changed ]]; then +if [[ "${changed[@]}" ]]; then echo ======================================================= echo flake8: running flake8 code checks on spack. echo echo Modified files: - echo $changed | perl -pe 's/^/ /;s/ +/\n /g' + echo "${changed[@]}" | perl -pe 's/^/ /;s/ +/\n /g' echo ======================================================= - if flake8 --format pylint $changed; then + if flake8 --format pylint "${changed[@]}"; then echo "Flake8 checks were clean." else echo "Flake8 found errors." @@ -66,7 +73,7 @@ else fi # Restore original package files after modifying them. -for file in $changed; do +for file in "${changed[@]}"; do if [[ -e "${file}.sbak~" ]]; then mv "${file}.sbak~" "${file}" fi -- cgit v1.2.3-70-g09d2 From 09c9786fab620f161e7be1dd2813b3d320724ebd Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Tue, 9 Aug 2016 11:49:37 -0500 Subject: Allow run-flake8 to be run from anywhere --- share/spack/qa/run-flake8 | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) (limited to 'share') diff --git a/share/spack/qa/run-flake8 b/share/spack/qa/run-flake8 index 2c01800ee9..2b758b7051 100755 --- a/share/spack/qa/run-flake8 +++ b/share/spack/qa/run-flake8 @@ -2,11 +2,6 @@ # # This script runs source code style checks on Spack. # -# It should be executed from the top-level directory of the repo, -# e.g.: -# -# share/spack/qa/run-flake8 -# # To run it, you'll need to have the Python flake8 installed locally. # PYTHONPATH=./lib/spack:$PYTHONPATH @@ -17,6 +12,9 @@ if [[ ! $flake8 ]]; then exit 1 fi +# Move to Spack root; allows script to be run from anywhere +cd "$(dirname "$0")/../../.." + # Add changed files that have been committed since branching off of develop changed=($(git diff --name-only --find-renames develop... -- '*.py')) # Add changed files that have been staged but not yet committed -- cgit v1.2.3-70-g09d2 From 87d0a7c3156070316ca28f356ef97c8208236fab Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Tue, 9 Aug 2016 13:40:28 -0500 Subject: Always clean up tmp files, even if killed --- share/spack/qa/run-flake8 | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) (limited to 'share') diff --git a/share/spack/qa/run-flake8 b/share/spack/qa/run-flake8 index 2b758b7051..e41cd0d471 100755 --- a/share/spack/qa/run-flake8 +++ b/share/spack/qa/run-flake8 @@ -25,6 +25,18 @@ changed+=($(git diff --name-only --find-renames -- '*.py')) # Ensure that each file in the array is unique changed=($(printf '%s\n' "${changed[@]}" | sort -u)) +function cleanup { + # Restore original package files after modifying them. + for file in "${changed[@]}"; do + if [[ -e "${file}.sbak~" ]]; then + mv "${file}.sbak~" "${file}" + fi + done +} + +# Cleanup temporary files upon exit or when script is killed +trap cleanup EXIT SIGINT SIGTERM + # Add approved style exemptions to the changed packages. for file in "${changed[@]}"; do # Make a backup to restore later @@ -52,7 +64,6 @@ for file in "${changed[@]}"; do perl -i -pe 's/^(.*(https?|file)\:.*)$/\1 # NOQA: ignore=E501/' $file done -return_code=0 if [[ "${changed[@]}" ]]; then echo ======================================================= echo flake8: running flake8 code checks on spack. @@ -64,17 +75,10 @@ if [[ "${changed[@]}" ]]; then echo "Flake8 checks were clean." else echo "Flake8 found errors." - return_code=1 + exit 1 fi else echo No core framework files modified. fi -# Restore original package files after modifying them. -for file in "${changed[@]}"; do - if [[ -e "${file}.sbak~" ]]; then - mv "${file}.sbak~" "${file}" - fi -done - -exit $return_code +exit 0 -- cgit v1.2.3-70-g09d2 From d39322e278b287f88f062cd0aa8a4edc345dee73 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Sun, 28 Aug 2016 21:35:09 -0500 Subject: Run flake8 checks on new untracked files (#1510) --- share/spack/qa/run-flake8 | 2 ++ 1 file changed, 2 insertions(+) (limited to 'share') diff --git a/share/spack/qa/run-flake8 b/share/spack/qa/run-flake8 index e41cd0d471..ffc82313a5 100755 --- a/share/spack/qa/run-flake8 +++ b/share/spack/qa/run-flake8 @@ -21,6 +21,8 @@ changed=($(git diff --name-only --find-renames develop... -- '*.py')) changed+=($(git diff --name-only --find-renames --cached -- '*.py')) # Add changed files that are unstaged changed+=($(git diff --name-only --find-renames -- '*.py')) +# Add new files that are untracked +changed+=($(git ls-files --exclude-standard --other -- '*.py')) # Ensure that each file in the array is unique changed=($(printf '%s\n' "${changed[@]}" | sort -u)) -- cgit v1.2.3-70-g09d2 From 732c1985efde2ef0ef1e0c57c2e0ef464be6a2e9 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Mon, 22 Aug 2016 10:22:42 -0500 Subject: Overhaul Spack's CI Infrastructure --- .travis.yml | 39 ++++++++++--------- README.md | 2 +- share/spack/qa/run-doc-tests | 4 ++ share/spack/qa/run-flake8 | 86 ----------------------------------------- share/spack/qa/run-flake8-tests | 86 +++++++++++++++++++++++++++++++++++++++++ 5 files changed, 112 insertions(+), 105 deletions(-) create mode 100755 share/spack/qa/run-doc-tests delete mode 100755 share/spack/qa/run-flake8 create mode 100755 share/spack/qa/run-flake8-tests (limited to 'share') diff --git a/.travis.yml b/.travis.yml index b376a33490..f6c6817121 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,27 +1,35 @@ language: python +# Construct build matrix python: - - "2.6" - - "2.7" + - 2.6 + - 2.7 + env: - - TEST_TYPE=unit - - TEST_TYPE=flake8 + - TEST_SUITE=unit + - TEST_SUITE=flake8 + - TEST_SUITE=doc -# Exclude flake8 from python 2.6 matrix: exclude: - - python: "2.6" + - python: 2.6 + # Flake8 no longer supports Python 2.6 env: TEST_TYPE=flake8 # Use new Travis infrastructure (Docker can't sudo yet) sudo: false -# Install coveralls to obtain code coverage +# Cache dependencies +cache: pip + +# Install various dependencies install: - - "pip install coveralls" - - "pip install flake8" + - pip install coveralls + - pip install flake8 + - pip install sphinx + - pip install mercurial -before_install: +before_script: # Need this for the git tests to succeed. - git config --global user.email "spack@example.com" - git config --global user.name "Test User" @@ -29,18 +37,13 @@ before_install: # Need this to be able to compute the list of changed files - git fetch origin develop:develop -script: - # Run unit tests with code coverage plus install libdwarf - - 'if [ "$TEST_TYPE" = "unit" ]; then share/spack/qa/run-unit-tests; fi' - # Run flake8 code style checks. - - 'if [ "$TEST_TYPE" = "flake8" ]; then share/spack/qa/run-flake8; fi' +script: share/spack/qa/run-$TEST_SUITE-tests after_success: - - 'if [ "$TEST_TYPE" = "unit" ] && [ "$TRAVIS_PYTHON_VERSION" = "2.7" ]; then coveralls; fi' + - if [[ $TEST_SUITE == unit && $TRAVIS_PYTHON_VERSION -eq 2.7 ]]; then coveralls; fi notifications: email: - recipients: - - tgamblin@llnl.gov + recipients: tgamblin@llnl.gov on_success: change on_failure: always diff --git a/README.md b/README.md index 27d62951a1..cf6f008ea6 100644 --- a/README.md +++ b/README.md @@ -68,7 +68,7 @@ Before you send a PR, your code should pass the following checks: * Your contribution will need to pass the `spack test` command. Run this before submitting your PR. -* Also run the `share/spack/qa/run-flake8` script to check for PEP8 compliance. +* Also run the `share/spack/qa/run-flake8-tests` script to check for PEP8 compliance. To encourage contributions and readability by a broad audience, Spack uses the [PEP8](https://www.python.org/dev/peps/pep-0008/) coding standard with [a few exceptions](https://github.com/LLNL/spack/blob/develop/.flake8). diff --git a/share/spack/qa/run-doc-tests b/share/spack/qa/run-doc-tests new file mode 100755 index 0000000000..e5b684fe59 --- /dev/null +++ b/share/spack/qa/run-doc-tests @@ -0,0 +1,4 @@ +#!/usr/bin/env bash + +# pass +exit 0 diff --git a/share/spack/qa/run-flake8 b/share/spack/qa/run-flake8 deleted file mode 100755 index ffc82313a5..0000000000 --- a/share/spack/qa/run-flake8 +++ /dev/null @@ -1,86 +0,0 @@ -#!/bin/bash -# -# This script runs source code style checks on Spack. -# -# To run it, you'll need to have the Python flake8 installed locally. -# -PYTHONPATH=./lib/spack:$PYTHONPATH - -flake8="$(which flake8)" -if [[ ! $flake8 ]]; then - echo "ERROR: flake8 is required to run this script." - exit 1 -fi - -# Move to Spack root; allows script to be run from anywhere -cd "$(dirname "$0")/../../.." - -# Add changed files that have been committed since branching off of develop -changed=($(git diff --name-only --find-renames develop... -- '*.py')) -# Add changed files that have been staged but not yet committed -changed+=($(git diff --name-only --find-renames --cached -- '*.py')) -# Add changed files that are unstaged -changed+=($(git diff --name-only --find-renames -- '*.py')) -# Add new files that are untracked -changed+=($(git ls-files --exclude-standard --other -- '*.py')) - -# Ensure that each file in the array is unique -changed=($(printf '%s\n' "${changed[@]}" | sort -u)) - -function cleanup { - # Restore original package files after modifying them. - for file in "${changed[@]}"; do - if [[ -e "${file}.sbak~" ]]; then - mv "${file}.sbak~" "${file}" - fi - done -} - -# Cleanup temporary files upon exit or when script is killed -trap cleanup EXIT SIGINT SIGTERM - -# Add approved style exemptions to the changed packages. -for file in "${changed[@]}"; do - # Make a backup to restore later - cp "$file" "$file.sbak~" - - # - # Exemptions for package.py files - # - if [[ $file = *package.py ]]; then - # Exempt lines with urls and descriptions from overlong line errors. - perl -i -pe 's/^(\s*homepage\s*=.*)$/\1 # NOQA: ignore=E501/' $file - perl -i -pe 's/^(\s*url\s*=.*)$/\1 # NOQA: ignore=E501/' $file - perl -i -pe 's/^(\s*version\(.*\).*)$/\1 # NOQA: ignore=E501/' $file - perl -i -pe 's/^(\s*variant\(.*\).*)$/\1 # NOQA: ignore=E501/' $file - perl -i -pe 's/^(\s*depends_on\(.*\).*)$/\1 # NOQA: ignore=E501/' $file - perl -i -pe 's/^(\s*extends\(.*\).*)$/\1 # NOQA: ignore=E501/' $file - - # Exempt '@when' decorated functions from redefinition errors. - perl -i -pe 's/^(\s*\@when\(.*\).*)$/\1 # NOQA: ignore=F811/' $file - fi - - # - # Exemptions for all files - # - perl -i -pe 's/^(.*(https?|file)\:.*)$/\1 # NOQA: ignore=E501/' $file -done - -if [[ "${changed[@]}" ]]; then - echo ======================================================= - echo flake8: running flake8 code checks on spack. - echo - echo Modified files: - echo "${changed[@]}" | perl -pe 's/^/ /;s/ +/\n /g' - echo ======================================================= - if flake8 --format pylint "${changed[@]}"; then - echo "Flake8 checks were clean." - else - echo "Flake8 found errors." - exit 1 - fi -else - echo No core framework files modified. -fi - -exit 0 diff --git a/share/spack/qa/run-flake8-tests b/share/spack/qa/run-flake8-tests new file mode 100755 index 0000000000..9556b0d250 --- /dev/null +++ b/share/spack/qa/run-flake8-tests @@ -0,0 +1,86 @@ +#!/usr/bin/env bash +# +# This script runs source code style checks on Spack. +# +# To run it, you'll need to have the Python flake8 installed locally. +# +PYTHONPATH=./lib/spack:$PYTHONPATH + +flake8="$(which flake8)" +if [[ ! $flake8 ]]; then + echo "ERROR: flake8 is required to run this script." + exit 1 +fi + +# Move to Spack root; allows script to be run from anywhere +cd "$(dirname "$0")/../../.." + +# Add changed files that have been committed since branching off of develop +changed=($(git diff --name-only --find-renames develop... -- '*.py')) +# Add changed files that have been staged but not yet committed +changed+=($(git diff --name-only --find-renames --cached -- '*.py')) +# Add changed files that are unstaged +changed+=($(git diff --name-only --find-renames -- '*.py')) +# Add new files that are untracked +changed+=($(git ls-files --exclude-standard --other -- '*.py')) + +# Ensure that each file in the array is unique +changed=($(printf '%s\n' "${changed[@]}" | sort -u)) + +function cleanup { + # Restore original package files after modifying them. + for file in "${changed[@]}"; do + if [[ -e "${file}.sbak~" ]]; then + mv "${file}.sbak~" "${file}" + fi + done +} + +# Cleanup temporary files upon exit or when script is killed +trap cleanup EXIT SIGINT SIGTERM + +# Add approved style exemptions to the changed packages. +for file in "${changed[@]}"; do + # Make a backup to restore later + cp "$file" "$file.sbak~" + + # + # Exemptions for package.py files + # + if [[ $file = *package.py ]]; then + # Exempt lines with urls and descriptions from overlong line errors. + perl -i -pe 's/^(\s*homepage\s*=.*)$/\1 # NOQA: ignore=E501/' $file + perl -i -pe 's/^(\s*url\s*=.*)$/\1 # NOQA: ignore=E501/' $file + perl -i -pe 's/^(\s*version\(.*\).*)$/\1 # NOQA: ignore=E501/' $file + perl -i -pe 's/^(\s*variant\(.*\).*)$/\1 # NOQA: ignore=E501/' $file + perl -i -pe 's/^(\s*depends_on\(.*\).*)$/\1 # NOQA: ignore=E501/' $file + perl -i -pe 's/^(\s*extends\(.*\).*)$/\1 # NOQA: ignore=E501/' $file + + # Exempt '@when' decorated functions from redefinition errors. + perl -i -pe 's/^(\s*\@when\(.*\).*)$/\1 # NOQA: ignore=F811/' $file + fi + + # + # Exemptions for all files + # + perl -i -pe 's/^(.*(https?|file)\:.*)$/\1 # NOQA: ignore=E501/' $file +done + +if [[ "${changed[@]}" ]]; then + echo ======================================================= + echo flake8: running flake8 code checks on spack. + echo + echo Modified files: + echo "${changed[@]}" | perl -pe 's/^/ /;s/ +/\n /g' + echo ======================================================= + if flake8 --format pylint "${changed[@]}"; then + echo "Flake8 checks were clean." + else + echo "Flake8 found errors." + exit 1 + fi +else + echo No core framework files modified. +fi + +exit 0 -- cgit v1.2.3-70-g09d2 From 679f787a65bf4d8b3aa0c7931da7771bb0b8fb1e Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Mon, 22 Aug 2016 14:35:41 -0500 Subject: Add generic changed_files script --- share/spack/qa/changed_files | 31 +++++++++++++++++++++++++++++++ share/spack/qa/run-flake8-tests | 39 +++++++++++++++++++++++---------------- 2 files changed, 54 insertions(+), 16 deletions(-) create mode 100755 share/spack/qa/changed_files (limited to 'share') diff --git a/share/spack/qa/changed_files b/share/spack/qa/changed_files new file mode 100755 index 0000000000..9c60b3b20b --- /dev/null +++ b/share/spack/qa/changed_files @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# +# Description: +# Returns a list of changed files. +# +# Usage: +# changed_files [ ...] +# changed_files [ ...] +# changed_files ["*." ...] +# +# Options: +# Directories, files, or globs to search for changed files. +# + +# Move to root directory of Spack +# Allows script to be run from anywhere +SPACK_ROOT="$(dirname "$0")/../../.." +cd "$SPACK_ROOT" + +# Add changed files that have been committed since branching off of develop +changed=($(git diff --name-only --find-renames develop... -- "$@")) +# Add changed files that have been staged but not yet committed +changed+=($(git diff --name-only --find-renames --cached -- "$@")) +# Add changed files that are unstaged +changed+=($(git diff --name-only --find-renames -- "$@")) +# Add new files that are untracked +changed+=($(git ls-files --exclude-standard --other -- "$@")) + +# Return array +# Ensure that each file in the array is unique +printf '%s\n' "${changed[@]}" | sort -u diff --git a/share/spack/qa/run-flake8-tests b/share/spack/qa/run-flake8-tests index 9556b0d250..6c9f8968e2 100755 --- a/share/spack/qa/run-flake8-tests +++ b/share/spack/qa/run-flake8-tests @@ -1,31 +1,38 @@ #!/usr/bin/env bash # -# This script runs source code style checks on Spack. +# Description: +# Runs source code style checks on Spack. +# See $SPACK_ROOT/.flake8 for a list of +# approved exceptions. # -# To run it, you'll need to have the Python flake8 installed locally. +# Usage: +# run-flake8-tests +# +# Notes: +# Requires flake8. Can be installed by running: +# `spack install py-flake8` +# or: +# `pip install flake8` +# and adding the bin directory to your PATH. # -PYTHONPATH=./lib/spack:$PYTHONPATH +# Check for dependencies flake8="$(which flake8)" if [[ ! $flake8 ]]; then echo "ERROR: flake8 is required to run this script." exit 1 fi -# Move to Spack root; allows script to be run from anywhere -cd "$(dirname "$0")/../../.." +QA_DIR="$(dirname "$0")" +SPACK_ROOT="$QA_DIR/../../.." -# Add changed files that have been committed since branching off of develop -changed=($(git diff --name-only --find-renames develop... -- '*.py')) -# Add changed files that have been staged but not yet committed -changed+=($(git diff --name-only --find-renames --cached -- '*.py')) -# Add changed files that are unstaged -changed+=($(git diff --name-only --find-renames -- '*.py')) -# Add new files that are untracked -changed+=($(git ls-files --exclude-standard --other -- '*.py')) +# Move to root directory of Spack +# Allows script to be run from anywhere +SPACK_ROOT="$(dirname "$0")/../../.." +cd "$SPACK_ROOT" -# Ensure that each file in the array is unique -changed=($(printf '%s\n' "${changed[@]}" | sort -u)) +# Gather array of changed files +changed=($("$QA_DIR/changed_files" "*.py")) function cleanup { # Restore original package files after modifying them. @@ -80,7 +87,7 @@ if [[ "${changed[@]}" ]]; then exit 1 fi else - echo No core framework files modified. + echo No Python files were modified. fi exit 0 -- cgit v1.2.3-70-g09d2 From d2d6c91b6664352ddc1d41dd8bec3cbe837992de Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Mon, 22 Aug 2016 16:57:53 -0500 Subject: Run documentation tests when documentation is modified --- share/spack/qa/run-doc-tests | 40 ++++++++++++++++++++++++++++++++++++++-- share/spack/qa/run-flake8-tests | 1 - 2 files changed, 38 insertions(+), 3 deletions(-) (limited to 'share') diff --git a/share/spack/qa/run-doc-tests b/share/spack/qa/run-doc-tests index e5b684fe59..2be5793d36 100755 --- a/share/spack/qa/run-doc-tests +++ b/share/spack/qa/run-doc-tests @@ -1,4 +1,40 @@ #!/usr/bin/env bash +# +# Description: +# Builds Spack documentation and checks for +# possible syntax errors. Treats warnings as +# fatal errors. +# +# Usage: +# run-doc-tests +# +# Notes: +# Requires sphinx. Can be installed by running: +# `spack install py-sphinx` +# or: +# `pip install sphinx` +# and adding the bin directory to your PATH. +# + +QA_DIR="$(dirname "$0")" +SPACK_ROOT="$QA_DIR/../../.." +DOC_DIR="$SPACK_ROOT/lib/spack/docs" + +# Move to documentation directory +# Allows script to be run from anywhere +cd "$DOC_DIR" + +# Gather array of changed files +changed=($("$QA_DIR/changed_files" lib/spack/docs)) + +# Cleanup temporary files upon exit or when script is killed +trap 'make clean' EXIT SIGINT SIGTERM + +# Only run tests if documentation was updated +if [[ "${changed[@]}" ]]; then + # Treat warnings as fatal errors + make SPHINXOPTS=-W +else + echo No documentation was modified. +fi -# pass -exit 0 diff --git a/share/spack/qa/run-flake8-tests b/share/spack/qa/run-flake8-tests index 6c9f8968e2..6c7b4f2c79 100755 --- a/share/spack/qa/run-flake8-tests +++ b/share/spack/qa/run-flake8-tests @@ -28,7 +28,6 @@ SPACK_ROOT="$QA_DIR/../../.." # Move to root directory of Spack # Allows script to be run from anywhere -SPACK_ROOT="$(dirname "$0")/../../.." cd "$SPACK_ROOT" # Gather array of changed files -- cgit v1.2.3-70-g09d2 From 1fc14fd7eddedccc6a24b34b42b9c26c96ded0db Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Mon, 22 Aug 2016 18:40:53 -0500 Subject: Only run unit tests when core Spack framework is modified --- share/spack/qa/check_dependencies | 32 +++++++++++++++++++ share/spack/qa/run-doc-tests | 25 +++++++++------ share/spack/qa/run-flake8-tests | 67 +++++++++++++++++++-------------------- share/spack/qa/run-unit-tests | 56 +++++++++++++++++++++++++++----- 4 files changed, 127 insertions(+), 53 deletions(-) create mode 100755 share/spack/qa/check_dependencies (limited to 'share') diff --git a/share/spack/qa/check_dependencies b/share/spack/qa/check_dependencies new file mode 100755 index 0000000000..cbcda0ce0f --- /dev/null +++ b/share/spack/qa/check_dependencies @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# +# Description: +# Check to see if dependencies are installed. +# If not, warn the user and tell them how to +# install these dependencies. +# +# Usage: +# check-deps ... +# +# Options: +# One or more dependencies. Must use name of binary. + +for dep in "$@"; do + if ! which $dep &> /dev/null; then + # sphinx-build comes from sphinx + package=$(echo $dep | cut -d - -f 1) + + cat << EOF +ERROR: $package is required to run this script. + +To install with Spack, run: + $ spack install py-$package +or, to install with pip, run: + $ pip install $package +Then add the bin directory to your PATH. +EOF + exit 1 + fi +done + +echo "Dependencies found." diff --git a/share/spack/qa/run-doc-tests b/share/spack/qa/run-doc-tests index 2be5793d36..bde30f74cc 100755 --- a/share/spack/qa/run-doc-tests +++ b/share/spack/qa/run-doc-tests @@ -9,32 +9,37 @@ # run-doc-tests # # Notes: -# Requires sphinx. Can be installed by running: -# `spack install py-sphinx` -# or: -# `pip install sphinx` -# and adding the bin directory to your PATH. +# Requires sphinx and mercurial. # QA_DIR="$(dirname "$0")" SPACK_ROOT="$QA_DIR/../../.." DOC_DIR="$SPACK_ROOT/lib/spack/docs" -# Move to documentation directory -# Allows script to be run from anywhere -cd "$DOC_DIR" +# Array of dependencies +deps=( + sphinx-build + hg +) + +# Check for dependencies +"$QA_DIR/check_dependencies" "${deps[@]}" || exit 1 # Gather array of changed files changed=($("$QA_DIR/changed_files" lib/spack/docs)) +# Move to documentation directory +# Allows script to be run from anywhere +cd "$DOC_DIR" + # Cleanup temporary files upon exit or when script is killed -trap 'make clean' EXIT SIGINT SIGTERM +trap 'make clean --silent' EXIT SIGINT SIGTERM # Only run tests if documentation was updated if [[ "${changed[@]}" ]]; then # Treat warnings as fatal errors make SPHINXOPTS=-W else - echo No documentation was modified. + echo "No documentation was modified." fi diff --git a/share/spack/qa/run-flake8-tests b/share/spack/qa/run-flake8-tests index 6c7b4f2c79..350ef3161f 100755 --- a/share/spack/qa/run-flake8-tests +++ b/share/spack/qa/run-flake8-tests @@ -9,23 +9,20 @@ # run-flake8-tests # # Notes: -# Requires flake8. Can be installed by running: -# `spack install py-flake8` -# or: -# `pip install flake8` -# and adding the bin directory to your PATH. +# Requires flake8. # -# Check for dependencies -flake8="$(which flake8)" -if [[ ! $flake8 ]]; then - echo "ERROR: flake8 is required to run this script." - exit 1 -fi - QA_DIR="$(dirname "$0")" SPACK_ROOT="$QA_DIR/../../.." +# Array of dependencies +deps=( + flake8 +) + +# Check for dependencies +"$QA_DIR/check_dependencies" "${deps[@]}" || exit 1 + # Move to root directory of Spack # Allows script to be run from anywhere cd "$SPACK_ROOT" @@ -33,6 +30,12 @@ cd "$SPACK_ROOT" # Gather array of changed files changed=($("$QA_DIR/changed_files" "*.py")) +# Exit if no Python files were modified +if [[ ! "${changed[@]}" ]]; then + echo "No Python files were modified." + exit 0 +fi + function cleanup { # Restore original package files after modifying them. for file in "${changed[@]}"; do @@ -55,15 +58,15 @@ for file in "${changed[@]}"; do # if [[ $file = *package.py ]]; then # Exempt lines with urls and descriptions from overlong line errors. - perl -i -pe 's/^(\s*homepage\s*=.*)$/\1 # NOQA: ignore=E501/' $file - perl -i -pe 's/^(\s*url\s*=.*)$/\1 # NOQA: ignore=E501/' $file - perl -i -pe 's/^(\s*version\(.*\).*)$/\1 # NOQA: ignore=E501/' $file - perl -i -pe 's/^(\s*variant\(.*\).*)$/\1 # NOQA: ignore=E501/' $file - perl -i -pe 's/^(\s*depends_on\(.*\).*)$/\1 # NOQA: ignore=E501/' $file - perl -i -pe 's/^(\s*extends\(.*\).*)$/\1 # NOQA: ignore=E501/' $file + perl -i -pe 's/^(\s*homepage\s*=.*)$/\1 # NOQA: ignore=E501/' "$file" + perl -i -pe 's/^(\s*url\s*=.*)$/\1 # NOQA: ignore=E501/' "$file" + perl -i -pe 's/^(\s*version\(.*\).*)$/\1 # NOQA: ignore=E501/' "$file" + perl -i -pe 's/^(\s*variant\(.*\).*)$/\1 # NOQA: ignore=E501/' "$file" + perl -i -pe 's/^(\s*depends_on\(.*\).*)$/\1 # NOQA: ignore=E501/' "$file" + perl -i -pe 's/^(\s*extends\(.*\).*)$/\1 # NOQA: ignore=E501/' "$file" # Exempt '@when' decorated functions from redefinition errors. - perl -i -pe 's/^(\s*\@when\(.*\).*)$/\1 # NOQA: ignore=F811/' $file + perl -i -pe 's/^(\s*\@when\(.*\).*)$/\1 # NOQA: ignore=F811/' "$file" fi # @@ -72,21 +75,15 @@ for file in "${changed[@]}"; do perl -i -pe 's/^(.*(https?|file)\:.*)$/\1 # NOQA: ignore=E501/' $file done -if [[ "${changed[@]}" ]]; then - echo ======================================================= - echo flake8: running flake8 code checks on spack. - echo - echo Modified files: - echo "${changed[@]}" | perl -pe 's/^/ /;s/ +/\n /g' - echo ======================================================= - if flake8 --format pylint "${changed[@]}"; then - echo "Flake8 checks were clean." - else - echo "Flake8 found errors." - exit 1 - fi +echo ======================================================= +echo flake8: running flake8 code checks on spack. +echo +echo Modified files: +echo "${changed[@]}" | perl -pe 's/^/ /;s/ +/\n /g' +echo ======================================================= +if flake8 --format pylint "${changed[@]}"; then + echo "Flake8 checks were clean." else - echo No Python files were modified. + echo "Flake8 found errors." + exit 1 fi - -exit 0 diff --git a/share/spack/qa/run-unit-tests b/share/spack/qa/run-unit-tests index 33fb1bfae2..9ce3062e58 100755 --- a/share/spack/qa/run-unit-tests +++ b/share/spack/qa/run-unit-tests @@ -1,20 +1,60 @@ #!/usr/bin/env bash # -# This script runs Spack unit tests. +# Description: +# Runs Spack unit tests. # -# It should be executed from the top-level directory of the repo, -# e.g.: +# Usage: +# run-unit-tests [test ...] # -# share/spack/qa/run-unit-tests +# Options: +# Optionally add one or more unit tests +# to only run these tests. # -# To run it, you'll need to have the Python coverage installed locally. +# Notes: +# Requires coverage. # -# Regular spack setup and tests -. ./share/spack/setup-env.sh +QA_DIR="$(dirname "$0")" +SPACK_ROOT="$QA_DIR/../../.." + +# Array of dependencies +deps=( + coverage +) + +# Check for dependencies +"$QA_DIR/check_dependencies" "${deps[@]}" || exit 1 + +# Add Spack to the PATH. +export PATH="$SPACK_ROOT/bin:$PATH" + +# Array of directories containing core Spack framework +core_dirs=( + bin + etc + # lib, but skip documentation + lib/spack/env + lib/spack/external + lib/spack/llnl + lib/spack/spack + share +) + +# Gather array of changed files +changed=($("$QA_DIR/changed_files" "${core_dirs[@]}")) + +# Exit if no core Spack framework files were modified +if [[ ! "${changed[@]}" ]]; then + echo "No core Spack framework files were modified." + exit 0 +fi + +# Run integration tests +# TODO: should these be separated into a different test suite? +source "$SPACK_ROOT/share/spack/setup-env.sh" spack compilers spack config get compilers spack install -v libdwarf # Run unit tests with code coverage -coverage run bin/spack test +coverage run spack test "$@" -- cgit v1.2.3-70-g09d2 From 7f9d098c2ff7e25a55bb3a69bd9d7314af1f9abc Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Tue, 23 Aug 2016 11:13:02 -0500 Subject: Add mercurial package, used as test dependency --- share/spack/qa/check_dependencies | 31 ++++++++++++--- share/spack/qa/run-doc-tests | 1 + share/spack/qa/run-unit-tests | 7 +++- .../repos/builtin/packages/py-coverage/package.py | 2 +- .../repos/builtin/packages/py-mercurial/package.py | 44 ++++++++++++++++++++++ 5 files changed, 78 insertions(+), 7 deletions(-) create mode 100644 var/spack/repos/builtin/packages/py-mercurial/package.py (limited to 'share') diff --git a/share/spack/qa/check_dependencies b/share/spack/qa/check_dependencies index cbcda0ce0f..292eac7dd3 100755 --- a/share/spack/qa/check_dependencies +++ b/share/spack/qa/check_dependencies @@ -13,16 +13,37 @@ for dep in "$@"; do if ! which $dep &> /dev/null; then - # sphinx-build comes from sphinx - package=$(echo $dep | cut -d - -f 1) + # Map binary name to package name + case $dep in + sphinx-apidoc|sphinx-build) + spack_package=py-sphinx + pip_package=sphinx + ;; + coverage) + spack_package=py-coverage + pip_package=coverage + ;; + flake8) + spack_package=py-flake8 + pip_package=flake8 + ;; + hg) + spack_package=py-mercurial + pip_package=mercurial + ;; + *) + spack_package=$dep + pip_package=$dep + ;; + esac cat << EOF -ERROR: $package is required to run this script. +ERROR: $dep is required to run this script. To install with Spack, run: - $ spack install py-$package + $ spack install $spack_package or, to install with pip, run: - $ pip install $package + $ pip install $pip_package Then add the bin directory to your PATH. EOF exit 1 diff --git a/share/spack/qa/run-doc-tests b/share/spack/qa/run-doc-tests index bde30f74cc..9feaa3b3e3 100755 --- a/share/spack/qa/run-doc-tests +++ b/share/spack/qa/run-doc-tests @@ -18,6 +18,7 @@ DOC_DIR="$SPACK_ROOT/lib/spack/docs" # Array of dependencies deps=( + sphinx-apidoc sphinx-build hg ) diff --git a/share/spack/qa/run-unit-tests b/share/spack/qa/run-unit-tests index 9ce3062e58..fe2166d2dc 100755 --- a/share/spack/qa/run-unit-tests +++ b/share/spack/qa/run-unit-tests @@ -20,6 +20,7 @@ SPACK_ROOT="$QA_DIR/../../.." # Array of dependencies deps=( coverage + hg ) # Check for dependencies @@ -28,6 +29,10 @@ deps=( # Add Spack to the PATH. export PATH="$SPACK_ROOT/bin:$PATH" +# Move to root directory of Spack +# Allows script to be run from anywhere +cd "$SPACK_ROOT" + # Array of directories containing core Spack framework core_dirs=( bin @@ -57,4 +62,4 @@ spack config get compilers spack install -v libdwarf # Run unit tests with code coverage -coverage run spack test "$@" +coverage run bin/spack test "$@" diff --git a/var/spack/repos/builtin/packages/py-coverage/package.py b/var/spack/repos/builtin/packages/py-coverage/package.py index c2a698b0bd..1a5b6df3d7 100644 --- a/var/spack/repos/builtin/packages/py-coverage/package.py +++ b/var/spack/repos/builtin/packages/py-coverage/package.py @@ -27,7 +27,7 @@ from spack import * class PyCoverage(Package): """ Testing coverage checker for python """ - # FIXME: add a proper url for your package's homepage here. + homepage = "http://nedbatchelder.com/code/coverage/" url = "https://pypi.python.org/packages/source/c/coverage/coverage-4.0a6.tar.gz" diff --git a/var/spack/repos/builtin/packages/py-mercurial/package.py b/var/spack/repos/builtin/packages/py-mercurial/package.py new file mode 100644 index 0000000000..b1b9ffa1b1 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-mercurial/package.py @@ -0,0 +1,44 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class PyMercurial(Package): + """Mercurial is a free, distributed source control management tool. + It efficiently handles projects of any size and offers an easy and + intuitive interface.""" + + homepage = "https://www.mercurial-scm.org/" + url = "https://pypi.python.org/packages/source/m/mercurial/mercurial-3.9.tar.gz" + + version('3.9', 'e2b355da744e94747daae3a5339d28a0', + url="https://pypi.python.org/packages/22/73/e8ef24d3cb13e4fa2695417e13fd22effa1c8e28465eea91a9f84aa922cd/mercurial-3.9.tar.gz") + + extends('python') + + depends_on('py-setuptools', type='build') + + def install(self, spec, prefix): + setup_py('install', '--prefix={0}'.format(prefix)) -- cgit v1.2.3-70-g09d2 From c36f13e44dae52086ca48e4f16b514642bc91106 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Tue, 23 Aug 2016 15:45:04 -0500 Subject: Added more missing dependencies --- share/spack/qa/check_dependencies | 30 ++++++++++++++++++++++-------- share/spack/qa/run-doc-tests | 4 +++- share/spack/qa/run-unit-tests | 4 +++- 3 files changed, 28 insertions(+), 10 deletions(-) (limited to 'share') diff --git a/share/spack/qa/check_dependencies b/share/spack/qa/check_dependencies index 292eac7dd3..ba9ede4ae0 100755 --- a/share/spack/qa/check_dependencies +++ b/share/spack/qa/check_dependencies @@ -27,25 +27,39 @@ for dep in "$@"; do spack_package=py-flake8 pip_package=flake8 ;; + git) + spack_package=git + ;; hg) spack_package=py-mercurial pip_package=mercurial ;; + svn) + spack_package=subversion + ;; *) spack_package=$dep pip_package=$dep ;; esac - cat << EOF -ERROR: $dep is required to run this script. + echo "ERROR: $dep is required to run this script." + echo + + if [[ $spack_package ]]; then + echo "To install with Spack, run:" + echo " $ spack install $spack_package" + fi + + if [[ $pip_package ]]; then + echo "To install with pip, run:" + echo " $ pip install $pip_package" + fi + + if [[ $spack_package || $pip_package ]]; then + echo "Then add the bin directory to your PATH." + fi -To install with Spack, run: - $ spack install $spack_package -or, to install with pip, run: - $ pip install $pip_package -Then add the bin directory to your PATH. -EOF exit 1 fi done diff --git a/share/spack/qa/run-doc-tests b/share/spack/qa/run-doc-tests index 9feaa3b3e3..102eb72d5e 100755 --- a/share/spack/qa/run-doc-tests +++ b/share/spack/qa/run-doc-tests @@ -9,7 +9,7 @@ # run-doc-tests # # Notes: -# Requires sphinx and mercurial. +# Requires sphinx, git, mercurial, and subversion. # QA_DIR="$(dirname "$0")" @@ -20,7 +20,9 @@ DOC_DIR="$SPACK_ROOT/lib/spack/docs" deps=( sphinx-apidoc sphinx-build + git hg + svn ) # Check for dependencies diff --git a/share/spack/qa/run-unit-tests b/share/spack/qa/run-unit-tests index fe2166d2dc..03dfe7cea1 100755 --- a/share/spack/qa/run-unit-tests +++ b/share/spack/qa/run-unit-tests @@ -11,7 +11,7 @@ # to only run these tests. # # Notes: -# Requires coverage. +# Requires coverage, git, mercurial, and subversion. # QA_DIR="$(dirname "$0")" @@ -20,7 +20,9 @@ SPACK_ROOT="$QA_DIR/../../.." # Array of dependencies deps=( coverage + git hg + svn ) # Check for dependencies -- cgit v1.2.3-70-g09d2 From f856952728811c267669962573aea15b9223c1e9 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Wed, 24 Aug 2016 16:56:59 -0500 Subject: Always run Documentation tests --- share/spack/qa/run-doc-tests | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) (limited to 'share') diff --git a/share/spack/qa/run-doc-tests b/share/spack/qa/run-doc-tests index 102eb72d5e..0b23bd9d3d 100755 --- a/share/spack/qa/run-doc-tests +++ b/share/spack/qa/run-doc-tests @@ -28,9 +28,6 @@ deps=( # Check for dependencies "$QA_DIR/check_dependencies" "${deps[@]}" || exit 1 -# Gather array of changed files -changed=($("$QA_DIR/changed_files" lib/spack/docs)) - # Move to documentation directory # Allows script to be run from anywhere cd "$DOC_DIR" @@ -38,11 +35,6 @@ cd "$DOC_DIR" # Cleanup temporary files upon exit or when script is killed trap 'make clean --silent' EXIT SIGINT SIGTERM -# Only run tests if documentation was updated -if [[ "${changed[@]}" ]]; then - # Treat warnings as fatal errors - make SPHINXOPTS=-W -else - echo "No documentation was modified." -fi +# Treat warnings as fatal errors +make SPHINXOPTS=-W -- cgit v1.2.3-70-g09d2 From a0c8aca3a2f50ddd6f39c2382afe8021ba14633b Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Mon, 29 Aug 2016 09:36:54 -0500 Subject: Remove py-mercurial, going with mercurial package instead --- share/spack/qa/check_dependencies | 2 +- .../repos/builtin/packages/py-mercurial/package.py | 44 ---------------------- 2 files changed, 1 insertion(+), 45 deletions(-) delete mode 100644 var/spack/repos/builtin/packages/py-mercurial/package.py (limited to 'share') diff --git a/share/spack/qa/check_dependencies b/share/spack/qa/check_dependencies index ba9ede4ae0..08fad9cdc9 100755 --- a/share/spack/qa/check_dependencies +++ b/share/spack/qa/check_dependencies @@ -31,7 +31,7 @@ for dep in "$@"; do spack_package=git ;; hg) - spack_package=py-mercurial + spack_package=mercurial pip_package=mercurial ;; svn) diff --git a/var/spack/repos/builtin/packages/py-mercurial/package.py b/var/spack/repos/builtin/packages/py-mercurial/package.py deleted file mode 100644 index b1b9ffa1b1..0000000000 --- a/var/spack/repos/builtin/packages/py-mercurial/package.py +++ /dev/null @@ -1,44 +0,0 @@ -############################################################################## -# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. -# Produced at the Lawrence Livermore National Laboratory. -# -# This file is part of Spack. -# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. -# LLNL-CODE-647188 -# -# For details, see https://github.com/llnl/spack -# Please also see the LICENSE file for our notice and the LGPL. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU Lesser General Public License (as -# published by the Free Software Foundation) version 2.1, February 1999. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and -# conditions of the GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -############################################################################## -from spack import * - - -class PyMercurial(Package): - """Mercurial is a free, distributed source control management tool. - It efficiently handles projects of any size and offers an easy and - intuitive interface.""" - - homepage = "https://www.mercurial-scm.org/" - url = "https://pypi.python.org/packages/source/m/mercurial/mercurial-3.9.tar.gz" - - version('3.9', 'e2b355da744e94747daae3a5339d28a0', - url="https://pypi.python.org/packages/22/73/e8ef24d3cb13e4fa2695417e13fd22effa1c8e28465eea91a9f84aa922cd/mercurial-3.9.tar.gz") - - extends('python') - - depends_on('py-setuptools', type='build') - - def install(self, spec, prefix): - setup_py('install', '--prefix={0}'.format(prefix)) -- cgit v1.2.3-70-g09d2 From e04b76c2c3ee397df16d77a768496e007458c3da Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Mon, 29 Aug 2016 11:10:06 -0500 Subject: Add spack to the PATH for doc tests --- share/spack/qa/run-doc-tests | 3 +++ 1 file changed, 3 insertions(+) (limited to 'share') diff --git a/share/spack/qa/run-doc-tests b/share/spack/qa/run-doc-tests index 0b23bd9d3d..96b76a216e 100755 --- a/share/spack/qa/run-doc-tests +++ b/share/spack/qa/run-doc-tests @@ -28,6 +28,9 @@ deps=( # Check for dependencies "$QA_DIR/check_dependencies" "${deps[@]}" || exit 1 +# Add Spack to the PATH. +export PATH="$SPACK_ROOT/bin:$PATH" + # Move to documentation directory # Allows script to be run from anywhere cd "$DOC_DIR" -- cgit v1.2.3-70-g09d2 From f6d07b54f15941ba25ed10ff9465df358a59d625 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Tue, 30 Aug 2016 16:01:00 -0500 Subject: Always run spack unit tests --- share/spack/qa/run-unit-tests | 21 --------------------- 1 file changed, 21 deletions(-) (limited to 'share') diff --git a/share/spack/qa/run-unit-tests b/share/spack/qa/run-unit-tests index 03dfe7cea1..6da919e18d 100755 --- a/share/spack/qa/run-unit-tests +++ b/share/spack/qa/run-unit-tests @@ -35,27 +35,6 @@ export PATH="$SPACK_ROOT/bin:$PATH" # Allows script to be run from anywhere cd "$SPACK_ROOT" -# Array of directories containing core Spack framework -core_dirs=( - bin - etc - # lib, but skip documentation - lib/spack/env - lib/spack/external - lib/spack/llnl - lib/spack/spack - share -) - -# Gather array of changed files -changed=($("$QA_DIR/changed_files" "${core_dirs[@]}")) - -# Exit if no core Spack framework files were modified -if [[ ! "${changed[@]}" ]]; then - echo "No core Spack framework files were modified." - exit 0 -fi - # Run integration tests # TODO: should these be separated into a different test suite? source "$SPACK_ROOT/share/spack/setup-env.sh" -- cgit v1.2.3-70-g09d2