From 732c1985efde2ef0ef1e0c57c2e0ef464be6a2e9 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Mon, 22 Aug 2016 10:22:42 -0500 Subject: Overhaul Spack's CI Infrastructure --- .travis.yml | 39 ++++++++++--------- README.md | 2 +- share/spack/qa/run-doc-tests | 4 ++ share/spack/qa/run-flake8 | 86 ----------------------------------------- share/spack/qa/run-flake8-tests | 86 +++++++++++++++++++++++++++++++++++++++++ 5 files changed, 112 insertions(+), 105 deletions(-) create mode 100755 share/spack/qa/run-doc-tests delete mode 100755 share/spack/qa/run-flake8 create mode 100755 share/spack/qa/run-flake8-tests diff --git a/.travis.yml b/.travis.yml index b376a33490..f6c6817121 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,27 +1,35 @@ language: python +# Construct build matrix python: - - "2.6" - - "2.7" + - 2.6 + - 2.7 + env: - - TEST_TYPE=unit - - TEST_TYPE=flake8 + - TEST_SUITE=unit + - TEST_SUITE=flake8 + - TEST_SUITE=doc -# Exclude flake8 from python 2.6 matrix: exclude: - - python: "2.6" + - python: 2.6 + # Flake8 no longer supports Python 2.6 env: TEST_TYPE=flake8 # Use new Travis infrastructure (Docker can't sudo yet) sudo: false -# Install coveralls to obtain code coverage +# Cache dependencies +cache: pip + +# Install various dependencies install: - - "pip install coveralls" - - "pip install flake8" + - pip install coveralls + - pip install flake8 + - pip install sphinx + - pip install mercurial -before_install: +before_script: # Need this for the git tests to succeed. - git config --global user.email "spack@example.com" - git config --global user.name "Test User" @@ -29,18 +37,13 @@ before_install: # Need this to be able to compute the list of changed files - git fetch origin develop:develop -script: - # Run unit tests with code coverage plus install libdwarf - - 'if [ "$TEST_TYPE" = "unit" ]; then share/spack/qa/run-unit-tests; fi' - # Run flake8 code style checks. - - 'if [ "$TEST_TYPE" = "flake8" ]; then share/spack/qa/run-flake8; fi' +script: share/spack/qa/run-$TEST_SUITE-tests after_success: - - 'if [ "$TEST_TYPE" = "unit" ] && [ "$TRAVIS_PYTHON_VERSION" = "2.7" ]; then coveralls; fi' + - if [[ $TEST_SUITE == unit && $TRAVIS_PYTHON_VERSION -eq 2.7 ]]; then coveralls; fi notifications: email: - recipients: - - tgamblin@llnl.gov + recipients: tgamblin@llnl.gov on_success: change on_failure: always diff --git a/README.md b/README.md index 27d62951a1..cf6f008ea6 100644 --- a/README.md +++ b/README.md @@ -68,7 +68,7 @@ Before you send a PR, your code should pass the following checks: * Your contribution will need to pass the `spack test` command. Run this before submitting your PR. -* Also run the `share/spack/qa/run-flake8` script to check for PEP8 compliance. +* Also run the `share/spack/qa/run-flake8-tests` script to check for PEP8 compliance. To encourage contributions and readability by a broad audience, Spack uses the [PEP8](https://www.python.org/dev/peps/pep-0008/) coding standard with [a few exceptions](https://github.com/LLNL/spack/blob/develop/.flake8). diff --git a/share/spack/qa/run-doc-tests b/share/spack/qa/run-doc-tests new file mode 100755 index 0000000000..e5b684fe59 --- /dev/null +++ b/share/spack/qa/run-doc-tests @@ -0,0 +1,4 @@ +#!/usr/bin/env bash + +# pass +exit 0 diff --git a/share/spack/qa/run-flake8 b/share/spack/qa/run-flake8 deleted file mode 100755 index ffc82313a5..0000000000 --- a/share/spack/qa/run-flake8 +++ /dev/null @@ -1,86 +0,0 @@ -#!/bin/bash -# -# This script runs source code style checks on Spack. -# -# To run it, you'll need to have the Python flake8 installed locally. -# -PYTHONPATH=./lib/spack:$PYTHONPATH - -flake8="$(which flake8)" -if [[ ! $flake8 ]]; then - echo "ERROR: flake8 is required to run this script." - exit 1 -fi - -# Move to Spack root; allows script to be run from anywhere -cd "$(dirname "$0")/../../.." - -# Add changed files that have been committed since branching off of develop -changed=($(git diff --name-only --find-renames develop... -- '*.py')) -# Add changed files that have been staged but not yet committed -changed+=($(git diff --name-only --find-renames --cached -- '*.py')) -# Add changed files that are unstaged -changed+=($(git diff --name-only --find-renames -- '*.py')) -# Add new files that are untracked -changed+=($(git ls-files --exclude-standard --other -- '*.py')) - -# Ensure that each file in the array is unique -changed=($(printf '%s\n' "${changed[@]}" | sort -u)) - -function cleanup { - # Restore original package files after modifying them. - for file in "${changed[@]}"; do - if [[ -e "${file}.sbak~" ]]; then - mv "${file}.sbak~" "${file}" - fi - done -} - -# Cleanup temporary files upon exit or when script is killed -trap cleanup EXIT SIGINT SIGTERM - -# Add approved style exemptions to the changed packages. -for file in "${changed[@]}"; do - # Make a backup to restore later - cp "$file" "$file.sbak~" - - # - # Exemptions for package.py files - # - if [[ $file = *package.py ]]; then - # Exempt lines with urls and descriptions from overlong line errors. - perl -i -pe 's/^(\s*homepage\s*=.*)$/\1 # NOQA: ignore=E501/' $file - perl -i -pe 's/^(\s*url\s*=.*)$/\1 # NOQA: ignore=E501/' $file - perl -i -pe 's/^(\s*version\(.*\).*)$/\1 # NOQA: ignore=E501/' $file - perl -i -pe 's/^(\s*variant\(.*\).*)$/\1 # NOQA: ignore=E501/' $file - perl -i -pe 's/^(\s*depends_on\(.*\).*)$/\1 # NOQA: ignore=E501/' $file - perl -i -pe 's/^(\s*extends\(.*\).*)$/\1 # NOQA: ignore=E501/' $file - - # Exempt '@when' decorated functions from redefinition errors. - perl -i -pe 's/^(\s*\@when\(.*\).*)$/\1 # NOQA: ignore=F811/' $file - fi - - # - # Exemptions for all files - # - perl -i -pe 's/^(.*(https?|file)\:.*)$/\1 # NOQA: ignore=E501/' $file -done - -if [[ "${changed[@]}" ]]; then - echo ======================================================= - echo flake8: running flake8 code checks on spack. - echo - echo Modified files: - echo "${changed[@]}" | perl -pe 's/^/ /;s/ +/\n /g' - echo ======================================================= - if flake8 --format pylint "${changed[@]}"; then - echo "Flake8 checks were clean." - else - echo "Flake8 found errors." - exit 1 - fi -else - echo No core framework files modified. -fi - -exit 0 diff --git a/share/spack/qa/run-flake8-tests b/share/spack/qa/run-flake8-tests new file mode 100755 index 0000000000..9556b0d250 --- /dev/null +++ b/share/spack/qa/run-flake8-tests @@ -0,0 +1,86 @@ +#!/usr/bin/env bash +# +# This script runs source code style checks on Spack. +# +# To run it, you'll need to have the Python flake8 installed locally. +# +PYTHONPATH=./lib/spack:$PYTHONPATH + +flake8="$(which flake8)" +if [[ ! $flake8 ]]; then + echo "ERROR: flake8 is required to run this script." + exit 1 +fi + +# Move to Spack root; allows script to be run from anywhere +cd "$(dirname "$0")/../../.." + +# Add changed files that have been committed since branching off of develop +changed=($(git diff --name-only --find-renames develop... -- '*.py')) +# Add changed files that have been staged but not yet committed +changed+=($(git diff --name-only --find-renames --cached -- '*.py')) +# Add changed files that are unstaged +changed+=($(git diff --name-only --find-renames -- '*.py')) +# Add new files that are untracked +changed+=($(git ls-files --exclude-standard --other -- '*.py')) + +# Ensure that each file in the array is unique +changed=($(printf '%s\n' "${changed[@]}" | sort -u)) + +function cleanup { + # Restore original package files after modifying them. + for file in "${changed[@]}"; do + if [[ -e "${file}.sbak~" ]]; then + mv "${file}.sbak~" "${file}" + fi + done +} + +# Cleanup temporary files upon exit or when script is killed +trap cleanup EXIT SIGINT SIGTERM + +# Add approved style exemptions to the changed packages. +for file in "${changed[@]}"; do + # Make a backup to restore later + cp "$file" "$file.sbak~" + + # + # Exemptions for package.py files + # + if [[ $file = *package.py ]]; then + # Exempt lines with urls and descriptions from overlong line errors. + perl -i -pe 's/^(\s*homepage\s*=.*)$/\1 # NOQA: ignore=E501/' $file + perl -i -pe 's/^(\s*url\s*=.*)$/\1 # NOQA: ignore=E501/' $file + perl -i -pe 's/^(\s*version\(.*\).*)$/\1 # NOQA: ignore=E501/' $file + perl -i -pe 's/^(\s*variant\(.*\).*)$/\1 # NOQA: ignore=E501/' $file + perl -i -pe 's/^(\s*depends_on\(.*\).*)$/\1 # NOQA: ignore=E501/' $file + perl -i -pe 's/^(\s*extends\(.*\).*)$/\1 # NOQA: ignore=E501/' $file + + # Exempt '@when' decorated functions from redefinition errors. + perl -i -pe 's/^(\s*\@when\(.*\).*)$/\1 # NOQA: ignore=F811/' $file + fi + + # + # Exemptions for all files + # + perl -i -pe 's/^(.*(https?|file)\:.*)$/\1 # NOQA: ignore=E501/' $file +done + +if [[ "${changed[@]}" ]]; then + echo ======================================================= + echo flake8: running flake8 code checks on spack. + echo + echo Modified files: + echo "${changed[@]}" | perl -pe 's/^/ /;s/ +/\n /g' + echo ======================================================= + if flake8 --format pylint "${changed[@]}"; then + echo "Flake8 checks were clean." + else + echo "Flake8 found errors." + exit 1 + fi +else + echo No core framework files modified. +fi + +exit 0 -- cgit v1.2.3-70-g09d2 From c6aa32bb3c96d3e9a6db3789fcdda9f2227e8129 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Mon, 22 Aug 2016 12:07:50 -0500 Subject: Some flake8 settings weren't documented --- .flake8 | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.flake8 b/.flake8 index b178a2da57..023f392952 100644 --- a/.flake8 +++ b/.flake8 @@ -5,8 +5,10 @@ # rationale is. # # Let people line things up nicely: +# - E129: visually indented line with same indent as next logical line # - E221: multiple spaces before operator -# - E241: multiple spaces after ‘,’ +# - E241: multiple spaces after ',' +# - E272: multiple spaces before keyword # # Let people use terse Python features: # - E731 : lambda expressions @@ -15,9 +17,10 @@ # - F403: disable wildcard import # # These are required to get the package.py files to test clean. -# - F821: undefined name (needed for cmake, configure, etc.) -# - F999: name name be undefined or undefined from star imports. +# - F405: `name` may be undefined, or undefined from star imports: `module` +# - F821: undefined name `name` (needed for cmake, configure, etc.) +# - F999: syntax error in doctest # [flake8] -ignore = E129,E221,E241,E272,E731,F403,F821,F999,F405 +ignore = E129,E221,E241,E272,E731,F403,F405,F821,F999 max-line-length = 79 -- cgit v1.2.3-70-g09d2 From 679f787a65bf4d8b3aa0c7931da7771bb0b8fb1e Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Mon, 22 Aug 2016 14:35:41 -0500 Subject: Add generic changed_files script --- share/spack/qa/changed_files | 31 +++++++++++++++++++++++++++++++ share/spack/qa/run-flake8-tests | 39 +++++++++++++++++++++++---------------- 2 files changed, 54 insertions(+), 16 deletions(-) create mode 100755 share/spack/qa/changed_files diff --git a/share/spack/qa/changed_files b/share/spack/qa/changed_files new file mode 100755 index 0000000000..9c60b3b20b --- /dev/null +++ b/share/spack/qa/changed_files @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# +# Description: +# Returns a list of changed files. +# +# Usage: +# changed_files [ ...] +# changed_files [ ...] +# changed_files ["*." ...] +# +# Options: +# Directories, files, or globs to search for changed files. +# + +# Move to root directory of Spack +# Allows script to be run from anywhere +SPACK_ROOT="$(dirname "$0")/../../.." +cd "$SPACK_ROOT" + +# Add changed files that have been committed since branching off of develop +changed=($(git diff --name-only --find-renames develop... -- "$@")) +# Add changed files that have been staged but not yet committed +changed+=($(git diff --name-only --find-renames --cached -- "$@")) +# Add changed files that are unstaged +changed+=($(git diff --name-only --find-renames -- "$@")) +# Add new files that are untracked +changed+=($(git ls-files --exclude-standard --other -- "$@")) + +# Return array +# Ensure that each file in the array is unique +printf '%s\n' "${changed[@]}" | sort -u diff --git a/share/spack/qa/run-flake8-tests b/share/spack/qa/run-flake8-tests index 9556b0d250..6c9f8968e2 100755 --- a/share/spack/qa/run-flake8-tests +++ b/share/spack/qa/run-flake8-tests @@ -1,31 +1,38 @@ #!/usr/bin/env bash # -# This script runs source code style checks on Spack. +# Description: +# Runs source code style checks on Spack. +# See $SPACK_ROOT/.flake8 for a list of +# approved exceptions. # -# To run it, you'll need to have the Python flake8 installed locally. +# Usage: +# run-flake8-tests +# +# Notes: +# Requires flake8. Can be installed by running: +# `spack install py-flake8` +# or: +# `pip install flake8` +# and adding the bin directory to your PATH. # -PYTHONPATH=./lib/spack:$PYTHONPATH +# Check for dependencies flake8="$(which flake8)" if [[ ! $flake8 ]]; then echo "ERROR: flake8 is required to run this script." exit 1 fi -# Move to Spack root; allows script to be run from anywhere -cd "$(dirname "$0")/../../.." +QA_DIR="$(dirname "$0")" +SPACK_ROOT="$QA_DIR/../../.." -# Add changed files that have been committed since branching off of develop -changed=($(git diff --name-only --find-renames develop... -- '*.py')) -# Add changed files that have been staged but not yet committed -changed+=($(git diff --name-only --find-renames --cached -- '*.py')) -# Add changed files that are unstaged -changed+=($(git diff --name-only --find-renames -- '*.py')) -# Add new files that are untracked -changed+=($(git ls-files --exclude-standard --other -- '*.py')) +# Move to root directory of Spack +# Allows script to be run from anywhere +SPACK_ROOT="$(dirname "$0")/../../.." +cd "$SPACK_ROOT" -# Ensure that each file in the array is unique -changed=($(printf '%s\n' "${changed[@]}" | sort -u)) +# Gather array of changed files +changed=($("$QA_DIR/changed_files" "*.py")) function cleanup { # Restore original package files after modifying them. @@ -80,7 +87,7 @@ if [[ "${changed[@]}" ]]; then exit 1 fi else - echo No core framework files modified. + echo No Python files were modified. fi exit 0 -- cgit v1.2.3-70-g09d2 From d2d6c91b6664352ddc1d41dd8bec3cbe837992de Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Mon, 22 Aug 2016 16:57:53 -0500 Subject: Run documentation tests when documentation is modified --- share/spack/qa/run-doc-tests | 40 ++++++++++++++++++++++++++++++++++++++-- share/spack/qa/run-flake8-tests | 1 - 2 files changed, 38 insertions(+), 3 deletions(-) diff --git a/share/spack/qa/run-doc-tests b/share/spack/qa/run-doc-tests index e5b684fe59..2be5793d36 100755 --- a/share/spack/qa/run-doc-tests +++ b/share/spack/qa/run-doc-tests @@ -1,4 +1,40 @@ #!/usr/bin/env bash +# +# Description: +# Builds Spack documentation and checks for +# possible syntax errors. Treats warnings as +# fatal errors. +# +# Usage: +# run-doc-tests +# +# Notes: +# Requires sphinx. Can be installed by running: +# `spack install py-sphinx` +# or: +# `pip install sphinx` +# and adding the bin directory to your PATH. +# + +QA_DIR="$(dirname "$0")" +SPACK_ROOT="$QA_DIR/../../.." +DOC_DIR="$SPACK_ROOT/lib/spack/docs" + +# Move to documentation directory +# Allows script to be run from anywhere +cd "$DOC_DIR" + +# Gather array of changed files +changed=($("$QA_DIR/changed_files" lib/spack/docs)) + +# Cleanup temporary files upon exit or when script is killed +trap 'make clean' EXIT SIGINT SIGTERM + +# Only run tests if documentation was updated +if [[ "${changed[@]}" ]]; then + # Treat warnings as fatal errors + make SPHINXOPTS=-W +else + echo No documentation was modified. +fi -# pass -exit 0 diff --git a/share/spack/qa/run-flake8-tests b/share/spack/qa/run-flake8-tests index 6c9f8968e2..6c7b4f2c79 100755 --- a/share/spack/qa/run-flake8-tests +++ b/share/spack/qa/run-flake8-tests @@ -28,7 +28,6 @@ SPACK_ROOT="$QA_DIR/../../.." # Move to root directory of Spack # Allows script to be run from anywhere -SPACK_ROOT="$(dirname "$0")/../../.." cd "$SPACK_ROOT" # Gather array of changed files -- cgit v1.2.3-70-g09d2 From 1fc14fd7eddedccc6a24b34b42b9c26c96ded0db Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Mon, 22 Aug 2016 18:40:53 -0500 Subject: Only run unit tests when core Spack framework is modified --- share/spack/qa/check_dependencies | 32 +++++++++++++++++++ share/spack/qa/run-doc-tests | 25 +++++++++------ share/spack/qa/run-flake8-tests | 67 +++++++++++++++++++-------------------- share/spack/qa/run-unit-tests | 56 +++++++++++++++++++++++++++----- 4 files changed, 127 insertions(+), 53 deletions(-) create mode 100755 share/spack/qa/check_dependencies diff --git a/share/spack/qa/check_dependencies b/share/spack/qa/check_dependencies new file mode 100755 index 0000000000..cbcda0ce0f --- /dev/null +++ b/share/spack/qa/check_dependencies @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# +# Description: +# Check to see if dependencies are installed. +# If not, warn the user and tell them how to +# install these dependencies. +# +# Usage: +# check-deps ... +# +# Options: +# One or more dependencies. Must use name of binary. + +for dep in "$@"; do + if ! which $dep &> /dev/null; then + # sphinx-build comes from sphinx + package=$(echo $dep | cut -d - -f 1) + + cat << EOF +ERROR: $package is required to run this script. + +To install with Spack, run: + $ spack install py-$package +or, to install with pip, run: + $ pip install $package +Then add the bin directory to your PATH. +EOF + exit 1 + fi +done + +echo "Dependencies found." diff --git a/share/spack/qa/run-doc-tests b/share/spack/qa/run-doc-tests index 2be5793d36..bde30f74cc 100755 --- a/share/spack/qa/run-doc-tests +++ b/share/spack/qa/run-doc-tests @@ -9,32 +9,37 @@ # run-doc-tests # # Notes: -# Requires sphinx. Can be installed by running: -# `spack install py-sphinx` -# or: -# `pip install sphinx` -# and adding the bin directory to your PATH. +# Requires sphinx and mercurial. # QA_DIR="$(dirname "$0")" SPACK_ROOT="$QA_DIR/../../.." DOC_DIR="$SPACK_ROOT/lib/spack/docs" -# Move to documentation directory -# Allows script to be run from anywhere -cd "$DOC_DIR" +# Array of dependencies +deps=( + sphinx-build + hg +) + +# Check for dependencies +"$QA_DIR/check_dependencies" "${deps[@]}" || exit 1 # Gather array of changed files changed=($("$QA_DIR/changed_files" lib/spack/docs)) +# Move to documentation directory +# Allows script to be run from anywhere +cd "$DOC_DIR" + # Cleanup temporary files upon exit or when script is killed -trap 'make clean' EXIT SIGINT SIGTERM +trap 'make clean --silent' EXIT SIGINT SIGTERM # Only run tests if documentation was updated if [[ "${changed[@]}" ]]; then # Treat warnings as fatal errors make SPHINXOPTS=-W else - echo No documentation was modified. + echo "No documentation was modified." fi diff --git a/share/spack/qa/run-flake8-tests b/share/spack/qa/run-flake8-tests index 6c7b4f2c79..350ef3161f 100755 --- a/share/spack/qa/run-flake8-tests +++ b/share/spack/qa/run-flake8-tests @@ -9,23 +9,20 @@ # run-flake8-tests # # Notes: -# Requires flake8. Can be installed by running: -# `spack install py-flake8` -# or: -# `pip install flake8` -# and adding the bin directory to your PATH. +# Requires flake8. # -# Check for dependencies -flake8="$(which flake8)" -if [[ ! $flake8 ]]; then - echo "ERROR: flake8 is required to run this script." - exit 1 -fi - QA_DIR="$(dirname "$0")" SPACK_ROOT="$QA_DIR/../../.." +# Array of dependencies +deps=( + flake8 +) + +# Check for dependencies +"$QA_DIR/check_dependencies" "${deps[@]}" || exit 1 + # Move to root directory of Spack # Allows script to be run from anywhere cd "$SPACK_ROOT" @@ -33,6 +30,12 @@ cd "$SPACK_ROOT" # Gather array of changed files changed=($("$QA_DIR/changed_files" "*.py")) +# Exit if no Python files were modified +if [[ ! "${changed[@]}" ]]; then + echo "No Python files were modified." + exit 0 +fi + function cleanup { # Restore original package files after modifying them. for file in "${changed[@]}"; do @@ -55,15 +58,15 @@ for file in "${changed[@]}"; do # if [[ $file = *package.py ]]; then # Exempt lines with urls and descriptions from overlong line errors. - perl -i -pe 's/^(\s*homepage\s*=.*)$/\1 # NOQA: ignore=E501/' $file - perl -i -pe 's/^(\s*url\s*=.*)$/\1 # NOQA: ignore=E501/' $file - perl -i -pe 's/^(\s*version\(.*\).*)$/\1 # NOQA: ignore=E501/' $file - perl -i -pe 's/^(\s*variant\(.*\).*)$/\1 # NOQA: ignore=E501/' $file - perl -i -pe 's/^(\s*depends_on\(.*\).*)$/\1 # NOQA: ignore=E501/' $file - perl -i -pe 's/^(\s*extends\(.*\).*)$/\1 # NOQA: ignore=E501/' $file + perl -i -pe 's/^(\s*homepage\s*=.*)$/\1 # NOQA: ignore=E501/' "$file" + perl -i -pe 's/^(\s*url\s*=.*)$/\1 # NOQA: ignore=E501/' "$file" + perl -i -pe 's/^(\s*version\(.*\).*)$/\1 # NOQA: ignore=E501/' "$file" + perl -i -pe 's/^(\s*variant\(.*\).*)$/\1 # NOQA: ignore=E501/' "$file" + perl -i -pe 's/^(\s*depends_on\(.*\).*)$/\1 # NOQA: ignore=E501/' "$file" + perl -i -pe 's/^(\s*extends\(.*\).*)$/\1 # NOQA: ignore=E501/' "$file" # Exempt '@when' decorated functions from redefinition errors. - perl -i -pe 's/^(\s*\@when\(.*\).*)$/\1 # NOQA: ignore=F811/' $file + perl -i -pe 's/^(\s*\@when\(.*\).*)$/\1 # NOQA: ignore=F811/' "$file" fi # @@ -72,21 +75,15 @@ for file in "${changed[@]}"; do perl -i -pe 's/^(.*(https?|file)\:.*)$/\1 # NOQA: ignore=E501/' $file done -if [[ "${changed[@]}" ]]; then - echo ======================================================= - echo flake8: running flake8 code checks on spack. - echo - echo Modified files: - echo "${changed[@]}" | perl -pe 's/^/ /;s/ +/\n /g' - echo ======================================================= - if flake8 --format pylint "${changed[@]}"; then - echo "Flake8 checks were clean." - else - echo "Flake8 found errors." - exit 1 - fi +echo ======================================================= +echo flake8: running flake8 code checks on spack. +echo +echo Modified files: +echo "${changed[@]}" | perl -pe 's/^/ /;s/ +/\n /g' +echo ======================================================= +if flake8 --format pylint "${changed[@]}"; then + echo "Flake8 checks were clean." else - echo No Python files were modified. + echo "Flake8 found errors." + exit 1 fi - -exit 0 diff --git a/share/spack/qa/run-unit-tests b/share/spack/qa/run-unit-tests index 33fb1bfae2..9ce3062e58 100755 --- a/share/spack/qa/run-unit-tests +++ b/share/spack/qa/run-unit-tests @@ -1,20 +1,60 @@ #!/usr/bin/env bash # -# This script runs Spack unit tests. +# Description: +# Runs Spack unit tests. # -# It should be executed from the top-level directory of the repo, -# e.g.: +# Usage: +# run-unit-tests [test ...] # -# share/spack/qa/run-unit-tests +# Options: +# Optionally add one or more unit tests +# to only run these tests. # -# To run it, you'll need to have the Python coverage installed locally. +# Notes: +# Requires coverage. # -# Regular spack setup and tests -. ./share/spack/setup-env.sh +QA_DIR="$(dirname "$0")" +SPACK_ROOT="$QA_DIR/../../.." + +# Array of dependencies +deps=( + coverage +) + +# Check for dependencies +"$QA_DIR/check_dependencies" "${deps[@]}" || exit 1 + +# Add Spack to the PATH. +export PATH="$SPACK_ROOT/bin:$PATH" + +# Array of directories containing core Spack framework +core_dirs=( + bin + etc + # lib, but skip documentation + lib/spack/env + lib/spack/external + lib/spack/llnl + lib/spack/spack + share +) + +# Gather array of changed files +changed=($("$QA_DIR/changed_files" "${core_dirs[@]}")) + +# Exit if no core Spack framework files were modified +if [[ ! "${changed[@]}" ]]; then + echo "No core Spack framework files were modified." + exit 0 +fi + +# Run integration tests +# TODO: should these be separated into a different test suite? +source "$SPACK_ROOT/share/spack/setup-env.sh" spack compilers spack config get compilers spack install -v libdwarf # Run unit tests with code coverage -coverage run bin/spack test +coverage run spack test "$@" -- cgit v1.2.3-70-g09d2 From 7f9d098c2ff7e25a55bb3a69bd9d7314af1f9abc Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Tue, 23 Aug 2016 11:13:02 -0500 Subject: Add mercurial package, used as test dependency --- share/spack/qa/check_dependencies | 31 ++++++++++++--- share/spack/qa/run-doc-tests | 1 + share/spack/qa/run-unit-tests | 7 +++- .../repos/builtin/packages/py-coverage/package.py | 2 +- .../repos/builtin/packages/py-mercurial/package.py | 44 ++++++++++++++++++++++ 5 files changed, 78 insertions(+), 7 deletions(-) create mode 100644 var/spack/repos/builtin/packages/py-mercurial/package.py diff --git a/share/spack/qa/check_dependencies b/share/spack/qa/check_dependencies index cbcda0ce0f..292eac7dd3 100755 --- a/share/spack/qa/check_dependencies +++ b/share/spack/qa/check_dependencies @@ -13,16 +13,37 @@ for dep in "$@"; do if ! which $dep &> /dev/null; then - # sphinx-build comes from sphinx - package=$(echo $dep | cut -d - -f 1) + # Map binary name to package name + case $dep in + sphinx-apidoc|sphinx-build) + spack_package=py-sphinx + pip_package=sphinx + ;; + coverage) + spack_package=py-coverage + pip_package=coverage + ;; + flake8) + spack_package=py-flake8 + pip_package=flake8 + ;; + hg) + spack_package=py-mercurial + pip_package=mercurial + ;; + *) + spack_package=$dep + pip_package=$dep + ;; + esac cat << EOF -ERROR: $package is required to run this script. +ERROR: $dep is required to run this script. To install with Spack, run: - $ spack install py-$package + $ spack install $spack_package or, to install with pip, run: - $ pip install $package + $ pip install $pip_package Then add the bin directory to your PATH. EOF exit 1 diff --git a/share/spack/qa/run-doc-tests b/share/spack/qa/run-doc-tests index bde30f74cc..9feaa3b3e3 100755 --- a/share/spack/qa/run-doc-tests +++ b/share/spack/qa/run-doc-tests @@ -18,6 +18,7 @@ DOC_DIR="$SPACK_ROOT/lib/spack/docs" # Array of dependencies deps=( + sphinx-apidoc sphinx-build hg ) diff --git a/share/spack/qa/run-unit-tests b/share/spack/qa/run-unit-tests index 9ce3062e58..fe2166d2dc 100755 --- a/share/spack/qa/run-unit-tests +++ b/share/spack/qa/run-unit-tests @@ -20,6 +20,7 @@ SPACK_ROOT="$QA_DIR/../../.." # Array of dependencies deps=( coverage + hg ) # Check for dependencies @@ -28,6 +29,10 @@ deps=( # Add Spack to the PATH. export PATH="$SPACK_ROOT/bin:$PATH" +# Move to root directory of Spack +# Allows script to be run from anywhere +cd "$SPACK_ROOT" + # Array of directories containing core Spack framework core_dirs=( bin @@ -57,4 +62,4 @@ spack config get compilers spack install -v libdwarf # Run unit tests with code coverage -coverage run spack test "$@" +coverage run bin/spack test "$@" diff --git a/var/spack/repos/builtin/packages/py-coverage/package.py b/var/spack/repos/builtin/packages/py-coverage/package.py index c2a698b0bd..1a5b6df3d7 100644 --- a/var/spack/repos/builtin/packages/py-coverage/package.py +++ b/var/spack/repos/builtin/packages/py-coverage/package.py @@ -27,7 +27,7 @@ from spack import * class PyCoverage(Package): """ Testing coverage checker for python """ - # FIXME: add a proper url for your package's homepage here. + homepage = "http://nedbatchelder.com/code/coverage/" url = "https://pypi.python.org/packages/source/c/coverage/coverage-4.0a6.tar.gz" diff --git a/var/spack/repos/builtin/packages/py-mercurial/package.py b/var/spack/repos/builtin/packages/py-mercurial/package.py new file mode 100644 index 0000000000..b1b9ffa1b1 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-mercurial/package.py @@ -0,0 +1,44 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class PyMercurial(Package): + """Mercurial is a free, distributed source control management tool. + It efficiently handles projects of any size and offers an easy and + intuitive interface.""" + + homepage = "https://www.mercurial-scm.org/" + url = "https://pypi.python.org/packages/source/m/mercurial/mercurial-3.9.tar.gz" + + version('3.9', 'e2b355da744e94747daae3a5339d28a0', + url="https://pypi.python.org/packages/22/73/e8ef24d3cb13e4fa2695417e13fd22effa1c8e28465eea91a9f84aa922cd/mercurial-3.9.tar.gz") + + extends('python') + + depends_on('py-setuptools', type='build') + + def install(self, spec, prefix): + setup_py('install', '--prefix={0}'.format(prefix)) -- cgit v1.2.3-70-g09d2 From a235d030ac6fda67da382d57f663bba004f32619 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Tue, 23 Aug 2016 11:51:44 -0500 Subject: Fix typos and bugs in Travis config file Forgot to change one of the TEST_TYPEs to TEST_SUITE. -eq not working, trying == instead. Cache doesn't seem to be working, removed. --- .travis.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index f6c6817121..f9bf19148f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,14 +14,11 @@ matrix: exclude: - python: 2.6 # Flake8 no longer supports Python 2.6 - env: TEST_TYPE=flake8 + env: TEST_SUITE=flake8 # Use new Travis infrastructure (Docker can't sudo yet) sudo: false -# Cache dependencies -cache: pip - # Install various dependencies install: - pip install coveralls @@ -40,7 +37,7 @@ before_script: script: share/spack/qa/run-$TEST_SUITE-tests after_success: - - if [[ $TEST_SUITE == unit && $TRAVIS_PYTHON_VERSION -eq 2.7 ]]; then coveralls; fi + - if [[ $TEST_SUITE == unit && $TRAVIS_PYTHON_VERSION == 2.7 ]]; then coveralls; fi notifications: email: -- cgit v1.2.3-70-g09d2 From fda7fcd73dba91bcd1016e3beffc47a4e692fad9 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Tue, 23 Aug 2016 13:58:33 -0500 Subject: Fix #1594, duplicate explicit target name --- lib/spack/docs/basic_usage.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/spack/docs/basic_usage.rst b/lib/spack/docs/basic_usage.rst index 342c93d609..ae7a3762c6 100644 --- a/lib/spack/docs/basic_usage.rst +++ b/lib/spack/docs/basic_usage.rst @@ -906,7 +906,7 @@ Integration with module systems versions. Spack provides some integration with -`Environment Modules `_ +`Environment Modules `__ and `Dotkit `_ to make it easier to use the packages it installed. @@ -972,7 +972,7 @@ When you install a package with Spack, it automatically generates a module file that lets you add the package to your environment. Currently, Spack supports the generation of `Environment Modules -`_ and `Dotkit +`__ and `Dotkit `_. Generated module files for each of these systems can be found in these directories: -- cgit v1.2.3-70-g09d2 From c36f13e44dae52086ca48e4f16b514642bc91106 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Tue, 23 Aug 2016 15:45:04 -0500 Subject: Added more missing dependencies --- share/spack/qa/check_dependencies | 30 ++++++++++++++++++++++-------- share/spack/qa/run-doc-tests | 4 +++- share/spack/qa/run-unit-tests | 4 +++- 3 files changed, 28 insertions(+), 10 deletions(-) diff --git a/share/spack/qa/check_dependencies b/share/spack/qa/check_dependencies index 292eac7dd3..ba9ede4ae0 100755 --- a/share/spack/qa/check_dependencies +++ b/share/spack/qa/check_dependencies @@ -27,25 +27,39 @@ for dep in "$@"; do spack_package=py-flake8 pip_package=flake8 ;; + git) + spack_package=git + ;; hg) spack_package=py-mercurial pip_package=mercurial ;; + svn) + spack_package=subversion + ;; *) spack_package=$dep pip_package=$dep ;; esac - cat << EOF -ERROR: $dep is required to run this script. + echo "ERROR: $dep is required to run this script." + echo + + if [[ $spack_package ]]; then + echo "To install with Spack, run:" + echo " $ spack install $spack_package" + fi + + if [[ $pip_package ]]; then + echo "To install with pip, run:" + echo " $ pip install $pip_package" + fi + + if [[ $spack_package || $pip_package ]]; then + echo "Then add the bin directory to your PATH." + fi -To install with Spack, run: - $ spack install $spack_package -or, to install with pip, run: - $ pip install $pip_package -Then add the bin directory to your PATH. -EOF exit 1 fi done diff --git a/share/spack/qa/run-doc-tests b/share/spack/qa/run-doc-tests index 9feaa3b3e3..102eb72d5e 100755 --- a/share/spack/qa/run-doc-tests +++ b/share/spack/qa/run-doc-tests @@ -9,7 +9,7 @@ # run-doc-tests # # Notes: -# Requires sphinx and mercurial. +# Requires sphinx, git, mercurial, and subversion. # QA_DIR="$(dirname "$0")" @@ -20,7 +20,9 @@ DOC_DIR="$SPACK_ROOT/lib/spack/docs" deps=( sphinx-apidoc sphinx-build + git hg + svn ) # Check for dependencies diff --git a/share/spack/qa/run-unit-tests b/share/spack/qa/run-unit-tests index fe2166d2dc..03dfe7cea1 100755 --- a/share/spack/qa/run-unit-tests +++ b/share/spack/qa/run-unit-tests @@ -11,7 +11,7 @@ # to only run these tests. # # Notes: -# Requires coverage. +# Requires coverage, git, mercurial, and subversion. # QA_DIR="$(dirname "$0")" @@ -20,7 +20,9 @@ SPACK_ROOT="$QA_DIR/../../.." # Array of dependencies deps=( coverage + git hg + svn ) # Check for dependencies -- cgit v1.2.3-70-g09d2 From 2326355497165f90717170e114ff9849b64be1fa Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Tue, 23 Aug 2016 17:03:29 -0500 Subject: Fix #1604 and fix #1605, title underline problems --- lib/spack/docs/basic_usage.rst | 203 ++++++++++++++--------- lib/spack/docs/configuration.rst | 23 +-- lib/spack/docs/developer_guide.rst | 67 ++++---- lib/spack/docs/features.rst | 23 ++- lib/spack/docs/getting_started.rst | 9 +- lib/spack/docs/index.rst | 7 +- lib/spack/docs/mirrors.rst | 31 ++-- lib/spack/docs/packaging_guide.rst | 325 ++++++++++++++++++++++--------------- 8 files changed, 411 insertions(+), 277 deletions(-) diff --git a/lib/spack/docs/basic_usage.rst b/lib/spack/docs/basic_usage.rst index ae7a3762c6..0a80fa7e2f 100644 --- a/lib/spack/docs/basic_usage.rst +++ b/lib/spack/docs/basic_usage.rst @@ -1,13 +1,14 @@ .. _basic-usage: +=========== Basic usage -===================== +=========== The ``spack`` command has many *subcommands*. You'll only need a small subset of them for typical usage. Note that Spack colorizes output. ``less -R`` should be used with -Spack to maintian this colorization. Eg:: +Spack to maintain this colorization. E.g.:: spack find | less -R @@ -16,8 +17,9 @@ It is recommend that the following be put in your ``.bashrc`` file:: alias less='less -R' +-------------------------- Listing available packages ------------------------------- +-------------------------- To install software with Spack, you need to know what software is available. You can see a list of available package names at the @@ -25,8 +27,9 @@ available. You can see a list of available package names at the .. _spack-list: +^^^^^^^^^^^^^^ ``spack list`` -~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^ The ``spack list`` command prints out a list of all of the packages Spack can install: @@ -59,8 +62,9 @@ All packages whose names contain documentation case insensitive: .. _spack-info: +^^^^^^^^^^^^^^ ``spack info`` -~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^ To get more information on a particular package from `spack list`, use `spack info`. Just supply the name of a package: @@ -77,8 +81,9 @@ viruses. .. _spack-versions: +^^^^^^^^^^^^^^^^^^ ``spack versions`` -~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^ To see *more* available versions of a package, run ``spack versions``. For example: @@ -95,14 +100,15 @@ by scraping it directly from package web pages. Depending on the package and how its releases are organized, Spack may or may not be able to find remote versions. - +--------------------------- Installing and uninstalling ------------------------------- +--------------------------- .. _spack-install: +^^^^^^^^^^^^^^^^^ ``spack install`` -~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^ ``spack install`` will install any package shown by ``spack list``. For example, To install the latest version of the ``mpileaks`` @@ -142,8 +148,9 @@ that the packages is installed: The last line, with the ``[+]``, indicates where the package is installed. +^^^^^^^^^^^^^^^^^^^^^^^^^^^ Building a specific version -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^^ Spack can also build *specific versions* of a package. To do this, just add ``@`` after the package name, followed by a version: @@ -172,8 +179,9 @@ customize an installation in :ref:`sec-specs`. .. _spack-uninstall: +^^^^^^^^^^^^^^^^^^^ ``spack uninstall`` -~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^ To uninstall a package, type ``spack uninstall ``. This will ask the user for confirmation, and in case will completely remove the directory in which the package was installed. @@ -216,17 +224,18 @@ You may force uninstall a package with the `--force` option but you risk breaking other installed packages. In general, it is safer to remove dependent packages *before* removing their dependencies or use the `--dependents` option. - +------------------------- Seeing installed packages ------------------------------------ +------------------------- We know that ``spack list`` shows you the names of available packages, but how do you figure out which are installed? .. _spack-find: +^^^^^^^^^^^^^^ ``spack find`` -~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^ ``spack find`` shows the *specs* of installed packages. A spec is like a name, but it has a version, compiler, architecture, and build @@ -367,11 +376,11 @@ will find every installed package with a 'debug' compile-time option enabled. The full spec syntax is discussed in detail in :ref:`sec-specs`. - .. _compiler-config: +---------------------- Compiler configuration ------------------------------------ +---------------------- Spack has the ability to build packages with multiple compilers and compiler versions. Spack searches for compilers on your machine @@ -380,8 +389,9 @@ your path. .. _spack-compilers: +^^^^^^^^^^^^^^^^^^^ ``spack compilers`` -~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^ You can see which compilers spack has found by running ``spack compilers`` or ``spack compiler list``:: @@ -408,15 +418,17 @@ how this is done is in :ref:`sec-specs`. .. _spack-compiler-add: +^^^^^^^^^^^^^^^^^^^^^^ ``spack compiler add`` -~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^ An alias for ``spack compiler find``. .. _spack-compiler-find: +^^^^^^^^^^^^^^^^^^^^^^^ ``spack compiler find`` -~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^ If you do not see a compiler in this list, but you want to use it with Spack, you can simply run ``spack compiler find`` with the path to @@ -441,8 +453,9 @@ This loads the environment module for gcc-4.9.0 to add it to .. _spack-compiler-info: +^^^^^^^^^^^^^^^^^^^^^^^ ``spack compiler info`` -~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^ If you want to see specifics on a particular compiler, you can run ``spack compiler info`` on it:: @@ -459,9 +472,9 @@ Notice also that we didn\'t have to be too specific about the version. We just said ``intel@15``, and information about the only matching Intel compiler was displayed. - +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Manual compiler configuration -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ If auto-detection fails, you can manually configure a compiler by editing your ``~/.spack/compilers.yaml`` file. You can do this by running @@ -518,8 +531,9 @@ following section. .. _sec-specs: +-------------------- Specs & dependencies -------------------------- +-------------------- We know that ``spack install``, ``spack uninstall``, and other commands take a package name with an optional version specifier. In @@ -609,7 +623,6 @@ some *other* version of ``mpich``. In general, such a configuration would likely behave unexpectedly at runtime, and Spack enforces this to ensure a consistent runtime environment. - The point of specs is to abstract this full DAG from Spack users. If a user does not care about the DAG at all, she can refer to mpileaks by simply writing ``mpileaks``. If she knows that ``mpileaks`` @@ -646,8 +659,9 @@ could depend on ``mpich@1.2:`` if it can only build with version Below are more details about the specifiers that you can add to specs. +^^^^^^^^^^^^^^^^^ Version specifier -~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^ A version specifier comes somewhere after a package name and starts with ``@``. It can be a single version, e.g. ``@1.0``, ``@3``, or @@ -669,9 +683,9 @@ policies set for the particular Spack installation. Details about how versions are compared and how Spack determines if one version is less than another are discussed in the developer guide. - +^^^^^^^^^^^^^^^^^^ Compiler specifier -~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^ A compiler specifier comes somewhere after a package name and starts with ``%``. It tells Spack what compiler(s) a particular package @@ -688,9 +702,9 @@ name or compiler specifier to their left in the spec. If the compiler spec is omitted, Spack will choose a default compiler based on site policies. - +^^^^^^^^ Variants -~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^ Variants are named options associated with a particular package. They are optional, as each package must provide default values for each variant it @@ -742,9 +756,9 @@ variants using the backwards compatibility syntax and uses only ``~`` for disabled boolean variants. We allow ``-`` and spaces on the command line is provided for convenience and legibility. - +^^^^^^^^^^^^^^ Compiler Flags -~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^ Compiler flags are specified using the same syntax as non-boolean variants, but fulfill a different purpose. While the function of a variant is set by @@ -764,9 +778,9 @@ in gnu autotools. If all flags are set, the order is ``$cppflags $cflags|$cxxflags $ldflags command $ldlibs`` for C and C++ and ``$fflags $cppflags $ldflags command $ldlibs`` for fortran. - +^^^^^^^^^^^^^^^^^^^^^^^ Architecture specifiers -~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^ The architecture can be specified by using the reserved words ``target`` and/or ``os`` (``target=x86-64 os=debian7``). You can also @@ -781,11 +795,11 @@ Spack will autodetect what kind of operating system is on your machine as well as the processor. For more information on how the architecture can be used on Cray machines, check here :ref:`cray-support` - .. _sec-virtual-dependencies: +-------------------- Virtual dependencies -------------------------- +-------------------- The dependence graph for ``mpileaks`` we saw above wasn't *quite* accurate. ``mpileaks`` uses MPI, which is an interface that has many @@ -828,9 +842,9 @@ version of a virtual package, e.g. if an application needs MPI-2 functions, it can depend on ``mpi@2:`` to indicate that it needs some implementation that provides MPI-2 functions. - +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Constraining virtual packages -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ When installing a package that depends on a virtual package, you can opt to specify the particular provider you want to use, or you can let @@ -857,8 +871,9 @@ any MPI implementation will do. If another package depends on error. Likewise, if you try to plug in some package that doesn't provide MPI, Spack will raise an error. +^^^^^^^^^^^^^^^^^^^^^^^^ Specifying Specs by Hash -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^ Complicated specs can become cumbersome to enter on the command line, especially when many of the qualifications are necessary to @@ -876,8 +891,9 @@ Note that this will not work to reinstall a depencency uninstalled by .. _spack-providers: +^^^^^^^^^^^^^^^^^^^ ``spack providers`` -~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^ You can see what packages provide a particular virtual package using ``spack providers``. If you wanted to see what packages provide @@ -895,6 +911,7 @@ versions are now filtered out. .. _shell-support: +------------------------------- Integration with module systems ------------------------------- @@ -910,10 +927,9 @@ Spack provides some integration with and `Dotkit `_ to make it easier to use the packages it installed. - - +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Installing Environment Modules -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ In order to use Spack's generated environment modules, you must have installed the *Environment Modules* package. On many Linux @@ -931,7 +947,6 @@ Environment Modules, you can get it with Spack: spack install environment-modules - In this case to activate it automatically you need to add the following two lines to your ``.bashrc`` profile (or similar): @@ -944,11 +959,12 @@ If you use a Unix shell other than ``bash``, modify the commands above accordingly and source the appropriate file in ``${MODULES_HOME}/Modules/init/``. - .. TODO : Add a similar section on how to install dotkit ? +^^^^^^^^^^^^^^^^^^^^^^^^ Spack and module systems -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^ + You can enable shell support by sourcing some files in the ``/share/spack`` directory. @@ -986,9 +1002,9 @@ The directories are automatically added to your ``MODULEPATH`` and ``DK_NODE`` environment variables when you enable Spack's `shell support `_. - +^^^^^^^^^^^^^^^^^^^^^^^ Using Modules & Dotkits -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^ If you have shell support enabled you should be able to run either ``module avail`` or ``use -l spack`` to see what modules/dotkits have @@ -1093,8 +1109,9 @@ a package, you can type unload or unuse similarly: only available if you have enabled Spack's shell support *and* you have dotkit or modules installed on your machine. +^^^^^^^^^^^^^^^^^^^^^^ Ambiguous module names -~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^ If a spec used with load/unload or use/unuse is ambiguous (i.e. more than one installed package matches it), then Spack will warn you: @@ -1118,8 +1135,9 @@ used ``gcc``. You could therefore just type: To identify just the one built with the Intel compiler. +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Module files generation and customization -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Environment Modules and Dotkit files are generated when packages are installed, and are placed in the following directories under the Spack root: @@ -1134,8 +1152,10 @@ The content that gets written in each module file can be customized in two ways: 1. overriding part of the ``spack.Package`` API within a ``package.py`` 2. writing dedicated configuration files +^^^^^^^^^^^^^^^^^^^^^^^^ Override ``Package`` API ^^^^^^^^^^^^^^^^^^^^^^^^ + There are currently two methods in ``spack.Package`` that may affect the content of module files: @@ -1145,9 +1165,9 @@ of module files: """Set up the compile and runtime environments for a package.""" pass - +""""""""""""""""" Recursive Modules -`````````````````` +""""""""""""""""" In some cases, it is desirable to load not just a module, but also all the modules it depends on. This is not required for most modules @@ -1165,8 +1185,9 @@ Modules may be loaded recursively with the ``load`` command's More than one spec may be placed on the command line here. +""""""""""""""""""""""""""""""""" Module Commands for Shell Scripts -`````````````````````````````````` +""""""""""""""""""""""""""""""""" Although Spack is flexible, the ``module`` command is much faster. This could become an issue when emitting a series of ``spack load`` @@ -1215,9 +1236,9 @@ This script may be directly executed in bash via source <( spack module find tcl --dependencies --shell py-numpy git ) - +^^^^^^^^^^^^^^^^^^^^^^^^^ Regenerating Module files -~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^ .. code-block:: python @@ -1230,6 +1251,7 @@ module file content for the package you are currently writing, the second allows for modifications to your dependees module file. In both cases one needs to fill ``run_env`` with the desired list of environment modifications. +"""""""""""""""""""""""""""""""""""""""""""""""" Example : ``builtin/packages/python/package.py`` """""""""""""""""""""""""""""""""""""""""""""""" @@ -1246,6 +1268,7 @@ overrides ``setup_dependent_environment`` in the following way: to insert the appropriate ``PYTHONPATH`` modifications in the module files of python packages. +^^^^^^^^^^^^^^^^^^^ Configuration files ^^^^^^^^^^^^^^^^^^^ @@ -1275,8 +1298,10 @@ For each module system that can be enabled a finer configuration is possible: The structure under the ``tcl`` and ``dotkit`` keys is almost equal, and will be showcased in the following by some examples. +""""""""""""""""""""""""""""""""""""""" Select module files by spec constraints """"""""""""""""""""""""""""""""""""""" + Using spec syntax it's possible to have different customizations for different groups of module files. @@ -1316,6 +1341,7 @@ what will happen is that: first, no matter where they appear in the configuration file. All the other spec constraints are instead evaluated top to bottom. +"""""""""""""""""""""""""""""""""""""""" Filter modifications out of module files """""""""""""""""""""""""""""""""""""""" @@ -1335,6 +1361,7 @@ will generate dotkit module files that will not contain modifications to either ``CPATH`` or ``LIBRARY_PATH`` and environment module files that instead will contain those modifications. +""""""""""""""""""""" Autoload dependencies """"""""""""""""""""" @@ -1357,6 +1384,7 @@ dependencies. ``prerequisites`` that accepts the same values and will add ``prereq`` statements instead of automatically loading other modules. +"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" Blacklist or whitelist the generation of specific module files """""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" @@ -1375,6 +1403,7 @@ A configuration file like: will skip module file generation for anything that satisfies ``%gcc@4.4.7``, with the exception of specs that satisfy ``gcc`` or ``llvm``. +"""""""""""""""""""""""""""""""""""""""""""""""" Customize the naming scheme and insert conflicts """""""""""""""""""""""""""""""""""""""""""""""" @@ -1400,6 +1429,7 @@ load two or more versions of the same software at the same time. .. note:: The ``conflict`` option is ``tcl`` specific +^^^^^^^^^^^^^^^^^^^^^^^^^ Regenerating module files ^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -1410,6 +1440,7 @@ new features. .. _spack-module: +"""""""""""""""""""""""" ``spack module refresh`` """""""""""""""""""""""" @@ -1426,8 +1457,9 @@ regenerate all module and dotkit files from scratch: .. _extensions: +---------------- Filesystem Views -------------------------------- +---------------- .. Maybe this is not the right location for this documentation. @@ -1469,8 +1501,9 @@ Views can be used for a variety of purposes including: - Providing an atomic and monolithic binary distribution, eg for delivery as a single tarball. - Producing ephemeral testing or developing environments. +^^^^^^^^^^^^^^^^^^^^^^ Using Filesystem Views -~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^ A filesystem view is created and packages are linked in by the ``spack view`` command's ``symlink`` and ``hardlink`` sub-commands. The @@ -1483,7 +1516,6 @@ files in the ``cmake`` package while retaining its dependencies. .. code-block:: sh - $ spack view -v symlink myview cmake@3.5.2 ==> Linking package: "ncurses" ==> Linking package: "zlib" @@ -1504,9 +1536,9 @@ files in the ``cmake`` package while retaining its dependencies. captoinfo c_rehash infotocap openssl tabs toe tset clear infocmp ncurses6-config reset tic tput - +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Limitations of Filesystem Views -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ This section describes some limitations that should be considered in using filesystems views. @@ -1536,12 +1568,9 @@ particular, if two packages require the same sub-tree of dependencies, removing one package (recursively) will remove its dependencies and leave the other package broken. - - - - +--------------------------- Extensions & Python support ------------------------------------- +--------------------------- Spack's installation model assumes that each package will live in its own install prefix. However, certain packages are typically installed @@ -1563,8 +1592,9 @@ an *extension*. Suppose you have Python installed like so: .. _spack-extensions: +^^^^^^^^^^^^^^^^^^^^ ``spack extensions`` -~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^ You can find extensions for your Python installation like this: @@ -1618,8 +1648,9 @@ directly when you run ``python``: ImportError: No module named numpy >>> +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Extensions & Environment Modules -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ There are two ways to get ``numpy`` working in Python. The first is to use :ref:`shell-support`. You can simply ``use`` or ``load`` the @@ -1643,9 +1674,9 @@ or, for dotkit: Now ``import numpy`` will succeed for as long as you keep your current session open. - +^^^^^^^^^^^^^^^^^^^^^ Activating Extensions -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^ It is often desirable to have certain packages *always* available as part of a Python installation. Spack offers a more permanent solution @@ -1655,8 +1686,9 @@ installation: .. _spack-activate: +^^^^^^^^^^^^^^^^^^ ``spack activate`` -^^^^^^^^^^^^^^^^^^^^^^^ +^^^^^^^^^^^^^^^^^^ .. code-block:: sh @@ -1700,7 +1732,6 @@ packages listed as activated: -- linux-debian7-x86_64 / gcc@4.4.7 -------------------------------- py-nose@1.3.4 py-numpy@1.9.1 py-setuptools@11.3.1 - Now, when a user runs python, ``numpy`` will be available for import *without* the user having to explicitly loaded. ``python@2.7.8`` now acts like a system Python installation with ``numpy`` installed inside @@ -1718,8 +1749,10 @@ into the same prefix. Users who want a different version of a package can still get it by using environment modules, but they will have to explicitly load their preferred version. +^^^^^^^^^^^^^^^^^^^^^ ``spack activate -f`` -^^^^^^^^^^^^^^^^^^^^^^^^^ +^^^^^^^^^^^^^^^^^^^^^ + If, for some reason, you want to activate a package *without* its dependencies, you can use ``spack activate -f``: @@ -1730,8 +1763,9 @@ dependencies, you can use ``spack activate -f``: .. _spack-deactivate: +^^^^^^^^^^^^^^^^^^^^ ``spack deactivate`` -^^^^^^^^^^^^^^^^^^^^^^^^^ +^^^^^^^^^^^^^^^^^^^^ We've seen how activating an extension can be used to set up a default version of a Python module. Obviously, you may want to change that at @@ -1751,8 +1785,9 @@ several variants: spack deactivate -a python +----------------------- Filesystem requirements --------------------------- +----------------------- Spack currently needs to be run from a filesystem that supports ``flock`` locking semantics. Nearly all local filesystems and recent @@ -1797,11 +1832,16 @@ This issue typically manifests with the error below: A nicer error message is TBD in future versions of Spack. +<<<<<<< c36f13e44dae52086ca48e4f16b514642bc91106 .. _cray-support: +======= +.. _spack-cray: +>>>>>>> Fix #1604 and fix #1605, title underline problems +------------- Spack on Cray ------------------------------ +------------- Spack differs slightly when used on a Cray system. The architecture spec can differentiate between the front-end and back-end processor and operating system. @@ -1835,10 +1875,9 @@ And the front-end operating system is: spack install zlib os=SuSE11 - - +^^^^^^^^^^^^^^^^^^^^^^^ Cray compiler detection -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^ Spack can detect compilers using two methods. For the front-end, we treat everything the same. The difference lies in back-end compiler detection. @@ -1880,8 +1919,9 @@ Spack sets the default Cray link type to dynamic, to better match other other platforms. Individual packages can enable static linking (which is the default outside of Spack on cray systems) using the -static flag. +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Setting defaults and using Cray modules -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ If you want to use default compilers for each PrgEnv and also be able to load cray external modules, you will need to set up a packages.yaml. @@ -1934,16 +1974,17 @@ if we want to build with intel compilers, use version 16.0.0.109. We add a spec for each compiler type for each cray modules. This ensures that for each compiler on our system we can use that external module. - For more on external packages check out the section :ref:`sec-external_packages`. +------------ Getting Help ------------------------ +------------ .. _spack-help: +^^^^^^^^^^^^^^ ``spack help`` -~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^ If you don't find what you need here, the ``help`` subcommand will print out out a list of *all* of ``spack``'s options and subcommands: diff --git a/lib/spack/docs/configuration.rst b/lib/spack/docs/configuration.rst index d39c932021..bce6005a7c 100644 --- a/lib/spack/docs/configuration.rst +++ b/lib/spack/docs/configuration.rst @@ -1,12 +1,14 @@ .. _configuration: +============= Configuration -=================================== +============= .. _temp-space: +--------------- Temporary space ----------------------------- +--------------- .. warning:: Temporary space configuration will eventually be moved to configuration files, but currently these settings are in @@ -55,8 +57,10 @@ directory is. .. _sec-external_packages: +----------------- External Packages ----------------------------- +----------------- + Spack can be configured to use externally-installed packages rather than building its own packages. This may be desirable if machines ship with system packages, such as a customized MPI @@ -126,9 +130,9 @@ The ``buildable`` does not need to be paired with external packages. It could also be used alone to forbid packages that may be buggy or otherwise undesirable. - +-------------------------- Concretization Preferences --------------------------------- +-------------------------- Spack can be configured to prefer certain compilers, package versions, depends_on, and variants during concretization. @@ -136,7 +140,6 @@ The preferred configuration can be controlled via the ``~/.spack/packages.yaml`` file for user configuations, or the ``etc/spack/packages.yaml`` site configuration. - Here's an example packages.yaml file that sets preferred packages: .. code-block:: sh @@ -152,7 +155,6 @@ Here's an example packages.yaml file that sets preferred packages: providers: mpi: [mvapich, mpich, openmpi] - At a high level, this example is specifying how packages should be concretized. The opencv package should prefer using gcc 4.9 and be built with debug options. The gperftools package should prefer version @@ -185,9 +187,9 @@ concretization rules. A provider lists a value that packages may ``depend_on`` (e.g, mpi) and a list of rules for fulfilling that dependency. - +--------- Profiling ------------------- +--------- Spack has some limited built-in support for profiling, and can report statistics using standard Python timing tools. To use this feature, @@ -195,8 +197,9 @@ supply ``-p`` to Spack on the command line, before any subcommands. .. _spack-p: +^^^^^^^^^^^^ ``spack -p`` -~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^ ``spack -p`` output looks like this: diff --git a/lib/spack/docs/developer_guide.rst b/lib/spack/docs/developer_guide.rst index d28fe4b2a5..5c712149b5 100644 --- a/lib/spack/docs/developer_guide.rst +++ b/lib/spack/docs/developer_guide.rst @@ -1,7 +1,8 @@ .. _developer_guide: +=============== Developer Guide -===================== +=============== This guide is intended for people who want to work on Spack itself. If you just want to develop packages, see the :ref:`packaging-guide`. @@ -11,8 +12,9 @@ It is assumed that you've read the :ref:`basic-usage` and concepts discussed there. If you're not, we recommend reading those first. +-------- Overview ------------------------ +-------- Spack is designed with three separate roles in mind: @@ -63,9 +65,9 @@ building the software off to the package object. The rest of this document describes all the pieces that come together to make that happen. - +------------------- Directory Structure -------------------------- +------------------- So that you can familiarize yourself with the project, we'll start with a high level view of Spack's directory structure:: @@ -123,15 +125,16 @@ use the thing that's supposed to spare them from the details of big, complicated packages. The end result is that Spack works out of the box: clone it and add ``bin`` to your PATH and you're ready to go. - +-------------- Code Structure -------------------------- +-------------- This section gives an overview of the various Python modules in Spack, grouped by functionality. +^^^^^^^^^^^^^^^^^^^^^^^ Package-related modules -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^ :mod:`spack.package` Contains the :class:`Package ` class, which @@ -158,9 +161,9 @@ Package-related modules decorator, which allows :ref:`multimethods ` in packages. - +^^^^^^^^^^^^^^^^^^^^ Spec-related modules -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^ :mod:`spack.spec` Contains :class:`Spec ` and :class:`SpecParser @@ -208,9 +211,9 @@ Spec-related modules Not yet implemented. Should eventually have architecture descriptions for cross-compiling. - +^^^^^^^^^^^^^^^^^ Build environment -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^ :mod:`spack.stage` Handles creating temporary directories for builds. @@ -224,15 +227,17 @@ Build environment Create more implementations of this to change the hierarchy and naming scheme in ``$spack_prefix/opt`` +^^^^^^^^^^^^^^^^^ Spack Subcommands -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^ :mod:`spack.cmd` Each module in this package implements a Spack subcommand. See :ref:`writing commands ` for details. +^^^^^^^^^^ Unit tests -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^ :mod:`spack.test` Implements Spack's test suite. Add a module and put its name in @@ -242,8 +247,9 @@ Unit tests This is a fake package hierarchy used to mock up packages for Spack's test suite. +^^^^^^^^^^^^^ Other Modules -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^ :mod:`spack.globals` Includes global settings for Spack. the default policy classes for @@ -269,13 +275,13 @@ Other Modules :class:`SpackError `, the base class for Spack's exception hierarchy. - +------------ Spec objects -------------------------- +------------ +--------------- Package objects -------------------------- - +--------------- Most spack commands look something like this: @@ -286,34 +292,37 @@ look something like this: #. Instantiate a package based on the spec, and #. Call methods (e.g., ``install()``) on the package object. - - The information in Package files is used at all stages in this process. - Conceptually, packages are overloaded. They contain +------------- Stage objects -------------------------- +------------- .. _writing-commands: +---------------- Writing commands -------------------------- +---------------- +---------- Unit tests -------------------------- +---------- +------------ Unit testing -------------------------- - +------------ +------------------ Developer commands -------------------------- +------------------ +^^^^^^^^^^^^^ ``spack doc`` -~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^ +^^^^^^^^^^^^^^ ``spack test`` -~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^ diff --git a/lib/spack/docs/features.rst b/lib/spack/docs/features.rst index 27a3b4b435..d746e8ad50 100644 --- a/lib/spack/docs/features.rst +++ b/lib/spack/docs/features.rst @@ -1,13 +1,15 @@ +================ Feature overview -================== +================ This is a high-level overview of features that make Spack different from other `package managers `_ and `port systems `_. +--------------------------- Simple package installation ----------------------------- +--------------------------- Installing the default version of a package is simple. This will install the latest version of the ``mpileaks`` package and all of its dependencies: @@ -16,8 +18,9 @@ the latest version of the ``mpileaks`` package and all of its dependencies: $ spack install mpileaks +-------------------------------- Custom versions & configurations -------------------------------------------- +-------------------------------- Spack allows installation to be customized. Users can specify the version, build compiler, compile-time options, and cross-compile @@ -47,9 +50,9 @@ Users can specify as many or few options as they care about. Spack will fill in the unspecified values with sensible defaults. The two listed syntaxes for variants are identical when the value is boolean. - +---------------------- Customize dependencies -------------------------------------- +---------------------- Spack allows *dependencies* of a particular installation to be customized extensively. Suppose that ``mpileaks`` depends indirectly @@ -61,23 +64,25 @@ configurations for the dependencies: # Install mpileaks and link it with specific versions of libelf and libdwarf $ spack install mpileaks@1.1.2 %gcc@4.7.3 +debug ^libelf@0.8.12 ^libdwarf@20130729+debug - +------------------------ Non-destructive installs -------------------------------------- +------------------------ Spack installs every unique package/dependency configuration into its own prefix, so new installs will not break existing ones. +------------------------------- Packages can peacefully coexist -------------------------------------- +------------------------------- Spack avoids library misconfiguration by using ``RPATH`` to link dependencies. When a user links a library or runs a program, it is tied to the dependencies it was built with, so there is no need to manipulate ``LD_LIBRARY_PATH`` at runtime. +------------------------- Creating packages is easy -------------------------------------- +------------------------- To create a new packages, all Spack needs is a URL for the source archive. The ``spack create`` command will create a boilerplate diff --git a/lib/spack/docs/getting_started.rst b/lib/spack/docs/getting_started.rst index 2c5b68ea65..47f66b429b 100644 --- a/lib/spack/docs/getting_started.rst +++ b/lib/spack/docs/getting_started.rst @@ -1,8 +1,10 @@ +=============== Getting Started -==================== +=============== +-------- Download --------------------- +-------- Getting spack is easy. You can clone it from the `github repository `_ using this command: @@ -35,8 +37,9 @@ For a richer experience, use Spack's `shell support This automatically adds Spack to your ``PATH``. +------------ Installation --------------------- +------------ You don't need to install Spack; it's ready to run as soon as you clone it from git. diff --git a/lib/spack/docs/index.rst b/lib/spack/docs/index.rst index a5bbd4e23b..603245b5dc 100644 --- a/lib/spack/docs/index.rst +++ b/lib/spack/docs/index.rst @@ -3,8 +3,9 @@ You can adapt this file completely to your liking, but it should at least contain the root `toctree` directive. +=================== Spack Documentation -================================= +=================== Spack is a package management tool designed to support multiple versions and configurations of software on a wide variety of platforms @@ -36,8 +37,9 @@ package: If you're new to spack and want to start using it, see :doc:`getting_started`, or refer to the full manual below. +----------------- Table of Contents ---------------------- +----------------- .. toctree:: :maxdepth: 2 @@ -54,6 +56,7 @@ Table of Contents package_list API Docs +================== Indices and tables ================== diff --git a/lib/spack/docs/mirrors.rst b/lib/spack/docs/mirrors.rst index 583575a565..41e039b838 100644 --- a/lib/spack/docs/mirrors.rst +++ b/lib/spack/docs/mirrors.rst @@ -1,7 +1,8 @@ .. _mirrors: +======= Mirrors -============================ +======= Some sites may not have access to the internet for fetching packages. These sites will need a local repository of tarballs from which they @@ -51,8 +52,9 @@ contains tarballs for each package, named after each package. .. _spack-mirror: +---------------- ``spack mirror`` ----------------------------- +---------------- Mirrors are managed with the ``spack mirror`` command. The help for ``spack mirror`` looks like this:: @@ -79,8 +81,9 @@ control the URL(s) from which Spack downloads its packages. .. _spack-mirror-create: +----------------------- ``spack mirror create`` ----------------------------- +----------------------- You can create a mirror using the ``spack mirror create`` command, assuming you're on a machine where you can access the internet. @@ -89,7 +92,6 @@ The command will iterate through all of Spack's packages and download the safe ones into a directory structure like the one above. Here is what it looks like: - .. code-block:: bash $ spack mirror create libelf libdwarf @@ -124,8 +126,9 @@ what it looks like: Once this is done, you can tar up the ``spack-mirror-2014-06-24`` directory and copy it over to the machine you want it hosted on. +^^^^^^^^^^^^^^^^^^^ Custom package sets -~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^ Normally, ``spack mirror create`` downloads all the archives it has checksums for. If you want to only create a mirror for a subset of @@ -138,8 +141,9 @@ command:: Will create a mirror for libelf versions greater than or equal to 0.8.12 and boost versions greater than or equal to 1.44. +^^^^^^^^^^^^ Mirror files -~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^ If you have a *very* large number of packages you want to mirror, you can supply a file with specs in it, one per line:: @@ -158,8 +162,9 @@ your site. .. _spack-mirror-add: +-------------------- ``spack mirror add`` ----------------------------- +-------------------- Once you have a mirror, you need to let spack know about it. This is relatively simple. First, figure out the URL for the mirror. If it's @@ -183,8 +188,9 @@ Each mirror has a name so that you can refer to it again later. .. _spack-mirror-list: +--------------------- ``spack mirror list`` ----------------------------- +--------------------- To see all the mirrors Spack knows about, run ``spack mirror list``:: @@ -193,8 +199,9 @@ To see all the mirrors Spack knows about, run ``spack mirror list``:: .. _spack-mirror-remove: +----------------------- ``spack mirror remove`` ----------------------------- +----------------------- To remove a mirror by name:: @@ -202,8 +209,9 @@ To remove a mirror by name:: $ spack mirror list ==> No mirrors configured. +----------------- Mirror precedence ----------------------------- +----------------- Adding a mirror really adds a line in ``~/.spack/mirrors.yaml``:: @@ -217,8 +225,9 @@ search the topmost mirror first and the bottom-most mirror last. .. _caching: +------------------- Local Default Cache ----------------------------- +------------------- Spack caches resources that are downloaded as part of installs. The cache is a valid spack mirror: it uses the same directory structure and naming scheme diff --git a/lib/spack/docs/packaging_guide.rst b/lib/spack/docs/packaging_guide.rst index 34fcb1e101..c6ac6dd7ef 100644 --- a/lib/spack/docs/packaging_guide.rst +++ b/lib/spack/docs/packaging_guide.rst @@ -1,7 +1,8 @@ .. _packaging-guide: +=============== Packaging Guide -===================== +=============== This guide is intended for developers or administrators who want to package software so that Spack can install it. It assumes that you @@ -28,13 +29,15 @@ ubiquitous in the scientific software community. Second, it's a modern language and has many powerful features to help make package writing easy. +--------------------------- Creating & editing packages ----------------------------------- +--------------------------- .. _spack-create: +^^^^^^^^^^^^^^^^ ``spack create`` -~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^ The ``spack create`` command creates a directory with the package name and generates a ``package.py`` file with a boilerplate package template from a URL. @@ -194,8 +197,9 @@ Before going into details, we'll cover a few more basics. .. _spack-edit: +^^^^^^^^^^^^^^ ``spack edit`` -~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^ One of the easiest ways to learn to write packages is to look at existing ones. You can edit a package file by name with the ``spack @@ -222,8 +226,9 @@ using ``spack create`` or ``spack edit -f``: .. _spack-edit-f: +^^^^^^^^^^^^^^^^^ ``spack edit -f`` -~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^ ``spack edit -f`` can be used to create a new, minimal boilerplate package: @@ -258,9 +263,9 @@ to fill in yourself: This is useful when ``spack create`` cannot figure out the name and version of your package from the archive URL. - +---------------------------- Naming & directory structure --------------------------------------- +---------------------------- .. note:: @@ -274,8 +279,9 @@ live in Spack's directory structure. In general, `spack-create`_ and `spack-edit`_ handle creating package files for you, so you can skip most of the details here. +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ``var/spack/repos/builtin/packages`` -~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ A Spack installation directory is structured like a standard UNIX install prefix (``bin``, ``lib``, ``include``, ``var``, ``opt``, @@ -298,9 +304,9 @@ package lives in:: Alongside the ``package.py`` file, a package may contain extra directories or files (like patches) that it needs to build. - +^^^^^^^^^^^^^ Package Names -~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^ Packages are named after the directory containing ``package.py``. It is preferred, but not required, that the directory, and thus the package name, are @@ -346,8 +352,9 @@ difficult to support these options. So, you can name a package ``3proxy`` or ``_foo`` and Spack won't care. It just needs to see that name in the package spec. +^^^^^^^^^^^^^^^^^^^ Package class names -~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^ Spack loads ``package.py`` files dynamically, and it needs to find a special class name in the file for the load to succeed. The **class @@ -368,9 +375,9 @@ some examples: In general, you won't have to remember this naming convention because `spack-create`_ and `spack-edit`_ handle the details for you. - +------------------- Adding new versions ------------------------- +------------------- The most straightforward way to add new versions to your package is to add a line like this in the package class: @@ -385,8 +392,9 @@ add a line like this in the package class: Versions should be listed with the newest version first. +^^^^^^^^^^^^ Version URLs -~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^ By default, each version's URL is extrapolated from the ``url`` field in the package. For example, Spack is smart enough to download @@ -423,8 +431,9 @@ construct the new one for ``8.2.1``. When you supply a custom URL for a version, Spack uses that URL *verbatim* and does not perform extrapolation. +^^^^^^^^^^^^^^^^^^^^^^^^ Skipping the expand step -~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^ Spack normally expands archives automatically after downloading them. If you want to skip this step (e.g., for self-extracting @@ -452,8 +461,9 @@ it executable, then runs it with some arguments. installer = Executable(self.stage.archive_file) installer('--prefix=%s' % prefix, 'arg1', 'arg2', 'etc.') +^^^^^^^^^ Checksums -~~~~~~~~~~~~~~~~~ +^^^^^^^^^ Spack uses a checksum to ensure that the downloaded package version is not corrupted or compromised. This is especially important when @@ -465,8 +475,9 @@ Spack can currently support checksums using the MD5, SHA-1, SHA-224, SHA-256, SHA-384, and SHA-512 algorithms. It determines the algorithm to use based on the hash length. +^^^^^^^^^^^^^ ``spack md5`` -^^^^^^^^^^^^^^^^^^^^^^ +^^^^^^^^^^^^^ If you have one or more files to checksum, you can use the ``spack md5`` command to do it: @@ -495,8 +506,9 @@ version of this process. .. _spack-checksum: +^^^^^^^^^^^^^^^^^^ ``spack checksum`` -^^^^^^^^^^^^^^^^^^^^^^ +^^^^^^^^^^^^^^^^^^ If you want to add new versions to a package you've already created, this is automated with the ``spack checksum`` command. Here's an @@ -559,8 +571,9 @@ versions. See the documentation on `attribute_list_url`_ and .. _vcs-fetch: +------------------------------ Fetching from VCS repositories --------------------------------------- +------------------------------ For some packages, source code is provided in a Version Control System (VCS) repository rather than in a tarball. Spack can fetch packages @@ -573,8 +586,9 @@ call to your package with parameters indicating the repository URL and any branch, tag, or revision to fetch. See below for the parameters you'll need for each VCS system. +^^^^^^^^^^^^^^^^^^^^^^^^^ Repositories and versions -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^ The package author is responsible for coming up with a sensible name for each version to be fetched from a repository. For example, if @@ -607,8 +621,9 @@ supported. .. _git-fetch: +^^^ Git -~~~~~~~~~~~~~~~~~~~~ +^^^ Git fetching is enabled with the following parameters to ``version``: @@ -683,9 +698,9 @@ Submodules version('1.0.1', git='https://github.com/example-project/example.git', tag='v1.0.1', submdoules=True) - +^^^^^^^^^^ Installing -^^^^^^^^^^^^^^ +^^^^^^^^^^ You can fetch and install any of the versions above as you'd expect, by using ``@`` in a spec: @@ -700,8 +715,9 @@ a user runs ``spack info ``. .. _hg-fetch: +^^^^^^^^^ Mercurial -~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^ Fetching with mercurial works much like `git `_, but you use the ``hg`` parameter. @@ -733,8 +749,9 @@ example@`` command-line syntax. .. _svn-fetch: +^^^^^^^^^^ Subversion -~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^ To fetch with subversion, use the ``svn`` and ``revision`` parameters: @@ -759,6 +776,7 @@ Fetching a revision Subversion branches are handled as part of the directory structure, so you can check out a branch or tag by changing the ``url``. +------------------------------------------------- Expanding additional resources in the source tree ------------------------------------------------- @@ -779,6 +797,7 @@ Based on the keywords present among the arguments the appropriate ``FetchStrateg will be used for the resource. The keyword ``destination`` is relative to the source root of the package and should point to where the resource is to be expanded. +------------------------------------------------------ Automatic caching of files fetched during installation ------------------------------------------------------ @@ -789,42 +808,48 @@ reinstalled on account of a change in the hashing scheme. .. _license: +----------------- Licensed software ------------------------------------------- +----------------- In order to install licensed software, Spack needs to know a few more details about a package. The following class attributes should be defined. +^^^^^^^^^^^^^^^^^^^^ ``license_required`` -~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^ Boolean. If set to ``True``, this software requires a license. If set to ``False``, all of the following attributes will be ignored. Defaults to ``False``. +^^^^^^^^^^^^^^^^^^^ ``license_comment`` -~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^ String. Contains the symbol used by the license manager to denote a comment. Defaults to ``#``. +^^^^^^^^^^^^^^^^^ ``license_files`` -~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^ List of strings. These are files that the software searches for when looking for a license. All file paths must be relative to the installation directory. More complex packages like Intel may require multiple licenses for individual components. Defaults to the empty list. +^^^^^^^^^^^^^^^^ ``license_vars`` -~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^ List of strings. Environment variables that can be set to tell the software where to look for a license if it is not in the usual location. Defaults to the empty list. +^^^^^^^^^^^^^^^ ``license_url`` -~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^ String. A URL pointing to license setup instructions for the software. Defaults to the empty string. @@ -909,8 +934,9 @@ documentation. .. _patching: +------- Patches ------------------------------------------- +------- Depending on the host architecture, package version, known bugs, or other issues, you may need to patch your software to get it to build @@ -918,8 +944,9 @@ correctly. Like many other package systems, spack allows you to store patches alongside your package files and apply them to source code after it's downloaded. +^^^^^^^^^ ``patch`` -~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^ You can specify patches in your package file with the ``patch()`` function. ``patch`` looks like this: @@ -999,8 +1026,9 @@ from the URL and then applied to your source code. applies cleanly with ``-p1``, but if you're using a patch you didn't create yourself, ``level`` can be handy. +^^^^^^^^^^^^^^^^^^^^^ ``patch()`` functions -~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^ In addition to supplying patch files, you can write a custom function to patch a package's source. For example, the ``py-pyside`` package @@ -1048,8 +1076,9 @@ if you run install, hit ctrl-C, and run install again, the code in the patch function is only run once. Also, you can tell Spack to run only the patching part of the build using the :ref:`spack-patch` command. +--------------- Handling RPATHs ----------------------------- +--------------- Spack installs each package in a way that ensures that all of its dependencies are found when it runs. It does this using `RPATHs @@ -1093,9 +1122,9 @@ RPATHs in Spack are handled in one of three ways: links. You can see this how this is used in the :ref:`PySide example ` above. - +-------------------- Finding new versions ----------------------------- +-------------------- You've already seen the ``homepage`` and ``url`` package attributes: @@ -1121,8 +1150,9 @@ Spack to find tarballs elsewhere. .. _attribute_list_url: +^^^^^^^^^^^^ ``list_url`` -~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^ When spack tries to find available versions of packages (e.g. with `spack checksum `_), it spiders the parent directory @@ -1156,8 +1186,9 @@ the ``list_url``, because that is where links to old versions are: .. _attribute_list_depth: +^^^^^^^^^^^^^^ ``list_depth`` -~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^ ``libdwarf`` and many other packages have a listing of available versions on a single webpage, but not all do. For example, ``mpich`` @@ -1187,8 +1218,9 @@ when spidering the page. .. _attribute_parallel: +--------------- Parallel builds ------------------- +--------------- By default, Spack will invoke ``make()`` with a ``-j `` argument, so that builds run in parallel. It figures out how many @@ -1240,11 +1272,11 @@ you set ``parallel`` to ``False`` at the package level, then each call to ``make()`` will be sequential by default, but packagers can call ``make(parallel=True)`` to override it. - .. _dependencies: +------------ Dependencies ------------------------------- +------------ We've covered how to build a simple package, but what if one package relies on another package to build? How do you express that in a @@ -1271,8 +1303,9 @@ Spack makes this relatively easy. Let's take a look at the def install(self, spec, prefix): ... +^^^^^^^^^^^^^^^^ ``depends_on()`` -~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^ The highlighted ``depends_on('libelf')`` call tells Spack that it needs to build and install the ``libelf`` package before it builds @@ -1280,8 +1313,9 @@ needs to build and install the ``libelf`` package before it builds guaranteed that ``libelf`` has been built and installed successfully, so you can rely on it for your libdwarf build. +^^^^^^^^^^^^^^^^ Dependency specs -~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^ ``depends_on`` doesn't just take the name of another package. It takes a full spec. This means that you can restrict the versions or @@ -1345,8 +1379,9 @@ Lua module loading). .. _setup-dependent-environment: +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ``setup_dependent_environment()`` -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Spack provides a mechanism for dependencies to provide variables that can be used in their dependents' build. Any package can declare a @@ -1412,11 +1447,11 @@ Python's ``setup_dependent_environment`` method also sets up some other variables, creates a directory, and sets up the ``PYTHONPATH`` so that dependent packages can find their dependencies at build time. - .. _packaging_extensions: +---------- Extensions -------------------------- +---------- Spack's support for package extensions is documented extensively in :ref:`extensions`. This section documents how to make your own @@ -1482,10 +1517,9 @@ linked in at activation time. ``depends_on('python')`` and ``extends(python)`` in the same package. ``extends`` implies ``depends_on``. - - +^^^^^^^^^^^^^^^^^^^^^^^^^ Activation & deactivation -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^ Spack's ``Package`` class has default ``activate`` and ``deactivate`` implementations that handle symbolically linking extensions' prefixes @@ -1535,9 +1569,9 @@ Deactivate behaves similarly to activate, but it unlinks files: Both of these methods call some custom functions in the Python package. See the source for Spack's Python package for details. - +^^^^^^^^^^^^^^^^^^^^ Activation arguments -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^ You may have noticed that the ``activate`` function defined above takes keyword arguments. These are the keyword arguments from @@ -1552,11 +1586,11 @@ The only keyword argument supported by default is the ``ignore`` argument, which can take a regex, list of regexes, or a predicate to determine which files *not* to symlink during activation. - .. _virtual-dependencies: +-------------------- Virtual dependencies ------------------------------ +-------------------- In some cases, more than one package can satisfy another package's dependency. One way this can happen is if a package depends on a @@ -1577,8 +1611,9 @@ similar package files, e.g., ``foo``, ``foo-mvapich``, ``foo-mpich``, but Spack avoids this explosion of package files by providing support for *virtual dependencies*. +^^^^^^^^^^^^ ``provides`` -~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^ In Spack, ``mpi`` is handled as a *virtual package*. A package like ``mpileaks`` can depend on it just like any other package, by @@ -1614,8 +1649,9 @@ The ``provides("mpi")`` call tells Spack that the ``mpich`` package can be used to satisfy the dependency of any package that ``depends_on('mpi')``. +^^^^^^^^^^^^^^^^^^^^ Versioned Interfaces -~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^ Just as you can pass a spec to ``depends_on``, so can you pass a spec to ``provides`` to add constraints. This allows Spack to support the @@ -1636,8 +1672,9 @@ This says that ``mpich2`` provides MPI support *up to* version 2, but if a package ``depends_on("mpi@3")``, then Spack will *not* build that package with ``mpich2``. +^^^^^^^^^^^^^^^^^ ``provides when`` -~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^ The same package may provide different versions of an interface depending on *its* version. Above, we simplified the ``provides`` @@ -1678,11 +1715,11 @@ Suppose a user invokes ``spack install`` like this: Spack will fail with a constraint violation, because the version of MPICH requested is too low for the ``mpi`` requirement in ``foo``. - .. _abstract-and-concrete: +------------------------- Abstract & concrete specs ------------------------------------------- +------------------------- Now that we've seen how spec constraints can be specified :ref:`on the command line ` and within package definitions, we can talk @@ -1716,7 +1753,6 @@ DAG, based on the constraints above:: ^libelf@0.8.11 ^mpi - .. graphviz:: digraph { @@ -1727,7 +1763,6 @@ DAG, based on the constraints above:: dyninst -> "libelf@0.8.11" } - This diagram shows a spec DAG output as a tree, where successive levels of indentation represent a depends-on relationship. In the above DAG, we can see some packages annotated with their constraints, @@ -1735,8 +1770,9 @@ and some packages with no annotations at all. When there are no annotations, it means the user doesn't care what configuration of that package is built, just so long as it works. +^^^^^^^^^^^^^^ Concretization -~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^ An abstract spec is useful for the user, but you can't install an abstract spec. Spack has to take the abstract spec and "fill in" the @@ -1776,8 +1812,9 @@ the preferences of their own users. .. _spack-spec: +^^^^^^^^^^^^^^ ``spack spec`` -~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^ For an arbitrary spec, you can see the result of concretization by running ``spack spec``. For example: @@ -1796,9 +1833,9 @@ running ``spack spec``. For example: This is useful when you want to know exactly what Spack will do when you ask for a particular spec. - +^^^^^^^^^^^^^^^^^^^^^^^^^^^ ``Concretization Policies`` -~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^^ A user may have certain preferences for how packages should be concretized on their system. For example, one user may prefer packages @@ -1810,8 +1847,9 @@ for more details. .. _install-method: +----------------------------------- Implementing the ``install`` method ------------------------------------------- +----------------------------------- The last element of a package is its ``install()`` method. This is where the real work of installation happens, and it's the main part of @@ -1825,15 +1863,12 @@ the package you'll need to customize for each piece of software. be built, and a ``prefix``: the path to the directory where the software should be installed. - Spack provides wrapper functions for ``configure`` and ``make`` so that you can call them in a similar way to how you'd call a shell command. In reality, these are Python functions. Spack provides these functions to make writing packages more natural. See the section on :ref:`shell wrappers `. - - Now that the metadata is out of the way, we can move on to the ``install()`` method. When a user runs ``spack install``, Spack fetches an archive for the correct version of the software, expands @@ -1883,8 +1918,9 @@ information. .. _install-environment: +----------------------- The install environment --------------------------- +----------------------- In general, you should not have to do much differently in your install method than you would when installing a package on the command line. @@ -1901,8 +1937,9 @@ custom Makefiles, you may need to add logic to modify the makefiles. The remainder of the section covers the way Spack's build environment works. +^^^^^^^^^^^^^^^^^^^^^ Environment variables -~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^ Spack sets a number of standard environment variables that serve two purposes: @@ -1970,8 +2007,9 @@ below. .. _compiler-wrappers: +^^^^^^^^^^^^^^^^^^^^^ Compiler interceptors -~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^ As mentioned, ``CC``, ``CXX``, ``F77``, and ``FC`` are set to point to Spack's compiler wrappers. These are simply called ``cc``, ``c++``, @@ -2037,8 +2075,10 @@ and/or ``ldlibs``). They do not override the canonical autotools flags with the same names (but in ALL-CAPS) that may be passed into the build by particularly challenging package scripts. +^^^^^^^^^^^^^^ Compiler flags -~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^ + In rare circumstances such as compiling and running small unit tests, a package developer may need to know what are the appropriate compiler flags to enable features like ``OpenMP``, ``c++11``, ``c++14`` and alike. To that end the @@ -2054,8 +2094,10 @@ package supports additional variants like variant('openmp', default=True, description="Enable OpenMP support.") +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Message Parsing Interface (MPI) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + It is common for high performance computing software/packages to use ``MPI``. As a result of conretization, a given package can be built using different implementations of MPI such as ``Openmpi``, ``MPICH`` or ``IntelMPI``. @@ -2070,9 +2112,10 @@ Package developers are advised to use these variables, for example ``self.spec[' instead of hard-coding ``join_path(self.spec['mpi'].prefix.bin, 'mpicc')`` for the reasons outlined above. - +^^^^^^^^^^^^^^^^^^^^^^^^^ Blas and Lapack libraries -~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^ + Different packages provide implementation of ``Blas`` and ``Lapack`` routines. The names of the resulting static and/or shared libraries differ from package to package. In order to make ``install()`` method indifferent to the @@ -2083,9 +2126,9 @@ applies to packages which provide ``Lapack``. Package developers are advised to use these variables, for example ``spec['blas'].blas_shared_lib`` instead of hard-coding ``join_path(spec['blas'].prefix.lib, 'libopenblas.so')``. - +^^^^^^^^^^^^^^^^^^^^^ Forking ``install()`` -~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^ To give packagers free reign over their install environment, Spack forks a new process each time it invokes a package's ``install()`` @@ -2097,9 +2140,9 @@ dedicated process. .. _prefix-objects: - +----------------- Failing the build ----------------------- +----------------- Sometimes you don't want a package to successfully install unless some condition is true. You can explicitly cause the build to fail from @@ -2110,9 +2153,9 @@ condition is true. You can explicitly cause the build to fail from if spec.architecture.startswith('darwin'): raise InstallError('This package does not build on Mac OS X!') - +-------------- Prefix objects ----------------------- +-------------- Spack passes the ``prefix`` parameter to the install method so that you can pass it to ``configure``, ``cmake``, or some other installer, @@ -2122,7 +2165,6 @@ e.g.: configure('--prefix=' + prefix) - For the most part, prefix objects behave exactly like strings. For packages that do not have their own install target, or for those that implement it poorly (like ``libdwarf``), you may need to manually copy @@ -2142,7 +2184,6 @@ yourself, e.g.: mkdirp(prefix.lib) install('libfoo.a', prefix.lib) - Most of the standard UNIX directory names are attributes on the ``prefix`` object. Here is a full list: @@ -2169,8 +2210,9 @@ Most of the standard UNIX directory names are attributes on the .. _spec-objects: +------------ Spec objects -------------------------- +------------ When ``install`` is called, most parts of the build process are set up for you. The correct version's tarball has been downloaded and @@ -2187,8 +2229,9 @@ special parameters to ``configure``, like need to supply special compiler flags depending on the compiler. All of this information is available in the spec. +^^^^^^^^^^^^^^^^^^^^^^^^ Testing spec constraints -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^ You can test whether your spec is configured a certain way by using the ``satisfies`` method. For example, if you want to check whether @@ -2253,9 +2296,9 @@ the two functions is that ``satisfies()`` tests whether spec constraints overlap at all, while ``in`` tests whether a spec or any of its dependencies satisfy the provided spec. - +^^^^^^^^^^^^^^^^^^^^^^ Accessing Dependencies -~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^ You may need to get at some file or binary that's in the prefix of one of your dependencies. You can do that by sub-scripting the spec: @@ -2277,8 +2320,9 @@ would do with the package's own spec: .. _multimethods: +^^^^^^^^^^^^^^^^^^^^^^^^^^ Multimethods and ``@when`` -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^ Spack allows you to make multiple versions of instance functions in packages, based on whether the package's spec satisfies particular @@ -2377,8 +2421,9 @@ method (the one without the ``@when`` decorator) will be called. .. _shell-wrappers: +----------------------- Shell command functions ----------------------------- +----------------------- Recall the install method from ``libelf``: @@ -2426,11 +2471,11 @@ to the ``make`` wrapper to disable parallel make. In the ``libelf`` package, this allows us to avoid race conditions in the library's build system. - .. _sanity-checks: +------------------------------- Sanity checking an installation --------------------------------- +------------------------------- By default, Spack assumes that a build has failed if nothing is written to the install prefix, and that it has succeeded if anything @@ -2455,9 +2500,9 @@ like this can falsely report that they were successfully installed if an error occurs before the install is complete but after files have been written to the ``prefix``. - +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ``sanity_check_is_file`` and ``sanity_check_is_dir`` -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ You can optionally specify *sanity checks* to deal with this problem. Add properties like this to your package: @@ -2482,11 +2527,11 @@ the build will fail and the install prefix will be removed. If they succeed, Spack considers the build succeeful and keeps the prefix in place. - .. _file-manipulation: +--------------------------- File manipulation functions ------------------------------- +--------------------------- Many builds are not perfect. If a build lacks an install target, or if it does not use systems like CMake or autotools, which have standard @@ -2508,9 +2553,9 @@ running: This is already part of the boilerplate for packages created with ``spack create`` or ``spack edit``. - +^^^^^^^^^^^^^^^^^^^ Filtering functions -~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^ :py:func:`filter_file(regex, repl, *filenames, **kwargs) ` Works like ``sed`` but with Python regular expression syntax. Takes @@ -2568,9 +2613,9 @@ Filtering functions change_sed_delimiter('@', ';', 'utils/FixMakefile') change_sed_delimiter('@', ';', 'utils/FixMakefile.sed.default') - +^^^^^^^^^^^^^^ File functions -~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^ :py:func:`ancestor(dir, n=1) ` Get the n\ :sup:`th` ancestor of the directory ``dir``. @@ -2638,22 +2683,21 @@ File functions The ``create=True`` keyword argument causes the command to create the directory if it does not exist. - :py:func:`touch(path) ` Create an empty file at ``path``. - .. _package-lifecycle: +----------------------- Coding Style Guidelines ---------------------------- +----------------------- The following guidelines are provided, in the interests of making Spack packages work in a consistent manner: - +^^^^^^^^^^^^^ Variant Names -~~~~~~~~~~~~~~ +^^^^^^^^^^^^^ Spack packages with variants similar to already-existing Spack packages should use the same name for their variants. Standard @@ -2671,14 +2715,15 @@ variant names are: If specified in this table, the corresponding default should be used when declaring a variant. - +^^^^^^^^^^^^^ Version Lists -~~~~~~~~~~~~~~ +^^^^^^^^^^^^^ Spack packges should list supported versions with the newest first. +^^^^^^^^^^^^^^^^ Special Versions -~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^ The following *special* version names may be used when building a package: @@ -2697,9 +2742,9 @@ The following *special* version names may be used when building a package: * *@local*: Indicates the version was built manually from some source tree of unknown provenance (see ``spack setup``). - +--------------------------- Packaging workflow commands ---------------------------------- +--------------------------- When you are building packages, you will likely not get things completely right the first time. @@ -2729,8 +2774,9 @@ control over the install process. .. _spack-fetch: +^^^^^^^^^^^^^^^ ``spack fetch`` -~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^ The first step of ``spack install``. Takes a spec and determines the correct download URL to use for the requested package version, then @@ -2743,8 +2789,9 @@ fetch`` is idempotent and will not download the archive again. .. _spack-stage: +^^^^^^^^^^^^^^^ ``spack stage`` -~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^ The second step in ``spack install`` after ``spack fetch``. Expands the downloaded archive in its temporary directory, where it will be @@ -2753,8 +2800,9 @@ already been expanded, ``stage`` is idempotent. .. _spack-patch: +^^^^^^^^^^^^^^^ ``spack patch`` -~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^ After staging, Spack applies patches to downloaded packages, if any have been specified in the package file. This command will run the @@ -2766,8 +2814,10 @@ package before patching. .. _spack-restage: +^^^^^^^^^^^^^^^^^ ``spack restage`` -~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^ + Restores the source code to pristine state, as it was before building. Does this in one of two ways: @@ -2780,18 +2830,21 @@ Does this in one of two ways: .. _spack-clean: +^^^^^^^^^^^^^^^ ``spack clean`` -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^ + Cleans up temporary files for a particular package, by deleting the expanded/checked out source code *and* any downloaded archive. If ``fetch``, ``stage``, or ``install`` are run again after this, Spack's build process will start from scratch. - .. _spack-purge: +^^^^^^^^^^^^^^^ ``spack purge`` -~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^ + Cleans up all of Spack's temporary and cached files. This can be used to recover disk space if temporary files from interrupted or failed installs accumulate in the staging area. @@ -2803,8 +2856,9 @@ running ``spack clean`` for every package you have fetched or staged. When called with ``--cache`` or ``--all`` this will clear all resources :ref:`cached ` during installs. +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Keeping the stage directory on success -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ By default, ``spack install`` will delete the staging area once a package has been successfully built and installed. Use @@ -2818,9 +2872,9 @@ This allows you to inspect the build directory and potentially debug the build. You can use ``purge`` or ``clean`` later to get rid of the unwanted temporary files. - +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Keeping the install prefix on failure -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ By default, ``spack install`` will delete any partially constructed install prefix if anything fails during ``install()``. If you want to @@ -2839,14 +2893,15 @@ to get rid of the install prefix before you build again: spack uninstall -f - +--------------------- Graphing dependencies --------------------------- +--------------------- .. _spack-graph: +^^^^^^^^^^^^^^^ ``spack graph`` -~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^ Spack provides the ``spack graph`` command for graphing dependencies. The command by default generates an ASCII rendering of a spec's @@ -2919,8 +2974,9 @@ example:: This graph can be provided as input to other graphing tools, such as those in `Graphviz `_. +------------------------- Interactive shell support --------------------------- +------------------------- Spack provides some limited shell support to make life easier for packagers. You can enable these commands by sourcing a setup file in @@ -2937,8 +2993,9 @@ For ``csh`` and ``tcsh`` run: .. _spack-cd: +^^^^^^^^^^^^ ``spack cd`` -~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^ ``spack cd`` allows you to quickly cd to pertinent directories in Spack. Suppose you've staged a package but you want to modify it before you @@ -2968,8 +3025,9 @@ the main python source directory of your spack install. .. _spack-env: +^^^^^^^^^^^^^ ``spack env`` -~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^ ``spack env`` functions much like the standard unix ``env`` command, but it takes a spec as an argument. You can use it to see the @@ -2993,11 +3051,11 @@ To run commands in a package's build environment, you can simply provided them a This will cd to the build directory and then run ``configure`` in the package's build environment. - .. _spack-location: +^^^^^^^^^^^^^^^^^^ ``spack location`` -~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^ ``spack location`` is the same as ``spack cd`` but it does not require shell support. It simply prints out the path you ask for, rather than @@ -3019,6 +3077,7 @@ might write: CXXFLAGS += -I$DWARF_PREFIX/include CXXFLAGS += -L$DWARF_PREFIX/lib +---------------------------------- Build System Configuration Support ---------------------------------- @@ -3064,8 +3123,9 @@ Notes: to manually re-run ``spconfig.py`` when a ``CMakeLists.txt`` file has changed. +^^^^^^^^^^^^ CMakePackage -~~~~~~~~~~~~ +^^^^^^^^^^^^ In order ot enable ``spack setup`` functionality, the author of ``myproject/package.py`` must subclass from ``CMakePackage`` instead @@ -3091,9 +3151,9 @@ translate variant flags into CMake definitions. For example: If needed, a packager may also override methods defined in ``StagedPackage`` (see below). - +^^^^^^^^^^^^^ StagedPackage -~~~~~~~~~~~~~ +^^^^^^^^^^^^^ ``CMakePackage`` is implemented by subclassing the ``StagedPackage`` superclass, which breaks down the standard ``Package.install()`` @@ -3115,8 +3175,9 @@ and ``install``. Details: ``install()`` method may be accessed via ``self.spec`` and ``self.prefix``. +^^^^^^^^^^^^^ GNU Autotools -~~~~~~~~~~~~~ +^^^^^^^^^^^^^ The ``setup`` functionality is currently only available for CMake-based packages. Extending this functionality to GNU -- cgit v1.2.3-70-g09d2 From 0614bd206fa8b5fb975536f055bfe008303e7ff6 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Wed, 24 Aug 2016 09:32:05 -0500 Subject: Fix #1608, Include file not found --- lib/spack/docs/basic_usage.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/spack/docs/basic_usage.rst b/lib/spack/docs/basic_usage.rst index 0a80fa7e2f..ef01f616cd 100644 --- a/lib/spack/docs/basic_usage.rst +++ b/lib/spack/docs/basic_usage.rst @@ -1278,8 +1278,8 @@ file can be placed either at *site* or *user* scope. The default site configuration reads: - .. literalinclude:: ../../../etc/spack/modules.yaml - :language: yaml +.. literalinclude:: ../../../etc/spack/defaults/modules.yaml + :language: yaml It basically inspects the installation prefixes for the existence of a few folders and, if they exist, it prepends a path to a given -- cgit v1.2.3-70-g09d2 From eb56744f4d4fbbccb25dbd2042a9b28397bcca81 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Wed, 24 Aug 2016 10:32:29 -0500 Subject: Fix package docstrings, #1612 --- var/spack/repos/builtin/packages/adios/package.py | 6 ++---- var/spack/repos/builtin/packages/antlr/package.py | 5 +++++ var/spack/repos/builtin/packages/arpack-ng/package.py | 7 +++---- var/spack/repos/builtin/packages/asciidoc/package.py | 3 ++- var/spack/repos/builtin/packages/astyle/package.py | 7 +++---- var/spack/repos/builtin/packages/caliper/package.py | 6 ++---- var/spack/repos/builtin/packages/cfitsio/package.py | 8 ++++---- var/spack/repos/builtin/packages/cityhash/package.py | 2 ++ var/spack/repos/builtin/packages/cube/package.py | 7 ++----- var/spack/repos/builtin/packages/datamash/package.py | 6 ++---- var/spack/repos/builtin/packages/eigen/package.py | 7 ++----- var/spack/repos/builtin/packages/espresso/package.py | 8 +++----- var/spack/repos/builtin/packages/gdal/package.py | 6 ++---- var/spack/repos/builtin/packages/gdb/package.py | 6 ++---- var/spack/repos/builtin/packages/gmsh/package.py | 7 +++---- var/spack/repos/builtin/packages/lmod/package.py | 7 +++---- var/spack/repos/builtin/packages/lua-luafilesystem/package.py | 7 +++---- var/spack/repos/builtin/packages/mxml/package.py | 6 ++---- var/spack/repos/builtin/packages/ncdu/package.py | 6 ++---- var/spack/repos/builtin/packages/opari2/package.py | 7 ++----- var/spack/repos/builtin/packages/openjpeg/package.py | 7 +++---- var/spack/repos/builtin/packages/parallel/package.py | 6 ++---- var/spack/repos/builtin/packages/petsc/package.py | 6 ++---- var/spack/repos/builtin/packages/py-prettytable/package.py | 10 ++++------ var/spack/repos/builtin/packages/py-tuiview/package.py | 7 +++---- var/spack/repos/builtin/packages/scorep/package.py | 9 +++------ var/spack/repos/builtin/packages/screen/package.py | 6 ++---- var/spack/repos/builtin/packages/swiftsim/package.py | 8 ++------ var/spack/repos/builtin/packages/tau/package.py | 10 +++------- var/spack/repos/builtin/packages/xerces-c/package.py | 9 ++++----- var/spack/repos/builtin/packages/zsh/package.py | 9 ++++----- 31 files changed, 83 insertions(+), 128 deletions(-) diff --git a/var/spack/repos/builtin/packages/adios/package.py b/var/spack/repos/builtin/packages/adios/package.py index e2e3b7d6b7..96fee2ac28 100644 --- a/var/spack/repos/builtin/packages/adios/package.py +++ b/var/spack/repos/builtin/packages/adios/package.py @@ -27,12 +27,10 @@ from spack import * class Adios(Package): - """ - The Adaptable IO System (ADIOS) provides a simple, + """The Adaptable IO System (ADIOS) provides a simple, flexible way for scientists to describe the data in their code that may need to be written, - read, or processed outside of the running simulation - """ + read, or processed outside of the running simulation.""" homepage = "http://www.olcf.ornl.gov/center-projects/adios/" url = "https://github.com/ornladios/ADIOS/archive/v1.10.0.tar.gz" diff --git a/var/spack/repos/builtin/packages/antlr/package.py b/var/spack/repos/builtin/packages/antlr/package.py index 891061c62f..804460ce14 100644 --- a/var/spack/repos/builtin/packages/antlr/package.py +++ b/var/spack/repos/builtin/packages/antlr/package.py @@ -26,6 +26,11 @@ from spack import * class Antlr(Package): + """ANTLR (ANother Tool for Language Recognition) is a powerful parser + generator for reading, processing, executing, or translating structured + text or binary files. It's widely used to build languages, tools, and + frameworks. From a grammar, ANTLR generates a parser that can build and + walk parse trees.""" homepage = "http://www.antlr.org" url = "https://github.com/antlr/antlr/tarball/v2.7.7" diff --git a/var/spack/repos/builtin/packages/arpack-ng/package.py b/var/spack/repos/builtin/packages/arpack-ng/package.py index 728c2345f2..2fa7608098 100644 --- a/var/spack/repos/builtin/packages/arpack-ng/package.py +++ b/var/spack/repos/builtin/packages/arpack-ng/package.py @@ -26,8 +26,7 @@ from spack import * class ArpackNg(Package): - """ - ARPACK-NG is a collection of Fortran77 subroutines designed to solve large + """ARPACK-NG is a collection of Fortran77 subroutines designed to solve large scale eigenvalue problems. Important Features: @@ -51,8 +50,8 @@ class ArpackNg(Package): forked it and implemented their own modifications, arpack-ng aims to tackle this by providing a common repository and maintained versions. - arpack-ng is replacing arpack almost everywhere. - """ + arpack-ng is replacing arpack almost everywhere.""" + homepage = 'https://github.com/opencollab/arpack-ng' url = 'https://github.com/opencollab/arpack-ng/archive/3.3.0.tar.gz' diff --git a/var/spack/repos/builtin/packages/asciidoc/package.py b/var/spack/repos/builtin/packages/asciidoc/package.py index be8db684f5..811d34afa8 100644 --- a/var/spack/repos/builtin/packages/asciidoc/package.py +++ b/var/spack/repos/builtin/packages/asciidoc/package.py @@ -26,8 +26,9 @@ from spack import * class Asciidoc(Package): - """ A presentable text document format for writing articles, UNIX man + """A presentable text document format for writing articles, UNIX man pages and other small to medium sized documents.""" + homepage = "http://asciidoc.org" url = "http://downloads.sourceforge.net/project/asciidoc/asciidoc/8.6.9/asciidoc-8.6.9.tar.gz" diff --git a/var/spack/repos/builtin/packages/astyle/package.py b/var/spack/repos/builtin/packages/astyle/package.py index 815c184577..7b450352ec 100644 --- a/var/spack/repos/builtin/packages/astyle/package.py +++ b/var/spack/repos/builtin/packages/astyle/package.py @@ -26,10 +26,9 @@ from spack import * class Astyle(Package): - """ - A Free, Fast, and Small Automatic Formatter for C, C++, C++/CLI, - Objective-C, C#, and Java Source Code. - """ + """A Free, Fast, and Small Automatic Formatter for C, C++, C++/CLI, + Objective-C, C#, and Java Source Code.""" + homepage = "http://astyle.sourceforge.net/" url = "http://downloads.sourceforge.net/project/astyle/astyle/astyle%202.04/astyle_2.04_linux.tar.gz" diff --git a/var/spack/repos/builtin/packages/caliper/package.py b/var/spack/repos/builtin/packages/caliper/package.py index e5aed5ed65..770037ab49 100644 --- a/var/spack/repos/builtin/packages/caliper/package.py +++ b/var/spack/repos/builtin/packages/caliper/package.py @@ -26,11 +26,9 @@ from spack import * class Caliper(Package): - """ - Caliper is a generic context annotation system. It gives programmers the + """Caliper is a generic context annotation system. It gives programmers the ability to provide arbitrary program context information to (performance) - tools at runtime. - """ + tools at runtime.""" homepage = "https://github.com/LLNL/Caliper" url = "" diff --git a/var/spack/repos/builtin/packages/cfitsio/package.py b/var/spack/repos/builtin/packages/cfitsio/package.py index c68b3ffc0d..fc5ae0eef5 100644 --- a/var/spack/repos/builtin/packages/cfitsio/package.py +++ b/var/spack/repos/builtin/packages/cfitsio/package.py @@ -26,11 +26,11 @@ from spack import * class Cfitsio(Package): - """ - CFITSIO is a library of C and Fortran subroutines for reading and writing - data files in FITS (Flexible Image Transport System) data format. - """ + """CFITSIO is a library of C and Fortran subroutines for reading and writing + data files in FITS (Flexible Image Transport System) data format.""" + homepage = 'http://heasarc.gsfc.nasa.gov/fitsio/' + version('3.370', 'abebd2d02ba5b0503c633581e3bfa116') def url_for_version(self, v): diff --git a/var/spack/repos/builtin/packages/cityhash/package.py b/var/spack/repos/builtin/packages/cityhash/package.py index d6d7f51558..85d948cc57 100644 --- a/var/spack/repos/builtin/packages/cityhash/package.py +++ b/var/spack/repos/builtin/packages/cityhash/package.py @@ -27,6 +27,8 @@ from spack.util.environment import * class Cityhash(Package): + """CityHash, a family of hash functions for strings.""" + homepage = "https://github.com/google/cityhash" url = "https://github.com/google/cityhash" diff --git a/var/spack/repos/builtin/packages/cube/package.py b/var/spack/repos/builtin/packages/cube/package.py index 09237f87c3..55f71ef681 100644 --- a/var/spack/repos/builtin/packages/cube/package.py +++ b/var/spack/repos/builtin/packages/cube/package.py @@ -22,18 +22,15 @@ # License along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## - from spack import * class Cube(Package): - """ - Cube the profile viewer for Score-P and Scalasca profiles. It displays a + """Cube the profile viewer for Score-P and Scalasca profiles. It displays a multi-dimensional performance space consisting of the dimensions: - performance metric - call path - - system resource - """ + - system resource""" homepage = "http://www.scalasca.org/software/cube-4.x/download.html" url = "http://apps.fz-juelich.de/scalasca/releases/cube/4.2/dist/cube-4.2.3.tar.gz" diff --git a/var/spack/repos/builtin/packages/datamash/package.py b/var/spack/repos/builtin/packages/datamash/package.py index ef30c50ebe..65773d1978 100644 --- a/var/spack/repos/builtin/packages/datamash/package.py +++ b/var/spack/repos/builtin/packages/datamash/package.py @@ -26,10 +26,8 @@ from spack import * class Datamash(Package): - """ - GNU datamash is a command-line program which performs basic numeric, - textual and statistical operations on input textual data files. - """ + """GNU datamash is a command-line program which performs basic numeric, + textual and statistical operations on input textual data files.""" homepage = "https://www.gnu.org/software/datamash/" url = "http://ftp.gnu.org/gnu/datamash/datamash-1.0.5.tar.gz" diff --git a/var/spack/repos/builtin/packages/eigen/package.py b/var/spack/repos/builtin/packages/eigen/package.py index 36d6850c1a..2dd43c0113 100644 --- a/var/spack/repos/builtin/packages/eigen/package.py +++ b/var/spack/repos/builtin/packages/eigen/package.py @@ -26,11 +26,8 @@ from spack import * class Eigen(Package): - """ - Eigen is a C++ template library for linear algebra - - Matrices, vectors, numerical solvers, and related algorithms - """ + """Eigen is a C++ template library for linear algebra matrices, + vectors, numerical solvers, and related algorithms.""" homepage = 'http://eigen.tuxfamily.org/' url = 'http://bitbucket.org/eigen/eigen/get/3.2.7.tar.bz2' diff --git a/var/spack/repos/builtin/packages/espresso/package.py b/var/spack/repos/builtin/packages/espresso/package.py index 447964f286..1d8c9805b1 100644 --- a/var/spack/repos/builtin/packages/espresso/package.py +++ b/var/spack/repos/builtin/packages/espresso/package.py @@ -23,17 +23,15 @@ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## from spack import * - import os class Espresso(Package): - """ - QE is an integrated suite of Open-Source computer codes for + """QE is an integrated suite of Open-Source computer codes for electronic-structure calculations and materials modeling at the nanoscale. It is based on density-functional theory, plane - waves, and pseudopotentials. - """ + waves, and pseudopotentials.""" + homepage = 'http://quantum-espresso.org' url = 'http://www.qe-forge.org/gf/download/frsrelease/204/912/espresso-5.3.0.tar.gz' diff --git a/var/spack/repos/builtin/packages/gdal/package.py b/var/spack/repos/builtin/packages/gdal/package.py index 4fdfafc992..cf8dce4ef1 100644 --- a/var/spack/repos/builtin/packages/gdal/package.py +++ b/var/spack/repos/builtin/packages/gdal/package.py @@ -26,15 +26,13 @@ from spack import * class Gdal(Package): - """ - GDAL is a translator library for raster and vector geospatial + """GDAL is a translator library for raster and vector geospatial data formats that is released under an X/MIT style Open Source license by the Open Source Geospatial Foundation. As a library, it presents a single raster abstract data model and vector abstract data model to the calling application for all supported formats. It also comes with a variety of useful command line - utilities for data translation and processing - """ + utilities for data translation and processing""" homepage = "http://www.gdal.org/" url = "http://download.osgeo.org/gdal/2.0.2/gdal-2.0.2.tar.gz" diff --git a/var/spack/repos/builtin/packages/gdb/package.py b/var/spack/repos/builtin/packages/gdb/package.py index 9145009fa4..fdde649270 100644 --- a/var/spack/repos/builtin/packages/gdb/package.py +++ b/var/spack/repos/builtin/packages/gdb/package.py @@ -22,16 +22,14 @@ # License along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## - from spack import * class Gdb(Package): """GDB, the GNU Project debugger, allows you to see what is going on - `inside' another program while it executes -- or what another - program was doing at the moment it crashed. + 'inside' another program while it executes -- or what another + program was doing at the moment it crashed.""" - """ homepage = "https://www.gnu.org/software/gdb" url = "http://ftp.gnu.org/gnu/gdb/gdb-7.10.tar.gz" diff --git a/var/spack/repos/builtin/packages/gmsh/package.py b/var/spack/repos/builtin/packages/gmsh/package.py index 72f490e2f4..77e70d675c 100644 --- a/var/spack/repos/builtin/packages/gmsh/package.py +++ b/var/spack/repos/builtin/packages/gmsh/package.py @@ -26,15 +26,14 @@ from spack import * class Gmsh(Package): - """ - Gmsh is a free 3D finite element grid generator with a built-in CAD engine + """Gmsh is a free 3D finite element grid generator with a built-in CAD engine and post-processor. Its design goal is to provide a fast, light and user-friendly meshing tool with parametric input and advanced visualization capabilities. Gmsh is built around four modules: geometry, mesh, solver and post-processing. The specification of any input to these modules is done either interactively using the graphical user interface or in ASCII text - files using Gmsh's own scripting language. - """ + files using Gmsh's own scripting language.""" + homepage = 'http://gmsh.info' url = 'http://gmsh.info/src/gmsh-2.11.0-source.tgz' diff --git a/var/spack/repos/builtin/packages/lmod/package.py b/var/spack/repos/builtin/packages/lmod/package.py index 69965bc423..15555e1200 100644 --- a/var/spack/repos/builtin/packages/lmod/package.py +++ b/var/spack/repos/builtin/packages/lmod/package.py @@ -27,14 +27,13 @@ from glob import glob class Lmod(Package): - """ - Lmod is a Lua based module system that easily handles the MODULEPATH + """Lmod is a Lua based module system that easily handles the MODULEPATH Hierarchical problem. Environment Modules provide a convenient way to dynamically change the users' environment through modulefiles. This includes easily adding or removing directories to the PATH environment variable. Modulefiles for Library packages provide environment variables - that specify where the library and header files can be found. - """ + that specify where the library and header files can be found.""" + homepage = 'https://www.tacc.utexas.edu/research-development/tacc-projects/lmod' url = 'https://github.com/TACC/Lmod/archive/6.4.1.tar.gz' diff --git a/var/spack/repos/builtin/packages/lua-luafilesystem/package.py b/var/spack/repos/builtin/packages/lua-luafilesystem/package.py index a61b9dd675..dca7c247fa 100644 --- a/var/spack/repos/builtin/packages/lua-luafilesystem/package.py +++ b/var/spack/repos/builtin/packages/lua-luafilesystem/package.py @@ -26,15 +26,14 @@ from spack import * class LuaLuafilesystem(Package): - """ - LuaFileSystem is a Lua library developed to complement the set of + """LuaFileSystem is a Lua library developed to complement the set of functions related to file systems offered by the standard Lua distribution. LuaFileSystem offers a portable way to access the underlying directory structure and file attributes. - LuaFileSystem is free software and uses the same license as Lua 5.1 - """ + LuaFileSystem is free software and uses the same license as Lua 5.1""" + homepage = 'http://keplerproject.github.io/luafilesystem' url = 'https://github.com/keplerproject/luafilesystem/archive/v_1_6_3.tar.gz' diff --git a/var/spack/repos/builtin/packages/mxml/package.py b/var/spack/repos/builtin/packages/mxml/package.py index 113c48c18f..db2e7eb441 100644 --- a/var/spack/repos/builtin/packages/mxml/package.py +++ b/var/spack/repos/builtin/packages/mxml/package.py @@ -26,11 +26,9 @@ from spack import * class Mxml(Package): - """ - Mini-XML is a small XML library that you can use to read and write XML + """Mini-XML is a small XML library that you can use to read and write XML and XML-like data files in your application without requiring large - non-standard libraries - """ + non-standard libraries.""" homepage = "http://www.msweet.org" url = "http://www.msweet.org/files/project3/mxml-2.9.tar.gz" diff --git a/var/spack/repos/builtin/packages/ncdu/package.py b/var/spack/repos/builtin/packages/ncdu/package.py index 2147319d3f..a22671250e 100644 --- a/var/spack/repos/builtin/packages/ncdu/package.py +++ b/var/spack/repos/builtin/packages/ncdu/package.py @@ -26,13 +26,11 @@ from spack import * class Ncdu(Package): - """ - Ncdu is a disk usage analyzer with an ncurses interface. It is designed + """Ncdu is a disk usage analyzer with an ncurses interface. It is designed to find space hogs on a remote server where you don't have an entire gaphical setup available, but it is a useful tool even on regular desktop systems. Ncdu aims to be fast, simple and easy to use, and should be able - to run in any minimal POSIX-like environment with ncurses installed. - """ + to run in any minimal POSIX-like environment with ncurses installed.""" homepage = "http://dev.yorhel.nl/ncdu" url = "http://dev.yorhel.nl/download/ncdu-1.11.tar.gz" diff --git a/var/spack/repos/builtin/packages/opari2/package.py b/var/spack/repos/builtin/packages/opari2/package.py index e901f8ed39..b270931d0d 100644 --- a/var/spack/repos/builtin/packages/opari2/package.py +++ b/var/spack/repos/builtin/packages/opari2/package.py @@ -22,21 +22,18 @@ # License along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## - from spack import * class Opari2(Package): - """ - OPARI2 is a source-to-source instrumentation tool for OpenMP and hybrid + """OPARI2 is a source-to-source instrumentation tool for OpenMP and hybrid codes. It surrounds OpenMP directives and runtime library calls with calls to the POMP2 measurement interface. OPARI2 will provide you with a new initialization method that allows for multi-directory and parallel builds as well as the usage of pre-instrumented libraries. Furthermore, an efficient way of tracking parent-child relationships was added. Additionally, we extended OPARI2 to support instrumentation of OpenMP 3.0 - tied tasks. - """ + tied tasks.""" homepage = "http://www.vi-hps.org/projects/score-p" url = "http://www.vi-hps.org/upload/packages/opari2/opari2-1.1.2.tar.gz" diff --git a/var/spack/repos/builtin/packages/openjpeg/package.py b/var/spack/repos/builtin/packages/openjpeg/package.py index 1bc5b04f6f..6ba368a0f7 100644 --- a/var/spack/repos/builtin/packages/openjpeg/package.py +++ b/var/spack/repos/builtin/packages/openjpeg/package.py @@ -26,14 +26,13 @@ from spack import * class Openjpeg(Package): - """ - OpenJPEG is an open-source JPEG 2000 codec written in C language. + """OpenJPEG is an open-source JPEG 2000 codec written in C language. It has been developed in order to promote the use of JPEG 2000, a still-image compression standard from the Joint Photographic Experts Group (JPEG). Since April 2015, it is officially recognized by ISO/IEC and - ITU-T as a JPEG 2000 Reference Software. - """ + ITU-T as a JPEG 2000 Reference Software.""" + homepage = "https://github.com/uclouvain/openjpeg" url = "https://github.com/uclouvain/openjpeg/archive/version.2.1.tar.gz" diff --git a/var/spack/repos/builtin/packages/parallel/package.py b/var/spack/repos/builtin/packages/parallel/package.py index 2306dace55..189a5a6054 100644 --- a/var/spack/repos/builtin/packages/parallel/package.py +++ b/var/spack/repos/builtin/packages/parallel/package.py @@ -26,11 +26,9 @@ from spack import * class Parallel(Package): - """ - GNU parallel is a shell tool for executing jobs in parallel using + """GNU parallel is a shell tool for executing jobs in parallel using one or more computers. A job can be a single command or a small - script that has to be run for each of the lines in the input. - """ + script that has to be run for each of the lines in the input.""" homepage = "http://www.gnu.org/software/parallel/" url = "http://ftp.gnu.org/gnu/parallel/parallel-20160422.tar.bz2" diff --git a/var/spack/repos/builtin/packages/petsc/package.py b/var/spack/repos/builtin/packages/petsc/package.py index 66e1abdf1a..794633b929 100644 --- a/var/spack/repos/builtin/packages/petsc/package.py +++ b/var/spack/repos/builtin/packages/petsc/package.py @@ -27,11 +27,9 @@ from spack import * class Petsc(Package): - """ - PETSc is a suite of data structures and routines for the scalable + """PETSc is a suite of data structures and routines for the scalable (parallel) solution of scientific applications modeled by partial - differential equations. - """ + differential equations.""" homepage = "http://www.mcs.anl.gov/petsc/index.html" url = "http://ftp.mcs.anl.gov/pub/petsc/release-snapshots/petsc-3.5.3.tar.gz" diff --git a/var/spack/repos/builtin/packages/py-prettytable/package.py b/var/spack/repos/builtin/packages/py-prettytable/package.py index cf814733d4..9d3edd4abb 100644 --- a/var/spack/repos/builtin/packages/py-prettytable/package.py +++ b/var/spack/repos/builtin/packages/py-prettytable/package.py @@ -22,16 +22,14 @@ # License along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -from spack import depends_on, extends, version -from spack import Package +from spack import * class PyPrettytable(Package): - """ - PrettyTable is a simple Python library designed to make + """PrettyTable is a simple Python library designed to make it quick and easy to represent tabular data in visually - appealing ASCII tables - """ + appealing ASCII tables.""" + homepage = "https://code.google.com/archive/p/prettytable/" url = "https://pypi.python.org/packages/e0/a1/36203205f77ccf98f3c6cf17cf068c972e6458d7e58509ca66da949ca347/prettytable-0.7.2.tar.gz" diff --git a/var/spack/repos/builtin/packages/py-tuiview/package.py b/var/spack/repos/builtin/packages/py-tuiview/package.py index 672a3ee587..a372ec2b67 100644 --- a/var/spack/repos/builtin/packages/py-tuiview/package.py +++ b/var/spack/repos/builtin/packages/py-tuiview/package.py @@ -26,10 +26,9 @@ from spack import * class PyTuiview(Package): - """ - TuiView is a lightweight raster GIS with powerful raster attribute - table manipulation abilities. - """ + """TuiView is a lightweight raster GIS with powerful raster attribute + table manipulation abilities.""" + homepage = "https://bitbucket.org/chchrsc/tuiview" url = "https://bitbucket.org/chchrsc/tuiview/get/tuiview-1.1.7.tar.gz" diff --git a/var/spack/repos/builtin/packages/scorep/package.py b/var/spack/repos/builtin/packages/scorep/package.py index d40e08740f..d2efd23c37 100644 --- a/var/spack/repos/builtin/packages/scorep/package.py +++ b/var/spack/repos/builtin/packages/scorep/package.py @@ -22,16 +22,13 @@ # License along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## - from spack import * class Scorep(Package): - """ - The Score-P measurement infrastructure is a highly scalable and easy-to-use - tool suite for profiling, event tracing, and online analysis of HPC - applications. - """ + """The Score-P measurement infrastructure is a highly scalable and + easy-to-use tool suite for profiling, event tracing, and online analysis + of HPC applications.""" homepage = "http://www.vi-hps.org/projects/score-p" url = "http://www.vi-hps.org/upload/packages/scorep/scorep-1.2.3.tar.gz" diff --git a/var/spack/repos/builtin/packages/screen/package.py b/var/spack/repos/builtin/packages/screen/package.py index 17335603e6..eeaebcbfb7 100644 --- a/var/spack/repos/builtin/packages/screen/package.py +++ b/var/spack/repos/builtin/packages/screen/package.py @@ -26,10 +26,8 @@ from spack import * class Screen(Package): - """ - Screen is a full-screen window manager that multiplexes a physical - terminal between several processes, typically interactive shells. - """ + """Screen is a full-screen window manager that multiplexes a physical + terminal between several processes, typically interactive shells.""" homepage = "https://www.gnu.org/software/screen/" url = "http://ftp.gnu.org/gnu/screen/screen-4.3.1.tar.gz" diff --git a/var/spack/repos/builtin/packages/swiftsim/package.py b/var/spack/repos/builtin/packages/swiftsim/package.py index 636aa26bd2..ec5e74bd15 100644 --- a/var/spack/repos/builtin/packages/swiftsim/package.py +++ b/var/spack/repos/builtin/packages/swiftsim/package.py @@ -22,18 +22,14 @@ # License along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## - from spack import * - import llnl.util.tty as tty class Swiftsim(Package): - """ - SPH With Inter-dependent Fine-grained Tasking (SWIFT) provides + """SPH With Inter-dependent Fine-grained Tasking (SWIFT) provides astrophysicists with a state of the art framework to perform - particle based simulations. - """ + particle based simulations.""" homepage = 'http://icc.dur.ac.uk/swift/' url = 'http://gitlab.cosma.dur.ac.uk/swift/swiftsim/repository/archive.tar.gz?ref=v0.3.0' diff --git a/var/spack/repos/builtin/packages/tau/package.py b/var/spack/repos/builtin/packages/tau/package.py index a77df8d37c..c70bf31018 100644 --- a/var/spack/repos/builtin/packages/tau/package.py +++ b/var/spack/repos/builtin/packages/tau/package.py @@ -22,21 +22,17 @@ # License along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## - from spack import * - import os import os.path - from llnl.util.filesystem import join_path class Tau(Package): - """ - A portable profiling and tracing toolkit for performance + """A portable profiling and tracing toolkit for performance analysis of parallel programs written in Fortran, C, C++, UPC, - Java, Python. - """ + Java, Python.""" + homepage = "http://www.cs.uoregon.edu/research/tau" url = "https://www.cs.uoregon.edu/research/tau/tau_releases/tau-2.25.tar.gz" diff --git a/var/spack/repos/builtin/packages/xerces-c/package.py b/var/spack/repos/builtin/packages/xerces-c/package.py index d0c2d3d497..61c7625f25 100644 --- a/var/spack/repos/builtin/packages/xerces-c/package.py +++ b/var/spack/repos/builtin/packages/xerces-c/package.py @@ -26,12 +26,11 @@ from spack import * class XercesC(Package): - """ Xerces-C++ is a validating XML parser written in a portable subset of C++. - Xerces-C++ makes it easy to give your application the ability to read and - write XML data. A shared library is provided for parsing, generating, + """Xerces-C++ is a validating XML parser written in a portable subset of + C++. Xerces-C++ makes it easy to give your application the ability to read + and write XML data. A shared library is provided for parsing, generating, manipulating, and validating XML documents using the DOM, SAX, and SAX2 - APIs. - """ + APIs.""" homepage = "https://xerces.apache.org/xerces-c" url = "https://www.apache.org/dist/xerces/c/3/sources/xerces-c-3.1.4.tar.bz2" diff --git a/var/spack/repos/builtin/packages/zsh/package.py b/var/spack/repos/builtin/packages/zsh/package.py index 4c27cd3ec2..b65866b1fd 100644 --- a/var/spack/repos/builtin/packages/zsh/package.py +++ b/var/spack/repos/builtin/packages/zsh/package.py @@ -26,11 +26,10 @@ from spack import * class Zsh(Package): - """ - Zsh is a shell designed for interactive use, although it is also a powerful - scripting language. Many of the useful features of bash, ksh, and tcsh were - incorporated into zsh; many original features were added. - """ + """Zsh is a shell designed for interactive use, although it is also a + powerful scripting language. Many of the useful features of bash, ksh, and + tcsh were incorporated into zsh; many original features were added.""" + homepage = "http://www.zsh.org" url = "http://downloads.sourceforge.net/project/zsh/zsh/5.1.1/zsh-5.1.1.tar.gz" -- cgit v1.2.3-70-g09d2 From 02239c094e6634912ad75e416a12794154449b01 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Wed, 24 Aug 2016 10:42:23 -0500 Subject: Fix unexpected indentation in .rst files, #1612 --- lib/spack/docs/basic_usage.rst | 2 +- lib/spack/docs/packaging_guide.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/spack/docs/basic_usage.rst b/lib/spack/docs/basic_usage.rst index ef01f616cd..a3cec2cc8b 100644 --- a/lib/spack/docs/basic_usage.rst +++ b/lib/spack/docs/basic_usage.rst @@ -1793,7 +1793,7 @@ Spack currently needs to be run from a filesystem that supports ``flock`` locking semantics. Nearly all local filesystems and recent versions of NFS support this, but parallel filesystems may be mounted without ``flock`` support enabled. You can determine how your - filesystems are mounted with ``mount -p``. The output for a Lustre +filesystems are mounted with ``mount -p``. The output for a Lustre filesystem might look like this: .. code-block:: sh diff --git a/lib/spack/docs/packaging_guide.rst b/lib/spack/docs/packaging_guide.rst index c6ac6dd7ef..0c0554c24a 100644 --- a/lib/spack/docs/packaging_guide.rst +++ b/lib/spack/docs/packaging_guide.rst @@ -607,7 +607,7 @@ always simple, though, so this is not strictly enforced. When fetching from from the branch corresponding to the development version (often called ``master``,``trunk`` or ``dev``), it is recommended to call this version ``develop``. Spack has special treatment for this version so - that ``@develop`` will satisfy dependencies like +that ``@develop`` will satisfy dependencies like ``depends_on(abc, when="@x.y.z:")``. In other words, ``@develop`` is greater than any other version. The rationale is that certain features or options first appear in the development branch. Therefore if a package author -- cgit v1.2.3-70-g09d2 From 69e50595bf97fa9cd8b6b3c740f75802a6f0f0dd Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Wed, 24 Aug 2016 13:46:29 -0500 Subject: Fix docstrings for core Spack libraries, fixes #1612 --- lib/spack/spack/build_environment.py | 14 ++--- lib/spack/spack/compilers/clang.py | 17 +++-- lib/spack/spack/compilers/intel.py | 12 ++-- lib/spack/spack/compilers/nag.py | 6 +- lib/spack/spack/compilers/pgi.py | 8 +-- lib/spack/spack/config.py | 26 ++++---- lib/spack/spack/directives.py | 9 ++- lib/spack/spack/environment.py | 10 +-- lib/spack/spack/mirror.py | 45 ++++++------- lib/spack/spack/package.py | 112 +++++++++++++++++---------------- lib/spack/spack/repository.py | 22 ++++--- lib/spack/spack/spec.py | 6 +- lib/spack/spack/stage.py | 40 ++++++------ lib/spack/spack/test/git_fetch.py | 13 ++-- lib/spack/spack/test/hg_fetch.py | 13 ++-- lib/spack/spack/test/mirror.py | 7 ++- lib/spack/spack/test/provider_index.py | 2 +- lib/spack/spack/test/svn_fetch.py | 13 ++-- lib/spack/spack/util/pattern.py | 12 ++-- lib/spack/spack/version.py | 19 +++--- 20 files changed, 205 insertions(+), 201 deletions(-) diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 5affd3c7c5..9fbf0536a4 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -476,14 +476,12 @@ def setup_package(pkg, dirty=False): def fork(pkg, function, dirty=False): """Fork a child process to do part of a spack build. - Arguments: + :param pkg: pkg whose environemnt we should set up the forked process for. + :param function: arg-less function to run in the child process. + :param dirty: If True, do NOT clean the environment before building. - pkg -- pkg whose environemnt we should set up the - forked process for. - function -- arg-less function to run in the child process. - dirty -- If True, do NOT clean the environment before building. + Usage:: - Usage: def child_fun(): # do stuff build_env.fork(pkg, child_fun) @@ -496,8 +494,8 @@ def fork(pkg, function, dirty=False): If something goes wrong, the child process is expected to print the error and the parent process will exit with error as well. If things go well, the child exits and the parent - carries on. - """ + carries on.""" + try: pid = os.fork() except OSError as e: diff --git a/lib/spack/spack/compilers/clang.py b/lib/spack/spack/compilers/clang.py index 36c91f6670..f7f1bf106b 100644 --- a/lib/spack/spack/compilers/clang.py +++ b/lib/spack/spack/compilers/clang.py @@ -77,18 +77,17 @@ class Clang(Compiler): @classmethod def default_version(cls, comp): """The '--version' option works for clang compilers. - On most platforms, output looks like this:: + On most platforms, output looks like this:: - clang version 3.1 (trunk 149096) - Target: x86_64-unknown-linux-gnu - Thread model: posix + clang version 3.1 (trunk 149096) + Target: x86_64-unknown-linux-gnu + Thread model: posix - On Mac OS X, it looks like this: - - Apple LLVM version 7.0.2 (clang-700.1.81) - Target: x86_64-apple-darwin15.2.0 - Thread model: posix + On Mac OS X, it looks like this:: + Apple LLVM version 7.0.2 (clang-700.1.81) + Target: x86_64-apple-darwin15.2.0 + Thread model: posix """ if comp not in cpr._version_cache: compiler = Executable(comp) diff --git a/lib/spack/spack/compilers/intel.py b/lib/spack/spack/compilers/intel.py index 8531ecd19a..4f0c12140f 100644 --- a/lib/spack/spack/compilers/intel.py +++ b/lib/spack/spack/compilers/intel.py @@ -68,15 +68,15 @@ class Intel(Compiler): @classmethod def default_version(cls, comp): """The '--version' option seems to be the most consistent one - for intel compilers. Output looks like this:: + for intel compilers. Output looks like this:: - icpc (ICC) 12.1.5 20120612 - Copyright (C) 1985-2012 Intel Corporation. All rights reserved. + icpc (ICC) 12.1.5 20120612 + Copyright (C) 1985-2012 Intel Corporation. All rights reserved. - or:: + or:: - ifort (IFORT) 12.1.5 20120612 - Copyright (C) 1985-2012 Intel Corporation. All rights reserved. + ifort (IFORT) 12.1.5 20120612 + Copyright (C) 1985-2012 Intel Corporation. All rights reserved. """ return get_compiler_version( comp, '--version', r'\((?:IFORT|ICC)\) ([^ ]+)') diff --git a/lib/spack/spack/compilers/nag.py b/lib/spack/spack/compilers/nag.py index fdfc078b5e..7beb0efdf5 100644 --- a/lib/spack/spack/compilers/nag.py +++ b/lib/spack/spack/compilers/nag.py @@ -70,10 +70,10 @@ class Nag(Compiler): @classmethod def default_version(self, comp): """The '-V' option works for nag compilers. - Output looks like this:: + Output looks like this:: - NAG Fortran Compiler Release 6.0(Hibiya) Build 1037 - Product NPL6A60NA for x86-64 Linux + NAG Fortran Compiler Release 6.0(Hibiya) Build 1037 + Product NPL6A60NA for x86-64 Linux """ return get_compiler_version( comp, '-V', r'NAG Fortran Compiler Release ([0-9.]+)') diff --git a/lib/spack/spack/compilers/pgi.py b/lib/spack/spack/compilers/pgi.py index 0e4be6e9ba..37ffe44028 100644 --- a/lib/spack/spack/compilers/pgi.py +++ b/lib/spack/spack/compilers/pgi.py @@ -58,11 +58,11 @@ class Pgi(Compiler): @classmethod def default_version(cls, comp): """The '-V' option works for all the PGI compilers. - Output looks like this:: + Output looks like this:: - pgcc 15.10-0 64-bit target on x86-64 Linux -tp sandybridge - The Portland Group - PGI Compilers and Tools - Copyright (c) 2015, NVIDIA CORPORATION. All rights reserved. + pgcc 15.10-0 64-bit target on x86-64 Linux -tp sandybridge + The Portland Group - PGI Compilers and Tools + Copyright (c) 2015, NVIDIA CORPORATION. All rights reserved. """ return get_compiler_version( comp, '-V', r'pg[^ ]* ([^ ]+) \d\d\d?-bit target') diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index c90eff4229..ebddfb328f 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -24,26 +24,28 @@ ############################################################################## """This module implements Spack's configuration file handling. +========================= Configuration file scopes -=============================== +========================= When Spack runs, it pulls configuration data from several config directories, each of which contains configuration files. In Spack, there are two configuration scopes: - 1. ``site``: Spack loads site-wide configuration options from +1. ``site``: Spack loads site-wide configuration options from ``$(prefix)/etc/spack/``. - 2. ``user``: Spack next loads per-user configuration options from - ~/.spack/. +2. ``user``: Spack next loads per-user configuration options from + ``~/.spack/``. Spack may read configuration files from both of these locations. When configurations conflict, the user config options take precedence over the site configurations. Each configuration directory may contain several configuration files, such as compilers.yaml or mirrors.yaml. +========================= Configuration file format -=============================== +========================= Configuration files are formatted using YAML syntax. This format is implemented by libyaml (included with Spack as an external module), @@ -63,13 +65,13 @@ Config files are structured as trees, like this ``compiler`` section:: cc: /usr/local/bin/mpixlc ... -In this example, entries like ''compilers'' and ''xlc@12.1'' are used to +In this example, entries like "compilers" and "xlc@12.1" are used to categorize entries beneath them in the tree. At the root of the tree, -entries like ''cc'' and ''cxx'' are specified as name/value pairs. +entries like "cc" and "cxx" are specified as name/value pairs. ``config.get_config()`` returns these trees as nested dicts, but it strips the first level off. So, ``config.get_config('compilers')`` -would return something like this for the above example: +would return something like this for the above example:: { 'chaos_5_x86_64_ib' : { 'gcc@4.4.7' : @@ -84,8 +86,9 @@ would return something like this for the above example: Likewise, the ``mirrors.yaml`` file's first line must be ``mirrors:``, but ``get_config()`` strips that off too. +========== Precedence -=============================== +========== ``config.py`` routines attempt to recursively merge configuration across scopes. So if there are ``compilers.py`` files in both the @@ -99,7 +102,7 @@ first. Sometimes, it is useful to *completely* override a site setting with a user one. To accomplish this, you can use *two* colons at the end of -a key in a configuration file. For example, this: +a key in a configuration file. For example, this:: compilers:: chaos_5_x86_64_ib: @@ -114,9 +117,8 @@ a key in a configuration file. For example, this: ... Will make Spack take compilers *only* from the user configuration, and -the site configuration will be ignored. +the site configuration will be ignored.""" -""" import copy import os import re diff --git a/lib/spack/spack/directives.py b/lib/spack/spack/directives.py index 2a151e0374..86acd075cd 100644 --- a/lib/spack/spack/directives.py +++ b/lib/spack/spack/directives.py @@ -288,8 +288,7 @@ def variant(pkg, name, default=False, description=""): @directive('resources') def resource(pkg, **kwargs): - """ - Define an external resource to be fetched and staged when building the + """Define an external resource to be fetched and staged when building the package. Based on the keywords present in the dictionary the appropriate FetchStrategy will be used for the resource. Resources are fetched and staged in their own folder inside spack stage area, and then moved into @@ -298,11 +297,11 @@ def resource(pkg, **kwargs): List of recognized keywords: * 'when' : (optional) represents the condition upon which the resource is - needed + needed * 'destination' : (optional) path where to move the resource. This path - must be relative to the main package stage area. + must be relative to the main package stage area. * 'placement' : (optional) gives the possibility to fine tune how the - resource is moved into the main package stage area. + resource is moved into the main package stage area. """ when = kwargs.get('when', pkg.name) destination = kwargs.get('destination', "") diff --git a/lib/spack/spack/environment.py b/lib/spack/spack/environment.py index 613ece2f45..64863510c3 100644 --- a/lib/spack/spack/environment.py +++ b/lib/spack/spack/environment.py @@ -261,17 +261,13 @@ class EnvironmentModifications(object): @staticmethod def from_sourcing_files(*args, **kwargs): - """ - Creates an instance of EnvironmentModifications that, if executed, + """Creates an instance of EnvironmentModifications that, if executed, has the same effect on the environment as sourcing the files passed as parameters - Args: - *args: list of files to be sourced + :param \*args: list of files to be sourced + :rtype: instance of EnvironmentModifications""" - Returns: - instance of EnvironmentModifications - """ env = EnvironmentModifications() # Check if the files are actually there if not all(os.path.isfile(file) for file in args): diff --git a/lib/spack/spack/mirror.py b/lib/spack/spack/mirror.py index f053e4405f..1e759848ea 100644 --- a/lib/spack/spack/mirror.py +++ b/lib/spack/spack/mirror.py @@ -120,28 +120,29 @@ def suggest_archive_basename(resource): def create(path, specs, **kwargs): """Create a directory to be used as a spack mirror, and fill it with - package archives. - - Arguments: - path Path to create a mirror directory hierarchy in. - specs Any package versions matching these specs will be added - to the mirror. - - Keyword args: - no_checksum: If True, do not checkpoint when fetching (default False) - num_versions: Max number of versions to fetch per spec, - if spec is ambiguous (default is 0 for all of them) - - Return Value: - Returns a tuple of lists: (present, mirrored, error) - * present: Package specs that were already present. - * mirrored: Package specs that were successfully mirrored. - * error: Package specs that failed to mirror due to some error. - - This routine iterates through all known package versions, and - it creates specs for those versions. If the version satisfies any spec - in the specs list, it is downloaded and added to the mirror. - """ + package archives. + + Arguments: + path: Path to create a mirror directory hierarchy in. + specs: Any package versions matching these specs will be added \ + to the mirror. + + Keyword args: + no_checksum: If True, do not checkpoint when fetching (default False) + num_versions: Max number of versions to fetch per spec, \ + if spec is ambiguous (default is 0 for all of them) + + Return Value: + Returns a tuple of lists: (present, mirrored, error) + + * present: Package specs that were already present. + * mirrored: Package specs that were successfully mirrored. + * error: Package specs that failed to mirror due to some error. + + This routine iterates through all known package versions, and + it creates specs for those versions. If the version satisfies any spec + in the specs list, it is downloaded and added to the mirror.""" + # Make sure nothing is in the way. if os.path.isfile(path): raise MirrorError("%s already exists and is a file." % path) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 8c1204402a..3ffedbbad9 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -84,9 +84,9 @@ class Package(object): with the package itself. Packages are written in pure python. Packages are all submodules of spack.packages. If spack is installed - in $prefix, all of its python files are in $prefix/lib/spack. Most - of them are in the spack module, so all the packages live in - $prefix/lib/spack/spack/packages. + in ``$prefix``, all of its python files are in ``$prefix/lib/spack``. + Most of them are in the spack module, so all the packages live in + ``$prefix/lib/spack/spack/packages``. All you have to do to create a package is make a new subclass of Package in this directory. Spack automatically scans the python files there @@ -95,7 +95,7 @@ class Package(object): **An example package** Let's look at the cmake package to start with. This package lives in - $prefix/lib/spack/spack/packages/cmake.py: + ``$prefix/var/spack/repos/builtin/packages/cmake/package.py``: .. code-block:: python @@ -118,19 +118,21 @@ class Package(object): 1. The module name, ``cmake``. * User will refers to this name, e.g. 'spack install cmake'. - * Corresponds to the name of the file, 'cmake.py', and it can - include ``_``, ``-``, and numbers (it can even start with a + * It can include ``_``, ``-``, and numbers (it can even start with a number). 2. The class name, "Cmake". This is formed by converting `-` or ``_`` in the module name to camel case. If the name starts with a number, we prefix the class name with ``_``. Examples: - Module Name Class Name - foo_bar FooBar - docbook-xml DocbookXml - FooBar Foobar - 3proxy _3proxy + =========== ========== + Module Name Class Name + =========== ========== + foo_bar FooBar + docbook-xml DocbookXml + FooBar Foobar + 3proxy _3proxy + =========== ========== The class name is what spack looks for when it loads a package module. @@ -139,30 +141,30 @@ class Package(object): Aside from proper naming, here is the bare minimum set of things you need when you make a package: - homepage - informational URL, so that users know what they're - installing. + homepage: + informational URL, so that users know what they're + installing. - url or url_for_version(self, version) + url or url_for_version(self, version): If url, then the URL of the source archive that spack will fetch. If url_for_version(), then a method returning the URL required to fetch a particular version. - install() - This function tells spack how to build and install the - software it downloaded. + install(): + This function tells spack how to build and install the + software it downloaded. **Optional Attributes** You can also optionally add these attributes, if needed: - list_url + list_url: Webpage to scrape for available version strings. Default is the directory containing the tarball; use this if the default isn't correct so that invoking 'spack versions' will work for this package. - url_version(self, version) + url_version(self, version): When spack downloads packages at particular versions, it just converts version to string with str(version). Override this if your package needs special version formatting in its URL. boost @@ -179,12 +181,15 @@ class Package(object): **spack create** - Most software comes in nicely packaged tarballs, like this one: - http://www.cmake.org/files/v2.8/cmake-2.8.10.2.tar.gz + Most software comes in nicely packaged tarballs, like this one + + http://www.cmake.org/files/v2.8/cmake-2.8.10.2.tar.gz + Taking a page from homebrew, spack deduces pretty much everything it - needs to know from the URL above. If you simply type this: + needs to know from the URL above. If you simply type this:: spack create http://www.cmake.org/files/v2.8/cmake-2.8.10.2.tar.gz + Spack will download the tarball, generate an md5 hash, figure out the version and the name of the package from the URL, and create a new package file for you with all the names and attributes set correctly. @@ -216,7 +221,6 @@ class Package(object): you can just run configure or cmake without any additional arguments and it will find the dependencies automatically. - **The Install Function** The install function is designed so that someone not too terribly familiar @@ -241,13 +245,12 @@ class Package(object): add_commands_to_module() function in this class. This is where most of them are created and set on the module. - **Parallel Builds** By default, Spack will run make in parallel when you run make() in your install function. Spack figures out how many cores are available on - your system and runs make with -j. If you do not want this behavior, - you can explicitly mark a package not to use parallel make: + your system and runs make with -j. If you do not want this + behavior, you can explicitly mark a package not to use parallel make: .. code-block:: python @@ -257,14 +260,15 @@ class Package(object): ... This changes thd default behavior so that make is sequential. If you still - want to build some parts in parallel, you can do this in your install function: + want to build some parts in parallel, you can do this in your install + function: .. code-block:: python make(parallel=True) - Likewise, if you do not supply parallel = True in your Package, you can keep - the default parallel behavior and run make like this when you want a + Likewise, if you do not supply parallel = True in your Package, you can + keep the default parallel behavior and run make like this when you want a sequential build: .. code-block:: python @@ -295,14 +299,13 @@ class Package(object): p.do_restage() # removes the build directory and # re-expands the archive. - The convention used here is that a do_* function is intended to be called - internally by Spack commands (in spack.cmd). These aren't for package - writers to override, and doing so may break the functionality of the Package - class. + The convention used here is that a ``do_*`` function is intended to be + called internally by Spack commands (in spack.cmd). These aren't for + package writers to override, and doing so may break the functionality + of the Package class. Package creators override functions like install() (all of them do this), clean() (some of them do this), and others to provide custom behavior. - """ # # These are default values for instance variables. @@ -862,19 +865,21 @@ class Package(object): Package implementations should override install() to describe their build process. - Args: - keep_prefix -- Keep install prefix on failure. By default, destroys it. - keep_stage -- By default, stage is destroyed only if there are no - exceptions during build. Set to True to keep the stage - even with exceptions. - ignore_deps -- Don't install dependencies before installing this + :param keep_prefix: Keep install prefix on failure. By default, \ + destroys it. + :param keep_stage: By default, stage is destroyed only if there are \ + no exceptions during build. Set to True to keep the stage + even with exceptions. + :param ignore_deps: Don't install dependencies before installing this \ package - fake -- Don't really build -- install fake stub files instead. - skip_patch -- Skip patch stage of build if True. - verbose -- Display verbose build output (by default, suppresses it) - dirty -- Don't clean the build environment before installing. - make_jobs -- Number of make jobs to use for install. Default is ncpus - run_tests -- Runn tests within the package's install() + :param fake: Don't really build; install fake stub files instead. + :param skip_patch: Skip patch stage of build if True. + :param verbose: Display verbose build output (by default, suppresses \ + it) + :param dirty: Don't clean the build environment before installing. + :param make_jobs: Number of make jobs to use for install. Default is \ + ncpus + :param run_tests: Run tests within the package's install() """ if not self.spec.concrete: raise ValueError("Can only install concrete packages: %s." @@ -1110,20 +1115,20 @@ class Package(object): def setup_dependent_environment(self, spack_env, run_env, dependent_spec): """Set up the environment of packages that depend on this one. - This is similar to `setup_environment`, but it is used to + This is similar to ``setup_environment``, but it is used to modify the compile and runtime environments of packages that *depend* on this one. This gives packages like Python and others that follow the extension model a way to implement common environment or compile-time settings for dependencies. - By default, this delegates to self.setup_environment() + By default, this delegates to ``self.setup_environment()`` - Example : + Example: 1. Installing python modules generally requires - `PYTHONPATH` to point to the lib/pythonX.Y/site-packages - directory in the module's install prefix. This could - set that variable. + `PYTHONPATH` to point to the lib/pythonX.Y/site-packages + directory in the module's install prefix. This could + set that variable. Args: @@ -1142,7 +1147,6 @@ class Package(object): This is useful if there are some common steps to installing all extensions for a certain package. - """ self.setup_environment(spack_env, run_env) diff --git a/lib/spack/spack/repository.py b/lib/spack/spack/repository.py index 2d8dc39648..1d4cddaa0f 100644 --- a/lib/spack/spack/repository.py +++ b/lib/spack/spack/repository.py @@ -198,15 +198,19 @@ class RepoPath(object): def get_repo(self, namespace, default=NOT_PROVIDED): """Get a repository by namespace. - Arguments - namespace - Look up this namespace in the RepoPath, and return - it if found. - - Optional Arguments - default - If default is provided, return it when the namespace - isn't found. If not, raise an UnknownNamespaceError. + + Arguments: + + namespace: + + Look up this namespace in the RepoPath, and return it if found. + + Optional Arguments: + + default: + + If default is provided, return it when the namespace + isn't found. If not, raise an UnknownNamespaceError. """ fullspace = '%s.%s' % (self.super_namespace, namespace) if fullspace not in self.by_namespace: diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 316b7d6a8c..19733c8a41 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -2085,7 +2085,7 @@ class Spec(object): $# 7-char prefix of DAG hash with '-' prefix $$ $ - You can also use full-string versions, which elide the prefixes: + You can also use full-string versions, which elide the prefixes:: ${PACKAGE} Package name ${VERSION} Version @@ -2101,9 +2101,9 @@ class Spec(object): ${SPACK_INSTALL} The default spack install directory, ${SPACK_PREFIX}/opt - Optionally you can provide a width, e.g. $20_ for a 20-wide name. + Optionally you can provide a width, e.g. ``$20_`` for a 20-wide name. Like printf, you can provide '-' for left justification, e.g. - $-20_ for a left-justified name. + ``$-20_`` for a left-justified name. Anything else is copied verbatim into the output stream. diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index 7676cb9ab6..c0705a89c8 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -49,16 +49,14 @@ class Stage(object): some source code is downloaded and built before being installed. It handles fetching the source code, either as an archive to be expanded or by checking it out of a repository. A stage's - lifecycle looks like this: + lifecycle looks like this:: - ``` - with Stage() as stage: # Context manager creates and destroys the - # stage directory - stage.fetch() # Fetch a source archive into the stage. - stage.expand_archive() # Expand the source archive. - # Build and install the archive. (handled by - # user of Stage) - ``` + with Stage() as stage: # Context manager creates and destroys the + # stage directory + stage.fetch() # Fetch a source archive into the stage. + stage.expand_archive() # Expand the source archive. + # Build and install the archive. + # (handled by user of Stage) When used as a context manager, the stage is automatically destroyed if no exception is raised by the context. If an @@ -66,19 +64,17 @@ class Stage(object): destroyed, for potential reuse later. You can also use the stage's create/destroy functions manually, - like this: - - ``` - stage = Stage() - try: - stage.create() # Explicitly create the stage directory. - stage.fetch() # Fetch a source archive into the stage. - stage.expand_archive() # Expand the source archive. - # Build and install the archive. (handled by - # user of Stage) - finally: - stage.destroy() # Explicitly destroy the stage directory. - ``` + like this:: + + stage = Stage() + try: + stage.create() # Explicitly create the stage directory. + stage.fetch() # Fetch a source archive into the stage. + stage.expand_archive() # Expand the source archive. + # Build and install the archive. + # (handled by user of Stage) + finally: + stage.destroy() # Explicitly destroy the stage directory. If spack.use_tmp_stage is True, spack will attempt to create stages in a tmp directory. Otherwise, stages are created directly diff --git a/lib/spack/spack/test/git_fetch.py b/lib/spack/spack/test/git_fetch.py index 0d1a8fe949..7aff98cc54 100644 --- a/lib/spack/spack/test/git_fetch.py +++ b/lib/spack/spack/test/git_fetch.py @@ -56,12 +56,13 @@ class GitFetchTest(MockPackagesTest): def try_fetch(self, rev, test_file, args): """Tries to: - 1. Fetch the repo using a fetch strategy constructed with - supplied args. - 2. Check if the test_file is in the checked out repository. - 3. Assert that the repository is at the revision supplied. - 4. Add and remove some files, then reset the repo, and - ensure it's all there again. + + 1. Fetch the repo using a fetch strategy constructed with + supplied args. + 2. Check if the test_file is in the checked out repository. + 3. Assert that the repository is at the revision supplied. + 4. Add and remove some files, then reset the repo, and + ensure it's all there again. """ self.pkg.versions[ver('git')] = args diff --git a/lib/spack/spack/test/hg_fetch.py b/lib/spack/spack/test/hg_fetch.py index 44af6730a1..03e35ea093 100644 --- a/lib/spack/spack/test/hg_fetch.py +++ b/lib/spack/spack/test/hg_fetch.py @@ -52,12 +52,13 @@ class HgFetchTest(MockPackagesTest): def try_fetch(self, rev, test_file, args): """Tries to: - 1. Fetch the repo using a fetch strategy constructed with - supplied args. - 2. Check if the test_file is in the checked out repository. - 3. Assert that the repository is at the revision supplied. - 4. Add and remove some files, then reset the repo, and - ensure it's all there again. + + 1. Fetch the repo using a fetch strategy constructed with + supplied args. + 2. Check if the test_file is in the checked out repository. + 3. Assert that the repository is at the revision supplied. + 4. Add and remove some files, then reset the repo, and + ensure it's all there again. """ self.pkg.versions[ver('hg')] = args diff --git a/lib/spack/spack/test/mirror.py b/lib/spack/spack/test/mirror.py index d6d7b30b7c..f9ff190468 100644 --- a/lib/spack/spack/test/mirror.py +++ b/lib/spack/spack/test/mirror.py @@ -52,9 +52,10 @@ class MirrorTest(MockPackagesTest): def set_up_package(self, name, MockRepoClass, url_attr): """Set up a mock package to be mirrored. - Each package needs us to: - 1. Set up a mock repo/archive to fetch from. - 2. Point the package's version args at that repo. + Each package needs us to: + + 1. Set up a mock repo/archive to fetch from. + 2. Point the package's version args at that repo. """ # Set up packages to point at mock repos. spec = Spec(name) diff --git a/lib/spack/spack/test/provider_index.py b/lib/spack/spack/test/provider_index.py index 9847dd05a6..d785038899 100644 --- a/lib/spack/spack/test/provider_index.py +++ b/lib/spack/spack/test/provider_index.py @@ -24,7 +24,7 @@ ############################################################################## """Tests for provider index cache files. -Tests assume that mock packages provide this: +Tests assume that mock packages provide this:: {'blas': { blas: set([netlib-blas, openblas, openblas-with-lapack])}, diff --git a/lib/spack/spack/test/svn_fetch.py b/lib/spack/spack/test/svn_fetch.py index 9ef7593ed1..01ffc488a7 100644 --- a/lib/spack/spack/test/svn_fetch.py +++ b/lib/spack/spack/test/svn_fetch.py @@ -63,12 +63,13 @@ class SvnFetchTest(MockPackagesTest): def try_fetch(self, rev, test_file, args): """Tries to: - 1. Fetch the repo using a fetch strategy constructed with - supplied args. - 2. Check if the test_file is in the checked out repository. - 3. Assert that the repository is at the revision supplied. - 4. Add and remove some files, then reset the repo, and - ensure it's all there again. + + 1. Fetch the repo using a fetch strategy constructed with + supplied args. + 2. Check if the test_file is in the checked out repository. + 3. Assert that the repository is at the revision supplied. + 4. Add and remove some files, then reset the repo, and + ensure it's all there again. """ self.pkg.versions[ver('svn')] = args diff --git a/lib/spack/spack/util/pattern.py b/lib/spack/spack/util/pattern.py index c36445193c..2b7f06d46e 100644 --- a/lib/spack/spack/util/pattern.py +++ b/lib/spack/spack/util/pattern.py @@ -31,15 +31,15 @@ def composite(interface=None, method_list=None, container=list): """Returns a class decorator that patches a class adding all the methods it needs to be a composite for a given interface. - :param interface: class exposing the interface to which the composite - object must conform. Only non-private and non-special methods will be - taken into account + :param interface: class exposing the interface to which the composite \ + object must conform. Only non-private and non-special methods will \ + be taken into account :param method_list: names of methods that should be part of the composite - :param container: container for the composite object (default = list). - Must fulfill the MutableSequence contract. The composite class will expose - the container API to manage object composition + :param container: container for the composite object (default = list). \ + Must fulfill the MutableSequence contract. The composite class will \ + expose the container API to manage object composition :return: class decorator """ diff --git a/lib/spack/spack/version.py b/lib/spack/spack/version.py index e1311eb0d9..fb87966d04 100644 --- a/lib/spack/spack/version.py +++ b/lib/spack/spack/version.py @@ -155,11 +155,11 @@ class Version(object): @coerced def satisfies(self, other): - """A Version 'satisfies' another if it is at least as specific and has a - common prefix. e.g., we want gcc@4.7.3 to satisfy a request for - gcc@4.7 so that when a user asks to build with gcc@4.7, we can find - a suitable compiler. - """ + """A Version 'satisfies' another if it is at least as specific and has + a common prefix. e.g., we want gcc@4.7.3 to satisfy a request for + gcc@4.7 so that when a user asks to build with gcc@4.7, we can find + a suitable compiler.""" + nself = len(self.version) nother = len(other.version) return nother <= nself and self.version[:nother] == other.version @@ -388,12 +388,12 @@ class VersionRange(object): @coerced def satisfies(self, other): - """ - A VersionRange satisfies another if some version in this range + """A VersionRange satisfies another if some version in this range would satisfy some version in the other range. To do this it must either: - a) Overlap with the other range - b) The start of this range satisfies the end of the other range. + + a) Overlap with the other range + b) The start of this range satisfies the end of the other range. This is essentially the same as overlaps(), but overlaps assumes that its arguments are specific. That is, 4.7 is interpreted as @@ -401,6 +401,7 @@ class VersionRange(object): by 4.7.3.5, etc. Rationale: + If a user asks for gcc@4.5:4.7, and a package is only compatible with gcc@4.7.3:4.8, then that package should be able to build under the constraints. Just using overlaps() would not work here. -- cgit v1.2.3-70-g09d2 From 867a92f08395aa2ceab3ba58f2fe229682b8c2c9 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Wed, 24 Aug 2016 16:28:17 -0500 Subject: Fixes #1618, Problems with spack package-list --- lib/spack/spack/cmd/package-list.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/lib/spack/spack/cmd/package-list.py b/lib/spack/spack/cmd/package-list.py index 9ed42de823..42f408af96 100644 --- a/lib/spack/spack/cmd/package-list.py +++ b/lib/spack/spack/cmd/package-list.py @@ -32,8 +32,8 @@ description = "Print a list of all packages in reStructuredText." def github_url(pkg): """Link to a package file on github.""" - url = "https://github.com/llnl/spack/blob/master/var/spack/packages/%s/package.py" - return (url % pkg.name) + url = "https://github.com/LLNL/spack/blob/develop/var/spack/repos/builtin/packages/{0}/package.py" + return url.format(pkg.name) def rst_table(elts): @@ -51,35 +51,41 @@ def print_rst_package_list(): print ".. _package-list:" print + print "============" print "Package List" - print "==================" - + print "============" + print print "This is a list of things you can install using Spack. It is" print "automatically generated based on the packages in the latest Spack" print "release." print - print "Spack currently has %d mainline packages:" % len(pkgs) print print rst_table("`%s`_" % p for p in pkg_names) print - print "-----" # Output some text for each package. for pkg in pkgs: + print "-----" print print ".. _%s:" % pkg.name print + # Must be at least 2 long, breaks for single letter packages like R. + print "-" * max(len(pkg.name), 2) print pkg.name - print "-" * len(pkg.name) - print "Links:" + print "-" * max(len(pkg.name), 2) + print + print "Homepage:" print " * `%s <%s>`__" % (cgi.escape(pkg.homepage), pkg.homepage) + print + print "Spack package:" print " * `%s/package.py <%s>`__" % (pkg.name, github_url(pkg)) print if pkg.versions: print "Versions:" print " " + ", ".join(str(v) for v in reversed(sorted(pkg.versions))) + print for deptype in spack.alldeps: deps = pkg.dependencies_of_type(deptype) @@ -92,7 +98,6 @@ def print_rst_package_list(): print "Description:" print pkg.format_doc(indent=2) print - print "-----" def package_list(parser, args): -- cgit v1.2.3-70-g09d2 From f78134efa54baef2c1ef13167bf49859c84d3643 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Wed, 24 Aug 2016 16:55:00 -0500 Subject: Fixes #1620, Missing references --- lib/spack/docs/configuration.rst | 2 ++ lib/spack/docs/packaging_guide.rst | 7 ++++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/spack/docs/configuration.rst b/lib/spack/docs/configuration.rst index bce6005a7c..7545acd62a 100644 --- a/lib/spack/docs/configuration.rst +++ b/lib/spack/docs/configuration.rst @@ -130,6 +130,8 @@ The ``buildable`` does not need to be paired with external packages. It could also be used alone to forbid packages that may be buggy or otherwise undesirable. +.. _concretization-preferences: + -------------------------- Concretization Preferences -------------------------- diff --git a/lib/spack/docs/packaging_guide.rst b/lib/spack/docs/packaging_guide.rst index 0c0554c24a..eae928ec76 100644 --- a/lib/spack/docs/packaging_guide.rst +++ b/lib/spack/docs/packaging_guide.rst @@ -1833,6 +1833,8 @@ running ``spack spec``. For example: This is useful when you want to know exactly what Spack will do when you ask for a particular spec. +.. _concretization-policies: + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ ``Concretization Policies`` ^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -1842,8 +1844,7 @@ be concretized on their system. For example, one user may prefer packages built with OpenMPI and the Intel compiler. Another user may prefer packages be built with MVAPICH and GCC. -See the `documentation in the config section `_ -for more details. +See the :ref:`concretization-preferences` section for more details. .. _install-method: @@ -2465,7 +2466,7 @@ Callable wrappers also allow spack to provide some special features. For example, in Spack, ``make`` is parallel by default, and Spack figures out the number of cores on your machine and passes an appropriate value for ``-j`` when it calls ``make`` (see the -``parallel`` package attribute under :ref:`metadata `). In +``parallel`` `package attribute `). In a package file, you can supply a keyword argument, ``parallel=False``, to the ``make`` wrapper to disable parallel make. In the ``libelf`` package, this allows us to avoid race conditions in the library's -- cgit v1.2.3-70-g09d2 From f856952728811c267669962573aea15b9223c1e9 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Wed, 24 Aug 2016 16:56:59 -0500 Subject: Always run Documentation tests --- share/spack/qa/run-doc-tests | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/share/spack/qa/run-doc-tests b/share/spack/qa/run-doc-tests index 102eb72d5e..0b23bd9d3d 100755 --- a/share/spack/qa/run-doc-tests +++ b/share/spack/qa/run-doc-tests @@ -28,9 +28,6 @@ deps=( # Check for dependencies "$QA_DIR/check_dependencies" "${deps[@]}" || exit 1 -# Gather array of changed files -changed=($("$QA_DIR/changed_files" lib/spack/docs)) - # Move to documentation directory # Allows script to be run from anywhere cd "$DOC_DIR" @@ -38,11 +35,6 @@ cd "$DOC_DIR" # Cleanup temporary files upon exit or when script is killed trap 'make clean --silent' EXIT SIGINT SIGTERM -# Only run tests if documentation was updated -if [[ "${changed[@]}" ]]; then - # Treat warnings as fatal errors - make SPHINXOPTS=-W -else - echo "No documentation was modified." -fi +# Treat warnings as fatal errors +make SPHINXOPTS=-W -- cgit v1.2.3-70-g09d2 From 6795f4ca12369b9aee8700b65f4150f60cde460b Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Fri, 26 Aug 2016 16:41:00 -0500 Subject: Use console instead of shell syntax lexer, fixes #1634 --- lib/spack/docs/basic_usage.rst | 831 ++++++++++++++++++------------------ lib/spack/docs/configuration.rst | 62 +-- lib/spack/docs/developer_guide.rst | 104 +++-- lib/spack/docs/features.rst | 26 +- lib/spack/docs/getting_started.rst | 11 +- lib/spack/docs/index.rst | 2 +- lib/spack/docs/mirrors.rst | 104 ++--- lib/spack/docs/packaging_guide.rst | 832 +++++++++++++++++-------------------- 8 files changed, 970 insertions(+), 1002 deletions(-) diff --git a/lib/spack/docs/basic_usage.rst b/lib/spack/docs/basic_usage.rst index a3cec2cc8b..fd701789ec 100644 --- a/lib/spack/docs/basic_usage.rst +++ b/lib/spack/docs/basic_usage.rst @@ -8,14 +8,17 @@ The ``spack`` command has many *subcommands*. You'll only need a small subset of them for typical usage. Note that Spack colorizes output. ``less -R`` should be used with -Spack to maintain this colorization. E.g.:: +Spack to maintain this colorization. E.g.: - spack find | less -R +.. code-block:: console -It is recommend that the following be put in your ``.bashrc`` file:: + $ spack find | less -R - alias less='less -R' +It is recommend that the following be put in your ``.bashrc`` file: +.. code-block:: sh + + alias less='less -R' -------------------------- Listing available packages @@ -114,18 +117,18 @@ Installing and uninstalling For example, To install the latest version of the ``mpileaks`` package, you might type this: -.. code-block:: sh +.. code-block:: console $ spack install mpileaks -If `mpileaks` depends on other packages, Spack will install the +If ``mpileaks`` depends on other packages, Spack will install the dependencies first. It then fetches the ``mpileaks`` tarball, expands it, verifies that it was downloaded without errors, builds it, and installs it in its own directory under ``$SPACK_ROOT/opt``. You'll see a number of messages from spack, a lot of build output, and a message that the packages is installed: -.. code-block:: sh +.. code-block:: console $ spack install mpileaks ==> Installing mpileaks @@ -155,7 +158,7 @@ Building a specific version Spack can also build *specific versions* of a package. To do this, just add ``@`` after the package name, followed by a version: -.. code-block:: sh +.. code-block:: console $ spack install mpich@3.0.4 @@ -174,8 +177,8 @@ two configurations of the same version of a package, one built with boost This can all be done on the command line using the *spec* syntax. Spack calls the descriptor used to refer to a particular package configuration a **spec**. In the commands above, ``mpileaks`` and -``mpileaks@3.0.4``. We'll talk more about how you can use them to -customize an installation in :ref:`sec-specs`. +``mpileaks@3.0.4`` are both valid *specs*. We'll talk more about how +you can use them to customize an installation in :ref:`sec-specs`. .. _spack-uninstall: @@ -183,53 +186,56 @@ customize an installation in :ref:`sec-specs`. ``spack uninstall`` ^^^^^^^^^^^^^^^^^^^ -To uninstall a package, type ``spack uninstall ``. This will ask the user for -confirmation, and in case will completely remove the directory in which the package was installed. +To uninstall a package, type ``spack uninstall ``. This will ask +the user for confirmation, and in case will completely remove the directory +in which the package was installed. -.. code-block:: sh +.. code-block:: console - spack uninstall mpich + $ spack uninstall mpich If there are still installed packages that depend on the package to be uninstalled, spack will refuse to uninstall it. To uninstall a package and every package that depends on it, you may give the -`--dependents` option. +``--dependents`` option. -.. code-block:: sh +.. code-block:: console - spack uninstall --dependents mpich + $ spack uninstall --dependents mpich -will display a list of all the packages that depend on `mpich` and, upon confirmation, -will uninstall them in the right order. +will display a list of all the packages that depend on ``mpich`` and, upon +confirmation, will uninstall them in the right order. -A line like +A command like -.. code-block:: sh +.. code-block:: console - spack uninstall mpich + $ spack uninstall mpich -may be ambiguous, if multiple ``mpich`` configurations are installed. For example, if both -``mpich@3.0.2`` and ``mpich@3.1`` are installed, ``mpich`` could refer -to either one. Because it cannot determine which one to uninstall, -Spack will ask you either to provide a version number to remove the -ambiguity or use the ``--all`` option to uninstall all of the matching packages. +may be ambiguous if multiple ``mpich`` configurations are installed. +For example, if both ``mpich@3.0.2`` and ``mpich@3.1`` are installed, +``mpich`` could refer to either one. Because it cannot determine which +one to uninstall, Spack will ask you either to provide a version number +to remove the ambiguity or use the ``--all`` option to uninstall all of +the matching packages. -You may force uninstall a package with the `--force` option +You may force uninstall a package with the ``--force`` option -.. code-block:: sh +.. code-block:: console - spack uninstall --force mpich + $ spack uninstall --force mpich -but you risk breaking other installed packages. In general, it is safer to remove dependent -packages *before* removing their dependencies or use the `--dependents` option. +but you risk breaking other installed packages. In general, it is safer to +remove dependent packages *before* removing their dependencies or use the +``--dependents`` option. ------------------------- Seeing installed packages ------------------------- We know that ``spack list`` shows you the names of available packages, -but how do you figure out which are installed? +but how do you figure out which are already installed? .. _spack-find: @@ -244,7 +250,7 @@ of the same package with different specs. Running ``spack find`` with no arguments lists installed packages: -.. code-block:: sh +.. code-block:: console $ spack find ==> 74 installed packages. @@ -283,17 +289,18 @@ only shows the version of installed packages. ``spack find`` can filter the package list based on the package name, spec, or a number of properties of their installation status. For example, missing -dependencies of a spec can be shown with ``-m``, packages which were +dependencies of a spec can be shown with ``--missing``, packages which were explicitly installed with ``spack install `` can be singled out with -``-e`` and those which have been pulled in only as dependencies with ``-E``. +``--explicit`` and those which have been pulled in only as dependencies with +``--implicit``. In some cases, there may be different configurations of the *same* version of a package installed. For example, there are two -installations of of ``libdwarf@20130729`` above. We can look at them -in more detail using ``spack find -d``, and by asking only to show +installations of ``libdwarf@20130729`` above. We can look at them +in more detail using ``spack find --deps``, and by asking only to show ``libdwarf`` packages: -.. code-block:: sh +.. code-block:: console $ spack find --deps libdwarf ==> 2 installed packages. @@ -307,25 +314,25 @@ Now we see that the two instances of ``libdwarf`` depend on *different* versions of ``libelf``: 0.8.12 and 0.8.13. This view can become complicated for packages with many dependencies. If you just want to know whether two packages' dependencies differ, you can use -``spack find -l``: +``spack find --long``: -.. code-block:: sh +.. code-block:: console - $ spack find -l libdwarf + $ spack find --long libdwarf ==> 2 installed packages. -- linux-debian7-x86_64 / gcc@4.4.7 -------------------------------- libdwarf@20130729-d9b90962 libdwarf@20130729-b52fac98 -Now the ``libwarf`` installs have hashes after their names. These are +Now the ``libdwarf`` installs have hashes after their names. These are hashes over all of the dependencies of each package. If the hashes are the same, then the packages have the same dependency configuration. If you want to know the path where each package is installed, you can -use ``spack find -p``: +use ``spack find --paths``: -.. code-block:: sh +.. code-block:: console - $ spack find -p + $ spack find --paths ==> 74 installed packages. -- linux-debian7-x86_64 / gcc@4.4.7 -------------------------------- ImageMagick@6.8.9-10 /home/gamblin2/spack/opt/linux-debian7-x86_64/gcc@4.4.7/ImageMagick@6.8.9-10-4df950dd @@ -340,9 +347,9 @@ use ``spack find -p``: And, finally, you can restrict your search to a particular package by supplying its name: -.. code-block:: sh +.. code-block:: console - $ spack find -p libelf + $ spack find --paths libelf -- linux-debian7-x86_64 / gcc@4.4.7 -------------------------------- libelf@0.8.11 /home/gamblin2/spack/opt/linux-debian7-x86_64/gcc@4.4.7/libelf@0.8.11 libelf@0.8.12 /home/gamblin2/spack/opt/linux-debian7-x86_64/gcc@4.4.7/libelf@0.8.12 @@ -353,7 +360,7 @@ by supplying its name: package. If you want to find only libelf versions greater than version 0.8.12, you could say: -.. code-block:: sh +.. code-block:: console $ spack find libelf@0.8.12: -- linux-debian7-x86_64 / gcc@4.4.7 -------------------------------- @@ -362,17 +369,16 @@ package. If you want to find only libelf versions greater than version Finding just the versions of libdwarf built with a particular version of libelf would look like this: -.. code-block:: sh +.. code-block:: console - $ spack find -l libdwarf ^libelf@0.8.12 + $ spack find --long libdwarf ^libelf@0.8.12 ==> 1 installed packages. -- linux-debian7-x86_64 / gcc@4.4.7 -------------------------------- libdwarf@20130729-d9b90962 We can also search for packages that have a certain attribute. For example, -``spack find -l libdwarf +debug`` will show only installations of libdwarf -with the 'debug' compile-time option enabled, while ``spack find -l +debug`` -will find every installed package with a 'debug' compile-time option enabled. +``spack find libdwarf +debug`` will show only installations of libdwarf +with the 'debug' compile-time option enabled. The full spec syntax is discussed in detail in :ref:`sec-specs`. @@ -385,7 +391,7 @@ Compiler configuration Spack has the ability to build packages with multiple compilers and compiler versions. Spack searches for compilers on your machine automatically the first time it is run. It does this by inspecting -your path. +your ``PATH``. .. _spack-compilers: @@ -394,24 +400,26 @@ your path. ^^^^^^^^^^^^^^^^^^^ You can see which compilers spack has found by running ``spack -compilers`` or ``spack compiler list``:: - - $ spack compilers - ==> Available compilers - -- gcc --------------------------------------------------------- - gcc@4.9.0 gcc@4.8.0 gcc@4.7.0 gcc@4.6.2 gcc@4.4.7 - gcc@4.8.2 gcc@4.7.1 gcc@4.6.3 gcc@4.6.1 gcc@4.1.2 - -- intel ------------------------------------------------------- - intel@15.0.0 intel@14.0.0 intel@13.0.0 intel@12.1.0 intel@10.0 - intel@14.0.3 intel@13.1.1 intel@12.1.5 intel@12.0.4 intel@9.1 - intel@14.0.2 intel@13.1.0 intel@12.1.3 intel@11.1 - intel@14.0.1 intel@13.0.1 intel@12.1.2 intel@10.1 - -- clang ------------------------------------------------------- - clang@3.4 clang@3.3 clang@3.2 clang@3.1 - -- pgi --------------------------------------------------------- - pgi@14.3-0 pgi@13.2-0 pgi@12.1-0 pgi@10.9-0 pgi@8.0-1 - pgi@13.10-0 pgi@13.1-1 pgi@11.10-0 pgi@10.2-0 pgi@7.1-3 - pgi@13.6-0 pgi@12.8-0 pgi@11.1-0 pgi@9.0-4 pgi@7.0-6 +compilers`` or ``spack compiler list``: + +.. code-block:: console + + $ spack compilers + ==> Available compilers + -- gcc --------------------------------------------------------- + gcc@4.9.0 gcc@4.8.0 gcc@4.7.0 gcc@4.6.2 gcc@4.4.7 + gcc@4.8.2 gcc@4.7.1 gcc@4.6.3 gcc@4.6.1 gcc@4.1.2 + -- intel ------------------------------------------------------- + intel@15.0.0 intel@14.0.0 intel@13.0.0 intel@12.1.0 intel@10.0 + intel@14.0.3 intel@13.1.1 intel@12.1.5 intel@12.0.4 intel@9.1 + intel@14.0.2 intel@13.1.0 intel@12.1.3 intel@11.1 + intel@14.0.1 intel@13.0.1 intel@12.1.2 intel@10.1 + -- clang ------------------------------------------------------- + clang@3.4 clang@3.3 clang@3.2 clang@3.1 + -- pgi --------------------------------------------------------- + pgi@14.3-0 pgi@13.2-0 pgi@12.1-0 pgi@10.9-0 pgi@8.0-1 + pgi@13.10-0 pgi@13.1-1 pgi@11.10-0 pgi@10.2-0 pgi@7.1-3 + pgi@13.6-0 pgi@12.8-0 pgi@11.1-0 pgi@9.0-4 pgi@7.0-6 Any of these compilers can be used to build Spack packages. More on how this is done is in :ref:`sec-specs`. @@ -432,21 +440,25 @@ An alias for ``spack compiler find``. If you do not see a compiler in this list, but you want to use it with Spack, you can simply run ``spack compiler find`` with the path to -where the compiler is installed. For example:: +where the compiler is installed. For example: + +.. code-block:: console - $ spack compiler find /usr/local/tools/ic-13.0.079 - ==> Added 1 new compiler to /Users/gamblin2/.spack/compilers.yaml - intel@13.0.079 + $ spack compiler find /usr/local/tools/ic-13.0.079 + ==> Added 1 new compiler to /Users/gamblin2/.spack/compilers.yaml + intel@13.0.079 Or you can run ``spack compiler find`` with no arguments to force auto-detection. This is useful if you do not know where compilers are installed, but you know that new compilers have been added to your -``PATH``. For example, using dotkit, you might do this:: +``PATH``. For example, using dotkit, you might do this: - $ module load gcc-4.9.0 - $ spack compiler find - ==> Added 1 new compiler to /Users/gamblin2/.spack/compilers.yaml - gcc@4.9.0 +.. code-block:: console + + $ module load gcc-4.9.0 + $ spack compiler find + ==> Added 1 new compiler to /Users/gamblin2/.spack/compilers.yaml + gcc@4.9.0 This loads the environment module for gcc-4.9.0 to add it to ``PATH``, and then it adds the compiler to Spack. @@ -458,7 +470,9 @@ This loads the environment module for gcc-4.9.0 to add it to ^^^^^^^^^^^^^^^^^^^^^^^ If you want to see specifics on a particular compiler, you can run -``spack compiler info`` on it:: +``spack compiler info`` on it: + +.. code-block:: console $ spack compiler info intel@15 intel@15.0.0: @@ -466,9 +480,11 @@ If you want to see specifics on a particular compiler, you can run cxx = /usr/local/bin/icpc-15.0.090 f77 = /usr/local/bin/ifort-15.0.090 fc = /usr/local/bin/ifort-15.0.090 + modules = [] + operating system = centos6 This shows which C, C++, and Fortran compilers were detected by Spack. -Notice also that we didn\'t have to be too specific about the +Notice also that we didn't have to be too specific about the version. We just said ``intel@15``, and information about the only matching Intel compiler was displayed. @@ -480,48 +496,54 @@ If auto-detection fails, you can manually configure a compiler by editing your ``~/.spack/compilers.yaml`` file. You can do this by running ``spack config edit compilers``, which will open the file in your ``$EDITOR``. -Each compiler configuration in the file looks like this:: +Each compiler configuration in the file looks like this: - ... - compilers: - - compiler: - modules = [] - operating_system: OS - paths: - cc: /usr/local/bin/icc-15.0.024-beta - cxx: /usr/local/bin/icpc-15.0.024-beta - f77: /usr/local/bin/ifort-15.0.024-beta - fc: /usr/local/bin/ifort-15.0.024-beta +.. code-block:: yaml - spec: intel@15.0.0: + compilers: + - compiler: + modules = [] + operating_system: centos6 + paths: + cc: /usr/local/bin/icc-15.0.024-beta + cxx: /usr/local/bin/icpc-15.0.024-beta + f77: /usr/local/bin/ifort-15.0.024-beta + fc: /usr/local/bin/ifort-15.0.024-beta + spec: intel@15.0.0: For compilers, like ``clang``, that do not support Fortran, put -``None`` for ``f77`` and ``fc``:: +``None`` for ``f77`` and ``fc``: + +.. code-block:: yaml - clang@3.3svn: - cc: /usr/bin/clang - cxx: /usr/bin/clang++ - f77: None - fc: None + paths: + cc: /usr/bin/clang + cxx: /usr/bin/clang++ + f77: None + fc: None + spec: clang@3.3svn: Once you save the file, the configured compilers will show up in the list displayed by ``spack compilers``. You can also add compiler flags to manually configured compilers. The valid flags are ``cflags``, ``cxxflags``, ``fflags``, ``cppflags``, -``ldflags``, and ``ldlibs``. For example:: - - ... - compilers: - - compiler: - ... - intel@15.0.0: - cc: /usr/local/bin/icc-15.0.024-beta - cxx: /usr/local/bin/icpc-15.0.024-beta - f77: /usr/local/bin/ifort-15.0.024-beta - fc: /usr/local/bin/ifort-15.0.024-beta - cppflags: -O3 -fPIC - ... +``ldflags``, and ``ldlibs``. For example: + +.. code-block:: yaml + + compilers: + - compiler: + modules = [] + operating_system: OS + paths: + cc: /usr/local/bin/icc-15.0.024-beta + cxx: /usr/local/bin/icpc-15.0.024-beta + f77: /usr/local/bin/ifort-15.0.024-beta + fc: /usr/local/bin/ifort-15.0.024-beta + parameters: + cppflags: -O3 -fPIC + spec: intel@15.0.0: These flags will be treated by spack as if they were enterred from the command line each time this compiler is used. The compiler wrappers @@ -541,10 +563,12 @@ Spack, that descriptor is called a *spec*. Spack uses specs to refer to a particular build configuration (or configurations) of a package. Specs are more than a package name and a version; you can use them to specify the compiler, compiler version, architecture, compile options, -and dependency options for a build. In this section, we\'ll go over +and dependency options for a build. In this section, we'll go over the full syntax of specs. -Here is an example of a much longer spec than we\'ve seen thus far:: +Here is an example of a much longer spec than we've seen thus far: + +.. code-block:: none mpileaks @1.2:1.4 %gcc@4.7.5 +debug -qt arch=bgq_os ^callpath @1.1 %gcc@4.7.2 @@ -638,7 +662,7 @@ of dependencies within a spec is that they appear *after* the root package. For example, these two specs represent exactly the same configuration: -.. code-block:: sh +.. code-block:: none mpileaks ^callpath@1.0 ^libelf@0.8.3 mpileaks ^libelf@0.8.3 ^callpath@1.0 @@ -712,7 +736,7 @@ makes available. Variants can be specified using a flexible parameter syntax ``name=``. For example, ``spack install libelf debug=True`` will install libelf build with debug flags. The names of particular variants available for a package depend on -what was provided by the package author. ``spack into `` will +what was provided by the package author. ``spack info `` will provide information on what build variants are available. For compatibility with earlier versions, variants which happen to be @@ -723,7 +747,7 @@ compile time option for the libelf package. Depending on the package a variant may have any default value. For ``libelf`` here, ``debug`` is ``False`` by default, and we turned it on -with ``debug=True`` or ``+debug``. If a package is ``True`` by default +with ``debug=True`` or ``+debug``. If a variant is ``True`` by default you can turn it off by either adding ``-name`` or ``~name`` to the spec. There are two syntaxes here because, depending on context, ``~`` and @@ -774,9 +798,9 @@ command line. From within python files, the same spec would be specified handles the quote symbols. The six compiler flags are injected in the order of implicit make commands -in gnu autotools. If all flags are set, the order is -``$cppflags $cflags|$cxxflags $ldflags command $ldlibs`` for C and C++ and -``$fflags $cppflags $ldflags command $ldlibs`` for fortran. +in GNU Autotools. If all flags are set, the order is +``$cppflags $cflags|$cxxflags $ldflags $ldlibs`` for C and C++ and +``$fflags $cppflags $ldflags $ldlibs`` for Fortran. ^^^^^^^^^^^^^^^^^^^^^^^ Architecture specifiers @@ -786,14 +810,14 @@ The architecture can be specified by using the reserved words ``target`` and/or ``os`` (``target=x86-64 os=debian7``). You can also use the triplet form of platform, operating system and processor. -.. code-block:: sh +.. code-block:: console - spack install libelf arch=cray_xc-CNL10-haswell + $ spack install libelf arch=cray-CNL10-haswell Users on non-Cray systems won't have to worry about specifying the architecture. Spack will autodetect what kind of operating system is on your machine as well as the processor. For more information on how the architecture can be -used on Cray machines, check here :ref:`cray-support` +used on Cray machines, see :ref:`cray-support` .. _sec-virtual-dependencies: @@ -848,21 +872,27 @@ Constraining virtual packages When installing a package that depends on a virtual package, you can opt to specify the particular provider you want to use, or you can let -Spack pick. For example, if you just type this:: +Spack pick. For example, if you just type this: + +.. code-block:: console - spack install mpileaks + $ spack install mpileaks Then spack will pick a provider for you according to site policies. -If you really want a particular version, say mpich, then you could -run this instead:: +If you really want a particular version, say ``mpich``, then you could +run this instead: - spack install mpileaks ^mpich +.. code-block:: console + + $ spack install mpileaks ^mpich This forces spack to use some version of ``mpich`` for its implementation. As always, you can be even more specific and require -a particular ``mpich`` version:: +a particular ``mpich`` version: + +.. code-block:: console - spack install mpileaks ^mpich@3 + $ spack install mpileaks ^mpich@3 The ``mpileaks`` package in particular only needs MPI-1 commands, so any MPI implementation will do. If another package depends on @@ -887,7 +917,7 @@ to be unique, Spack will replace the reference with the spec to which it refers. Otherwise, it will prompt for a more qualified hash. Note that this will not work to reinstall a depencency uninstalled by -``spack uninstall -f``. +``spack uninstall --force``. .. _spack-providers: @@ -937,15 +967,15 @@ distributions, this can be installed from the vendor's repository: .. code-block:: sh - yum install environment-modules # (Fedora/RHEL/CentOS) - apt-get install environment-modules # (Ubuntu/Debian) + $ yum install environment-modules # (Fedora/RHEL/CentOS) + $ apt-get install environment-modules # (Ubuntu/Debian) If your Linux distribution does not have Environment Modules, you can get it with Spack: -.. code-block:: sh +.. code-block:: console - spack install environment-modules + $ spack install environment-modules In this case to activate it automatically you need to add the following two lines to your ``.bashrc`` profile (or similar): @@ -972,7 +1002,8 @@ For ``bash`` or ``ksh``, run: .. code-block:: sh - . ${SPACK_ROOT}/share/spack/setup-env.sh + export SPACK_ROOT=/path/to/spack + . $SPACK_ROOT/share/spack/setup-env.sh For ``csh`` and ``tcsh`` run: @@ -995,8 +1026,8 @@ directories: .. code-block:: sh - ${SPACK_ROOT}/share/spack/modules - ${SPACK_ROOT}/share/spack/dotkit + $SPACK_ROOT/share/spack/modules + $SPACK_ROOT/share/spack/dotkit The directories are automatically added to your ``MODULEPATH`` and ``DK_NODE`` environment variables when you enable Spack's `shell @@ -1011,56 +1042,56 @@ If you have shell support enabled you should be able to run either been installed. Here is sample output of those programs, showing lots of installed packages. - .. code-block:: sh - - $ module avail - - ------- /home/gamblin2/spack/share/spack/modules/linux-debian7-x86_64 -------- - adept-utils@1.0%gcc@4.4.7-5adef8da libelf@0.8.13%gcc@4.4.7 - automaded@1.0%gcc@4.4.7-d9691bb0 libelf@0.8.13%intel@15.0.0 - boost@1.55.0%gcc@4.4.7 mpc@1.0.2%gcc@4.4.7-559607f5 - callpath@1.0.1%gcc@4.4.7-5dce4318 mpfr@3.1.2%gcc@4.4.7 - dyninst@8.1.2%gcc@4.4.7-b040c20e mpich@3.0.4%gcc@4.4.7 - gcc@4.9.1%gcc@4.4.7-93ab98c5 mpich@3.0.4%gcc@4.9.0 - gmp@6.0.0a%gcc@4.4.7 mrnet@4.1.0%gcc@4.4.7-72b7881d - graphlib@2.0.0%gcc@4.4.7 netgauge@2.4.6%gcc@4.9.0-27912b7b - launchmon@1.0.1%gcc@4.4.7 stat@2.1.0%gcc@4.4.7-51101207 - libNBC@1.1.1%gcc@4.9.0-27912b7b sundials@2.5.0%gcc@4.9.0-27912b7b - libdwarf@20130729%gcc@4.4.7-b52fac98 - - .. code-block:: sh - - $ use -l spack - - spack ---------- - adept-utils@1.0%gcc@4.4.7-5adef8da - adept-utils @1.0 - automaded@1.0%gcc@4.4.7-d9691bb0 - automaded @1.0 - boost@1.55.0%gcc@4.4.7 - boost @1.55.0 - callpath@1.0.1%gcc@4.4.7-5dce4318 - callpath @1.0.1 - dyninst@8.1.2%gcc@4.4.7-b040c20e - dyninst @8.1.2 - gmp@6.0.0a%gcc@4.4.7 - gmp @6.0.0a - libNBC@1.1.1%gcc@4.9.0-27912b7b - libNBC @1.1.1 - libdwarf@20130729%gcc@4.4.7-b52fac98 - libdwarf @20130729 - libelf@0.8.13%gcc@4.4.7 - libelf @0.8.13 - libelf@0.8.13%intel@15.0.0 - libelf @0.8.13 - mpc@1.0.2%gcc@4.4.7-559607f5 - mpc @1.0.2 - mpfr@3.1.2%gcc@4.4.7 - mpfr @3.1.2 - mpich@3.0.4%gcc@4.4.7 - mpich @3.0.4 - mpich@3.0.4%gcc@4.9.0 - mpich @3.0.4 - netgauge@2.4.6%gcc@4.9.0-27912b7b - netgauge @2.4.6 - sundials@2.5.0%gcc@4.9.0-27912b7b - sundials @2.5.0 +.. code-block:: console + + $ module avail + + ------- /home/gamblin2/spack/share/spack/modules/linux-debian7-x86_64 -------- + adept-utils@1.0%gcc@4.4.7-5adef8da libelf@0.8.13%gcc@4.4.7 + automaded@1.0%gcc@4.4.7-d9691bb0 libelf@0.8.13%intel@15.0.0 + boost@1.55.0%gcc@4.4.7 mpc@1.0.2%gcc@4.4.7-559607f5 + callpath@1.0.1%gcc@4.4.7-5dce4318 mpfr@3.1.2%gcc@4.4.7 + dyninst@8.1.2%gcc@4.4.7-b040c20e mpich@3.0.4%gcc@4.4.7 + gcc@4.9.1%gcc@4.4.7-93ab98c5 mpich@3.0.4%gcc@4.9.0 + gmp@6.0.0a%gcc@4.4.7 mrnet@4.1.0%gcc@4.4.7-72b7881d + graphlib@2.0.0%gcc@4.4.7 netgauge@2.4.6%gcc@4.9.0-27912b7b + launchmon@1.0.1%gcc@4.4.7 stat@2.1.0%gcc@4.4.7-51101207 + libNBC@1.1.1%gcc@4.9.0-27912b7b sundials@2.5.0%gcc@4.9.0-27912b7b + libdwarf@20130729%gcc@4.4.7-b52fac98 + +.. code-block:: console + + $ use -l spack + + spack ---------- + adept-utils@1.0%gcc@4.4.7-5adef8da - adept-utils @1.0 + automaded@1.0%gcc@4.4.7-d9691bb0 - automaded @1.0 + boost@1.55.0%gcc@4.4.7 - boost @1.55.0 + callpath@1.0.1%gcc@4.4.7-5dce4318 - callpath @1.0.1 + dyninst@8.1.2%gcc@4.4.7-b040c20e - dyninst @8.1.2 + gmp@6.0.0a%gcc@4.4.7 - gmp @6.0.0a + libNBC@1.1.1%gcc@4.9.0-27912b7b - libNBC @1.1.1 + libdwarf@20130729%gcc@4.4.7-b52fac98 - libdwarf @20130729 + libelf@0.8.13%gcc@4.4.7 - libelf @0.8.13 + libelf@0.8.13%intel@15.0.0 - libelf @0.8.13 + mpc@1.0.2%gcc@4.4.7-559607f5 - mpc @1.0.2 + mpfr@3.1.2%gcc@4.4.7 - mpfr @3.1.2 + mpich@3.0.4%gcc@4.4.7 - mpich @3.0.4 + mpich@3.0.4%gcc@4.9.0 - mpich @3.0.4 + netgauge@2.4.6%gcc@4.9.0-27912b7b - netgauge @2.4.6 + sundials@2.5.0%gcc@4.9.0-27912b7b - sundials @2.5.0 The names here should look familiar, they're the same ones from ``spack find``. You *can* use the names here directly. For example, you could type either of these commands to load the callpath module: -.. code-block:: sh +.. code-block:: console - use callpath@1.0.1%gcc@4.4.7-5dce4318 + $ use callpath@1.0.1%gcc@4.4.7-5dce4318 -.. code-block:: sh +.. code-block:: console - module load callpath@1.0.1%gcc@4.4.7-5dce4318 + $ module load callpath@1.0.1%gcc@4.4.7-5dce4318 Neither of these is particularly pretty, easy to remember, or easy to type. Luckily, Spack has its own interface for using modules @@ -1077,7 +1108,7 @@ And you can use the same shortened names you use everywhere else in Spack. For example, this will add the ``mpich`` package built with ``gcc`` to your path: -.. code-block:: sh +.. code-block:: console $ spack install mpich %gcc@4.4.7 @@ -1090,18 +1121,18 @@ Spack. For example, this will add the ``mpich`` package built with Or, similarly with modules, you could type: -.. code-block:: sh +.. code-block:: console $ spack load mpich %gcc@4.4.7 These commands will add appropriate directories to your ``PATH``, -``MANPATH``, ``CPATH``, and ``LD_LIBRARY_PATH``. When you no longer want to use -a package, you can type unload or unuse similarly: +``MANPATH``, ``CPATH``, and ``LD_LIBRARY_PATH``. When you no longer +want to use a package, you can type unload or unuse similarly: -.. code-block:: sh +.. code-block:: console - $ spack unload mpich %gcc@4.4.7 # modules - $ spack unuse mpich %gcc@4.4.7 # dotkit + $ spack unload mpich %gcc@4.4.7 # modules + $ spack unuse mpich %gcc@4.4.7 # dotkit .. note:: @@ -1116,7 +1147,7 @@ Ambiguous module names If a spec used with load/unload or use/unuse is ambiguous (i.e. more than one installed package matches it), then Spack will warn you: -.. code-block:: sh +.. code-block:: console $ spack load libelf ==> Error: Multiple matches for spec libelf. Choose one: @@ -1129,7 +1160,7 @@ identify one package. For example, above, the key differentiator is that one ``libelf`` is built with the Intel compiler, while the other used ``gcc``. You could therefore just type: -.. code-block:: sh +.. code-block:: console $ spack load libelf %intel @@ -1144,13 +1175,13 @@ and are placed in the following directories under the Spack root: .. code-block:: sh - ${SPACK_ROOT}/share/spack/modules - ${SPACK_ROOT}/share/spack/dotkit + $SPACK_ROOT/share/spack/modules + $SPACK_ROOT/share/spack/dotkit The content that gets written in each module file can be customized in two ways: - 1. overriding part of the ``spack.Package`` API within a ``package.py`` - 2. writing dedicated configuration files +#. overriding part of the ``spack.Package`` API within a ``package.py`` +#. writing dedicated configuration files ^^^^^^^^^^^^^^^^^^^^^^^^ Override ``Package`` API @@ -1161,9 +1192,9 @@ of module files: .. code-block:: python - def setup_environment(self, spack_env, run_env): - """Set up the compile and runtime environments for a package.""" - pass + def setup_environment(self, spack_env, run_env): + """Set up the compile and runtime environments for a package.""" + pass """"""""""""""""" Recursive Modules @@ -1179,9 +1210,9 @@ RPATH. Modules may be loaded recursively with the ``load`` command's ``--dependencies`` or ``-r`` argument: -.. code-block:: sh +.. code-block:: console - $ spack load --dependencies ... + $ spack load --dependencies ... More than one spec may be placed on the command line here. @@ -1195,46 +1226,46 @@ commands inside a shell script. By adding the ``--shell`` flag, ``spack module find`` may also be used to generate code that can be cut-and-pasted into a shell script. For example: -.. code-block:: sh - - $ spack module find tcl --dependencies --shell py-numpy git - # bzip2@1.0.6%gcc@4.9.3=linux-x86_64 - module load bzip2-1.0.6-gcc-4.9.3-ktnrhkrmbbtlvnagfatrarzjojmkvzsx - # ncurses@6.0%gcc@4.9.3=linux-x86_64 - module load ncurses-6.0-gcc-4.9.3-kaazyneh3bjkfnalunchyqtygoe2mncv - # zlib@1.2.8%gcc@4.9.3=linux-x86_64 - module load zlib-1.2.8-gcc-4.9.3-v3ufwaahjnviyvgjcelo36nywx2ufj7z - # sqlite@3.8.5%gcc@4.9.3=linux-x86_64 - module load sqlite-3.8.5-gcc-4.9.3-a3eediswgd5f3rmto7g3szoew5nhehbr - # readline@6.3%gcc@4.9.3=linux-x86_64 - module load readline-6.3-gcc-4.9.3-se6r3lsycrwxyhreg4lqirp6xixxejh3 - # python@3.5.1%gcc@4.9.3=linux-x86_64 - module load python-3.5.1-gcc-4.9.3-5q5rsrtjld4u6jiicuvtnx52m7tfhegi - # py-setuptools@20.5%gcc@4.9.3=linux-x86_64 - module load py-setuptools-20.5-gcc-4.9.3-4qr2suj6p6glepnedmwhl4f62x64wxw2 - # py-nose@1.3.7%gcc@4.9.3=linux-x86_64 - module load py-nose-1.3.7-gcc-4.9.3-pwhtjw2dvdvfzjwuuztkzr7b4l6zepli - # openblas@0.2.17%gcc@4.9.3+shared=linux-x86_64 - module load openblas-0.2.17-gcc-4.9.3-pw6rmlom7apfsnjtzfttyayzc7nx5e7y - # py-numpy@1.11.0%gcc@4.9.3+blas+lapack=linux-x86_64 - module load py-numpy-1.11.0-gcc-4.9.3-mulodttw5pcyjufva4htsktwty4qd52r - # curl@7.47.1%gcc@4.9.3=linux-x86_64 - module load curl-7.47.1-gcc-4.9.3-ohz3fwsepm3b462p5lnaquv7op7naqbi - # autoconf@2.69%gcc@4.9.3=linux-x86_64 - module load autoconf-2.69-gcc-4.9.3-bkibjqhgqm5e3o423ogfv2y3o6h2uoq4 - # cmake@3.5.0%gcc@4.9.3~doc+ncurses+openssl~qt=linux-x86_64 - module load cmake-3.5.0-gcc-4.9.3-x7xnsklmgwla3ubfgzppamtbqk5rwn7t - # expat@2.1.0%gcc@4.9.3=linux-x86_64 - module load expat-2.1.0-gcc-4.9.3-6pkz2ucnk2e62imwakejjvbv6egncppd - # git@2.8.0-rc2%gcc@4.9.3+curl+expat=linux-x86_64 - module load git-2.8.0-rc2-gcc-4.9.3-3bib4hqtnv5xjjoq5ugt3inblt4xrgkd +.. code-block:: console + + $ spack module find tcl --dependencies --shell py-numpy git + # bzip2@1.0.6%gcc@4.9.3=linux-x86_64 + module load bzip2-1.0.6-gcc-4.9.3-ktnrhkrmbbtlvnagfatrarzjojmkvzsx + # ncurses@6.0%gcc@4.9.3=linux-x86_64 + module load ncurses-6.0-gcc-4.9.3-kaazyneh3bjkfnalunchyqtygoe2mncv + # zlib@1.2.8%gcc@4.9.3=linux-x86_64 + module load zlib-1.2.8-gcc-4.9.3-v3ufwaahjnviyvgjcelo36nywx2ufj7z + # sqlite@3.8.5%gcc@4.9.3=linux-x86_64 + module load sqlite-3.8.5-gcc-4.9.3-a3eediswgd5f3rmto7g3szoew5nhehbr + # readline@6.3%gcc@4.9.3=linux-x86_64 + module load readline-6.3-gcc-4.9.3-se6r3lsycrwxyhreg4lqirp6xixxejh3 + # python@3.5.1%gcc@4.9.3=linux-x86_64 + module load python-3.5.1-gcc-4.9.3-5q5rsrtjld4u6jiicuvtnx52m7tfhegi + # py-setuptools@20.5%gcc@4.9.3=linux-x86_64 + module load py-setuptools-20.5-gcc-4.9.3-4qr2suj6p6glepnedmwhl4f62x64wxw2 + # py-nose@1.3.7%gcc@4.9.3=linux-x86_64 + module load py-nose-1.3.7-gcc-4.9.3-pwhtjw2dvdvfzjwuuztkzr7b4l6zepli + # openblas@0.2.17%gcc@4.9.3+shared=linux-x86_64 + module load openblas-0.2.17-gcc-4.9.3-pw6rmlom7apfsnjtzfttyayzc7nx5e7y + # py-numpy@1.11.0%gcc@4.9.3+blas+lapack=linux-x86_64 + module load py-numpy-1.11.0-gcc-4.9.3-mulodttw5pcyjufva4htsktwty4qd52r + # curl@7.47.1%gcc@4.9.3=linux-x86_64 + module load curl-7.47.1-gcc-4.9.3-ohz3fwsepm3b462p5lnaquv7op7naqbi + # autoconf@2.69%gcc@4.9.3=linux-x86_64 + module load autoconf-2.69-gcc-4.9.3-bkibjqhgqm5e3o423ogfv2y3o6h2uoq4 + # cmake@3.5.0%gcc@4.9.3~doc+ncurses+openssl~qt=linux-x86_64 + module load cmake-3.5.0-gcc-4.9.3-x7xnsklmgwla3ubfgzppamtbqk5rwn7t + # expat@2.1.0%gcc@4.9.3=linux-x86_64 + module load expat-2.1.0-gcc-4.9.3-6pkz2ucnk2e62imwakejjvbv6egncppd + # git@2.8.0-rc2%gcc@4.9.3+curl+expat=linux-x86_64 + module load git-2.8.0-rc2-gcc-4.9.3-3bib4hqtnv5xjjoq5ugt3inblt4xrgkd The script may be further edited by removing unnecessary modules. -This script may be directly executed in bash via +This script may be directly executed in bash via: -.. code-block :: sh +.. code-block:: sh - source <( spack module find tcl --dependencies --shell py-numpy git ) + source < (spack module find tcl --dependencies --shell py-numpy git) ^^^^^^^^^^^^^^^^^^^^^^^^^ Regenerating Module files @@ -1242,9 +1273,9 @@ Regenerating Module files .. code-block:: python - def setup_dependent_environment(self, spack_env, run_env, dependent_spec): - """Set up the environment of packages that depend on this one""" - pass + def setup_dependent_environment(self, spack_env, run_env, dependent_spec): + """Set up the environment of packages that depend on this one""" + pass As briefly stated in the comments, the first method lets you customize the module file content for the package you are currently writing, the second @@ -1260,10 +1291,9 @@ overrides ``setup_dependent_environment`` in the following way: .. code-block:: python - def setup_dependent_environment(self, spack_env, run_env, extension_spec): - # ... - if extension_spec.package.extends(self.spec): - run_env.prepend_path('PYTHONPATH', os.path.join(extension_spec.prefix, self.site_packages_dir)) + def setup_dependent_environment(self, spack_env, run_env, extension_spec): + if extension_spec.package.extends(self.spec): + run_env.prepend_path('PYTHONPATH', os.path.join(extension_spec.prefix, self.site_packages_dir)) to insert the appropriate ``PYTHONPATH`` modifications in the module files of python packages. @@ -1289,11 +1319,11 @@ For each module system that can be enabled a finer configuration is possible: .. code-block:: yaml - modules: - tcl: - # contains environment modules specific customizations - dotkit: - # contains dotkit specific customizations + modules: + tcl: + # contains environment modules specific customizations + dotkit: + # contains dotkit specific customizations The structure under the ``tcl`` and ``dotkit`` keys is almost equal, and will be showcased in the following by some examples. @@ -1309,24 +1339,24 @@ Considering : .. code-block:: yaml - modules: - tcl: - all: # Default addition for every package - environment: - set: - BAR: 'bar' - ^openmpi:: # A double ':' overrides previous rules - environment: - set: - BAR: 'baz' - zlib: - environment: - prepend_path: - LD_LIBRARY_PATH: 'foo' - zlib%gcc@4.8: - environment: - unset: - - FOOBAR + modules: + tcl: + all: # Default addition for every package + environment: + set: + BAR: 'bar' + ^openmpi:: # A double ':' overrides previous rules + environment: + set: + BAR: 'baz' + zlib: + environment: + prepend_path: + LD_LIBRARY_PATH: 'foo' + zlib%gcc@4.8: + environment: + unset: + - FOOBAR what will happen is that: @@ -1336,10 +1366,10 @@ what will happen is that: - any spec that satisfies ``zlib%gcc@4.8`` will additionally unset ``FOOBAR`` .. note:: - Order does matter - The modifications associated with the ``all`` keyword are always evaluated - first, no matter where they appear in the configuration file. All the other - spec constraints are instead evaluated top to bottom. + Order does matter + The modifications associated with the ``all`` keyword are always evaluated + first, no matter where they appear in the configuration file. All the other + spec constraints are instead evaluated top to bottom. """""""""""""""""""""""""""""""""""""""" Filter modifications out of module files @@ -1351,11 +1381,12 @@ modified by your dotkit modules. Then : .. code-block:: yaml - modules: - dotkit: - all: - filter: - environment_blacklist: ['CPATH', 'LIBRARY_PATH'] # Exclude changes to any of these variables + modules: + dotkit: + all: + filter: + # Exclude changes to any of these variables + environment_blacklist: ['CPATH', 'LIBRARY_PATH'] will generate dotkit module files that will not contain modifications to either ``CPATH`` or ``LIBRARY_PATH`` and environment module files that instead will @@ -1369,20 +1400,20 @@ The following lines in ``modules.yaml``: .. code-block:: yaml - modules: - tcl: - all: - autoload: 'direct' + modules: + tcl: + all: + autoload: 'direct' will produce environment module files that will automatically load their direct dependencies. .. note:: - Allowed values for ``autoload`` statements - Allowed values for ``autoload`` statements are either ``none``, ``direct`` - or ``all``. In ``tcl`` configuration it is possible to use the option - ``prerequisites`` that accepts the same values and will add ``prereq`` - statements instead of automatically loading other modules. + Allowed values for ``autoload`` statements + Allowed values for ``autoload`` statements are either ``none``, ``direct`` + or ``all``. In ``tcl`` configuration it is possible to use the option + ``prerequisites`` that accepts the same values and will add ``prereq`` + statements instead of automatically loading other modules. """""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" Blacklist or whitelist the generation of specific module files @@ -1395,10 +1426,10 @@ A configuration file like: .. code-block:: yaml - modules: - tcl: - whitelist: ['gcc', 'llvm'] # Whitelist will have precedence over blacklist - blacklist: ['%gcc@4.4.7'] # Assuming gcc@4.4.7 is the system compiler + modules: + tcl: + whitelist: ['gcc', 'llvm'] # Whitelist will have precedence over blacklist + blacklist: ['%gcc@4.4.7'] # Assuming gcc@4.4.7 is the system compiler will skip module file generation for anything that satisfies ``%gcc@4.4.7``, with the exception of specs that satisfy ``gcc`` or ``llvm``. @@ -1411,23 +1442,23 @@ A configuration file like: .. code-block:: yaml - modules: - tcl: - naming_scheme: '{name}/{version}-{compiler.name}-{compiler.version}' - all: - conflict: ['{name}', 'intel/14.0.1'] + modules: + tcl: + naming_scheme: '{name}/{version}-{compiler.name}-{compiler.version}' + all: + conflict: ['{name}', 'intel/14.0.1'] will create module files that will conflict with ``intel/14.0.1`` and with the base directory of the same module, effectively preventing the possibility to load two or more versions of the same software at the same time. .. note:: - Tokens available for the naming scheme - currently only the tokens shown in the example are available to construct - the naming scheme + Tokens available for the naming scheme + currently only the tokens shown in the example are available to construct + the naming scheme .. note:: - The ``conflict`` option is ``tcl`` specific + The ``conflict`` option is ``tcl`` specific ^^^^^^^^^^^^^^^^^^^^^^^^^ Regenerating module files @@ -1448,15 +1479,12 @@ Running ``spack module refresh`` will remove the ``share/spack/modules`` and ``share/spack/dotkit`` directories, then regenerate all module and dotkit files from scratch: -.. code-block:: sh +.. code-block:: console $ spack module refresh ==> Regenerating tcl module files. ==> Regenerating dotkit module files. - -.. _extensions: - ---------------- Filesystem Views ---------------- @@ -1496,10 +1524,10 @@ built by Spack. Views can be used for a variety of purposes including: -- A central installation in a traditional layout, eg ``/usr/local`` maintained over time by the sysadmin. -- A self-contained installation area which may for the basis of a top-level atomic versioning scheme, eg ``/opt/pro`` vs ``/opt/dev``. -- Providing an atomic and monolithic binary distribution, eg for delivery as a single tarball. -- Producing ephemeral testing or developing environments. +* A central installation in a traditional layout, eg ``/usr/local`` maintained over time by the sysadmin. +* A self-contained installation area which may for the basis of a top-level atomic versioning scheme, eg ``/opt/pro`` vs ``/opt/dev``. +* Providing an atomic and monolithic binary distribution, eg for delivery as a single tarball. +* Producing ephemeral testing or developing environments. ^^^^^^^^^^^^^^^^^^^^^^ Using Filesystem Views @@ -1514,27 +1542,27 @@ The following example creates a filesystem view based on an installed ``cmake`` package and then removes from the view the files in the ``cmake`` package while retaining its dependencies. -.. code-block:: sh +.. code-block:: console - $ spack view -v symlink myview cmake@3.5.2 - ==> Linking package: "ncurses" - ==> Linking package: "zlib" - ==> Linking package: "openssl" - ==> Linking package: "cmake" + $ spack view --verbose symlink myview cmake@3.5.2 + ==> Linking package: "ncurses" + ==> Linking package: "zlib" + ==> Linking package: "openssl" + ==> Linking package: "cmake" - $ ls myview/ - bin doc etc include lib share + $ ls myview/ + bin doc etc include lib share - $ ls myview/bin/ - captoinfo clear cpack ctest infotocap openssl tabs toe tset - ccmake cmake c_rehash infocmp ncurses6-config reset tic tput + $ ls myview/bin/ + captoinfo clear cpack ctest infotocap openssl tabs toe tset + ccmake cmake c_rehash infocmp ncurses6-config reset tic tput - $ spack view -v -d false rm myview cmake@3.5.2 - ==> Removing package: "cmake" + $ spack view --verbose --dependencies false rm myview cmake@3.5.2 + ==> Removing package: "cmake" - $ ls myview/bin/ - captoinfo c_rehash infotocap openssl tabs toe tset - clear infocmp ncurses6-config reset tic tput + $ ls myview/bin/ + captoinfo c_rehash infotocap openssl tabs toe tset + clear infocmp ncurses6-config reset tic tput ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Limitations of Filesystem Views @@ -1568,6 +1596,8 @@ particular, if two packages require the same sub-tree of dependencies, removing one package (recursively) will remove its dependencies and leave the other package broken. +.. _extensions: + --------------------------- Extensions & Python support --------------------------- @@ -1583,7 +1613,7 @@ Spack has support for this type of installation as well. In Spack, a package that can live inside the prefix of another package is called an *extension*. Suppose you have Python installed like so: -.. code-block:: sh +.. code-block:: console $ spack find python ==> 1 installed packages. @@ -1598,7 +1628,7 @@ an *extension*. Suppose you have Python installed like so: You can find extensions for your Python installation like this: -.. code-block:: sh +.. code-block:: console $ spack extensions python ==> python@2.7.8%gcc@4.4.7 arch=linux-debian7-x86_64-703c7a96 @@ -1623,11 +1653,11 @@ You can find extensions for your Python installation like this: The extensions are a subset of what's returned by ``spack list``, and they are packages like any other. They are installed into their own -prefixes, and you can see this with ``spack find -p``: +prefixes, and you can see this with ``spack find --paths``: -.. code-block:: sh +.. code-block:: console - $ spack find -p py-numpy + $ spack find --paths py-numpy ==> 1 installed packages. -- linux-debian7-x86_64 / gcc@4.4.7 -------------------------------- py-numpy@1.9.1 /g/g21/gamblin2/src/spack/opt/linux-debian7-x86_64/gcc@4.4.7/py-numpy@1.9.1-66733244 @@ -1635,7 +1665,7 @@ prefixes, and you can see this with ``spack find -p``: However, even though this package is installed, you cannot use it directly when you run ``python``: -.. code-block:: sh +.. code-block:: console $ spack load python $ python @@ -1659,14 +1689,14 @@ in your current shell. For tcl modules: -.. code-block:: sh +.. code-block:: console $ spack load python $ spack load py-numpy or, for dotkit: -.. code-block:: sh +.. code-block:: console $ spack use python $ spack use py-numpy @@ -1690,7 +1720,7 @@ installation: ``spack activate`` ^^^^^^^^^^^^^^^^^^ -.. code-block:: sh +.. code-block:: console $ spack activate py-numpy ==> Activated extension py-setuptools@11.3.1%gcc@4.4.7 arch=linux-debian7-x86_64-3c74eb69 for python@2.7.8%gcc@4.4.7. @@ -1707,7 +1737,7 @@ once all dependencies were activated in the ``python`` installation, If we run ``spack extensions`` again, we now see the three new packages listed as activated: -.. code-block:: sh +.. code-block:: console $ spack extensions python ==> python@2.7.8%gcc@4.4.7 arch=linux-debian7-x86_64-703c7a96 @@ -1756,7 +1786,7 @@ explicitly load their preferred version. If, for some reason, you want to activate a package *without* its dependencies, you can use ``spack activate -f``: -.. code-block:: sh +.. code-block:: console $ spack activate -f py-numpy ==> Activated extension py-numpy@1.9.1%gcc@4.4.7 arch=linux-debian7-x86_64-66733244 for python@2.7.8%gcc@4.4.7. @@ -1772,18 +1802,20 @@ version of a Python module. Obviously, you may want to change that at some point. ``spack deactivate`` is the command for this. There are several variants: - * ``spack deactivate `` will deactivate a single - extension. If another activated extension depends on this one, - Spack will warn you and exit with an error. - * ``spack deactivate -f `` deactivates an extension - regardless of packages that depend on it. - * ``spack deactivate -a `` deactivates an extension and - all of its dependencies. Use ``-f`` to disregard dependents. - * ``spack deactivate -a `` deactivates *all* activated - extensions of a package. For example, to deactivate *all* python - extensions, use:: +* ``spack deactivate `` will deactivate a single + extension. If another activated extension depends on this one, + Spack will warn you and exit with an error. +* ``spack deactivate --force `` deactivates an extension + regardless of packages that depend on it. +* ``spack deactivate --all `` deactivates an extension and + all of its dependencies. Use ``--force`` to disregard dependents. +* ``spack deactivate --all `` deactivates *all* activated + extensions of a package. For example, to deactivate *all* python + extensions, use: - spack deactivate -a python + .. code-block:: console + + $ spack deactivate --all python ----------------------- Filesystem requirements @@ -1796,7 +1828,7 @@ without ``flock`` support enabled. You can determine how your filesystems are mounted with ``mount -p``. The output for a Lustre filesystem might look like this: -.. code-block:: sh +.. code-block:: console $ mount -l | grep lscratch pilsner-mds1-lnet0@o2ib100:/lsd on /p/lscratchd type lustre (rw,nosuid,noauto,_netdev,lazystatfs,flock) @@ -1808,7 +1840,7 @@ system administrator to enable ``flock``. This issue typically manifests with the error below: -.. code-block:: sh +.. code-block:: console $ ./spack find Traceback (most recent call last): @@ -1817,7 +1849,7 @@ This issue typically manifests with the error below: File "./spack", line 154,' in main return_val = command(parser, args) File "./spack/lib/spack/spack/cmd/find.py", line 170, in find - specs = set(spack.installed_db.query(**q_args)) + specs = set(spack.installed_db.query(\**q_args)) File "./spack/lib/spack/spack/database.py", line 551, in query with self.read_transaction(): File "./spack/lib/spack/spack/database.py", line 598, in __enter__ @@ -1832,12 +1864,7 @@ This issue typically manifests with the error below: A nicer error message is TBD in future versions of Spack. -<<<<<<< c36f13e44dae52086ca48e4f16b514642bc91106 - .. _cray-support: -======= -.. _spack-cray: ->>>>>>> Fix #1604 and fix #1605, title underline problems ------------- Spack on Cray @@ -1846,34 +1873,34 @@ Spack on Cray Spack differs slightly when used on a Cray system. The architecture spec can differentiate between the front-end and back-end processor and operating system. For example, on Edison at NERSC, the back-end target processor -is \"Ivy Bridge\", so you can specify to use the back-end this way: +is "Ivy Bridge", so you can specify to use the back-end this way: -.. code-block:: sh +.. code-block:: console - spack install zlib target=ivybridge + $ spack install zlib target=ivybridge You can also use the operating system to build against the back-end: -.. code-block:: sh +.. code-block:: console - spack install zlib os=CNL10 + $ spack install zlib os=CNL10 Notice that the name includes both the operating system name and the major version number concatenated together. Alternatively, if you want to build something for the front-end, you can specify the front-end target processor. The processor for a login node -on Edison is \"Sandy bridge\" so we specify on the command line like so: +on Edison is "Sandy bridge" so we specify on the command line like so: -.. code-block:: sh +.. code-block:: console - spack install zlib target=sandybridge + $ spack install zlib target=sandybridge And the front-end operating system is: -.. code-block:: sh +.. code-block:: console - spack install zlib os=SuSE11 + $ spack install zlib os=SuSE11 ^^^^^^^^^^^^^^^^^^^^^^^ Cray compiler detection @@ -1888,27 +1915,27 @@ compiler wrapper names (i.e. cc, CC, ftn). During build time, Spack will load the correct PrgEnv and compiler module and will call appropriate wrapper. The compilers.yaml config file will also differ. There is a -modules section that is filled with the compiler\'s Programming Environment -and module name. On other systems, this field is empty []:: +modules section that is filled with the compiler's Programming Environment +and module name. On other systems, this field is empty []: - ... - - compiler: - modules: - - PrgEnv-intel - - intel/15.0.109 - ... +.. code-block:: yaml + + - compiler: + modules: + - PrgEnv-intel + - intel/15.0.109 As mentioned earlier, the compiler paths will look different on a Cray system. Since most compilers are invoked using cc, CC and ftn, the paths for each -compiler are replaced with their respective Cray compiler wrapper names:: +compiler are replaced with their respective Cray compiler wrapper names: + +.. code-block:: yaml - ... paths: cc: cc cxx: CC f77: ftn fc: ftn - ... As opposed to an explicit path to the compiler executable. This allows Spack to call the Cray compiler wrappers during build time. @@ -1917,24 +1944,24 @@ For more on compiler configuration, check out :ref:`compiler-config`. Spack sets the default Cray link type to dynamic, to better match other other platforms. Individual packages can enable static linking (which is the -default outside of Spack on cray systems) using the -static flag. +default outside of Spack on cray systems) using the ``-static`` flag. ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Setting defaults and using Cray modules ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ If you want to use default compilers for each PrgEnv and also be able -to load cray external modules, you will need to set up a packages.yaml. +to load cray external modules, you will need to set up a ``packages.yaml``. -Here\'s an example of an external configuration for cray modules: +Here's an example of an external configuration for cray modules: .. code-block:: yaml - packages: - mpi: - modules: - mpich@7.3.1%gcc@5.2.0 arch=cray_xc-haswell-CNL10: cray-mpich - mpich@7.3.1%intel@16.0.0.109 arch=cray_xc-haswell-CNL10: cray-mpich + packages: + mpi: + modules: + mpich@7.3.1%gcc@5.2.0 arch=cray_xc-haswell-CNL10: cray-mpich + mpich@7.3.1%intel@16.0.0.109 arch=cray_xc-haswell-CNL10: cray-mpich This tells Spack that for whatever package that depends on mpi, load the cray-mpich module into the environment. You can then be able to use whatever @@ -1942,7 +1969,7 @@ environment variables, libraries, etc, that are brought into the environment via module load. You can set the default compiler that Spack can use for each compiler type. -If you want to use the Cray defaults, then set them under *all:* in packages.yaml. +If you want to use the Cray defaults, then set them under ``all:`` in packages.yaml. In the compiler field, set the compiler specs in your order of preference. Whenever you build with that compiler type, Spack will concretize to that version. @@ -1951,23 +1978,23 @@ Here is an example of a full packages.yaml used at NERSC .. code-block:: yaml packages: - mpi: - modules: - mpich@7.3.1%gcc@5.2.0 arch=cray_xc-CNL10-ivybridge: cray-mpich - mpich@7.3.1%intel@16.0.0.109 arch=cray_xc-SuSE11-ivybridge: cray-mpich - buildable: False - netcdf: - modules: - netcdf@4.3.3.1%gcc@5.2.0 arch=cray_xc-CNL10-ivybridge: cray-netcdf - netcdf@4.3.3.1%intel@16.0.0.109 arch=cray_xc-CNL10-ivybridge: cray-netcdf - buildable: False - hdf5: - modules: - hdf5@1.8.14%gcc@5.2.0 arch=cray_xc-CNL10-ivybridge: cray-hdf5 - hdf5@1.8.14%intel@16.0.0.109 arch=cray_xc-CNL10-ivybridge: cray-hdf5 - buildable: False - all: - compiler: [gcc@5.2.0, intel@16.0.0.109] + mpi: + modules: + mpich@7.3.1%gcc@5.2.0 arch=cray_xc-CNL10-ivybridge: cray-mpich + mpich@7.3.1%intel@16.0.0.109 arch=cray_xc-SuSE11-ivybridge: cray-mpich + buildable: False + netcdf: + modules: + netcdf@4.3.3.1%gcc@5.2.0 arch=cray_xc-CNL10-ivybridge: cray-netcdf + netcdf@4.3.3.1%intel@16.0.0.109 arch=cray_xc-CNL10-ivybridge: cray-netcdf + buildable: False + hdf5: + modules: + hdf5@1.8.14%gcc@5.2.0 arch=cray_xc-CNL10-ivybridge: cray-hdf5 + hdf5@1.8.14%intel@16.0.0.109 arch=cray_xc-CNL10-ivybridge: cray-hdf5 + buildable: False + all: + compiler: [gcc@5.2.0, intel@16.0.0.109] Here we tell spack that whenever we want to build with gcc use version 5.2.0 or if we want to build with intel compilers, use version 16.0.0.109. We add a spec @@ -1987,7 +2014,7 @@ Getting Help ^^^^^^^^^^^^^^ If you don't find what you need here, the ``help`` subcommand will -print out out a list of *all* of ``spack``'s options and subcommands: +print out out a list of *all* of spack's options and subcommands: .. command-output:: spack help @@ -1996,5 +2023,5 @@ usage information for a particular subcommand: .. command-output:: spack help install -Alternately, you can use ``spack -h`` in place of ``spack help``, or -``spack -h`` to get help on a particular subcommand. +Alternately, you can use ``spack --help`` in place of ``spack help``, or +``spack --help`` to get help on a particular subcommand. diff --git a/lib/spack/docs/configuration.rst b/lib/spack/docs/configuration.rst index 7545acd62a..0c88dd1d9d 100644 --- a/lib/spack/docs/configuration.rst +++ b/lib/spack/docs/configuration.rst @@ -10,7 +10,9 @@ Configuration Temporary space --------------- -.. warning:: Temporary space configuration will eventually be moved to +.. warning:: + + Temporary space configuration will eventually be moved to configuration files, but currently these settings are in ``lib/spack/spack/__init__.py`` @@ -73,11 +75,11 @@ directory. Here's an example of an external configuration: .. code-block:: yaml packages: - openmpi: - paths: - openmpi@1.4.3%gcc@4.4.7 arch=linux-x86_64-debian7: /opt/openmpi-1.4.3 - openmpi@1.4.3%gcc@4.4.7 arch=linux-x86_64-debian7+debug: /opt/openmpi-1.4.3-debug - openmpi@1.6.5%intel@10.1 arch=linux-x86_64-debian7: /opt/openmpi-1.6.5-intel + openmpi: + paths: + openmpi@1.4.3%gcc@4.4.7 arch=linux-x86_64-debian7: /opt/openmpi-1.4.3 + openmpi@1.4.3%gcc@4.4.7 arch=linux-x86_64-debian7+debug: /opt/openmpi-1.4.3-debug + openmpi@1.6.5%intel@10.1 arch=linux-x86_64-debian7: /opt/openmpi-1.6.5-intel This example lists three installations of OpenMPI, one built with gcc, one built with gcc and debug information, and another built with Intel. @@ -110,13 +112,13 @@ be: .. code-block:: yaml - packages: - openmpi: - paths: - openmpi@1.4.3%gcc@4.4.7 arch=linux-x86_64-debian7: /opt/openmpi-1.4.3 - openmpi@1.4.3%gcc@4.4.7 arch=linux-x86_64-debian7+debug: /opt/openmpi-1.4.3-debug - openmpi@1.6.5%intel@10.1 arch=linux-x86_64-debian7: /opt/openmpi-1.6.5-intel - buildable: False + packages: + openmpi: + paths: + openmpi@1.4.3%gcc@4.4.7 arch=linux-x86_64-debian7: /opt/openmpi-1.4.3 + openmpi@1.4.3%gcc@4.4.7 arch=linux-x86_64-debian7+debug: /opt/openmpi-1.4.3-debug + openmpi@1.6.5%intel@10.1 arch=linux-x86_64-debian7: /opt/openmpi-1.6.5-intel + buildable: False The addition of the ``buildable`` flag tells Spack that it should never build its own version of OpenMPI, and it will instead always rely on a pre-built @@ -144,18 +146,18 @@ The preferred configuration can be controlled via the Here's an example packages.yaml file that sets preferred packages: -.. code-block:: sh +.. code-block:: yaml - packages: - opencv: - compiler: [gcc@4.9] - variants: +debug - gperftools: - version: [2.2, 2.4, 2.3] - all: - compiler: [gcc@4.4.7, gcc@4.6:, intel, clang, pgi] - providers: - mpi: [mvapich, mpich, openmpi] + packages: + opencv: + compiler: [gcc@4.9] + variants: +debug + gperftools: + version: [2.2, 2.4, 2.3] + all: + compiler: [gcc@4.4.7, gcc@4.6:, intel, clang, pgi] + providers: + mpi: [mvapich, mpich, openmpi] At a high level, this example is specifying how packages should be concretized. The opencv package should prefer using gcc 4.9 and @@ -205,15 +207,15 @@ supply ``-p`` to Spack on the command line, before any subcommands. ``spack -p`` output looks like this: -.. code-block:: sh +.. code-block:: console $ spack -p graph dyninst o dyninst - |\ - | |\ - | o | libdwarf - |/ / - o | libelf + \|\ + \| \|\ + \| o \| libdwarf + \|/ / + o \| libelf / o boost diff --git a/lib/spack/docs/developer_guide.rst b/lib/spack/docs/developer_guide.rst index 5c712149b5..04ae8fe1a1 100644 --- a/lib/spack/docs/developer_guide.rst +++ b/lib/spack/docs/developer_guide.rst @@ -18,12 +18,12 @@ Overview Spack is designed with three separate roles in mind: - #. **Users**, who need to install software *without* knowing all the - details about how it is built. - #. **Packagers** who know how a particular software package is - built and encode this information in package files. - #. **Developers** who work on Spack, add new features, and try to - make the jobs of packagers and users easier. +#. **Users**, who need to install software *without* knowing all the + details about how it is built. +#. **Packagers** who know how a particular software package is + built and encode this information in package files. +#. **Developers** who work on Spack, add new features, and try to + make the jobs of packagers and users easier. Users could be end users installing software in their home directory, or administrators installing software to a shared directory on a @@ -43,9 +43,9 @@ specification. This gets us to the two key concepts in Spack's software design: - #. **Specs**: expressions for describing builds of software, and - #. **Packages**: Python modules that build software according to a - spec. +#. **Specs**: expressions for describing builds of software, and +#. **Packages**: Python modules that build software according to a + spec. A package is a template for building particular software, and a spec as a descriptor for one or more instances of that template. Users @@ -70,56 +70,55 @@ Directory Structure ------------------- So that you can familiarize yourself with the project, we'll start -with a high level view of Spack's directory structure:: +with a high level view of Spack's directory structure: - spack/ <- installation root - bin/ - spack <- main spack executable +.. code-block:: none - etc/ - spack/ <- Spack config files. - Can be overridden by files in ~/.spack. + spack/ <- installation root + bin/ + spack <- main spack executable - var/ - spack/ <- build & stage directories - repos/ <- contains package repositories - builtin/ <- pkg repository that comes with Spack - repo.yaml <- descriptor for the builtin repository - packages/ <- directories under here contain packages - cache/ <- saves resources downloaded during installs + etc/ + spack/ <- Spack config files. + Can be overridden by files in ~/.spack. - opt/ - spack/ <- packages are installed here + var/ + spack/ <- build & stage directories + repos/ <- contains package repositories + builtin/ <- pkg repository that comes with Spack + repo.yaml <- descriptor for the builtin repository + packages/ <- directories under here contain packages + cache/ <- saves resources downloaded during installs - lib/ - spack/ - docs/ <- source for this documentation - env/ <- compiler wrappers for build environment + opt/ + spack/ <- packages are installed here - external/ <- external libs included in Spack distro - llnl/ <- some general-use libraries + lib/ + spack/ + docs/ <- source for this documentation + env/ <- compiler wrappers for build environment - spack/ <- spack module; contains Python code - cmd/ <- each file in here is a spack subcommand - compilers/ <- compiler description files - test/ <- unit test modules - util/ <- common code + external/ <- external libs included in Spack distro + llnl/ <- some general-use libraries + + spack/ <- spack module; contains Python code + cmd/ <- each file in here is a spack subcommand + compilers/ <- compiler description files + test/ <- unit test modules + util/ <- common code Spack is designed so that it could live within a `standard UNIX directory hierarchy `_, so ``lib``, ``var``, and ``opt`` all contain a ``spack`` subdirectory in case Spack is installed alongside other software. Most of the interesting -parts of Spack live in ``lib/spack``. Files under ``var`` are created -as needed, so there is no ``var`` directory when you initially clone -Spack from the repository. +parts of Spack live in ``lib/spack``. Spack has *one* directory layout and there is no install process. -version and the source code. Most Python programs don't look like -this (they use distutils, ``setup.py``, etc.) but we wanted to make -Spack *very* easy to use. The simple layout spares users from the -need to install Spack into a Python environment. Many users don't -have write access to a Python installation, and installing an entire -new instance of Python to bootstrap Spack would be very complicated. +Most Python programs don't look like this (they use distutils, ``setup.py``, +etc.) but we wanted to make Spack *very* easy to use. The simple layout +spares users from the need to install Spack into a Python environment. +Many users don't have write access to a Python installation, and installing +an entire new instance of Python to bootstrap Spack would be very complicated. Users should not have to install install a big, complicated package to use the thing that's supposed to spare them from the details of big, complicated packages. The end result is that Spack works out of the @@ -283,19 +282,18 @@ Spec objects Package objects --------------- -Most spack commands -look something like this: +Most spack commands look something like this: - #. Parse an abstract spec (or specs) from the command line, - #. *Normalize* the spec based on information in package files, - #. *Concretize* the spec according to some customizable policies, - #. Instantiate a package based on the spec, and - #. Call methods (e.g., ``install()``) on the package object. +#. Parse an abstract spec (or specs) from the command line, +#. *Normalize* the spec based on information in package files, +#. *Concretize* the spec according to some customizable policies, +#. Instantiate a package based on the spec, and +#. Call methods (e.g., ``install()``) on the package object. The information in Package files is used at all stages in this process. -Conceptually, packages are overloaded. They contain +Conceptually, packages are overloaded. They contain: ------------- Stage objects diff --git a/lib/spack/docs/features.rst b/lib/spack/docs/features.rst index d746e8ad50..57629997aa 100644 --- a/lib/spack/docs/features.rst +++ b/lib/spack/docs/features.rst @@ -14,7 +14,7 @@ Simple package installation Installing the default version of a package is simple. This will install the latest version of the ``mpileaks`` package and all of its dependencies: -.. code-block:: sh +.. code-block:: console $ spack install mpileaks @@ -26,7 +26,7 @@ Spack allows installation to be customized. Users can specify the version, build compiler, compile-time options, and cross-compile platform, all on the command line. -.. code-block:: sh +.. code-block:: console # Install a particular version by appending @ $ spack install mpileaks@1.1.2 @@ -59,7 +59,7 @@ customized extensively. Suppose that ``mpileaks`` depends indirectly on ``libelf`` and ``libdwarf``. Using ``^``, users can add custom configurations for the dependencies: -.. code-block:: sh +.. code-block:: console # Install mpileaks and link it with specific versions of libelf and libdwarf $ spack install mpileaks@1.1.2 %gcc@4.7.3 +debug ^libelf@0.8.12 ^libdwarf@20130729+debug @@ -91,7 +91,7 @@ in pure Python. For example, this command: -.. code-block:: sh +.. code-block:: console $ spack create http://www.mr511.de/software/libelf-0.8.13.tar.gz @@ -101,16 +101,26 @@ creates a simple python file: from spack import * + class Libelf(Package): - homepage = "http://www.example.com/" + """FIXME: Put a proper description of your package here.""" + + # FIXME: Add a proper url for your package's homepage here. + homepage = "http://www.example.com" url = "http://www.mr511.de/software/libelf-0.8.13.tar.gz" version('0.8.13', '4136d7b4c04df68b686570afa26988ac') - def install(self, prefix): - configure("--prefix=%s" % prefix) + # FIXME: Add dependencies if required. + # depends_on('foo') + + def install(self, spec, prefix): + # FIXME: Modify the configure line to suit your build system here. + configure('--prefix={0}'.format(prefix)) + + # FIXME: Add logic to build and install here. make() - make("install") + make('install') It doesn't take much python coding to get from there to a working package: diff --git a/lib/spack/docs/getting_started.rst b/lib/spack/docs/getting_started.rst index 47f66b429b..676697a549 100644 --- a/lib/spack/docs/getting_started.rst +++ b/lib/spack/docs/getting_started.rst @@ -9,7 +9,7 @@ Download Getting spack is easy. You can clone it from the `github repository `_ using this command: -.. code-block:: sh +.. code-block:: console $ git clone https://github.com/llnl/spack.git @@ -18,7 +18,7 @@ full path to this directory is in the ``SPACK_ROOT`` environment variable. Add ``$SPACK_ROOT/bin`` to your path and you're ready to go: -.. code-block:: sh +.. code-block:: console $ export PATH=$SPACK_ROOT/bin:$PATH $ spack install libelf @@ -26,9 +26,10 @@ go: For a richer experience, use Spack's `shell support `_: -.. code-block:: sh +.. code-block:: console # For bash users + $ export SPACK_ROOT=/path/to/spack $ . $SPACK_ROOT/share/spack/setup-env.sh # For tcsh or csh users (note you must set SPACK_ROOT) @@ -48,9 +49,9 @@ You may want to run it out of a prefix other than the git repository you cloned. The ``spack bootstrap`` command provides this functionality. To install spack in a new directory, simply type: -.. code-block:: sh +.. code-block:: console - $ spack bootstrap /my/favorite/prefix + $ spack bootstrap /my/favorite/prefix This will install a new spack script in ``/my/favorite/prefix/bin``, which you can use just like you would the regular spack script. Each diff --git a/lib/spack/docs/index.rst b/lib/spack/docs/index.rst index 603245b5dc..45efcf131f 100644 --- a/lib/spack/docs/index.rst +++ b/lib/spack/docs/index.rst @@ -28,7 +28,7 @@ Get spack from the `github repository `_ and install your first package: -.. code-block:: sh +.. code-block:: console $ git clone https://github.com/llnl/spack.git $ cd spack/bin diff --git a/lib/spack/docs/mirrors.rst b/lib/spack/docs/mirrors.rst index 41e039b838..c4d275a8fc 100644 --- a/lib/spack/docs/mirrors.rst +++ b/lib/spack/docs/mirrors.rst @@ -11,27 +11,29 @@ mirror is a URL that points to a directory, either on the local filesystem or on some server, containing tarballs for all of Spack's packages. -Here's an example of a mirror's directory structure:: - - mirror/ - cmake/ - cmake-2.8.10.2.tar.gz - dyninst/ - dyninst-8.1.1.tgz - dyninst-8.1.2.tgz - libdwarf/ - libdwarf-20130126.tar.gz - libdwarf-20130207.tar.gz - libdwarf-20130729.tar.gz - libelf/ - libelf-0.8.12.tar.gz - libelf-0.8.13.tar.gz - libunwind/ - libunwind-1.1.tar.gz - mpich/ - mpich-3.0.4.tar.gz - mvapich2/ - mvapich2-1.9.tgz +Here's an example of a mirror's directory structure: + +.. code-block:: none + + mirror/ + cmake/ + cmake-2.8.10.2.tar.gz + dyninst/ + dyninst-8.1.1.tgz + dyninst-8.1.2.tgz + libdwarf/ + libdwarf-20130126.tar.gz + libdwarf-20130207.tar.gz + libdwarf-20130729.tar.gz + libelf/ + libelf-0.8.12.tar.gz + libelf-0.8.13.tar.gz + libunwind/ + libunwind-1.1.tar.gz + mpich/ + mpich-3.0.4.tar.gz + mvapich2/ + mvapich2-1.9.tgz The structure is very simple. There is a top-level directory. The second level directories are named after packages, and the third level @@ -57,21 +59,9 @@ contains tarballs for each package, named after each package. ---------------- Mirrors are managed with the ``spack mirror`` command. The help for -``spack mirror`` looks like this:: +``spack mirror`` looks like this: - $ spack mirror -h - usage: spack mirror [-h] SUBCOMMAND ... - - positional arguments: - SUBCOMMAND - create Create a directory to be used as a spack mirror, and fill - it with package archives. - add Add a mirror to Spack. - remove Remove a mirror by name. - list Print out available mirrors to the console. - - optional arguments: - -h, --help show this help message and exit +.. command-output:: spack help mirror The ``create`` command actually builds a mirror by fetching all of its packages from the internet and checksumming them. @@ -92,7 +82,7 @@ The command will iterate through all of Spack's packages and download the safe ones into a directory structure like the one above. Here is what it looks like: -.. code-block:: bash +.. code-block:: console $ spack mirror create libelf libdwarf ==> Created new mirror in spack-mirror-2014-06-24 @@ -134,9 +124,11 @@ Normally, ``spack mirror create`` downloads all the archives it has checksums for. If you want to only create a mirror for a subset of packages, you can do that by supplying a list of package specs on the command line after ``spack mirror create``. For example, this -command:: +command: - $ spack mirror create libelf@0.8.12: boost@1.44: +.. code-block:: console + + $ spack mirror create libelf@0.8.12: boost@1.44: Will create a mirror for libelf versions greater than or equal to 0.8.12 and boost versions greater than or equal to 1.44. @@ -146,7 +138,9 @@ Mirror files ^^^^^^^^^^^^ If you have a *very* large number of packages you want to mirror, you -can supply a file with specs in it, one per line:: +can supply a file with specs in it, one per line: + +.. code-block:: console $ cat specs.txt libdwarf @@ -168,19 +162,21 @@ your site. Once you have a mirror, you need to let spack know about it. This is relatively simple. First, figure out the URL for the mirror. If it's -a file, you can use a file URL like this one:: +a file, you can use a file URL like this one: - file:///Users/gamblin2/spack-mirror-2014-06-24 +.. code-block:: none + + file:///Users/gamblin2/spack-mirror-2014-06-24 That points to the directory on the local filesystem. If it were on a web server, you could use a URL like this one: - https://example.com/some/web-hosted/directory/spack-mirror-2014-06-24 +https://example.com/some/web-hosted/directory/spack-mirror-2014-06-24 Spack will use the URL as the root for all of the packages it fetches. You can tell your Spack installation to use that mirror like this: -.. code-block:: bash +.. code-block:: console $ spack mirror add local_filesystem file:///Users/gamblin2/spack-mirror-2014-06-24 @@ -192,7 +188,9 @@ Each mirror has a name so that you can refer to it again later. ``spack mirror list`` --------------------- -To see all the mirrors Spack knows about, run ``spack mirror list``:: +To see all the mirrors Spack knows about, run ``spack mirror list``: + +.. code-block:: console $ spack mirror list local_filesystem file:///Users/gamblin2/spack-mirror-2014-06-24 @@ -203,7 +201,9 @@ To see all the mirrors Spack knows about, run ``spack mirror list``:: ``spack mirror remove`` ----------------------- -To remove a mirror by name:: +To remove a mirror by name, run: + +.. code-block:: console $ spack mirror remove local_filesystem $ spack mirror list @@ -213,7 +213,9 @@ To remove a mirror by name:: Mirror precedence ----------------- -Adding a mirror really adds a line in ``~/.spack/mirrors.yaml``:: +Adding a mirror really adds a line in ``~/.spack/mirrors.yaml``: + +.. code-block:: yaml mirrors: local_filesystem: file:///Users/gamblin2/spack-mirror-2014-06-24 @@ -232,12 +234,12 @@ Local Default Cache Spack caches resources that are downloaded as part of installs. The cache is a valid spack mirror: it uses the same directory structure and naming scheme as other Spack mirrors (so it can be copied anywhere and referenced with a URL -like other mirrors). The mirror is maintained locally (within the Spack -installation directory) at :file:`var/spack/cache/`. It is always enabled (and -is always searched first when attempting to retrieve files for an installation) +like other mirrors). The mirror is maintained locally (within the Spack +installation directory) at :file:`var/spack/cache/`. It is always enabled (and +is always searched first when attempting to retrieve files for an installation) but can be cleared with :ref:`purge `; the cache directory can also -be deleted manually without issue. +be deleted manually without issue. Caching includes retrieved tarball archives and source control repositories, but -only resources with an associated digest or commit ID (e.g. a revision number +only resources with an associated digest or commit ID (e.g. a revision number for SVN) will be cached. diff --git a/lib/spack/docs/packaging_guide.rst b/lib/spack/docs/packaging_guide.rst index eae928ec76..08d39a266a 100644 --- a/lib/spack/docs/packaging_guide.rst +++ b/lib/spack/docs/packaging_guide.rst @@ -12,9 +12,9 @@ have at least some familiarity with Python, and that you've read the There are two key parts of Spack: - #. **Specs**: expressions for describing builds of software, and - #. **Packages**: Python modules that describe how to build - software according to a spec. +#. **Specs**: expressions for describing builds of software, and +#. **Packages**: Python modules that describe how to build + software according to a spec. Specs allow a user to describe a *particular* build in a way that a package author can understand. Packages allow a the packager to @@ -47,7 +47,7 @@ working. Here's an example: -.. code-block:: sh +.. code-block:: console $ spack create http://www.cmake.org/files/v2.8/cmake-2.8.12.1.tar.gz @@ -62,7 +62,7 @@ strings look like for this package. Using this information, it will try to find versions, Spack prompts you to tell it how many versions you want to download and checksum: -.. code-block:: sh +.. code-block:: console $ spack create http://www.cmake.org/files/v2.8/cmake-2.8.12.1.tar.gz ==> This looks like a URL for cmake version 2.8.12.1. @@ -86,7 +86,9 @@ always choose to download just one tarball initially, and run .. note:: If ``spack create`` fails to detect the package name correctly, - you can try supplying it yourself, e.g.:: + you can try supplying it yourself, e.g.: + + .. code-block:: console $ spack create --name cmake http://www.cmake.org/files/v2.8/cmake-2.8.12.1.tar.gz @@ -104,7 +106,7 @@ always choose to download just one tarball initially, and run Let's say you download 3 tarballs: -.. code-block:: sh +.. code-block:: none Include how many checksums in the package file? (default is 5, q to abort) 3 ==> Downloading... @@ -121,27 +123,28 @@ file in your favorite ``$EDITOR``: .. code-block:: python :linenos: - # FIXME: - # This is a template package file for Spack. We've conveniently - # put "FIXME" labels next to all the things you'll want to change. # - # Once you've edited all the FIXME's, delete this whole message, - # save this file, and test out your package like this: + # This is a template package file for Spack. We've put "FIXME" + # next to all the things you'll want to change. Once you've handled + # them, you can save this file and test your package like this: # # spack install cmake # - # You can always get back here to change things with: + # You can edit this file again by typing: # # spack edit cmake # - # See the spack documentation for more information on building - # packages. + # See the Spack documentation for more information on packaging. + # If you submit this package back to Spack as a pull request, + # please first remove this boilerplate and all FIXME comments. # from spack import * + class Cmake(Package): - """FIXME: put a proper description of your package here.""" - # FIXME: add a proper url for your package's homepage here. + """FIXME: Put a proper description of your package here.""" + + # FIXME: Add a proper url for your package's homepage here. homepage = "http://www.example.com" url = "http://www.cmake.org/files/v2.8/cmake-2.8.12.1.tar.gz" @@ -165,33 +168,33 @@ done for you. In the generated package, the download ``url`` attribute is already set. All the things you still need to change are marked with -``FIXME`` labels. The first ``FIXME`` refers to the commented -instructions at the top of the file. You can delete these -instructions after reading them. The rest of them are as follows: +``FIXME`` labels. You can delete the commented instructions between +the license and the first import statement after reading them. +The rest of the tasks you need to do are as follows: - #. Add a description. +#. Add a description. - Immediately inside the package class is a *docstring* in - triple-quotes (``"""``). It's used to generate the description - shown when users run ``spack info``. + Immediately inside the package class is a *docstring* in + triple-quotes (``"""``). It's used to generate the description + shown when users run ``spack info``. - #. Change the ``homepage`` to a useful URL. +#. Change the ``homepage`` to a useful URL. - The ``homepage`` is displayed when users run ``spack info`` so - that they can learn about packages. + The ``homepage`` is displayed when users run ``spack info`` so + that they can learn about packages. - #. Add ``depends_on()`` calls for the package's dependencies. +#. Add ``depends_on()`` calls for the package's dependencies. - ``depends_on`` tells Spack that other packages need to be built - and installed before this one. See `dependencies_`. + ``depends_on`` tells Spack that other packages need to be built + and installed before this one. See :ref:`dependencies`. - #. Get the ``install()`` method working. +#. Get the ``install()`` method working. - The ``install()`` method implements the logic to build a - package. The code should look familiar; it is designed to look - like a shell script. Specifics will differ depending on the package, - and :ref:`implementing the install method ` is - covered in detail later. + The ``install()`` method implements the logic to build a + package. The code should look familiar; it is designed to look + like a shell script. Specifics will differ depending on the package, + and :ref:`implementing the install method ` is + covered in detail later. Before going into details, we'll cover a few more basics. @@ -205,41 +208,41 @@ One of the easiest ways to learn to write packages is to look at existing ones. You can edit a package file by name with the ``spack edit`` command: -.. code-block:: sh +.. code-block:: console - spack edit cmake + $ spack edit cmake So, if you used ``spack create`` to create a package, then saved and closed the resulting file, you can get back to it with ``spack edit``. The ``cmake`` package actually lives in -``$SPACK_ROOT/var/spack/repos/builtin/packages/cmake/package.py``, but this provides -a much simpler shortcut and saves you the trouble of typing the full -path. +``$SPACK_ROOT/var/spack/repos/builtin/packages/cmake/package.py``, +but this provides a much simpler shortcut and saves you the trouble +of typing the full path. If you try to edit a package that doesn't exist, Spack will recommend -using ``spack create`` or ``spack edit -f``: +using ``spack create`` or ``spack edit --force``: -.. code-block:: sh +.. code-block:: console $ spack edit foo ==> Error: No package 'foo'. Use spack create, or supply -f/--force to edit a new file. .. _spack-edit-f: -^^^^^^^^^^^^^^^^^ -``spack edit -f`` -^^^^^^^^^^^^^^^^^ +^^^^^^^^^^^^^^^^^^^^^^ +``spack edit --force`` +^^^^^^^^^^^^^^^^^^^^^^ -``spack edit -f`` can be used to create a new, minimal boilerplate +``spack edit --force`` can be used to create a new, minimal boilerplate package: -.. code-block:: sh +.. code-block:: console $ spack edit -f foo Unlike ``spack create``, which infers names and versions, and which actually downloads the tarball and checksums it for you, ``spack edit --f`` has no such fanciness. It will substitute dummy values for you +--force`` has no such fanciness. It will substitute dummy values for you to fill in yourself: .. code-block:: python @@ -256,7 +259,7 @@ to fill in yourself: version('1.0', '0123456789abcdef0123456789abcdef') def install(self, spec, prefix): - configure("--prefix=" + prefix) + configure("--prefix=%s" % prefix) make() make("install") @@ -267,16 +270,9 @@ version of your package from the archive URL. Naming & directory structure ---------------------------- -.. note:: - - Spack's default naming and directory structure will change in - version 0.9. Specifically, 0.9 will stop using directory names - with special characters like ``@``, to avoid interfering with - certain packages that do not handle this well. - This section describes how packages need to be named, and where they -live in Spack's directory structure. In general, `spack-create`_ and -`spack-edit`_ handle creating package files for you, so you can skip +live in Spack's directory structure. In general, :ref:`spack-create` and +:ref:`spack-edit` handle creating package files for you, so you can skip most of the details here. ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -291,13 +287,15 @@ Packages themselves live in ``$SPACK_ROOT/var/spack/repos/builtin/packages``. If you ``cd`` to that directory, you will see directories for each package: -.. command-output:: cd $SPACK_ROOT/var/spack/repos/builtin/packages; ls -CF +.. command-output:: cd $SPACK_ROOT/var/spack/repos/builtin/packages && ls :shell: :ellipsis: 10 Each directory contains a file called ``package.py``, which is where all the python code for the package goes. For example, the ``libelf`` -package lives in:: +package lives in: + +.. code-block:: none $SPACK_ROOT/var/spack/repos/builtin/packages/libelf/package.py @@ -312,7 +310,7 @@ Packages are named after the directory containing ``package.py``. It is preferred, but not required, that the directory, and thus the package name, are lower case. So, ``libelf``'s ``package.py`` lives in a directory called ``libelf``. The ``package.py`` file defines a class called ``Libelf``, which -extends Spack's ``Package`` class. for example, here is +extends Spack's ``Package`` class. For example, here is ``$SPACK_ROOT/var/spack/repos/builtin/packages/libelf/package.py``: .. code-block:: python @@ -334,7 +332,7 @@ The **directory name** (``libelf``) determines the package name that users should provide on the command line. e.g., if you type any of these: -.. code-block:: sh +.. code-block:: console $ spack install libelf $ spack install libelf@0.8.13 @@ -373,7 +371,7 @@ some examples: ================= ================= In general, you won't have to remember this naming convention because -`spack-create`_ and `spack-edit`_ handle the details for you. +:ref:`spack-create` and :ref:`spack-edit` handle the details for you. ------------------- Adding new versions @@ -482,7 +480,7 @@ to use based on the hash length. If you have one or more files to checksum, you can use the ``spack md5`` command to do it: -.. code-block:: sh +.. code-block:: console $ spack md5 foo-8.2.1.tar.gz foo-8.2.2.tar.gz ==> 2 MD5 checksums: @@ -492,7 +490,7 @@ command to do it: ``spack md5`` also accepts one or more URLs and automatically downloads the files for you: -.. code-block:: sh +.. code-block:: console $ spack md5 http://example.com/foo-8.2.1.tar.gz ==> Trying to fetch from http://example.com/foo-8.2.1.tar.gz @@ -514,7 +512,7 @@ If you want to add new versions to a package you've already created, this is automated with the ``spack checksum`` command. Here's an example for ``libelf``: -.. code-block:: sh +.. code-block:: console $ spack checksum libelf ==> Found 16 versions of libelf. @@ -538,7 +536,7 @@ they're released). It fetches the tarballs you ask for and prints out a list of ``version`` commands ready to copy/paste into your package file: -.. code-block:: sh +.. code-block:: console ==> Checksummed new versions of libelf: version('0.8.13', '4136d7b4c04df68b686570afa26988ac') @@ -605,7 +603,7 @@ same thing every time you fetch a particular version. Life isn't always simple, though, so this is not strictly enforced. When fetching from from the branch corresponding to the development version -(often called ``master``,``trunk`` or ``dev``), it is recommended to +(often called ``master``, ``trunk``, or ``dev``), it is recommended to call this version ``develop``. Spack has special treatment for this version so that ``@develop`` will satisfy dependencies like ``depends_on(abc, when="@x.y.z:")``. In other words, ``@develop`` is @@ -627,11 +625,11 @@ Git Git fetching is enabled with the following parameters to ``version``: - * ``git``: URL of the git repository. - * ``tag``: name of a tag to fetch. - * ``branch``: name of a branch to fetch. - * ``commit``: SHA hash (or prefix) of a commit to fetch. - * ``submodules``: Also fetch submodules when checking out this repository. +* ``git``: URL of the git repository. +* ``tag``: name of a tag to fetch. +* ``branch``: name of a branch to fetch. +* ``commit``: SHA hash (or prefix) of a commit to fetch. +* ``submodules``: Also fetch submodules when checking out this repository. Only one of ``tag``, ``branch``, or ``commit`` can be used at a time. @@ -675,7 +673,7 @@ Commits version('2014-10-08', git='https://github.com/example-project/example.git', commit='9d38cd4e2c94c3cea97d0e2924814acc') - This doesn't have to be a full hash; You can abbreviate it as you'd + This doesn't have to be a full hash; you can abbreviate it as you'd expect with git: .. code-block:: python @@ -705,14 +703,13 @@ Installing You can fetch and install any of the versions above as you'd expect, by using ``@`` in a spec: -.. code-block:: sh +.. code-block:: console - spack install example@2014-10-08 + $ spack install example@2014-10-08 Git and other VCS versions will show up in the list of versions when a user runs ``spack info ``. - .. _hg-fetch: ^^^^^^^^^ @@ -858,12 +855,12 @@ For example, let's take a look at the package for the PGI compilers. .. code-block:: python - # Licensing - license_required = True - license_comment = '#' - license_files = ['license.dat'] - license_vars = ['PGROUPD_LICENSE_FILE', 'LM_LICENSE_FILE'] - license_url = 'http://www.pgroup.com/doc/pgiinstall.pdf' + # Licensing + license_required = True + license_comment = '#' + license_files = ['license.dat'] + license_vars = ['PGROUPD_LICENSE_FILE', 'LM_LICENSE_FILE'] + license_url = 'http://www.pgroup.com/doc/pgiinstall.pdf' As you can see, PGI requires a license. Its license manager, FlexNet, uses the ``#`` symbol to denote a comment. It expects the license file to be @@ -886,39 +883,39 @@ this: .. code-block:: sh - # A license is required to use pgi. - # - # The recommended solution is to store your license key in this global - # license file. After installation, the following symlink(s) will be - # added to point to this file (relative to the installation prefix): - # - # license.dat - # - # Alternatively, use one of the following environment variable(s): - # - # PGROUPD_LICENSE_FILE - # LM_LICENSE_FILE - # - # If you choose to store your license in a non-standard location, you may - # set one of these variable(s) to the full pathname to the license file, or - # port@host if you store your license keys on a dedicated license server. - # You will likely want to set this variable in a module file so that it - # gets loaded every time someone tries to use pgi. - # - # For further information on how to acquire a license, please refer to: - # - # http://www.pgroup.com/doc/pgiinstall.pdf - # - # You may enter your license below. + # A license is required to use pgi. + # + # The recommended solution is to store your license key in this global + # license file. After installation, the following symlink(s) will be + # added to point to this file (relative to the installation prefix): + # + # license.dat + # + # Alternatively, use one of the following environment variable(s): + # + # PGROUPD_LICENSE_FILE + # LM_LICENSE_FILE + # + # If you choose to store your license in a non-standard location, you may + # set one of these variable(s) to the full pathname to the license file, or + # port@host if you store your license keys on a dedicated license server. + # You will likely want to set this variable in a module file so that it + # gets loaded every time someone tries to use pgi. + # + # For further information on how to acquire a license, please refer to: + # + # http://www.pgroup.com/doc/pgiinstall.pdf + # + # You may enter your license below. You can add your license directly to this file, or tell FlexNet to use a license stored on a separate license server. Here is an example that points to a license server called licman1: -.. code-block:: sh +.. code-block:: none - SERVER licman1.mcs.anl.gov 00163eb7fba5 27200 - USE_SERVER + SERVER licman1.mcs.anl.gov 00163eb7fba5 27200 + USE_SERVER If your package requires the license to install, you can reference the location of this global license using ``self.global_license_file``. @@ -949,7 +946,7 @@ after it's downloaded. ^^^^^^^^^ You can specify patches in your package file with the ``patch()`` -function. ``patch`` looks like this: +directive. ``patch`` looks like this: .. code-block:: python @@ -961,7 +958,9 @@ The first argument can be either a URL or a filename. It specifies a patch file that should be applied to your source. If the patch you supply is a filename, then the patch needs to live within the spack source tree. For example, the patch above lives in a directory -structure like this:: +structure like this: + +.. code-block:: none $SPACK_ROOT/var/spack/repos/builtin/packages/ mvapich2/ @@ -982,52 +981,58 @@ from the URL and then applied to your source code. ``patch`` can take two options keyword arguments. They are: +"""""""" ``when`` - If supplied, this is a spec that tells spack when to apply - the patch. If the installed package spec matches this spec, the - patch will be applied. In our example above, the patch is applied - when mvapich is at version ``1.9`` or higher. +"""""""" + +If supplied, this is a spec that tells spack when to apply +the patch. If the installed package spec matches this spec, the +patch will be applied. In our example above, the patch is applied +when mvapich is at version ``1.9`` or higher. +""""""""" ``level`` - This tells spack how to run the ``patch`` command. By default, - the level is 1 and spack runs ``patch -p1``. If level is 2, - spack will run ``patch -p2``, and so on. - - A lot of people are confused by level, so here's a primer. If you - look in your patch file, you may see something like this: - - .. code-block:: diff - :linenos: - - --- a/src/mpi/romio/adio/ad_lustre/ad_lustre_rwcontig.c 2013-12-10 12:05:44.806417000 -0800 - +++ b/src/mpi/romio/adio/ad_lustre/ad_lustre_rwcontig.c 2013-12-10 11:53:03.295622000 -0800 - @@ -8,7 +8,7 @@ - * Copyright (C) 2008 Sun Microsystems, Lustre group - */ - - -#define _XOPEN_SOURCE 600 - +//#define _XOPEN_SOURCE 600 - #include - #include - #include "ad_lustre.h" - - Lines 1-2 show paths with synthetic ``a/`` and ``b/`` prefixes. These - are placeholders for the two ``mvapich2`` source directories that - ``diff`` compared when it created the patch file. This is git's - default behavior when creating patch files, but other programs may - behave differently. - - ``-p1`` strips off the first level of the prefix in both paths, - allowing the patch to be applied from the root of an expanded mvapich2 - archive. If you set level to ``2``, it would strip off ``src``, and - so on. - - It's generally easier to just structure your patch file so that it - applies cleanly with ``-p1``, but if you're using a patch you didn't - create yourself, ``level`` can be handy. +""""""""" + +This tells spack how to run the ``patch`` command. By default, +the level is 1 and spack runs ``patch -p 1``. If level is 2, +spack will run ``patch -p 2``, and so on. + +A lot of people are confused by level, so here's a primer. If you +look in your patch file, you may see something like this: + +.. code-block:: diff + :linenos: + + --- a/src/mpi/romio/adio/ad_lustre/ad_lustre_rwcontig.c 2013-12-10 12:05:44.806417000 -0800 + +++ b/src/mpi/romio/adio/ad_lustre/ad_lustre_rwcontig.c 2013-12-10 11:53:03.295622000 -0800 + @@ -8,7 +8,7 @@ + * Copyright (C) 2008 Sun Microsystems, Lustre group + \*/ + + -#define _XOPEN_SOURCE 600 + +//#define _XOPEN_SOURCE 600 + #include + #include + #include "ad_lustre.h" + +Lines 1-2 show paths with synthetic ``a/`` and ``b/`` prefixes. These +are placeholders for the two ``mvapich2`` source directories that +``diff`` compared when it created the patch file. This is git's +default behavior when creating patch files, but other programs may +behave differently. + +``-p1`` strips off the first level of the prefix in both paths, +allowing the patch to be applied from the root of an expanded mvapich2 +archive. If you set level to ``2``, it would strip off ``src``, and +so on. + +It's generally easier to just structure your patch file so that it +applies cleanly with ``-p1``, but if you're using a patch you didn't +create yourself, ``level`` can be handy. ^^^^^^^^^^^^^^^^^^^^^ -``patch()`` functions +Patch functions ^^^^^^^^^^^^^^^^^^^^^ In addition to supplying patch files, you can write a custom function @@ -1037,35 +1042,10 @@ handles ``RPATH``: .. _pyside-patch: -.. code-block:: python +.. literalinclude:: ../../../var/spack/repos/builtin/packages/py-pyside/package.py + :pyobject: PyPyside.patch :linenos: - class PyPyside(Package): - ... - - def patch(self): - """Undo PySide RPATH handling and add Spack RPATH.""" - # Figure out the special RPATH - pypkg = self.spec['python'].package - rpath = self.rpath - rpath.append(os.path.join(self.prefix, pypkg.site_packages_dir, 'PySide')) - - # Add Spack's standard CMake args to the sub-builds. - # They're called BY setup.py so we have to patch it. - filter_file( - r'OPTION_CMAKE,', - r'OPTION_CMAKE, ' + ( - '"-DCMAKE_INSTALL_RPATH_USE_LINK_PATH=FALSE", ' - '"-DCMAKE_INSTALL_RPATH=%s",' % ':'.join(rpath)), - 'setup.py') - - # PySide tries to patch ELF files to remove RPATHs - # Disable this and go with the one we set. - filter_file( - r'^\s*rpath_cmd\(pyside_path, srcpath\)', - r'#rpath_cmd(pyside_path, srcpath)', - 'pyside_postinstall.py') - A ``patch`` function, if present, will be run after patch files are applied and before ``install()`` is run. @@ -1085,9 +1065,9 @@ dependencies are found when it runs. It does this using `RPATHs `_. An RPATH is a search path, stored in a binary (an executable or library), that tells the dynamic loader where to find its dependencies at runtime. You may be -familiar with ```LD_LIBRARY_PATH`` +familiar with `LD_LIBRARY_PATH `_ -on Linux or ```DYLD_LIBRARY_PATH`` +on Linux or `DYLD_LIBRARY_PATH ` on Mac OS X. RPATH is similar to these paths, in that it tells the loader where to find libraries. Unlike them, it is embedded in @@ -1095,32 +1075,32 @@ the binary and not set in each user's environment. RPATHs in Spack are handled in one of three ways: - 1. For most packages, RPATHs are handled automatically using Spack's - :ref:`compiler wrappers `. These wrappers are - set in standard variables like ``CC``, ``CXX``, and ``FC``, so - most build systems (autotools and many gmake systems) pick them - up and use them. - 2. CMake also respects Spack's compiler wrappers, but many CMake - builds have logic to overwrite RPATHs when binaries are - installed. Spack provides the ``std_cmake_args`` variable, which - includes parameters necessary for CMake build use the right - installation RPATH. It can be used like this when ``cmake`` is - invoked: - - .. code-block:: python - - class MyPackage(Package): - ... - def install(self, spec, prefix): - cmake('..', *std_cmake_args) - make() - make('install') - - 3. If you need to modify the build to add your own RPATHs, you can - use the ``self.rpath`` property of your package, which will - return a list of all the RPATHs that Spack will use when it - links. You can see this how this is used in the :ref:`PySide - example ` above. +#. For most packages, RPATHs are handled automatically using Spack's + :ref:`compiler wrappers `. These wrappers are + set in standard variables like ``CC``, ``CXX``, ``F77``, and ``FC``, + so most build systems (autotools and many gmake systems) pick them + up and use them. +#. CMake also respects Spack's compiler wrappers, but many CMake + builds have logic to overwrite RPATHs when binaries are + installed. Spack provides the ``std_cmake_args`` variable, which + includes parameters necessary for CMake build use the right + installation RPATH. It can be used like this when ``cmake`` is + invoked: + + .. code-block:: python + + class MyPackage(Package): + ... + def install(self, spec, prefix): + cmake('..', *std_cmake_args) + make() + make('install') + +#. If you need to modify the build to add your own RPATHs, you can + use the ``self.rpath`` property of your package, which will + return a list of all the RPATHs that Spack will use when it + links. You can see this how this is used in the :ref:`PySide + example ` above. -------------------- Finding new versions @@ -1133,6 +1113,7 @@ You've already seen the ``homepage`` and ``url`` package attributes: from spack import * + class Mpich(Package): """MPICH is a high performance and widely portable implementation of the Message Passing Interface (MPI) standard.""" @@ -1194,7 +1175,9 @@ the ``list_url``, because that is where links to old versions are: versions on a single webpage, but not all do. For example, ``mpich`` has a tarball URL that looks like this: - url = "http://www.mpich.org/static/downloads/3.0.4/mpich-3.0.4.tar.gz" +.. code-block:: python + + url = "http://www.mpich.org/static/downloads/3.0.4/mpich-3.0.4.tar.gz" But its downloads are in many different subdirectories of ``http://www.mpich.org/static/downloads/``. So, we need to add a @@ -1344,7 +1327,7 @@ Or a requirement for a particular variant or compiler flags: depends_on("libelf@0.8+debug") depends_on('libelf debug=True') - depends_on('libelf cppflags="-fPIC") + depends_on('libelf cppflags="-fPIC"') Both users *and* package authors can use the same spec syntax to refer to different package configurations. Users use the spec syntax on the @@ -1356,9 +1339,9 @@ Additionally, dependencies may be specified for specific use cases: .. code-block:: python - depends_on("cmake", type="build") - depends_on("libelf", type=("build", "link")) - depends_on("python", type="run") + depends_on("cmake", type="build") + depends_on("libelf", type=("build", "link")) + depends_on("python", type="run") The dependency types are: @@ -1405,34 +1388,16 @@ packages that depend on a particular Qt installation will find it. The arguments to this function are: - * **module**: the module of the dependent package, where global - properties can be assigned. - * **spec**: the spec of the *dependency package* (the one the function is called on). - * **dep_spec**: the spec of the dependent package (i.e. dep_spec depends on spec). +* **module**: the module of the dependent package, where global + properties can be assigned. +* **spec**: the spec of the *dependency package* (the one the function is called on). +* **dep_spec**: the spec of the dependent package (i.e. dep_spec depends on spec). A good example of using these is in the Python package: -.. code-block:: python - - def setup_dependent_environment(self, module, spec, dep_spec): - # Python extension builds can have a global python executable function - module.python = Executable(join_path(spec.prefix.bin, 'python')) - - # Add variables for lib/pythonX.Y and lib/pythonX.Y/site-packages dirs. - module.python_lib_dir = os.path.join(dep_spec.prefix, self.python_lib_dir) - module.python_include_dir = os.path.join(dep_spec.prefix, self.python_include_dir) - module.site_packages_dir = os.path.join(dep_spec.prefix, self.site_packages_dir) - - # Make the site packages directory if it does not exist already. - mkdirp(module.site_packages_dir) - - # Set PYTHONPATH to include site-packages dir for the - # extension and any other python extensions it depends on. - python_paths = [] - for d in dep_spec.traverse(): - if d.package.extends(self.spec): - python_paths.append(os.path.join(d.prefix, self.site_packages_dir)) - os.environ['PYTHONPATH'] = ':'.join(python_paths) +.. literalinclude:: ../../../var/spack/repos/builtin/packages/python/package.py + :pyobject: Python.setup_dependent_environment + :linenos: The first thing that happens here is that the ``python`` command is inserted into module scope of the dependent. This allows most python @@ -1441,7 +1406,7 @@ packages to have a very simple install method, like this: .. code-block:: python def install(self, spec, prefix): - python('setup.py', 'install', '--prefix=%s' % prefix) + python('setup.py', 'install', '--prefix={0}'.format(prefix)) Python's ``setup_dependent_environment`` method also sets up some other variables, creates a directory, and sets up the ``PYTHONPATH`` @@ -1483,7 +1448,7 @@ activate``. When it is activated, all the files in its prefix will be symbolically linked into the prefix of the python package. Many packages produce Python extensions for *some* variants, but not -others: they should extend ``python`` only if the apropriate +others: they should extend ``python`` only if the appropriate variant(s) are selected. This may be accomplished with conditional ``extends()`` declarations: @@ -1502,13 +1467,15 @@ when it does the activation: .. code-block:: python - class PyNose(Package): + class PySncosmo(Package): ... - extends('python', ignore=r'bin/nosetests.*$') + # py-sncosmo binaries are duplicates of those from py-astropy + extends('python', ignore=r'bin/.*') + depends_on('py-astropy') ... -The code above will prevent ``$prefix/bin/nosetests`` from being -linked in at activation time. +The code above will prevent everything in the ``$prefix/bin/`` directory +from being linked in at activation time. .. note:: @@ -1537,15 +1504,9 @@ same way that Python does. Let's look at Python's activate function: -.. code-block:: python - - def activate(self, ext_pkg, **kwargs): - kwargs.update(ignore=self.python_ignore(ext_pkg, kwargs)) - super(Python, self).activate(ext_pkg, **kwargs) - - exts = spack.install_layout.extension_map(self.spec) - exts[ext_pkg.name] = ext_pkg.spec - self.write_easy_install_pth(exts) +.. literalinclude:: ../../../var/spack/repos/builtin/packages/python/package.py + :pyobject: Python.activate + :linenos: This function is called on the *extendee* (Python). It first calls ``activate`` in the superclass, which handles symlinking the @@ -1555,16 +1516,9 @@ Python's setuptools. Deactivate behaves similarly to activate, but it unlinks files: -.. code-block:: python - - def deactivate(self, ext_pkg, **kwargs): - kwargs.update(ignore=self.python_ignore(ext_pkg, kwargs)) - super(Python, self).deactivate(ext_pkg, **kwargs) - - exts = spack.install_layout.extension_map(self.spec) - if ext_pkg.name in exts: # Make deactivate idempotent. - del exts[ext_pkg.name] - self.write_easy_install_pth(exts) +.. literalinclude:: ../../../var/spack/repos/builtin/packages/python/package.py + :pyobject: Python.deactivate + :linenos: Both of these methods call some custom functions in the Python package. See the source for Spack's Python package for details. @@ -1708,7 +1662,7 @@ the package ``foo`` declares this: Suppose a user invokes ``spack install`` like this: -.. code-block:: sh +.. code-block:: console $ spack install foo ^mpich@1.0 @@ -1726,9 +1680,9 @@ command line ` and within package definitions, we can talk about how Spack puts all of this information together. When you run this: -.. code-block:: sh +.. code-block:: console - spack install mpileaks ^callpath@1.0+debug ^libelf@0.8.11 + $ spack install mpileaks ^callpath@1.0+debug ^libelf@0.8.11 Spack parses the command line and builds a spec from the description. The spec says that ``mpileaks`` should be built with the ``callpath`` @@ -1744,7 +1698,9 @@ abstract spec is partially specified. In other words, it could describe more than one build of a package. Spack does this to make things easier on the user: they should only have to specify as much of the package spec as they care about. Here's an example partial spec -DAG, based on the constraints above:: +DAG, based on the constraints above: + +.. code-block:: none mpileaks ^callpath@1.0+debug @@ -1780,7 +1736,9 @@ remaining unspecified parts in order to install. This process is called **concretization**. Concretization happens in between the time the user runs ``spack install`` and the time the ``install()`` method is called. The concretized version of the spec above might look like -this:: +this: + +.. code-block:: none mpileaks@2.3%gcc@4.7.3 arch=linux-debian7-x86_64 ^callpath@1.0%gcc@4.7.3+debug arch=linux-debian7-x86_64 @@ -1819,7 +1777,7 @@ the preferences of their own users. For an arbitrary spec, you can see the result of concretization by running ``spack spec``. For example: -.. code-block:: sh +.. code-block:: console $ spack spec dyninst@8.0.1 dyninst@8.0.1 @@ -1857,7 +1815,7 @@ where the real work of installation happens, and it's the main part of the package you'll need to customize for each piece of software. .. literalinclude:: ../../../var/spack/repos/builtin/packages/libelf/package.py - :start-after: 0.8.12 + :pyobject: Libelf.install :linenos: ``install`` takes a ``spec``: a description of how the package should @@ -1945,8 +1903,8 @@ Environment variables Spack sets a number of standard environment variables that serve two purposes: - #. Make build systems use Spack's compiler wrappers for their builds. - #. Allow build systems to find dependencies more easily +#. Make build systems use Spack's compiler wrappers for their builds. +#. Allow build systems to find dependencies more easily The Compiler environment variables that Spack sets are: @@ -1960,7 +1918,7 @@ The Compiler environment variables that Spack sets are: ============ =============================== All of these are standard variables respected by most build systems. -If your project uses ``autotools`` or ``CMake``, then it should pick +If your project uses ``Autotools`` or ``CMake``, then it should pick them up automatically when you run ``configure`` or ``cmake`` in the ``install()`` function. Many traditional builds using GNU Make and BSD make also respect these variables, so they may work with these @@ -1977,12 +1935,12 @@ In addition to the compiler variables, these variables are set before entering ``install()`` so that packages can locate dependencies easily: - ======================= ============================= - ``PATH`` Set to point to ``/bin`` directories of dependencies - ``CMAKE_PREFIX_PATH`` Path to dependency prefixes for CMake - ``PKG_CONFIG_PATH`` Path to any pkgconfig directories for dependencies - ``PYTHONPATH`` Path to site-packages dir of any python dependencies - ======================= ============================= +===================== ==================================================== +``PATH`` Set to point to ``/bin`` directories of dependencies +``CMAKE_PREFIX_PATH`` Path to dependency prefixes for CMake +``PKG_CONFIG_PATH`` Path to any pkgconfig directories for dependencies +``PYTHONPATH`` Path to site-packages dir of any python dependencies +===================== ==================================================== ``PATH`` is set up to point to dependencies ``/bin`` directories so that you can use tools installed by dependency packages at build time. @@ -1993,7 +1951,7 @@ For example, ``$MPICH_ROOT/bin/mpicc`` is frequently used by dependencies of where ``cmake`` will search for dependency libraries and headers. This causes all standard CMake find commands to look in the paths of your dependencies, so you *do not* have to manually specify arguments -like ``-D DEPENDENCY_DIR=/path/to/dependency`` to ``cmake``. More on +like ``-DDEPENDENCY_DIR=/path/to/dependency`` to ``cmake``. More on this is `in the CMake documentation `_. ``PKG_CONFIG_PATH`` is for packages that attempt to discover @@ -2031,13 +1989,15 @@ flags to the compile line so that dependencies can be easily found. These flags are added for each dependency, if they exist: Compile-time library search paths - * ``-L$dep_prefix/lib`` - * ``-L$dep_prefix/lib64`` +* ``-L$dep_prefix/lib`` +* ``-L$dep_prefix/lib64`` + Runtime library search paths (RPATHs) - * ``$rpath_flag$dep_prefix/lib`` - * ``$rpath_flag$dep_prefix/lib64`` +* ``$rpath_flag$dep_prefix/lib`` +* ``$rpath_flag$dep_prefix/lib64`` + Include search paths - * ``-I$dep_prefix/include`` +* ``-I$dep_prefix/include`` An example of this would be the ``libdwarf`` build, which has one dependency: ``libelf``. Every call to ``cc`` in the ``libdwarf`` @@ -2083,7 +2043,7 @@ Compiler flags In rare circumstances such as compiling and running small unit tests, a package developer may need to know what are the appropriate compiler flags to enable features like ``OpenMP``, ``c++11``, ``c++14`` and alike. To that end the -compiler classes in ``spack`` implement the following _properties_ : +compiler classes in ``spack`` implement the following **properties**: ``openmp_flag``, ``cxx11_flag``, ``cxx14_flag``, which can be accessed in a package by ``self.compiler.cxx11_flag`` and alike. Note that the implementation is such that if a given compiler version does not support this feature, an @@ -2102,7 +2062,7 @@ Message Parsing Interface (MPI) It is common for high performance computing software/packages to use ``MPI``. As a result of conretization, a given package can be built using different implementations of MPI such as ``Openmpi``, ``MPICH`` or ``IntelMPI``. -In some scenarios to configure a package one have to provide it with appropriate MPI +In some scenarios, to configure a package, one has to provide it with appropriate MPI compiler wrappers such as ``mpicc``, ``mpic++``. However different implementations of ``MPI`` may have different names for those wrappers. In order to make package's ``install()`` method indifferent to the @@ -2119,9 +2079,9 @@ Blas and Lapack libraries Different packages provide implementation of ``Blas`` and ``Lapack`` routines. The names of the resulting static and/or shared libraries differ from package -to package. In order to make ``install()`` method indifferent to the +to package. In order to make the ``install()`` method indifferent to the choice of ``Blas`` implementation, each package which provides it -sets up ``self.spec.blas_shared_lib`` and ``self.spec.blas_static_lib `` to +sets up ``self.spec.blas_shared_lib`` and ``self.spec.blas_static_lib`` to point to the shared and static ``Blas`` libraries, respectively. The same applies to packages which provide ``Lapack``. Package developers are advised to use these variables, for example ``spec['blas'].blas_shared_lib`` instead of @@ -2241,9 +2201,14 @@ do that, e.g.: .. code-block:: python + configure_args = [ + '--prefix={0}'.format(prefix) + ] + if spec.satisfies('@1.2:1.4'): configure_args.append("CXXFLAGS='-DWITH_FEATURE'") - configure('--prefix=' + prefix, *configure_args) + + configure(*configure_args) This works for compilers, too: @@ -2306,18 +2271,18 @@ of your dependencies. You can do that by sub-scripting the spec: .. code-block:: python - my_mpi = spec['mpich'] + my_mpi = spec['mpi'] The value in the brackets needs to be some package name, and spec needs to depend on that package, or the operation will fail. For example, the above code will fail if the ``spec`` doesn't depend on -``mpich``. The value returned and assigned to ``my_mpi``, is itself +``mpi``. The value returned and assigned to ``my_mpi``, is itself just another ``Spec`` object, so you can do all the same things you would do with the package's own spec: .. code-block:: python - mpicc = new_path(my_mpi.prefix.bin, 'mpicc') + mpicc = join_path(my_mpi.prefix.bin, 'mpicc') .. _multimethods: @@ -2330,7 +2295,7 @@ packages, based on whether the package's spec satisfies particular criteria. The ``@when`` annotation lets packages declare multiple versions of -methods like install() that depend on the package's spec. For +methods like ``install()`` that depend on the package's spec. For example: .. code-block:: python @@ -2341,16 +2306,17 @@ example: def install(self, prefix): # Do default install - @when('arch=linux-debian7-x86_64') + @when('arch=chaos_5_x86_64_ib') def install(self, prefix): # This will be executed instead of the default install if # the package's sys_type() is chaos_5_x86_64_ib. - @when('arch=linux-debian7-x86_64") + @when('arch=linux-debian7-x86_64') def install(self, prefix): - # This will be executed if the package's sys_type is bgqos_0 + # This will be executed if the package's sys_type() is + # linux-debian7-x86_64. -In the above code there are three versions of install(), two of which +In the above code there are three versions of ``install()``, two of which are specialized for particular platforms. The version that is called depends on the architecture of the package spec. @@ -2428,17 +2394,9 @@ Shell command functions Recall the install method from ``libelf``: -.. code-block:: python - - def install(self, spec, prefix): - configure("--prefix=" + prefix, - "--enable-shared", - "--disable-dependency-tracking", - "--disable-debug") - make() - - # The mkdir commands in libelf's install can fail in parallel - make("install", parallel=False) +.. literalinclude:: ../../../var/spack/repos/builtin/packages/libelf/package.py + :pyobject: Libelf.install + :linenos: Normally in Python, you'd have to write something like this in order to execute shell commands: @@ -2446,7 +2404,7 @@ to execute shell commands: .. code-block:: python import subprocess - subprocess.check_call('configure', '--prefix=' + prefix) + subprocess.check_call('configure', '--prefix={0}'.format(prefix)) We've tried to make this a bit easier by providing callable wrapper objects for some shell commands. By default, ``configure``, @@ -2488,7 +2446,7 @@ Consider a simple autotools build like this: .. code-block:: python def install(self, spec, prefix): - configure("--prefix=" + prefix) + configure("--prefix={0}".format(prefix)) make() make("install") @@ -2525,7 +2483,7 @@ Now, after ``install()`` runs, Spack will check whether ``$prefix/include/libelf.h`` exists and is a file, and whether ``$prefix/lib`` exists and is a directory. If the checks fail, then the build will fail and the install prefix will be removed. If they -succeed, Spack considers the build succeeful and keeps the prefix in +succeed, Spack considers the build successful and keeps the prefix in place. .. _file-manipulation: @@ -2633,8 +2591,8 @@ File functions install('my-header.h', join_path(prefix.include)) -:py:func:`join_path(prefix, *args) ` Like - ``os.path.join``, this joins paths using the OS path separator. +:py:func:`join_path(prefix, *args) ` + Like ``os.path.join``, this joins paths using the OS path separator. However, this version allows an arbitrary number of arguments, so you can string together many path components. @@ -2708,9 +2666,9 @@ variant names are: Name Default Description ======= ======== ======================== shared True Build shared libraries - static Build static libraries - mpi Use MPI - python Build Python extension + static True Build static libraries + mpi True Use MPI + python False Build Python extension ======= ======== ======================== If specified in this table, the corresponding default should be used @@ -2720,7 +2678,7 @@ when declaring a variant. Version Lists ^^^^^^^^^^^^^ -Spack packges should list supported versions with the newest first. +Spack packages should list supported versions with the newest first. ^^^^^^^^^^^^^^^^ Special Versions @@ -2728,20 +2686,30 @@ Special Versions The following *special* version names may be used when building a package: -* *@system*: Indicates a hook to the OS-installed version of the - package. This is useful, for example, to tell Spack to use the - OS-installed version in ``packages.yaml``:: +""""""""""" +``@system`` +""""""""""" + +Indicates a hook to the OS-installed version of the +package. This is useful, for example, to tell Spack to use the +OS-installed version in ``packages.yaml``: - openssl: - paths: - openssl@system: /usr - buildable: False +.. code-block:: yaml - Certain Spack internals look for the *@system* version and do - appropriate things in that case. + openssl: + paths: + openssl@system: /usr + buildable: False -* *@local*: Indicates the version was built manually from some source - tree of unknown provenance (see ``spack setup``). +Certain Spack internals look for the ``@system`` version and do +appropriate things in that case. + +"""""""""" +``@local`` +"""""""""" + +Indicates the version was built manually from some source +tree of unknown provenance (see ``spack setup``). --------------------------- Packaging workflow commands @@ -2760,7 +2728,7 @@ of the build. A typical package workflow might look like this: -.. code-block:: sh +.. code-block:: console $ spack edit mypackage $ spack install mypackage @@ -2823,11 +2791,11 @@ Restores the source code to pristine state, as it was before building. Does this in one of two ways: - 1. If the source was fetched as a tarball, deletes the entire build - directory and re-expands the tarball. +#. If the source was fetched as a tarball, deletes the entire build + directory and re-expands the tarball. - 2. If the source was checked out from a repository, this deletes the - build directory and checks it out again. +#. If the source was checked out from a repository, this deletes the + build directory and checks it out again. .. _spack-clean: @@ -2865,9 +2833,9 @@ By default, ``spack install`` will delete the staging area once a package has been successfully built and installed. Use ``--keep-stage`` to leave the build directory intact: -.. code-block:: sh +.. code-block:: console - spack install --keep-stage + $ spack install --keep-stage This allows you to inspect the build directory and potentially debug the build. You can use ``purge`` or ``clean`` later to get rid of the @@ -2882,17 +2850,17 @@ install prefix if anything fails during ``install()``. If you want to keep the prefix anyway (e.g. to diagnose a bug), you can use ``--keep-prefix``: -.. code-block:: sh +.. code-block:: console - spack install --keep-prefix + $ spack install --keep-prefix Note that this may confuse Spack into thinking that the package has -been installed properly, so you may need to use ``spack uninstall -f`` +been installed properly, so you may need to use ``spack uninstall --force`` to get rid of the install prefix before you build again: -.. code-block:: sh +.. code-block:: console - spack uninstall -f + $ spack uninstall --force --------------------- Graphing dependencies @@ -2906,31 +2874,9 @@ Graphing dependencies Spack provides the ``spack graph`` command for graphing dependencies. The command by default generates an ASCII rendering of a spec's -dependency graph. For example:: - - $ spack graph mpileaks - o mpileaks - |\ - | |\ - | o | callpath - |/| | - | |\| - | |\ \ - | | |\ \ - | | | | o adept-utils - | |_|_|/| - |/| | | | - o | | | | mpi - / / / / - | | o | dyninst - | |/| | - |/|/| | - | | |/ - | o | libdwarf - |/ / - o | libelf - / - o boost +dependency graph. For example: + +.. command-output:: spack graph mpileaks At the top is the root package in the DAG, with dependency edges emerging from it. On a color terminal, the edges are colored by which @@ -2938,39 +2884,9 @@ dependency they lead to. You can also use ``spack graph`` to generate graphs in the widely used `Dot `_ format. For -example:: - - $ spack graph --dot mpileaks - digraph G { - label = "Spack Dependencies" - labelloc = "b" - rankdir = "LR" - ranksep = "5" - - "boost" [label="boost"] - "callpath" [label="callpath"] - "libdwarf" [label="libdwarf"] - "mpileaks" [label="mpileaks"] - "mpi" [label="mpi"] - "adept-utils" [label="adept-utils"] - "dyninst" [label="dyninst"] - "libelf" [label="libelf"] - - "callpath" -> "dyninst" - "callpath" -> "adept-utils" - "callpath" -> "mpi" - "callpath" -> "libelf" - "callpath" -> "libdwarf" - "libdwarf" -> "libelf" - "mpileaks" -> "adept-utils" - "mpileaks" -> "callpath" - "mpileaks" -> "mpi" - "adept-utils" -> "boost" - "adept-utils" -> "mpi" - "dyninst" -> "boost" - "dyninst" -> "libelf" - "dyninst" -> "libdwarf" - } +example: + +.. command-output:: spack graph --dot mpileaks This graph can be provided as input to other graphing tools, such as those in `Graphviz `_. @@ -2981,14 +2897,19 @@ Interactive shell support Spack provides some limited shell support to make life easier for packagers. You can enable these commands by sourcing a setup file in -the ``/share/spack`` directory. For ``bash`` or ``ksh``, run:: +the ``share/spack`` directory. For ``bash`` or ``ksh``, run: - . $SPACK_ROOT/share/spack/setup-env.sh +.. code-block:: sh + + export SPACK_ROOT=/path/to/spack + . $SPACK_ROOT/share/spack/setup-env.sh For ``csh`` and ``tcsh`` run: - setenv SPACK_ROOT /path/to/spack - source $SPACK_ROOT/share/spack/setup-env.csh +.. code-block:: csh + + setenv SPACK_ROOT /path/to/spack + source $SPACK_ROOT/share/spack/setup-env.csh ``spack cd`` will then be available. @@ -3002,7 +2923,7 @@ For ``csh`` and ``tcsh`` run: Suppose you've staged a package but you want to modify it before you build it: -.. code-block:: sh +.. code-block:: console $ spack stage libelf ==> Trying to fetch from http://www.mr511.de/software/libelf-0.8.13.tar.gz @@ -3013,7 +2934,7 @@ build it: $ pwd /Users/gamblin2/src/spack/var/spack/stage/libelf@0.8.13%gcc@4.8.3 arch=linux-debian7-x86_64/libelf-0.8.13 -``spack cd`` here changed he current working directory to the +``spack cd`` here changed the current working directory to the directory containing the expanded ``libelf`` source code. There are a number of other places you can cd to in the spack directory hierarchy: @@ -3035,16 +2956,17 @@ but it takes a spec as an argument. You can use it to see the environment variables that will be set when a particular build runs, for example: -.. code-block:: sh +.. code-block:: console $ spack env mpileaks@1.1%intel This will display the entire environment that will be set when the ``mpileaks@1.1%intel`` build runs. -To run commands in a package's build environment, you can simply provided them after the spec argument to ``spack env``: +To run commands in a package's build environment, you can simply +provide them after the spec argument to ``spack env``: -.. code-block:: sh +.. code-block:: console $ spack cd mpileaks@1.1%intel $ spack env mpileaks@1.1%intel ./configure @@ -3060,13 +2982,17 @@ package's build environment. ``spack location`` is the same as ``spack cd`` but it does not require shell support. It simply prints out the path you ask for, rather than -cd'ing to it. In bash, this:: +cd'ing to it. In bash, this: - cd $(spack location -b ) +.. code-block:: console -is the same as:: + $ cd $(spack location -b ) - spack cd -b +is the same as: + +.. code-block:: console + + $ spack cd -b ``spack location`` is intended for use in scripts or makefiles that need to know where packages are installed. e.g., in a makefile you @@ -3082,7 +3008,7 @@ might write: Build System Configuration Support ---------------------------------- -Imagine a developer creating a CMake-based (or Autotools) project in a local +Imagine a developer creating a CMake or Autotools-based project in a local directory, which depends on libraries A-Z. Once Spack has installed those dependencies, one would like to run ``cmake`` with appropriate command line and environment so CMake can find them. The ``spack @@ -3091,44 +3017,45 @@ configuration that is essentially the same as how Spack *would have* configured the project. This can be demonstrated with a usage example: -.. code-block:: bash +.. code-block:: console - cd myproject - spack setup myproject@local - mkdir build; cd build - ../spconfig.py .. - make - make install + $ cd myproject + $ spack setup myproject@local + $ mkdir build; cd build + $ ../spconfig.py .. + $ make + $ make install Notes: - * Spack must have ``myproject/package.py`` in its repository for - this to work. - * ``spack setup`` produces the executable script ``spconfig.py`` in - the local directory, and also creates the module file for the - package. ``spconfig.py`` is normally run from the user's - out-of-source build directory. - * The version number given to ``spack setup`` is arbitrary, just - like ``spack diy``. ``myproject/package.py`` does not need to - have any valid downloadable versions listed (typical when a - project is new). - * spconfig.py produces a CMake configuration that *does not* use the - Spack wrappers. Any resulting binaries *will not* use RPATH, - unless the user has enabled it. This is recommended for - development purposes, not production. - * ``spconfig.py`` is human readable, and can serve as a developer - reference of what dependencies are being used. - * ``make install`` installs the package into the Spack repository, - where it may be used by other Spack packages. - * CMake-generated makefiles re-run CMake in some circumstances. Use - of ``spconfig.py`` breaks this behavior, requiring the developer - to manually re-run ``spconfig.py`` when a ``CMakeLists.txt`` file - has changed. + +* Spack must have ``myproject/package.py`` in its repository for + this to work. +* ``spack setup`` produces the executable script ``spconfig.py`` in + the local directory, and also creates the module file for the + package. ``spconfig.py`` is normally run from the user's + out-of-source build directory. +* The version number given to ``spack setup`` is arbitrary, just + like ``spack diy``. ``myproject/package.py`` does not need to + have any valid downloadable versions listed (typical when a + project is new). +* spconfig.py produces a CMake configuration that *does not* use the + Spack wrappers. Any resulting binaries *will not* use RPATH, + unless the user has enabled it. This is recommended for + development purposes, not production. +* ``spconfig.py`` is human readable, and can serve as a developer + reference of what dependencies are being used. +* ``make install`` installs the package into the Spack repository, + where it may be used by other Spack packages. +* CMake-generated makefiles re-run CMake in some circumstances. Use + of ``spconfig.py`` breaks this behavior, requiring the developer + to manually re-run ``spconfig.py`` when a ``CMakeLists.txt`` file + has changed. ^^^^^^^^^^^^ CMakePackage ^^^^^^^^^^^^ -In order ot enable ``spack setup`` functionality, the author of +In order to enable ``spack setup`` functionality, the author of ``myproject/package.py`` must subclass from ``CMakePackage`` instead of the standard ``Package`` superclass. Because CMake is standardized, the packager does not need to tell Spack how to run @@ -3140,14 +3067,15 @@ translate variant flags into CMake definitions. For example: .. code-block:: python - def configure_args(self): - spec = self.spec - return [ - '-DUSE_EVERYTRACE=%s' % ('YES' if '+everytrace' in spec else 'NO'), - '-DBUILD_PYTHON=%s' % ('YES' if '+python' in spec else 'NO'), - '-DBUILD_GRIDGEN=%s' % ('YES' if '+gridgen' in spec else 'NO'), - '-DBUILD_COUPLER=%s' % ('YES' if '+coupler' in spec else 'NO'), - '-DUSE_PISM=%s' % ('YES' if '+pism' in spec else 'NO')] + def configure_args(self): + spec = self.spec + return [ + '-DUSE_EVERYTRACE=%s' % ('YES' if '+everytrace' in spec else 'NO'), + '-DBUILD_PYTHON=%s' % ('YES' if '+python' in spec else 'NO'), + '-DBUILD_GRIDGEN=%s' % ('YES' if '+gridgen' in spec else 'NO'), + '-DBUILD_COUPLER=%s' % ('YES' if '+coupler' in spec else 'NO'), + '-DUSE_PISM=%s' % ('YES' if '+pism' in spec else 'NO') + ] If needed, a packager may also override methods defined in ``StagedPackage`` (see below). -- cgit v1.2.3-70-g09d2 From a0c8aca3a2f50ddd6f39c2382afe8021ba14633b Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Mon, 29 Aug 2016 09:36:54 -0500 Subject: Remove py-mercurial, going with mercurial package instead --- share/spack/qa/check_dependencies | 2 +- .../repos/builtin/packages/py-mercurial/package.py | 44 ---------------------- 2 files changed, 1 insertion(+), 45 deletions(-) delete mode 100644 var/spack/repos/builtin/packages/py-mercurial/package.py diff --git a/share/spack/qa/check_dependencies b/share/spack/qa/check_dependencies index ba9ede4ae0..08fad9cdc9 100755 --- a/share/spack/qa/check_dependencies +++ b/share/spack/qa/check_dependencies @@ -31,7 +31,7 @@ for dep in "$@"; do spack_package=git ;; hg) - spack_package=py-mercurial + spack_package=mercurial pip_package=mercurial ;; svn) diff --git a/var/spack/repos/builtin/packages/py-mercurial/package.py b/var/spack/repos/builtin/packages/py-mercurial/package.py deleted file mode 100644 index b1b9ffa1b1..0000000000 --- a/var/spack/repos/builtin/packages/py-mercurial/package.py +++ /dev/null @@ -1,44 +0,0 @@ -############################################################################## -# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. -# Produced at the Lawrence Livermore National Laboratory. -# -# This file is part of Spack. -# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. -# LLNL-CODE-647188 -# -# For details, see https://github.com/llnl/spack -# Please also see the LICENSE file for our notice and the LGPL. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU Lesser General Public License (as -# published by the Free Software Foundation) version 2.1, February 1999. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and -# conditions of the GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -############################################################################## -from spack import * - - -class PyMercurial(Package): - """Mercurial is a free, distributed source control management tool. - It efficiently handles projects of any size and offers an easy and - intuitive interface.""" - - homepage = "https://www.mercurial-scm.org/" - url = "https://pypi.python.org/packages/source/m/mercurial/mercurial-3.9.tar.gz" - - version('3.9', 'e2b355da744e94747daae3a5339d28a0', - url="https://pypi.python.org/packages/22/73/e8ef24d3cb13e4fa2695417e13fd22effa1c8e28465eea91a9f84aa922cd/mercurial-3.9.tar.gz") - - extends('python') - - depends_on('py-setuptools', type='build') - - def install(self, spec, prefix): - setup_py('install', '--prefix={0}'.format(prefix)) -- cgit v1.2.3-70-g09d2 From e04b76c2c3ee397df16d77a768496e007458c3da Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Mon, 29 Aug 2016 11:10:06 -0500 Subject: Add spack to the PATH for doc tests --- share/spack/qa/run-doc-tests | 3 +++ 1 file changed, 3 insertions(+) diff --git a/share/spack/qa/run-doc-tests b/share/spack/qa/run-doc-tests index 0b23bd9d3d..96b76a216e 100755 --- a/share/spack/qa/run-doc-tests +++ b/share/spack/qa/run-doc-tests @@ -28,6 +28,9 @@ deps=( # Check for dependencies "$QA_DIR/check_dependencies" "${deps[@]}" || exit 1 +# Add Spack to the PATH. +export PATH="$SPACK_ROOT/bin:$PATH" + # Move to documentation directory # Allows script to be run from anywhere cd "$DOC_DIR" -- cgit v1.2.3-70-g09d2 From 8d1a753a1b21a4ac6ae94cbb97ba2cbf7da8367d Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Mon, 29 Aug 2016 11:58:22 -0500 Subject: Fix spack --profile, fixes #1639 --- lib/spack/docs/configuration.rst | 43 ++++++++-------------------------------- 1 file changed, 8 insertions(+), 35 deletions(-) diff --git a/lib/spack/docs/configuration.rst b/lib/spack/docs/configuration.rst index 0c88dd1d9d..ba534d1e62 100644 --- a/lib/spack/docs/configuration.rst +++ b/lib/spack/docs/configuration.rst @@ -201,41 +201,14 @@ supply ``-p`` to Spack on the command line, before any subcommands. .. _spack-p: -^^^^^^^^^^^^ -``spack -p`` -^^^^^^^^^^^^ - -``spack -p`` output looks like this: - -.. code-block:: console - - $ spack -p graph dyninst - o dyninst - \|\ - \| \|\ - \| o \| libdwarf - \|/ / - o \| libelf - / - o boost - - 307670 function calls (305943 primitive calls) in 0.127 seconds - - Ordered by: internal time - - ncalls tottime percall cumtime percall filename:lineno(function) - 853 0.021 0.000 0.066 0.000 inspect.py:472(getmodule) - 51197 0.011 0.000 0.018 0.000 inspect.py:51(ismodule) - 73961 0.010 0.000 0.010 0.000 {isinstance} - 1762 0.006 0.000 0.053 0.000 inspect.py:440(getsourcefile) - 32075 0.006 0.000 0.006 0.000 {hasattr} - 1760 0.004 0.000 0.004 0.000 {posix.stat} - 2240 0.004 0.000 0.004 0.000 {posix.lstat} - 2602 0.004 0.000 0.011 0.000 inspect.py:398(getfile) - 771 0.004 0.000 0.077 0.000 inspect.py:518(findsource) - 2656 0.004 0.000 0.004 0.000 {method 'match' of '_sre.SRE_Pattern' objects} - 30772 0.003 0.000 0.003 0.000 {method 'get' of 'dict' objects} - ... +^^^^^^^^^^^^^^^^^^^ +``spack --profile`` +^^^^^^^^^^^^^^^^^^^ + +``spack --profile`` output looks like this: + +.. command-output:: spack --profile graph dyninst + :ellipsis: 25 The bottom of the output shows the top most time consuming functions, slowest on top. The profiling support is from Python's built-in tool, -- cgit v1.2.3-70-g09d2 From ae2a803496772db9ce6f9245d165b44f587e4389 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Tue, 30 Aug 2016 10:47:38 -0500 Subject: Make subcommands importable, '-' -> '_', fixes #1642 --- bin/spack | 11 +- lib/spack/spack/cmd/package-list.py | 104 ------------- lib/spack/spack/cmd/package_list.py | 104 +++++++++++++ lib/spack/spack/cmd/test-install.py | 245 ------------------------------- lib/spack/spack/cmd/test_install.py | 245 +++++++++++++++++++++++++++++++ lib/spack/spack/cmd/url-parse.py | 75 ---------- lib/spack/spack/cmd/url_parse.py | 75 ++++++++++ lib/spack/spack/test/cmd/test_install.py | 6 +- 8 files changed, 434 insertions(+), 431 deletions(-) delete mode 100644 lib/spack/spack/cmd/package-list.py create mode 100644 lib/spack/spack/cmd/package_list.py delete mode 100644 lib/spack/spack/cmd/test-install.py create mode 100644 lib/spack/spack/cmd/test_install.py delete mode 100644 lib/spack/spack/cmd/url-parse.py create mode 100644 lib/spack/spack/cmd/url_parse.py diff --git a/bin/spack b/bin/spack index 7efa88f1ee..323a06aa53 100755 --- a/bin/spack +++ b/bin/spack @@ -56,8 +56,15 @@ with warnings.catch_warnings(): # Spack, were removed, but shadow system modules that Spack still # imports. If we leave them, Spack will fail in mysterious ways. # TODO: more elegant solution for orphaned pyc files. -orphaned_pyc_files = [os.path.join(SPACK_EXTERNAL_LIBS, n) - for n in ('functools.pyc', 'ordereddict.pyc')] +orphaned_pyc_files = [ + os.path.join(SPACK_EXTERNAL_LIBS, 'functools.pyc'), + os.path.join(SPACK_EXTERNAL_LIBS, 'ordereddict.pyc'), + os.path.join(SPACK_LIB_PATH, 'spack', 'platforms', 'cray_xc.pyc'), + os.path.join(SPACK_LIB_PATH, 'spack', 'cmd', 'package-list.pyc'), + os.path.join(SPACK_LIB_PATH, 'spack', 'cmd', 'test-install.pyc'), + os.path.join(SPACK_LIB_PATH, 'spack', 'cmd', 'url-parse.pyc') +] + for pyc_file in orphaned_pyc_files: if not os.path.exists(pyc_file): continue diff --git a/lib/spack/spack/cmd/package-list.py b/lib/spack/spack/cmd/package-list.py deleted file mode 100644 index 42f408af96..0000000000 --- a/lib/spack/spack/cmd/package-list.py +++ /dev/null @@ -1,104 +0,0 @@ -############################################################################## -# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. -# Produced at the Lawrence Livermore National Laboratory. -# -# This file is part of Spack. -# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. -# LLNL-CODE-647188 -# -# For details, see https://github.com/llnl/spack -# Please also see the LICENSE file for our notice and the LGPL. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU Lesser General Public License (as -# published by the Free Software Foundation) version 2.1, February 1999. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and -# conditions of the GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -############################################################################## -import cgi -from StringIO import StringIO -from llnl.util.tty.colify import * -import spack - -description = "Print a list of all packages in reStructuredText." - - -def github_url(pkg): - """Link to a package file on github.""" - url = "https://github.com/LLNL/spack/blob/develop/var/spack/repos/builtin/packages/{0}/package.py" - return url.format(pkg.name) - - -def rst_table(elts): - """Print out a RST-style table.""" - cols = StringIO() - ncol, widths = colify(elts, output=cols, tty=True) - header = " ".join("=" * (w - 1) for w in widths) - return "%s\n%s%s" % (header, cols.getvalue(), header) - - -def print_rst_package_list(): - """Print out information on all packages in restructured text.""" - pkgs = sorted(spack.repo.all_packages(), key=lambda s: s.name.lower()) - pkg_names = [p.name for p in pkgs] - - print ".. _package-list:" - print - print "============" - print "Package List" - print "============" - print - print "This is a list of things you can install using Spack. It is" - print "automatically generated based on the packages in the latest Spack" - print "release." - print - print "Spack currently has %d mainline packages:" % len(pkgs) - print - print rst_table("`%s`_" % p for p in pkg_names) - print - - # Output some text for each package. - for pkg in pkgs: - print "-----" - print - print ".. _%s:" % pkg.name - print - # Must be at least 2 long, breaks for single letter packages like R. - print "-" * max(len(pkg.name), 2) - print pkg.name - print "-" * max(len(pkg.name), 2) - print - print "Homepage:" - print " * `%s <%s>`__" % (cgi.escape(pkg.homepage), pkg.homepage) - print - print "Spack package:" - print " * `%s/package.py <%s>`__" % (pkg.name, github_url(pkg)) - print - if pkg.versions: - print "Versions:" - print " " + ", ".join(str(v) for v in - reversed(sorted(pkg.versions))) - print - - for deptype in spack.alldeps: - deps = pkg.dependencies_of_type(deptype) - if deps: - print "%s Dependencies" % deptype.capitalize() - print " " + ", ".join("%s_" % d if d in pkg_names - else d for d in deps) - print - - print "Description:" - print pkg.format_doc(indent=2) - print - - -def package_list(parser, args): - print_rst_package_list() diff --git a/lib/spack/spack/cmd/package_list.py b/lib/spack/spack/cmd/package_list.py new file mode 100644 index 0000000000..42f408af96 --- /dev/null +++ b/lib/spack/spack/cmd/package_list.py @@ -0,0 +1,104 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +import cgi +from StringIO import StringIO +from llnl.util.tty.colify import * +import spack + +description = "Print a list of all packages in reStructuredText." + + +def github_url(pkg): + """Link to a package file on github.""" + url = "https://github.com/LLNL/spack/blob/develop/var/spack/repos/builtin/packages/{0}/package.py" + return url.format(pkg.name) + + +def rst_table(elts): + """Print out a RST-style table.""" + cols = StringIO() + ncol, widths = colify(elts, output=cols, tty=True) + header = " ".join("=" * (w - 1) for w in widths) + return "%s\n%s%s" % (header, cols.getvalue(), header) + + +def print_rst_package_list(): + """Print out information on all packages in restructured text.""" + pkgs = sorted(spack.repo.all_packages(), key=lambda s: s.name.lower()) + pkg_names = [p.name for p in pkgs] + + print ".. _package-list:" + print + print "============" + print "Package List" + print "============" + print + print "This is a list of things you can install using Spack. It is" + print "automatically generated based on the packages in the latest Spack" + print "release." + print + print "Spack currently has %d mainline packages:" % len(pkgs) + print + print rst_table("`%s`_" % p for p in pkg_names) + print + + # Output some text for each package. + for pkg in pkgs: + print "-----" + print + print ".. _%s:" % pkg.name + print + # Must be at least 2 long, breaks for single letter packages like R. + print "-" * max(len(pkg.name), 2) + print pkg.name + print "-" * max(len(pkg.name), 2) + print + print "Homepage:" + print " * `%s <%s>`__" % (cgi.escape(pkg.homepage), pkg.homepage) + print + print "Spack package:" + print " * `%s/package.py <%s>`__" % (pkg.name, github_url(pkg)) + print + if pkg.versions: + print "Versions:" + print " " + ", ".join(str(v) for v in + reversed(sorted(pkg.versions))) + print + + for deptype in spack.alldeps: + deps = pkg.dependencies_of_type(deptype) + if deps: + print "%s Dependencies" % deptype.capitalize() + print " " + ", ".join("%s_" % d if d in pkg_names + else d for d in deps) + print + + print "Description:" + print pkg.format_doc(indent=2) + print + + +def package_list(parser, args): + print_rst_package_list() diff --git a/lib/spack/spack/cmd/test-install.py b/lib/spack/spack/cmd/test-install.py deleted file mode 100644 index 8e7173e9a2..0000000000 --- a/lib/spack/spack/cmd/test-install.py +++ /dev/null @@ -1,245 +0,0 @@ -############################################################################## -# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. -# Produced at the Lawrence Livermore National Laboratory. -# -# This file is part of Spack. -# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. -# LLNL-CODE-647188 -# -# For details, see https://github.com/llnl/spack -# Please also see the LICENSE file for our notice and the LGPL. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU Lesser General Public License (as -# published by the Free Software Foundation) version 2.1, February 1999. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and -# conditions of the GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -############################################################################## -import argparse -import codecs -import os -import time -import xml.dom.minidom -import xml.etree.ElementTree as ET - -import llnl.util.tty as tty -import spack -import spack.cmd -from llnl.util.filesystem import * -from spack.build_environment import InstallError -from spack.fetch_strategy import FetchError - -description = "Run package install as a unit test, output formatted results." - - -def setup_parser(subparser): - subparser.add_argument( - '-j', '--jobs', action='store', type=int, - help="Explicitly set number of make jobs. Default is #cpus.") - - subparser.add_argument( - '-n', '--no-checksum', action='store_true', dest='no_checksum', - help="Do not check packages against checksum") - - subparser.add_argument( - '-o', '--output', action='store', - help="test output goes in this file") - - subparser.add_argument( - 'package', nargs=argparse.REMAINDER, - help="spec of package to install") - - -class TestResult(object): - PASSED = 0 - FAILED = 1 - SKIPPED = 2 - ERRORED = 3 - - -class TestSuite(object): - - def __init__(self, filename): - self.filename = filename - self.root = ET.Element('testsuite') - self.tests = [] - - def __enter__(self): - return self - - def append(self, item): - if not isinstance(item, TestCase): - raise TypeError( - 'only TestCase instances may be appended to TestSuite') - self.tests.append(item) # Append the item to the list of tests - - def __exit__(self, exc_type, exc_val, exc_tb): - # Prepare the header for the entire test suite - number_of_errors = sum( - x.result_type == TestResult.ERRORED for x in self.tests) - self.root.set('errors', str(number_of_errors)) - number_of_failures = sum( - x.result_type == TestResult.FAILED for x in self.tests) - self.root.set('failures', str(number_of_failures)) - self.root.set('tests', str(len(self.tests))) - - for item in self.tests: - self.root.append(item.element) - - with open(self.filename, 'wb') as file: - xml_string = ET.tostring(self.root) - xml_string = xml.dom.minidom.parseString(xml_string).toprettyxml() - file.write(xml_string) - - -class TestCase(object): - - results = { - TestResult.PASSED: None, - TestResult.SKIPPED: 'skipped', - TestResult.FAILED: 'failure', - TestResult.ERRORED: 'error', - } - - def __init__(self, classname, name, time=None): - self.element = ET.Element('testcase') - self.element.set('classname', str(classname)) - self.element.set('name', str(name)) - if time is not None: - self.element.set('time', str(time)) - self.result_type = None - - def set_result(self, result_type, - message=None, error_type=None, text=None): - self.result_type = result_type - result = TestCase.results[self.result_type] - if result is not None and result is not TestResult.PASSED: - subelement = ET.SubElement(self.element, result) - if error_type is not None: - subelement.set('type', error_type) - if message is not None: - subelement.set('message', str(message)) - if text is not None: - subelement.text = text - - -def fetch_log(path): - if not os.path.exists(path): - return list() - with codecs.open(path, 'rb', 'utf-8') as F: - return list(line.strip() for line in F.readlines()) - - -def failed_dependencies(spec): - def get_deps(deptype): - return set(item for item in spec.dependencies(deptype) - if not spack.repo.get(item).installed) - link_deps = get_deps('link') - run_deps = get_deps('run') - return link_deps.union(run_deps) - - -def get_top_spec_or_die(args): - specs = spack.cmd.parse_specs(args.package, concretize=True) - if len(specs) > 1: - tty.die("Only 1 top-level package can be specified") - top_spec = iter(specs).next() - return top_spec - - -def install_single_spec(spec, number_of_jobs): - package = spack.repo.get(spec) - - # If it is already installed, skip the test - if spack.repo.get(spec).installed: - testcase = TestCase(package.name, package.spec.short_spec, time=0.0) - testcase.set_result( - TestResult.SKIPPED, - message='Skipped [already installed]', - error_type='already_installed') - return testcase - - # If it relies on dependencies that did not install, skip - if failed_dependencies(spec): - testcase = TestCase(package.name, package.spec.short_spec, time=0.0) - testcase.set_result( - TestResult.SKIPPED, - message='Skipped [failed dependencies]', - error_type='dep_failed') - return testcase - - # Otherwise try to install the spec - try: - start_time = time.time() - package.do_install(keep_prefix=False, - keep_stage=True, - ignore_deps=False, - make_jobs=number_of_jobs, - verbose=True, - fake=False) - duration = time.time() - start_time - testcase = TestCase(package.name, package.spec.short_spec, duration) - testcase.set_result(TestResult.PASSED) - except InstallError: - # An InstallError is considered a failure (the recipe didn't work - # correctly) - duration = time.time() - start_time - # Try to get the log - lines = fetch_log(package.build_log_path) - text = '\n'.join(lines) - testcase = TestCase(package.name, package.spec.short_spec, duration) - testcase.set_result(TestResult.FAILED, - message='Installation failure', text=text) - - except FetchError: - # A FetchError is considered an error (we didn't even start building) - duration = time.time() - start_time - testcase = TestCase(package.name, package.spec.short_spec, duration) - testcase.set_result(TestResult.ERRORED, - message='Unable to fetch package') - - return testcase - - -def get_filename(args, top_spec): - if not args.output: - fname = 'test-{x.name}-{x.version}-{hash}.xml'.format( - x=top_spec, hash=top_spec.dag_hash()) - output_directory = join_path(os.getcwd(), 'test-output') - if not os.path.exists(output_directory): - os.mkdir(output_directory) - output_filename = join_path(output_directory, fname) - else: - output_filename = args.output - return output_filename - - -def test_install(parser, args): - # Check the input - if not args.package: - tty.die("install requires a package argument") - - if args.jobs is not None: - if args.jobs <= 0: - tty.die("The -j option must be a positive integer!") - - if args.no_checksum: - spack.do_checksum = False # TODO: remove this global. - - # Get the one and only top spec - top_spec = get_top_spec_or_die(args) - # Get the filename of the test - output_filename = get_filename(args, top_spec) - # TEST SUITE - with TestSuite(output_filename) as test_suite: - # Traverse in post order : each spec is a test case - for spec in top_spec.traverse(order='post'): - test_case = install_single_spec(spec, args.jobs) - test_suite.append(test_case) diff --git a/lib/spack/spack/cmd/test_install.py b/lib/spack/spack/cmd/test_install.py new file mode 100644 index 0000000000..8e7173e9a2 --- /dev/null +++ b/lib/spack/spack/cmd/test_install.py @@ -0,0 +1,245 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +import argparse +import codecs +import os +import time +import xml.dom.minidom +import xml.etree.ElementTree as ET + +import llnl.util.tty as tty +import spack +import spack.cmd +from llnl.util.filesystem import * +from spack.build_environment import InstallError +from spack.fetch_strategy import FetchError + +description = "Run package install as a unit test, output formatted results." + + +def setup_parser(subparser): + subparser.add_argument( + '-j', '--jobs', action='store', type=int, + help="Explicitly set number of make jobs. Default is #cpus.") + + subparser.add_argument( + '-n', '--no-checksum', action='store_true', dest='no_checksum', + help="Do not check packages against checksum") + + subparser.add_argument( + '-o', '--output', action='store', + help="test output goes in this file") + + subparser.add_argument( + 'package', nargs=argparse.REMAINDER, + help="spec of package to install") + + +class TestResult(object): + PASSED = 0 + FAILED = 1 + SKIPPED = 2 + ERRORED = 3 + + +class TestSuite(object): + + def __init__(self, filename): + self.filename = filename + self.root = ET.Element('testsuite') + self.tests = [] + + def __enter__(self): + return self + + def append(self, item): + if not isinstance(item, TestCase): + raise TypeError( + 'only TestCase instances may be appended to TestSuite') + self.tests.append(item) # Append the item to the list of tests + + def __exit__(self, exc_type, exc_val, exc_tb): + # Prepare the header for the entire test suite + number_of_errors = sum( + x.result_type == TestResult.ERRORED for x in self.tests) + self.root.set('errors', str(number_of_errors)) + number_of_failures = sum( + x.result_type == TestResult.FAILED for x in self.tests) + self.root.set('failures', str(number_of_failures)) + self.root.set('tests', str(len(self.tests))) + + for item in self.tests: + self.root.append(item.element) + + with open(self.filename, 'wb') as file: + xml_string = ET.tostring(self.root) + xml_string = xml.dom.minidom.parseString(xml_string).toprettyxml() + file.write(xml_string) + + +class TestCase(object): + + results = { + TestResult.PASSED: None, + TestResult.SKIPPED: 'skipped', + TestResult.FAILED: 'failure', + TestResult.ERRORED: 'error', + } + + def __init__(self, classname, name, time=None): + self.element = ET.Element('testcase') + self.element.set('classname', str(classname)) + self.element.set('name', str(name)) + if time is not None: + self.element.set('time', str(time)) + self.result_type = None + + def set_result(self, result_type, + message=None, error_type=None, text=None): + self.result_type = result_type + result = TestCase.results[self.result_type] + if result is not None and result is not TestResult.PASSED: + subelement = ET.SubElement(self.element, result) + if error_type is not None: + subelement.set('type', error_type) + if message is not None: + subelement.set('message', str(message)) + if text is not None: + subelement.text = text + + +def fetch_log(path): + if not os.path.exists(path): + return list() + with codecs.open(path, 'rb', 'utf-8') as F: + return list(line.strip() for line in F.readlines()) + + +def failed_dependencies(spec): + def get_deps(deptype): + return set(item for item in spec.dependencies(deptype) + if not spack.repo.get(item).installed) + link_deps = get_deps('link') + run_deps = get_deps('run') + return link_deps.union(run_deps) + + +def get_top_spec_or_die(args): + specs = spack.cmd.parse_specs(args.package, concretize=True) + if len(specs) > 1: + tty.die("Only 1 top-level package can be specified") + top_spec = iter(specs).next() + return top_spec + + +def install_single_spec(spec, number_of_jobs): + package = spack.repo.get(spec) + + # If it is already installed, skip the test + if spack.repo.get(spec).installed: + testcase = TestCase(package.name, package.spec.short_spec, time=0.0) + testcase.set_result( + TestResult.SKIPPED, + message='Skipped [already installed]', + error_type='already_installed') + return testcase + + # If it relies on dependencies that did not install, skip + if failed_dependencies(spec): + testcase = TestCase(package.name, package.spec.short_spec, time=0.0) + testcase.set_result( + TestResult.SKIPPED, + message='Skipped [failed dependencies]', + error_type='dep_failed') + return testcase + + # Otherwise try to install the spec + try: + start_time = time.time() + package.do_install(keep_prefix=False, + keep_stage=True, + ignore_deps=False, + make_jobs=number_of_jobs, + verbose=True, + fake=False) + duration = time.time() - start_time + testcase = TestCase(package.name, package.spec.short_spec, duration) + testcase.set_result(TestResult.PASSED) + except InstallError: + # An InstallError is considered a failure (the recipe didn't work + # correctly) + duration = time.time() - start_time + # Try to get the log + lines = fetch_log(package.build_log_path) + text = '\n'.join(lines) + testcase = TestCase(package.name, package.spec.short_spec, duration) + testcase.set_result(TestResult.FAILED, + message='Installation failure', text=text) + + except FetchError: + # A FetchError is considered an error (we didn't even start building) + duration = time.time() - start_time + testcase = TestCase(package.name, package.spec.short_spec, duration) + testcase.set_result(TestResult.ERRORED, + message='Unable to fetch package') + + return testcase + + +def get_filename(args, top_spec): + if not args.output: + fname = 'test-{x.name}-{x.version}-{hash}.xml'.format( + x=top_spec, hash=top_spec.dag_hash()) + output_directory = join_path(os.getcwd(), 'test-output') + if not os.path.exists(output_directory): + os.mkdir(output_directory) + output_filename = join_path(output_directory, fname) + else: + output_filename = args.output + return output_filename + + +def test_install(parser, args): + # Check the input + if not args.package: + tty.die("install requires a package argument") + + if args.jobs is not None: + if args.jobs <= 0: + tty.die("The -j option must be a positive integer!") + + if args.no_checksum: + spack.do_checksum = False # TODO: remove this global. + + # Get the one and only top spec + top_spec = get_top_spec_or_die(args) + # Get the filename of the test + output_filename = get_filename(args, top_spec) + # TEST SUITE + with TestSuite(output_filename) as test_suite: + # Traverse in post order : each spec is a test case + for spec in top_spec.traverse(order='post'): + test_case = install_single_spec(spec, args.jobs) + test_suite.append(test_case) diff --git a/lib/spack/spack/cmd/url-parse.py b/lib/spack/spack/cmd/url-parse.py deleted file mode 100644 index b8c7c95040..0000000000 --- a/lib/spack/spack/cmd/url-parse.py +++ /dev/null @@ -1,75 +0,0 @@ -############################################################################## -# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. -# Produced at the Lawrence Livermore National Laboratory. -# -# This file is part of Spack. -# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. -# LLNL-CODE-647188 -# -# For details, see https://github.com/llnl/spack -# Please also see the LICENSE file for our notice and the LGPL. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU Lesser General Public License (as -# published by the Free Software Foundation) version 2.1, February 1999. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and -# conditions of the GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -############################################################################## -import llnl.util.tty as tty - -import spack -import spack.url -from spack.util.web import find_versions_of_archive - -description = "Show parsing of a URL, optionally spider web for versions." - - -def setup_parser(subparser): - subparser.add_argument('url', help="url of a package archive") - subparser.add_argument( - '-s', '--spider', action='store_true', - help="Spider the source page for versions.") - - -def print_name_and_version(url): - name, ns, nl, ntup, ver, vs, vl, vtup = spack.url.substitution_offsets(url) - underlines = [" "] * max(ns + nl, vs + vl) - for i in range(ns, ns + nl): - underlines[i] = '-' - for i in range(vs, vs + vl): - underlines[i] = '~' - - print " %s" % url - print " %s" % ''.join(underlines) - - -def url_parse(parser, args): - url = args.url - - ver, vs, vl = spack.url.parse_version_offset(url) - name, ns, nl = spack.url.parse_name_offset(url, ver) - - tty.msg("Parsing URL:") - try: - print_name_and_version(url) - except spack.url.UrlParseError as e: - tty.error(str(e)) - - print - tty.msg("Substituting version 9.9.9b:") - newurl = spack.url.substitute_version(url, '9.9.9b') - print_name_and_version(newurl) - - if args.spider: - print - tty.msg("Spidering for versions:") - versions = find_versions_of_archive(url) - for v in sorted(versions): - print "%-20s%s" % (v, versions[v]) diff --git a/lib/spack/spack/cmd/url_parse.py b/lib/spack/spack/cmd/url_parse.py new file mode 100644 index 0000000000..b8c7c95040 --- /dev/null +++ b/lib/spack/spack/cmd/url_parse.py @@ -0,0 +1,75 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +import llnl.util.tty as tty + +import spack +import spack.url +from spack.util.web import find_versions_of_archive + +description = "Show parsing of a URL, optionally spider web for versions." + + +def setup_parser(subparser): + subparser.add_argument('url', help="url of a package archive") + subparser.add_argument( + '-s', '--spider', action='store_true', + help="Spider the source page for versions.") + + +def print_name_and_version(url): + name, ns, nl, ntup, ver, vs, vl, vtup = spack.url.substitution_offsets(url) + underlines = [" "] * max(ns + nl, vs + vl) + for i in range(ns, ns + nl): + underlines[i] = '-' + for i in range(vs, vs + vl): + underlines[i] = '~' + + print " %s" % url + print " %s" % ''.join(underlines) + + +def url_parse(parser, args): + url = args.url + + ver, vs, vl = spack.url.parse_version_offset(url) + name, ns, nl = spack.url.parse_name_offset(url, ver) + + tty.msg("Parsing URL:") + try: + print_name_and_version(url) + except spack.url.UrlParseError as e: + tty.error(str(e)) + + print + tty.msg("Substituting version 9.9.9b:") + newurl = spack.url.substitute_version(url, '9.9.9b') + print_name_and_version(newurl) + + if args.spider: + print + tty.msg("Spidering for versions:") + versions = find_versions_of_archive(url) + for v in sorted(versions): + print "%-20s%s" % (v, versions[v]) diff --git a/lib/spack/spack/test/cmd/test_install.py b/lib/spack/spack/test/cmd/test_install.py index 39287d5d6d..4734fe1267 100644 --- a/lib/spack/spack/test/cmd/test_install.py +++ b/lib/spack/spack/test/cmd/test_install.py @@ -30,6 +30,7 @@ import contextlib import spack import spack.cmd +from spack.cmd import test_install FILE_REGISTRY = collections.defaultdict(StringIO.StringIO) @@ -51,11 +52,6 @@ def mock_open(filename, mode): handle.close() -# The use of __import__ is necessary to maintain a name with hyphen (which -# cannot be an identifier in python) -test_install = __import__("spack.cmd.test-install", fromlist=['test_install']) - - class MockSpec(object): def __init__(self, name, version, hashStr=None): -- cgit v1.2.3-70-g09d2 From cc7df29e810b0733d7457720a18fe4400ef90990 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Tue, 30 Aug 2016 12:05:14 -0500 Subject: Keep dashes in command names, translate to underscores --- bin/spack | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bin/spack b/bin/spack index 323a06aa53..17586cc1f6 100755 --- a/bin/spack +++ b/bin/spack @@ -127,7 +127,8 @@ subparsers = parser.add_subparsers(metavar='SUBCOMMAND', dest="command") import spack.cmd for cmd in spack.cmd.commands: module = spack.cmd.get_module(cmd) - subparser = subparsers.add_parser(cmd, help=module.description) + cmd_name = cmd.replace('_', '-') + subparser = subparsers.add_parser(cmd_name, help=module.description) module.setup_parser(subparser) # Just print help and exit if run with no arguments at all @@ -163,7 +164,7 @@ def main(): spack.curl.add_default_arg('-k') # Try to load the particular command asked for and run it - command = spack.cmd.get_command(args.command) + command = spack.cmd.get_command(args.command.replace('-', '_')) try: return_val = command(parser, args) except SpackError as e: -- cgit v1.2.3-70-g09d2 From 9b455e925488bd44cfbb591fc55d4dfb01e30e90 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Tue, 30 Aug 2016 15:58:22 -0500 Subject: Resolve new documentation issues that have cropped up --- lib/spack/docs/case_studies.rst | 106 ++++++++++++--------- .../repos/builtin/packages/libsplash/package.py | 7 +- .../repos/builtin/packages/opencoarrays/package.py | 3 +- .../repos/builtin/packages/pngwriter/package.py | 15 ++- 4 files changed, 71 insertions(+), 60 deletions(-) diff --git a/lib/spack/docs/case_studies.rst b/lib/spack/docs/case_studies.rst index bcd754fdcd..fcec636c27 100644 --- a/lib/spack/docs/case_studies.rst +++ b/lib/spack/docs/case_studies.rst @@ -1,5 +1,6 @@ +======================================= Using Spack for CMake-based Development -========================================== +======================================= These are instructions on how to use Spack to aid in the development of a CMake-based project. Spack is used to help find the dependencies @@ -7,24 +8,26 @@ for the project, configure it at development time, and then package it it in a way that others can install. Using Spack for CMake-based development consists of three parts: -1. Setting up the CMake build in your software -2. Writing the Spack Package -3. Using it from Spack. - +#. Setting up the CMake build in your software +#. Writing the Spack Package +#. Using it from Spack. +-------------------------- Setting Up the CMake Build ---------------------------------------- +-------------------------- You should follow standard CMake conventions in setting up your software, your CMake build should NOT depend on or require Spack to build. See here for an example: - https://github.com/citibeth/icebin + +https://github.com/citibeth/icebin Note that there's one exception here to the rule I mentioned above. -In ``CMakeLists.txt``, I have the following line:: +In ``CMakeLists.txt``, I have the following line: - include_directories($ENV{CMAKE_TRANSITIVE_INCLUDE_PATH}) +.. code-block:: none + include_directories($ENV{CMAKE_TRANSITIVE_INCLUDE_PATH}) This is a hook into Spack, and it ensures that all transitive dependencies are included in the include path. It's not needed if @@ -48,40 +51,44 @@ correctly. Not only is this a good idea and nice, but it also ensures that your package will build the same with or without ``spack install``. +------------------------- Writing the Spack Package ---------------------------------------- +------------------------- Now that you have a CMake build, you want to tell Spack how to configure it. This is done by writing a Spack package for your software. See here for example: - https://github.com/citibeth/spack/blob/efischer/develop/var/spack/repos/builtin/packages/icebin/package.py + +https://github.com/citibeth/spack/blob/efischer/develop/var/spack/repos/builtin/packages/icebin/package.py You need to subclass ``CMakePackage``, as is done in this example. This enables advanced features of Spack for helping you in configuring your software (keep reading...). Instead of an ``install()`` method used when subclassing ``Package``, you write ``configure_args()``. See here for more info on how this works: - https://github.com/LLNL/spack/pull/543/files + +https://github.com/LLNL/spack/pull/543/files NOTE: if your software is not publicly available, you do not need to set the URL or version. Or you can set up bogus URLs and versions... whatever causes Spack to not crash. - +------------------- Using it from Spack --------------------------------- +------------------- Now that you have a Spack package, you can get Spack to setup your CMake project for you. Use the following to setup, configure and -build your project:: +build your project: - cd myproject - spack spconfig myproject@local - mkdir build; cd build - ../spconfig.py .. - make - make install +.. code-block:: console + $ cd myproject + $ spack spconfig myproject@local + $ mkdir build; cd build + $ ../spconfig.py .. + $ make + $ make install Everything here should look pretty familiar here from a CMake perspective, except that ``spack spconfig`` creates the file @@ -94,68 +101,75 @@ If your project is publicly available (eg on GitHub), then you can ALSO use this setup to "just install" a release version without going through the manual configuration/build step. Just do: -1. Put tag(s) on the version(s) in your GitHub repo you want to be release versions. +#. Put tag(s) on the version(s) in your GitHub repo you want to be release versions. -2. Set the ``url`` in your ``package.py`` to download a tarball for +#. Set the ``url`` in your ``package.py`` to download a tarball for the appropriate version. (GitHub will give you a tarball for any - version in the repo, if you tickle it the right way). For example:: + version in the repo, if you tickle it the right way). For example: - https://github.com/citibeth/icebin/tarball/v0.1.0 + https://github.com/citibeth/icebin/tarball/v0.1.0 Set up versions as appropriate in your ``package.py``. (Manually download the tarball and run ``md5sum`` to determine the appropriate checksum for it). -3. Now you should be able to say ``spack install myproject@version`` +#. Now you should be able to say ``spack install myproject@version`` and things "just work." NOTE... in order to use the features outlined in this post, you currently need to use the following branch of Spack: - https://github.com/citibeth/spack/tree/efischer/develop + +https://github.com/citibeth/spack/tree/efischer/develop There is a pull request open on this branch ( https://github.com/LLNL/spack/pull/543 ) and we are working to get it integrated into the main ``develop`` branch. - +------------------------ Activating your Software -------------------------------------- +------------------------ Once you've built your software, you will want to load it up. You can use ``spack load mypackage@local`` for that in your ``.bashrc``, but that is slow. Try stuff like the following instead: The following command will load the Spack-installed packages needed -for basic Python use of IceBin:: +for basic Python use of IceBin: - module load `spack module find tcl icebin netcdf cmake@3.5.1` - module load `spack module find --dependencies tcl py-basemap py-giss` +.. code-block:: console + + $ module load `spack module find tcl icebin netcdf cmake@3.5.1` + $ module load `spack module find --dependencies tcl py-basemap py-giss` You can speed up shell startup by turning these into ``module load`` commands. -1. Cut-n-paste the script ``make_spackenv``:: +#. Cut-n-paste the script ``make_spackenv``: + + .. code-block:: sh - #!/bin/sh - # - # Generate commands to load the Spack environment + #!/bin/sh + # + # Generate commands to load the Spack environment - SPACKENV=$HOME/spackenv.sh + SPACKENV=$HOME/spackenv.sh - spack module find --shell tcl git icebin@local ibmisc netcdf cmake@3.5.1 >$SPACKENV - spack module find --dependencies --shell tcl py-basemap py-giss >>$SPACKENV + spack module find --shell tcl git icebin@local ibmisc netcdf cmake@3.5.1 > $SPACKENV + spack module find --dependencies --shell tcl py-basemap py-giss >> $SPACKENV -2. Add the following to your ``.bashrc`` file:: +#. Add the following to your ``.bashrc`` file: - source $HOME/spackenv.sh - # Preferentially use your checked-out Python source - export PYTHONPATH=$HOME/icebin/pylib:$PYTHONPATH + .. code-block:: sh -3. Run ``sh make_spackenv`` whenever your Spack installation changes (including right now). + source $HOME/spackenv.sh + # Preferentially use your checked-out Python source + export PYTHONPATH=$HOME/icebin/pylib:$PYTHONPATH +#. Run ``sh make_spackenv`` whenever your Spack installation changes (including right now). +----------- Giving Back -------------------- +----------- If your software is publicly available, you should submit the ``package.py`` for it as a pull request to the main Spack GitHub @@ -164,4 +178,4 @@ project. This will ensure that anyone can install your software for how that has turned into detailed instructions that have successfully enabled collaborators to install complex software: - https://github.com/citibeth/icebin/blob/develop/README.rst +https://github.com/citibeth/icebin/blob/develop/README.rst diff --git a/var/spack/repos/builtin/packages/libsplash/package.py b/var/spack/repos/builtin/packages/libsplash/package.py index 0659f7d650..21a6eede3f 100644 --- a/var/spack/repos/builtin/packages/libsplash/package.py +++ b/var/spack/repos/builtin/packages/libsplash/package.py @@ -26,10 +26,9 @@ from spack import * class Libsplash(Package): - """ - libSplash aims at developing a HDF5-based I/O library for HPC simulations. - It is created as an easy-to-use frontend for the standard HDF5 library - with support for MPI processes in a cluster environment. While the + """libSplash aims at developing a HDF5-based I/O library for HPC + simulations. It is created as an easy-to-use frontend for the standard HDF5 + library with support for MPI processes in a cluster environment. While the standard HDF5 library provides detailed low-level control, libSplash simplifies tasks commonly found in large-scale HPC simulations, such as iterative computations and MPI distributed processes. diff --git a/var/spack/repos/builtin/packages/opencoarrays/package.py b/var/spack/repos/builtin/packages/opencoarrays/package.py index 0003157985..d9760e2afc 100644 --- a/var/spack/repos/builtin/packages/opencoarrays/package.py +++ b/var/spack/repos/builtin/packages/opencoarrays/package.py @@ -26,8 +26,7 @@ from spack import * class Opencoarrays(CMakePackage): - """ - OpenCoarrays is an open-source software project that produces an + """OpenCoarrays is an open-source software project that produces an application binary interface (ABI) supporting coarray Fortran (CAF) compilers, an application programming interface (API) that supports users of non-CAF compilers, and an associated compiler wrapper and program diff --git a/var/spack/repos/builtin/packages/pngwriter/package.py b/var/spack/repos/builtin/packages/pngwriter/package.py index 037a2eb4f3..c51f1f82a8 100644 --- a/var/spack/repos/builtin/packages/pngwriter/package.py +++ b/var/spack/repos/builtin/packages/pngwriter/package.py @@ -26,14 +26,13 @@ from spack import * class Pngwriter(Package): - """ - PNGwriter is a very easy to use open source graphics library that uses PNG - as its output format. The interface has been designed to be as simple and - intuitive as possible. It supports plotting and reading pixels in the RGB - (red, green, blue), HSV (hue, saturation, value/brightness) and CMYK (cyan, - magenta, yellow, black) colour spaces, basic shapes, scaling, bilinear - interpolation, full TrueType antialiased and rotated text support, bezier - curves, opening existing PNG images and more. + """PNGwriter is a very easy to use open source graphics library that uses + PNG as its output format. The interface has been designed to be as simple + and intuitive as possible. It supports plotting and reading pixels in the + RGB (red, green, blue), HSV (hue, saturation, value/brightness) and CMYK + (cyan, magenta, yellow, black) colour spaces, basic shapes, scaling, + bilinear interpolation, full TrueType antialiased and rotated text support, + bezier curves, opening existing PNG images and more. """ homepage = "http://pngwriter.sourceforge.net/" -- cgit v1.2.3-70-g09d2 From f6d07b54f15941ba25ed10ff9465df358a59d625 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Tue, 30 Aug 2016 16:01:00 -0500 Subject: Always run spack unit tests --- share/spack/qa/run-unit-tests | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/share/spack/qa/run-unit-tests b/share/spack/qa/run-unit-tests index 03dfe7cea1..6da919e18d 100755 --- a/share/spack/qa/run-unit-tests +++ b/share/spack/qa/run-unit-tests @@ -35,27 +35,6 @@ export PATH="$SPACK_ROOT/bin:$PATH" # Allows script to be run from anywhere cd "$SPACK_ROOT" -# Array of directories containing core Spack framework -core_dirs=( - bin - etc - # lib, but skip documentation - lib/spack/env - lib/spack/external - lib/spack/llnl - lib/spack/spack - share -) - -# Gather array of changed files -changed=($("$QA_DIR/changed_files" "${core_dirs[@]}")) - -# Exit if no core Spack framework files were modified -if [[ ! "${changed[@]}" ]]; then - echo "No core Spack framework files were modified." - exit 0 -fi - # Run integration tests # TODO: should these be separated into a different test suite? source "$SPACK_ROOT/share/spack/setup-env.sh" -- cgit v1.2.3-70-g09d2 From 1be6267149ba2fdd3d510bde713b237eaa0248e9 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Tue, 30 Aug 2016 16:28:55 -0500 Subject: Undo changes to trailing triple quotes in docstring --- lib/spack/spack/build_environment.py | 3 ++- lib/spack/spack/config.py | 3 ++- lib/spack/spack/environment.py | 3 ++- lib/spack/spack/mirror.py | 4 ++-- lib/spack/spack/spec.py | 4 ++-- lib/spack/spack/version.py | 3 ++- var/spack/repos/builtin/packages/adios/package.py | 3 ++- var/spack/repos/builtin/packages/arpack-ng/package.py | 3 ++- var/spack/repos/builtin/packages/astyle/package.py | 3 ++- var/spack/repos/builtin/packages/caliper/package.py | 3 ++- var/spack/repos/builtin/packages/cfitsio/package.py | 3 ++- var/spack/repos/builtin/packages/cube/package.py | 3 ++- var/spack/repos/builtin/packages/datamash/package.py | 3 ++- var/spack/repos/builtin/packages/eigen/package.py | 3 ++- var/spack/repos/builtin/packages/espresso/package.py | 3 ++- var/spack/repos/builtin/packages/gdal/package.py | 3 ++- var/spack/repos/builtin/packages/gdb/package.py | 3 ++- var/spack/repos/builtin/packages/gmsh/package.py | 3 ++- var/spack/repos/builtin/packages/lmod/package.py | 3 ++- var/spack/repos/builtin/packages/lua-luafilesystem/package.py | 3 ++- var/spack/repos/builtin/packages/mxml/package.py | 3 ++- var/spack/repos/builtin/packages/ncdu/package.py | 3 ++- var/spack/repos/builtin/packages/opari2/package.py | 3 ++- var/spack/repos/builtin/packages/openjpeg/package.py | 3 ++- var/spack/repos/builtin/packages/parallel/package.py | 3 ++- var/spack/repos/builtin/packages/petsc/package.py | 3 ++- var/spack/repos/builtin/packages/py-prettytable/package.py | 3 ++- var/spack/repos/builtin/packages/py-tuiview/package.py | 3 ++- var/spack/repos/builtin/packages/scorep/package.py | 3 ++- var/spack/repos/builtin/packages/screen/package.py | 3 ++- var/spack/repos/builtin/packages/swiftsim/package.py | 3 ++- var/spack/repos/builtin/packages/tau/package.py | 3 ++- var/spack/repos/builtin/packages/xerces-c/package.py | 3 ++- var/spack/repos/builtin/packages/zsh/package.py | 3 ++- 34 files changed, 68 insertions(+), 36 deletions(-) diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 9fbf0536a4..9b5ed367d1 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -494,7 +494,8 @@ def fork(pkg, function, dirty=False): If something goes wrong, the child process is expected to print the error and the parent process will exit with error as well. If things go well, the child exits and the parent - carries on.""" + carries on. + """ try: pid = os.fork() diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index ebddfb328f..dec43726a0 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -117,7 +117,8 @@ a key in a configuration file. For example, this:: ... Will make Spack take compilers *only* from the user configuration, and -the site configuration will be ignored.""" +the site configuration will be ignored. +""" import copy import os diff --git a/lib/spack/spack/environment.py b/lib/spack/spack/environment.py index 64863510c3..b0eadef7a6 100644 --- a/lib/spack/spack/environment.py +++ b/lib/spack/spack/environment.py @@ -266,7 +266,8 @@ class EnvironmentModifications(object): parameters :param \*args: list of files to be sourced - :rtype: instance of EnvironmentModifications""" + :rtype: instance of EnvironmentModifications + """ env = EnvironmentModifications() # Check if the files are actually there diff --git a/lib/spack/spack/mirror.py b/lib/spack/spack/mirror.py index 1e759848ea..2361fb5448 100644 --- a/lib/spack/spack/mirror.py +++ b/lib/spack/spack/mirror.py @@ -141,8 +141,8 @@ def create(path, specs, **kwargs): This routine iterates through all known package versions, and it creates specs for those versions. If the version satisfies any spec - in the specs list, it is downloaded and added to the mirror.""" - + in the specs list, it is downloaded and added to the mirror. + """ # Make sure nothing is in the way. if os.path.isfile(path): raise MirrorError("%s already exists and is a file." % path) diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 19733c8a41..d8a7cf9d7b 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -2110,8 +2110,8 @@ class Spec(object): *Example:* ``$_$@$+`` translates to the name, version, and options of the package, but no dependencies, arch, or compiler. - TODO: allow, e.g., $6# to customize short hash length - TODO: allow, e.g., $## for full hash. + TODO: allow, e.g., ``$6#`` to customize short hash length + TODO: allow, e.g., ``$##`` for full hash. """ color = kwargs.get('color', False) length = len(format_string) diff --git a/lib/spack/spack/version.py b/lib/spack/spack/version.py index fb87966d04..bc96dcd716 100644 --- a/lib/spack/spack/version.py +++ b/lib/spack/spack/version.py @@ -158,7 +158,8 @@ class Version(object): """A Version 'satisfies' another if it is at least as specific and has a common prefix. e.g., we want gcc@4.7.3 to satisfy a request for gcc@4.7 so that when a user asks to build with gcc@4.7, we can find - a suitable compiler.""" + a suitable compiler. + """ nself = len(self.version) nother = len(other.version) diff --git a/var/spack/repos/builtin/packages/adios/package.py b/var/spack/repos/builtin/packages/adios/package.py index 96fee2ac28..01834383b8 100644 --- a/var/spack/repos/builtin/packages/adios/package.py +++ b/var/spack/repos/builtin/packages/adios/package.py @@ -30,7 +30,8 @@ class Adios(Package): """The Adaptable IO System (ADIOS) provides a simple, flexible way for scientists to describe the data in their code that may need to be written, - read, or processed outside of the running simulation.""" + read, or processed outside of the running simulation. + """ homepage = "http://www.olcf.ornl.gov/center-projects/adios/" url = "https://github.com/ornladios/ADIOS/archive/v1.10.0.tar.gz" diff --git a/var/spack/repos/builtin/packages/arpack-ng/package.py b/var/spack/repos/builtin/packages/arpack-ng/package.py index 2fa7608098..0e71125d41 100644 --- a/var/spack/repos/builtin/packages/arpack-ng/package.py +++ b/var/spack/repos/builtin/packages/arpack-ng/package.py @@ -50,7 +50,8 @@ class ArpackNg(Package): forked it and implemented their own modifications, arpack-ng aims to tackle this by providing a common repository and maintained versions. - arpack-ng is replacing arpack almost everywhere.""" + arpack-ng is replacing arpack almost everywhere. + """ homepage = 'https://github.com/opencollab/arpack-ng' url = 'https://github.com/opencollab/arpack-ng/archive/3.3.0.tar.gz' diff --git a/var/spack/repos/builtin/packages/astyle/package.py b/var/spack/repos/builtin/packages/astyle/package.py index 7b450352ec..eecdcfdad7 100644 --- a/var/spack/repos/builtin/packages/astyle/package.py +++ b/var/spack/repos/builtin/packages/astyle/package.py @@ -27,7 +27,8 @@ from spack import * class Astyle(Package): """A Free, Fast, and Small Automatic Formatter for C, C++, C++/CLI, - Objective-C, C#, and Java Source Code.""" + Objective-C, C#, and Java Source Code. + """ homepage = "http://astyle.sourceforge.net/" url = "http://downloads.sourceforge.net/project/astyle/astyle/astyle%202.04/astyle_2.04_linux.tar.gz" diff --git a/var/spack/repos/builtin/packages/caliper/package.py b/var/spack/repos/builtin/packages/caliper/package.py index 770037ab49..113bef61b8 100644 --- a/var/spack/repos/builtin/packages/caliper/package.py +++ b/var/spack/repos/builtin/packages/caliper/package.py @@ -28,7 +28,8 @@ from spack import * class Caliper(Package): """Caliper is a generic context annotation system. It gives programmers the ability to provide arbitrary program context information to (performance) - tools at runtime.""" + tools at runtime. + """ homepage = "https://github.com/LLNL/Caliper" url = "" diff --git a/var/spack/repos/builtin/packages/cfitsio/package.py b/var/spack/repos/builtin/packages/cfitsio/package.py index fc5ae0eef5..35d9662f6f 100644 --- a/var/spack/repos/builtin/packages/cfitsio/package.py +++ b/var/spack/repos/builtin/packages/cfitsio/package.py @@ -27,7 +27,8 @@ from spack import * class Cfitsio(Package): """CFITSIO is a library of C and Fortran subroutines for reading and writing - data files in FITS (Flexible Image Transport System) data format.""" + data files in FITS (Flexible Image Transport System) data format. + """ homepage = 'http://heasarc.gsfc.nasa.gov/fitsio/' diff --git a/var/spack/repos/builtin/packages/cube/package.py b/var/spack/repos/builtin/packages/cube/package.py index 55f71ef681..8c835b3886 100644 --- a/var/spack/repos/builtin/packages/cube/package.py +++ b/var/spack/repos/builtin/packages/cube/package.py @@ -30,7 +30,8 @@ class Cube(Package): multi-dimensional performance space consisting of the dimensions: - performance metric - call path - - system resource""" + - system resource + """ homepage = "http://www.scalasca.org/software/cube-4.x/download.html" url = "http://apps.fz-juelich.de/scalasca/releases/cube/4.2/dist/cube-4.2.3.tar.gz" diff --git a/var/spack/repos/builtin/packages/datamash/package.py b/var/spack/repos/builtin/packages/datamash/package.py index 65773d1978..85adeca996 100644 --- a/var/spack/repos/builtin/packages/datamash/package.py +++ b/var/spack/repos/builtin/packages/datamash/package.py @@ -27,7 +27,8 @@ from spack import * class Datamash(Package): """GNU datamash is a command-line program which performs basic numeric, - textual and statistical operations on input textual data files.""" + textual and statistical operations on input textual data files. + """ homepage = "https://www.gnu.org/software/datamash/" url = "http://ftp.gnu.org/gnu/datamash/datamash-1.0.5.tar.gz" diff --git a/var/spack/repos/builtin/packages/eigen/package.py b/var/spack/repos/builtin/packages/eigen/package.py index 2dd43c0113..42dc24e373 100644 --- a/var/spack/repos/builtin/packages/eigen/package.py +++ b/var/spack/repos/builtin/packages/eigen/package.py @@ -27,7 +27,8 @@ from spack import * class Eigen(Package): """Eigen is a C++ template library for linear algebra matrices, - vectors, numerical solvers, and related algorithms.""" + vectors, numerical solvers, and related algorithms. + """ homepage = 'http://eigen.tuxfamily.org/' url = 'http://bitbucket.org/eigen/eigen/get/3.2.7.tar.bz2' diff --git a/var/spack/repos/builtin/packages/espresso/package.py b/var/spack/repos/builtin/packages/espresso/package.py index 1d8c9805b1..3dca1ba187 100644 --- a/var/spack/repos/builtin/packages/espresso/package.py +++ b/var/spack/repos/builtin/packages/espresso/package.py @@ -30,7 +30,8 @@ class Espresso(Package): """QE is an integrated suite of Open-Source computer codes for electronic-structure calculations and materials modeling at the nanoscale. It is based on density-functional theory, plane - waves, and pseudopotentials.""" + waves, and pseudopotentials. + """ homepage = 'http://quantum-espresso.org' url = 'http://www.qe-forge.org/gf/download/frsrelease/204/912/espresso-5.3.0.tar.gz' diff --git a/var/spack/repos/builtin/packages/gdal/package.py b/var/spack/repos/builtin/packages/gdal/package.py index cf8dce4ef1..4f54b3a841 100644 --- a/var/spack/repos/builtin/packages/gdal/package.py +++ b/var/spack/repos/builtin/packages/gdal/package.py @@ -32,7 +32,8 @@ class Gdal(Package): it presents a single raster abstract data model and vector abstract data model to the calling application for all supported formats. It also comes with a variety of useful command line - utilities for data translation and processing""" + utilities for data translation and processing. + """ homepage = "http://www.gdal.org/" url = "http://download.osgeo.org/gdal/2.0.2/gdal-2.0.2.tar.gz" diff --git a/var/spack/repos/builtin/packages/gdb/package.py b/var/spack/repos/builtin/packages/gdb/package.py index fdde649270..a14e166ed4 100644 --- a/var/spack/repos/builtin/packages/gdb/package.py +++ b/var/spack/repos/builtin/packages/gdb/package.py @@ -28,7 +28,8 @@ from spack import * class Gdb(Package): """GDB, the GNU Project debugger, allows you to see what is going on 'inside' another program while it executes -- or what another - program was doing at the moment it crashed.""" + program was doing at the moment it crashed. + """ homepage = "https://www.gnu.org/software/gdb" url = "http://ftp.gnu.org/gnu/gdb/gdb-7.10.tar.gz" diff --git a/var/spack/repos/builtin/packages/gmsh/package.py b/var/spack/repos/builtin/packages/gmsh/package.py index 77e70d675c..1d375f2186 100644 --- a/var/spack/repos/builtin/packages/gmsh/package.py +++ b/var/spack/repos/builtin/packages/gmsh/package.py @@ -32,7 +32,8 @@ class Gmsh(Package): capabilities. Gmsh is built around four modules: geometry, mesh, solver and post-processing. The specification of any input to these modules is done either interactively using the graphical user interface or in ASCII text - files using Gmsh's own scripting language.""" + files using Gmsh's own scripting language. + """ homepage = 'http://gmsh.info' url = 'http://gmsh.info/src/gmsh-2.11.0-source.tgz' diff --git a/var/spack/repos/builtin/packages/lmod/package.py b/var/spack/repos/builtin/packages/lmod/package.py index 15555e1200..c2daca80a5 100644 --- a/var/spack/repos/builtin/packages/lmod/package.py +++ b/var/spack/repos/builtin/packages/lmod/package.py @@ -32,7 +32,8 @@ class Lmod(Package): dynamically change the users' environment through modulefiles. This includes easily adding or removing directories to the PATH environment variable. Modulefiles for Library packages provide environment variables - that specify where the library and header files can be found.""" + that specify where the library and header files can be found. + """ homepage = 'https://www.tacc.utexas.edu/research-development/tacc-projects/lmod' url = 'https://github.com/TACC/Lmod/archive/6.4.1.tar.gz' diff --git a/var/spack/repos/builtin/packages/lua-luafilesystem/package.py b/var/spack/repos/builtin/packages/lua-luafilesystem/package.py index dca7c247fa..9e0e449813 100644 --- a/var/spack/repos/builtin/packages/lua-luafilesystem/package.py +++ b/var/spack/repos/builtin/packages/lua-luafilesystem/package.py @@ -32,7 +32,8 @@ class LuaLuafilesystem(Package): LuaFileSystem offers a portable way to access the underlying directory structure and file attributes. - LuaFileSystem is free software and uses the same license as Lua 5.1""" + LuaFileSystem is free software and uses the same license as Lua 5.1 + """ homepage = 'http://keplerproject.github.io/luafilesystem' url = 'https://github.com/keplerproject/luafilesystem/archive/v_1_6_3.tar.gz' diff --git a/var/spack/repos/builtin/packages/mxml/package.py b/var/spack/repos/builtin/packages/mxml/package.py index db2e7eb441..29e3b27d6e 100644 --- a/var/spack/repos/builtin/packages/mxml/package.py +++ b/var/spack/repos/builtin/packages/mxml/package.py @@ -28,7 +28,8 @@ from spack import * class Mxml(Package): """Mini-XML is a small XML library that you can use to read and write XML and XML-like data files in your application without requiring large - non-standard libraries.""" + non-standard libraries. + """ homepage = "http://www.msweet.org" url = "http://www.msweet.org/files/project3/mxml-2.9.tar.gz" diff --git a/var/spack/repos/builtin/packages/ncdu/package.py b/var/spack/repos/builtin/packages/ncdu/package.py index a22671250e..0842a592cc 100644 --- a/var/spack/repos/builtin/packages/ncdu/package.py +++ b/var/spack/repos/builtin/packages/ncdu/package.py @@ -30,7 +30,8 @@ class Ncdu(Package): to find space hogs on a remote server where you don't have an entire gaphical setup available, but it is a useful tool even on regular desktop systems. Ncdu aims to be fast, simple and easy to use, and should be able - to run in any minimal POSIX-like environment with ncurses installed.""" + to run in any minimal POSIX-like environment with ncurses installed. + """ homepage = "http://dev.yorhel.nl/ncdu" url = "http://dev.yorhel.nl/download/ncdu-1.11.tar.gz" diff --git a/var/spack/repos/builtin/packages/opari2/package.py b/var/spack/repos/builtin/packages/opari2/package.py index b270931d0d..20c67716a4 100644 --- a/var/spack/repos/builtin/packages/opari2/package.py +++ b/var/spack/repos/builtin/packages/opari2/package.py @@ -33,7 +33,8 @@ class Opari2(Package): as well as the usage of pre-instrumented libraries. Furthermore, an efficient way of tracking parent-child relationships was added. Additionally, we extended OPARI2 to support instrumentation of OpenMP 3.0 - tied tasks.""" + tied tasks. + """ homepage = "http://www.vi-hps.org/projects/score-p" url = "http://www.vi-hps.org/upload/packages/opari2/opari2-1.1.2.tar.gz" diff --git a/var/spack/repos/builtin/packages/openjpeg/package.py b/var/spack/repos/builtin/packages/openjpeg/package.py index 6ba368a0f7..fc505e19a9 100644 --- a/var/spack/repos/builtin/packages/openjpeg/package.py +++ b/var/spack/repos/builtin/packages/openjpeg/package.py @@ -31,7 +31,8 @@ class Openjpeg(Package): still-image compression standard from the Joint Photographic Experts Group (JPEG). Since April 2015, it is officially recognized by ISO/IEC and - ITU-T as a JPEG 2000 Reference Software.""" + ITU-T as a JPEG 2000 Reference Software. + """ homepage = "https://github.com/uclouvain/openjpeg" url = "https://github.com/uclouvain/openjpeg/archive/version.2.1.tar.gz" diff --git a/var/spack/repos/builtin/packages/parallel/package.py b/var/spack/repos/builtin/packages/parallel/package.py index 189a5a6054..81c0195651 100644 --- a/var/spack/repos/builtin/packages/parallel/package.py +++ b/var/spack/repos/builtin/packages/parallel/package.py @@ -28,7 +28,8 @@ from spack import * class Parallel(Package): """GNU parallel is a shell tool for executing jobs in parallel using one or more computers. A job can be a single command or a small - script that has to be run for each of the lines in the input.""" + script that has to be run for each of the lines in the input. + """ homepage = "http://www.gnu.org/software/parallel/" url = "http://ftp.gnu.org/gnu/parallel/parallel-20160422.tar.bz2" diff --git a/var/spack/repos/builtin/packages/petsc/package.py b/var/spack/repos/builtin/packages/petsc/package.py index 794633b929..73091bb182 100644 --- a/var/spack/repos/builtin/packages/petsc/package.py +++ b/var/spack/repos/builtin/packages/petsc/package.py @@ -29,7 +29,8 @@ from spack import * class Petsc(Package): """PETSc is a suite of data structures and routines for the scalable (parallel) solution of scientific applications modeled by partial - differential equations.""" + differential equations. + """ homepage = "http://www.mcs.anl.gov/petsc/index.html" url = "http://ftp.mcs.anl.gov/pub/petsc/release-snapshots/petsc-3.5.3.tar.gz" diff --git a/var/spack/repos/builtin/packages/py-prettytable/package.py b/var/spack/repos/builtin/packages/py-prettytable/package.py index 9d3edd4abb..fa1c17ae8c 100644 --- a/var/spack/repos/builtin/packages/py-prettytable/package.py +++ b/var/spack/repos/builtin/packages/py-prettytable/package.py @@ -28,7 +28,8 @@ from spack import * class PyPrettytable(Package): """PrettyTable is a simple Python library designed to make it quick and easy to represent tabular data in visually - appealing ASCII tables.""" + appealing ASCII tables. + """ homepage = "https://code.google.com/archive/p/prettytable/" url = "https://pypi.python.org/packages/e0/a1/36203205f77ccf98f3c6cf17cf068c972e6458d7e58509ca66da949ca347/prettytable-0.7.2.tar.gz" diff --git a/var/spack/repos/builtin/packages/py-tuiview/package.py b/var/spack/repos/builtin/packages/py-tuiview/package.py index a372ec2b67..5caf3ff143 100644 --- a/var/spack/repos/builtin/packages/py-tuiview/package.py +++ b/var/spack/repos/builtin/packages/py-tuiview/package.py @@ -27,7 +27,8 @@ from spack import * class PyTuiview(Package): """TuiView is a lightweight raster GIS with powerful raster attribute - table manipulation abilities.""" + table manipulation abilities. + """ homepage = "https://bitbucket.org/chchrsc/tuiview" url = "https://bitbucket.org/chchrsc/tuiview/get/tuiview-1.1.7.tar.gz" diff --git a/var/spack/repos/builtin/packages/scorep/package.py b/var/spack/repos/builtin/packages/scorep/package.py index d2efd23c37..e0f7972304 100644 --- a/var/spack/repos/builtin/packages/scorep/package.py +++ b/var/spack/repos/builtin/packages/scorep/package.py @@ -28,7 +28,8 @@ from spack import * class Scorep(Package): """The Score-P measurement infrastructure is a highly scalable and easy-to-use tool suite for profiling, event tracing, and online analysis - of HPC applications.""" + of HPC applications. + """ homepage = "http://www.vi-hps.org/projects/score-p" url = "http://www.vi-hps.org/upload/packages/scorep/scorep-1.2.3.tar.gz" diff --git a/var/spack/repos/builtin/packages/screen/package.py b/var/spack/repos/builtin/packages/screen/package.py index eeaebcbfb7..7edfb44a4d 100644 --- a/var/spack/repos/builtin/packages/screen/package.py +++ b/var/spack/repos/builtin/packages/screen/package.py @@ -27,7 +27,8 @@ from spack import * class Screen(Package): """Screen is a full-screen window manager that multiplexes a physical - terminal between several processes, typically interactive shells.""" + terminal between several processes, typically interactive shells. + """ homepage = "https://www.gnu.org/software/screen/" url = "http://ftp.gnu.org/gnu/screen/screen-4.3.1.tar.gz" diff --git a/var/spack/repos/builtin/packages/swiftsim/package.py b/var/spack/repos/builtin/packages/swiftsim/package.py index ec5e74bd15..c591c48d19 100644 --- a/var/spack/repos/builtin/packages/swiftsim/package.py +++ b/var/spack/repos/builtin/packages/swiftsim/package.py @@ -29,7 +29,8 @@ import llnl.util.tty as tty class Swiftsim(Package): """SPH With Inter-dependent Fine-grained Tasking (SWIFT) provides astrophysicists with a state of the art framework to perform - particle based simulations.""" + particle based simulations. + """ homepage = 'http://icc.dur.ac.uk/swift/' url = 'http://gitlab.cosma.dur.ac.uk/swift/swiftsim/repository/archive.tar.gz?ref=v0.3.0' diff --git a/var/spack/repos/builtin/packages/tau/package.py b/var/spack/repos/builtin/packages/tau/package.py index c70bf31018..1801b41c37 100644 --- a/var/spack/repos/builtin/packages/tau/package.py +++ b/var/spack/repos/builtin/packages/tau/package.py @@ -31,7 +31,8 @@ from llnl.util.filesystem import join_path class Tau(Package): """A portable profiling and tracing toolkit for performance analysis of parallel programs written in Fortran, C, C++, UPC, - Java, Python.""" + Java, Python. + """ homepage = "http://www.cs.uoregon.edu/research/tau" url = "https://www.cs.uoregon.edu/research/tau/tau_releases/tau-2.25.tar.gz" diff --git a/var/spack/repos/builtin/packages/xerces-c/package.py b/var/spack/repos/builtin/packages/xerces-c/package.py index 61c7625f25..9f3ad8a4dc 100644 --- a/var/spack/repos/builtin/packages/xerces-c/package.py +++ b/var/spack/repos/builtin/packages/xerces-c/package.py @@ -30,7 +30,8 @@ class XercesC(Package): C++. Xerces-C++ makes it easy to give your application the ability to read and write XML data. A shared library is provided for parsing, generating, manipulating, and validating XML documents using the DOM, SAX, and SAX2 - APIs.""" + APIs. + """ homepage = "https://xerces.apache.org/xerces-c" url = "https://www.apache.org/dist/xerces/c/3/sources/xerces-c-3.1.4.tar.bz2" diff --git a/var/spack/repos/builtin/packages/zsh/package.py b/var/spack/repos/builtin/packages/zsh/package.py index b65866b1fd..a70d307be9 100644 --- a/var/spack/repos/builtin/packages/zsh/package.py +++ b/var/spack/repos/builtin/packages/zsh/package.py @@ -28,7 +28,8 @@ from spack import * class Zsh(Package): """Zsh is a shell designed for interactive use, although it is also a powerful scripting language. Many of the useful features of bash, ksh, and - tcsh were incorporated into zsh; many original features were added.""" + tcsh were incorporated into zsh; many original features were added. + """ homepage = "http://www.zsh.org" url = "http://downloads.sourceforge.net/project/zsh/zsh/5.1.1/zsh-5.1.1.tar.gz" -- cgit v1.2.3-70-g09d2 From fc748eb3d06db5fd6d72c7b1f9247fe9c05b900e Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 30 Aug 2016 11:28:17 -0700 Subject: Exclude spack.__all__ from documentation. Everything in the __all__ list in the spack module is from some other module, so only do their documentation in their original location. This also avoids issues like the fact that some directive names shadow spack core module names. --- lib/spack/docs/conf.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/lib/spack/docs/conf.py b/lib/spack/docs/conf.py index 1416074356..a702e70e51 100644 --- a/lib/spack/docs/conf.py +++ b/lib/spack/docs/conf.py @@ -47,6 +47,7 @@ from glob import glob # documentation root, use os.path.abspath to make it absolute, like shown here. sys.path.insert(0, os.path.abspath('exts')) sys.path.insert(0, os.path.abspath('../external')) +sys.path.append(os.path.abspath('../spack')) # Add the Spack bin directory to the path so that we can use its output in docs. spack_root = '../../..' @@ -82,6 +83,27 @@ with open('command_index.rst', 'a') as index: for cmd in command_names: index.write(' * :ref:`%s`\n' % cmd) +# +# Exclude everything in spack.__all__ from indexing. All of these +# symbols are imported from elsewhere in spack; their inclusion in +# __all__ simply allows package authors to use `from spack import *`. +# Excluding them ensures they're only documented in their "real" module. +# +# This also avoids issues where some of these symbols shadow core spack +# modules. Sphinx will complain about duplicate docs when this happens. +# +import fileinput, spack +handling_spack = False +for line in fileinput.input('spack.rst', inplace=1): + if handling_spack: + if not line.startswith(' :noindex:'): + print ' :noindex: %s' % ' '.join(spack.__all__) + handling_spack = False + + if line.startswith('.. automodule::'): + handling_spack = (line == '.. automodule:: spack\n') + + print line, # Set an environment variable so that colify will print output like it would to # a terminal. -- cgit v1.2.3-70-g09d2 From d9c51915638a410ce4dd8fd8bf602f5f10a6500c Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 30 Aug 2016 19:57:33 -0700 Subject: Fix issue with path to Spack. --- lib/spack/docs/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/docs/conf.py b/lib/spack/docs/conf.py index a702e70e51..b23e365960 100644 --- a/lib/spack/docs/conf.py +++ b/lib/spack/docs/conf.py @@ -52,7 +52,7 @@ sys.path.append(os.path.abspath('../spack')) # Add the Spack bin directory to the path so that we can use its output in docs. spack_root = '../../..' os.environ['SPACK_ROOT'] = spack_root -os.environ['PATH'] += os.pathsep + '$SPACK_ROOT/bin' +os.environ['PATH'] += '%s%s/bin' % (os.pathsep, spack_root) # Get the spack version for use in the docs spack_version = subprocess.Popen( -- cgit v1.2.3-70-g09d2 From c470ffe1ac0fd15a88637493c82bbaad5e36258f Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 30 Aug 2016 20:07:21 -0700 Subject: Install graphviz before build. --- .travis.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.travis.yml b/.travis.yml index f9bf19148f..7961133c1a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,6 +19,12 @@ matrix: # Use new Travis infrastructure (Docker can't sudo yet) sudo: false +# Docs need graphviz to build +addons: + apt: + packages: + - graphviz + # Install various dependencies install: - pip install coveralls -- cgit v1.2.3-70-g09d2 From 176a84a8285b4117c9465f92d323dbb6aa4eb82c Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 30 Aug 2016 20:30:43 -0700 Subject: Don't test sphinx-generated conf.py for flake8 issues. --- lib/spack/docs/conf.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/spack/docs/conf.py b/lib/spack/docs/conf.py index b23e365960..95549e3b1e 100644 --- a/lib/spack/docs/conf.py +++ b/lib/spack/docs/conf.py @@ -1,3 +1,4 @@ +# flake8: noqa ############################################################################## # Copyright (c) 2013, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory. -- cgit v1.2.3-70-g09d2