diff options
79 files changed, 2075 insertions, 1784 deletions
@@ -5,8 +5,10 @@ # rationale is. # # Let people line things up nicely: +# - E129: visually indented line with same indent as next logical line # - E221: multiple spaces before operator -# - E241: multiple spaces after ‘,’ +# - E241: multiple spaces after ',' +# - E272: multiple spaces before keyword # # Let people use terse Python features: # - E731 : lambda expressions @@ -15,9 +17,10 @@ # - F403: disable wildcard import # # These are required to get the package.py files to test clean. -# - F821: undefined name (needed for cmake, configure, etc.) -# - F999: name name be undefined or undefined from star imports. +# - F405: `name` may be undefined, or undefined from star imports: `module` +# - F821: undefined name `name` (needed for cmake, configure, etc.) +# - F999: syntax error in doctest # [flake8] -ignore = E129,E221,E241,E272,E731,F403,F821,F999,F405 +ignore = E129,E221,E241,E272,E731,F403,F405,F821,F999 max-line-length = 79 diff --git a/.travis.yml b/.travis.yml index b376a33490..7961133c1a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,27 +1,38 @@ language: python +# Construct build matrix python: - - "2.6" - - "2.7" + - 2.6 + - 2.7 + env: - - TEST_TYPE=unit - - TEST_TYPE=flake8 + - TEST_SUITE=unit + - TEST_SUITE=flake8 + - TEST_SUITE=doc -# Exclude flake8 from python 2.6 matrix: exclude: - - python: "2.6" - env: TEST_TYPE=flake8 + - python: 2.6 + # Flake8 no longer supports Python 2.6 + env: TEST_SUITE=flake8 # Use new Travis infrastructure (Docker can't sudo yet) sudo: false -# Install coveralls to obtain code coverage +# Docs need graphviz to build +addons: + apt: + packages: + - graphviz + +# Install various dependencies install: - - "pip install coveralls" - - "pip install flake8" + - pip install coveralls + - pip install flake8 + - pip install sphinx + - pip install mercurial -before_install: +before_script: # Need this for the git tests to succeed. - git config --global user.email "spack@example.com" - git config --global user.name "Test User" @@ -29,18 +40,13 @@ before_install: # Need this to be able to compute the list of changed files - git fetch origin develop:develop -script: - # Run unit tests with code coverage plus install libdwarf - - 'if [ "$TEST_TYPE" = "unit" ]; then share/spack/qa/run-unit-tests; fi' - # Run flake8 code style checks. - - 'if [ "$TEST_TYPE" = "flake8" ]; then share/spack/qa/run-flake8; fi' +script: share/spack/qa/run-$TEST_SUITE-tests after_success: - - 'if [ "$TEST_TYPE" = "unit" ] && [ "$TRAVIS_PYTHON_VERSION" = "2.7" ]; then coveralls; fi' + - if [[ $TEST_SUITE == unit && $TRAVIS_PYTHON_VERSION == 2.7 ]]; then coveralls; fi notifications: email: - recipients: - - tgamblin@llnl.gov + recipients: tgamblin@llnl.gov on_success: change on_failure: always @@ -68,7 +68,7 @@ Before you send a PR, your code should pass the following checks: * Your contribution will need to pass the `spack test` command. Run this before submitting your PR. -* Also run the `share/spack/qa/run-flake8` script to check for PEP8 compliance. +* Also run the `share/spack/qa/run-flake8-tests` script to check for PEP8 compliance. To encourage contributions and readability by a broad audience, Spack uses the [PEP8](https://www.python.org/dev/peps/pep-0008/) coding standard with [a few exceptions](https://github.com/LLNL/spack/blob/develop/.flake8). @@ -56,8 +56,15 @@ with warnings.catch_warnings(): # Spack, were removed, but shadow system modules that Spack still # imports. If we leave them, Spack will fail in mysterious ways. # TODO: more elegant solution for orphaned pyc files. -orphaned_pyc_files = [os.path.join(SPACK_EXTERNAL_LIBS, n) - for n in ('functools.pyc', 'ordereddict.pyc')] +orphaned_pyc_files = [ + os.path.join(SPACK_EXTERNAL_LIBS, 'functools.pyc'), + os.path.join(SPACK_EXTERNAL_LIBS, 'ordereddict.pyc'), + os.path.join(SPACK_LIB_PATH, 'spack', 'platforms', 'cray_xc.pyc'), + os.path.join(SPACK_LIB_PATH, 'spack', 'cmd', 'package-list.pyc'), + os.path.join(SPACK_LIB_PATH, 'spack', 'cmd', 'test-install.pyc'), + os.path.join(SPACK_LIB_PATH, 'spack', 'cmd', 'url-parse.pyc') +] + for pyc_file in orphaned_pyc_files: if not os.path.exists(pyc_file): continue @@ -120,7 +127,8 @@ subparsers = parser.add_subparsers(metavar='SUBCOMMAND', dest="command") import spack.cmd for cmd in spack.cmd.commands: module = spack.cmd.get_module(cmd) - subparser = subparsers.add_parser(cmd, help=module.description) + cmd_name = cmd.replace('_', '-') + subparser = subparsers.add_parser(cmd_name, help=module.description) module.setup_parser(subparser) # Just print help and exit if run with no arguments at all @@ -156,7 +164,7 @@ def main(): spack.curl.add_default_arg('-k') # Try to load the particular command asked for and run it - command = spack.cmd.get_command(args.command) + command = spack.cmd.get_command(args.command.replace('-', '_')) try: return_val = command(parser, args) except SpackError as e: diff --git a/lib/spack/docs/basic_usage.rst b/lib/spack/docs/basic_usage.rst index 342c93d609..fd701789ec 100644 --- a/lib/spack/docs/basic_usage.rst +++ b/lib/spack/docs/basic_usage.rst @@ -1,23 +1,28 @@ .. _basic-usage: +=========== Basic usage -===================== +=========== The ``spack`` command has many *subcommands*. You'll only need a small subset of them for typical usage. Note that Spack colorizes output. ``less -R`` should be used with -Spack to maintian this colorization. Eg:: +Spack to maintain this colorization. E.g.: - spack find | less -R +.. code-block:: console -It is recommend that the following be put in your ``.bashrc`` file:: + $ spack find | less -R - alias less='less -R' +It is recommend that the following be put in your ``.bashrc`` file: +.. code-block:: sh + alias less='less -R' + +-------------------------- Listing available packages ------------------------------- +-------------------------- To install software with Spack, you need to know what software is available. You can see a list of available package names at the @@ -25,8 +30,9 @@ available. You can see a list of available package names at the .. _spack-list: +^^^^^^^^^^^^^^ ``spack list`` -~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^ The ``spack list`` command prints out a list of all of the packages Spack can install: @@ -59,8 +65,9 @@ All packages whose names contain documentation case insensitive: .. _spack-info: +^^^^^^^^^^^^^^ ``spack info`` -~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^ To get more information on a particular package from `spack list`, use `spack info`. Just supply the name of a package: @@ -77,8 +84,9 @@ viruses. .. _spack-versions: +^^^^^^^^^^^^^^^^^^ ``spack versions`` -~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^ To see *more* available versions of a package, run ``spack versions``. For example: @@ -95,31 +103,32 @@ by scraping it directly from package web pages. Depending on the package and how its releases are organized, Spack may or may not be able to find remote versions. - +--------------------------- Installing and uninstalling ------------------------------- +--------------------------- .. _spack-install: +^^^^^^^^^^^^^^^^^ ``spack install`` -~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^ ``spack install`` will install any package shown by ``spack list``. For example, To install the latest version of the ``mpileaks`` package, you might type this: -.. code-block:: sh +.. code-block:: console $ spack install mpileaks -If `mpileaks` depends on other packages, Spack will install the +If ``mpileaks`` depends on other packages, Spack will install the dependencies first. It then fetches the ``mpileaks`` tarball, expands it, verifies that it was downloaded without errors, builds it, and installs it in its own directory under ``$SPACK_ROOT/opt``. You'll see a number of messages from spack, a lot of build output, and a message that the packages is installed: -.. code-block:: sh +.. code-block:: console $ spack install mpileaks ==> Installing mpileaks @@ -142,13 +151,14 @@ that the packages is installed: The last line, with the ``[+]``, indicates where the package is installed. +^^^^^^^^^^^^^^^^^^^^^^^^^^^ Building a specific version -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^^ Spack can also build *specific versions* of a package. To do this, just add ``@`` after the package name, followed by a version: -.. code-block:: sh +.. code-block:: console $ spack install mpich@3.0.4 @@ -167,66 +177,71 @@ two configurations of the same version of a package, one built with boost This can all be done on the command line using the *spec* syntax. Spack calls the descriptor used to refer to a particular package configuration a **spec**. In the commands above, ``mpileaks`` and -``mpileaks@3.0.4``. We'll talk more about how you can use them to -customize an installation in :ref:`sec-specs`. +``mpileaks@3.0.4`` are both valid *specs*. We'll talk more about how +you can use them to customize an installation in :ref:`sec-specs`. .. _spack-uninstall: +^^^^^^^^^^^^^^^^^^^ ``spack uninstall`` -~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^ -To uninstall a package, type ``spack uninstall <package>``. This will ask the user for -confirmation, and in case will completely remove the directory in which the package was installed. +To uninstall a package, type ``spack uninstall <package>``. This will ask +the user for confirmation, and in case will completely remove the directory +in which the package was installed. -.. code-block:: sh +.. code-block:: console - spack uninstall mpich + $ spack uninstall mpich If there are still installed packages that depend on the package to be uninstalled, spack will refuse to uninstall it. To uninstall a package and every package that depends on it, you may give the -`--dependents` option. - -.. code-block:: sh +``--dependents`` option. - spack uninstall --dependents mpich +.. code-block:: console -will display a list of all the packages that depend on `mpich` and, upon confirmation, -will uninstall them in the right order. + $ spack uninstall --dependents mpich -A line like +will display a list of all the packages that depend on ``mpich`` and, upon +confirmation, will uninstall them in the right order. -.. code-block:: sh +A command like - spack uninstall mpich +.. code-block:: console -may be ambiguous, if multiple ``mpich`` configurations are installed. For example, if both -``mpich@3.0.2`` and ``mpich@3.1`` are installed, ``mpich`` could refer -to either one. Because it cannot determine which one to uninstall, -Spack will ask you either to provide a version number to remove the -ambiguity or use the ``--all`` option to uninstall all of the matching packages. + $ spack uninstall mpich -You may force uninstall a package with the `--force` option +may be ambiguous if multiple ``mpich`` configurations are installed. +For example, if both ``mpich@3.0.2`` and ``mpich@3.1`` are installed, +``mpich`` could refer to either one. Because it cannot determine which +one to uninstall, Spack will ask you either to provide a version number +to remove the ambiguity or use the ``--all`` option to uninstall all of +the matching packages. -.. code-block:: sh +You may force uninstall a package with the ``--force`` option - spack uninstall --force mpich +.. code-block:: console -but you risk breaking other installed packages. In general, it is safer to remove dependent -packages *before* removing their dependencies or use the `--dependents` option. + $ spack uninstall --force mpich +but you risk breaking other installed packages. In general, it is safer to +remove dependent packages *before* removing their dependencies or use the +``--dependents`` option. +------------------------- Seeing installed packages ------------------------------------ +------------------------- We know that ``spack list`` shows you the names of available packages, -but how do you figure out which are installed? +but how do you figure out which are already installed? .. _spack-find: +^^^^^^^^^^^^^^ ``spack find`` -~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^ ``spack find`` shows the *specs* of installed packages. A spec is like a name, but it has a version, compiler, architecture, and build @@ -235,7 +250,7 @@ of the same package with different specs. Running ``spack find`` with no arguments lists installed packages: -.. code-block:: sh +.. code-block:: console $ spack find ==> 74 installed packages. @@ -274,17 +289,18 @@ only shows the version of installed packages. ``spack find`` can filter the package list based on the package name, spec, or a number of properties of their installation status. For example, missing -dependencies of a spec can be shown with ``-m``, packages which were +dependencies of a spec can be shown with ``--missing``, packages which were explicitly installed with ``spack install <package>`` can be singled out with -``-e`` and those which have been pulled in only as dependencies with ``-E``. +``--explicit`` and those which have been pulled in only as dependencies with +``--implicit``. In some cases, there may be different configurations of the *same* version of a package installed. For example, there are two -installations of of ``libdwarf@20130729`` above. We can look at them -in more detail using ``spack find -d``, and by asking only to show +installations of ``libdwarf@20130729`` above. We can look at them +in more detail using ``spack find --deps``, and by asking only to show ``libdwarf`` packages: -.. code-block:: sh +.. code-block:: console $ spack find --deps libdwarf ==> 2 installed packages. @@ -298,25 +314,25 @@ Now we see that the two instances of ``libdwarf`` depend on *different* versions of ``libelf``: 0.8.12 and 0.8.13. This view can become complicated for packages with many dependencies. If you just want to know whether two packages' dependencies differ, you can use -``spack find -l``: +``spack find --long``: -.. code-block:: sh +.. code-block:: console - $ spack find -l libdwarf + $ spack find --long libdwarf ==> 2 installed packages. -- linux-debian7-x86_64 / gcc@4.4.7 -------------------------------- libdwarf@20130729-d9b90962 libdwarf@20130729-b52fac98 -Now the ``libwarf`` installs have hashes after their names. These are +Now the ``libdwarf`` installs have hashes after their names. These are hashes over all of the dependencies of each package. If the hashes are the same, then the packages have the same dependency configuration. If you want to know the path where each package is installed, you can -use ``spack find -p``: +use ``spack find --paths``: -.. code-block:: sh +.. code-block:: console - $ spack find -p + $ spack find --paths ==> 74 installed packages. -- linux-debian7-x86_64 / gcc@4.4.7 -------------------------------- ImageMagick@6.8.9-10 /home/gamblin2/spack/opt/linux-debian7-x86_64/gcc@4.4.7/ImageMagick@6.8.9-10-4df950dd @@ -331,9 +347,9 @@ use ``spack find -p``: And, finally, you can restrict your search to a particular package by supplying its name: -.. code-block:: sh +.. code-block:: console - $ spack find -p libelf + $ spack find --paths libelf -- linux-debian7-x86_64 / gcc@4.4.7 -------------------------------- libelf@0.8.11 /home/gamblin2/spack/opt/linux-debian7-x86_64/gcc@4.4.7/libelf@0.8.11 libelf@0.8.12 /home/gamblin2/spack/opt/linux-debian7-x86_64/gcc@4.4.7/libelf@0.8.12 @@ -344,7 +360,7 @@ by supplying its name: package. If you want to find only libelf versions greater than version 0.8.12, you could say: -.. code-block:: sh +.. code-block:: console $ spack find libelf@0.8.12: -- linux-debian7-x86_64 / gcc@4.4.7 -------------------------------- @@ -353,99 +369,110 @@ package. If you want to find only libelf versions greater than version Finding just the versions of libdwarf built with a particular version of libelf would look like this: -.. code-block:: sh +.. code-block:: console - $ spack find -l libdwarf ^libelf@0.8.12 + $ spack find --long libdwarf ^libelf@0.8.12 ==> 1 installed packages. -- linux-debian7-x86_64 / gcc@4.4.7 -------------------------------- libdwarf@20130729-d9b90962 We can also search for packages that have a certain attribute. For example, -``spack find -l libdwarf +debug`` will show only installations of libdwarf -with the 'debug' compile-time option enabled, while ``spack find -l +debug`` -will find every installed package with a 'debug' compile-time option enabled. +``spack find libdwarf +debug`` will show only installations of libdwarf +with the 'debug' compile-time option enabled. The full spec syntax is discussed in detail in :ref:`sec-specs`. - .. _compiler-config: +---------------------- Compiler configuration ------------------------------------ +---------------------- Spack has the ability to build packages with multiple compilers and compiler versions. Spack searches for compilers on your machine automatically the first time it is run. It does this by inspecting -your path. +your ``PATH``. .. _spack-compilers: +^^^^^^^^^^^^^^^^^^^ ``spack compilers`` -~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^ You can see which compilers spack has found by running ``spack -compilers`` or ``spack compiler list``:: - - $ spack compilers - ==> Available compilers - -- gcc --------------------------------------------------------- - gcc@4.9.0 gcc@4.8.0 gcc@4.7.0 gcc@4.6.2 gcc@4.4.7 - gcc@4.8.2 gcc@4.7.1 gcc@4.6.3 gcc@4.6.1 gcc@4.1.2 - -- intel ------------------------------------------------------- - intel@15.0.0 intel@14.0.0 intel@13.0.0 intel@12.1.0 intel@10.0 - intel@14.0.3 intel@13.1.1 intel@12.1.5 intel@12.0.4 intel@9.1 - intel@14.0.2 intel@13.1.0 intel@12.1.3 intel@11.1 - intel@14.0.1 intel@13.0.1 intel@12.1.2 intel@10.1 - -- clang ------------------------------------------------------- - clang@3.4 clang@3.3 clang@3.2 clang@3.1 - -- pgi --------------------------------------------------------- - pgi@14.3-0 pgi@13.2-0 pgi@12.1-0 pgi@10.9-0 pgi@8.0-1 - pgi@13.10-0 pgi@13.1-1 pgi@11.10-0 pgi@10.2-0 pgi@7.1-3 - pgi@13.6-0 pgi@12.8-0 pgi@11.1-0 pgi@9.0-4 pgi@7.0-6 +compilers`` or ``spack compiler list``: + +.. code-block:: console + + $ spack compilers + ==> Available compilers + -- gcc --------------------------------------------------------- + gcc@4.9.0 gcc@4.8.0 gcc@4.7.0 gcc@4.6.2 gcc@4.4.7 + gcc@4.8.2 gcc@4.7.1 gcc@4.6.3 gcc@4.6.1 gcc@4.1.2 + -- intel ------------------------------------------------------- + intel@15.0.0 intel@14.0.0 intel@13.0.0 intel@12.1.0 intel@10.0 + intel@14.0.3 intel@13.1.1 intel@12.1.5 intel@12.0.4 intel@9.1 + intel@14.0.2 intel@13.1.0 intel@12.1.3 intel@11.1 + intel@14.0.1 intel@13.0.1 intel@12.1.2 intel@10.1 + -- clang ------------------------------------------------------- + clang@3.4 clang@3.3 clang@3.2 clang@3.1 + -- pgi --------------------------------------------------------- + pgi@14.3-0 pgi@13.2-0 pgi@12.1-0 pgi@10.9-0 pgi@8.0-1 + pgi@13.10-0 pgi@13.1-1 pgi@11.10-0 pgi@10.2-0 pgi@7.1-3 + pgi@13.6-0 pgi@12.8-0 pgi@11.1-0 pgi@9.0-4 pgi@7.0-6 Any of these compilers can be used to build Spack packages. More on how this is done is in :ref:`sec-specs`. .. _spack-compiler-add: +^^^^^^^^^^^^^^^^^^^^^^ ``spack compiler add`` -~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^ An alias for ``spack compiler find``. .. _spack-compiler-find: +^^^^^^^^^^^^^^^^^^^^^^^ ``spack compiler find`` -~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^ If you do not see a compiler in this list, but you want to use it with Spack, you can simply run ``spack compiler find`` with the path to -where the compiler is installed. For example:: +where the compiler is installed. For example: + +.. code-block:: console - $ spack compiler find /usr/local/tools/ic-13.0.079 - ==> Added 1 new compiler to /Users/gamblin2/.spack/compilers.yaml - intel@13.0.079 + $ spack compiler find /usr/local/tools/ic-13.0.079 + ==> Added 1 new compiler to /Users/gamblin2/.spack/compilers.yaml + intel@13.0.079 Or you can run ``spack compiler find`` with no arguments to force auto-detection. This is useful if you do not know where compilers are installed, but you know that new compilers have been added to your -``PATH``. For example, using dotkit, you might do this:: +``PATH``. For example, using dotkit, you might do this: + +.. code-block:: console - $ module load gcc-4.9.0 - $ spack compiler find - ==> Added 1 new compiler to /Users/gamblin2/.spack/compilers.yaml - gcc@4.9.0 + $ module load gcc-4.9.0 + $ spack compiler find + ==> Added 1 new compiler to /Users/gamblin2/.spack/compilers.yaml + gcc@4.9.0 This loads the environment module for gcc-4.9.0 to add it to ``PATH``, and then it adds the compiler to Spack. .. _spack-compiler-info: +^^^^^^^^^^^^^^^^^^^^^^^ ``spack compiler info`` -~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^ If you want to see specifics on a particular compiler, you can run -``spack compiler info`` on it:: +``spack compiler info`` on it: + +.. code-block:: console $ spack compiler info intel@15 intel@15.0.0: @@ -453,62 +480,70 @@ If you want to see specifics on a particular compiler, you can run cxx = /usr/local/bin/icpc-15.0.090 f77 = /usr/local/bin/ifort-15.0.090 fc = /usr/local/bin/ifort-15.0.090 + modules = [] + operating system = centos6 This shows which C, C++, and Fortran compilers were detected by Spack. -Notice also that we didn\'t have to be too specific about the +Notice also that we didn't have to be too specific about the version. We just said ``intel@15``, and information about the only matching Intel compiler was displayed. - +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Manual compiler configuration -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ If auto-detection fails, you can manually configure a compiler by editing your ``~/.spack/compilers.yaml`` file. You can do this by running ``spack config edit compilers``, which will open the file in your ``$EDITOR``. -Each compiler configuration in the file looks like this:: +Each compiler configuration in the file looks like this: - ... - compilers: - - compiler: - modules = [] - operating_system: OS - paths: - cc: /usr/local/bin/icc-15.0.024-beta - cxx: /usr/local/bin/icpc-15.0.024-beta - f77: /usr/local/bin/ifort-15.0.024-beta - fc: /usr/local/bin/ifort-15.0.024-beta +.. code-block:: yaml - spec: intel@15.0.0: + compilers: + - compiler: + modules = [] + operating_system: centos6 + paths: + cc: /usr/local/bin/icc-15.0.024-beta + cxx: /usr/local/bin/icpc-15.0.024-beta + f77: /usr/local/bin/ifort-15.0.024-beta + fc: /usr/local/bin/ifort-15.0.024-beta + spec: intel@15.0.0: For compilers, like ``clang``, that do not support Fortran, put -``None`` for ``f77`` and ``fc``:: +``None`` for ``f77`` and ``fc``: - clang@3.3svn: - cc: /usr/bin/clang - cxx: /usr/bin/clang++ - f77: None - fc: None +.. code-block:: yaml + + paths: + cc: /usr/bin/clang + cxx: /usr/bin/clang++ + f77: None + fc: None + spec: clang@3.3svn: Once you save the file, the configured compilers will show up in the list displayed by ``spack compilers``. You can also add compiler flags to manually configured compilers. The valid flags are ``cflags``, ``cxxflags``, ``fflags``, ``cppflags``, -``ldflags``, and ``ldlibs``. For example:: - - ... - compilers: - - compiler: - ... - intel@15.0.0: - cc: /usr/local/bin/icc-15.0.024-beta - cxx: /usr/local/bin/icpc-15.0.024-beta - f77: /usr/local/bin/ifort-15.0.024-beta - fc: /usr/local/bin/ifort-15.0.024-beta - cppflags: -O3 -fPIC - ... +``ldflags``, and ``ldlibs``. For example: + +.. code-block:: yaml + + compilers: + - compiler: + modules = [] + operating_system: OS + paths: + cc: /usr/local/bin/icc-15.0.024-beta + cxx: /usr/local/bin/icpc-15.0.024-beta + f77: /usr/local/bin/ifort-15.0.024-beta + fc: /usr/local/bin/ifort-15.0.024-beta + parameters: + cppflags: -O3 -fPIC + spec: intel@15.0.0: These flags will be treated by spack as if they were enterred from the command line each time this compiler is used. The compiler wrappers @@ -518,8 +553,9 @@ following section. .. _sec-specs: +-------------------- Specs & dependencies -------------------------- +-------------------- We know that ``spack install``, ``spack uninstall``, and other commands take a package name with an optional version specifier. In @@ -527,10 +563,12 @@ Spack, that descriptor is called a *spec*. Spack uses specs to refer to a particular build configuration (or configurations) of a package. Specs are more than a package name and a version; you can use them to specify the compiler, compiler version, architecture, compile options, -and dependency options for a build. In this section, we\'ll go over +and dependency options for a build. In this section, we'll go over the full syntax of specs. -Here is an example of a much longer spec than we\'ve seen thus far:: +Here is an example of a much longer spec than we've seen thus far: + +.. code-block:: none mpileaks @1.2:1.4 %gcc@4.7.5 +debug -qt arch=bgq_os ^callpath @1.1 %gcc@4.7.2 @@ -609,7 +647,6 @@ some *other* version of ``mpich``. In general, such a configuration would likely behave unexpectedly at runtime, and Spack enforces this to ensure a consistent runtime environment. - The point of specs is to abstract this full DAG from Spack users. If a user does not care about the DAG at all, she can refer to mpileaks by simply writing ``mpileaks``. If she knows that ``mpileaks`` @@ -625,7 +662,7 @@ of dependencies within a spec is that they appear *after* the root package. For example, these two specs represent exactly the same configuration: -.. code-block:: sh +.. code-block:: none mpileaks ^callpath@1.0 ^libelf@0.8.3 mpileaks ^libelf@0.8.3 ^callpath@1.0 @@ -646,8 +683,9 @@ could depend on ``mpich@1.2:`` if it can only build with version Below are more details about the specifiers that you can add to specs. +^^^^^^^^^^^^^^^^^ Version specifier -~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^ A version specifier comes somewhere after a package name and starts with ``@``. It can be a single version, e.g. ``@1.0``, ``@3``, or @@ -669,9 +707,9 @@ policies set for the particular Spack installation. Details about how versions are compared and how Spack determines if one version is less than another are discussed in the developer guide. - +^^^^^^^^^^^^^^^^^^ Compiler specifier -~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^ A compiler specifier comes somewhere after a package name and starts with ``%``. It tells Spack what compiler(s) a particular package @@ -688,9 +726,9 @@ name or compiler specifier to their left in the spec. If the compiler spec is omitted, Spack will choose a default compiler based on site policies. - +^^^^^^^^ Variants -~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^ Variants are named options associated with a particular package. They are optional, as each package must provide default values for each variant it @@ -698,7 +736,7 @@ makes available. Variants can be specified using a flexible parameter syntax ``name=<value>``. For example, ``spack install libelf debug=True`` will install libelf build with debug flags. The names of particular variants available for a package depend on -what was provided by the package author. ``spack into <package>`` will +what was provided by the package author. ``spack info <package>`` will provide information on what build variants are available. For compatibility with earlier versions, variants which happen to be @@ -709,7 +747,7 @@ compile time option for the libelf package. Depending on the package a variant may have any default value. For ``libelf`` here, ``debug`` is ``False`` by default, and we turned it on -with ``debug=True`` or ``+debug``. If a package is ``True`` by default +with ``debug=True`` or ``+debug``. If a variant is ``True`` by default you can turn it off by either adding ``-name`` or ``~name`` to the spec. There are two syntaxes here because, depending on context, ``~`` and @@ -742,9 +780,9 @@ variants using the backwards compatibility syntax and uses only ``~`` for disabled boolean variants. We allow ``-`` and spaces on the command line is provided for convenience and legibility. - +^^^^^^^^^^^^^^ Compiler Flags -~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^ Compiler flags are specified using the same syntax as non-boolean variants, but fulfill a different purpose. While the function of a variant is set by @@ -760,32 +798,32 @@ command line. From within python files, the same spec would be specified handles the quote symbols. The six compiler flags are injected in the order of implicit make commands -in gnu autotools. If all flags are set, the order is -``$cppflags $cflags|$cxxflags $ldflags command $ldlibs`` for C and C++ and -``$fflags $cppflags $ldflags command $ldlibs`` for fortran. - +in GNU Autotools. If all flags are set, the order is +``$cppflags $cflags|$cxxflags $ldflags <command> $ldlibs`` for C and C++ and +``$fflags $cppflags $ldflags <command> $ldlibs`` for Fortran. +^^^^^^^^^^^^^^^^^^^^^^^ Architecture specifiers -~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^ The architecture can be specified by using the reserved words ``target`` and/or ``os`` (``target=x86-64 os=debian7``). You can also use the triplet form of platform, operating system and processor. -.. code-block:: sh +.. code-block:: console - spack install libelf arch=cray_xc-CNL10-haswell + $ spack install libelf arch=cray-CNL10-haswell Users on non-Cray systems won't have to worry about specifying the architecture. Spack will autodetect what kind of operating system is on your machine as well as the processor. For more information on how the architecture can be -used on Cray machines, check here :ref:`cray-support` - +used on Cray machines, see :ref:`cray-support` .. _sec-virtual-dependencies: +-------------------- Virtual dependencies -------------------------- +-------------------- The dependence graph for ``mpileaks`` we saw above wasn't *quite* accurate. ``mpileaks`` uses MPI, which is an interface that has many @@ -828,27 +866,33 @@ version of a virtual package, e.g. if an application needs MPI-2 functions, it can depend on ``mpi@2:`` to indicate that it needs some implementation that provides MPI-2 functions. - +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Constraining virtual packages -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ When installing a package that depends on a virtual package, you can opt to specify the particular provider you want to use, or you can let -Spack pick. For example, if you just type this:: +Spack pick. For example, if you just type this: + +.. code-block:: console - spack install mpileaks + $ spack install mpileaks Then spack will pick a provider for you according to site policies. -If you really want a particular version, say mpich, then you could -run this instead:: +If you really want a particular version, say ``mpich``, then you could +run this instead: + +.. code-block:: console - spack install mpileaks ^mpich + $ spack install mpileaks ^mpich This forces spack to use some version of ``mpich`` for its implementation. As always, you can be even more specific and require -a particular ``mpich`` version:: +a particular ``mpich`` version: + +.. code-block:: console - spack install mpileaks ^mpich@3 + $ spack install mpileaks ^mpich@3 The ``mpileaks`` package in particular only needs MPI-1 commands, so any MPI implementation will do. If another package depends on @@ -857,8 +901,9 @@ any MPI implementation will do. If another package depends on error. Likewise, if you try to plug in some package that doesn't provide MPI, Spack will raise an error. +^^^^^^^^^^^^^^^^^^^^^^^^ Specifying Specs by Hash -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^ Complicated specs can become cumbersome to enter on the command line, especially when many of the qualifications are necessary to @@ -872,12 +917,13 @@ to be unique, Spack will replace the reference with the spec to which it refers. Otherwise, it will prompt for a more qualified hash. Note that this will not work to reinstall a depencency uninstalled by -``spack uninstall -f``. +``spack uninstall --force``. .. _spack-providers: +^^^^^^^^^^^^^^^^^^^ ``spack providers`` -~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^ You can see what packages provide a particular virtual package using ``spack providers``. If you wanted to see what packages provide @@ -895,6 +941,7 @@ versions are now filtered out. .. _shell-support: +------------------------------- Integration with module systems ------------------------------- @@ -906,14 +953,13 @@ Integration with module systems versions. Spack provides some integration with -`Environment Modules <http://modules.sourceforge.net/>`_ +`Environment Modules <http://modules.sourceforge.net/>`__ and `Dotkit <https://computing.llnl.gov/?set=jobs&page=dotkit>`_ to make it easier to use the packages it installed. - - +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Installing Environment Modules -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ In order to use Spack's generated environment modules, you must have installed the *Environment Modules* package. On many Linux @@ -921,16 +967,15 @@ distributions, this can be installed from the vendor's repository: .. code-block:: sh - yum install environment-modules # (Fedora/RHEL/CentOS) - apt-get install environment-modules # (Ubuntu/Debian) + $ yum install environment-modules # (Fedora/RHEL/CentOS) + $ apt-get install environment-modules # (Ubuntu/Debian) If your Linux distribution does not have Environment Modules, you can get it with Spack: -.. code-block:: sh - - spack install environment-modules +.. code-block:: console + $ spack install environment-modules In this case to activate it automatically you need to add the following two lines to your ``.bashrc`` profile (or similar): @@ -944,11 +989,12 @@ If you use a Unix shell other than ``bash``, modify the commands above accordingly and source the appropriate file in ``${MODULES_HOME}/Modules/init/``. - .. TODO : Add a similar section on how to install dotkit ? +^^^^^^^^^^^^^^^^^^^^^^^^ Spack and module systems -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^ + You can enable shell support by sourcing some files in the ``/share/spack`` directory. @@ -956,7 +1002,8 @@ For ``bash`` or ``ksh``, run: .. code-block:: sh - . ${SPACK_ROOT}/share/spack/setup-env.sh + export SPACK_ROOT=/path/to/spack + . $SPACK_ROOT/share/spack/setup-env.sh For ``csh`` and ``tcsh`` run: @@ -972,79 +1019,79 @@ When you install a package with Spack, it automatically generates a module file that lets you add the package to your environment. Currently, Spack supports the generation of `Environment Modules -<http://wiki.tcl.tk/12999>`_ and `Dotkit +<http://wiki.tcl.tk/12999>`__ and `Dotkit <https://computing.llnl.gov/?set=jobs&page=dotkit>`_. Generated module files for each of these systems can be found in these directories: .. code-block:: sh - ${SPACK_ROOT}/share/spack/modules - ${SPACK_ROOT}/share/spack/dotkit + $SPACK_ROOT/share/spack/modules + $SPACK_ROOT/share/spack/dotkit The directories are automatically added to your ``MODULEPATH`` and ``DK_NODE`` environment variables when you enable Spack's `shell support <shell-support_>`_. - +^^^^^^^^^^^^^^^^^^^^^^^ Using Modules & Dotkits -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^ If you have shell support enabled you should be able to run either ``module avail`` or ``use -l spack`` to see what modules/dotkits have been installed. Here is sample output of those programs, showing lots of installed packages. - .. code-block:: sh - - $ module avail - - ------- /home/gamblin2/spack/share/spack/modules/linux-debian7-x86_64 -------- - adept-utils@1.0%gcc@4.4.7-5adef8da libelf@0.8.13%gcc@4.4.7 - automaded@1.0%gcc@4.4.7-d9691bb0 libelf@0.8.13%intel@15.0.0 - boost@1.55.0%gcc@4.4.7 mpc@1.0.2%gcc@4.4.7-559607f5 - callpath@1.0.1%gcc@4.4.7-5dce4318 mpfr@3.1.2%gcc@4.4.7 - dyninst@8.1.2%gcc@4.4.7-b040c20e mpich@3.0.4%gcc@4.4.7 - gcc@4.9.1%gcc@4.4.7-93ab98c5 mpich@3.0.4%gcc@4.9.0 - gmp@6.0.0a%gcc@4.4.7 mrnet@4.1.0%gcc@4.4.7-72b7881d - graphlib@2.0.0%gcc@4.4.7 netgauge@2.4.6%gcc@4.9.0-27912b7b - launchmon@1.0.1%gcc@4.4.7 stat@2.1.0%gcc@4.4.7-51101207 - libNBC@1.1.1%gcc@4.9.0-27912b7b sundials@2.5.0%gcc@4.9.0-27912b7b - libdwarf@20130729%gcc@4.4.7-b52fac98 - - .. code-block:: sh - - $ use -l spack - - spack ---------- - adept-utils@1.0%gcc@4.4.7-5adef8da - adept-utils @1.0 - automaded@1.0%gcc@4.4.7-d9691bb0 - automaded @1.0 - boost@1.55.0%gcc@4.4.7 - boost @1.55.0 - callpath@1.0.1%gcc@4.4.7-5dce4318 - callpath @1.0.1 - dyninst@8.1.2%gcc@4.4.7-b040c20e - dyninst @8.1.2 - gmp@6.0.0a%gcc@4.4.7 - gmp @6.0.0a - libNBC@1.1.1%gcc@4.9.0-27912b7b - libNBC @1.1.1 - libdwarf@20130729%gcc@4.4.7-b52fac98 - libdwarf @20130729 - libelf@0.8.13%gcc@4.4.7 - libelf @0.8.13 - libelf@0.8.13%intel@15.0.0 - libelf @0.8.13 - mpc@1.0.2%gcc@4.4.7-559607f5 - mpc @1.0.2 - mpfr@3.1.2%gcc@4.4.7 - mpfr @3.1.2 - mpich@3.0.4%gcc@4.4.7 - mpich @3.0.4 - mpich@3.0.4%gcc@4.9.0 - mpich @3.0.4 - netgauge@2.4.6%gcc@4.9.0-27912b7b - netgauge @2.4.6 - sundials@2.5.0%gcc@4.9.0-27912b7b - sundials @2.5.0 +.. code-block:: console + + $ module avail + + ------- /home/gamblin2/spack/share/spack/modules/linux-debian7-x86_64 -------- + adept-utils@1.0%gcc@4.4.7-5adef8da libelf@0.8.13%gcc@4.4.7 + automaded@1.0%gcc@4.4.7-d9691bb0 libelf@0.8.13%intel@15.0.0 + boost@1.55.0%gcc@4.4.7 mpc@1.0.2%gcc@4.4.7-559607f5 + callpath@1.0.1%gcc@4.4.7-5dce4318 mpfr@3.1.2%gcc@4.4.7 + dyninst@8.1.2%gcc@4.4.7-b040c20e mpich@3.0.4%gcc@4.4.7 + gcc@4.9.1%gcc@4.4.7-93ab98c5 mpich@3.0.4%gcc@4.9.0 + gmp@6.0.0a%gcc@4.4.7 mrnet@4.1.0%gcc@4.4.7-72b7881d + graphlib@2.0.0%gcc@4.4.7 netgauge@2.4.6%gcc@4.9.0-27912b7b + launchmon@1.0.1%gcc@4.4.7 stat@2.1.0%gcc@4.4.7-51101207 + libNBC@1.1.1%gcc@4.9.0-27912b7b sundials@2.5.0%gcc@4.9.0-27912b7b + libdwarf@20130729%gcc@4.4.7-b52fac98 + +.. code-block:: console + + $ use -l spack + + spack ---------- + adept-utils@1.0%gcc@4.4.7-5adef8da - adept-utils @1.0 + automaded@1.0%gcc@4.4.7-d9691bb0 - automaded @1.0 + boost@1.55.0%gcc@4.4.7 - boost @1.55.0 + callpath@1.0.1%gcc@4.4.7-5dce4318 - callpath @1.0.1 + dyninst@8.1.2%gcc@4.4.7-b040c20e - dyninst @8.1.2 + gmp@6.0.0a%gcc@4.4.7 - gmp @6.0.0a + libNBC@1.1.1%gcc@4.9.0-27912b7b - libNBC @1.1.1 + libdwarf@20130729%gcc@4.4.7-b52fac98 - libdwarf @20130729 + libelf@0.8.13%gcc@4.4.7 - libelf @0.8.13 + libelf@0.8.13%intel@15.0.0 - libelf @0.8.13 + mpc@1.0.2%gcc@4.4.7-559607f5 - mpc @1.0.2 + mpfr@3.1.2%gcc@4.4.7 - mpfr @3.1.2 + mpich@3.0.4%gcc@4.4.7 - mpich @3.0.4 + mpich@3.0.4%gcc@4.9.0 - mpich @3.0.4 + netgauge@2.4.6%gcc@4.9.0-27912b7b - netgauge @2.4.6 + sundials@2.5.0%gcc@4.9.0-27912b7b - sundials @2.5.0 The names here should look familiar, they're the same ones from ``spack find``. You *can* use the names here directly. For example, you could type either of these commands to load the callpath module: -.. code-block:: sh +.. code-block:: console - use callpath@1.0.1%gcc@4.4.7-5dce4318 + $ use callpath@1.0.1%gcc@4.4.7-5dce4318 -.. code-block:: sh +.. code-block:: console - module load callpath@1.0.1%gcc@4.4.7-5dce4318 + $ module load callpath@1.0.1%gcc@4.4.7-5dce4318 Neither of these is particularly pretty, easy to remember, or easy to type. Luckily, Spack has its own interface for using modules @@ -1061,7 +1108,7 @@ And you can use the same shortened names you use everywhere else in Spack. For example, this will add the ``mpich`` package built with ``gcc`` to your path: -.. code-block:: sh +.. code-block:: console $ spack install mpich %gcc@4.4.7 @@ -1074,18 +1121,18 @@ Spack. For example, this will add the ``mpich`` package built with Or, similarly with modules, you could type: -.. code-block:: sh +.. code-block:: console $ spack load mpich %gcc@4.4.7 These commands will add appropriate directories to your ``PATH``, -``MANPATH``, ``CPATH``, and ``LD_LIBRARY_PATH``. When you no longer want to use -a package, you can type unload or unuse similarly: +``MANPATH``, ``CPATH``, and ``LD_LIBRARY_PATH``. When you no longer +want to use a package, you can type unload or unuse similarly: -.. code-block:: sh +.. code-block:: console - $ spack unload mpich %gcc@4.4.7 # modules - $ spack unuse mpich %gcc@4.4.7 # dotkit + $ spack unload mpich %gcc@4.4.7 # modules + $ spack unuse mpich %gcc@4.4.7 # dotkit .. note:: @@ -1093,13 +1140,14 @@ a package, you can type unload or unuse similarly: only available if you have enabled Spack's shell support *and* you have dotkit or modules installed on your machine. +^^^^^^^^^^^^^^^^^^^^^^ Ambiguous module names -~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^ If a spec used with load/unload or use/unuse is ambiguous (i.e. more than one installed package matches it), then Spack will warn you: -.. code-block:: sh +.. code-block:: console $ spack load libelf ==> Error: Multiple matches for spec libelf. Choose one: @@ -1112,42 +1160,45 @@ identify one package. For example, above, the key differentiator is that one ``libelf`` is built with the Intel compiler, while the other used ``gcc``. You could therefore just type: -.. code-block:: sh +.. code-block:: console $ spack load libelf %intel To identify just the one built with the Intel compiler. +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Module files generation and customization -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Environment Modules and Dotkit files are generated when packages are installed, and are placed in the following directories under the Spack root: .. code-block:: sh - ${SPACK_ROOT}/share/spack/modules - ${SPACK_ROOT}/share/spack/dotkit + $SPACK_ROOT/share/spack/modules + $SPACK_ROOT/share/spack/dotkit The content that gets written in each module file can be customized in two ways: - 1. overriding part of the ``spack.Package`` API within a ``package.py`` - 2. writing dedicated configuration files +#. overriding part of the ``spack.Package`` API within a ``package.py`` +#. writing dedicated configuration files +^^^^^^^^^^^^^^^^^^^^^^^^ Override ``Package`` API ^^^^^^^^^^^^^^^^^^^^^^^^ + There are currently two methods in ``spack.Package`` that may affect the content of module files: .. code-block:: python - def setup_environment(self, spack_env, run_env): - """Set up the compile and runtime environments for a package.""" - pass - + def setup_environment(self, spack_env, run_env): + """Set up the compile and runtime environments for a package.""" + pass +""""""""""""""""" Recursive Modules -`````````````````` +""""""""""""""""" In some cases, it is desirable to load not just a module, but also all the modules it depends on. This is not required for most modules @@ -1159,14 +1210,15 @@ RPATH. Modules may be loaded recursively with the ``load`` command's ``--dependencies`` or ``-r`` argument: -.. code-block:: sh +.. code-block:: console - $ spack load --dependencies <spec> ... + $ spack load --dependencies <spec> ... More than one spec may be placed on the command line here. +""""""""""""""""""""""""""""""""" Module Commands for Shell Scripts -`````````````````````````````````` +""""""""""""""""""""""""""""""""" Although Spack is flexible, the ``module`` command is much faster. This could become an issue when emitting a series of ``spack load`` @@ -1174,62 +1226,63 @@ commands inside a shell script. By adding the ``--shell`` flag, ``spack module find`` may also be used to generate code that can be cut-and-pasted into a shell script. For example: -.. code-block:: sh - - $ spack module find tcl --dependencies --shell py-numpy git - # bzip2@1.0.6%gcc@4.9.3=linux-x86_64 - module load bzip2-1.0.6-gcc-4.9.3-ktnrhkrmbbtlvnagfatrarzjojmkvzsx - # ncurses@6.0%gcc@4.9.3=linux-x86_64 - module load ncurses-6.0-gcc-4.9.3-kaazyneh3bjkfnalunchyqtygoe2mncv - # zlib@1.2.8%gcc@4.9.3=linux-x86_64 - module load zlib-1.2.8-gcc-4.9.3-v3ufwaahjnviyvgjcelo36nywx2ufj7z - # sqlite@3.8.5%gcc@4.9.3=linux-x86_64 - module load sqlite-3.8.5-gcc-4.9.3-a3eediswgd5f3rmto7g3szoew5nhehbr - # readline@6.3%gcc@4.9.3=linux-x86_64 - module load readline-6.3-gcc-4.9.3-se6r3lsycrwxyhreg4lqirp6xixxejh3 - # python@3.5.1%gcc@4.9.3=linux-x86_64 - module load python-3.5.1-gcc-4.9.3-5q5rsrtjld4u6jiicuvtnx52m7tfhegi - # py-setuptools@20.5%gcc@4.9.3=linux-x86_64 - module load py-setuptools-20.5-gcc-4.9.3-4qr2suj6p6glepnedmwhl4f62x64wxw2 - # py-nose@1.3.7%gcc@4.9.3=linux-x86_64 - module load py-nose-1.3.7-gcc-4.9.3-pwhtjw2dvdvfzjwuuztkzr7b4l6zepli - # openblas@0.2.17%gcc@4.9.3+shared=linux-x86_64 - module load openblas-0.2.17-gcc-4.9.3-pw6rmlom7apfsnjtzfttyayzc7nx5e7y - # py-numpy@1.11.0%gcc@4.9.3+blas+lapack=linux-x86_64 - module load py-numpy-1.11.0-gcc-4.9.3-mulodttw5pcyjufva4htsktwty4qd52r - # curl@7.47.1%gcc@4.9.3=linux-x86_64 - module load curl-7.47.1-gcc-4.9.3-ohz3fwsepm3b462p5lnaquv7op7naqbi - # autoconf@2.69%gcc@4.9.3=linux-x86_64 - module load autoconf-2.69-gcc-4.9.3-bkibjqhgqm5e3o423ogfv2y3o6h2uoq4 - # cmake@3.5.0%gcc@4.9.3~doc+ncurses+openssl~qt=linux-x86_64 - module load cmake-3.5.0-gcc-4.9.3-x7xnsklmgwla3ubfgzppamtbqk5rwn7t - # expat@2.1.0%gcc@4.9.3=linux-x86_64 - module load expat-2.1.0-gcc-4.9.3-6pkz2ucnk2e62imwakejjvbv6egncppd - # git@2.8.0-rc2%gcc@4.9.3+curl+expat=linux-x86_64 - module load git-2.8.0-rc2-gcc-4.9.3-3bib4hqtnv5xjjoq5ugt3inblt4xrgkd +.. code-block:: console + + $ spack module find tcl --dependencies --shell py-numpy git + # bzip2@1.0.6%gcc@4.9.3=linux-x86_64 + module load bzip2-1.0.6-gcc-4.9.3-ktnrhkrmbbtlvnagfatrarzjojmkvzsx + # ncurses@6.0%gcc@4.9.3=linux-x86_64 + module load ncurses-6.0-gcc-4.9.3-kaazyneh3bjkfnalunchyqtygoe2mncv + # zlib@1.2.8%gcc@4.9.3=linux-x86_64 + module load zlib-1.2.8-gcc-4.9.3-v3ufwaahjnviyvgjcelo36nywx2ufj7z + # sqlite@3.8.5%gcc@4.9.3=linux-x86_64 + module load sqlite-3.8.5-gcc-4.9.3-a3eediswgd5f3rmto7g3szoew5nhehbr + # readline@6.3%gcc@4.9.3=linux-x86_64 + module load readline-6.3-gcc-4.9.3-se6r3lsycrwxyhreg4lqirp6xixxejh3 + # python@3.5.1%gcc@4.9.3=linux-x86_64 + module load python-3.5.1-gcc-4.9.3-5q5rsrtjld4u6jiicuvtnx52m7tfhegi + # py-setuptools@20.5%gcc@4.9.3=linux-x86_64 + module load py-setuptools-20.5-gcc-4.9.3-4qr2suj6p6glepnedmwhl4f62x64wxw2 + # py-nose@1.3.7%gcc@4.9.3=linux-x86_64 + module load py-nose-1.3.7-gcc-4.9.3-pwhtjw2dvdvfzjwuuztkzr7b4l6zepli + # openblas@0.2.17%gcc@4.9.3+shared=linux-x86_64 + module load openblas-0.2.17-gcc-4.9.3-pw6rmlom7apfsnjtzfttyayzc7nx5e7y + # py-numpy@1.11.0%gcc@4.9.3+blas+lapack=linux-x86_64 + module load py-numpy-1.11.0-gcc-4.9.3-mulodttw5pcyjufva4htsktwty4qd52r + # curl@7.47.1%gcc@4.9.3=linux-x86_64 + module load curl-7.47.1-gcc-4.9.3-ohz3fwsepm3b462p5lnaquv7op7naqbi + # autoconf@2.69%gcc@4.9.3=linux-x86_64 + module load autoconf-2.69-gcc-4.9.3-bkibjqhgqm5e3o423ogfv2y3o6h2uoq4 + # cmake@3.5.0%gcc@4.9.3~doc+ncurses+openssl~qt=linux-x86_64 + module load cmake-3.5.0-gcc-4.9.3-x7xnsklmgwla3ubfgzppamtbqk5rwn7t + # expat@2.1.0%gcc@4.9.3=linux-x86_64 + module load expat-2.1.0-gcc-4.9.3-6pkz2ucnk2e62imwakejjvbv6egncppd + # git@2.8.0-rc2%gcc@4.9.3+curl+expat=linux-x86_64 + module load git-2.8.0-rc2-gcc-4.9.3-3bib4hqtnv5xjjoq5ugt3inblt4xrgkd The script may be further edited by removing unnecessary modules. -This script may be directly executed in bash via +This script may be directly executed in bash via: -.. code-block :: sh - - source <( spack module find tcl --dependencies --shell py-numpy git ) +.. code-block:: sh + source < (spack module find tcl --dependencies --shell py-numpy git) +^^^^^^^^^^^^^^^^^^^^^^^^^ Regenerating Module files -~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^ .. code-block:: python - def setup_dependent_environment(self, spack_env, run_env, dependent_spec): - """Set up the environment of packages that depend on this one""" - pass + def setup_dependent_environment(self, spack_env, run_env, dependent_spec): + """Set up the environment of packages that depend on this one""" + pass As briefly stated in the comments, the first method lets you customize the module file content for the package you are currently writing, the second allows for modifications to your dependees module file. In both cases one needs to fill ``run_env`` with the desired list of environment modifications. +"""""""""""""""""""""""""""""""""""""""""""""""" Example : ``builtin/packages/python/package.py`` """""""""""""""""""""""""""""""""""""""""""""""" @@ -1238,14 +1291,14 @@ overrides ``setup_dependent_environment`` in the following way: .. code-block:: python - def setup_dependent_environment(self, spack_env, run_env, extension_spec): - # ... - if extension_spec.package.extends(self.spec): - run_env.prepend_path('PYTHONPATH', os.path.join(extension_spec.prefix, self.site_packages_dir)) + def setup_dependent_environment(self, spack_env, run_env, extension_spec): + if extension_spec.package.extends(self.spec): + run_env.prepend_path('PYTHONPATH', os.path.join(extension_spec.prefix, self.site_packages_dir)) to insert the appropriate ``PYTHONPATH`` modifications in the module files of python packages. +^^^^^^^^^^^^^^^^^^^ Configuration files ^^^^^^^^^^^^^^^^^^^ @@ -1255,8 +1308,8 @@ file can be placed either at *site* or *user* scope. The default site configuration reads: - .. literalinclude:: ../../../etc/spack/modules.yaml - :language: yaml +.. literalinclude:: ../../../etc/spack/defaults/modules.yaml + :language: yaml It basically inspects the installation prefixes for the existence of a few folders and, if they exist, it prepends a path to a given @@ -1266,17 +1319,19 @@ For each module system that can be enabled a finer configuration is possible: .. code-block:: yaml - modules: - tcl: - # contains environment modules specific customizations - dotkit: - # contains dotkit specific customizations + modules: + tcl: + # contains environment modules specific customizations + dotkit: + # contains dotkit specific customizations The structure under the ``tcl`` and ``dotkit`` keys is almost equal, and will be showcased in the following by some examples. +""""""""""""""""""""""""""""""""""""""" Select module files by spec constraints """"""""""""""""""""""""""""""""""""""" + Using spec syntax it's possible to have different customizations for different groups of module files. @@ -1284,24 +1339,24 @@ Considering : .. code-block:: yaml - modules: - tcl: - all: # Default addition for every package - environment: - set: - BAR: 'bar' - ^openmpi:: # A double ':' overrides previous rules - environment: - set: - BAR: 'baz' - zlib: - environment: - prepend_path: - LD_LIBRARY_PATH: 'foo' - zlib%gcc@4.8: - environment: - unset: - - FOOBAR + modules: + tcl: + all: # Default addition for every package + environment: + set: + BAR: 'bar' + ^openmpi:: # A double ':' overrides previous rules + environment: + set: + BAR: 'baz' + zlib: + environment: + prepend_path: + LD_LIBRARY_PATH: 'foo' + zlib%gcc@4.8: + environment: + unset: + - FOOBAR what will happen is that: @@ -1311,11 +1366,12 @@ what will happen is that: - any spec that satisfies ``zlib%gcc@4.8`` will additionally unset ``FOOBAR`` .. note:: - Order does matter - The modifications associated with the ``all`` keyword are always evaluated - first, no matter where they appear in the configuration file. All the other - spec constraints are instead evaluated top to bottom. + Order does matter + The modifications associated with the ``all`` keyword are always evaluated + first, no matter where they appear in the configuration file. All the other + spec constraints are instead evaluated top to bottom. +"""""""""""""""""""""""""""""""""""""""" Filter modifications out of module files """""""""""""""""""""""""""""""""""""""" @@ -1325,16 +1381,18 @@ modified by your dotkit modules. Then : .. code-block:: yaml - modules: - dotkit: - all: - filter: - environment_blacklist: ['CPATH', 'LIBRARY_PATH'] # Exclude changes to any of these variables + modules: + dotkit: + all: + filter: + # Exclude changes to any of these variables + environment_blacklist: ['CPATH', 'LIBRARY_PATH'] will generate dotkit module files that will not contain modifications to either ``CPATH`` or ``LIBRARY_PATH`` and environment module files that instead will contain those modifications. +""""""""""""""""""""" Autoload dependencies """"""""""""""""""""" @@ -1342,21 +1400,22 @@ The following lines in ``modules.yaml``: .. code-block:: yaml - modules: - tcl: - all: - autoload: 'direct' + modules: + tcl: + all: + autoload: 'direct' will produce environment module files that will automatically load their direct dependencies. .. note:: - Allowed values for ``autoload`` statements - Allowed values for ``autoload`` statements are either ``none``, ``direct`` - or ``all``. In ``tcl`` configuration it is possible to use the option - ``prerequisites`` that accepts the same values and will add ``prereq`` - statements instead of automatically loading other modules. + Allowed values for ``autoload`` statements + Allowed values for ``autoload`` statements are either ``none``, ``direct`` + or ``all``. In ``tcl`` configuration it is possible to use the option + ``prerequisites`` that accepts the same values and will add ``prereq`` + statements instead of automatically loading other modules. +"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" Blacklist or whitelist the generation of specific module files """""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" @@ -1367,14 +1426,15 @@ A configuration file like: .. code-block:: yaml - modules: - tcl: - whitelist: ['gcc', 'llvm'] # Whitelist will have precedence over blacklist - blacklist: ['%gcc@4.4.7'] # Assuming gcc@4.4.7 is the system compiler + modules: + tcl: + whitelist: ['gcc', 'llvm'] # Whitelist will have precedence over blacklist + blacklist: ['%gcc@4.4.7'] # Assuming gcc@4.4.7 is the system compiler will skip module file generation for anything that satisfies ``%gcc@4.4.7``, with the exception of specs that satisfy ``gcc`` or ``llvm``. +"""""""""""""""""""""""""""""""""""""""""""""""" Customize the naming scheme and insert conflicts """""""""""""""""""""""""""""""""""""""""""""""" @@ -1382,24 +1442,25 @@ A configuration file like: .. code-block:: yaml - modules: - tcl: - naming_scheme: '{name}/{version}-{compiler.name}-{compiler.version}' - all: - conflict: ['{name}', 'intel/14.0.1'] + modules: + tcl: + naming_scheme: '{name}/{version}-{compiler.name}-{compiler.version}' + all: + conflict: ['{name}', 'intel/14.0.1'] will create module files that will conflict with ``intel/14.0.1`` and with the base directory of the same module, effectively preventing the possibility to load two or more versions of the same software at the same time. .. note:: - Tokens available for the naming scheme - currently only the tokens shown in the example are available to construct - the naming scheme + Tokens available for the naming scheme + currently only the tokens shown in the example are available to construct + the naming scheme .. note:: - The ``conflict`` option is ``tcl`` specific + The ``conflict`` option is ``tcl`` specific +^^^^^^^^^^^^^^^^^^^^^^^^^ Regenerating module files ^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -1410,6 +1471,7 @@ new features. .. _spack-module: +"""""""""""""""""""""""" ``spack module refresh`` """""""""""""""""""""""" @@ -1417,17 +1479,15 @@ Running ``spack module refresh`` will remove the ``share/spack/modules`` and ``share/spack/dotkit`` directories, then regenerate all module and dotkit files from scratch: -.. code-block:: sh +.. code-block:: console $ spack module refresh ==> Regenerating tcl module files. ==> Regenerating dotkit module files. - -.. _extensions: - +---------------- Filesystem Views -------------------------------- +---------------- .. Maybe this is not the right location for this documentation. @@ -1464,13 +1524,14 @@ built by Spack. Views can be used for a variety of purposes including: -- A central installation in a traditional layout, eg ``/usr/local`` maintained over time by the sysadmin. -- A self-contained installation area which may for the basis of a top-level atomic versioning scheme, eg ``/opt/pro`` vs ``/opt/dev``. -- Providing an atomic and monolithic binary distribution, eg for delivery as a single tarball. -- Producing ephemeral testing or developing environments. +* A central installation in a traditional layout, eg ``/usr/local`` maintained over time by the sysadmin. +* A self-contained installation area which may for the basis of a top-level atomic versioning scheme, eg ``/opt/pro`` vs ``/opt/dev``. +* Providing an atomic and monolithic binary distribution, eg for delivery as a single tarball. +* Producing ephemeral testing or developing environments. +^^^^^^^^^^^^^^^^^^^^^^ Using Filesystem Views -~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^ A filesystem view is created and packages are linked in by the ``spack view`` command's ``symlink`` and ``hardlink`` sub-commands. The @@ -1481,32 +1542,31 @@ The following example creates a filesystem view based on an installed ``cmake`` package and then removes from the view the files in the ``cmake`` package while retaining its dependencies. -.. code-block:: sh - +.. code-block:: console - $ spack view -v symlink myview cmake@3.5.2 - ==> Linking package: "ncurses" - ==> Linking package: "zlib" - ==> Linking package: "openssl" - ==> Linking package: "cmake" + $ spack view --verbose symlink myview cmake@3.5.2 + ==> Linking package: "ncurses" + ==> Linking package: "zlib" + ==> Linking package: "openssl" + ==> Linking package: "cmake" - $ ls myview/ - bin doc etc include lib share + $ ls myview/ + bin doc etc include lib share - $ ls myview/bin/ - captoinfo clear cpack ctest infotocap openssl tabs toe tset - ccmake cmake c_rehash infocmp ncurses6-config reset tic tput + $ ls myview/bin/ + captoinfo clear cpack ctest infotocap openssl tabs toe tset + ccmake cmake c_rehash infocmp ncurses6-config reset tic tput - $ spack view -v -d false rm myview cmake@3.5.2 - ==> Removing package: "cmake" - - $ ls myview/bin/ - captoinfo c_rehash infotocap openssl tabs toe tset - clear infocmp ncurses6-config reset tic tput + $ spack view --verbose --dependencies false rm myview cmake@3.5.2 + ==> Removing package: "cmake" + $ ls myview/bin/ + captoinfo c_rehash infotocap openssl tabs toe tset + clear infocmp ncurses6-config reset tic tput +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Limitations of Filesystem Views -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ This section describes some limitations that should be considered in using filesystems views. @@ -1536,12 +1596,11 @@ particular, if two packages require the same sub-tree of dependencies, removing one package (recursively) will remove its dependencies and leave the other package broken. +.. _extensions: - - - +--------------------------- Extensions & Python support ------------------------------------- +--------------------------- Spack's installation model assumes that each package will live in its own install prefix. However, certain packages are typically installed @@ -1554,7 +1613,7 @@ Spack has support for this type of installation as well. In Spack, a package that can live inside the prefix of another package is called an *extension*. Suppose you have Python installed like so: -.. code-block:: sh +.. code-block:: console $ spack find python ==> 1 installed packages. @@ -1563,12 +1622,13 @@ an *extension*. Suppose you have Python installed like so: .. _spack-extensions: +^^^^^^^^^^^^^^^^^^^^ ``spack extensions`` -~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^ You can find extensions for your Python installation like this: -.. code-block:: sh +.. code-block:: console $ spack extensions python ==> python@2.7.8%gcc@4.4.7 arch=linux-debian7-x86_64-703c7a96 @@ -1593,11 +1653,11 @@ You can find extensions for your Python installation like this: The extensions are a subset of what's returned by ``spack list``, and they are packages like any other. They are installed into their own -prefixes, and you can see this with ``spack find -p``: +prefixes, and you can see this with ``spack find --paths``: -.. code-block:: sh +.. code-block:: console - $ spack find -p py-numpy + $ spack find --paths py-numpy ==> 1 installed packages. -- linux-debian7-x86_64 / gcc@4.4.7 -------------------------------- py-numpy@1.9.1 /g/g21/gamblin2/src/spack/opt/linux-debian7-x86_64/gcc@4.4.7/py-numpy@1.9.1-66733244 @@ -1605,7 +1665,7 @@ prefixes, and you can see this with ``spack find -p``: However, even though this package is installed, you cannot use it directly when you run ``python``: -.. code-block:: sh +.. code-block:: console $ spack load python $ python @@ -1618,8 +1678,9 @@ directly when you run ``python``: ImportError: No module named numpy >>> +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Extensions & Environment Modules -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ There are two ways to get ``numpy`` working in Python. The first is to use :ref:`shell-support`. You can simply ``use`` or ``load`` the @@ -1628,14 +1689,14 @@ in your current shell. For tcl modules: -.. code-block:: sh +.. code-block:: console $ spack load python $ spack load py-numpy or, for dotkit: -.. code-block:: sh +.. code-block:: console $ spack use python $ spack use py-numpy @@ -1643,9 +1704,9 @@ or, for dotkit: Now ``import numpy`` will succeed for as long as you keep your current session open. - +^^^^^^^^^^^^^^^^^^^^^ Activating Extensions -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^ It is often desirable to have certain packages *always* available as part of a Python installation. Spack offers a more permanent solution @@ -1655,10 +1716,11 @@ installation: .. _spack-activate: +^^^^^^^^^^^^^^^^^^ ``spack activate`` -^^^^^^^^^^^^^^^^^^^^^^^ +^^^^^^^^^^^^^^^^^^ -.. code-block:: sh +.. code-block:: console $ spack activate py-numpy ==> Activated extension py-setuptools@11.3.1%gcc@4.4.7 arch=linux-debian7-x86_64-3c74eb69 for python@2.7.8%gcc@4.4.7. @@ -1675,7 +1737,7 @@ once all dependencies were activated in the ``python`` installation, If we run ``spack extensions`` again, we now see the three new packages listed as activated: -.. code-block:: sh +.. code-block:: console $ spack extensions python ==> python@2.7.8%gcc@4.4.7 arch=linux-debian7-x86_64-703c7a96 @@ -1700,7 +1762,6 @@ packages listed as activated: -- linux-debian7-x86_64 / gcc@4.4.7 -------------------------------- py-nose@1.3.4 py-numpy@1.9.1 py-setuptools@11.3.1 - Now, when a user runs python, ``numpy`` will be available for import *without* the user having to explicitly loaded. ``python@2.7.8`` now acts like a system Python installation with ``numpy`` installed inside @@ -1718,50 +1779,56 @@ into the same prefix. Users who want a different version of a package can still get it by using environment modules, but they will have to explicitly load their preferred version. +^^^^^^^^^^^^^^^^^^^^^ ``spack activate -f`` -^^^^^^^^^^^^^^^^^^^^^^^^^ +^^^^^^^^^^^^^^^^^^^^^ + If, for some reason, you want to activate a package *without* its dependencies, you can use ``spack activate -f``: -.. code-block:: sh +.. code-block:: console $ spack activate -f py-numpy ==> Activated extension py-numpy@1.9.1%gcc@4.4.7 arch=linux-debian7-x86_64-66733244 for python@2.7.8%gcc@4.4.7. .. _spack-deactivate: +^^^^^^^^^^^^^^^^^^^^ ``spack deactivate`` -^^^^^^^^^^^^^^^^^^^^^^^^^ +^^^^^^^^^^^^^^^^^^^^ We've seen how activating an extension can be used to set up a default version of a Python module. Obviously, you may want to change that at some point. ``spack deactivate`` is the command for this. There are several variants: - * ``spack deactivate <extension>`` will deactivate a single - extension. If another activated extension depends on this one, - Spack will warn you and exit with an error. - * ``spack deactivate -f <extension>`` deactivates an extension - regardless of packages that depend on it. - * ``spack deactivate -a <extension>`` deactivates an extension and - all of its dependencies. Use ``-f`` to disregard dependents. - * ``spack deactivate -a <extendee>`` deactivates *all* activated - extensions of a package. For example, to deactivate *all* python - extensions, use:: +* ``spack deactivate <extension>`` will deactivate a single + extension. If another activated extension depends on this one, + Spack will warn you and exit with an error. +* ``spack deactivate --force <extension>`` deactivates an extension + regardless of packages that depend on it. +* ``spack deactivate --all <extension>`` deactivates an extension and + all of its dependencies. Use ``--force`` to disregard dependents. +* ``spack deactivate --all <extendee>`` deactivates *all* activated + extensions of a package. For example, to deactivate *all* python + extensions, use: + + .. code-block:: console - spack deactivate -a python + $ spack deactivate --all python +----------------------- Filesystem requirements --------------------------- +----------------------- Spack currently needs to be run from a filesystem that supports ``flock`` locking semantics. Nearly all local filesystems and recent versions of NFS support this, but parallel filesystems may be mounted without ``flock`` support enabled. You can determine how your - filesystems are mounted with ``mount -p``. The output for a Lustre +filesystems are mounted with ``mount -p``. The output for a Lustre filesystem might look like this: -.. code-block:: sh +.. code-block:: console $ mount -l | grep lscratch pilsner-mds1-lnet0@o2ib100:/lsd on /p/lscratchd type lustre (rw,nosuid,noauto,_netdev,lazystatfs,flock) @@ -1773,7 +1840,7 @@ system administrator to enable ``flock``. This issue typically manifests with the error below: -.. code-block:: sh +.. code-block:: console $ ./spack find Traceback (most recent call last): @@ -1782,7 +1849,7 @@ This issue typically manifests with the error below: File "./spack", line 154,' in main return_val = command(parser, args) File "./spack/lib/spack/spack/cmd/find.py", line 170, in find - specs = set(spack.installed_db.query(**q_args)) + specs = set(spack.installed_db.query(\**q_args)) File "./spack/lib/spack/spack/database.py", line 551, in query with self.read_transaction(): File "./spack/lib/spack/spack/database.py", line 598, in __enter__ @@ -1797,48 +1864,47 @@ This issue typically manifests with the error below: A nicer error message is TBD in future versions of Spack. - .. _cray-support: +------------- Spack on Cray ------------------------------ +------------- Spack differs slightly when used on a Cray system. The architecture spec can differentiate between the front-end and back-end processor and operating system. For example, on Edison at NERSC, the back-end target processor -is \"Ivy Bridge\", so you can specify to use the back-end this way: +is "Ivy Bridge", so you can specify to use the back-end this way: -.. code-block:: sh +.. code-block:: console - spack install zlib target=ivybridge + $ spack install zlib target=ivybridge You can also use the operating system to build against the back-end: -.. code-block:: sh +.. code-block:: console - spack install zlib os=CNL10 + $ spack install zlib os=CNL10 Notice that the name includes both the operating system name and the major version number concatenated together. Alternatively, if you want to build something for the front-end, you can specify the front-end target processor. The processor for a login node -on Edison is \"Sandy bridge\" so we specify on the command line like so: +on Edison is "Sandy bridge" so we specify on the command line like so: -.. code-block:: sh +.. code-block:: console - spack install zlib target=sandybridge + $ spack install zlib target=sandybridge And the front-end operating system is: -.. code-block:: sh - - spack install zlib os=SuSE11 - +.. code-block:: console + $ spack install zlib os=SuSE11 +^^^^^^^^^^^^^^^^^^^^^^^ Cray compiler detection -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^ Spack can detect compilers using two methods. For the front-end, we treat everything the same. The difference lies in back-end compiler detection. @@ -1849,27 +1915,27 @@ compiler wrapper names (i.e. cc, CC, ftn). During build time, Spack will load the correct PrgEnv and compiler module and will call appropriate wrapper. The compilers.yaml config file will also differ. There is a -modules section that is filled with the compiler\'s Programming Environment -and module name. On other systems, this field is empty []:: +modules section that is filled with the compiler's Programming Environment +and module name. On other systems, this field is empty []: - ... - - compiler: - modules: - - PrgEnv-intel - - intel/15.0.109 - ... +.. code-block:: yaml + + - compiler: + modules: + - PrgEnv-intel + - intel/15.0.109 As mentioned earlier, the compiler paths will look different on a Cray system. Since most compilers are invoked using cc, CC and ftn, the paths for each -compiler are replaced with their respective Cray compiler wrapper names:: +compiler are replaced with their respective Cray compiler wrapper names: + +.. code-block:: yaml - ... paths: cc: cc cxx: CC f77: ftn fc: ftn - ... As opposed to an explicit path to the compiler executable. This allows Spack to call the Cray compiler wrappers during build time. @@ -1878,23 +1944,24 @@ For more on compiler configuration, check out :ref:`compiler-config`. Spack sets the default Cray link type to dynamic, to better match other other platforms. Individual packages can enable static linking (which is the -default outside of Spack on cray systems) using the -static flag. +default outside of Spack on cray systems) using the ``-static`` flag. +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Setting defaults and using Cray modules -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ If you want to use default compilers for each PrgEnv and also be able -to load cray external modules, you will need to set up a packages.yaml. +to load cray external modules, you will need to set up a ``packages.yaml``. -Here\'s an example of an external configuration for cray modules: +Here's an example of an external configuration for cray modules: .. code-block:: yaml - packages: - mpi: - modules: - mpich@7.3.1%gcc@5.2.0 arch=cray_xc-haswell-CNL10: cray-mpich - mpich@7.3.1%intel@16.0.0.109 arch=cray_xc-haswell-CNL10: cray-mpich + packages: + mpi: + modules: + mpich@7.3.1%gcc@5.2.0 arch=cray_xc-haswell-CNL10: cray-mpich + mpich@7.3.1%intel@16.0.0.109 arch=cray_xc-haswell-CNL10: cray-mpich This tells Spack that for whatever package that depends on mpi, load the cray-mpich module into the environment. You can then be able to use whatever @@ -1902,7 +1969,7 @@ environment variables, libraries, etc, that are brought into the environment via module load. You can set the default compiler that Spack can use for each compiler type. -If you want to use the Cray defaults, then set them under *all:* in packages.yaml. +If you want to use the Cray defaults, then set them under ``all:`` in packages.yaml. In the compiler field, set the compiler specs in your order of preference. Whenever you build with that compiler type, Spack will concretize to that version. @@ -1911,42 +1978,43 @@ Here is an example of a full packages.yaml used at NERSC .. code-block:: yaml packages: - mpi: - modules: - mpich@7.3.1%gcc@5.2.0 arch=cray_xc-CNL10-ivybridge: cray-mpich - mpich@7.3.1%intel@16.0.0.109 arch=cray_xc-SuSE11-ivybridge: cray-mpich - buildable: False - netcdf: - modules: - netcdf@4.3.3.1%gcc@5.2.0 arch=cray_xc-CNL10-ivybridge: cray-netcdf - netcdf@4.3.3.1%intel@16.0.0.109 arch=cray_xc-CNL10-ivybridge: cray-netcdf - buildable: False - hdf5: - modules: - hdf5@1.8.14%gcc@5.2.0 arch=cray_xc-CNL10-ivybridge: cray-hdf5 - hdf5@1.8.14%intel@16.0.0.109 arch=cray_xc-CNL10-ivybridge: cray-hdf5 - buildable: False - all: - compiler: [gcc@5.2.0, intel@16.0.0.109] + mpi: + modules: + mpich@7.3.1%gcc@5.2.0 arch=cray_xc-CNL10-ivybridge: cray-mpich + mpich@7.3.1%intel@16.0.0.109 arch=cray_xc-SuSE11-ivybridge: cray-mpich + buildable: False + netcdf: + modules: + netcdf@4.3.3.1%gcc@5.2.0 arch=cray_xc-CNL10-ivybridge: cray-netcdf + netcdf@4.3.3.1%intel@16.0.0.109 arch=cray_xc-CNL10-ivybridge: cray-netcdf + buildable: False + hdf5: + modules: + hdf5@1.8.14%gcc@5.2.0 arch=cray_xc-CNL10-ivybridge: cray-hdf5 + hdf5@1.8.14%intel@16.0.0.109 arch=cray_xc-CNL10-ivybridge: cray-hdf5 + buildable: False + all: + compiler: [gcc@5.2.0, intel@16.0.0.109] Here we tell spack that whenever we want to build with gcc use version 5.2.0 or if we want to build with intel compilers, use version 16.0.0.109. We add a spec for each compiler type for each cray modules. This ensures that for each compiler on our system we can use that external module. - For more on external packages check out the section :ref:`sec-external_packages`. +------------ Getting Help ------------------------ +------------ .. _spack-help: +^^^^^^^^^^^^^^ ``spack help`` -~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^ If you don't find what you need here, the ``help`` subcommand will -print out out a list of *all* of ``spack``'s options and subcommands: +print out out a list of *all* of spack's options and subcommands: .. command-output:: spack help @@ -1955,5 +2023,5 @@ usage information for a particular subcommand: .. command-output:: spack help install -Alternately, you can use ``spack -h`` in place of ``spack help``, or -``spack <subcommand> -h`` to get help on a particular subcommand. +Alternately, you can use ``spack --help`` in place of ``spack help``, or +``spack <subcommand> --help`` to get help on a particular subcommand. diff --git a/lib/spack/docs/case_studies.rst b/lib/spack/docs/case_studies.rst index bcd754fdcd..fcec636c27 100644 --- a/lib/spack/docs/case_studies.rst +++ b/lib/spack/docs/case_studies.rst @@ -1,5 +1,6 @@ +======================================= Using Spack for CMake-based Development -========================================== +======================================= These are instructions on how to use Spack to aid in the development of a CMake-based project. Spack is used to help find the dependencies @@ -7,24 +8,26 @@ for the project, configure it at development time, and then package it it in a way that others can install. Using Spack for CMake-based development consists of three parts: -1. Setting up the CMake build in your software -2. Writing the Spack Package -3. Using it from Spack. - +#. Setting up the CMake build in your software +#. Writing the Spack Package +#. Using it from Spack. +-------------------------- Setting Up the CMake Build ---------------------------------------- +-------------------------- You should follow standard CMake conventions in setting up your software, your CMake build should NOT depend on or require Spack to build. See here for an example: - https://github.com/citibeth/icebin + +https://github.com/citibeth/icebin Note that there's one exception here to the rule I mentioned above. -In ``CMakeLists.txt``, I have the following line:: +In ``CMakeLists.txt``, I have the following line: - include_directories($ENV{CMAKE_TRANSITIVE_INCLUDE_PATH}) +.. code-block:: none + include_directories($ENV{CMAKE_TRANSITIVE_INCLUDE_PATH}) This is a hook into Spack, and it ensures that all transitive dependencies are included in the include path. It's not needed if @@ -48,40 +51,44 @@ correctly. Not only is this a good idea and nice, but it also ensures that your package will build the same with or without ``spack install``. +------------------------- Writing the Spack Package ---------------------------------------- +------------------------- Now that you have a CMake build, you want to tell Spack how to configure it. This is done by writing a Spack package for your software. See here for example: - https://github.com/citibeth/spack/blob/efischer/develop/var/spack/repos/builtin/packages/icebin/package.py + +https://github.com/citibeth/spack/blob/efischer/develop/var/spack/repos/builtin/packages/icebin/package.py You need to subclass ``CMakePackage``, as is done in this example. This enables advanced features of Spack for helping you in configuring your software (keep reading...). Instead of an ``install()`` method used when subclassing ``Package``, you write ``configure_args()``. See here for more info on how this works: - https://github.com/LLNL/spack/pull/543/files + +https://github.com/LLNL/spack/pull/543/files NOTE: if your software is not publicly available, you do not need to set the URL or version. Or you can set up bogus URLs and versions... whatever causes Spack to not crash. - +------------------- Using it from Spack --------------------------------- +------------------- Now that you have a Spack package, you can get Spack to setup your CMake project for you. Use the following to setup, configure and -build your project:: +build your project: - cd myproject - spack spconfig myproject@local - mkdir build; cd build - ../spconfig.py .. - make - make install +.. code-block:: console + $ cd myproject + $ spack spconfig myproject@local + $ mkdir build; cd build + $ ../spconfig.py .. + $ make + $ make install Everything here should look pretty familiar here from a CMake perspective, except that ``spack spconfig`` creates the file @@ -94,68 +101,75 @@ If your project is publicly available (eg on GitHub), then you can ALSO use this setup to "just install" a release version without going through the manual configuration/build step. Just do: -1. Put tag(s) on the version(s) in your GitHub repo you want to be release versions. +#. Put tag(s) on the version(s) in your GitHub repo you want to be release versions. -2. Set the ``url`` in your ``package.py`` to download a tarball for +#. Set the ``url`` in your ``package.py`` to download a tarball for the appropriate version. (GitHub will give you a tarball for any - version in the repo, if you tickle it the right way). For example:: + version in the repo, if you tickle it the right way). For example: - https://github.com/citibeth/icebin/tarball/v0.1.0 + https://github.com/citibeth/icebin/tarball/v0.1.0 Set up versions as appropriate in your ``package.py``. (Manually download the tarball and run ``md5sum`` to determine the appropriate checksum for it). -3. Now you should be able to say ``spack install myproject@version`` +#. Now you should be able to say ``spack install myproject@version`` and things "just work." NOTE... in order to use the features outlined in this post, you currently need to use the following branch of Spack: - https://github.com/citibeth/spack/tree/efischer/develop + +https://github.com/citibeth/spack/tree/efischer/develop There is a pull request open on this branch ( https://github.com/LLNL/spack/pull/543 ) and we are working to get it integrated into the main ``develop`` branch. - +------------------------ Activating your Software -------------------------------------- +------------------------ Once you've built your software, you will want to load it up. You can use ``spack load mypackage@local`` for that in your ``.bashrc``, but that is slow. Try stuff like the following instead: The following command will load the Spack-installed packages needed -for basic Python use of IceBin:: +for basic Python use of IceBin: - module load `spack module find tcl icebin netcdf cmake@3.5.1` - module load `spack module find --dependencies tcl py-basemap py-giss` +.. code-block:: console + + $ module load `spack module find tcl icebin netcdf cmake@3.5.1` + $ module load `spack module find --dependencies tcl py-basemap py-giss` You can speed up shell startup by turning these into ``module load`` commands. -1. Cut-n-paste the script ``make_spackenv``:: +#. Cut-n-paste the script ``make_spackenv``: + + .. code-block:: sh - #!/bin/sh - # - # Generate commands to load the Spack environment + #!/bin/sh + # + # Generate commands to load the Spack environment - SPACKENV=$HOME/spackenv.sh + SPACKENV=$HOME/spackenv.sh - spack module find --shell tcl git icebin@local ibmisc netcdf cmake@3.5.1 >$SPACKENV - spack module find --dependencies --shell tcl py-basemap py-giss >>$SPACKENV + spack module find --shell tcl git icebin@local ibmisc netcdf cmake@3.5.1 > $SPACKENV + spack module find --dependencies --shell tcl py-basemap py-giss >> $SPACKENV -2. Add the following to your ``.bashrc`` file:: +#. Add the following to your ``.bashrc`` file: - source $HOME/spackenv.sh - # Preferentially use your checked-out Python source - export PYTHONPATH=$HOME/icebin/pylib:$PYTHONPATH + .. code-block:: sh -3. Run ``sh make_spackenv`` whenever your Spack installation changes (including right now). + source $HOME/spackenv.sh + # Preferentially use your checked-out Python source + export PYTHONPATH=$HOME/icebin/pylib:$PYTHONPATH +#. Run ``sh make_spackenv`` whenever your Spack installation changes (including right now). +----------- Giving Back -------------------- +----------- If your software is publicly available, you should submit the ``package.py`` for it as a pull request to the main Spack GitHub @@ -164,4 +178,4 @@ project. This will ensure that anyone can install your software for how that has turned into detailed instructions that have successfully enabled collaborators to install complex software: - https://github.com/citibeth/icebin/blob/develop/README.rst +https://github.com/citibeth/icebin/blob/develop/README.rst diff --git a/lib/spack/docs/conf.py b/lib/spack/docs/conf.py index 1416074356..95549e3b1e 100644 --- a/lib/spack/docs/conf.py +++ b/lib/spack/docs/conf.py @@ -1,3 +1,4 @@ +# flake8: noqa ############################################################################## # Copyright (c) 2013, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory. @@ -47,11 +48,12 @@ from glob import glob # documentation root, use os.path.abspath to make it absolute, like shown here. sys.path.insert(0, os.path.abspath('exts')) sys.path.insert(0, os.path.abspath('../external')) +sys.path.append(os.path.abspath('../spack')) # Add the Spack bin directory to the path so that we can use its output in docs. spack_root = '../../..' os.environ['SPACK_ROOT'] = spack_root -os.environ['PATH'] += os.pathsep + '$SPACK_ROOT/bin' +os.environ['PATH'] += '%s%s/bin' % (os.pathsep, spack_root) # Get the spack version for use in the docs spack_version = subprocess.Popen( @@ -82,6 +84,27 @@ with open('command_index.rst', 'a') as index: for cmd in command_names: index.write(' * :ref:`%s`\n' % cmd) +# +# Exclude everything in spack.__all__ from indexing. All of these +# symbols are imported from elsewhere in spack; their inclusion in +# __all__ simply allows package authors to use `from spack import *`. +# Excluding them ensures they're only documented in their "real" module. +# +# This also avoids issues where some of these symbols shadow core spack +# modules. Sphinx will complain about duplicate docs when this happens. +# +import fileinput, spack +handling_spack = False +for line in fileinput.input('spack.rst', inplace=1): + if handling_spack: + if not line.startswith(' :noindex:'): + print ' :noindex: %s' % ' '.join(spack.__all__) + handling_spack = False + + if line.startswith('.. automodule::'): + handling_spack = (line == '.. automodule:: spack\n') + + print line, # Set an environment variable so that colify will print output like it would to # a terminal. diff --git a/lib/spack/docs/configuration.rst b/lib/spack/docs/configuration.rst index d39c932021..ba534d1e62 100644 --- a/lib/spack/docs/configuration.rst +++ b/lib/spack/docs/configuration.rst @@ -1,14 +1,18 @@ .. _configuration: +============= Configuration -=================================== +============= .. _temp-space: +--------------- Temporary space ----------------------------- +--------------- -.. warning:: Temporary space configuration will eventually be moved to +.. warning:: + + Temporary space configuration will eventually be moved to configuration files, but currently these settings are in ``lib/spack/spack/__init__.py`` @@ -55,8 +59,10 @@ directory is. .. _sec-external_packages: +----------------- External Packages ----------------------------- +----------------- + Spack can be configured to use externally-installed packages rather than building its own packages. This may be desirable if machines ship with system packages, such as a customized MPI @@ -69,11 +75,11 @@ directory. Here's an example of an external configuration: .. code-block:: yaml packages: - openmpi: - paths: - openmpi@1.4.3%gcc@4.4.7 arch=linux-x86_64-debian7: /opt/openmpi-1.4.3 - openmpi@1.4.3%gcc@4.4.7 arch=linux-x86_64-debian7+debug: /opt/openmpi-1.4.3-debug - openmpi@1.6.5%intel@10.1 arch=linux-x86_64-debian7: /opt/openmpi-1.6.5-intel + openmpi: + paths: + openmpi@1.4.3%gcc@4.4.7 arch=linux-x86_64-debian7: /opt/openmpi-1.4.3 + openmpi@1.4.3%gcc@4.4.7 arch=linux-x86_64-debian7+debug: /opt/openmpi-1.4.3-debug + openmpi@1.6.5%intel@10.1 arch=linux-x86_64-debian7: /opt/openmpi-1.6.5-intel This example lists three installations of OpenMPI, one built with gcc, one built with gcc and debug information, and another built with Intel. @@ -106,13 +112,13 @@ be: .. code-block:: yaml - packages: - openmpi: - paths: - openmpi@1.4.3%gcc@4.4.7 arch=linux-x86_64-debian7: /opt/openmpi-1.4.3 - openmpi@1.4.3%gcc@4.4.7 arch=linux-x86_64-debian7+debug: /opt/openmpi-1.4.3-debug - openmpi@1.6.5%intel@10.1 arch=linux-x86_64-debian7: /opt/openmpi-1.6.5-intel - buildable: False + packages: + openmpi: + paths: + openmpi@1.4.3%gcc@4.4.7 arch=linux-x86_64-debian7: /opt/openmpi-1.4.3 + openmpi@1.4.3%gcc@4.4.7 arch=linux-x86_64-debian7+debug: /opt/openmpi-1.4.3-debug + openmpi@1.6.5%intel@10.1 arch=linux-x86_64-debian7: /opt/openmpi-1.6.5-intel + buildable: False The addition of the ``buildable`` flag tells Spack that it should never build its own version of OpenMPI, and it will instead always rely on a pre-built @@ -126,9 +132,11 @@ The ``buildable`` does not need to be paired with external packages. It could also be used alone to forbid packages that may be buggy or otherwise undesirable. +.. _concretization-preferences: +-------------------------- Concretization Preferences --------------------------------- +-------------------------- Spack can be configured to prefer certain compilers, package versions, depends_on, and variants during concretization. @@ -136,22 +144,20 @@ The preferred configuration can be controlled via the ``~/.spack/packages.yaml`` file for user configuations, or the ``etc/spack/packages.yaml`` site configuration. - Here's an example packages.yaml file that sets preferred packages: -.. code-block:: sh - - packages: - opencv: - compiler: [gcc@4.9] - variants: +debug - gperftools: - version: [2.2, 2.4, 2.3] - all: - compiler: [gcc@4.4.7, gcc@4.6:, intel, clang, pgi] - providers: - mpi: [mvapich, mpich, openmpi] +.. code-block:: yaml + packages: + opencv: + compiler: [gcc@4.9] + variants: +debug + gperftools: + version: [2.2, 2.4, 2.3] + all: + compiler: [gcc@4.4.7, gcc@4.6:, intel, clang, pgi] + providers: + mpi: [mvapich, mpich, openmpi] At a high level, this example is specifying how packages should be concretized. The opencv package should prefer using gcc 4.9 and @@ -185,9 +191,9 @@ concretization rules. A provider lists a value that packages may ``depend_on`` (e.g, mpi) and a list of rules for fulfilling that dependency. - +--------- Profiling ------------------- +--------- Spack has some limited built-in support for profiling, and can report statistics using standard Python timing tools. To use this feature, @@ -195,40 +201,14 @@ supply ``-p`` to Spack on the command line, before any subcommands. .. _spack-p: -``spack -p`` -~~~~~~~~~~~~~~~~~ - -``spack -p`` output looks like this: - -.. code-block:: sh - - $ spack -p graph dyninst - o dyninst - |\ - | |\ - | o | libdwarf - |/ / - o | libelf - / - o boost - - 307670 function calls (305943 primitive calls) in 0.127 seconds - - Ordered by: internal time - - ncalls tottime percall cumtime percall filename:lineno(function) - 853 0.021 0.000 0.066 0.000 inspect.py:472(getmodule) - 51197 0.011 0.000 0.018 0.000 inspect.py:51(ismodule) - 73961 0.010 0.000 0.010 0.000 {isinstance} - 1762 0.006 0.000 0.053 0.000 inspect.py:440(getsourcefile) - 32075 0.006 0.000 0.006 0.000 {hasattr} - 1760 0.004 0.000 0.004 0.000 {posix.stat} - 2240 0.004 0.000 0.004 0.000 {posix.lstat} - 2602 0.004 0.000 0.011 0.000 inspect.py:398(getfile) - 771 0.004 0.000 0.077 0.000 inspect.py:518(findsource) - 2656 0.004 0.000 0.004 0.000 {method 'match' of '_sre.SRE_Pattern' objects} - 30772 0.003 0.000 0.003 0.000 {method 'get' of 'dict' objects} - ... +^^^^^^^^^^^^^^^^^^^ +``spack --profile`` +^^^^^^^^^^^^^^^^^^^ + +``spack --profile`` output looks like this: + +.. command-output:: spack --profile graph dyninst + :ellipsis: 25 The bottom of the output shows the top most time consuming functions, slowest on top. The profiling support is from Python's built-in tool, diff --git a/lib/spack/docs/developer_guide.rst b/lib/spack/docs/developer_guide.rst index d28fe4b2a5..04ae8fe1a1 100644 --- a/lib/spack/docs/developer_guide.rst +++ b/lib/spack/docs/developer_guide.rst @@ -1,7 +1,8 @@ .. _developer_guide: +=============== Developer Guide -===================== +=============== This guide is intended for people who want to work on Spack itself. If you just want to develop packages, see the :ref:`packaging-guide`. @@ -11,17 +12,18 @@ It is assumed that you've read the :ref:`basic-usage` and concepts discussed there. If you're not, we recommend reading those first. +-------- Overview ------------------------ +-------- Spack is designed with three separate roles in mind: - #. **Users**, who need to install software *without* knowing all the - details about how it is built. - #. **Packagers** who know how a particular software package is - built and encode this information in package files. - #. **Developers** who work on Spack, add new features, and try to - make the jobs of packagers and users easier. +#. **Users**, who need to install software *without* knowing all the + details about how it is built. +#. **Packagers** who know how a particular software package is + built and encode this information in package files. +#. **Developers** who work on Spack, add new features, and try to + make the jobs of packagers and users easier. Users could be end users installing software in their home directory, or administrators installing software to a shared directory on a @@ -41,9 +43,9 @@ specification. This gets us to the two key concepts in Spack's software design: - #. **Specs**: expressions for describing builds of software, and - #. **Packages**: Python modules that build software according to a - spec. +#. **Specs**: expressions for describing builds of software, and +#. **Packages**: Python modules that build software according to a + spec. A package is a template for building particular software, and a spec as a descriptor for one or more instances of that template. Users @@ -63,75 +65,75 @@ building the software off to the package object. The rest of this document describes all the pieces that come together to make that happen. - +------------------- Directory Structure -------------------------- +------------------- So that you can familiarize yourself with the project, we'll start -with a high level view of Spack's directory structure:: +with a high level view of Spack's directory structure: + +.. code-block:: none - spack/ <- installation root - bin/ - spack <- main spack executable + spack/ <- installation root + bin/ + spack <- main spack executable - etc/ - spack/ <- Spack config files. - Can be overridden by files in ~/.spack. + etc/ + spack/ <- Spack config files. + Can be overridden by files in ~/.spack. - var/ - spack/ <- build & stage directories - repos/ <- contains package repositories - builtin/ <- pkg repository that comes with Spack - repo.yaml <- descriptor for the builtin repository - packages/ <- directories under here contain packages - cache/ <- saves resources downloaded during installs + var/ + spack/ <- build & stage directories + repos/ <- contains package repositories + builtin/ <- pkg repository that comes with Spack + repo.yaml <- descriptor for the builtin repository + packages/ <- directories under here contain packages + cache/ <- saves resources downloaded during installs - opt/ - spack/ <- packages are installed here + opt/ + spack/ <- packages are installed here - lib/ - spack/ - docs/ <- source for this documentation - env/ <- compiler wrappers for build environment + lib/ + spack/ + docs/ <- source for this documentation + env/ <- compiler wrappers for build environment - external/ <- external libs included in Spack distro - llnl/ <- some general-use libraries + external/ <- external libs included in Spack distro + llnl/ <- some general-use libraries - spack/ <- spack module; contains Python code - cmd/ <- each file in here is a spack subcommand - compilers/ <- compiler description files - test/ <- unit test modules - util/ <- common code + spack/ <- spack module; contains Python code + cmd/ <- each file in here is a spack subcommand + compilers/ <- compiler description files + test/ <- unit test modules + util/ <- common code Spack is designed so that it could live within a `standard UNIX directory hierarchy <http://linux.die.net/man/7/hier>`_, so ``lib``, ``var``, and ``opt`` all contain a ``spack`` subdirectory in case Spack is installed alongside other software. Most of the interesting -parts of Spack live in ``lib/spack``. Files under ``var`` are created -as needed, so there is no ``var`` directory when you initially clone -Spack from the repository. +parts of Spack live in ``lib/spack``. Spack has *one* directory layout and there is no install process. -version and the source code. Most Python programs don't look like -this (they use distutils, ``setup.py``, etc.) but we wanted to make -Spack *very* easy to use. The simple layout spares users from the -need to install Spack into a Python environment. Many users don't -have write access to a Python installation, and installing an entire -new instance of Python to bootstrap Spack would be very complicated. +Most Python programs don't look like this (they use distutils, ``setup.py``, +etc.) but we wanted to make Spack *very* easy to use. The simple layout +spares users from the need to install Spack into a Python environment. +Many users don't have write access to a Python installation, and installing +an entire new instance of Python to bootstrap Spack would be very complicated. Users should not have to install install a big, complicated package to use the thing that's supposed to spare them from the details of big, complicated packages. The end result is that Spack works out of the box: clone it and add ``bin`` to your PATH and you're ready to go. - +-------------- Code Structure -------------------------- +-------------- This section gives an overview of the various Python modules in Spack, grouped by functionality. +^^^^^^^^^^^^^^^^^^^^^^^ Package-related modules -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^ :mod:`spack.package` Contains the :class:`Package <spack.package.Package>` class, which @@ -158,9 +160,9 @@ Package-related modules decorator, which allows :ref:`multimethods <multimethods>` in packages. - +^^^^^^^^^^^^^^^^^^^^ Spec-related modules -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^ :mod:`spack.spec` Contains :class:`Spec <spack.spec.Spec>` and :class:`SpecParser @@ -208,9 +210,9 @@ Spec-related modules Not yet implemented. Should eventually have architecture descriptions for cross-compiling. - +^^^^^^^^^^^^^^^^^ Build environment -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^ :mod:`spack.stage` Handles creating temporary directories for builds. @@ -224,15 +226,17 @@ Build environment Create more implementations of this to change the hierarchy and naming scheme in ``$spack_prefix/opt`` +^^^^^^^^^^^^^^^^^ Spack Subcommands -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^ :mod:`spack.cmd` Each module in this package implements a Spack subcommand. See :ref:`writing commands <writing-commands>` for details. +^^^^^^^^^^ Unit tests -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^ :mod:`spack.test` Implements Spack's test suite. Add a module and put its name in @@ -242,8 +246,9 @@ Unit tests This is a fake package hierarchy used to mock up packages for Spack's test suite. +^^^^^^^^^^^^^ Other Modules -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^ :mod:`spack.globals` Includes global settings for Spack. the default policy classes for @@ -269,51 +274,53 @@ Other Modules :class:`SpackError <spack.error.SpackError>`, the base class for Spack's exception hierarchy. - +------------ Spec objects -------------------------- +------------ +--------------- Package objects -------------------------- - - -Most spack commands -look something like this: - - #. Parse an abstract spec (or specs) from the command line, - #. *Normalize* the spec based on information in package files, - #. *Concretize* the spec according to some customizable policies, - #. Instantiate a package based on the spec, and - #. Call methods (e.g., ``install()``) on the package object. +--------------- +Most spack commands look something like this: +#. Parse an abstract spec (or specs) from the command line, +#. *Normalize* the spec based on information in package files, +#. *Concretize* the spec according to some customizable policies, +#. Instantiate a package based on the spec, and +#. Call methods (e.g., ``install()``) on the package object. The information in Package files is used at all stages in this process. +Conceptually, packages are overloaded. They contain: -Conceptually, packages are overloaded. They contain - +------------- Stage objects -------------------------- +------------- .. _writing-commands: +---------------- Writing commands -------------------------- +---------------- +---------- Unit tests -------------------------- +---------- +------------ Unit testing -------------------------- - +------------ +------------------ Developer commands -------------------------- +------------------ +^^^^^^^^^^^^^ ``spack doc`` -~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^ +^^^^^^^^^^^^^^ ``spack test`` -~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^ diff --git a/lib/spack/docs/features.rst b/lib/spack/docs/features.rst index 27a3b4b435..57629997aa 100644 --- a/lib/spack/docs/features.rst +++ b/lib/spack/docs/features.rst @@ -1,29 +1,32 @@ +================ Feature overview -================== +================ This is a high-level overview of features that make Spack different from other `package managers <http://en.wikipedia.org/wiki/Package_management_system>`_ and `port systems <http://en.wikipedia.org/wiki/Ports_collection>`_. +--------------------------- Simple package installation ----------------------------- +--------------------------- Installing the default version of a package is simple. This will install the latest version of the ``mpileaks`` package and all of its dependencies: -.. code-block:: sh +.. code-block:: console $ spack install mpileaks +-------------------------------- Custom versions & configurations -------------------------------------------- +-------------------------------- Spack allows installation to be customized. Users can specify the version, build compiler, compile-time options, and cross-compile platform, all on the command line. -.. code-block:: sh +.. code-block:: console # Install a particular version by appending @ $ spack install mpileaks@1.1.2 @@ -47,37 +50,39 @@ Users can specify as many or few options as they care about. Spack will fill in the unspecified values with sensible defaults. The two listed syntaxes for variants are identical when the value is boolean. - +---------------------- Customize dependencies -------------------------------------- +---------------------- Spack allows *dependencies* of a particular installation to be customized extensively. Suppose that ``mpileaks`` depends indirectly on ``libelf`` and ``libdwarf``. Using ``^``, users can add custom configurations for the dependencies: -.. code-block:: sh +.. code-block:: console # Install mpileaks and link it with specific versions of libelf and libdwarf $ spack install mpileaks@1.1.2 %gcc@4.7.3 +debug ^libelf@0.8.12 ^libdwarf@20130729+debug - +------------------------ Non-destructive installs -------------------------------------- +------------------------ Spack installs every unique package/dependency configuration into its own prefix, so new installs will not break existing ones. +------------------------------- Packages can peacefully coexist -------------------------------------- +------------------------------- Spack avoids library misconfiguration by using ``RPATH`` to link dependencies. When a user links a library or runs a program, it is tied to the dependencies it was built with, so there is no need to manipulate ``LD_LIBRARY_PATH`` at runtime. +------------------------- Creating packages is easy -------------------------------------- +------------------------- To create a new packages, all Spack needs is a URL for the source archive. The ``spack create`` command will create a boilerplate @@ -86,7 +91,7 @@ in pure Python. For example, this command: -.. code-block:: sh +.. code-block:: console $ spack create http://www.mr511.de/software/libelf-0.8.13.tar.gz @@ -96,16 +101,26 @@ creates a simple python file: from spack import * + class Libelf(Package): - homepage = "http://www.example.com/" + """FIXME: Put a proper description of your package here.""" + + # FIXME: Add a proper url for your package's homepage here. + homepage = "http://www.example.com" url = "http://www.mr511.de/software/libelf-0.8.13.tar.gz" version('0.8.13', '4136d7b4c04df68b686570afa26988ac') - def install(self, prefix): - configure("--prefix=%s" % prefix) + # FIXME: Add dependencies if required. + # depends_on('foo') + + def install(self, spec, prefix): + # FIXME: Modify the configure line to suit your build system here. + configure('--prefix={0}'.format(prefix)) + + # FIXME: Add logic to build and install here. make() - make("install") + make('install') It doesn't take much python coding to get from there to a working package: diff --git a/lib/spack/docs/getting_started.rst b/lib/spack/docs/getting_started.rst index 2c5b68ea65..676697a549 100644 --- a/lib/spack/docs/getting_started.rst +++ b/lib/spack/docs/getting_started.rst @@ -1,13 +1,15 @@ +=============== Getting Started -==================== +=============== +-------- Download --------------------- +-------- Getting spack is easy. You can clone it from the `github repository <https://github.com/llnl/spack>`_ using this command: -.. code-block:: sh +.. code-block:: console $ git clone https://github.com/llnl/spack.git @@ -16,7 +18,7 @@ full path to this directory is in the ``SPACK_ROOT`` environment variable. Add ``$SPACK_ROOT/bin`` to your path and you're ready to go: -.. code-block:: sh +.. code-block:: console $ export PATH=$SPACK_ROOT/bin:$PATH $ spack install libelf @@ -24,9 +26,10 @@ go: For a richer experience, use Spack's `shell support <http://software.llnl.gov/spack/basic_usage.html#environment-modules>`_: -.. code-block:: sh +.. code-block:: console # For bash users + $ export SPACK_ROOT=/path/to/spack $ . $SPACK_ROOT/share/spack/setup-env.sh # For tcsh or csh users (note you must set SPACK_ROOT) @@ -35,8 +38,9 @@ For a richer experience, use Spack's `shell support This automatically adds Spack to your ``PATH``. +------------ Installation --------------------- +------------ You don't need to install Spack; it's ready to run as soon as you clone it from git. @@ -45,9 +49,9 @@ You may want to run it out of a prefix other than the git repository you cloned. The ``spack bootstrap`` command provides this functionality. To install spack in a new directory, simply type: -.. code-block:: sh +.. code-block:: console - $ spack bootstrap /my/favorite/prefix + $ spack bootstrap /my/favorite/prefix This will install a new spack script in ``/my/favorite/prefix/bin``, which you can use just like you would the regular spack script. Each diff --git a/lib/spack/docs/index.rst b/lib/spack/docs/index.rst index a5bbd4e23b..45efcf131f 100644 --- a/lib/spack/docs/index.rst +++ b/lib/spack/docs/index.rst @@ -3,8 +3,9 @@ You can adapt this file completely to your liking, but it should at least contain the root `toctree` directive. +=================== Spack Documentation -================================= +=================== Spack is a package management tool designed to support multiple versions and configurations of software on a wide variety of platforms @@ -27,7 +28,7 @@ Get spack from the `github repository <https://github.com/llnl/spack>`_ and install your first package: -.. code-block:: sh +.. code-block:: console $ git clone https://github.com/llnl/spack.git $ cd spack/bin @@ -36,8 +37,9 @@ package: If you're new to spack and want to start using it, see :doc:`getting_started`, or refer to the full manual below. +----------------- Table of Contents ---------------------- +----------------- .. toctree:: :maxdepth: 2 @@ -54,6 +56,7 @@ Table of Contents package_list API Docs <spack> +================== Indices and tables ================== diff --git a/lib/spack/docs/mirrors.rst b/lib/spack/docs/mirrors.rst index 583575a565..c4d275a8fc 100644 --- a/lib/spack/docs/mirrors.rst +++ b/lib/spack/docs/mirrors.rst @@ -1,7 +1,8 @@ .. _mirrors: +======= Mirrors -============================ +======= Some sites may not have access to the internet for fetching packages. These sites will need a local repository of tarballs from which they @@ -10,27 +11,29 @@ mirror is a URL that points to a directory, either on the local filesystem or on some server, containing tarballs for all of Spack's packages. -Here's an example of a mirror's directory structure:: - - mirror/ - cmake/ - cmake-2.8.10.2.tar.gz - dyninst/ - dyninst-8.1.1.tgz - dyninst-8.1.2.tgz - libdwarf/ - libdwarf-20130126.tar.gz - libdwarf-20130207.tar.gz - libdwarf-20130729.tar.gz - libelf/ - libelf-0.8.12.tar.gz - libelf-0.8.13.tar.gz - libunwind/ - libunwind-1.1.tar.gz - mpich/ - mpich-3.0.4.tar.gz - mvapich2/ - mvapich2-1.9.tgz +Here's an example of a mirror's directory structure: + +.. code-block:: none + + mirror/ + cmake/ + cmake-2.8.10.2.tar.gz + dyninst/ + dyninst-8.1.1.tgz + dyninst-8.1.2.tgz + libdwarf/ + libdwarf-20130126.tar.gz + libdwarf-20130207.tar.gz + libdwarf-20130729.tar.gz + libelf/ + libelf-0.8.12.tar.gz + libelf-0.8.13.tar.gz + libunwind/ + libunwind-1.1.tar.gz + mpich/ + mpich-3.0.4.tar.gz + mvapich2/ + mvapich2-1.9.tgz The structure is very simple. There is a top-level directory. The second level directories are named after packages, and the third level @@ -51,25 +54,14 @@ contains tarballs for each package, named after each package. .. _spack-mirror: +---------------- ``spack mirror`` ----------------------------- +---------------- Mirrors are managed with the ``spack mirror`` command. The help for -``spack mirror`` looks like this:: +``spack mirror`` looks like this: - $ spack mirror -h - usage: spack mirror [-h] SUBCOMMAND ... - - positional arguments: - SUBCOMMAND - create Create a directory to be used as a spack mirror, and fill - it with package archives. - add Add a mirror to Spack. - remove Remove a mirror by name. - list Print out available mirrors to the console. - - optional arguments: - -h, --help show this help message and exit +.. command-output:: spack help mirror The ``create`` command actually builds a mirror by fetching all of its packages from the internet and checksumming them. @@ -79,8 +71,9 @@ control the URL(s) from which Spack downloads its packages. .. _spack-mirror-create: +----------------------- ``spack mirror create`` ----------------------------- +----------------------- You can create a mirror using the ``spack mirror create`` command, assuming you're on a machine where you can access the internet. @@ -89,8 +82,7 @@ The command will iterate through all of Spack's packages and download the safe ones into a directory structure like the one above. Here is what it looks like: - -.. code-block:: bash +.. code-block:: console $ spack mirror create libelf libdwarf ==> Created new mirror in spack-mirror-2014-06-24 @@ -124,25 +116,31 @@ what it looks like: Once this is done, you can tar up the ``spack-mirror-2014-06-24`` directory and copy it over to the machine you want it hosted on. +^^^^^^^^^^^^^^^^^^^ Custom package sets -~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^ Normally, ``spack mirror create`` downloads all the archives it has checksums for. If you want to only create a mirror for a subset of packages, you can do that by supplying a list of package specs on the command line after ``spack mirror create``. For example, this -command:: +command: - $ spack mirror create libelf@0.8.12: boost@1.44: +.. code-block:: console + + $ spack mirror create libelf@0.8.12: boost@1.44: Will create a mirror for libelf versions greater than or equal to 0.8.12 and boost versions greater than or equal to 1.44. +^^^^^^^^^^^^ Mirror files -~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^ If you have a *very* large number of packages you want to mirror, you -can supply a file with specs in it, one per line:: +can supply a file with specs in it, one per line: + +.. code-block:: console $ cat specs.txt libdwarf @@ -158,24 +156,27 @@ your site. .. _spack-mirror-add: +-------------------- ``spack mirror add`` ----------------------------- +-------------------- Once you have a mirror, you need to let spack know about it. This is relatively simple. First, figure out the URL for the mirror. If it's -a file, you can use a file URL like this one:: +a file, you can use a file URL like this one: - file:///Users/gamblin2/spack-mirror-2014-06-24 +.. code-block:: none + + file:///Users/gamblin2/spack-mirror-2014-06-24 That points to the directory on the local filesystem. If it were on a web server, you could use a URL like this one: - https://example.com/some/web-hosted/directory/spack-mirror-2014-06-24 +https://example.com/some/web-hosted/directory/spack-mirror-2014-06-24 Spack will use the URL as the root for all of the packages it fetches. You can tell your Spack installation to use that mirror like this: -.. code-block:: bash +.. code-block:: console $ spack mirror add local_filesystem file:///Users/gamblin2/spack-mirror-2014-06-24 @@ -183,29 +184,38 @@ Each mirror has a name so that you can refer to it again later. .. _spack-mirror-list: +--------------------- ``spack mirror list`` ----------------------------- +--------------------- + +To see all the mirrors Spack knows about, run ``spack mirror list``: -To see all the mirrors Spack knows about, run ``spack mirror list``:: +.. code-block:: console $ spack mirror list local_filesystem file:///Users/gamblin2/spack-mirror-2014-06-24 .. _spack-mirror-remove: +----------------------- ``spack mirror remove`` ----------------------------- +----------------------- -To remove a mirror by name:: +To remove a mirror by name, run: + +.. code-block:: console $ spack mirror remove local_filesystem $ spack mirror list ==> No mirrors configured. +----------------- Mirror precedence ----------------------------- +----------------- + +Adding a mirror really adds a line in ``~/.spack/mirrors.yaml``: -Adding a mirror really adds a line in ``~/.spack/mirrors.yaml``:: +.. code-block:: yaml mirrors: local_filesystem: file:///Users/gamblin2/spack-mirror-2014-06-24 @@ -217,18 +227,19 @@ search the topmost mirror first and the bottom-most mirror last. .. _caching: +------------------- Local Default Cache ----------------------------- +------------------- Spack caches resources that are downloaded as part of installs. The cache is a valid spack mirror: it uses the same directory structure and naming scheme as other Spack mirrors (so it can be copied anywhere and referenced with a URL -like other mirrors). The mirror is maintained locally (within the Spack -installation directory) at :file:`var/spack/cache/`. It is always enabled (and -is always searched first when attempting to retrieve files for an installation) +like other mirrors). The mirror is maintained locally (within the Spack +installation directory) at :file:`var/spack/cache/`. It is always enabled (and +is always searched first when attempting to retrieve files for an installation) but can be cleared with :ref:`purge <spack-purge>`; the cache directory can also -be deleted manually without issue. +be deleted manually without issue. Caching includes retrieved tarball archives and source control repositories, but -only resources with an associated digest or commit ID (e.g. a revision number +only resources with an associated digest or commit ID (e.g. a revision number for SVN) will be cached. diff --git a/lib/spack/docs/packaging_guide.rst b/lib/spack/docs/packaging_guide.rst index 34fcb1e101..08d39a266a 100644 --- a/lib/spack/docs/packaging_guide.rst +++ b/lib/spack/docs/packaging_guide.rst @@ -1,7 +1,8 @@ .. _packaging-guide: +=============== Packaging Guide -===================== +=============== This guide is intended for developers or administrators who want to package software so that Spack can install it. It assumes that you @@ -11,9 +12,9 @@ have at least some familiarity with Python, and that you've read the There are two key parts of Spack: - #. **Specs**: expressions for describing builds of software, and - #. **Packages**: Python modules that describe how to build - software according to a spec. +#. **Specs**: expressions for describing builds of software, and +#. **Packages**: Python modules that describe how to build + software according to a spec. Specs allow a user to describe a *particular* build in a way that a package author can understand. Packages allow a the packager to @@ -28,13 +29,15 @@ ubiquitous in the scientific software community. Second, it's a modern language and has many powerful features to help make package writing easy. +--------------------------- Creating & editing packages ----------------------------------- +--------------------------- .. _spack-create: +^^^^^^^^^^^^^^^^ ``spack create`` -~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^ The ``spack create`` command creates a directory with the package name and generates a ``package.py`` file with a boilerplate package template from a URL. @@ -44,7 +47,7 @@ working. Here's an example: -.. code-block:: sh +.. code-block:: console $ spack create http://www.cmake.org/files/v2.8/cmake-2.8.12.1.tar.gz @@ -59,7 +62,7 @@ strings look like for this package. Using this information, it will try to find versions, Spack prompts you to tell it how many versions you want to download and checksum: -.. code-block:: sh +.. code-block:: console $ spack create http://www.cmake.org/files/v2.8/cmake-2.8.12.1.tar.gz ==> This looks like a URL for cmake version 2.8.12.1. @@ -83,7 +86,9 @@ always choose to download just one tarball initially, and run .. note:: If ``spack create`` fails to detect the package name correctly, - you can try supplying it yourself, e.g.:: + you can try supplying it yourself, e.g.: + + .. code-block:: console $ spack create --name cmake http://www.cmake.org/files/v2.8/cmake-2.8.12.1.tar.gz @@ -101,7 +106,7 @@ always choose to download just one tarball initially, and run Let's say you download 3 tarballs: -.. code-block:: sh +.. code-block:: none Include how many checksums in the package file? (default is 5, q to abort) 3 ==> Downloading... @@ -118,27 +123,28 @@ file in your favorite ``$EDITOR``: .. code-block:: python :linenos: - # FIXME: - # This is a template package file for Spack. We've conveniently - # put "FIXME" labels next to all the things you'll want to change. # - # Once you've edited all the FIXME's, delete this whole message, - # save this file, and test out your package like this: + # This is a template package file for Spack. We've put "FIXME" + # next to all the things you'll want to change. Once you've handled + # them, you can save this file and test your package like this: # # spack install cmake # - # You can always get back here to change things with: + # You can edit this file again by typing: # # spack edit cmake # - # See the spack documentation for more information on building - # packages. + # See the Spack documentation for more information on packaging. + # If you submit this package back to Spack as a pull request, + # please first remove this boilerplate and all FIXME comments. # from spack import * + class Cmake(Package): - """FIXME: put a proper description of your package here.""" - # FIXME: add a proper url for your package's homepage here. + """FIXME: Put a proper description of your package here.""" + + # FIXME: Add a proper url for your package's homepage here. homepage = "http://www.example.com" url = "http://www.cmake.org/files/v2.8/cmake-2.8.12.1.tar.gz" @@ -162,79 +168,81 @@ done for you. In the generated package, the download ``url`` attribute is already set. All the things you still need to change are marked with -``FIXME`` labels. The first ``FIXME`` refers to the commented -instructions at the top of the file. You can delete these -instructions after reading them. The rest of them are as follows: +``FIXME`` labels. You can delete the commented instructions between +the license and the first import statement after reading them. +The rest of the tasks you need to do are as follows: - #. Add a description. +#. Add a description. - Immediately inside the package class is a *docstring* in - triple-quotes (``"""``). It's used to generate the description - shown when users run ``spack info``. + Immediately inside the package class is a *docstring* in + triple-quotes (``"""``). It's used to generate the description + shown when users run ``spack info``. - #. Change the ``homepage`` to a useful URL. +#. Change the ``homepage`` to a useful URL. - The ``homepage`` is displayed when users run ``spack info`` so - that they can learn about packages. + The ``homepage`` is displayed when users run ``spack info`` so + that they can learn about packages. - #. Add ``depends_on()`` calls for the package's dependencies. +#. Add ``depends_on()`` calls for the package's dependencies. - ``depends_on`` tells Spack that other packages need to be built - and installed before this one. See `dependencies_`. + ``depends_on`` tells Spack that other packages need to be built + and installed before this one. See :ref:`dependencies`. - #. Get the ``install()`` method working. +#. Get the ``install()`` method working. - The ``install()`` method implements the logic to build a - package. The code should look familiar; it is designed to look - like a shell script. Specifics will differ depending on the package, - and :ref:`implementing the install method <install-method>` is - covered in detail later. + The ``install()`` method implements the logic to build a + package. The code should look familiar; it is designed to look + like a shell script. Specifics will differ depending on the package, + and :ref:`implementing the install method <install-method>` is + covered in detail later. Before going into details, we'll cover a few more basics. .. _spack-edit: +^^^^^^^^^^^^^^ ``spack edit`` -~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^ One of the easiest ways to learn to write packages is to look at existing ones. You can edit a package file by name with the ``spack edit`` command: -.. code-block:: sh +.. code-block:: console - spack edit cmake + $ spack edit cmake So, if you used ``spack create`` to create a package, then saved and closed the resulting file, you can get back to it with ``spack edit``. The ``cmake`` package actually lives in -``$SPACK_ROOT/var/spack/repos/builtin/packages/cmake/package.py``, but this provides -a much simpler shortcut and saves you the trouble of typing the full -path. +``$SPACK_ROOT/var/spack/repos/builtin/packages/cmake/package.py``, +but this provides a much simpler shortcut and saves you the trouble +of typing the full path. If you try to edit a package that doesn't exist, Spack will recommend -using ``spack create`` or ``spack edit -f``: +using ``spack create`` or ``spack edit --force``: -.. code-block:: sh +.. code-block:: console $ spack edit foo ==> Error: No package 'foo'. Use spack create, or supply -f/--force to edit a new file. .. _spack-edit-f: -``spack edit -f`` -~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^ +``spack edit --force`` +^^^^^^^^^^^^^^^^^^^^^^ -``spack edit -f`` can be used to create a new, minimal boilerplate +``spack edit --force`` can be used to create a new, minimal boilerplate package: -.. code-block:: sh +.. code-block:: console $ spack edit -f foo Unlike ``spack create``, which infers names and versions, and which actually downloads the tarball and checksums it for you, ``spack edit --f`` has no such fanciness. It will substitute dummy values for you +--force`` has no such fanciness. It will substitute dummy values for you to fill in yourself: .. code-block:: python @@ -251,31 +259,25 @@ to fill in yourself: version('1.0', '0123456789abcdef0123456789abcdef') def install(self, spec, prefix): - configure("--prefix=" + prefix) + configure("--prefix=%s" % prefix) make() make("install") This is useful when ``spack create`` cannot figure out the name and version of your package from the archive URL. - +---------------------------- Naming & directory structure --------------------------------------- - -.. note:: - - Spack's default naming and directory structure will change in - version 0.9. Specifically, 0.9 will stop using directory names - with special characters like ``@``, to avoid interfering with - certain packages that do not handle this well. +---------------------------- This section describes how packages need to be named, and where they -live in Spack's directory structure. In general, `spack-create`_ and -`spack-edit`_ handle creating package files for you, so you can skip +live in Spack's directory structure. In general, :ref:`spack-create` and +:ref:`spack-edit` handle creating package files for you, so you can skip most of the details here. +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ``var/spack/repos/builtin/packages`` -~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ A Spack installation directory is structured like a standard UNIX install prefix (``bin``, ``lib``, ``include``, ``var``, ``opt``, @@ -285,28 +287,30 @@ Packages themselves live in ``$SPACK_ROOT/var/spack/repos/builtin/packages``. If you ``cd`` to that directory, you will see directories for each package: -.. command-output:: cd $SPACK_ROOT/var/spack/repos/builtin/packages; ls -CF +.. command-output:: cd $SPACK_ROOT/var/spack/repos/builtin/packages && ls :shell: :ellipsis: 10 Each directory contains a file called ``package.py``, which is where all the python code for the package goes. For example, the ``libelf`` -package lives in:: +package lives in: + +.. code-block:: none $SPACK_ROOT/var/spack/repos/builtin/packages/libelf/package.py Alongside the ``package.py`` file, a package may contain extra directories or files (like patches) that it needs to build. - +^^^^^^^^^^^^^ Package Names -~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^ Packages are named after the directory containing ``package.py``. It is preferred, but not required, that the directory, and thus the package name, are lower case. So, ``libelf``'s ``package.py`` lives in a directory called ``libelf``. The ``package.py`` file defines a class called ``Libelf``, which -extends Spack's ``Package`` class. for example, here is +extends Spack's ``Package`` class. For example, here is ``$SPACK_ROOT/var/spack/repos/builtin/packages/libelf/package.py``: .. code-block:: python @@ -328,7 +332,7 @@ The **directory name** (``libelf``) determines the package name that users should provide on the command line. e.g., if you type any of these: -.. code-block:: sh +.. code-block:: console $ spack install libelf $ spack install libelf@0.8.13 @@ -346,8 +350,9 @@ difficult to support these options. So, you can name a package ``3proxy`` or ``_foo`` and Spack won't care. It just needs to see that name in the package spec. +^^^^^^^^^^^^^^^^^^^ Package class names -~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^ Spack loads ``package.py`` files dynamically, and it needs to find a special class name in the file for the load to succeed. The **class @@ -366,11 +371,11 @@ some examples: ================= ================= In general, you won't have to remember this naming convention because -`spack-create`_ and `spack-edit`_ handle the details for you. - +:ref:`spack-create` and :ref:`spack-edit` handle the details for you. +------------------- Adding new versions ------------------------- +------------------- The most straightforward way to add new versions to your package is to add a line like this in the package class: @@ -385,8 +390,9 @@ add a line like this in the package class: Versions should be listed with the newest version first. +^^^^^^^^^^^^ Version URLs -~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^ By default, each version's URL is extrapolated from the ``url`` field in the package. For example, Spack is smart enough to download @@ -423,8 +429,9 @@ construct the new one for ``8.2.1``. When you supply a custom URL for a version, Spack uses that URL *verbatim* and does not perform extrapolation. +^^^^^^^^^^^^^^^^^^^^^^^^ Skipping the expand step -~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^ Spack normally expands archives automatically after downloading them. If you want to skip this step (e.g., for self-extracting @@ -452,8 +459,9 @@ it executable, then runs it with some arguments. installer = Executable(self.stage.archive_file) installer('--prefix=%s' % prefix, 'arg1', 'arg2', 'etc.') +^^^^^^^^^ Checksums -~~~~~~~~~~~~~~~~~ +^^^^^^^^^ Spack uses a checksum to ensure that the downloaded package version is not corrupted or compromised. This is especially important when @@ -465,13 +473,14 @@ Spack can currently support checksums using the MD5, SHA-1, SHA-224, SHA-256, SHA-384, and SHA-512 algorithms. It determines the algorithm to use based on the hash length. +^^^^^^^^^^^^^ ``spack md5`` -^^^^^^^^^^^^^^^^^^^^^^ +^^^^^^^^^^^^^ If you have one or more files to checksum, you can use the ``spack md5`` command to do it: -.. code-block:: sh +.. code-block:: console $ spack md5 foo-8.2.1.tar.gz foo-8.2.2.tar.gz ==> 2 MD5 checksums: @@ -481,7 +490,7 @@ command to do it: ``spack md5`` also accepts one or more URLs and automatically downloads the files for you: -.. code-block:: sh +.. code-block:: console $ spack md5 http://example.com/foo-8.2.1.tar.gz ==> Trying to fetch from http://example.com/foo-8.2.1.tar.gz @@ -495,14 +504,15 @@ version of this process. .. _spack-checksum: +^^^^^^^^^^^^^^^^^^ ``spack checksum`` -^^^^^^^^^^^^^^^^^^^^^^ +^^^^^^^^^^^^^^^^^^ If you want to add new versions to a package you've already created, this is automated with the ``spack checksum`` command. Here's an example for ``libelf``: -.. code-block:: sh +.. code-block:: console $ spack checksum libelf ==> Found 16 versions of libelf. @@ -526,7 +536,7 @@ they're released). It fetches the tarballs you ask for and prints out a list of ``version`` commands ready to copy/paste into your package file: -.. code-block:: sh +.. code-block:: console ==> Checksummed new versions of libelf: version('0.8.13', '4136d7b4c04df68b686570afa26988ac') @@ -559,8 +569,9 @@ versions. See the documentation on `attribute_list_url`_ and .. _vcs-fetch: +------------------------------ Fetching from VCS repositories --------------------------------------- +------------------------------ For some packages, source code is provided in a Version Control System (VCS) repository rather than in a tarball. Spack can fetch packages @@ -573,8 +584,9 @@ call to your package with parameters indicating the repository URL and any branch, tag, or revision to fetch. See below for the parameters you'll need for each VCS system. +^^^^^^^^^^^^^^^^^^^^^^^^^ Repositories and versions -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^ The package author is responsible for coming up with a sensible name for each version to be fetched from a repository. For example, if @@ -591,9 +603,9 @@ same thing every time you fetch a particular version. Life isn't always simple, though, so this is not strictly enforced. When fetching from from the branch corresponding to the development version -(often called ``master``,``trunk`` or ``dev``), it is recommended to +(often called ``master``, ``trunk``, or ``dev``), it is recommended to call this version ``develop``. Spack has special treatment for this version so - that ``@develop`` will satisfy dependencies like +that ``@develop`` will satisfy dependencies like ``depends_on(abc, when="@x.y.z:")``. In other words, ``@develop`` is greater than any other version. The rationale is that certain features or options first appear in the development branch. Therefore if a package author @@ -607,16 +619,17 @@ supported. .. _git-fetch: +^^^ Git -~~~~~~~~~~~~~~~~~~~~ +^^^ Git fetching is enabled with the following parameters to ``version``: - * ``git``: URL of the git repository. - * ``tag``: name of a tag to fetch. - * ``branch``: name of a branch to fetch. - * ``commit``: SHA hash (or prefix) of a commit to fetch. - * ``submodules``: Also fetch submodules when checking out this repository. +* ``git``: URL of the git repository. +* ``tag``: name of a tag to fetch. +* ``branch``: name of a branch to fetch. +* ``commit``: SHA hash (or prefix) of a commit to fetch. +* ``submodules``: Also fetch submodules when checking out this repository. Only one of ``tag``, ``branch``, or ``commit`` can be used at a time. @@ -660,7 +673,7 @@ Commits version('2014-10-08', git='https://github.com/example-project/example.git', commit='9d38cd4e2c94c3cea97d0e2924814acc') - This doesn't have to be a full hash; You can abbreviate it as you'd + This doesn't have to be a full hash; you can abbreviate it as you'd expect with git: .. code-block:: python @@ -683,25 +696,25 @@ Submodules version('1.0.1', git='https://github.com/example-project/example.git', tag='v1.0.1', submdoules=True) - +^^^^^^^^^^ Installing -^^^^^^^^^^^^^^ +^^^^^^^^^^ You can fetch and install any of the versions above as you'd expect, by using ``@<version>`` in a spec: -.. code-block:: sh +.. code-block:: console - spack install example@2014-10-08 + $ spack install example@2014-10-08 Git and other VCS versions will show up in the list of versions when a user runs ``spack info <package name>``. - .. _hg-fetch: +^^^^^^^^^ Mercurial -~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^ Fetching with mercurial works much like `git <git-fetch>`_, but you use the ``hg`` parameter. @@ -733,8 +746,9 @@ example@<version>`` command-line syntax. .. _svn-fetch: +^^^^^^^^^^ Subversion -~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^ To fetch with subversion, use the ``svn`` and ``revision`` parameters: @@ -759,6 +773,7 @@ Fetching a revision Subversion branches are handled as part of the directory structure, so you can check out a branch or tag by changing the ``url``. +------------------------------------------------- Expanding additional resources in the source tree ------------------------------------------------- @@ -779,6 +794,7 @@ Based on the keywords present among the arguments the appropriate ``FetchStrateg will be used for the resource. The keyword ``destination`` is relative to the source root of the package and should point to where the resource is to be expanded. +------------------------------------------------------ Automatic caching of files fetched during installation ------------------------------------------------------ @@ -789,42 +805,48 @@ reinstalled on account of a change in the hashing scheme. .. _license: +----------------- Licensed software ------------------------------------------- +----------------- In order to install licensed software, Spack needs to know a few more details about a package. The following class attributes should be defined. +^^^^^^^^^^^^^^^^^^^^ ``license_required`` -~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^ Boolean. If set to ``True``, this software requires a license. If set to ``False``, all of the following attributes will be ignored. Defaults to ``False``. +^^^^^^^^^^^^^^^^^^^ ``license_comment`` -~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^ String. Contains the symbol used by the license manager to denote a comment. Defaults to ``#``. +^^^^^^^^^^^^^^^^^ ``license_files`` -~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^ List of strings. These are files that the software searches for when looking for a license. All file paths must be relative to the installation directory. More complex packages like Intel may require multiple licenses for individual components. Defaults to the empty list. +^^^^^^^^^^^^^^^^ ``license_vars`` -~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^ List of strings. Environment variables that can be set to tell the software where to look for a license if it is not in the usual location. Defaults to the empty list. +^^^^^^^^^^^^^^^ ``license_url`` -~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^ String. A URL pointing to license setup instructions for the software. Defaults to the empty string. @@ -833,12 +855,12 @@ For example, let's take a look at the package for the PGI compilers. .. code-block:: python - # Licensing - license_required = True - license_comment = '#' - license_files = ['license.dat'] - license_vars = ['PGROUPD_LICENSE_FILE', 'LM_LICENSE_FILE'] - license_url = 'http://www.pgroup.com/doc/pgiinstall.pdf' + # Licensing + license_required = True + license_comment = '#' + license_files = ['license.dat'] + license_vars = ['PGROUPD_LICENSE_FILE', 'LM_LICENSE_FILE'] + license_url = 'http://www.pgroup.com/doc/pgiinstall.pdf' As you can see, PGI requires a license. Its license manager, FlexNet, uses the ``#`` symbol to denote a comment. It expects the license file to be @@ -861,39 +883,39 @@ this: .. code-block:: sh - # A license is required to use pgi. - # - # The recommended solution is to store your license key in this global - # license file. After installation, the following symlink(s) will be - # added to point to this file (relative to the installation prefix): - # - # license.dat - # - # Alternatively, use one of the following environment variable(s): - # - # PGROUPD_LICENSE_FILE - # LM_LICENSE_FILE - # - # If you choose to store your license in a non-standard location, you may - # set one of these variable(s) to the full pathname to the license file, or - # port@host if you store your license keys on a dedicated license server. - # You will likely want to set this variable in a module file so that it - # gets loaded every time someone tries to use pgi. - # - # For further information on how to acquire a license, please refer to: - # - # http://www.pgroup.com/doc/pgiinstall.pdf - # - # You may enter your license below. + # A license is required to use pgi. + # + # The recommended solution is to store your license key in this global + # license file. After installation, the following symlink(s) will be + # added to point to this file (relative to the installation prefix): + # + # license.dat + # + # Alternatively, use one of the following environment variable(s): + # + # PGROUPD_LICENSE_FILE + # LM_LICENSE_FILE + # + # If you choose to store your license in a non-standard location, you may + # set one of these variable(s) to the full pathname to the license file, or + # port@host if you store your license keys on a dedicated license server. + # You will likely want to set this variable in a module file so that it + # gets loaded every time someone tries to use pgi. + # + # For further information on how to acquire a license, please refer to: + # + # http://www.pgroup.com/doc/pgiinstall.pdf + # + # You may enter your license below. You can add your license directly to this file, or tell FlexNet to use a license stored on a separate license server. Here is an example that points to a license server called licman1: -.. code-block:: sh +.. code-block:: none - SERVER licman1.mcs.anl.gov 00163eb7fba5 27200 - USE_SERVER + SERVER licman1.mcs.anl.gov 00163eb7fba5 27200 + USE_SERVER If your package requires the license to install, you can reference the location of this global license using ``self.global_license_file``. @@ -909,8 +931,9 @@ documentation. .. _patching: +------- Patches ------------------------------------------- +------- Depending on the host architecture, package version, known bugs, or other issues, you may need to patch your software to get it to build @@ -918,11 +941,12 @@ correctly. Like many other package systems, spack allows you to store patches alongside your package files and apply them to source code after it's downloaded. +^^^^^^^^^ ``patch`` -~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^ You can specify patches in your package file with the ``patch()`` -function. ``patch`` looks like this: +directive. ``patch`` looks like this: .. code-block:: python @@ -934,7 +958,9 @@ The first argument can be either a URL or a filename. It specifies a patch file that should be applied to your source. If the patch you supply is a filename, then the patch needs to live within the spack source tree. For example, the patch above lives in a directory -structure like this:: +structure like this: + +.. code-block:: none $SPACK_ROOT/var/spack/repos/builtin/packages/ mvapich2/ @@ -955,52 +981,59 @@ from the URL and then applied to your source code. ``patch`` can take two options keyword arguments. They are: +"""""""" ``when`` - If supplied, this is a spec that tells spack when to apply - the patch. If the installed package spec matches this spec, the - patch will be applied. In our example above, the patch is applied - when mvapich is at version ``1.9`` or higher. +"""""""" +If supplied, this is a spec that tells spack when to apply +the patch. If the installed package spec matches this spec, the +patch will be applied. In our example above, the patch is applied +when mvapich is at version ``1.9`` or higher. + +""""""""" ``level`` - This tells spack how to run the ``patch`` command. By default, - the level is 1 and spack runs ``patch -p1``. If level is 2, - spack will run ``patch -p2``, and so on. - - A lot of people are confused by level, so here's a primer. If you - look in your patch file, you may see something like this: - - .. code-block:: diff - :linenos: - - --- a/src/mpi/romio/adio/ad_lustre/ad_lustre_rwcontig.c 2013-12-10 12:05:44.806417000 -0800 - +++ b/src/mpi/romio/adio/ad_lustre/ad_lustre_rwcontig.c 2013-12-10 11:53:03.295622000 -0800 - @@ -8,7 +8,7 @@ - * Copyright (C) 2008 Sun Microsystems, Lustre group - */ - - -#define _XOPEN_SOURCE 600 - +//#define _XOPEN_SOURCE 600 - #include <stdlib.h> - #include <malloc.h> - #include "ad_lustre.h" - - Lines 1-2 show paths with synthetic ``a/`` and ``b/`` prefixes. These - are placeholders for the two ``mvapich2`` source directories that - ``diff`` compared when it created the patch file. This is git's - default behavior when creating patch files, but other programs may - behave differently. - - ``-p1`` strips off the first level of the prefix in both paths, - allowing the patch to be applied from the root of an expanded mvapich2 - archive. If you set level to ``2``, it would strip off ``src``, and - so on. - - It's generally easier to just structure your patch file so that it - applies cleanly with ``-p1``, but if you're using a patch you didn't - create yourself, ``level`` can be handy. - -``patch()`` functions -~~~~~~~~~~~~~~~~~~~~~~~~ +""""""""" + +This tells spack how to run the ``patch`` command. By default, +the level is 1 and spack runs ``patch -p 1``. If level is 2, +spack will run ``patch -p 2``, and so on. + +A lot of people are confused by level, so here's a primer. If you +look in your patch file, you may see something like this: + +.. code-block:: diff + :linenos: + + --- a/src/mpi/romio/adio/ad_lustre/ad_lustre_rwcontig.c 2013-12-10 12:05:44.806417000 -0800 + +++ b/src/mpi/romio/adio/ad_lustre/ad_lustre_rwcontig.c 2013-12-10 11:53:03.295622000 -0800 + @@ -8,7 +8,7 @@ + * Copyright (C) 2008 Sun Microsystems, Lustre group + \*/ + + -#define _XOPEN_SOURCE 600 + +//#define _XOPEN_SOURCE 600 + #include <stdlib.h> + #include <malloc.h> + #include "ad_lustre.h" + +Lines 1-2 show paths with synthetic ``a/`` and ``b/`` prefixes. These +are placeholders for the two ``mvapich2`` source directories that +``diff`` compared when it created the patch file. This is git's +default behavior when creating patch files, but other programs may +behave differently. + +``-p1`` strips off the first level of the prefix in both paths, +allowing the patch to be applied from the root of an expanded mvapich2 +archive. If you set level to ``2``, it would strip off ``src``, and +so on. + +It's generally easier to just structure your patch file so that it +applies cleanly with ``-p1``, but if you're using a patch you didn't +create yourself, ``level`` can be handy. + +^^^^^^^^^^^^^^^^^^^^^ +Patch functions +^^^^^^^^^^^^^^^^^^^^^ In addition to supplying patch files, you can write a custom function to patch a package's source. For example, the ``py-pyside`` package @@ -1009,35 +1042,10 @@ handles ``RPATH``: .. _pyside-patch: -.. code-block:: python +.. literalinclude:: ../../../var/spack/repos/builtin/packages/py-pyside/package.py + :pyobject: PyPyside.patch :linenos: - class PyPyside(Package): - ... - - def patch(self): - """Undo PySide RPATH handling and add Spack RPATH.""" - # Figure out the special RPATH - pypkg = self.spec['python'].package - rpath = self.rpath - rpath.append(os.path.join(self.prefix, pypkg.site_packages_dir, 'PySide')) - - # Add Spack's standard CMake args to the sub-builds. - # They're called BY setup.py so we have to patch it. - filter_file( - r'OPTION_CMAKE,', - r'OPTION_CMAKE, ' + ( - '"-DCMAKE_INSTALL_RPATH_USE_LINK_PATH=FALSE", ' - '"-DCMAKE_INSTALL_RPATH=%s",' % ':'.join(rpath)), - 'setup.py') - - # PySide tries to patch ELF files to remove RPATHs - # Disable this and go with the one we set. - filter_file( - r'^\s*rpath_cmd\(pyside_path, srcpath\)', - r'#rpath_cmd(pyside_path, srcpath)', - 'pyside_postinstall.py') - A ``patch`` function, if present, will be run after patch files are applied and before ``install()`` is run. @@ -1048,17 +1056,18 @@ if you run install, hit ctrl-C, and run install again, the code in the patch function is only run once. Also, you can tell Spack to run only the patching part of the build using the :ref:`spack-patch` command. +--------------- Handling RPATHs ----------------------------- +--------------- Spack installs each package in a way that ensures that all of its dependencies are found when it runs. It does this using `RPATHs <http://en.wikipedia.org/wiki/Rpath>`_. An RPATH is a search path, stored in a binary (an executable or library), that tells the dynamic loader where to find its dependencies at runtime. You may be -familiar with ```LD_LIBRARY_PATH`` +familiar with `LD_LIBRARY_PATH <http://tldp.org/HOWTO/Program-Library-HOWTO/shared-libraries.html>`_ -on Linux or ```DYLD_LIBRARY_PATH`` +on Linux or `DYLD_LIBRARY_PATH <https://developer.apple.com/library/mac/documentation/Darwin/Reference/ManPages/man1/dyld.1.html>` on Mac OS X. RPATH is similar to these paths, in that it tells the loader where to find libraries. Unlike them, it is embedded in @@ -1066,36 +1075,36 @@ the binary and not set in each user's environment. RPATHs in Spack are handled in one of three ways: - 1. For most packages, RPATHs are handled automatically using Spack's - :ref:`compiler wrappers <compiler-wrappers>`. These wrappers are - set in standard variables like ``CC``, ``CXX``, and ``FC``, so - most build systems (autotools and many gmake systems) pick them - up and use them. - 2. CMake also respects Spack's compiler wrappers, but many CMake - builds have logic to overwrite RPATHs when binaries are - installed. Spack provides the ``std_cmake_args`` variable, which - includes parameters necessary for CMake build use the right - installation RPATH. It can be used like this when ``cmake`` is - invoked: - - .. code-block:: python - - class MyPackage(Package): - ... - def install(self, spec, prefix): - cmake('..', *std_cmake_args) - make() - make('install') - - 3. If you need to modify the build to add your own RPATHs, you can - use the ``self.rpath`` property of your package, which will - return a list of all the RPATHs that Spack will use when it - links. You can see this how this is used in the :ref:`PySide - example <pyside-patch>` above. - - +#. For most packages, RPATHs are handled automatically using Spack's + :ref:`compiler wrappers <compiler-wrappers>`. These wrappers are + set in standard variables like ``CC``, ``CXX``, ``F77``, and ``FC``, + so most build systems (autotools and many gmake systems) pick them + up and use them. +#. CMake also respects Spack's compiler wrappers, but many CMake + builds have logic to overwrite RPATHs when binaries are + installed. Spack provides the ``std_cmake_args`` variable, which + includes parameters necessary for CMake build use the right + installation RPATH. It can be used like this when ``cmake`` is + invoked: + + .. code-block:: python + + class MyPackage(Package): + ... + def install(self, spec, prefix): + cmake('..', *std_cmake_args) + make() + make('install') + +#. If you need to modify the build to add your own RPATHs, you can + use the ``self.rpath`` property of your package, which will + return a list of all the RPATHs that Spack will use when it + links. You can see this how this is used in the :ref:`PySide + example <pyside-patch>` above. + +-------------------- Finding new versions ----------------------------- +-------------------- You've already seen the ``homepage`` and ``url`` package attributes: @@ -1104,6 +1113,7 @@ You've already seen the ``homepage`` and ``url`` package attributes: from spack import * + class Mpich(Package): """MPICH is a high performance and widely portable implementation of the Message Passing Interface (MPI) standard.""" @@ -1121,8 +1131,9 @@ Spack to find tarballs elsewhere. .. _attribute_list_url: +^^^^^^^^^^^^ ``list_url`` -~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^ When spack tries to find available versions of packages (e.g. with `spack checksum <spack-checksum_>`_), it spiders the parent directory @@ -1156,14 +1167,17 @@ the ``list_url``, because that is where links to old versions are: .. _attribute_list_depth: +^^^^^^^^^^^^^^ ``list_depth`` -~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^ ``libdwarf`` and many other packages have a listing of available versions on a single webpage, but not all do. For example, ``mpich`` has a tarball URL that looks like this: - url = "http://www.mpich.org/static/downloads/3.0.4/mpich-3.0.4.tar.gz" +.. code-block:: python + + url = "http://www.mpich.org/static/downloads/3.0.4/mpich-3.0.4.tar.gz" But its downloads are in many different subdirectories of ``http://www.mpich.org/static/downloads/``. So, we need to add a @@ -1187,8 +1201,9 @@ when spidering the page. .. _attribute_parallel: +--------------- Parallel builds ------------------- +--------------- By default, Spack will invoke ``make()`` with a ``-j <njobs>`` argument, so that builds run in parallel. It figures out how many @@ -1240,11 +1255,11 @@ you set ``parallel`` to ``False`` at the package level, then each call to ``make()`` will be sequential by default, but packagers can call ``make(parallel=True)`` to override it. - .. _dependencies: +------------ Dependencies ------------------------------- +------------ We've covered how to build a simple package, but what if one package relies on another package to build? How do you express that in a @@ -1271,8 +1286,9 @@ Spack makes this relatively easy. Let's take a look at the def install(self, spec, prefix): ... +^^^^^^^^^^^^^^^^ ``depends_on()`` -~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^ The highlighted ``depends_on('libelf')`` call tells Spack that it needs to build and install the ``libelf`` package before it builds @@ -1280,8 +1296,9 @@ needs to build and install the ``libelf`` package before it builds guaranteed that ``libelf`` has been built and installed successfully, so you can rely on it for your libdwarf build. +^^^^^^^^^^^^^^^^ Dependency specs -~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^ ``depends_on`` doesn't just take the name of another package. It takes a full spec. This means that you can restrict the versions or @@ -1310,7 +1327,7 @@ Or a requirement for a particular variant or compiler flags: depends_on("libelf@0.8+debug") depends_on('libelf debug=True') - depends_on('libelf cppflags="-fPIC") + depends_on('libelf cppflags="-fPIC"') Both users *and* package authors can use the same spec syntax to refer to different package configurations. Users use the spec syntax on the @@ -1322,9 +1339,9 @@ Additionally, dependencies may be specified for specific use cases: .. code-block:: python - depends_on("cmake", type="build") - depends_on("libelf", type=("build", "link")) - depends_on("python", type="run") + depends_on("cmake", type="build") + depends_on("libelf", type=("build", "link")) + depends_on("python", type="run") The dependency types are: @@ -1345,8 +1362,9 @@ Lua module loading). .. _setup-dependent-environment: +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ``setup_dependent_environment()`` -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Spack provides a mechanism for dependencies to provide variables that can be used in their dependents' build. Any package can declare a @@ -1370,34 +1388,16 @@ packages that depend on a particular Qt installation will find it. The arguments to this function are: - * **module**: the module of the dependent package, where global - properties can be assigned. - * **spec**: the spec of the *dependency package* (the one the function is called on). - * **dep_spec**: the spec of the dependent package (i.e. dep_spec depends on spec). +* **module**: the module of the dependent package, where global + properties can be assigned. +* **spec**: the spec of the *dependency package* (the one the function is called on). +* **dep_spec**: the spec of the dependent package (i.e. dep_spec depends on spec). A good example of using these is in the Python package: -.. code-block:: python - - def setup_dependent_environment(self, module, spec, dep_spec): - # Python extension builds can have a global python executable function - module.python = Executable(join_path(spec.prefix.bin, 'python')) - - # Add variables for lib/pythonX.Y and lib/pythonX.Y/site-packages dirs. - module.python_lib_dir = os.path.join(dep_spec.prefix, self.python_lib_dir) - module.python_include_dir = os.path.join(dep_spec.prefix, self.python_include_dir) - module.site_packages_dir = os.path.join(dep_spec.prefix, self.site_packages_dir) - - # Make the site packages directory if it does not exist already. - mkdirp(module.site_packages_dir) - - # Set PYTHONPATH to include site-packages dir for the - # extension and any other python extensions it depends on. - python_paths = [] - for d in dep_spec.traverse(): - if d.package.extends(self.spec): - python_paths.append(os.path.join(d.prefix, self.site_packages_dir)) - os.environ['PYTHONPATH'] = ':'.join(python_paths) +.. literalinclude:: ../../../var/spack/repos/builtin/packages/python/package.py + :pyobject: Python.setup_dependent_environment + :linenos: The first thing that happens here is that the ``python`` command is inserted into module scope of the dependent. This allows most python @@ -1406,17 +1406,17 @@ packages to have a very simple install method, like this: .. code-block:: python def install(self, spec, prefix): - python('setup.py', 'install', '--prefix=%s' % prefix) + python('setup.py', 'install', '--prefix={0}'.format(prefix)) Python's ``setup_dependent_environment`` method also sets up some other variables, creates a directory, and sets up the ``PYTHONPATH`` so that dependent packages can find their dependencies at build time. - .. _packaging_extensions: +---------- Extensions -------------------------- +---------- Spack's support for package extensions is documented extensively in :ref:`extensions`. This section documents how to make your own @@ -1448,7 +1448,7 @@ activate``. When it is activated, all the files in its prefix will be symbolically linked into the prefix of the python package. Many packages produce Python extensions for *some* variants, but not -others: they should extend ``python`` only if the apropriate +others: they should extend ``python`` only if the appropriate variant(s) are selected. This may be accomplished with conditional ``extends()`` declarations: @@ -1467,13 +1467,15 @@ when it does the activation: .. code-block:: python - class PyNose(Package): + class PySncosmo(Package): ... - extends('python', ignore=r'bin/nosetests.*$') + # py-sncosmo binaries are duplicates of those from py-astropy + extends('python', ignore=r'bin/.*') + depends_on('py-astropy') ... -The code above will prevent ``$prefix/bin/nosetests`` from being -linked in at activation time. +The code above will prevent everything in the ``$prefix/bin/`` directory +from being linked in at activation time. .. note:: @@ -1482,10 +1484,9 @@ linked in at activation time. ``depends_on('python')`` and ``extends(python)`` in the same package. ``extends`` implies ``depends_on``. - - +^^^^^^^^^^^^^^^^^^^^^^^^^ Activation & deactivation -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^ Spack's ``Package`` class has default ``activate`` and ``deactivate`` implementations that handle symbolically linking extensions' prefixes @@ -1503,15 +1504,9 @@ same way that Python does. Let's look at Python's activate function: -.. code-block:: python - - def activate(self, ext_pkg, **kwargs): - kwargs.update(ignore=self.python_ignore(ext_pkg, kwargs)) - super(Python, self).activate(ext_pkg, **kwargs) - - exts = spack.install_layout.extension_map(self.spec) - exts[ext_pkg.name] = ext_pkg.spec - self.write_easy_install_pth(exts) +.. literalinclude:: ../../../var/spack/repos/builtin/packages/python/package.py + :pyobject: Python.activate + :linenos: This function is called on the *extendee* (Python). It first calls ``activate`` in the superclass, which handles symlinking the @@ -1521,23 +1516,16 @@ Python's setuptools. Deactivate behaves similarly to activate, but it unlinks files: -.. code-block:: python - - def deactivate(self, ext_pkg, **kwargs): - kwargs.update(ignore=self.python_ignore(ext_pkg, kwargs)) - super(Python, self).deactivate(ext_pkg, **kwargs) - - exts = spack.install_layout.extension_map(self.spec) - if ext_pkg.name in exts: # Make deactivate idempotent. - del exts[ext_pkg.name] - self.write_easy_install_pth(exts) +.. literalinclude:: ../../../var/spack/repos/builtin/packages/python/package.py + :pyobject: Python.deactivate + :linenos: Both of these methods call some custom functions in the Python package. See the source for Spack's Python package for details. - +^^^^^^^^^^^^^^^^^^^^ Activation arguments -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^ You may have noticed that the ``activate`` function defined above takes keyword arguments. These are the keyword arguments from @@ -1552,11 +1540,11 @@ The only keyword argument supported by default is the ``ignore`` argument, which can take a regex, list of regexes, or a predicate to determine which files *not* to symlink during activation. - .. _virtual-dependencies: +-------------------- Virtual dependencies ------------------------------ +-------------------- In some cases, more than one package can satisfy another package's dependency. One way this can happen is if a package depends on a @@ -1577,8 +1565,9 @@ similar package files, e.g., ``foo``, ``foo-mvapich``, ``foo-mpich``, but Spack avoids this explosion of package files by providing support for *virtual dependencies*. +^^^^^^^^^^^^ ``provides`` -~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^ In Spack, ``mpi`` is handled as a *virtual package*. A package like ``mpileaks`` can depend on it just like any other package, by @@ -1614,8 +1603,9 @@ The ``provides("mpi")`` call tells Spack that the ``mpich`` package can be used to satisfy the dependency of any package that ``depends_on('mpi')``. +^^^^^^^^^^^^^^^^^^^^ Versioned Interfaces -~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^ Just as you can pass a spec to ``depends_on``, so can you pass a spec to ``provides`` to add constraints. This allows Spack to support the @@ -1636,8 +1626,9 @@ This says that ``mpich2`` provides MPI support *up to* version 2, but if a package ``depends_on("mpi@3")``, then Spack will *not* build that package with ``mpich2``. +^^^^^^^^^^^^^^^^^ ``provides when`` -~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^ The same package may provide different versions of an interface depending on *its* version. Above, we simplified the ``provides`` @@ -1671,27 +1662,27 @@ the package ``foo`` declares this: Suppose a user invokes ``spack install`` like this: -.. code-block:: sh +.. code-block:: console $ spack install foo ^mpich@1.0 Spack will fail with a constraint violation, because the version of MPICH requested is too low for the ``mpi`` requirement in ``foo``. - .. _abstract-and-concrete: +------------------------- Abstract & concrete specs ------------------------------------------- +------------------------- Now that we've seen how spec constraints can be specified :ref:`on the command line <sec-specs>` and within package definitions, we can talk about how Spack puts all of this information together. When you run this: -.. code-block:: sh +.. code-block:: console - spack install mpileaks ^callpath@1.0+debug ^libelf@0.8.11 + $ spack install mpileaks ^callpath@1.0+debug ^libelf@0.8.11 Spack parses the command line and builds a spec from the description. The spec says that ``mpileaks`` should be built with the ``callpath`` @@ -1707,7 +1698,9 @@ abstract spec is partially specified. In other words, it could describe more than one build of a package. Spack does this to make things easier on the user: they should only have to specify as much of the package spec as they care about. Here's an example partial spec -DAG, based on the constraints above:: +DAG, based on the constraints above: + +.. code-block:: none mpileaks ^callpath@1.0+debug @@ -1716,7 +1709,6 @@ DAG, based on the constraints above:: ^libelf@0.8.11 ^mpi - .. graphviz:: digraph { @@ -1727,7 +1719,6 @@ DAG, based on the constraints above:: dyninst -> "libelf@0.8.11" } - This diagram shows a spec DAG output as a tree, where successive levels of indentation represent a depends-on relationship. In the above DAG, we can see some packages annotated with their constraints, @@ -1735,8 +1726,9 @@ and some packages with no annotations at all. When there are no annotations, it means the user doesn't care what configuration of that package is built, just so long as it works. +^^^^^^^^^^^^^^ Concretization -~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^ An abstract spec is useful for the user, but you can't install an abstract spec. Spack has to take the abstract spec and "fill in" the @@ -1744,7 +1736,9 @@ remaining unspecified parts in order to install. This process is called **concretization**. Concretization happens in between the time the user runs ``spack install`` and the time the ``install()`` method is called. The concretized version of the spec above might look like -this:: +this: + +.. code-block:: none mpileaks@2.3%gcc@4.7.3 arch=linux-debian7-x86_64 ^callpath@1.0%gcc@4.7.3+debug arch=linux-debian7-x86_64 @@ -1776,13 +1770,14 @@ the preferences of their own users. .. _spack-spec: +^^^^^^^^^^^^^^ ``spack spec`` -~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^ For an arbitrary spec, you can see the result of concretization by running ``spack spec``. For example: -.. code-block:: sh +.. code-block:: console $ spack spec dyninst@8.0.1 dyninst@8.0.1 @@ -1796,44 +1791,43 @@ running ``spack spec``. For example: This is useful when you want to know exactly what Spack will do when you ask for a particular spec. +.. _concretization-policies: +^^^^^^^^^^^^^^^^^^^^^^^^^^^ ``Concretization Policies`` -~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^^ A user may have certain preferences for how packages should be concretized on their system. For example, one user may prefer packages built with OpenMPI and the Intel compiler. Another user may prefer packages be built with MVAPICH and GCC. -See the `documentation in the config section <concretization-preferences_>`_ -for more details. +See the :ref:`concretization-preferences` section for more details. .. _install-method: +----------------------------------- Implementing the ``install`` method ------------------------------------------- +----------------------------------- The last element of a package is its ``install()`` method. This is where the real work of installation happens, and it's the main part of the package you'll need to customize for each piece of software. .. literalinclude:: ../../../var/spack/repos/builtin/packages/libelf/package.py - :start-after: 0.8.12 + :pyobject: Libelf.install :linenos: ``install`` takes a ``spec``: a description of how the package should be built, and a ``prefix``: the path to the directory where the software should be installed. - Spack provides wrapper functions for ``configure`` and ``make`` so that you can call them in a similar way to how you'd call a shell command. In reality, these are Python functions. Spack provides these functions to make writing packages more natural. See the section on :ref:`shell wrappers <shell-wrappers>`. - - Now that the metadata is out of the way, we can move on to the ``install()`` method. When a user runs ``spack install``, Spack fetches an archive for the correct version of the software, expands @@ -1883,8 +1877,9 @@ information. .. _install-environment: +----------------------- The install environment --------------------------- +----------------------- In general, you should not have to do much differently in your install method than you would when installing a package on the command line. @@ -1901,14 +1896,15 @@ custom Makefiles, you may need to add logic to modify the makefiles. The remainder of the section covers the way Spack's build environment works. +^^^^^^^^^^^^^^^^^^^^^ Environment variables -~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^ Spack sets a number of standard environment variables that serve two purposes: - #. Make build systems use Spack's compiler wrappers for their builds. - #. Allow build systems to find dependencies more easily +#. Make build systems use Spack's compiler wrappers for their builds. +#. Allow build systems to find dependencies more easily The Compiler environment variables that Spack sets are: @@ -1922,7 +1918,7 @@ The Compiler environment variables that Spack sets are: ============ =============================== All of these are standard variables respected by most build systems. -If your project uses ``autotools`` or ``CMake``, then it should pick +If your project uses ``Autotools`` or ``CMake``, then it should pick them up automatically when you run ``configure`` or ``cmake`` in the ``install()`` function. Many traditional builds using GNU Make and BSD make also respect these variables, so they may work with these @@ -1939,12 +1935,12 @@ In addition to the compiler variables, these variables are set before entering ``install()`` so that packages can locate dependencies easily: - ======================= ============================= - ``PATH`` Set to point to ``/bin`` directories of dependencies - ``CMAKE_PREFIX_PATH`` Path to dependency prefixes for CMake - ``PKG_CONFIG_PATH`` Path to any pkgconfig directories for dependencies - ``PYTHONPATH`` Path to site-packages dir of any python dependencies - ======================= ============================= +===================== ==================================================== +``PATH`` Set to point to ``/bin`` directories of dependencies +``CMAKE_PREFIX_PATH`` Path to dependency prefixes for CMake +``PKG_CONFIG_PATH`` Path to any pkgconfig directories for dependencies +``PYTHONPATH`` Path to site-packages dir of any python dependencies +===================== ==================================================== ``PATH`` is set up to point to dependencies ``/bin`` directories so that you can use tools installed by dependency packages at build time. @@ -1955,7 +1951,7 @@ For example, ``$MPICH_ROOT/bin/mpicc`` is frequently used by dependencies of where ``cmake`` will search for dependency libraries and headers. This causes all standard CMake find commands to look in the paths of your dependencies, so you *do not* have to manually specify arguments -like ``-D DEPENDENCY_DIR=/path/to/dependency`` to ``cmake``. More on +like ``-DDEPENDENCY_DIR=/path/to/dependency`` to ``cmake``. More on this is `in the CMake documentation <http://www.cmake.org/cmake/help/v3.0/variable/CMAKE_PREFIX_PATH.html>`_. ``PKG_CONFIG_PATH`` is for packages that attempt to discover @@ -1970,8 +1966,9 @@ below. .. _compiler-wrappers: +^^^^^^^^^^^^^^^^^^^^^ Compiler interceptors -~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^ As mentioned, ``CC``, ``CXX``, ``F77``, and ``FC`` are set to point to Spack's compiler wrappers. These are simply called ``cc``, ``c++``, @@ -1992,13 +1989,15 @@ flags to the compile line so that dependencies can be easily found. These flags are added for each dependency, if they exist: Compile-time library search paths - * ``-L$dep_prefix/lib`` - * ``-L$dep_prefix/lib64`` +* ``-L$dep_prefix/lib`` +* ``-L$dep_prefix/lib64`` + Runtime library search paths (RPATHs) - * ``$rpath_flag$dep_prefix/lib`` - * ``$rpath_flag$dep_prefix/lib64`` +* ``$rpath_flag$dep_prefix/lib`` +* ``$rpath_flag$dep_prefix/lib64`` + Include search paths - * ``-I$dep_prefix/include`` +* ``-I$dep_prefix/include`` An example of this would be the ``libdwarf`` build, which has one dependency: ``libelf``. Every call to ``cc`` in the ``libdwarf`` @@ -2037,12 +2036,14 @@ and/or ``ldlibs``). They do not override the canonical autotools flags with the same names (but in ALL-CAPS) that may be passed into the build by particularly challenging package scripts. +^^^^^^^^^^^^^^ Compiler flags -~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^ + In rare circumstances such as compiling and running small unit tests, a package developer may need to know what are the appropriate compiler flags to enable features like ``OpenMP``, ``c++11``, ``c++14`` and alike. To that end the -compiler classes in ``spack`` implement the following _properties_ : +compiler classes in ``spack`` implement the following **properties**: ``openmp_flag``, ``cxx11_flag``, ``cxx14_flag``, which can be accessed in a package by ``self.compiler.cxx11_flag`` and alike. Note that the implementation is such that if a given compiler version does not support this feature, an @@ -2054,12 +2055,14 @@ package supports additional variants like variant('openmp', default=True, description="Enable OpenMP support.") +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Message Parsing Interface (MPI) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + It is common for high performance computing software/packages to use ``MPI``. As a result of conretization, a given package can be built using different implementations of MPI such as ``Openmpi``, ``MPICH`` or ``IntelMPI``. -In some scenarios to configure a package one have to provide it with appropriate MPI +In some scenarios, to configure a package, one has to provide it with appropriate MPI compiler wrappers such as ``mpicc``, ``mpic++``. However different implementations of ``MPI`` may have different names for those wrappers. In order to make package's ``install()`` method indifferent to the @@ -2070,22 +2073,23 @@ Package developers are advised to use these variables, for example ``self.spec[' instead of hard-coding ``join_path(self.spec['mpi'].prefix.bin, 'mpicc')`` for the reasons outlined above. - +^^^^^^^^^^^^^^^^^^^^^^^^^ Blas and Lapack libraries -~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^ + Different packages provide implementation of ``Blas`` and ``Lapack`` routines. The names of the resulting static and/or shared libraries differ from package -to package. In order to make ``install()`` method indifferent to the +to package. In order to make the ``install()`` method indifferent to the choice of ``Blas`` implementation, each package which provides it -sets up ``self.spec.blas_shared_lib`` and ``self.spec.blas_static_lib `` to +sets up ``self.spec.blas_shared_lib`` and ``self.spec.blas_static_lib`` to point to the shared and static ``Blas`` libraries, respectively. The same applies to packages which provide ``Lapack``. Package developers are advised to use these variables, for example ``spec['blas'].blas_shared_lib`` instead of hard-coding ``join_path(spec['blas'].prefix.lib, 'libopenblas.so')``. - +^^^^^^^^^^^^^^^^^^^^^ Forking ``install()`` -~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^ To give packagers free reign over their install environment, Spack forks a new process each time it invokes a package's ``install()`` @@ -2097,9 +2101,9 @@ dedicated process. .. _prefix-objects: - +----------------- Failing the build ----------------------- +----------------- Sometimes you don't want a package to successfully install unless some condition is true. You can explicitly cause the build to fail from @@ -2110,9 +2114,9 @@ condition is true. You can explicitly cause the build to fail from if spec.architecture.startswith('darwin'): raise InstallError('This package does not build on Mac OS X!') - +-------------- Prefix objects ----------------------- +-------------- Spack passes the ``prefix`` parameter to the install method so that you can pass it to ``configure``, ``cmake``, or some other installer, @@ -2122,7 +2126,6 @@ e.g.: configure('--prefix=' + prefix) - For the most part, prefix objects behave exactly like strings. For packages that do not have their own install target, or for those that implement it poorly (like ``libdwarf``), you may need to manually copy @@ -2142,7 +2145,6 @@ yourself, e.g.: mkdirp(prefix.lib) install('libfoo.a', prefix.lib) - Most of the standard UNIX directory names are attributes on the ``prefix`` object. Here is a full list: @@ -2169,8 +2171,9 @@ Most of the standard UNIX directory names are attributes on the .. _spec-objects: +------------ Spec objects -------------------------- +------------ When ``install`` is called, most parts of the build process are set up for you. The correct version's tarball has been downloaded and @@ -2187,8 +2190,9 @@ special parameters to ``configure``, like need to supply special compiler flags depending on the compiler. All of this information is available in the spec. +^^^^^^^^^^^^^^^^^^^^^^^^ Testing spec constraints -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^ You can test whether your spec is configured a certain way by using the ``satisfies`` method. For example, if you want to check whether @@ -2197,9 +2201,14 @@ do that, e.g.: .. code-block:: python + configure_args = [ + '--prefix={0}'.format(prefix) + ] + if spec.satisfies('@1.2:1.4'): configure_args.append("CXXFLAGS='-DWITH_FEATURE'") - configure('--prefix=' + prefix, *configure_args) + + configure(*configure_args) This works for compilers, too: @@ -2253,39 +2262,40 @@ the two functions is that ``satisfies()`` tests whether spec constraints overlap at all, while ``in`` tests whether a spec or any of its dependencies satisfy the provided spec. - +^^^^^^^^^^^^^^^^^^^^^^ Accessing Dependencies -~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^ You may need to get at some file or binary that's in the prefix of one of your dependencies. You can do that by sub-scripting the spec: .. code-block:: python - my_mpi = spec['mpich'] + my_mpi = spec['mpi'] The value in the brackets needs to be some package name, and spec needs to depend on that package, or the operation will fail. For example, the above code will fail if the ``spec`` doesn't depend on -``mpich``. The value returned and assigned to ``my_mpi``, is itself +``mpi``. The value returned and assigned to ``my_mpi``, is itself just another ``Spec`` object, so you can do all the same things you would do with the package's own spec: .. code-block:: python - mpicc = new_path(my_mpi.prefix.bin, 'mpicc') + mpicc = join_path(my_mpi.prefix.bin, 'mpicc') .. _multimethods: +^^^^^^^^^^^^^^^^^^^^^^^^^^ Multimethods and ``@when`` -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^ Spack allows you to make multiple versions of instance functions in packages, based on whether the package's spec satisfies particular criteria. The ``@when`` annotation lets packages declare multiple versions of -methods like install() that depend on the package's spec. For +methods like ``install()`` that depend on the package's spec. For example: .. code-block:: python @@ -2296,16 +2306,17 @@ example: def install(self, prefix): # Do default install - @when('arch=linux-debian7-x86_64') + @when('arch=chaos_5_x86_64_ib') def install(self, prefix): # This will be executed instead of the default install if # the package's sys_type() is chaos_5_x86_64_ib. - @when('arch=linux-debian7-x86_64") + @when('arch=linux-debian7-x86_64') def install(self, prefix): - # This will be executed if the package's sys_type is bgqos_0 + # This will be executed if the package's sys_type() is + # linux-debian7-x86_64. -In the above code there are three versions of install(), two of which +In the above code there are three versions of ``install()``, two of which are specialized for particular platforms. The version that is called depends on the architecture of the package spec. @@ -2377,22 +2388,15 @@ method (the one without the ``@when`` decorator) will be called. .. _shell-wrappers: +----------------------- Shell command functions ----------------------------- +----------------------- Recall the install method from ``libelf``: -.. code-block:: python - - def install(self, spec, prefix): - configure("--prefix=" + prefix, - "--enable-shared", - "--disable-dependency-tracking", - "--disable-debug") - make() - - # The mkdir commands in libelf's install can fail in parallel - make("install", parallel=False) +.. literalinclude:: ../../../var/spack/repos/builtin/packages/libelf/package.py + :pyobject: Libelf.install + :linenos: Normally in Python, you'd have to write something like this in order to execute shell commands: @@ -2400,7 +2404,7 @@ to execute shell commands: .. code-block:: python import subprocess - subprocess.check_call('configure', '--prefix=' + prefix) + subprocess.check_call('configure', '--prefix={0}'.format(prefix)) We've tried to make this a bit easier by providing callable wrapper objects for some shell commands. By default, ``configure``, @@ -2420,17 +2424,17 @@ Callable wrappers also allow spack to provide some special features. For example, in Spack, ``make`` is parallel by default, and Spack figures out the number of cores on your machine and passes an appropriate value for ``-j<numjobs>`` when it calls ``make`` (see the -``parallel`` package attribute under :ref:`metadata <metadata>`). In +``parallel`` `package attribute <attribute_parallel>`). In a package file, you can supply a keyword argument, ``parallel=False``, to the ``make`` wrapper to disable parallel make. In the ``libelf`` package, this allows us to avoid race conditions in the library's build system. - .. _sanity-checks: +------------------------------- Sanity checking an installation --------------------------------- +------------------------------- By default, Spack assumes that a build has failed if nothing is written to the install prefix, and that it has succeeded if anything @@ -2442,7 +2446,7 @@ Consider a simple autotools build like this: .. code-block:: python def install(self, spec, prefix): - configure("--prefix=" + prefix) + configure("--prefix={0}".format(prefix)) make() make("install") @@ -2455,9 +2459,9 @@ like this can falsely report that they were successfully installed if an error occurs before the install is complete but after files have been written to the ``prefix``. - +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ``sanity_check_is_file`` and ``sanity_check_is_dir`` -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ You can optionally specify *sanity checks* to deal with this problem. Add properties like this to your package: @@ -2479,14 +2483,14 @@ Now, after ``install()`` runs, Spack will check whether ``$prefix/include/libelf.h`` exists and is a file, and whether ``$prefix/lib`` exists and is a directory. If the checks fail, then the build will fail and the install prefix will be removed. If they -succeed, Spack considers the build succeeful and keeps the prefix in +succeed, Spack considers the build successful and keeps the prefix in place. - .. _file-manipulation: +--------------------------- File manipulation functions ------------------------------- +--------------------------- Many builds are not perfect. If a build lacks an install target, or if it does not use systems like CMake or autotools, which have standard @@ -2508,9 +2512,9 @@ running: This is already part of the boilerplate for packages created with ``spack create`` or ``spack edit``. - +^^^^^^^^^^^^^^^^^^^ Filtering functions -~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^ :py:func:`filter_file(regex, repl, *filenames, **kwargs) <spack.filter_file>` Works like ``sed`` but with Python regular expression syntax. Takes @@ -2568,9 +2572,9 @@ Filtering functions change_sed_delimiter('@', ';', 'utils/FixMakefile') change_sed_delimiter('@', ';', 'utils/FixMakefile.sed.default') - +^^^^^^^^^^^^^^ File functions -~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^ :py:func:`ancestor(dir, n=1) <spack.ancestor>` Get the n\ :sup:`th` ancestor of the directory ``dir``. @@ -2587,8 +2591,8 @@ File functions install('my-header.h', join_path(prefix.include)) -:py:func:`join_path(prefix, *args) <spack.join_path>` Like - ``os.path.join``, this joins paths using the OS path separator. +:py:func:`join_path(prefix, *args) <spack.join_path>` + Like ``os.path.join``, this joins paths using the OS path separator. However, this version allows an arbitrary number of arguments, so you can string together many path components. @@ -2638,22 +2642,21 @@ File functions The ``create=True`` keyword argument causes the command to create the directory if it does not exist. - :py:func:`touch(path) <spack.touch>` Create an empty file at ``path``. - .. _package-lifecycle: +----------------------- Coding Style Guidelines ---------------------------- +----------------------- The following guidelines are provided, in the interests of making Spack packages work in a consistent manner: - +^^^^^^^^^^^^^ Variant Names -~~~~~~~~~~~~~~ +^^^^^^^^^^^^^ Spack packages with variants similar to already-existing Spack packages should use the same name for their variants. Standard @@ -2663,43 +2666,54 @@ variant names are: Name Default Description ======= ======== ======================== shared True Build shared libraries - static Build static libraries - mpi Use MPI - python Build Python extension + static True Build static libraries + mpi True Use MPI + python False Build Python extension ======= ======== ======================== If specified in this table, the corresponding default should be used when declaring a variant. - +^^^^^^^^^^^^^ Version Lists -~~~~~~~~~~~~~~ +^^^^^^^^^^^^^ -Spack packges should list supported versions with the newest first. +Spack packages should list supported versions with the newest first. +^^^^^^^^^^^^^^^^ Special Versions -~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^ The following *special* version names may be used when building a package: -* *@system*: Indicates a hook to the OS-installed version of the - package. This is useful, for example, to tell Spack to use the - OS-installed version in ``packages.yaml``:: +""""""""""" +``@system`` +""""""""""" + +Indicates a hook to the OS-installed version of the +package. This is useful, for example, to tell Spack to use the +OS-installed version in ``packages.yaml``: + +.. code-block:: yaml - openssl: - paths: - openssl@system: /usr - buildable: False + openssl: + paths: + openssl@system: /usr + buildable: False - Certain Spack internals look for the *@system* version and do - appropriate things in that case. +Certain Spack internals look for the ``@system`` version and do +appropriate things in that case. -* *@local*: Indicates the version was built manually from some source - tree of unknown provenance (see ``spack setup``). +"""""""""" +``@local`` +"""""""""" +Indicates the version was built manually from some source +tree of unknown provenance (see ``spack setup``). +--------------------------- Packaging workflow commands ---------------------------------- +--------------------------- When you are building packages, you will likely not get things completely right the first time. @@ -2714,7 +2728,7 @@ of the build. A typical package workflow might look like this: -.. code-block:: sh +.. code-block:: console $ spack edit mypackage $ spack install mypackage @@ -2729,8 +2743,9 @@ control over the install process. .. _spack-fetch: +^^^^^^^^^^^^^^^ ``spack fetch`` -~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^ The first step of ``spack install``. Takes a spec and determines the correct download URL to use for the requested package version, then @@ -2743,8 +2758,9 @@ fetch`` is idempotent and will not download the archive again. .. _spack-stage: +^^^^^^^^^^^^^^^ ``spack stage`` -~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^ The second step in ``spack install`` after ``spack fetch``. Expands the downloaded archive in its temporary directory, where it will be @@ -2753,8 +2769,9 @@ already been expanded, ``stage`` is idempotent. .. _spack-patch: +^^^^^^^^^^^^^^^ ``spack patch`` -~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^ After staging, Spack applies patches to downloaded packages, if any have been specified in the package file. This command will run the @@ -2766,32 +2783,37 @@ package before patching. .. _spack-restage: +^^^^^^^^^^^^^^^^^ ``spack restage`` -~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^ + Restores the source code to pristine state, as it was before building. Does this in one of two ways: - 1. If the source was fetched as a tarball, deletes the entire build - directory and re-expands the tarball. +#. If the source was fetched as a tarball, deletes the entire build + directory and re-expands the tarball. - 2. If the source was checked out from a repository, this deletes the - build directory and checks it out again. +#. If the source was checked out from a repository, this deletes the + build directory and checks it out again. .. _spack-clean: +^^^^^^^^^^^^^^^ ``spack clean`` -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^ + Cleans up temporary files for a particular package, by deleting the expanded/checked out source code *and* any downloaded archive. If ``fetch``, ``stage``, or ``install`` are run again after this, Spack's build process will start from scratch. - .. _spack-purge: +^^^^^^^^^^^^^^^ ``spack purge`` -~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^ + Cleans up all of Spack's temporary and cached files. This can be used to recover disk space if temporary files from interrupted or failed installs accumulate in the staging area. @@ -2803,78 +2825,58 @@ running ``spack clean`` for every package you have fetched or staged. When called with ``--cache`` or ``--all`` this will clear all resources :ref:`cached <caching>` during installs. +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Keeping the stage directory on success -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ By default, ``spack install`` will delete the staging area once a package has been successfully built and installed. Use ``--keep-stage`` to leave the build directory intact: -.. code-block:: sh +.. code-block:: console - spack install --keep-stage <spec> + $ spack install --keep-stage <spec> This allows you to inspect the build directory and potentially debug the build. You can use ``purge`` or ``clean`` later to get rid of the unwanted temporary files. - +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Keeping the install prefix on failure -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ By default, ``spack install`` will delete any partially constructed install prefix if anything fails during ``install()``. If you want to keep the prefix anyway (e.g. to diagnose a bug), you can use ``--keep-prefix``: -.. code-block:: sh +.. code-block:: console - spack install --keep-prefix <spec> + $ spack install --keep-prefix <spec> Note that this may confuse Spack into thinking that the package has -been installed properly, so you may need to use ``spack uninstall -f`` +been installed properly, so you may need to use ``spack uninstall --force`` to get rid of the install prefix before you build again: -.. code-block:: sh - - spack uninstall -f <spec> +.. code-block:: console + $ spack uninstall --force <spec> +--------------------- Graphing dependencies --------------------------- +--------------------- .. _spack-graph: +^^^^^^^^^^^^^^^ ``spack graph`` -~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^ Spack provides the ``spack graph`` command for graphing dependencies. The command by default generates an ASCII rendering of a spec's -dependency graph. For example:: - - $ spack graph mpileaks - o mpileaks - |\ - | |\ - | o | callpath - |/| | - | |\| - | |\ \ - | | |\ \ - | | | | o adept-utils - | |_|_|/| - |/| | | | - o | | | | mpi - / / / / - | | o | dyninst - | |/| | - |/|/| | - | | |/ - | o | libdwarf - |/ / - o | libelf - / - o boost +dependency graph. For example: + +.. command-output:: spack graph mpileaks At the top is the root package in the DAG, with dependency edges emerging from it. On a color terminal, the edges are colored by which @@ -2882,69 +2884,46 @@ dependency they lead to. You can also use ``spack graph`` to generate graphs in the widely used `Dot <http://www.graphviz.org/doc/info/lang.html>`_ format. For -example:: - - $ spack graph --dot mpileaks - digraph G { - label = "Spack Dependencies" - labelloc = "b" - rankdir = "LR" - ranksep = "5" - - "boost" [label="boost"] - "callpath" [label="callpath"] - "libdwarf" [label="libdwarf"] - "mpileaks" [label="mpileaks"] - "mpi" [label="mpi"] - "adept-utils" [label="adept-utils"] - "dyninst" [label="dyninst"] - "libelf" [label="libelf"] - - "callpath" -> "dyninst" - "callpath" -> "adept-utils" - "callpath" -> "mpi" - "callpath" -> "libelf" - "callpath" -> "libdwarf" - "libdwarf" -> "libelf" - "mpileaks" -> "adept-utils" - "mpileaks" -> "callpath" - "mpileaks" -> "mpi" - "adept-utils" -> "boost" - "adept-utils" -> "mpi" - "dyninst" -> "boost" - "dyninst" -> "libelf" - "dyninst" -> "libdwarf" - } +example: + +.. command-output:: spack graph --dot mpileaks This graph can be provided as input to other graphing tools, such as those in `Graphviz <http://www.graphviz.org>`_. +------------------------- Interactive shell support --------------------------- +------------------------- Spack provides some limited shell support to make life easier for packagers. You can enable these commands by sourcing a setup file in -the ``/share/spack`` directory. For ``bash`` or ``ksh``, run:: +the ``share/spack`` directory. For ``bash`` or ``ksh``, run: - . $SPACK_ROOT/share/spack/setup-env.sh +.. code-block:: sh + + export SPACK_ROOT=/path/to/spack + . $SPACK_ROOT/share/spack/setup-env.sh For ``csh`` and ``tcsh`` run: - setenv SPACK_ROOT /path/to/spack - source $SPACK_ROOT/share/spack/setup-env.csh +.. code-block:: csh + + setenv SPACK_ROOT /path/to/spack + source $SPACK_ROOT/share/spack/setup-env.csh ``spack cd`` will then be available. .. _spack-cd: +^^^^^^^^^^^^ ``spack cd`` -~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^ ``spack cd`` allows you to quickly cd to pertinent directories in Spack. Suppose you've staged a package but you want to modify it before you build it: -.. code-block:: sh +.. code-block:: console $ spack stage libelf ==> Trying to fetch from http://www.mr511.de/software/libelf-0.8.13.tar.gz @@ -2955,7 +2934,7 @@ build it: $ pwd /Users/gamblin2/src/spack/var/spack/stage/libelf@0.8.13%gcc@4.8.3 arch=linux-debian7-x86_64/libelf-0.8.13 -``spack cd`` here changed he current working directory to the +``spack cd`` here changed the current working directory to the directory containing the expanded ``libelf`` source code. There are a number of other places you can cd to in the spack directory hierarchy: @@ -2968,24 +2947,26 @@ the main python source directory of your spack install. .. _spack-env: +^^^^^^^^^^^^^ ``spack env`` -~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^ ``spack env`` functions much like the standard unix ``env`` command, but it takes a spec as an argument. You can use it to see the environment variables that will be set when a particular build runs, for example: -.. code-block:: sh +.. code-block:: console $ spack env mpileaks@1.1%intel This will display the entire environment that will be set when the ``mpileaks@1.1%intel`` build runs. -To run commands in a package's build environment, you can simply provided them after the spec argument to ``spack env``: +To run commands in a package's build environment, you can simply +provide them after the spec argument to ``spack env``: -.. code-block:: sh +.. code-block:: console $ spack cd mpileaks@1.1%intel $ spack env mpileaks@1.1%intel ./configure @@ -2993,21 +2974,25 @@ To run commands in a package's build environment, you can simply provided them a This will cd to the build directory and then run ``configure`` in the package's build environment. - .. _spack-location: +^^^^^^^^^^^^^^^^^^ ``spack location`` -~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^ ``spack location`` is the same as ``spack cd`` but it does not require shell support. It simply prints out the path you ask for, rather than -cd'ing to it. In bash, this:: +cd'ing to it. In bash, this: + +.. code-block:: console + + $ cd $(spack location -b <spec>) - cd $(spack location -b <spec>) +is the same as: -is the same as:: +.. code-block:: console - spack cd -b <spec> + $ spack cd -b <spec> ``spack location`` is intended for use in scripts or makefiles that need to know where packages are installed. e.g., in a makefile you @@ -3019,10 +3004,11 @@ might write: CXXFLAGS += -I$DWARF_PREFIX/include CXXFLAGS += -L$DWARF_PREFIX/lib +---------------------------------- Build System Configuration Support ---------------------------------- -Imagine a developer creating a CMake-based (or Autotools) project in a local +Imagine a developer creating a CMake or Autotools-based project in a local directory, which depends on libraries A-Z. Once Spack has installed those dependencies, one would like to run ``cmake`` with appropriate command line and environment so CMake can find them. The ``spack @@ -3031,43 +3017,45 @@ configuration that is essentially the same as how Spack *would have* configured the project. This can be demonstrated with a usage example: -.. code-block:: bash +.. code-block:: console - cd myproject - spack setup myproject@local - mkdir build; cd build - ../spconfig.py .. - make - make install + $ cd myproject + $ spack setup myproject@local + $ mkdir build; cd build + $ ../spconfig.py .. + $ make + $ make install Notes: - * Spack must have ``myproject/package.py`` in its repository for - this to work. - * ``spack setup`` produces the executable script ``spconfig.py`` in - the local directory, and also creates the module file for the - package. ``spconfig.py`` is normally run from the user's - out-of-source build directory. - * The version number given to ``spack setup`` is arbitrary, just - like ``spack diy``. ``myproject/package.py`` does not need to - have any valid downloadable versions listed (typical when a - project is new). - * spconfig.py produces a CMake configuration that *does not* use the - Spack wrappers. Any resulting binaries *will not* use RPATH, - unless the user has enabled it. This is recommended for - development purposes, not production. - * ``spconfig.py`` is human readable, and can serve as a developer - reference of what dependencies are being used. - * ``make install`` installs the package into the Spack repository, - where it may be used by other Spack packages. - * CMake-generated makefiles re-run CMake in some circumstances. Use - of ``spconfig.py`` breaks this behavior, requiring the developer - to manually re-run ``spconfig.py`` when a ``CMakeLists.txt`` file - has changed. +* Spack must have ``myproject/package.py`` in its repository for + this to work. +* ``spack setup`` produces the executable script ``spconfig.py`` in + the local directory, and also creates the module file for the + package. ``spconfig.py`` is normally run from the user's + out-of-source build directory. +* The version number given to ``spack setup`` is arbitrary, just + like ``spack diy``. ``myproject/package.py`` does not need to + have any valid downloadable versions listed (typical when a + project is new). +* spconfig.py produces a CMake configuration that *does not* use the + Spack wrappers. Any resulting binaries *will not* use RPATH, + unless the user has enabled it. This is recommended for + development purposes, not production. +* ``spconfig.py`` is human readable, and can serve as a developer + reference of what dependencies are being used. +* ``make install`` installs the package into the Spack repository, + where it may be used by other Spack packages. +* CMake-generated makefiles re-run CMake in some circumstances. Use + of ``spconfig.py`` breaks this behavior, requiring the developer + to manually re-run ``spconfig.py`` when a ``CMakeLists.txt`` file + has changed. + +^^^^^^^^^^^^ CMakePackage -~~~~~~~~~~~~ +^^^^^^^^^^^^ -In order ot enable ``spack setup`` functionality, the author of +In order to enable ``spack setup`` functionality, the author of ``myproject/package.py`` must subclass from ``CMakePackage`` instead of the standard ``Package`` superclass. Because CMake is standardized, the packager does not need to tell Spack how to run @@ -3079,21 +3067,22 @@ translate variant flags into CMake definitions. For example: .. code-block:: python - def configure_args(self): - spec = self.spec - return [ - '-DUSE_EVERYTRACE=%s' % ('YES' if '+everytrace' in spec else 'NO'), - '-DBUILD_PYTHON=%s' % ('YES' if '+python' in spec else 'NO'), - '-DBUILD_GRIDGEN=%s' % ('YES' if '+gridgen' in spec else 'NO'), - '-DBUILD_COUPLER=%s' % ('YES' if '+coupler' in spec else 'NO'), - '-DUSE_PISM=%s' % ('YES' if '+pism' in spec else 'NO')] + def configure_args(self): + spec = self.spec + return [ + '-DUSE_EVERYTRACE=%s' % ('YES' if '+everytrace' in spec else 'NO'), + '-DBUILD_PYTHON=%s' % ('YES' if '+python' in spec else 'NO'), + '-DBUILD_GRIDGEN=%s' % ('YES' if '+gridgen' in spec else 'NO'), + '-DBUILD_COUPLER=%s' % ('YES' if '+coupler' in spec else 'NO'), + '-DUSE_PISM=%s' % ('YES' if '+pism' in spec else 'NO') + ] If needed, a packager may also override methods defined in ``StagedPackage`` (see below). - +^^^^^^^^^^^^^ StagedPackage -~~~~~~~~~~~~~ +^^^^^^^^^^^^^ ``CMakePackage`` is implemented by subclassing the ``StagedPackage`` superclass, which breaks down the standard ``Package.install()`` @@ -3115,8 +3104,9 @@ and ``install``. Details: ``install()`` method may be accessed via ``self.spec`` and ``self.prefix``. +^^^^^^^^^^^^^ GNU Autotools -~~~~~~~~~~~~~ +^^^^^^^^^^^^^ The ``setup`` functionality is currently only available for CMake-based packages. Extending this functionality to GNU diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 5affd3c7c5..9b5ed367d1 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -476,14 +476,12 @@ def setup_package(pkg, dirty=False): def fork(pkg, function, dirty=False): """Fork a child process to do part of a spack build. - Arguments: + :param pkg: pkg whose environemnt we should set up the forked process for. + :param function: arg-less function to run in the child process. + :param dirty: If True, do NOT clean the environment before building. - pkg -- pkg whose environemnt we should set up the - forked process for. - function -- arg-less function to run in the child process. - dirty -- If True, do NOT clean the environment before building. + Usage:: - Usage: def child_fun(): # do stuff build_env.fork(pkg, child_fun) @@ -498,6 +496,7 @@ def fork(pkg, function, dirty=False): well. If things go well, the child exits and the parent carries on. """ + try: pid = os.fork() except OSError as e: diff --git a/lib/spack/spack/cmd/package-list.py b/lib/spack/spack/cmd/package_list.py index 9ed42de823..42f408af96 100644 --- a/lib/spack/spack/cmd/package-list.py +++ b/lib/spack/spack/cmd/package_list.py @@ -32,8 +32,8 @@ description = "Print a list of all packages in reStructuredText." def github_url(pkg): """Link to a package file on github.""" - url = "https://github.com/llnl/spack/blob/master/var/spack/packages/%s/package.py" - return (url % pkg.name) + url = "https://github.com/LLNL/spack/blob/develop/var/spack/repos/builtin/packages/{0}/package.py" + return url.format(pkg.name) def rst_table(elts): @@ -51,35 +51,41 @@ def print_rst_package_list(): print ".. _package-list:" print + print "============" print "Package List" - print "==================" - + print "============" + print print "This is a list of things you can install using Spack. It is" print "automatically generated based on the packages in the latest Spack" print "release." print - print "Spack currently has %d mainline packages:" % len(pkgs) print print rst_table("`%s`_" % p for p in pkg_names) print - print "-----" # Output some text for each package. for pkg in pkgs: + print "-----" print print ".. _%s:" % pkg.name print + # Must be at least 2 long, breaks for single letter packages like R. + print "-" * max(len(pkg.name), 2) print pkg.name - print "-" * len(pkg.name) - print "Links:" + print "-" * max(len(pkg.name), 2) + print + print "Homepage:" print " * `%s <%s>`__" % (cgi.escape(pkg.homepage), pkg.homepage) + print + print "Spack package:" print " * `%s/package.py <%s>`__" % (pkg.name, github_url(pkg)) print if pkg.versions: print "Versions:" print " " + ", ".join(str(v) for v in reversed(sorted(pkg.versions))) + print for deptype in spack.alldeps: deps = pkg.dependencies_of_type(deptype) @@ -92,7 +98,6 @@ def print_rst_package_list(): print "Description:" print pkg.format_doc(indent=2) print - print "-----" def package_list(parser, args): diff --git a/lib/spack/spack/cmd/test-install.py b/lib/spack/spack/cmd/test_install.py index 8e7173e9a2..8e7173e9a2 100644 --- a/lib/spack/spack/cmd/test-install.py +++ b/lib/spack/spack/cmd/test_install.py diff --git a/lib/spack/spack/cmd/url-parse.py b/lib/spack/spack/cmd/url_parse.py index b8c7c95040..b8c7c95040 100644 --- a/lib/spack/spack/cmd/url-parse.py +++ b/lib/spack/spack/cmd/url_parse.py diff --git a/lib/spack/spack/compilers/clang.py b/lib/spack/spack/compilers/clang.py index 36c91f6670..f7f1bf106b 100644 --- a/lib/spack/spack/compilers/clang.py +++ b/lib/spack/spack/compilers/clang.py @@ -77,18 +77,17 @@ class Clang(Compiler): @classmethod def default_version(cls, comp): """The '--version' option works for clang compilers. - On most platforms, output looks like this:: + On most platforms, output looks like this:: - clang version 3.1 (trunk 149096) - Target: x86_64-unknown-linux-gnu - Thread model: posix + clang version 3.1 (trunk 149096) + Target: x86_64-unknown-linux-gnu + Thread model: posix - On Mac OS X, it looks like this: - - Apple LLVM version 7.0.2 (clang-700.1.81) - Target: x86_64-apple-darwin15.2.0 - Thread model: posix + On Mac OS X, it looks like this:: + Apple LLVM version 7.0.2 (clang-700.1.81) + Target: x86_64-apple-darwin15.2.0 + Thread model: posix """ if comp not in cpr._version_cache: compiler = Executable(comp) diff --git a/lib/spack/spack/compilers/intel.py b/lib/spack/spack/compilers/intel.py index 8531ecd19a..4f0c12140f 100644 --- a/lib/spack/spack/compilers/intel.py +++ b/lib/spack/spack/compilers/intel.py @@ -68,15 +68,15 @@ class Intel(Compiler): @classmethod def default_version(cls, comp): """The '--version' option seems to be the most consistent one - for intel compilers. Output looks like this:: + for intel compilers. Output looks like this:: - icpc (ICC) 12.1.5 20120612 - Copyright (C) 1985-2012 Intel Corporation. All rights reserved. + icpc (ICC) 12.1.5 20120612 + Copyright (C) 1985-2012 Intel Corporation. All rights reserved. - or:: + or:: - ifort (IFORT) 12.1.5 20120612 - Copyright (C) 1985-2012 Intel Corporation. All rights reserved. + ifort (IFORT) 12.1.5 20120612 + Copyright (C) 1985-2012 Intel Corporation. All rights reserved. """ return get_compiler_version( comp, '--version', r'\((?:IFORT|ICC)\) ([^ ]+)') diff --git a/lib/spack/spack/compilers/nag.py b/lib/spack/spack/compilers/nag.py index fdfc078b5e..7beb0efdf5 100644 --- a/lib/spack/spack/compilers/nag.py +++ b/lib/spack/spack/compilers/nag.py @@ -70,10 +70,10 @@ class Nag(Compiler): @classmethod def default_version(self, comp): """The '-V' option works for nag compilers. - Output looks like this:: + Output looks like this:: - NAG Fortran Compiler Release 6.0(Hibiya) Build 1037 - Product NPL6A60NA for x86-64 Linux + NAG Fortran Compiler Release 6.0(Hibiya) Build 1037 + Product NPL6A60NA for x86-64 Linux """ return get_compiler_version( comp, '-V', r'NAG Fortran Compiler Release ([0-9.]+)') diff --git a/lib/spack/spack/compilers/pgi.py b/lib/spack/spack/compilers/pgi.py index 0e4be6e9ba..37ffe44028 100644 --- a/lib/spack/spack/compilers/pgi.py +++ b/lib/spack/spack/compilers/pgi.py @@ -58,11 +58,11 @@ class Pgi(Compiler): @classmethod def default_version(cls, comp): """The '-V' option works for all the PGI compilers. - Output looks like this:: + Output looks like this:: - pgcc 15.10-0 64-bit target on x86-64 Linux -tp sandybridge - The Portland Group - PGI Compilers and Tools - Copyright (c) 2015, NVIDIA CORPORATION. All rights reserved. + pgcc 15.10-0 64-bit target on x86-64 Linux -tp sandybridge + The Portland Group - PGI Compilers and Tools + Copyright (c) 2015, NVIDIA CORPORATION. All rights reserved. """ return get_compiler_version( comp, '-V', r'pg[^ ]* ([^ ]+) \d\d\d?-bit target') diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index c90eff4229..dec43726a0 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -24,26 +24,28 @@ ############################################################################## """This module implements Spack's configuration file handling. +========================= Configuration file scopes -=============================== +========================= When Spack runs, it pulls configuration data from several config directories, each of which contains configuration files. In Spack, there are two configuration scopes: - 1. ``site``: Spack loads site-wide configuration options from +1. ``site``: Spack loads site-wide configuration options from ``$(prefix)/etc/spack/``. - 2. ``user``: Spack next loads per-user configuration options from - ~/.spack/. +2. ``user``: Spack next loads per-user configuration options from + ``~/.spack/``. Spack may read configuration files from both of these locations. When configurations conflict, the user config options take precedence over the site configurations. Each configuration directory may contain several configuration files, such as compilers.yaml or mirrors.yaml. +========================= Configuration file format -=============================== +========================= Configuration files are formatted using YAML syntax. This format is implemented by libyaml (included with Spack as an external module), @@ -63,13 +65,13 @@ Config files are structured as trees, like this ``compiler`` section:: cc: /usr/local/bin/mpixlc ... -In this example, entries like ''compilers'' and ''xlc@12.1'' are used to +In this example, entries like "compilers" and "xlc@12.1" are used to categorize entries beneath them in the tree. At the root of the tree, -entries like ''cc'' and ''cxx'' are specified as name/value pairs. +entries like "cc" and "cxx" are specified as name/value pairs. ``config.get_config()`` returns these trees as nested dicts, but it strips the first level off. So, ``config.get_config('compilers')`` -would return something like this for the above example: +would return something like this for the above example:: { 'chaos_5_x86_64_ib' : { 'gcc@4.4.7' : @@ -84,8 +86,9 @@ would return something like this for the above example: Likewise, the ``mirrors.yaml`` file's first line must be ``mirrors:``, but ``get_config()`` strips that off too. +========== Precedence -=============================== +========== ``config.py`` routines attempt to recursively merge configuration across scopes. So if there are ``compilers.py`` files in both the @@ -99,7 +102,7 @@ first. Sometimes, it is useful to *completely* override a site setting with a user one. To accomplish this, you can use *two* colons at the end of -a key in a configuration file. For example, this: +a key in a configuration file. For example, this:: compilers:: chaos_5_x86_64_ib: @@ -115,8 +118,8 @@ a key in a configuration file. For example, this: Will make Spack take compilers *only* from the user configuration, and the site configuration will be ignored. - """ + import copy import os import re diff --git a/lib/spack/spack/directives.py b/lib/spack/spack/directives.py index 2a151e0374..86acd075cd 100644 --- a/lib/spack/spack/directives.py +++ b/lib/spack/spack/directives.py @@ -288,8 +288,7 @@ def variant(pkg, name, default=False, description=""): @directive('resources') def resource(pkg, **kwargs): - """ - Define an external resource to be fetched and staged when building the + """Define an external resource to be fetched and staged when building the package. Based on the keywords present in the dictionary the appropriate FetchStrategy will be used for the resource. Resources are fetched and staged in their own folder inside spack stage area, and then moved into @@ -298,11 +297,11 @@ def resource(pkg, **kwargs): List of recognized keywords: * 'when' : (optional) represents the condition upon which the resource is - needed + needed * 'destination' : (optional) path where to move the resource. This path - must be relative to the main package stage area. + must be relative to the main package stage area. * 'placement' : (optional) gives the possibility to fine tune how the - resource is moved into the main package stage area. + resource is moved into the main package stage area. """ when = kwargs.get('when', pkg.name) destination = kwargs.get('destination', "") diff --git a/lib/spack/spack/environment.py b/lib/spack/spack/environment.py index 613ece2f45..b0eadef7a6 100644 --- a/lib/spack/spack/environment.py +++ b/lib/spack/spack/environment.py @@ -261,17 +261,14 @@ class EnvironmentModifications(object): @staticmethod def from_sourcing_files(*args, **kwargs): - """ - Creates an instance of EnvironmentModifications that, if executed, + """Creates an instance of EnvironmentModifications that, if executed, has the same effect on the environment as sourcing the files passed as parameters - Args: - *args: list of files to be sourced - - Returns: - instance of EnvironmentModifications + :param \*args: list of files to be sourced + :rtype: instance of EnvironmentModifications """ + env = EnvironmentModifications() # Check if the files are actually there if not all(os.path.isfile(file) for file in args): diff --git a/lib/spack/spack/mirror.py b/lib/spack/spack/mirror.py index f053e4405f..2361fb5448 100644 --- a/lib/spack/spack/mirror.py +++ b/lib/spack/spack/mirror.py @@ -120,27 +120,28 @@ def suggest_archive_basename(resource): def create(path, specs, **kwargs): """Create a directory to be used as a spack mirror, and fill it with - package archives. - - Arguments: - path Path to create a mirror directory hierarchy in. - specs Any package versions matching these specs will be added - to the mirror. - - Keyword args: - no_checksum: If True, do not checkpoint when fetching (default False) - num_versions: Max number of versions to fetch per spec, - if spec is ambiguous (default is 0 for all of them) - - Return Value: - Returns a tuple of lists: (present, mirrored, error) - * present: Package specs that were already present. - * mirrored: Package specs that were successfully mirrored. - * error: Package specs that failed to mirror due to some error. - - This routine iterates through all known package versions, and - it creates specs for those versions. If the version satisfies any spec - in the specs list, it is downloaded and added to the mirror. + package archives. + + Arguments: + path: Path to create a mirror directory hierarchy in. + specs: Any package versions matching these specs will be added \ + to the mirror. + + Keyword args: + no_checksum: If True, do not checkpoint when fetching (default False) + num_versions: Max number of versions to fetch per spec, \ + if spec is ambiguous (default is 0 for all of them) + + Return Value: + Returns a tuple of lists: (present, mirrored, error) + + * present: Package specs that were already present. + * mirrored: Package specs that were successfully mirrored. + * error: Package specs that failed to mirror due to some error. + + This routine iterates through all known package versions, and + it creates specs for those versions. If the version satisfies any spec + in the specs list, it is downloaded and added to the mirror. """ # Make sure nothing is in the way. if os.path.isfile(path): diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 8c1204402a..3ffedbbad9 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -84,9 +84,9 @@ class Package(object): with the package itself. Packages are written in pure python. Packages are all submodules of spack.packages. If spack is installed - in $prefix, all of its python files are in $prefix/lib/spack. Most - of them are in the spack module, so all the packages live in - $prefix/lib/spack/spack/packages. + in ``$prefix``, all of its python files are in ``$prefix/lib/spack``. + Most of them are in the spack module, so all the packages live in + ``$prefix/lib/spack/spack/packages``. All you have to do to create a package is make a new subclass of Package in this directory. Spack automatically scans the python files there @@ -95,7 +95,7 @@ class Package(object): **An example package** Let's look at the cmake package to start with. This package lives in - $prefix/lib/spack/spack/packages/cmake.py: + ``$prefix/var/spack/repos/builtin/packages/cmake/package.py``: .. code-block:: python @@ -118,19 +118,21 @@ class Package(object): 1. The module name, ``cmake``. * User will refers to this name, e.g. 'spack install cmake'. - * Corresponds to the name of the file, 'cmake.py', and it can - include ``_``, ``-``, and numbers (it can even start with a + * It can include ``_``, ``-``, and numbers (it can even start with a number). 2. The class name, "Cmake". This is formed by converting `-` or ``_`` in the module name to camel case. If the name starts with a number, we prefix the class name with ``_``. Examples: - Module Name Class Name - foo_bar FooBar - docbook-xml DocbookXml - FooBar Foobar - 3proxy _3proxy + =========== ========== + Module Name Class Name + =========== ========== + foo_bar FooBar + docbook-xml DocbookXml + FooBar Foobar + 3proxy _3proxy + =========== ========== The class name is what spack looks for when it loads a package module. @@ -139,30 +141,30 @@ class Package(object): Aside from proper naming, here is the bare minimum set of things you need when you make a package: - homepage - informational URL, so that users know what they're - installing. + homepage: + informational URL, so that users know what they're + installing. - url or url_for_version(self, version) + url or url_for_version(self, version): If url, then the URL of the source archive that spack will fetch. If url_for_version(), then a method returning the URL required to fetch a particular version. - install() - This function tells spack how to build and install the - software it downloaded. + install(): + This function tells spack how to build and install the + software it downloaded. **Optional Attributes** You can also optionally add these attributes, if needed: - list_url + list_url: Webpage to scrape for available version strings. Default is the directory containing the tarball; use this if the default isn't correct so that invoking 'spack versions' will work for this package. - url_version(self, version) + url_version(self, version): When spack downloads packages at particular versions, it just converts version to string with str(version). Override this if your package needs special version formatting in its URL. boost @@ -179,12 +181,15 @@ class Package(object): **spack create** - Most software comes in nicely packaged tarballs, like this one: - http://www.cmake.org/files/v2.8/cmake-2.8.10.2.tar.gz + Most software comes in nicely packaged tarballs, like this one + + http://www.cmake.org/files/v2.8/cmake-2.8.10.2.tar.gz + Taking a page from homebrew, spack deduces pretty much everything it - needs to know from the URL above. If you simply type this: + needs to know from the URL above. If you simply type this:: spack create http://www.cmake.org/files/v2.8/cmake-2.8.10.2.tar.gz + Spack will download the tarball, generate an md5 hash, figure out the version and the name of the package from the URL, and create a new package file for you with all the names and attributes set correctly. @@ -216,7 +221,6 @@ class Package(object): you can just run configure or cmake without any additional arguments and it will find the dependencies automatically. - **The Install Function** The install function is designed so that someone not too terribly familiar @@ -241,13 +245,12 @@ class Package(object): add_commands_to_module() function in this class. This is where most of them are created and set on the module. - **Parallel Builds** By default, Spack will run make in parallel when you run make() in your install function. Spack figures out how many cores are available on - your system and runs make with -j<cores>. If you do not want this behavior, - you can explicitly mark a package not to use parallel make: + your system and runs make with -j<cores>. If you do not want this + behavior, you can explicitly mark a package not to use parallel make: .. code-block:: python @@ -257,14 +260,15 @@ class Package(object): ... This changes thd default behavior so that make is sequential. If you still - want to build some parts in parallel, you can do this in your install function: + want to build some parts in parallel, you can do this in your install + function: .. code-block:: python make(parallel=True) - Likewise, if you do not supply parallel = True in your Package, you can keep - the default parallel behavior and run make like this when you want a + Likewise, if you do not supply parallel = True in your Package, you can + keep the default parallel behavior and run make like this when you want a sequential build: .. code-block:: python @@ -295,14 +299,13 @@ class Package(object): p.do_restage() # removes the build directory and # re-expands the archive. - The convention used here is that a do_* function is intended to be called - internally by Spack commands (in spack.cmd). These aren't for package - writers to override, and doing so may break the functionality of the Package - class. + The convention used here is that a ``do_*`` function is intended to be + called internally by Spack commands (in spack.cmd). These aren't for + package writers to override, and doing so may break the functionality + of the Package class. Package creators override functions like install() (all of them do this), clean() (some of them do this), and others to provide custom behavior. - """ # # These are default values for instance variables. @@ -862,19 +865,21 @@ class Package(object): Package implementations should override install() to describe their build process. - Args: - keep_prefix -- Keep install prefix on failure. By default, destroys it. - keep_stage -- By default, stage is destroyed only if there are no - exceptions during build. Set to True to keep the stage - even with exceptions. - ignore_deps -- Don't install dependencies before installing this + :param keep_prefix: Keep install prefix on failure. By default, \ + destroys it. + :param keep_stage: By default, stage is destroyed only if there are \ + no exceptions during build. Set to True to keep the stage + even with exceptions. + :param ignore_deps: Don't install dependencies before installing this \ package - fake -- Don't really build -- install fake stub files instead. - skip_patch -- Skip patch stage of build if True. - verbose -- Display verbose build output (by default, suppresses it) - dirty -- Don't clean the build environment before installing. - make_jobs -- Number of make jobs to use for install. Default is ncpus - run_tests -- Runn tests within the package's install() + :param fake: Don't really build; install fake stub files instead. + :param skip_patch: Skip patch stage of build if True. + :param verbose: Display verbose build output (by default, suppresses \ + it) + :param dirty: Don't clean the build environment before installing. + :param make_jobs: Number of make jobs to use for install. Default is \ + ncpus + :param run_tests: Run tests within the package's install() """ if not self.spec.concrete: raise ValueError("Can only install concrete packages: %s." @@ -1110,20 +1115,20 @@ class Package(object): def setup_dependent_environment(self, spack_env, run_env, dependent_spec): """Set up the environment of packages that depend on this one. - This is similar to `setup_environment`, but it is used to + This is similar to ``setup_environment``, but it is used to modify the compile and runtime environments of packages that *depend* on this one. This gives packages like Python and others that follow the extension model a way to implement common environment or compile-time settings for dependencies. - By default, this delegates to self.setup_environment() + By default, this delegates to ``self.setup_environment()`` - Example : + Example: 1. Installing python modules generally requires - `PYTHONPATH` to point to the lib/pythonX.Y/site-packages - directory in the module's install prefix. This could - set that variable. + `PYTHONPATH` to point to the lib/pythonX.Y/site-packages + directory in the module's install prefix. This could + set that variable. Args: @@ -1142,7 +1147,6 @@ class Package(object): This is useful if there are some common steps to installing all extensions for a certain package. - """ self.setup_environment(spack_env, run_env) diff --git a/lib/spack/spack/repository.py b/lib/spack/spack/repository.py index 2d8dc39648..1d4cddaa0f 100644 --- a/lib/spack/spack/repository.py +++ b/lib/spack/spack/repository.py @@ -198,15 +198,19 @@ class RepoPath(object): def get_repo(self, namespace, default=NOT_PROVIDED): """Get a repository by namespace. - Arguments - namespace - Look up this namespace in the RepoPath, and return - it if found. - - Optional Arguments - default - If default is provided, return it when the namespace - isn't found. If not, raise an UnknownNamespaceError. + + Arguments: + + namespace: + + Look up this namespace in the RepoPath, and return it if found. + + Optional Arguments: + + default: + + If default is provided, return it when the namespace + isn't found. If not, raise an UnknownNamespaceError. """ fullspace = '%s.%s' % (self.super_namespace, namespace) if fullspace not in self.by_namespace: diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 316b7d6a8c..d8a7cf9d7b 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -2085,7 +2085,7 @@ class Spec(object): $# 7-char prefix of DAG hash with '-' prefix $$ $ - You can also use full-string versions, which elide the prefixes: + You can also use full-string versions, which elide the prefixes:: ${PACKAGE} Package name ${VERSION} Version @@ -2101,17 +2101,17 @@ class Spec(object): ${SPACK_INSTALL} The default spack install directory, ${SPACK_PREFIX}/opt - Optionally you can provide a width, e.g. $20_ for a 20-wide name. + Optionally you can provide a width, e.g. ``$20_`` for a 20-wide name. Like printf, you can provide '-' for left justification, e.g. - $-20_ for a left-justified name. + ``$-20_`` for a left-justified name. Anything else is copied verbatim into the output stream. *Example:* ``$_$@$+`` translates to the name, version, and options of the package, but no dependencies, arch, or compiler. - TODO: allow, e.g., $6# to customize short hash length - TODO: allow, e.g., $## for full hash. + TODO: allow, e.g., ``$6#`` to customize short hash length + TODO: allow, e.g., ``$##`` for full hash. """ color = kwargs.get('color', False) length = len(format_string) diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index 7676cb9ab6..c0705a89c8 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -49,16 +49,14 @@ class Stage(object): some source code is downloaded and built before being installed. It handles fetching the source code, either as an archive to be expanded or by checking it out of a repository. A stage's - lifecycle looks like this: + lifecycle looks like this:: - ``` - with Stage() as stage: # Context manager creates and destroys the - # stage directory - stage.fetch() # Fetch a source archive into the stage. - stage.expand_archive() # Expand the source archive. - <install> # Build and install the archive. (handled by - # user of Stage) - ``` + with Stage() as stage: # Context manager creates and destroys the + # stage directory + stage.fetch() # Fetch a source archive into the stage. + stage.expand_archive() # Expand the source archive. + <install> # Build and install the archive. + # (handled by user of Stage) When used as a context manager, the stage is automatically destroyed if no exception is raised by the context. If an @@ -66,19 +64,17 @@ class Stage(object): destroyed, for potential reuse later. You can also use the stage's create/destroy functions manually, - like this: - - ``` - stage = Stage() - try: - stage.create() # Explicitly create the stage directory. - stage.fetch() # Fetch a source archive into the stage. - stage.expand_archive() # Expand the source archive. - <install> # Build and install the archive. (handled by - # user of Stage) - finally: - stage.destroy() # Explicitly destroy the stage directory. - ``` + like this:: + + stage = Stage() + try: + stage.create() # Explicitly create the stage directory. + stage.fetch() # Fetch a source archive into the stage. + stage.expand_archive() # Expand the source archive. + <install> # Build and install the archive. + # (handled by user of Stage) + finally: + stage.destroy() # Explicitly destroy the stage directory. If spack.use_tmp_stage is True, spack will attempt to create stages in a tmp directory. Otherwise, stages are created directly diff --git a/lib/spack/spack/test/cmd/test_install.py b/lib/spack/spack/test/cmd/test_install.py index 39287d5d6d..4734fe1267 100644 --- a/lib/spack/spack/test/cmd/test_install.py +++ b/lib/spack/spack/test/cmd/test_install.py @@ -30,6 +30,7 @@ import contextlib import spack import spack.cmd +from spack.cmd import test_install FILE_REGISTRY = collections.defaultdict(StringIO.StringIO) @@ -51,11 +52,6 @@ def mock_open(filename, mode): handle.close() -# The use of __import__ is necessary to maintain a name with hyphen (which -# cannot be an identifier in python) -test_install = __import__("spack.cmd.test-install", fromlist=['test_install']) - - class MockSpec(object): def __init__(self, name, version, hashStr=None): diff --git a/lib/spack/spack/test/git_fetch.py b/lib/spack/spack/test/git_fetch.py index 0d1a8fe949..7aff98cc54 100644 --- a/lib/spack/spack/test/git_fetch.py +++ b/lib/spack/spack/test/git_fetch.py @@ -56,12 +56,13 @@ class GitFetchTest(MockPackagesTest): def try_fetch(self, rev, test_file, args): """Tries to: - 1. Fetch the repo using a fetch strategy constructed with - supplied args. - 2. Check if the test_file is in the checked out repository. - 3. Assert that the repository is at the revision supplied. - 4. Add and remove some files, then reset the repo, and - ensure it's all there again. + + 1. Fetch the repo using a fetch strategy constructed with + supplied args. + 2. Check if the test_file is in the checked out repository. + 3. Assert that the repository is at the revision supplied. + 4. Add and remove some files, then reset the repo, and + ensure it's all there again. """ self.pkg.versions[ver('git')] = args diff --git a/lib/spack/spack/test/hg_fetch.py b/lib/spack/spack/test/hg_fetch.py index 44af6730a1..03e35ea093 100644 --- a/lib/spack/spack/test/hg_fetch.py +++ b/lib/spack/spack/test/hg_fetch.py @@ -52,12 +52,13 @@ class HgFetchTest(MockPackagesTest): def try_fetch(self, rev, test_file, args): """Tries to: - 1. Fetch the repo using a fetch strategy constructed with - supplied args. - 2. Check if the test_file is in the checked out repository. - 3. Assert that the repository is at the revision supplied. - 4. Add and remove some files, then reset the repo, and - ensure it's all there again. + + 1. Fetch the repo using a fetch strategy constructed with + supplied args. + 2. Check if the test_file is in the checked out repository. + 3. Assert that the repository is at the revision supplied. + 4. Add and remove some files, then reset the repo, and + ensure it's all there again. """ self.pkg.versions[ver('hg')] = args diff --git a/lib/spack/spack/test/mirror.py b/lib/spack/spack/test/mirror.py index d6d7b30b7c..f9ff190468 100644 --- a/lib/spack/spack/test/mirror.py +++ b/lib/spack/spack/test/mirror.py @@ -52,9 +52,10 @@ class MirrorTest(MockPackagesTest): def set_up_package(self, name, MockRepoClass, url_attr): """Set up a mock package to be mirrored. - Each package needs us to: - 1. Set up a mock repo/archive to fetch from. - 2. Point the package's version args at that repo. + Each package needs us to: + + 1. Set up a mock repo/archive to fetch from. + 2. Point the package's version args at that repo. """ # Set up packages to point at mock repos. spec = Spec(name) diff --git a/lib/spack/spack/test/provider_index.py b/lib/spack/spack/test/provider_index.py index 9847dd05a6..d785038899 100644 --- a/lib/spack/spack/test/provider_index.py +++ b/lib/spack/spack/test/provider_index.py @@ -24,7 +24,7 @@ ############################################################################## """Tests for provider index cache files. -Tests assume that mock packages provide this: +Tests assume that mock packages provide this:: {'blas': { blas: set([netlib-blas, openblas, openblas-with-lapack])}, diff --git a/lib/spack/spack/test/svn_fetch.py b/lib/spack/spack/test/svn_fetch.py index 9ef7593ed1..01ffc488a7 100644 --- a/lib/spack/spack/test/svn_fetch.py +++ b/lib/spack/spack/test/svn_fetch.py @@ -63,12 +63,13 @@ class SvnFetchTest(MockPackagesTest): def try_fetch(self, rev, test_file, args): """Tries to: - 1. Fetch the repo using a fetch strategy constructed with - supplied args. - 2. Check if the test_file is in the checked out repository. - 3. Assert that the repository is at the revision supplied. - 4. Add and remove some files, then reset the repo, and - ensure it's all there again. + + 1. Fetch the repo using a fetch strategy constructed with + supplied args. + 2. Check if the test_file is in the checked out repository. + 3. Assert that the repository is at the revision supplied. + 4. Add and remove some files, then reset the repo, and + ensure it's all there again. """ self.pkg.versions[ver('svn')] = args diff --git a/lib/spack/spack/util/pattern.py b/lib/spack/spack/util/pattern.py index c36445193c..2b7f06d46e 100644 --- a/lib/spack/spack/util/pattern.py +++ b/lib/spack/spack/util/pattern.py @@ -31,15 +31,15 @@ def composite(interface=None, method_list=None, container=list): """Returns a class decorator that patches a class adding all the methods it needs to be a composite for a given interface. - :param interface: class exposing the interface to which the composite - object must conform. Only non-private and non-special methods will be - taken into account + :param interface: class exposing the interface to which the composite \ + object must conform. Only non-private and non-special methods will \ + be taken into account :param method_list: names of methods that should be part of the composite - :param container: container for the composite object (default = list). - Must fulfill the MutableSequence contract. The composite class will expose - the container API to manage object composition + :param container: container for the composite object (default = list). \ + Must fulfill the MutableSequence contract. The composite class will \ + expose the container API to manage object composition :return: class decorator """ diff --git a/lib/spack/spack/version.py b/lib/spack/spack/version.py index e1311eb0d9..bc96dcd716 100644 --- a/lib/spack/spack/version.py +++ b/lib/spack/spack/version.py @@ -155,11 +155,12 @@ class Version(object): @coerced def satisfies(self, other): - """A Version 'satisfies' another if it is at least as specific and has a - common prefix. e.g., we want gcc@4.7.3 to satisfy a request for - gcc@4.7 so that when a user asks to build with gcc@4.7, we can find - a suitable compiler. + """A Version 'satisfies' another if it is at least as specific and has + a common prefix. e.g., we want gcc@4.7.3 to satisfy a request for + gcc@4.7 so that when a user asks to build with gcc@4.7, we can find + a suitable compiler. """ + nself = len(self.version) nother = len(other.version) return nother <= nself and self.version[:nother] == other.version @@ -388,12 +389,12 @@ class VersionRange(object): @coerced def satisfies(self, other): - """ - A VersionRange satisfies another if some version in this range + """A VersionRange satisfies another if some version in this range would satisfy some version in the other range. To do this it must either: - a) Overlap with the other range - b) The start of this range satisfies the end of the other range. + + a) Overlap with the other range + b) The start of this range satisfies the end of the other range. This is essentially the same as overlaps(), but overlaps assumes that its arguments are specific. That is, 4.7 is interpreted as @@ -401,6 +402,7 @@ class VersionRange(object): by 4.7.3.5, etc. Rationale: + If a user asks for gcc@4.5:4.7, and a package is only compatible with gcc@4.7.3:4.8, then that package should be able to build under the constraints. Just using overlaps() would not work here. diff --git a/share/spack/qa/changed_files b/share/spack/qa/changed_files new file mode 100755 index 0000000000..9c60b3b20b --- /dev/null +++ b/share/spack/qa/changed_files @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# +# Description: +# Returns a list of changed files. +# +# Usage: +# changed_files [<directory> ...] +# changed_files [<file> ...] +# changed_files ["*.<extension>" ...] +# +# Options: +# Directories, files, or globs to search for changed files. +# + +# Move to root directory of Spack +# Allows script to be run from anywhere +SPACK_ROOT="$(dirname "$0")/../../.." +cd "$SPACK_ROOT" + +# Add changed files that have been committed since branching off of develop +changed=($(git diff --name-only --find-renames develop... -- "$@")) +# Add changed files that have been staged but not yet committed +changed+=($(git diff --name-only --find-renames --cached -- "$@")) +# Add changed files that are unstaged +changed+=($(git diff --name-only --find-renames -- "$@")) +# Add new files that are untracked +changed+=($(git ls-files --exclude-standard --other -- "$@")) + +# Return array +# Ensure that each file in the array is unique +printf '%s\n' "${changed[@]}" | sort -u diff --git a/share/spack/qa/check_dependencies b/share/spack/qa/check_dependencies new file mode 100755 index 0000000000..08fad9cdc9 --- /dev/null +++ b/share/spack/qa/check_dependencies @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +# +# Description: +# Check to see if dependencies are installed. +# If not, warn the user and tell them how to +# install these dependencies. +# +# Usage: +# check-deps <dep> ... +# +# Options: +# One or more dependencies. Must use name of binary. + +for dep in "$@"; do + if ! which $dep &> /dev/null; then + # Map binary name to package name + case $dep in + sphinx-apidoc|sphinx-build) + spack_package=py-sphinx + pip_package=sphinx + ;; + coverage) + spack_package=py-coverage + pip_package=coverage + ;; + flake8) + spack_package=py-flake8 + pip_package=flake8 + ;; + git) + spack_package=git + ;; + hg) + spack_package=mercurial + pip_package=mercurial + ;; + svn) + spack_package=subversion + ;; + *) + spack_package=$dep + pip_package=$dep + ;; + esac + + echo "ERROR: $dep is required to run this script." + echo + + if [[ $spack_package ]]; then + echo "To install with Spack, run:" + echo " $ spack install $spack_package" + fi + + if [[ $pip_package ]]; then + echo "To install with pip, run:" + echo " $ pip install $pip_package" + fi + + if [[ $spack_package || $pip_package ]]; then + echo "Then add the bin directory to your PATH." + fi + + exit 1 + fi +done + +echo "Dependencies found." diff --git a/share/spack/qa/run-doc-tests b/share/spack/qa/run-doc-tests new file mode 100755 index 0000000000..96b76a216e --- /dev/null +++ b/share/spack/qa/run-doc-tests @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# +# Description: +# Builds Spack documentation and checks for +# possible syntax errors. Treats warnings as +# fatal errors. +# +# Usage: +# run-doc-tests +# +# Notes: +# Requires sphinx, git, mercurial, and subversion. +# + +QA_DIR="$(dirname "$0")" +SPACK_ROOT="$QA_DIR/../../.." +DOC_DIR="$SPACK_ROOT/lib/spack/docs" + +# Array of dependencies +deps=( + sphinx-apidoc + sphinx-build + git + hg + svn +) + +# Check for dependencies +"$QA_DIR/check_dependencies" "${deps[@]}" || exit 1 + +# Add Spack to the PATH. +export PATH="$SPACK_ROOT/bin:$PATH" + +# Move to documentation directory +# Allows script to be run from anywhere +cd "$DOC_DIR" + +# Cleanup temporary files upon exit or when script is killed +trap 'make clean --silent' EXIT SIGINT SIGTERM + +# Treat warnings as fatal errors +make SPHINXOPTS=-W + diff --git a/share/spack/qa/run-flake8 b/share/spack/qa/run-flake8 deleted file mode 100755 index ffc82313a5..0000000000 --- a/share/spack/qa/run-flake8 +++ /dev/null @@ -1,86 +0,0 @@ -#!/bin/bash -# -# This script runs source code style checks on Spack. -# -# To run it, you'll need to have the Python flake8 installed locally. -# -PYTHONPATH=./lib/spack:$PYTHONPATH - -flake8="$(which flake8)" -if [[ ! $flake8 ]]; then - echo "ERROR: flake8 is required to run this script." - exit 1 -fi - -# Move to Spack root; allows script to be run from anywhere -cd "$(dirname "$0")/../../.." - -# Add changed files that have been committed since branching off of develop -changed=($(git diff --name-only --find-renames develop... -- '*.py')) -# Add changed files that have been staged but not yet committed -changed+=($(git diff --name-only --find-renames --cached -- '*.py')) -# Add changed files that are unstaged -changed+=($(git diff --name-only --find-renames -- '*.py')) -# Add new files that are untracked -changed+=($(git ls-files --exclude-standard --other -- '*.py')) - -# Ensure that each file in the array is unique -changed=($(printf '%s\n' "${changed[@]}" | sort -u)) - -function cleanup { - # Restore original package files after modifying them. - for file in "${changed[@]}"; do - if [[ -e "${file}.sbak~" ]]; then - mv "${file}.sbak~" "${file}" - fi - done -} - -# Cleanup temporary files upon exit or when script is killed -trap cleanup EXIT SIGINT SIGTERM - -# Add approved style exemptions to the changed packages. -for file in "${changed[@]}"; do - # Make a backup to restore later - cp "$file" "$file.sbak~" - - # - # Exemptions for package.py files - # - if [[ $file = *package.py ]]; then - # Exempt lines with urls and descriptions from overlong line errors. - perl -i -pe 's/^(\s*homepage\s*=.*)$/\1 # NOQA: ignore=E501/' $file - perl -i -pe 's/^(\s*url\s*=.*)$/\1 # NOQA: ignore=E501/' $file - perl -i -pe 's/^(\s*version\(.*\).*)$/\1 # NOQA: ignore=E501/' $file - perl -i -pe 's/^(\s*variant\(.*\).*)$/\1 # NOQA: ignore=E501/' $file - perl -i -pe 's/^(\s*depends_on\(.*\).*)$/\1 # NOQA: ignore=E501/' $file - perl -i -pe 's/^(\s*extends\(.*\).*)$/\1 # NOQA: ignore=E501/' $file - - # Exempt '@when' decorated functions from redefinition errors. - perl -i -pe 's/^(\s*\@when\(.*\).*)$/\1 # NOQA: ignore=F811/' $file - fi - - # - # Exemptions for all files - # - perl -i -pe 's/^(.*(https?|file)\:.*)$/\1 # NOQA: ignore=E501/' $file -done - -if [[ "${changed[@]}" ]]; then - echo ======================================================= - echo flake8: running flake8 code checks on spack. - echo - echo Modified files: - echo "${changed[@]}" | perl -pe 's/^/ /;s/ +/\n /g' - echo ======================================================= - if flake8 --format pylint "${changed[@]}"; then - echo "Flake8 checks were clean." - else - echo "Flake8 found errors." - exit 1 - fi -else - echo No core framework files modified. -fi - -exit 0 diff --git a/share/spack/qa/run-flake8-tests b/share/spack/qa/run-flake8-tests new file mode 100755 index 0000000000..350ef3161f --- /dev/null +++ b/share/spack/qa/run-flake8-tests @@ -0,0 +1,89 @@ +#!/usr/bin/env bash +# +# Description: +# Runs source code style checks on Spack. +# See $SPACK_ROOT/.flake8 for a list of +# approved exceptions. +# +# Usage: +# run-flake8-tests +# +# Notes: +# Requires flake8. +# + +QA_DIR="$(dirname "$0")" +SPACK_ROOT="$QA_DIR/../../.." + +# Array of dependencies +deps=( + flake8 +) + +# Check for dependencies +"$QA_DIR/check_dependencies" "${deps[@]}" || exit 1 + +# Move to root directory of Spack +# Allows script to be run from anywhere +cd "$SPACK_ROOT" + +# Gather array of changed files +changed=($("$QA_DIR/changed_files" "*.py")) + +# Exit if no Python files were modified +if [[ ! "${changed[@]}" ]]; then + echo "No Python files were modified." + exit 0 +fi + +function cleanup { + # Restore original package files after modifying them. + for file in "${changed[@]}"; do + if [[ -e "${file}.sbak~" ]]; then + mv "${file}.sbak~" "${file}" + fi + done +} + +# Cleanup temporary files upon exit or when script is killed +trap cleanup EXIT SIGINT SIGTERM + +# Add approved style exemptions to the changed packages. +for file in "${changed[@]}"; do + # Make a backup to restore later + cp "$file" "$file.sbak~" + + # + # Exemptions for package.py files + # + if [[ $file = *package.py ]]; then + # Exempt lines with urls and descriptions from overlong line errors. + perl -i -pe 's/^(\s*homepage\s*=.*)$/\1 # NOQA: ignore=E501/' "$file" + perl -i -pe 's/^(\s*url\s*=.*)$/\1 # NOQA: ignore=E501/' "$file" + perl -i -pe 's/^(\s*version\(.*\).*)$/\1 # NOQA: ignore=E501/' "$file" + perl -i -pe 's/^(\s*variant\(.*\).*)$/\1 # NOQA: ignore=E501/' "$file" + perl -i -pe 's/^(\s*depends_on\(.*\).*)$/\1 # NOQA: ignore=E501/' "$file" + perl -i -pe 's/^(\s*extends\(.*\).*)$/\1 # NOQA: ignore=E501/' "$file" + + # Exempt '@when' decorated functions from redefinition errors. + perl -i -pe 's/^(\s*\@when\(.*\).*)$/\1 # NOQA: ignore=F811/' "$file" + fi + + # + # Exemptions for all files + # + perl -i -pe 's/^(.*(https?|file)\:.*)$/\1 # NOQA: ignore=E501/' $file +done + +echo ======================================================= +echo flake8: running flake8 code checks on spack. +echo +echo Modified files: +echo "${changed[@]}" | perl -pe 's/^/ /;s/ +/\n /g' +echo ======================================================= +if flake8 --format pylint "${changed[@]}"; then + echo "Flake8 checks were clean." +else + echo "Flake8 found errors." + exit 1 +fi diff --git a/share/spack/qa/run-unit-tests b/share/spack/qa/run-unit-tests index 33fb1bfae2..6da919e18d 100755 --- a/share/spack/qa/run-unit-tests +++ b/share/spack/qa/run-unit-tests @@ -1,20 +1,46 @@ #!/usr/bin/env bash # -# This script runs Spack unit tests. +# Description: +# Runs Spack unit tests. # -# It should be executed from the top-level directory of the repo, -# e.g.: +# Usage: +# run-unit-tests [test ...] # -# share/spack/qa/run-unit-tests +# Options: +# Optionally add one or more unit tests +# to only run these tests. # -# To run it, you'll need to have the Python coverage installed locally. +# Notes: +# Requires coverage, git, mercurial, and subversion. # -# Regular spack setup and tests -. ./share/spack/setup-env.sh +QA_DIR="$(dirname "$0")" +SPACK_ROOT="$QA_DIR/../../.." + +# Array of dependencies +deps=( + coverage + git + hg + svn +) + +# Check for dependencies +"$QA_DIR/check_dependencies" "${deps[@]}" || exit 1 + +# Add Spack to the PATH. +export PATH="$SPACK_ROOT/bin:$PATH" + +# Move to root directory of Spack +# Allows script to be run from anywhere +cd "$SPACK_ROOT" + +# Run integration tests +# TODO: should these be separated into a different test suite? +source "$SPACK_ROOT/share/spack/setup-env.sh" spack compilers spack config get compilers spack install -v libdwarf # Run unit tests with code coverage -coverage run bin/spack test +coverage run bin/spack test "$@" diff --git a/var/spack/repos/builtin/packages/adios/package.py b/var/spack/repos/builtin/packages/adios/package.py index e2e3b7d6b7..01834383b8 100644 --- a/var/spack/repos/builtin/packages/adios/package.py +++ b/var/spack/repos/builtin/packages/adios/package.py @@ -27,11 +27,10 @@ from spack import * class Adios(Package): - """ - The Adaptable IO System (ADIOS) provides a simple, + """The Adaptable IO System (ADIOS) provides a simple, flexible way for scientists to describe the data in their code that may need to be written, - read, or processed outside of the running simulation + read, or processed outside of the running simulation. """ homepage = "http://www.olcf.ornl.gov/center-projects/adios/" diff --git a/var/spack/repos/builtin/packages/antlr/package.py b/var/spack/repos/builtin/packages/antlr/package.py index 891061c62f..804460ce14 100644 --- a/var/spack/repos/builtin/packages/antlr/package.py +++ b/var/spack/repos/builtin/packages/antlr/package.py @@ -26,6 +26,11 @@ from spack import * class Antlr(Package): + """ANTLR (ANother Tool for Language Recognition) is a powerful parser + generator for reading, processing, executing, or translating structured + text or binary files. It's widely used to build languages, tools, and + frameworks. From a grammar, ANTLR generates a parser that can build and + walk parse trees.""" homepage = "http://www.antlr.org" url = "https://github.com/antlr/antlr/tarball/v2.7.7" diff --git a/var/spack/repos/builtin/packages/arpack-ng/package.py b/var/spack/repos/builtin/packages/arpack-ng/package.py index 728c2345f2..0e71125d41 100644 --- a/var/spack/repos/builtin/packages/arpack-ng/package.py +++ b/var/spack/repos/builtin/packages/arpack-ng/package.py @@ -26,8 +26,7 @@ from spack import * class ArpackNg(Package): - """ - ARPACK-NG is a collection of Fortran77 subroutines designed to solve large + """ARPACK-NG is a collection of Fortran77 subroutines designed to solve large scale eigenvalue problems. Important Features: @@ -53,6 +52,7 @@ class ArpackNg(Package): arpack-ng is replacing arpack almost everywhere. """ + homepage = 'https://github.com/opencollab/arpack-ng' url = 'https://github.com/opencollab/arpack-ng/archive/3.3.0.tar.gz' diff --git a/var/spack/repos/builtin/packages/asciidoc/package.py b/var/spack/repos/builtin/packages/asciidoc/package.py index be8db684f5..811d34afa8 100644 --- a/var/spack/repos/builtin/packages/asciidoc/package.py +++ b/var/spack/repos/builtin/packages/asciidoc/package.py @@ -26,8 +26,9 @@ from spack import * class Asciidoc(Package): - """ A presentable text document format for writing articles, UNIX man + """A presentable text document format for writing articles, UNIX man pages and other small to medium sized documents.""" + homepage = "http://asciidoc.org" url = "http://downloads.sourceforge.net/project/asciidoc/asciidoc/8.6.9/asciidoc-8.6.9.tar.gz" diff --git a/var/spack/repos/builtin/packages/astyle/package.py b/var/spack/repos/builtin/packages/astyle/package.py index 815c184577..eecdcfdad7 100644 --- a/var/spack/repos/builtin/packages/astyle/package.py +++ b/var/spack/repos/builtin/packages/astyle/package.py @@ -26,10 +26,10 @@ from spack import * class Astyle(Package): - """ - A Free, Fast, and Small Automatic Formatter for C, C++, C++/CLI, + """A Free, Fast, and Small Automatic Formatter for C, C++, C++/CLI, Objective-C, C#, and Java Source Code. """ + homepage = "http://astyle.sourceforge.net/" url = "http://downloads.sourceforge.net/project/astyle/astyle/astyle%202.04/astyle_2.04_linux.tar.gz" diff --git a/var/spack/repos/builtin/packages/caliper/package.py b/var/spack/repos/builtin/packages/caliper/package.py index e5aed5ed65..113bef61b8 100644 --- a/var/spack/repos/builtin/packages/caliper/package.py +++ b/var/spack/repos/builtin/packages/caliper/package.py @@ -26,8 +26,7 @@ from spack import * class Caliper(Package): - """ - Caliper is a generic context annotation system. It gives programmers the + """Caliper is a generic context annotation system. It gives programmers the ability to provide arbitrary program context information to (performance) tools at runtime. """ diff --git a/var/spack/repos/builtin/packages/cfitsio/package.py b/var/spack/repos/builtin/packages/cfitsio/package.py index c68b3ffc0d..35d9662f6f 100644 --- a/var/spack/repos/builtin/packages/cfitsio/package.py +++ b/var/spack/repos/builtin/packages/cfitsio/package.py @@ -26,11 +26,12 @@ from spack import * class Cfitsio(Package): - """ - CFITSIO is a library of C and Fortran subroutines for reading and writing + """CFITSIO is a library of C and Fortran subroutines for reading and writing data files in FITS (Flexible Image Transport System) data format. """ + homepage = 'http://heasarc.gsfc.nasa.gov/fitsio/' + version('3.370', 'abebd2d02ba5b0503c633581e3bfa116') def url_for_version(self, v): diff --git a/var/spack/repos/builtin/packages/cityhash/package.py b/var/spack/repos/builtin/packages/cityhash/package.py index d6d7f51558..85d948cc57 100644 --- a/var/spack/repos/builtin/packages/cityhash/package.py +++ b/var/spack/repos/builtin/packages/cityhash/package.py @@ -27,6 +27,8 @@ from spack.util.environment import * class Cityhash(Package): + """CityHash, a family of hash functions for strings.""" + homepage = "https://github.com/google/cityhash" url = "https://github.com/google/cityhash" diff --git a/var/spack/repos/builtin/packages/cube/package.py b/var/spack/repos/builtin/packages/cube/package.py index 09237f87c3..8c835b3886 100644 --- a/var/spack/repos/builtin/packages/cube/package.py +++ b/var/spack/repos/builtin/packages/cube/package.py @@ -22,13 +22,11 @@ # License along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## - from spack import * class Cube(Package): - """ - Cube the profile viewer for Score-P and Scalasca profiles. It displays a + """Cube the profile viewer for Score-P and Scalasca profiles. It displays a multi-dimensional performance space consisting of the dimensions: - performance metric - call path diff --git a/var/spack/repos/builtin/packages/datamash/package.py b/var/spack/repos/builtin/packages/datamash/package.py index ef30c50ebe..85adeca996 100644 --- a/var/spack/repos/builtin/packages/datamash/package.py +++ b/var/spack/repos/builtin/packages/datamash/package.py @@ -26,8 +26,7 @@ from spack import * class Datamash(Package): - """ - GNU datamash is a command-line program which performs basic numeric, + """GNU datamash is a command-line program which performs basic numeric, textual and statistical operations on input textual data files. """ diff --git a/var/spack/repos/builtin/packages/eigen/package.py b/var/spack/repos/builtin/packages/eigen/package.py index 36d6850c1a..42dc24e373 100644 --- a/var/spack/repos/builtin/packages/eigen/package.py +++ b/var/spack/repos/builtin/packages/eigen/package.py @@ -26,10 +26,8 @@ from spack import * class Eigen(Package): - """ - Eigen is a C++ template library for linear algebra - - Matrices, vectors, numerical solvers, and related algorithms + """Eigen is a C++ template library for linear algebra matrices, + vectors, numerical solvers, and related algorithms. """ homepage = 'http://eigen.tuxfamily.org/' diff --git a/var/spack/repos/builtin/packages/espresso/package.py b/var/spack/repos/builtin/packages/espresso/package.py index 447964f286..3dca1ba187 100644 --- a/var/spack/repos/builtin/packages/espresso/package.py +++ b/var/spack/repos/builtin/packages/espresso/package.py @@ -23,17 +23,16 @@ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## from spack import * - import os class Espresso(Package): - """ - QE is an integrated suite of Open-Source computer codes for + """QE is an integrated suite of Open-Source computer codes for electronic-structure calculations and materials modeling at the nanoscale. It is based on density-functional theory, plane waves, and pseudopotentials. """ + homepage = 'http://quantum-espresso.org' url = 'http://www.qe-forge.org/gf/download/frsrelease/204/912/espresso-5.3.0.tar.gz' diff --git a/var/spack/repos/builtin/packages/gdal/package.py b/var/spack/repos/builtin/packages/gdal/package.py index 4fdfafc992..4f54b3a841 100644 --- a/var/spack/repos/builtin/packages/gdal/package.py +++ b/var/spack/repos/builtin/packages/gdal/package.py @@ -26,14 +26,13 @@ from spack import * class Gdal(Package): - """ - GDAL is a translator library for raster and vector geospatial + """GDAL is a translator library for raster and vector geospatial data formats that is released under an X/MIT style Open Source license by the Open Source Geospatial Foundation. As a library, it presents a single raster abstract data model and vector abstract data model to the calling application for all supported formats. It also comes with a variety of useful command line - utilities for data translation and processing + utilities for data translation and processing. """ homepage = "http://www.gdal.org/" diff --git a/var/spack/repos/builtin/packages/gdb/package.py b/var/spack/repos/builtin/packages/gdb/package.py index 9145009fa4..a14e166ed4 100644 --- a/var/spack/repos/builtin/packages/gdb/package.py +++ b/var/spack/repos/builtin/packages/gdb/package.py @@ -22,16 +22,15 @@ # License along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## - from spack import * class Gdb(Package): """GDB, the GNU Project debugger, allows you to see what is going on - `inside' another program while it executes -- or what another - program was doing at the moment it crashed. - + 'inside' another program while it executes -- or what another + program was doing at the moment it crashed. """ + homepage = "https://www.gnu.org/software/gdb" url = "http://ftp.gnu.org/gnu/gdb/gdb-7.10.tar.gz" diff --git a/var/spack/repos/builtin/packages/gmsh/package.py b/var/spack/repos/builtin/packages/gmsh/package.py index 72f490e2f4..1d375f2186 100644 --- a/var/spack/repos/builtin/packages/gmsh/package.py +++ b/var/spack/repos/builtin/packages/gmsh/package.py @@ -26,8 +26,7 @@ from spack import * class Gmsh(Package): - """ - Gmsh is a free 3D finite element grid generator with a built-in CAD engine + """Gmsh is a free 3D finite element grid generator with a built-in CAD engine and post-processor. Its design goal is to provide a fast, light and user-friendly meshing tool with parametric input and advanced visualization capabilities. Gmsh is built around four modules: geometry, mesh, solver and @@ -35,6 +34,7 @@ class Gmsh(Package): either interactively using the graphical user interface or in ASCII text files using Gmsh's own scripting language. """ + homepage = 'http://gmsh.info' url = 'http://gmsh.info/src/gmsh-2.11.0-source.tgz' diff --git a/var/spack/repos/builtin/packages/libsplash/package.py b/var/spack/repos/builtin/packages/libsplash/package.py index 0659f7d650..21a6eede3f 100644 --- a/var/spack/repos/builtin/packages/libsplash/package.py +++ b/var/spack/repos/builtin/packages/libsplash/package.py @@ -26,10 +26,9 @@ from spack import * class Libsplash(Package): - """ - libSplash aims at developing a HDF5-based I/O library for HPC simulations. - It is created as an easy-to-use frontend for the standard HDF5 library - with support for MPI processes in a cluster environment. While the + """libSplash aims at developing a HDF5-based I/O library for HPC + simulations. It is created as an easy-to-use frontend for the standard HDF5 + library with support for MPI processes in a cluster environment. While the standard HDF5 library provides detailed low-level control, libSplash simplifies tasks commonly found in large-scale HPC simulations, such as iterative computations and MPI distributed processes. diff --git a/var/spack/repos/builtin/packages/lmod/package.py b/var/spack/repos/builtin/packages/lmod/package.py index 69965bc423..c2daca80a5 100644 --- a/var/spack/repos/builtin/packages/lmod/package.py +++ b/var/spack/repos/builtin/packages/lmod/package.py @@ -27,14 +27,14 @@ from glob import glob class Lmod(Package): - """ - Lmod is a Lua based module system that easily handles the MODULEPATH + """Lmod is a Lua based module system that easily handles the MODULEPATH Hierarchical problem. Environment Modules provide a convenient way to dynamically change the users' environment through modulefiles. This includes easily adding or removing directories to the PATH environment variable. Modulefiles for Library packages provide environment variables that specify where the library and header files can be found. """ + homepage = 'https://www.tacc.utexas.edu/research-development/tacc-projects/lmod' url = 'https://github.com/TACC/Lmod/archive/6.4.1.tar.gz' diff --git a/var/spack/repos/builtin/packages/lua-luafilesystem/package.py b/var/spack/repos/builtin/packages/lua-luafilesystem/package.py index a61b9dd675..9e0e449813 100644 --- a/var/spack/repos/builtin/packages/lua-luafilesystem/package.py +++ b/var/spack/repos/builtin/packages/lua-luafilesystem/package.py @@ -26,8 +26,7 @@ from spack import * class LuaLuafilesystem(Package): - """ - LuaFileSystem is a Lua library developed to complement the set of + """LuaFileSystem is a Lua library developed to complement the set of functions related to file systems offered by the standard Lua distribution. LuaFileSystem offers a portable way to access the underlying directory @@ -35,6 +34,7 @@ class LuaLuafilesystem(Package): LuaFileSystem is free software and uses the same license as Lua 5.1 """ + homepage = 'http://keplerproject.github.io/luafilesystem' url = 'https://github.com/keplerproject/luafilesystem/archive/v_1_6_3.tar.gz' diff --git a/var/spack/repos/builtin/packages/mxml/package.py b/var/spack/repos/builtin/packages/mxml/package.py index 113c48c18f..29e3b27d6e 100644 --- a/var/spack/repos/builtin/packages/mxml/package.py +++ b/var/spack/repos/builtin/packages/mxml/package.py @@ -26,10 +26,9 @@ from spack import * class Mxml(Package): - """ - Mini-XML is a small XML library that you can use to read and write XML + """Mini-XML is a small XML library that you can use to read and write XML and XML-like data files in your application without requiring large - non-standard libraries + non-standard libraries. """ homepage = "http://www.msweet.org" diff --git a/var/spack/repos/builtin/packages/ncdu/package.py b/var/spack/repos/builtin/packages/ncdu/package.py index 2147319d3f..0842a592cc 100644 --- a/var/spack/repos/builtin/packages/ncdu/package.py +++ b/var/spack/repos/builtin/packages/ncdu/package.py @@ -26,8 +26,7 @@ from spack import * class Ncdu(Package): - """ - Ncdu is a disk usage analyzer with an ncurses interface. It is designed + """Ncdu is a disk usage analyzer with an ncurses interface. It is designed to find space hogs on a remote server where you don't have an entire gaphical setup available, but it is a useful tool even on regular desktop systems. Ncdu aims to be fast, simple and easy to use, and should be able diff --git a/var/spack/repos/builtin/packages/opari2/package.py b/var/spack/repos/builtin/packages/opari2/package.py index e901f8ed39..20c67716a4 100644 --- a/var/spack/repos/builtin/packages/opari2/package.py +++ b/var/spack/repos/builtin/packages/opari2/package.py @@ -22,13 +22,11 @@ # License along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## - from spack import * class Opari2(Package): - """ - OPARI2 is a source-to-source instrumentation tool for OpenMP and hybrid + """OPARI2 is a source-to-source instrumentation tool for OpenMP and hybrid codes. It surrounds OpenMP directives and runtime library calls with calls to the POMP2 measurement interface. OPARI2 will provide you with a new initialization method that allows for multi-directory and parallel builds diff --git a/var/spack/repos/builtin/packages/opencoarrays/package.py b/var/spack/repos/builtin/packages/opencoarrays/package.py index 0003157985..d9760e2afc 100644 --- a/var/spack/repos/builtin/packages/opencoarrays/package.py +++ b/var/spack/repos/builtin/packages/opencoarrays/package.py @@ -26,8 +26,7 @@ from spack import * class Opencoarrays(CMakePackage): - """ - OpenCoarrays is an open-source software project that produces an + """OpenCoarrays is an open-source software project that produces an application binary interface (ABI) supporting coarray Fortran (CAF) compilers, an application programming interface (API) that supports users of non-CAF compilers, and an associated compiler wrapper and program diff --git a/var/spack/repos/builtin/packages/openjpeg/package.py b/var/spack/repos/builtin/packages/openjpeg/package.py index 1bc5b04f6f..fc505e19a9 100644 --- a/var/spack/repos/builtin/packages/openjpeg/package.py +++ b/var/spack/repos/builtin/packages/openjpeg/package.py @@ -26,14 +26,14 @@ from spack import * class Openjpeg(Package): - """ - OpenJPEG is an open-source JPEG 2000 codec written in C language. + """OpenJPEG is an open-source JPEG 2000 codec written in C language. It has been developed in order to promote the use of JPEG 2000, a still-image compression standard from the Joint Photographic Experts Group (JPEG). Since April 2015, it is officially recognized by ISO/IEC and ITU-T as a JPEG 2000 Reference Software. """ + homepage = "https://github.com/uclouvain/openjpeg" url = "https://github.com/uclouvain/openjpeg/archive/version.2.1.tar.gz" diff --git a/var/spack/repos/builtin/packages/parallel/package.py b/var/spack/repos/builtin/packages/parallel/package.py index 2306dace55..81c0195651 100644 --- a/var/spack/repos/builtin/packages/parallel/package.py +++ b/var/spack/repos/builtin/packages/parallel/package.py @@ -26,8 +26,7 @@ from spack import * class Parallel(Package): - """ - GNU parallel is a shell tool for executing jobs in parallel using + """GNU parallel is a shell tool for executing jobs in parallel using one or more computers. A job can be a single command or a small script that has to be run for each of the lines in the input. """ diff --git a/var/spack/repos/builtin/packages/petsc/package.py b/var/spack/repos/builtin/packages/petsc/package.py index 66e1abdf1a..73091bb182 100644 --- a/var/spack/repos/builtin/packages/petsc/package.py +++ b/var/spack/repos/builtin/packages/petsc/package.py @@ -27,8 +27,7 @@ from spack import * class Petsc(Package): - """ - PETSc is a suite of data structures and routines for the scalable + """PETSc is a suite of data structures and routines for the scalable (parallel) solution of scientific applications modeled by partial differential equations. """ diff --git a/var/spack/repos/builtin/packages/pngwriter/package.py b/var/spack/repos/builtin/packages/pngwriter/package.py index 037a2eb4f3..c51f1f82a8 100644 --- a/var/spack/repos/builtin/packages/pngwriter/package.py +++ b/var/spack/repos/builtin/packages/pngwriter/package.py @@ -26,14 +26,13 @@ from spack import * class Pngwriter(Package): - """ - PNGwriter is a very easy to use open source graphics library that uses PNG - as its output format. The interface has been designed to be as simple and - intuitive as possible. It supports plotting and reading pixels in the RGB - (red, green, blue), HSV (hue, saturation, value/brightness) and CMYK (cyan, - magenta, yellow, black) colour spaces, basic shapes, scaling, bilinear - interpolation, full TrueType antialiased and rotated text support, bezier - curves, opening existing PNG images and more. + """PNGwriter is a very easy to use open source graphics library that uses + PNG as its output format. The interface has been designed to be as simple + and intuitive as possible. It supports plotting and reading pixels in the + RGB (red, green, blue), HSV (hue, saturation, value/brightness) and CMYK + (cyan, magenta, yellow, black) colour spaces, basic shapes, scaling, + bilinear interpolation, full TrueType antialiased and rotated text support, + bezier curves, opening existing PNG images and more. """ homepage = "http://pngwriter.sourceforge.net/" diff --git a/var/spack/repos/builtin/packages/py-coverage/package.py b/var/spack/repos/builtin/packages/py-coverage/package.py index c2a698b0bd..1a5b6df3d7 100644 --- a/var/spack/repos/builtin/packages/py-coverage/package.py +++ b/var/spack/repos/builtin/packages/py-coverage/package.py @@ -27,7 +27,7 @@ from spack import * class PyCoverage(Package): """ Testing coverage checker for python """ - # FIXME: add a proper url for your package's homepage here. + homepage = "http://nedbatchelder.com/code/coverage/" url = "https://pypi.python.org/packages/source/c/coverage/coverage-4.0a6.tar.gz" diff --git a/var/spack/repos/builtin/packages/py-prettytable/package.py b/var/spack/repos/builtin/packages/py-prettytable/package.py index cf814733d4..fa1c17ae8c 100644 --- a/var/spack/repos/builtin/packages/py-prettytable/package.py +++ b/var/spack/repos/builtin/packages/py-prettytable/package.py @@ -22,16 +22,15 @@ # License along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -from spack import depends_on, extends, version -from spack import Package +from spack import * class PyPrettytable(Package): - """ - PrettyTable is a simple Python library designed to make + """PrettyTable is a simple Python library designed to make it quick and easy to represent tabular data in visually - appealing ASCII tables + appealing ASCII tables. """ + homepage = "https://code.google.com/archive/p/prettytable/" url = "https://pypi.python.org/packages/e0/a1/36203205f77ccf98f3c6cf17cf068c972e6458d7e58509ca66da949ca347/prettytable-0.7.2.tar.gz" diff --git a/var/spack/repos/builtin/packages/py-tuiview/package.py b/var/spack/repos/builtin/packages/py-tuiview/package.py index 672a3ee587..5caf3ff143 100644 --- a/var/spack/repos/builtin/packages/py-tuiview/package.py +++ b/var/spack/repos/builtin/packages/py-tuiview/package.py @@ -26,10 +26,10 @@ from spack import * class PyTuiview(Package): + """TuiView is a lightweight raster GIS with powerful raster attribute + table manipulation abilities. """ - TuiView is a lightweight raster GIS with powerful raster attribute - table manipulation abilities. - """ + homepage = "https://bitbucket.org/chchrsc/tuiview" url = "https://bitbucket.org/chchrsc/tuiview/get/tuiview-1.1.7.tar.gz" diff --git a/var/spack/repos/builtin/packages/scorep/package.py b/var/spack/repos/builtin/packages/scorep/package.py index d40e08740f..e0f7972304 100644 --- a/var/spack/repos/builtin/packages/scorep/package.py +++ b/var/spack/repos/builtin/packages/scorep/package.py @@ -22,15 +22,13 @@ # License along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## - from spack import * class Scorep(Package): - """ - The Score-P measurement infrastructure is a highly scalable and easy-to-use - tool suite for profiling, event tracing, and online analysis of HPC - applications. + """The Score-P measurement infrastructure is a highly scalable and + easy-to-use tool suite for profiling, event tracing, and online analysis + of HPC applications. """ homepage = "http://www.vi-hps.org/projects/score-p" diff --git a/var/spack/repos/builtin/packages/screen/package.py b/var/spack/repos/builtin/packages/screen/package.py index 17335603e6..7edfb44a4d 100644 --- a/var/spack/repos/builtin/packages/screen/package.py +++ b/var/spack/repos/builtin/packages/screen/package.py @@ -26,8 +26,7 @@ from spack import * class Screen(Package): - """ - Screen is a full-screen window manager that multiplexes a physical + """Screen is a full-screen window manager that multiplexes a physical terminal between several processes, typically interactive shells. """ diff --git a/var/spack/repos/builtin/packages/swiftsim/package.py b/var/spack/repos/builtin/packages/swiftsim/package.py index 636aa26bd2..c591c48d19 100644 --- a/var/spack/repos/builtin/packages/swiftsim/package.py +++ b/var/spack/repos/builtin/packages/swiftsim/package.py @@ -22,15 +22,12 @@ # License along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## - from spack import * - import llnl.util.tty as tty class Swiftsim(Package): - """ - SPH With Inter-dependent Fine-grained Tasking (SWIFT) provides + """SPH With Inter-dependent Fine-grained Tasking (SWIFT) provides astrophysicists with a state of the art framework to perform particle based simulations. """ diff --git a/var/spack/repos/builtin/packages/tau/package.py b/var/spack/repos/builtin/packages/tau/package.py index a77df8d37c..1801b41c37 100644 --- a/var/spack/repos/builtin/packages/tau/package.py +++ b/var/spack/repos/builtin/packages/tau/package.py @@ -22,21 +22,18 @@ # License along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## - from spack import * - import os import os.path - from llnl.util.filesystem import join_path class Tau(Package): - """ - A portable profiling and tracing toolkit for performance + """A portable profiling and tracing toolkit for performance analysis of parallel programs written in Fortran, C, C++, UPC, Java, Python. """ + homepage = "http://www.cs.uoregon.edu/research/tau" url = "https://www.cs.uoregon.edu/research/tau/tau_releases/tau-2.25.tar.gz" diff --git a/var/spack/repos/builtin/packages/xerces-c/package.py b/var/spack/repos/builtin/packages/xerces-c/package.py index d0c2d3d497..9f3ad8a4dc 100644 --- a/var/spack/repos/builtin/packages/xerces-c/package.py +++ b/var/spack/repos/builtin/packages/xerces-c/package.py @@ -26,9 +26,9 @@ from spack import * class XercesC(Package): - """ Xerces-C++ is a validating XML parser written in a portable subset of C++. - Xerces-C++ makes it easy to give your application the ability to read and - write XML data. A shared library is provided for parsing, generating, + """Xerces-C++ is a validating XML parser written in a portable subset of + C++. Xerces-C++ makes it easy to give your application the ability to read + and write XML data. A shared library is provided for parsing, generating, manipulating, and validating XML documents using the DOM, SAX, and SAX2 APIs. """ diff --git a/var/spack/repos/builtin/packages/zsh/package.py b/var/spack/repos/builtin/packages/zsh/package.py index 4c27cd3ec2..a70d307be9 100644 --- a/var/spack/repos/builtin/packages/zsh/package.py +++ b/var/spack/repos/builtin/packages/zsh/package.py @@ -26,11 +26,11 @@ from spack import * class Zsh(Package): + """Zsh is a shell designed for interactive use, although it is also a + powerful scripting language. Many of the useful features of bash, ksh, and + tcsh were incorporated into zsh; many original features were added. """ - Zsh is a shell designed for interactive use, although it is also a powerful - scripting language. Many of the useful features of bash, ksh, and tcsh were - incorporated into zsh; many original features were added. - """ + homepage = "http://www.zsh.org" url = "http://downloads.sourceforge.net/project/zsh/zsh/5.1.1/zsh-5.1.1.tar.gz" |