diff options
204 files changed, 6754 insertions, 1892 deletions
@@ -1,13 +1,20 @@ Todd Gamblin <tgamblin@llnl.gov> George Todd Gamblin <gamblin2@llnl.gov> +Todd Gamblin <tgamblin@llnl.gov> Todd Gamblin <gamblin2@llnl.gov> Adam Moody <moody20@llnl.gov> Adam T. Moody <moody20@llnl.gov> Alfredo Gimenez <gimenez1@llnl.gov> Alfredo Gimenez <alfredo.gimenez@gmail.com> David Boehme <boehme3@llnl.gov> David Boehme <boehme3@sierra324.llnl.gov> David Boehme <boehme3@llnl.gov> David Boehme <boehme3@sierra648.llnl.gov> Kevin Brandstatter <kjbrandstatter@gmail.com> Kevin Brandstatter <kbrandst@hawk.iit.edu> Luc Jaulmes <luc.jaulmes@bsc.es> Luc Jaulmes <jaulmes1@llnl.gov> -Saravan Pantham <saravan.pantham@gmail.com> Saravan Pantham <pantham1@surface86.llnl.gov +Saravan Pantham <saravan.pantham@gmail.com> Saravan Pantham <pantham1@surface86.llnl.gov> Tom Scogland <tscogland@llnl.gov> Tom Scogland <scogland1@llnl.gov> Tom Scogland <tscogland@llnl.gov> Tom Scogland <tom.scogland@gmail.com> Joachim Protze <protze@rz.rwth-aachen.de> jprotze <protze@rz.rwth-aachen.de> Gregory L. Lee <lee218@llnl.gov> Gregory L. Lee <lee218@surface86.llnl.gov> +Gregory L. Lee <lee218@llnl.gov> Gregory L. Lee <lee218@cab687.llnl.gov> +Gregory L. Lee <lee218@llnl.gov> Gregory L. Lee <lee218@cab690.llnl.gov> +Gregory L. Lee <lee218@llnl.gov> Gregory L. Lee <lee218@catalyst159.llnl.gov> Gregory L. Lee <lee218@llnl.gov> Gregory Lee <lee218@llnl.gov> +Massimiliano Culpo <massimiliano.culpo@epfl.ch> Massimiliano Culpo <massimiliano.culpo@googlemail.com> +Massimiliano Culpo <massimiliano.culpo@epfl.ch> alalazo <massimiliano.culpo@googlemail.com> +Mark Miller <miller86@llnl.gov> miller86 <miller86@llnl.gov> diff --git a/.travis.yml b/.travis.yml index 082b747bc2..465a86faf7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,6 +20,9 @@ before_install: script: - . share/spack/setup-env.sh + - spack compilers + - spack config get compilers + - spack install -v libdwarf # Run unit tests with code coverage - coverage run --source=lib --omit=lib/spack/spack/test/*,lib/spack/env/*,lib/spack/docs/* bin/spack test # Checks if the file that have been changed are flake8 conformant @@ -19,7 +19,7 @@ written in pure Python, and specs allow package authors to write a single build script for many different builds of the same package. See the -[Feature Overview](http://llnl.github.io/spack/features.html) +[Feature Overview](http://software.llnl.gov/spack/features.html) for examples and highlights. To install spack and install your first package: @@ -31,7 +31,7 @@ To install spack and install your first package: Documentation ---------------- -[**Full documentation**](http://llnl.github.io/spack) for Spack is +[**Full documentation**](http://software.llnl.gov/spack) for Spack is the first place to look. See also: diff --git a/bin/sbang b/bin/sbang new file mode 100755 index 0000000000..ebfbe2e7a1 --- /dev/null +++ b/bin/sbang @@ -0,0 +1,84 @@ +#!/bin/bash +# +# `sbang`: Run scripts with long shebang lines. +# +# Many operating systems limit the length of shebang lines, making it +# hard to use interpreters that are deep in the directory hierarchy. +# `sbang` can run such scripts, either as a shebang interpreter, or +# directly on the command line. +# +# Usage +# ----------------------------- +# Suppose you have a script, long-shebang.sh, like this: +# +# 1 #!/very/long/path/to/some/interpreter +# 2 +# 3 echo "success!" +# +# Invoking this script will result in an error on some OS's. On +# Linux, you get this: +# +# $ ./long-shebang.sh +# -bash: ./long: /very/long/path/to/some/interp: bad interpreter: +# No such file or directory +# +# On Mac OS X, the system simply assumes the interpreter is the shell +# and tries to run with it, which is likely not what you want. +# +# +# `sbang` on the command line +# ----------------------------- +# You can use `sbang` in two ways. The first is to use it directly, +# from the command line, like this: +# +# $ sbang ./long-shebang.sh +# success! +# +# +# `sbang` as the interpreter +# ----------------------------- +# You can also use `sbang` *as* the interpreter for your script. Put +# `#!/bin/bash /path/to/sbang` on line 1, and move the original +# shebang to line 2 of the script: +# +# 1 #!/bin/bash /path/to/sbang +# 2 #!/long/path/to/real/interpreter with arguments +# 3 +# 4 echo "success!" +# +# $ ./long-shebang.sh +# success! +# +# On Linux, you could shorten line 1 to `#!/path/to/sbang`, but other +# operating systems like Mac OS X require the interpreter to be a +# binary, so it's best to use `sbang` as a `bash` argument. +# Obviously, for this to work, `sbang` needs to have a short enough +# path that *it* will run without hitting OS limits. +# +# +# How it works +# ----------------------------- +# `sbang` is a very simple bash script. It looks at the first two +# lines of a script argument and runs the last line starting with +# `#!`, with the script as an argument. It also forwards arguments. +# + +# First argument is the script we want to actually run. +script="$1" + +# Search the first two lines of script for interpreters. +lines=0 +while read line && ((lines < 2)) ; do + if [[ "$line" = '#!'* ]]; then + interpreter="${line#\#!}" + fi + lines=$((lines+1)) +done < "$script" + +# Invoke any interpreter found, or raise an error if none was found. +if [ -n "$interpreter" ]; then + exec $interpreter "$@" +else + echo "error: sbang found no interpreter in $script" + exit 1 +fi @@ -42,9 +42,9 @@ SPACK_EXTERNAL_LIBS = os.path.join(SPACK_LIB_PATH, "external") sys.path.insert(0, SPACK_EXTERNAL_LIBS) import warnings -# Avoid warnings when nose is installed with the python exe being used to run -# spack. Note this must be done after Spack's external libs directory is added -# to sys.path. +# Avoid warnings when nose is installed with the python exe being used to run +# spack. Note this must be done after Spack's external libs directory is added +# to sys.path. with warnings.catch_warnings(): warnings.filterwarnings("ignore", ".*nose was already imported") import nose @@ -62,7 +62,7 @@ for pyc_file in orphaned_pyc_files: try: os.remove(pyc_file) except OSError as e: - print "WARNING: Spack may fail mysteriously. Couldn't remove orphaned .pyc file: %s" % pyc + print "WARNING: Spack may fail mysteriously. Couldn't remove orphaned .pyc file: %s" % pyc_file # If there is no working directory, use the spack prefix. try: diff --git a/lib/spack/docs/basic_usage.rst b/lib/spack/docs/basic_usage.rst index 0578f0c8db..accf09cc2a 100644 --- a/lib/spack/docs/basic_usage.rst +++ b/lib/spack/docs/basic_usage.rst @@ -357,7 +357,7 @@ Spack, you can simply run ``spack compiler add`` with the path to where the compiler is installed. For example:: $ spack compiler add /usr/local/tools/ic-13.0.079 - ==> Added 1 new compiler to /Users/gamblin2/.spackconfig + ==> Added 1 new compiler to /Users/gamblin2/.spack/compilers.yaml intel@13.0.079 Or you can run ``spack compiler add`` with no arguments to force @@ -367,7 +367,7 @@ installed, but you know that new compilers have been added to your $ module load gcc-4.9.0 $ spack compiler add - ==> Added 1 new compiler to /Users/gamblin2/.spackconfig + ==> Added 1 new compiler to /Users/gamblin2/.spack/compilers.yaml gcc@4.9.0 This loads the environment module for gcc-4.9.0 to get it into the @@ -398,27 +398,34 @@ Manual compiler configuration ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ If auto-detection fails, you can manually configure a compiler by -editing your ``~/.spackconfig`` file. You can do this by running -``spack config edit``, which will open the file in your ``$EDITOR``. +editing your ``~/.spack/compilers.yaml`` file. You can do this by running +``spack config edit compilers``, which will open the file in your ``$EDITOR``. Each compiler configuration in the file looks like this:: ... - [compiler "intel@15.0.0"] - cc = /usr/local/bin/icc-15.0.024-beta - cxx = /usr/local/bin/icpc-15.0.024-beta - f77 = /usr/local/bin/ifort-15.0.024-beta - fc = /usr/local/bin/ifort-15.0.024-beta - ... + chaos_5_x86_64_ib: + ... + intel@15.0.0: + cc: /usr/local/bin/icc-15.0.024-beta + cxx: /usr/local/bin/icpc-15.0.024-beta + f77: /usr/local/bin/ifort-15.0.024-beta + fc: /usr/local/bin/ifort-15.0.024-beta + ... + +The chaos_5_x86_64_ib string is an architecture string, and multiple +compilers can be listed underneath an architecture. The architecture +string may be replaced with the string 'all' to signify compilers that +work on all architectures. For compilers, like ``clang``, that do not support Fortran, put ``None`` for ``f77`` and ``fc``:: - [compiler "clang@3.3svn"] - cc = /usr/bin/clang - cxx = /usr/bin/clang++ - f77 = None - fc = None + clang@3.3svn: + cc: /usr/bin/clang + cxx: /usr/bin/clang++ + f77: None + fc: None Once you save the file, the configured compilers will show up in the list displayed by ``spack compilers``. @@ -896,7 +903,7 @@ Or, similarly with modules, you could type: $ spack load mpich %gcc@4.4.7 These commands will add appropriate directories to your ``PATH``, -``MANPATH``, and ``LD_LIBRARY_PATH``. When you no longer want to use +``MANPATH``, ``CPATH``, and ``LD_LIBRARY_PATH``. When you no longer want to use a package, you can type unload or unuse similarly: .. code-block:: sh diff --git a/lib/spack/docs/developer_guide.rst b/lib/spack/docs/developer_guide.rst index db47de80f5..0b618aa683 100644 --- a/lib/spack/docs/developer_guide.rst +++ b/lib/spack/docs/developer_guide.rst @@ -73,19 +73,32 @@ with a high level view of Spack's directory structure:: spack/ <- installation root bin/ spack <- main spack executable + + etc/ + spack/ <- Spack config files. + Can be overridden by files in ~/.spack. + var/ spack/ <- build & stage directories + repos/ <- contains package repositories + builtin/ <- pkg repository that comes with Spack + repo.yaml <- descriptor for the builtin repository + packages/ <- directories under here contain packages + opt/ spack/ <- packages are installed here + lib/ spack/ docs/ <- source for this documentation env/ <- compiler wrappers for build environment + external/ <- external libs included in Spack distro + llnl/ <- some general-use libraries + spack/ <- spack module; contains Python code cmd/ <- each file in here is a spack subcommand compilers/ <- compiler description files - packages/ <- each file in here is a spack package test/ <- unit test modules util/ <- common code diff --git a/lib/spack/docs/features.rst b/lib/spack/docs/features.rst index fcb810086d..0998ba8da4 100644 --- a/lib/spack/docs/features.rst +++ b/lib/spack/docs/features.rst @@ -103,7 +103,7 @@ creates a simple python file: It doesn't take much python coding to get from there to a working package: -.. literalinclude:: ../../../var/spack/packages/libelf/package.py +.. literalinclude:: ../../../var/spack/repos/builtin/packages/libelf/package.py :lines: 25- Spack also provides wrapper functions around common commands like diff --git a/lib/spack/docs/getting_started.rst b/lib/spack/docs/getting_started.rst index 67ca18e71a..2c5b68ea65 100644 --- a/lib/spack/docs/getting_started.rst +++ b/lib/spack/docs/getting_started.rst @@ -22,7 +22,7 @@ go: $ spack install libelf For a richer experience, use Spack's `shell support -<http://llnl.github.io/spack/basic_usage.html#environment-modules>`_: +<http://software.llnl.gov/spack/basic_usage.html#environment-modules>`_: .. code-block:: sh diff --git a/lib/spack/docs/index.rst b/lib/spack/docs/index.rst index 79757208c9..d6ce52b747 100644 --- a/lib/spack/docs/index.rst +++ b/lib/spack/docs/index.rst @@ -18,7 +18,7 @@ configurations can coexist on the same system. Most importantly, Spack is *simple*. It offers a simple *spec* syntax so that users can specify versions and configuration options concisely. Spack is also simple for package authors: package files -are writtin in pure Python, and specs allow package authors to +are written in pure Python, and specs allow package authors to maintain a single file for many different builds of the same package. See the :doc:`features` for examples and highlights. diff --git a/lib/spack/docs/mirrors.rst b/lib/spack/docs/mirrors.rst index d732a3dd54..dad04d053b 100644 --- a/lib/spack/docs/mirrors.rst +++ b/lib/spack/docs/mirrors.rst @@ -38,7 +38,7 @@ contains tarballs for each package, named after each package. .. note:: - Archives are **not** named exactly they were in the package's fetch + Archives are **not** named exactly the way they were in the package's fetch URL. They have the form ``<name>-<version>.<extension>``, where ``<name>`` is Spack's name for the package, ``<version>`` is the version of the tarball, and ``<extension>`` is whatever format the @@ -186,7 +186,7 @@ Each mirror has a name so that you can refer to it again later. ``spack mirror list`` ---------------------------- -If you want to see all the mirrors Spack knows about you can run ``spack mirror list``:: +To see all the mirrors Spack knows about, run ``spack mirror list``:: $ spack mirror list local_filesystem file:///Users/gamblin2/spack-mirror-2014-06-24 @@ -196,7 +196,7 @@ If you want to see all the mirrors Spack knows about you can run ``spack mirror ``spack mirror remove`` ---------------------------- -And, if you want to remove a mirror, just remove it by name:: +To remove a mirror by name:: $ spack mirror remove local_filesystem $ spack mirror list @@ -205,12 +205,11 @@ And, if you want to remove a mirror, just remove it by name:: Mirror precedence ---------------------------- -Adding a mirror really just adds a section in ``~/.spackconfig``:: +Adding a mirror really adds a line in ``~/.spack/mirrors.yaml``:: - [mirror "local_filesystem"] - url = file:///Users/gamblin2/spack-mirror-2014-06-24 - [mirror "remote_server"] - url = https://example.com/some/web-hosted/directory/spack-mirror-2014-06-24 + mirrors: + local_filesystem: file:///Users/gamblin2/spack-mirror-2014-06-24 + remote_server: https://example.com/some/web-hosted/directory/spack-mirror-2014-06-24 If you want to change the order in which mirrors are searched for packages, you can edit this file and reorder the sections. Spack will diff --git a/lib/spack/docs/packaging_guide.rst b/lib/spack/docs/packaging_guide.rst index 59ba63fa35..519c0da232 100644 --- a/lib/spack/docs/packaging_guide.rst +++ b/lib/spack/docs/packaging_guide.rst @@ -84,7 +84,7 @@ always choose to download just one tarball initially, and run If it fails entirely, you can get minimal boilerplate by using :ref:`spack-edit-f`, or you can manually create a directory and - ``package.py`` file for the package in ``var/spack/packages``. + ``package.py`` file for the package in ``var/spack/repos/builtin/packages``. .. note:: @@ -203,7 +203,7 @@ edit`` command: So, if you used ``spack create`` to create a package, then saved and closed the resulting file, you can get back to it with ``spack edit``. The ``cmake`` package actually lives in -``$SPACK_ROOT/var/spack/packages/cmake/package.py``, but this provides +``$SPACK_ROOT/var/spack/repos/builtin/packages/cmake/package.py``, but this provides a much simpler shortcut and saves you the trouble of typing the full path. @@ -269,18 +269,18 @@ live in Spack's directory structure. In general, `spack-create`_ and `spack-edit`_ handle creating package files for you, so you can skip most of the details here. -``var/spack/packages`` +``var/spack/repos/builtin/packages`` ~~~~~~~~~~~~~~~~~~~~~~~ A Spack installation directory is structured like a standard UNIX install prefix (``bin``, ``lib``, ``include``, ``var``, ``opt``, etc.). Most of the code for Spack lives in ``$SPACK_ROOT/lib/spack``. -Packages themselves live in ``$SPACK_ROOT/var/spack/packages``. +Packages themselves live in ``$SPACK_ROOT/var/spack/repos/builtin/packages``. If you ``cd`` to that directory, you will see directories for each package: -.. command-output:: cd $SPACK_ROOT/var/spack/packages; ls -CF +.. command-output:: cd $SPACK_ROOT/var/spack/repos/builtin/packages; ls -CF :shell: :ellipsis: 10 @@ -288,7 +288,7 @@ Each directory contains a file called ``package.py``, which is where all the python code for the package goes. For example, the ``libelf`` package lives in:: - $SPACK_ROOT/var/spack/packages/libelf/package.py + $SPACK_ROOT/var/spack/repos/builtin/packages/libelf/package.py Alongside the ``package.py`` file, a package may contain extra directories or files (like patches) that it needs to build. @@ -301,7 +301,7 @@ Packages are named after the directory containing ``package.py``. So, ``libelf``'s ``package.py`` lives in a directory called ``libelf``. The ``package.py`` file defines a class called ``Libelf``, which extends Spack's ``Package`` class. for example, here is -``$SPACK_ROOT/var/spack/packages/libelf/package.py``: +``$SPACK_ROOT/var/spack/repos/builtin/packages/libelf/package.py``: .. code-block:: python :linenos: @@ -328,7 +328,7 @@ these: $ spack install libelf@0.8.13 Spack sees the package name in the spec and looks for -``libelf/package.py`` in ``var/spack/packages``. Likewise, if you say +``libelf/package.py`` in ``var/spack/repos/builtin/packages``. Likewise, if you say ``spack install py-numpy``, then Spack looks for ``py-numpy/package.py``. @@ -401,6 +401,35 @@ construct the new one for ``8.2.1``. When you supply a custom URL for a version, Spack uses that URL *verbatim* and does not perform extrapolation. +Skipping the expand step +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Spack normally expands archives automatically after downloading +them. If you want to skip this step (e.g., for self-extracting +executables and other custom archive types), you can add +``expand=False`` to a ``version`` directive. + +.. code-block:: python + + version('8.2.1', '4136d7b4c04df68b686570afa26988ac', + url='http://example.com/foo-8.2.1-special-version.tar.gz', 'expand=False') + +When ``expand`` is set to ``False``, Spack sets the current working +directory to the directory containing the downloaded archive before it +calls your ``install`` method. Within ``install``, the path to the +downloaded archive is available as ``self.stage.archive_file``. + +Here is an example snippet for packages distributed as self-extracting +archives. The example sets permissions on the downloaded file to make +it executable, then runs it with some arguments. + +.. code-block:: python + + def install(self, spec, prefix): + set_executable(self.stage.archive_file) + installer = Executable(self.stage.archive_file) + installer('--prefix=%s' % prefix, 'arg1', 'arg2', 'etc.') + Checksums ~~~~~~~~~~~~~~~~~ @@ -632,7 +661,7 @@ Default revision instead. Revisions - Add ``hg`` and ``revision``parameters: + Add ``hg`` and ``revision`` parameters: .. code-block:: python @@ -703,7 +732,7 @@ supply is a filename, then the patch needs to live within the spack source tree. For example, the patch above lives in a directory structure like this:: - $SPACK_ROOT/var/spack/packages/ + $SPACK_ROOT/var/spack/repos/builtin/packages/ mvapich2/ package.py ad_lustre_rwcontig_open_source.patch @@ -1524,6 +1553,69 @@ This is useful when you want to know exactly what Spack will do when you ask for a particular spec. +``Concretization Policies`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +A user may have certain preferences for how packages should +be concretized on their system. For example, one user may prefer packages +built with OpenMPI and the Intel compiler. Another user may prefer +packages be built with MVAPICH and GCC. + +Spack can be configured to prefer certain compilers, package +versions, depends_on, and variants during concretization. +The preferred configuration can be controlled via the +``~/.spack/packages.yaml`` file for user configuations, or the +``etc/spack/packages.yaml`` site configuration. + + +Here's an example packages.yaml file that sets preferred packages: + +.. code-block:: sh + + packages: + dyninst: + compiler: [gcc@4.9] + variants: +debug + gperftools: + version: [2.2, 2.4, 2.3] + all: + compiler: [gcc@4.4.7, gcc@4.6:, intel, clang, pgi] + providers: + mpi: [mvapich, mpich, openmpi] + + +At a high level, this example is specifying how packages should be +concretized. The dyninst package should prefer using gcc 4.9 and +be built with debug options. The gperftools package should prefer version +2.2 over 2.4. Every package on the system should prefer mvapich for +its MPI and gcc 4.4.7 (except for Dyninst, which overrides this by preferring gcc 4.9). +These options are used to fill in implicit defaults. Any of them can be overwritten +on the command line if explicitly requested. + +Each packages.yaml file begins with the string ``packages:`` and +package names are specified on the next level. The special string ``all`` +applies settings to each package. Underneath each package name is +one or more components: ``compiler``, ``variants``, ``version``, +or ``providers``. Each component has an ordered list of spec +``constraints``, with earlier entries in the list being preferred over +later entries. + +Sometimes a package installation may have constraints that forbid +the first concretization rule, in which case Spack will use the first +legal concretization rule. Going back to the example, if a user +requests gperftools 2.3 or later, then Spack will install version 2.4 +as the 2.4 version of gperftools is preferred over 2.3. + +An explicit concretization rule in the preferred section will always +take preference over unlisted concretizations. In the above example, +xlc isn't listed in the compiler list. Every listed compiler from +gcc to pgi will thus be preferred over the xlc compiler. + +The syntax for the ``provider`` section differs slightly from other +concretization rules. A provider lists a value that packages may +``depend_on`` (e.g, mpi) and a list of rules for fulfilling that +dependency. + .. _install-method: Implementing the ``install`` method @@ -1533,7 +1625,7 @@ The last element of a package is its ``install()`` method. This is where the real work of installation happens, and it's the main part of the package you'll need to customize for each piece of software. -.. literalinclude:: ../../../var/spack/packages/libelf/package.py +.. literalinclude:: ../../../var/spack/repos/builtin/packages/libelf/package.py :start-after: 0.8.12 :linenos: @@ -1711,15 +1803,15 @@ Compile-time library search paths * ``-L$dep_prefix/lib`` * ``-L$dep_prefix/lib64`` Runtime library search paths (RPATHs) - * ``-Wl,-rpath=$dep_prefix/lib`` - * ``-Wl,-rpath=$dep_prefix/lib64`` + * ``-Wl,-rpath,$dep_prefix/lib`` + * ``-Wl,-rpath,$dep_prefix/lib64`` Include search paths * ``-I$dep_prefix/include`` An example of this would be the ``libdwarf`` build, which has one dependency: ``libelf``. Every call to ``cc`` in the ``libdwarf`` build will have ``-I$LIBELF_PREFIX/include``, -``-L$LIBELF_PREFIX/lib``, and ``-Wl,-rpath=$LIBELF_PREFIX/lib`` +``-L$LIBELF_PREFIX/lib``, and ``-Wl,-rpath,$LIBELF_PREFIX/lib`` inserted on the command line. This is done transparently to the project's build system, which will just think it's using a system where ``libelf`` is readily available. Because of this, you **do @@ -1752,6 +1844,20 @@ dedicated process. .. _prefix-objects: + +Failing the build +---------------------- + +Sometimes you don't want a package to successfully install unless some +condition is true. You can explicitly cause the build to fail from +``install()`` by raising an ``InstallError``, for example: + +.. code-block:: python + + if spec.architecture.startswith('darwin'): + raise InstallError('This package does not build on Mac OS X!') + + Prefix objects ---------------------- @@ -2068,6 +2174,62 @@ package, this allows us to avoid race conditions in the library's build system. +.. _sanity-checks: + +Sanity checking an intallation +-------------------------------- + +By default, Spack assumes that a build has failed if nothing is +written to the install prefix, and that it has succeeded if anything +(a file, a directory, etc.) is written to the install prefix after +``install()`` completes. + +Consider a simple autotools build like this: + +.. code-block:: python + + def install(self, spec, prefix): + configure("--prefix=" + prefix) + make() + make("install") + +If you are using using standard autotools or CMake, ``configure`` and +``make`` will not write anything to the install prefix. Only ``make +install`` writes the files, and only once the build is already +complete. Not all builds are like this. Many builds of scientific +software modify the install prefix *before* ``make install``. Builds +like this can falsely report that they were successfully installed if +an error occurs before the install is complete but after files have +been written to the ``prefix``. + + +``sanity_check_is_file`` and ``sanity_check_is_dir`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +You can optionally specify *sanity checks* to deal with this problem. +Add properties like this to your package: + +.. code-block:: python + + class MyPackage(Package): + ... + + sanity_check_is_file = ['include/libelf.h'] + sanity_check_is_dir = [lib] + + def install(self, spec, prefix): + configure("--prefix=" + prefix) + make() + make("install") + +Now, after ``install()`` runs, Spack will check whether +``$prefix/include/libelf.h`` exists and is a file, and whether +``$prefix/lib`` exists and is a directory. If the checks fail, then +the build will fail and the install prefix will be removed. If they +succeed, Spack considers the build succeeful and keeps the prefix in +place. + + .. _file-manipulation: File manipulation functions @@ -2108,6 +2270,15 @@ Filtering functions Examples: + #. Filtering a Makefile to force it to use Spack's compiler wrappers: + + .. code-block:: python + + filter_file(r'^CC\s*=.*', spack_cc, 'Makefile') + filter_file(r'^CXX\s*=.*', spack_cxx, 'Makefile') + filter_file(r'^F77\s*=.*', spack_f77, 'Makefile') + filter_file(r'^FC\s*=.*', spack_fc, 'Makefile') + #. Replacing ``#!/usr/bin/perl`` with ``#!/usr/bin/env perl`` in ``bib2xhtml``: .. code-block:: python diff --git a/lib/spack/docs/site_configuration.rst b/lib/spack/docs/site_configuration.rst index b03df29573..3abfa21a9d 100644 --- a/lib/spack/docs/site_configuration.rst +++ b/lib/spack/docs/site_configuration.rst @@ -54,87 +54,73 @@ more elements to the list to indicate where your own site's temporary directory is. -.. _concretization-policies: - -Concretization policies ----------------------------- - -When a user asks for a package like ``mpileaks`` to be installed, -Spack has to make decisions like what version should be installed, -what compiler to use, and how its dependencies should be configured. -This process is called *concretization*, and it's covered in detail in -:ref:`its own section <abstract-and-concrete>`. - -The default concretization policies are in the -:py:mod:`spack.concretize` module, specifically in the -:py:class:`spack.concretize.DefaultConcretizer` class. These are the -important methods used in the concretization process: - -* :py:meth:`concretize_version(self, spec) <spack.concretize.DefaultConcretizer.concretize_version>` -* :py:meth:`concretize_architecture(self, spec) <spack.concretize.DefaultConcretizer.concretize_architecture>` -* :py:meth:`concretize_compiler(self, spec) <spack.concretize.DefaultConcretizer.concretize_compiler>` -* :py:meth:`choose_provider(self, spec, providers) <spack.concretize.DefaultConcretizer.choose_provider>` - -The first three take a :py:class:`Spec <spack.spec.Spec>` object and -modify it by adding constraints for the version. For example, if the -input spec had a version range like `1.0:5.0.3`, then the -``concretize_version`` method should set the spec's version to a -*single* version in that range. Likewise, ``concretize_architecture`` -selects an architecture when the input spec does not have one, and -``concretize_compiler`` needs to set both a concrete compiler and a -concrete compiler version. - -``choose_provider()`` affects how concrete implementations are chosen -based on a virtual dependency spec. The input spec is some virtual -dependency and the ``providers`` index is a :py:class:`ProviderIndex -<spack.packages.ProviderIndex>` object. The ``ProviderIndex`` maps -the virtual spec to specs for possible implementations, and -``choose_provider()`` should simply choose one of these. The -``concretize_*`` methods will be called on the chosen implementation -later, so there is no need to fully concretize the spec when returning -it. - -The ``DefaultConcretizer`` is intended to provide sensible defaults -for each policy, but there are certain choices that it can't know -about. For example, one site might prefer ``OpenMPI`` over ``MPICH``, -or another might prefer an old version of some packages. These types -of special cases can be integrated with custom concretizers. - -Writing a custom concretizer -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -To write your own concretizer, you need only subclass -``DefaultConcretizer`` and override the methods you want to change. -For example, you might write a class like this to change *only* the -``concretize_version()`` behavior: - -.. code-block:: python - - from spack.concretize import DefaultConcretizer - - class MyConcretizer(DefaultConcretizer): - def concretize_version(self, spec): - # implement custom logic here. - -Once you have written your custom concretizer, you can make Spack use -it by editing ``globals.py``. Find this part of the file: - -.. code-block:: python - - # - # This controls how things are concretized in spack. - # Replace it with a subclass if you want different - # policies. - # - concretizer = DefaultConcretizer() - -Set concretizer to *your own* class instead of the default: - -.. code-block:: python - - concretizer = MyConcretizer() - -The next time you run Spack, your changes should take effect. +External Packages +~~~~~~~~~~~~~~~~~~~~~ +Spack can be configured to use externally-installed +packages rather than building its own packages. This may be desirable +if machines ship with system packages, such as a customized MPI +that should be used instead of Spack building its own MPI. + +External packages are configured through the ``packages.yaml`` file found +in a Spack installation's ``etc/spack/`` or a user's ``~/.spack/`` +directory. Here's an example of an external configuration: + +.. code-block:: yaml + + packages: + openmpi: + paths: + openmpi@1.4.3%gcc@4.4.7=chaos_5_x86_64_ib: /opt/openmpi-1.4.3 + openmpi@1.4.3%gcc@4.4.7=chaos_5_x86_64_ib+debug: /opt/openmpi-1.4.3-debug + openmpi@1.6.5%intel@10.1=chaos_5_x86_64_ib: /opt/openmpi-1.6.5-intel + +This example lists three installations of OpenMPI, one built with gcc, +one built with gcc and debug information, and another built with Intel. +If Spack is asked to build a package that uses one of these MPIs as a +dependency, it will use the the pre-installed OpenMPI in +the given directory. + +Each ``packages.yaml`` begins with a ``packages:`` token, followed +by a list of package names. To specify externals, add a ``paths`` +token under the package name, which lists externals in a +``spec : /path`` format. Each spec should be as +well-defined as reasonably possible. If a +package lacks a spec component, such as missing a compiler or +package version, then Spack will guess the missing component based +on its most-favored packages, and it may guess incorrectly. + +Each package version and compilers listed in an external should +have entries in Spack's packages and compiler configuration, even +though the package and compiler may not every be built. + +The packages configuration can tell Spack to use an external location +for certain package versions, but it does not restrict Spack to using +external packages. In the above example, if an OpenMPI 1.8.4 became +available Spack may choose to start building and linking with that version +rather than continue using the pre-installed OpenMPI versions. + +To prevent this, the ``packages.yaml`` configuration also allows packages +to be flagged as non-buildable. The previous example could be modified to +be: + +.. code-block:: yaml + + packages: + openmpi: + paths: + openmpi@1.4.3%gcc@4.4.7=chaos_5_x86_64_ib: /opt/openmpi-1.4.3 + openmpi@1.4.3%gcc@4.4.7=chaos_5_x86_64_ib+debug: /opt/openmpi-1.4.3-debug + openmpi@1.6.5%intel@10.1=chaos_5_x86_64_ib: /opt/openmpi-1.6.5-intel + buildable: False + +The addition of the ``buildable`` flag tells Spack that it should never build +its own version of OpenMPI, and it will instead always rely on a pre-built +OpenMPI. Similar to ``paths``, ``buildable`` is specified as a property under +a package name. + +The ``buildable`` does not need to be paired with external packages. +It could also be used alone to forbid packages that may be +buggy or otherwise undesirable. Profiling diff --git a/lib/spack/env/cc b/lib/spack/env/cc index 0966277a91..4a3e6eddc9 100755 --- a/lib/spack/env/cc +++ b/lib/spack/env/cc @@ -90,15 +90,15 @@ case "$command" in command="$SPACK_CC" language="C" ;; - c++|CC|g++|clang++|icpc|pgCC|xlc++) + c++|CC|g++|clang++|icpc|pgc++|xlc++) command="$SPACK_CXX" language="C++" ;; - f90|fc|f95|gfortran|ifort|pgf90|xlf90) + f90|fc|f95|gfortran|ifort|pgfortran|xlf90|nagfor) command="$SPACK_FC" language="Fortran 90" ;; - f77|gfortran|ifort|pgf77|xlf) + f77|gfortran|ifort|pgfortran|xlf|nagfor) command="$SPACK_F77" language="Fortran 77" ;; @@ -113,14 +113,22 @@ case "$command" in ;; esac -# Finish setting up the mode. +# If any of the arguments below is present then the mode is vcheck. In vcheck mode nothing is added in terms of extra search paths or libraries if [ -z "$mode" ]; then - mode=ccld for arg in "$@"; do if [ "$arg" = -v -o "$arg" = -V -o "$arg" = --version -o "$arg" = -dumpversion ]; then mode=vcheck break - elif [ "$arg" = -E ]; then + fi + done +fi + +# Finish setting up the mode. + +if [ -z "$mode" ]; then + mode=ccld + for arg in "$@"; do + if [ "$arg" = -E ]; then mode=cpp break elif [ "$arg" = -c ]; then @@ -130,7 +138,7 @@ if [ -z "$mode" ]; then done fi -# Dump the version and exist if we're in testing mode. +# Dump the version and exit if we're in testing mode. if [ "$SPACK_TEST_COMMAND" = "dump-mode" ]; then echo "$mode" exit @@ -145,6 +153,10 @@ fi # Save original command for debug logging input_command="$@" +if [ "$mode" == vcheck ] ; then + exec ${command} "$@" +fi + # # Now do real parsing of the command line args, trying hard to keep # non-rpath linker arguments in the proper order w.r.t. other command @@ -175,32 +187,44 @@ while [ -n "$1" ]; do ;; -Wl,*) arg="${1#-Wl,}" - if [ -z "$arg" ]; then shift; arg="$1"; fi - if [[ "$arg" = -rpath=* ]]; then - rpaths+=("${arg#-rpath=}") - elif [[ "$arg" = -rpath ]]; then + # TODO: Handle multiple -Wl, continuations of -Wl,-rpath + if [[ $arg == -rpath=* ]]; then + arg="${arg#-rpath=}" + for rpath in ${arg//,/ }; do + rpaths+=("$rpath") + done + elif [[ $arg == -rpath,* ]]; then + arg="${arg#-rpath,}" + for rpath in ${arg//,/ }; do + rpaths+=("$rpath") + done + elif [[ $arg == -rpath ]]; then shift; arg="$1" - if [[ "$arg" != -Wl,* ]]; then + if [[ $arg != '-Wl,'* ]]; then die "-Wl,-rpath was not followed by -Wl,*" fi - rpaths+=("${arg#-Wl,}") + arg="${arg#-Wl,}" + for rpath in ${arg//,/ }; do + rpaths+=("$rpath") + done else other_args+=("-Wl,$arg") fi ;; - -Xlinker,*) - arg="${1#-Xlinker,}" - if [ -z "$arg" ]; then shift; arg="$1"; fi - if [[ "$arg" = -rpath=* ]]; then + -Xlinker) + shift; arg="$1"; + if [[ $arg = -rpath=* ]]; then rpaths+=("${arg#-rpath=}") - elif [[ "$arg" = -rpath ]]; then + elif [[ $arg = -rpath ]]; then shift; arg="$1" - if [[ "$arg" != -Xlinker,* ]]; then - die "-Xlinker,-rpath was not followed by -Xlinker,*" + if [[ $arg != -Xlinker ]]; then + die "-Xlinker -rpath was not followed by -Xlinker <arg>" fi - rpaths+=("${arg#-Xlinker,}") + shift; arg="$1" + rpaths+=("$arg") else - other_args+=("-Xlinker,$arg") + other_args+=("-Xlinker") + other_args+=("$arg") fi ;; *) diff --git a/lib/spack/env/pgi/pgf77 b/lib/spack/env/nag/nagfor index 82c2b8e90a..82c2b8e90a 120000 --- a/lib/spack/env/pgi/pgf77 +++ b/lib/spack/env/nag/nagfor diff --git a/lib/spack/env/pgi/case-insensitive/pgCC b/lib/spack/env/pgi/case-insensitive/pgCC deleted file mode 120000 index e2deb67f3b..0000000000 --- a/lib/spack/env/pgi/case-insensitive/pgCC +++ /dev/null @@ -1 +0,0 @@ -../../cc
\ No newline at end of file diff --git a/lib/spack/env/pgi/pgf90 b/lib/spack/env/pgi/pgc++ index 82c2b8e90a..82c2b8e90a 120000 --- a/lib/spack/env/pgi/pgf90 +++ b/lib/spack/env/pgi/pgc++ diff --git a/lib/spack/env/pgi/pgfortran b/lib/spack/env/pgi/pgfortran new file mode 120000 index 0000000000..82c2b8e90a --- /dev/null +++ b/lib/spack/env/pgi/pgfortran @@ -0,0 +1 @@ +../cc
\ No newline at end of file diff --git a/lib/spack/llnl/util/filesystem.py b/lib/spack/llnl/util/filesystem.py index 24cfbfde71..c4665c284c 100644 --- a/lib/spack/llnl/util/filesystem.py +++ b/lib/spack/llnl/util/filesystem.py @@ -25,7 +25,9 @@ __all__ = ['set_install_permissions', 'install', 'install_tree', 'traverse_tree', 'expand_user', 'working_dir', 'touch', 'touchp', 'mkdirp', 'force_remove', 'join_path', 'ancestor', 'can_access', 'filter_file', - 'FileFilter', 'change_sed_delimiter', 'is_exe', 'force_symlink'] + 'FileFilter', 'change_sed_delimiter', 'is_exe', 'force_symlink', + 'set_executable', 'copy_mode', 'unset_executable_mode', + 'remove_dead_links', 'remove_linked_tree'] import os import sys @@ -152,15 +154,28 @@ def set_install_permissions(path): def copy_mode(src, dest): src_mode = os.stat(src).st_mode dest_mode = os.stat(dest).st_mode - if src_mode | stat.S_IXUSR: dest_mode |= stat.S_IXUSR - if src_mode | stat.S_IXGRP: dest_mode |= stat.S_IXGRP - if src_mode | stat.S_IXOTH: dest_mode |= stat.S_IXOTH + if src_mode & stat.S_IXUSR: dest_mode |= stat.S_IXUSR + if src_mode & stat.S_IXGRP: dest_mode |= stat.S_IXGRP + if src_mode & stat.S_IXOTH: dest_mode |= stat.S_IXOTH os.chmod(dest, dest_mode) +def unset_executable_mode(path): + mode = os.stat(path).st_mode + mode &= ~stat.S_IXUSR + mode &= ~stat.S_IXGRP + mode &= ~stat.S_IXOTH + os.chmod(path, mode) + + def install(src, dest): """Manually install a file to a particular location.""" tty.debug("Installing %s to %s" % (src, dest)) + + # Expand dsst to its eventual full path if it is a directory. + if os.path.isdir(dest): + dest = join_path(dest, os.path.basename(src)) + shutil.copy(src, dest) set_install_permissions(dest) copy_mode(src, dest) @@ -235,7 +250,7 @@ def touchp(path): def force_symlink(src, dest): try: os.symlink(src, dest) - except OSError, e: + except OSError as e: os.remove(dest) os.symlink(src, dest) @@ -339,3 +354,41 @@ def traverse_tree(source_root, dest_root, rel_path='', **kwargs): if order == 'post': yield (source_path, dest_path) + + +def set_executable(path): + st = os.stat(path) + os.chmod(path, st.st_mode | stat.S_IEXEC) + + +def remove_dead_links(root): + """ + Removes any dead link that is present in root + + Args: + root: path where to search for dead links + + """ + for file in os.listdir(root): + path = join_path(root, file) + if os.path.islink(path): + real_path = os.path.realpath(path) + if not os.path.exists(real_path): + os.unlink(path) + +def remove_linked_tree(path): + """ + Removes a directory and its contents. If the directory is a + symlink, follows the link and removes the real directory before + removing the link. + + Args: + path: directory to be removed + + """ + if os.path.exists(path): + if os.path.islink(path): + shutil.rmtree(os.path.realpath(path), True) + os.unlink(path) + else: + shutil.rmtree(path, True) diff --git a/lib/spack/llnl/util/lang.py b/lib/spack/llnl/util/lang.py index 1c4d1ed623..13d301f84e 100644 --- a/lib/spack/llnl/util/lang.py +++ b/lib/spack/llnl/util/lang.py @@ -235,11 +235,11 @@ def key_ordering(cls): if not has_method(cls, '_cmp_key'): raise TypeError("'%s' doesn't define _cmp_key()." % cls.__name__) - setter('__eq__', lambda s,o: o is not None and s._cmp_key() == o._cmp_key()) + setter('__eq__', lambda s,o: (s is o) or (o is not None and s._cmp_key() == o._cmp_key())) setter('__lt__', lambda s,o: o is not None and s._cmp_key() < o._cmp_key()) setter('__le__', lambda s,o: o is not None and s._cmp_key() <= o._cmp_key()) - setter('__ne__', lambda s,o: o is None or s._cmp_key() != o._cmp_key()) + setter('__ne__', lambda s,o: (s is not o) and (o is None or s._cmp_key() != o._cmp_key())) setter('__gt__', lambda s,o: o is None or s._cmp_key() > o._cmp_key()) setter('__ge__', lambda s,o: o is None or s._cmp_key() >= o._cmp_key()) diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index ab78ecef30..aee11f061f 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -82,6 +82,20 @@ from spack.directory_layout import YamlDirectoryLayout install_layout = YamlDirectoryLayout(install_path) # +# This controls how packages are sorted when trying to choose +# the most preferred package. More preferred packages are sorted +# first. +# +from spack.preferred_packages import PreferredPackages +pkgsort = PreferredPackages() + +# +# This tests ABI compatibility between packages +# +from spack.abi import ABI +abi = ABI() + +# # This controls how things are concretized in spack. # Replace it with a subclass if you want different # policies. @@ -174,3 +188,10 @@ __all__ += spack.directives.__all__ import spack.util.executable from spack.util.executable import * __all__ += spack.util.executable.__all__ + +from spack.package import \ + install_dependency_symlinks, flatten_dependencies, DependencyConflictError, \ + InstallError, ExternalPackageError +__all__ += [ + 'install_dependency_symlinks', 'flatten_dependencies', 'DependencyConflictError', + 'InstallError', 'ExternalPackageError'] diff --git a/lib/spack/spack/abi.py b/lib/spack/spack/abi.py new file mode 100644 index 0000000000..7e565bcbf9 --- /dev/null +++ b/lib/spack/spack/abi.py @@ -0,0 +1,128 @@ +############################################################################## +# Copyright (c) 2015, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## + +import os +import spack +import spack.spec +from spack.spec import CompilerSpec +from spack.util.executable import Executable, ProcessError +from llnl.util.lang import memoized + +class ABI(object): + """This class provides methods to test ABI compatibility between specs. + The current implementation is rather rough and could be improved.""" + + def architecture_compatible(self, parent, child): + """Returns true iff the parent and child specs have ABI compatible architectures.""" + return not parent.architecture or not child.architecture or parent.architecture == child.architecture + + + @memoized + def _gcc_get_libstdcxx_version(self, version): + """Returns gcc ABI compatibility info by getting the library version of + a compiler's libstdc++.so or libgcc_s.so""" + spec = CompilerSpec("gcc", version) + compilers = spack.compilers.compilers_for_spec(spec) + if not compilers: + return None + compiler = compilers[0] + rungcc = None + libname = None + output = None + if compiler.cxx: + rungcc = Executable(compiler.cxx) + libname = "libstdc++.so" + elif compiler.cc: + rungcc = Executable(compiler.cc) + libname = "libgcc_s.so" + else: + return None + try: + output = rungcc("--print-file-name=%s" % libname, return_output=True) + except ProcessError, e: + return None + if not output: + return None + libpath = os.readlink(output.strip()) + if not libpath: + return None + return os.path.basename(libpath) + + + @memoized + def _gcc_compiler_compare(self, pversion, cversion): + """Returns true iff the gcc version pversion and cversion + are ABI compatible.""" + plib = self._gcc_get_libstdcxx_version(pversion) + clib = self._gcc_get_libstdcxx_version(cversion) + if not plib or not clib: + return False + return plib == clib + + + def _intel_compiler_compare(self, pversion, cversion): + """Returns true iff the intel version pversion and cversion + are ABI compatible""" + + # Test major and minor versions. Ignore build version. + if (len(pversion.version) < 2 or len(cversion.version) < 2): + return False + return pversion.version[:2] == cversion.version[:2] + + + def compiler_compatible(self, parent, child, **kwargs): + """Returns true iff the compilers for parent and child specs are ABI compatible""" + if not parent.compiler or not child.compiler: + return True + + if parent.compiler.name != child.compiler.name: + # Different compiler families are assumed ABI incompatible + return False + + if kwargs.get('loose', False): + return True + + # TODO: Can we move the specialized ABI matching stuff + # TODO: into compiler classes? + for pversion in parent.compiler.versions: + for cversion in child.compiler.versions: + # For a few compilers use specialized comparisons. Otherwise + # match on version match. + if pversion.satisfies(cversion): + return True + elif (parent.compiler.name == "gcc" and + self._gcc_compiler_compare(pversion, cversion)): + return True + elif (parent.compiler.name == "intel" and + self._intel_compiler_compare(pversion, cversion)): + return True + return False + + + def compatible(self, parent, child, **kwargs): + """Returns true iff a parent and child spec are ABI compatible""" + loosematch = kwargs.get('loose', False) + return self.architecture_compatible(parent, child) and \ + self.compiler_compatible(parent, child, loose=loosematch) diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index b2db83acb7..119a255a34 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -3,7 +3,7 @@ This module contains all routines related to setting up the package build environment. All of this is set up by package.py just before install() is called. -There are two parts to the bulid environment: +There are two parts to the build environment: 1. Python build environment (i.e. install() method) @@ -13,7 +13,7 @@ There are two parts to the bulid environment: the package's module scope. Ths allows package writers to call them all directly in Package.install() without writing 'self.' everywhere. No, this isn't Pythonic. Yes, it makes the code more - readable and more like the shell script from whcih someone is + readable and more like the shell script from which someone is likely porting. 2. Build execution environment @@ -27,17 +27,18 @@ There are two parts to the bulid environment: Skimming this module is a nice way to get acquainted with the types of calls you can make from within the install() function. """ -import os -import sys -import shutil import multiprocessing +import os import platform -from llnl.util.filesystem import * +import shutil +import sys import spack -import spack.compilers as compilers -from spack.util.executable import Executable, which +import llnl.util.tty as tty +from llnl.util.filesystem import * +from spack.environment import EnvironmentModifications, validate from spack.util.environment import * +from spack.util.executable import Executable, which # # This can be set by the user to globally disable parallel builds. @@ -83,85 +84,88 @@ class MakeExecutable(Executable): return super(MakeExecutable, self).__call__(*args, **kwargs) -def set_compiler_environment_variables(pkg): - assert(pkg.spec.concrete) - compiler = pkg.compiler - +def set_compiler_environment_variables(pkg, env): + assert pkg.spec.concrete # Set compiler variables used by CMake and autotools - assert all(key in pkg.compiler.link_paths - for key in ('cc', 'cxx', 'f77', 'fc')) + assert all(key in pkg.compiler.link_paths for key in ('cc', 'cxx', 'f77', 'fc')) + # Populate an object with the list of environment modifications + # and return it + # TODO : add additional kwargs for better diagnostics, like requestor, ttyout, ttyerr, etc. link_dir = spack.build_env_path - os.environ['CC'] = join_path(link_dir, pkg.compiler.link_paths['cc']) - os.environ['CXX'] = join_path(link_dir, pkg.compiler.link_paths['cxx']) - os.environ['F77'] = join_path(link_dir, pkg.compiler.link_paths['f77']) - os.environ['FC'] = join_path(link_dir, pkg.compiler.link_paths['fc']) + env.set('CC', join_path(link_dir, pkg.compiler.link_paths['cc'])) + env.set('CXX', join_path(link_dir, pkg.compiler.link_paths['cxx'])) + env.set('F77', join_path(link_dir, pkg.compiler.link_paths['f77'])) + env.set('FC', join_path(link_dir, pkg.compiler.link_paths['fc'])) # Set SPACK compiler variables so that our wrapper knows what to call + compiler = pkg.compiler if compiler.cc: - os.environ['SPACK_CC'] = compiler.cc + env.set('SPACK_CC', compiler.cc) if compiler.cxx: - os.environ['SPACK_CXX'] = compiler.cxx + env.set('SPACK_CXX', compiler.cxx) if compiler.f77: - os.environ['SPACK_F77'] = compiler.f77 + env.set('SPACK_F77', compiler.f77) if compiler.fc: - os.environ['SPACK_FC'] = compiler.fc + env.set('SPACK_FC', compiler.fc) - os.environ['SPACK_COMPILER_SPEC'] = str(pkg.spec.compiler) + env.set('SPACK_COMPILER_SPEC', str(pkg.spec.compiler)) + return env -def set_build_environment_variables(pkg): - """This ensures a clean install environment when we build packages. +def set_build_environment_variables(pkg, env): + """ + This ensures a clean install environment when we build packages """ # Add spack build environment path with compiler wrappers first in # the path. We add both spack.env_path, which includes default # wrappers (cc, c++, f77, f90), AND a subdirectory containing # compiler-specific symlinks. The latter ensures that builds that # are sensitive to the *name* of the compiler see the right name - # when we're building wtih the wrappers. + # when we're building with the wrappers. # # Conflicts on case-insensitive systems (like "CC" and "cc") are # handled by putting one in the <build_env_path>/case-insensitive # directory. Add that to the path too. env_paths = [] - def add_env_path(path): - env_paths.append(path) - ci = join_path(path, 'case-insensitive') - if os.path.isdir(ci): env_paths.append(ci) - add_env_path(spack.build_env_path) - add_env_path(join_path(spack.build_env_path, pkg.compiler.name)) - - path_put_first("PATH", env_paths) - path_set(SPACK_ENV_PATH, env_paths) - - # Prefixes of all of the package's dependencies go in - # SPACK_DEPENDENCIES + for item in [spack.build_env_path, join_path(spack.build_env_path, pkg.compiler.name)]: + env_paths.append(item) + ci = join_path(item, 'case-insensitive') + if os.path.isdir(ci): + env_paths.append(ci) + + for item in reversed(env_paths): + env.prepend_path('PATH', item) + env.set_path(SPACK_ENV_PATH, env_paths) + + # Prefixes of all of the package's dependencies go in SPACK_DEPENDENCIES dep_prefixes = [d.prefix for d in pkg.spec.traverse(root=False)] - path_set(SPACK_DEPENDENCIES, dep_prefixes) + env.set_path(SPACK_DEPENDENCIES, dep_prefixes) + env.set_path('CMAKE_PREFIX_PATH', dep_prefixes) # Add dependencies to CMAKE_PREFIX_PATH # Install prefix - os.environ[SPACK_PREFIX] = pkg.prefix + env.set(SPACK_PREFIX, pkg.prefix) # Install root prefix - os.environ[SPACK_INSTALL] = spack.install_path + env.set(SPACK_INSTALL, spack.install_path) # Remove these vars from the environment during build because they # can affect how some packages find libraries. We want to make # sure that builds never pull in unintended external dependencies. - pop_keys(os.environ, "LD_LIBRARY_PATH", "LD_RUN_PATH", "DYLD_LIBRARY_PATH") + env.unset('LD_LIBRARY_PATH') + env.unset('LD_RUN_PATH') + env.unset('DYLD_LIBRARY_PATH') # Add bin directories from dependencies to the PATH for the build. - bin_dirs = ['%s/bin' % prefix for prefix in dep_prefixes] - path_put_first('PATH', [bin for bin in bin_dirs if os.path.isdir(bin)]) + bin_dirs = reversed(filter(os.path.isdir, ['%s/bin' % prefix for prefix in dep_prefixes])) + for item in bin_dirs: + env.prepend_path('PATH', item) # Working directory for the spack command itself, for debug logs. if spack.debug: - os.environ[SPACK_DEBUG] = "TRUE" - os.environ[SPACK_SHORT_SPEC] = pkg.spec.short_spec - os.environ[SPACK_DEBUG_LOG_DIR] = spack.spack_working_dir - - # Add dependencies to CMAKE_PREFIX_PATH - path_set("CMAKE_PREFIX_PATH", dep_prefixes) + env.set(SPACK_DEBUG, 'TRUE') + env.set(SPACK_SHORT_SPEC, pkg.spec.short_spec) + env.set(SPACK_DEBUG_LOG_DIR, spack.spack_working_dir) # Add any pkgconfig directories to PKG_CONFIG_PATH pkg_config_dirs = [] @@ -170,21 +174,23 @@ def set_build_environment_variables(pkg): pcdir = join_path(p, libdir, 'pkgconfig') if os.path.isdir(pcdir): pkg_config_dirs.append(pcdir) - path_set("PKG_CONFIG_PATH", pkg_config_dirs) + env.set_path('PKG_CONFIG_PATH', pkg_config_dirs) + + return env -def set_module_variables_for_package(pkg, m): +def set_module_variables_for_package(pkg, module): """Populate the module scope of install() with some useful functions. This makes things easier for package writers. """ - m = pkg.module - # number of jobs spack will to build with. jobs = multiprocessing.cpu_count() if not pkg.parallel: jobs = 1 elif pkg.make_jobs: jobs = pkg.make_jobs + + m = module m.make_jobs = jobs # TODO: make these build deps that can be installed if not found. @@ -214,6 +220,13 @@ def set_module_variables_for_package(pkg, m): m.std_cmake_args.append('-DCMAKE_INSTALL_RPATH_USE_LINK_PATH=FALSE') m.std_cmake_args.append('-DCMAKE_INSTALL_RPATH=%s' % ":".join(get_rpaths(pkg))) + # Put spack compiler paths in module scope. + link_dir = spack.build_env_path + m.spack_cc = join_path(link_dir, pkg.compiler.link_paths['cc']) + m.spack_cxx = join_path(link_dir, pkg.compiler.link_paths['cxx']) + m.spack_f77 = join_path(link_dir, pkg.compiler.link_paths['f77']) + m.spack_fc = join_path(link_dir, pkg.compiler.link_paths['fc']) + # Emulate some shell commands for convenience m.pwd = os.getcwd m.cd = os.chdir @@ -237,9 +250,9 @@ def set_module_variables_for_package(pkg, m): def get_rpaths(pkg): """Get a list of all the rpaths for a package.""" rpaths = [pkg.prefix.lib, pkg.prefix.lib64] - rpaths.extend(d.prefix.lib for d in pkg.spec.traverse(root=False) + rpaths.extend(d.prefix.lib for d in pkg.spec.dependencies.values() if os.path.isdir(d.prefix.lib)) - rpaths.extend(d.prefix.lib64 for d in pkg.spec.traverse(root=False) + rpaths.extend(d.prefix.lib64 for d in pkg.spec.dependencies.values() if os.path.isdir(d.prefix.lib64)) return rpaths @@ -257,24 +270,63 @@ def parent_class_modules(cls): return result +def setup_module_variables_for_dag(pkg): + """Set module-scope variables for all packages in the DAG.""" + for spec in pkg.spec.traverse(order='post'): + # If a user makes their own package repo, e.g. + # spack.repos.mystuff.libelf.Libelf, and they inherit from + # an existing class like spack.repos.original.libelf.Libelf, + # then set the module variables for both classes so the + # parent class can still use them if it gets called. + spkg = spec.package + modules = parent_class_modules(spkg.__class__) + for mod in modules: + set_module_variables_for_package(spkg, mod) + set_module_variables_for_package(spkg, spkg.module) + + def setup_package(pkg): """Execute all environment setup routines.""" - set_compiler_environment_variables(pkg) - set_build_environment_variables(pkg) + spack_env = EnvironmentModifications() + run_env = EnvironmentModifications() + + # Before proceeding, ensure that specs and packages are consistent + # + # This is a confusing behavior due to how packages are + # constructed. `setup_dependent_package` may set attributes on + # specs in the DAG for use by other packages' install + # method. However, spec.package will look up a package via + # spack.repo, which defensively copies specs into packages. This + # code ensures that all packages in the DAG have pieces of the + # same spec object at build time. + # + # This is safe for the build process, b/c the build process is a + # throwaway environment, but it is kind of dirty. + # + # TODO: Think about how to avoid this fix and do something cleaner. + for s in pkg.spec.traverse(): s.package.spec = s + + set_compiler_environment_variables(pkg, spack_env) + set_build_environment_variables(pkg, spack_env) + setup_module_variables_for_dag(pkg) + + # Allow dependencies to modify the module + spec = pkg.spec + for dependency_spec in spec.traverse(root=False): + dpkg = dependency_spec.package + dpkg.setup_dependent_package(pkg.module, spec) + + # Allow dependencies to set up environment as well + for dependency_spec in spec.traverse(root=False): + dpkg = dependency_spec.package + dpkg.setup_dependent_environment(spack_env, run_env, spec) - # If a user makes their own package repo, e.g. - # spack.repos.mystuff.libelf.Libelf, and they inherit from - # an existing class like spack.repos.original.libelf.Libelf, - # then set the module variables for both classes so the - # parent class can still use them if it gets called. - modules = parent_class_modules(pkg.__class__) - for mod in modules: - set_module_variables_for_package(pkg, mod) + # Allow the package to apply some settings. + pkg.setup_environment(spack_env, run_env) - # Allow dependencies to set up environment as well. - for dep_spec in pkg.spec.traverse(root=False): - dep_spec.package.setup_dependent_environment( - pkg.module, dep_spec, pkg.spec) + # Make sure nothing's strange about the Spack environment. + validate(spack_env, tty.warn) + spack_env.apply_modifications() def fork(pkg, function): @@ -291,23 +343,23 @@ def fork(pkg, function): # do stuff build_env.fork(pkg, child_fun) - Forked processes are run with the build environemnt set up by + Forked processes are run with the build environment set up by spack.build_environment. This allows package authors to have - full control over the environment, etc. without offecting + full control over the environment, etc. without affecting other builds that might be executed in the same spack call. - If something goes wrong, the child process is expected toprint + If something goes wrong, the child process is expected to print the error and the parent process will exit with error as well. If things go well, the child exits and the parent carries on. """ try: pid = os.fork() - except OSError, e: + except OSError as e: raise InstallError("Unable to fork build process: %s" % e) if pid == 0: - # Give the child process the package's build environemnt. + # Give the child process the package's build environment. setup_package(pkg) try: @@ -318,7 +370,7 @@ def fork(pkg, function): # which interferes with unit tests. os._exit(0) - except spack.error.SpackError, e: + except spack.error.SpackError as e: e.die() except: @@ -333,8 +385,7 @@ def fork(pkg, function): # message. Just make the parent exit with an error code. pid, returncode = os.waitpid(pid, 0) if returncode != 0: - raise InstallError("Installation process had nonzero exit code." - .format(str(returncode))) + raise InstallError("Installation process had nonzero exit code.".format(str(returncode))) class InstallError(spack.error.SpackError): diff --git a/lib/spack/spack/cmd/bootstrap.py b/lib/spack/spack/cmd/bootstrap.py index e4ec7da35d..bdbd623b39 100644 --- a/lib/spack/spack/cmd/bootstrap.py +++ b/lib/spack/spack/cmd/bootstrap.py @@ -42,7 +42,7 @@ def get_origin_url(): git = which('git', required=True) origin_url = git( '--git-dir=%s' % git_dir, 'config', '--get', 'remote.origin.url', - return_output=True) + output=str) return origin_url.strip() diff --git a/lib/spack/spack/cmd/checksum.py b/lib/spack/spack/cmd/checksum.py index b1ad89dbb8..518d2703dc 100644 --- a/lib/spack/spack/cmd/checksum.py +++ b/lib/spack/spack/cmd/checksum.py @@ -22,23 +22,18 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import os -import re import argparse import hashlib -from pprint import pprint -from subprocess import CalledProcessError import llnl.util.tty as tty -from llnl.util.tty.colify import colify - import spack import spack.cmd import spack.util.crypto from spack.stage import Stage, FailedDownloadError from spack.version import * -description ="Checksum available versions of a package." +description = "Checksum available versions of a package." + def setup_parser(subparser): subparser.add_argument( @@ -58,25 +53,23 @@ def get_checksums(versions, urls, **kwargs): tty.msg("Downloading...") hashes = [] - for i, (url, version) in enumerate(zip(urls, versions)): - stage = Stage(url) + i = 0 + for url, version in zip(urls, versions): try: - stage.fetch() - if i == 0 and first_stage_function: - first_stage_function(stage) - - hashes.append( - spack.util.crypto.checksum(hashlib.md5, stage.archive_file)) - except FailedDownloadError, e: + with Stage(url, keep=keep_stage) as stage: + stage.fetch() + if i == 0 and first_stage_function: + first_stage_function(stage) + + hashes.append((version, + spack.util.crypto.checksum(hashlib.md5, stage.archive_file))) + i += 1 + except FailedDownloadError as e: tty.msg("Failed to fetch %s" % url) - continue - - finally: - if not keep_stage: - stage.destroy() - - return zip(versions, hashes) + except Exception as e: + tty.msg('Something failed on %s, skipping.\n (%s)' % (url, e)) + return hashes def checksum(parser, args): @@ -95,13 +88,13 @@ def checksum(parser, args): else: versions = pkg.fetch_remote_versions() if not versions: - tty.die("Could not fetch any versions for %s." % pkg.name) + tty.die("Could not fetch any versions for %s" % pkg.name) sorted_versions = sorted(versions, reverse=True) - tty.msg("Found %s versions of %s." % (len(versions), pkg.name), + tty.msg("Found %s versions of %s" % (len(versions), pkg.name), *spack.cmd.elide_list( - ["%-10s%s" % (v, versions[v]) for v in sorted_versions])) + ["%-10s%s" % (v, versions[v]) for v in sorted_versions])) print archives_to_fetch = tty.get_number( "How many would you like to checksum?", default=5, abort='q') @@ -116,7 +109,7 @@ def checksum(parser, args): keep_stage=args.keep_stage) if not version_hashes: - tty.die("Could not fetch any versions for %s." % pkg.name) + tty.die("Could not fetch any versions for %s" % pkg.name) version_lines = [" version('%s', '%s')" % (v, h) for v, h in version_hashes] tty.msg("Checksummed new versions of %s:" % pkg.name, *version_lines) diff --git a/lib/spack/spack/cmd/compiler.py b/lib/spack/spack/cmd/compiler.py index 75b51f6b49..3e58e82184 100644 --- a/lib/spack/spack/cmd/compiler.py +++ b/lib/spack/spack/cmd/compiler.py @@ -96,7 +96,7 @@ def compiler_remove(args): compilers = spack.compilers.compilers_for_spec(cspec, scope=args.scope) if not compilers: - tty.die("No compilers match spec %s." % cspec) + tty.die("No compilers match spec %s" % cspec) elif not args.all and len(compilers) > 1: tty.error("Multiple compilers match spec %s. Choose one:" % cspec) colify(reversed(sorted([c.spec for c in compilers])), indent=4) @@ -105,7 +105,7 @@ def compiler_remove(args): for compiler in compilers: spack.compilers.remove_compiler_from_config(compiler.spec, scope=args.scope) - tty.msg("Removed compiler %s." % compiler.spec) + tty.msg("Removed compiler %s" % compiler.spec) def compiler_info(args): @@ -114,7 +114,7 @@ def compiler_info(args): compilers = spack.compilers.compilers_for_spec(cspec, scope=args.scope) if not compilers: - tty.error("No compilers match spec %s." % cspec) + tty.error("No compilers match spec %s" % cspec) else: for c in compilers: print str(c.spec) + ":" diff --git a/lib/spack/spack/cmd/create.py b/lib/spack/spack/cmd/create.py index 7cea39cb55..f0cd50b8df 100644 --- a/lib/spack/spack/cmd/create.py +++ b/lib/spack/spack/cmd/create.py @@ -132,7 +132,7 @@ class ConfigureGuesser(object): # Peek inside the tarball. tar = which('tar') output = tar( - "--exclude=*/*/*", "-tf", stage.archive_file, return_output=True) + "--exclude=*/*/*", "-tf", stage.archive_file, output=str) lines = output.split("\n") # Set the configure line to the one that matched. @@ -156,7 +156,7 @@ def guess_name_and_version(url, args): # Try to deduce name and version of the new package from the URL version = spack.url.parse_version(url) if not version: - tty.die("Couldn't guess a version string from %s." % url) + tty.die("Couldn't guess a version string from %s" % url) # Try to guess a name. If it doesn't work, allow the user to override. if args.alternate_name: @@ -189,7 +189,7 @@ def find_repository(spec, args): try: repo = Repo(repo_path) if spec.namespace and spec.namespace != repo.namespace: - tty.die("Can't create package with namespace %s in repo with namespace %s." + tty.die("Can't create package with namespace %s in repo with namespace %s" % (spec.namespace, repo.namespace)) except RepoError as e: tty.die(str(e)) @@ -208,7 +208,7 @@ def find_repository(spec, args): return repo -def fetch_tarballs(url, name, args): +def fetch_tarballs(url, name, version): """Try to find versions of the supplied archive by scraping the web. Prompts the user to select how many to download if many are found. @@ -252,11 +252,11 @@ def create(parser, args): name = spec.name # factors out namespace, if any repo = find_repository(spec, args) - tty.msg("This looks like a URL for %s version %s." % (name, version)) + tty.msg("This looks like a URL for %s version %s" % (name, version)) tty.msg("Creating template for package %s" % name) # Fetch tarballs (prompting user if necessary) - versions, urls = fetch_tarballs(url, name, args) + versions, urls = fetch_tarballs(url, name, version) # Try to guess what configure system is used. guesser = ConfigureGuesser() @@ -266,7 +266,7 @@ def create(parser, args): keep_stage=args.keep_stage) if not ver_hash_tuples: - tty.die("Could not fetch any tarballs for %s." % name) + tty.die("Could not fetch any tarballs for %s" % name) # Prepend 'py-' to python package names, by convention. if guesser.build_system == 'python': @@ -291,4 +291,4 @@ def create(parser, args): # If everything checks out, go ahead and edit. spack.editor(pkg_path) - tty.msg("Created package %s." % pkg_path) + tty.msg("Created package %s" % pkg_path) diff --git a/lib/spack/spack/cmd/deactivate.py b/lib/spack/spack/cmd/deactivate.py index a0c78bf755..d6b23d6a08 100644 --- a/lib/spack/spack/cmd/deactivate.py +++ b/lib/spack/spack/cmd/deactivate.py @@ -37,7 +37,7 @@ def setup_parser(subparser): help="Run deactivation even if spec is NOT currently activated.") subparser.add_argument( '-a', '--all', action='store_true', - help="Deactivate all extensions of an extendable pacakge, or " + help="Deactivate all extensions of an extendable package, or " "deactivate an extension AND its dependencies.") subparser.add_argument( 'spec', nargs=argparse.REMAINDER, help="spec of package extension to deactivate.") diff --git a/lib/spack/spack/cmd/diy.py b/lib/spack/spack/cmd/diy.py index 9df53312f8..45f13e4463 100644 --- a/lib/spack/spack/cmd/diy.py +++ b/lib/spack/spack/cmd/diy.py @@ -46,6 +46,9 @@ def setup_parser(subparser): '--skip-patch', action='store_true', help="Skip patching for the DIY build.") subparser.add_argument( + '-q', '--quiet', action='store_true', dest='quiet', + help="Do not display verbose build output while installing.") + subparser.add_argument( 'spec', nargs=argparse.REMAINDER, help="specs to use for install. Must contain package AND verison.") @@ -72,8 +75,8 @@ def diy(self, args): edit_package(spec.name, spack.repo.first_repo(), None, True) return - if not spec.version.concrete: - tty.die("spack diy spec must have a single, concrete version.") + if not spec.versions.concrete: + tty.die("spack diy spec must have a single, concrete version. Did you forget a package version number?") spec.concretize() package = spack.repo.get(spec) @@ -92,4 +95,5 @@ def diy(self, args): package.do_install( keep_prefix=args.keep_prefix, ignore_deps=args.ignore_deps, + verbose=not args.quiet, keep_stage=True) # don't remove source dir for DIY. diff --git a/lib/spack/spack/cmd/location.py b/lib/spack/spack/cmd/location.py index 39c225e9b2..307ee8982d 100644 --- a/lib/spack/spack/cmd/location.py +++ b/lib/spack/spack/cmd/location.py @@ -32,7 +32,7 @@ from llnl.util.filesystem import join_path import spack import spack.cmd -description="Print out locations of various diectories used by Spack" +description="Print out locations of various directories used by Spack" def setup_parser(subparser): global directories diff --git a/lib/spack/spack/cmd/md5.py b/lib/spack/spack/cmd/md5.py index ef1e4f3475..f99fc0f8c2 100644 --- a/lib/spack/spack/cmd/md5.py +++ b/lib/spack/spack/cmd/md5.py @@ -22,32 +22,51 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import os -import hashlib import argparse +import hashlib +import os import llnl.util.tty as tty -from llnl.util.filesystem import * - import spack.util.crypto +from spack.stage import Stage, FailedDownloadError + +description = "Calculate md5 checksums for files/urls." -description = "Calculate md5 checksums for files." def setup_parser(subparser): setup_parser.parser = subparser subparser.add_argument('files', nargs=argparse.REMAINDER, help="Files to checksum.") + +def compute_md5_checksum(url): + if not os.path.isfile(url): + with Stage(url) as stage: + stage.fetch() + value = spack.util.crypto.checksum(hashlib.md5, stage.archive_file) + else: + value = spack.util.crypto.checksum(hashlib.md5, url) + return value + + def md5(parser, args): if not args.files: setup_parser.parser.print_help() return 1 - for f in args.files: - if not os.path.isfile(f): - tty.die("Not a file: %s" % f) - if not can_access(f): - tty.die("Cannot read file: %s" % f) + results = [] + for url in args.files: + try: + checksum = compute_md5_checksum(url) + results.append((checksum, url)) + except FailedDownloadError as e: + tty.warn("Failed to fetch %s" % url) + tty.warn("%s" % e) + except IOError as e: + tty.warn("Error when reading %s" % url) + tty.warn("%s" % e) - checksum = spack.util.crypto.checksum(hashlib.md5, f) - print "%s %s" % (checksum, f) + # Dump the MD5s at last without interleaving them with downloads + tty.msg("%d MD5 checksums:" % len(results)) + for checksum, url in results: + print "%s %s" % (checksum, url) diff --git a/lib/spack/spack/cmd/mirror.py b/lib/spack/spack/cmd/mirror.py index 885483a840..fcd15a6a90 100644 --- a/lib/spack/spack/cmd/mirror.py +++ b/lib/spack/spack/cmd/mirror.py @@ -53,11 +53,13 @@ def setup_parser(subparser): create_parser.add_argument('-d', '--directory', default=None, help="Directory in which to create mirror.") create_parser.add_argument( - 'specs', nargs=argparse.REMAINDER, help="Specs of packages to put in mirror") + 'specs', nargs=argparse.REMAINDER, + help="Specs of packages to put in mirror") create_parser.add_argument( '-f', '--file', help="File with specs of packages to put in mirror.") create_parser.add_argument( - '-D', '--dependencies', action='store_true', help="Also fetch all dependencies") + '-D', '--dependencies', action='store_true', + help="Also fetch all dependencies") create_parser.add_argument( '-o', '--one-version-per-spec', action='store_const', const=1, default=0, help="Only fetch one 'preferred' version per spec, not all known versions.") @@ -74,7 +76,8 @@ def setup_parser(subparser): help="Configuration scope to modify.") # Remove - remove_parser = sp.add_parser('remove', aliases=['rm'], help=mirror_remove.__doc__) + remove_parser = sp.add_parser('remove', aliases=['rm'], + help=mirror_remove.__doc__) remove_parser.add_argument('name') remove_parser.add_argument( '--scope', choices=scopes, default=spack.cmd.default_modify_scope, @@ -123,7 +126,7 @@ def mirror_remove(args): old_value = mirrors.pop(name) spack.config.update_config('mirrors', mirrors, scope=args.scope) - tty.msg("Removed mirror %s with url %s." % (name, old_value)) + tty.msg("Removed mirror %s with url %s" % (name, old_value)) def mirror_list(args): @@ -141,15 +144,17 @@ def mirror_list(args): def _read_specs_from_file(filename): + specs = [] with open(filename, "r") as stream: for i, string in enumerate(stream): try: s = Spec(string) s.package - args.specs.append(s) + specs.append(s) except SpackError, e: tty.die("Parse error in %s, line %d:" % (args.file, i+1), ">>> " + string, str(e)) + return specs def mirror_create(args): @@ -169,6 +174,7 @@ def mirror_create(args): specs = [Spec(n) for n in spack.repo.all_package_names()] specs.sort(key=lambda s: s.format("$_$@").lower()) + # If the user asked for dependencies, traverse spec DAG get them. if args.dependencies: new_specs = set() for spec in specs: @@ -197,7 +203,7 @@ def mirror_create(args): verb = "updated" if existed else "created" tty.msg( - "Successfully %s mirror in %s." % (verb, directory), + "Successfully %s mirror in %s" % (verb, directory), "Archive stats:", " %-4d already present" % p, " %-4d added" % m, diff --git a/lib/spack/spack/cmd/module.py b/lib/spack/spack/cmd/module.py index a5a9570eb5..315d9fc926 100644 --- a/lib/spack/spack/cmd/module.py +++ b/lib/spack/spack/cmd/module.py @@ -58,7 +58,7 @@ def module_find(mtype, spec_array): should type to use that package's module. """ if mtype not in module_types: - tty.die("Invalid module type: '%s'. Options are %s." % (mtype, comma_or(module_types))) + tty.die("Invalid module type: '%s'. Options are %s" % (mtype, comma_or(module_types))) specs = spack.cmd.parse_specs(spec_array) if len(specs) > 1: @@ -78,9 +78,9 @@ def module_find(mtype, spec_array): mt = module_types[mtype] mod = mt(specs[0]) if not os.path.isfile(mod.file_name): - tty.die("No %s module is installed for %s." % (mtype, spec)) + tty.die("No %s module is installed for %s" % (mtype, spec)) - print mod.use_name + print(mod.use_name) def module_refresh(): @@ -94,7 +94,7 @@ def module_refresh(): shutil.rmtree(cls.path, ignore_errors=False) mkdirp(cls.path) for spec in specs: - tty.debug(" Writing file for %s." % spec) + tty.debug(" Writing file for %s" % spec) cls(spec).write() diff --git a/lib/spack/spack/cmd/patch.py b/lib/spack/spack/cmd/patch.py index 44fc8696db..b04b402738 100644 --- a/lib/spack/spack/cmd/patch.py +++ b/lib/spack/spack/cmd/patch.py @@ -24,6 +24,7 @@ ############################################################################## import argparse +import llnl.util.tty as tty import spack.cmd import spack diff --git a/lib/spack/spack/cmd/pkg.py b/lib/spack/spack/cmd/pkg.py index 448f762841..cf478d3763 100644 --- a/lib/spack/spack/cmd/pkg.py +++ b/lib/spack/spack/cmd/pkg.py @@ -79,7 +79,7 @@ def list_packages(rev): git = get_git() relpath = spack.packages_path[len(spack.prefix + os.path.sep):] + os.path.sep output = git('ls-tree', '--full-tree', '--name-only', rev, relpath, - return_output=True) + output=str) return sorted(line[len(relpath):] for line in output.split('\n') if line) diff --git a/lib/spack/spack/cmd/repo.py b/lib/spack/spack/cmd/repo.py index 34c755fb67..87c782000f 100644 --- a/lib/spack/spack/cmd/repo.py +++ b/lib/spack/spack/cmd/repo.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://llnl.github.io/spack +# For details, see https://software.llnl.gov/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify @@ -74,51 +74,7 @@ def setup_parser(subparser): def repo_create(args): """Create a new package repository.""" - root = canonicalize_path(args.directory) - namespace = args.namespace - - if not args.namespace: - namespace = os.path.basename(root) - - if not re.match(r'\w[\.\w-]*', namespace): - tty.die("'%s' is not a valid namespace." % namespace) - - existed = False - if os.path.exists(root): - if os.path.isfile(root): - tty.die('File %s already exists and is not a directory' % root) - elif os.path.isdir(root): - if not os.access(root, os.R_OK | os.W_OK): - tty.die('Cannot create new repo in %s: cannot access directory.' % root) - if os.listdir(root): - tty.die('Cannot create new repo in %s: directory is not empty.' % root) - existed = True - - full_path = os.path.realpath(root) - parent = os.path.dirname(full_path) - if not os.access(parent, os.R_OK | os.W_OK): - tty.die("Cannot create repository in %s: can't access parent!" % root) - - try: - config_path = os.path.join(root, repo_config_name) - packages_path = os.path.join(root, packages_dir_name) - - mkdirp(packages_path) - with open(config_path, 'w') as config: - config.write("repo:\n") - config.write(" namespace: '%s'\n" % namespace) - - except (IOError, OSError) as e: - tty.die('Failed to create new repository in %s.' % root, - "Caused by %s: %s" % (type(e), e)) - - # try to clean up. - if existed: - shutil.rmtree(config_path, ignore_errors=True) - shutil.rmtree(packages_path, ignore_errors=True) - else: - shutil.rmtree(root, ignore_errors=True) - + full_path, namespace = create_repo(args.directory, args.namespace) tty.msg("Created repo with namespace '%s'." % namespace) tty.msg("To register it with spack, run this command:", 'spack repo add %s' % full_path) @@ -133,11 +89,11 @@ def repo_add(args): # check if the path exists if not os.path.exists(canon_path): - tty.die("No such file or directory: '%s'." % path) + tty.die("No such file or directory: %s" % path) # Make sure the path is a directory. if not os.path.isdir(canon_path): - tty.die("Not a Spack repository: '%s'." % path) + tty.die("Not a Spack repository: %s" % path) # Make sure it's actually a spack repository by constructing it. repo = Repo(canon_path) @@ -147,7 +103,7 @@ def repo_add(args): if not repos: repos = [] if repo.root in repos or path in repos: - tty.die("Repository is already registered with Spack: '%s'" % path) + tty.die("Repository is already registered with Spack: %s" % path) repos.insert(0, canon_path) spack.config.update_config('repos', repos, args.scope) @@ -166,7 +122,7 @@ def repo_remove(args): if canon_path == repo_canon_path: repos.remove(repo_path) spack.config.update_config('repos', repos, args.scope) - tty.msg("Removed repository '%s'." % repo_path) + tty.msg("Removed repository %s" % repo_path) return # If it is a namespace, remove corresponding repo @@ -176,13 +132,13 @@ def repo_remove(args): if repo.namespace == path_or_namespace: repos.remove(path) spack.config.update_config('repos', repos, args.scope) - tty.msg("Removed repository '%s' with namespace %s." + tty.msg("Removed repository %s with namespace '%s'." % (repo.root, repo.namespace)) return except RepoError as e: continue - tty.die("No repository with path or namespace: '%s'" + tty.die("No repository with path or namespace: %s" % path_or_namespace) diff --git a/lib/spack/spack/cmd/test-install.py b/lib/spack/spack/cmd/test-install.py index 74017f59fb..656873a2f0 100644 --- a/lib/spack/spack/cmd/test-install.py +++ b/lib/spack/spack/cmd/test-install.py @@ -37,20 +37,20 @@ from spack.build_environment import InstallError from spack.fetch_strategy import FetchError import spack.cmd -description = "Treat package installations as unit tests and output formatted test results" +description = "Run package installation as a unit test, output formatted results." def setup_parser(subparser): subparser.add_argument( '-j', '--jobs', action='store', type=int, help="Explicitly set number of make jobs. Default is #cpus.") - + subparser.add_argument( '-n', '--no-checksum', action='store_true', dest='no_checksum', help="Do not check packages against checksum") - + subparser.add_argument( '-o', '--output', action='store', help="test output goes in this file") - + subparser.add_argument( 'package', nargs=argparse.REMAINDER, help="spec of package to install") @@ -59,10 +59,10 @@ class JunitResultFormat(object): def __init__(self): self.root = ET.Element('testsuite') self.tests = [] - + def add_test(self, buildId, testResult, buildInfo=None): self.tests.append((buildId, testResult, buildInfo)) - + def write_to(self, stream): self.root.set('tests', '{0}'.format(len(self.tests))) for buildId, testResult, buildInfo in self.tests: @@ -84,25 +84,25 @@ class TestResult(object): PASSED = 0 FAILED = 1 SKIPPED = 2 - + class BuildId(object): def __init__(self, spec): self.name = spec.name self.version = spec.version self.hashId = spec.dag_hash() - + def stringId(self): return "-".join(str(x) for x in (self.name, self.version, self.hashId)) def __hash__(self): return hash((self.name, self.version, self.hashId)) - + def __eq__(self, other): if not isinstance(other, BuildId): return False - - return ((self.name, self.version, self.hashId) == + + return ((self.name, self.version, self.hashId) == (other.name, other.version, other.hashId)) @@ -114,12 +114,12 @@ def fetch_log(path): def failed_dependencies(spec): - return set(childSpec for childSpec in spec.dependencies.itervalues() if not + return set(childSpec for childSpec in spec.dependencies.itervalues() if not spack.repo.get(childSpec).installed) def create_test_output(topSpec, newInstalls, output, getLogFunc=fetch_log): - # Post-order traversal is not strictly required but it makes sense to output + # Post-order traversal is not strictly required but it makes sense to output # tests for dependencies first. for spec in topSpec.traverse(order='post'): if spec not in newInstalls: @@ -143,12 +143,12 @@ def create_test_output(topSpec, newInstalls, output, getLogFunc=fetch_log): re.search('error:', line, re.IGNORECASE)) errOutput = errMessages if errMessages else lines[-10:] errOutput = '\n'.join(itertools.chain( - [spec.to_yaml(), "Errors:"], errOutput, + [spec.to_yaml(), "Errors:"], errOutput, ["Build Log:", package.build_log_path])) else: result = TestResult.PASSED errOutput = None - + bId = BuildId(spec) output.add_test(bId, result, errOutput) @@ -163,18 +163,18 @@ def test_install(parser, args): if args.no_checksum: spack.do_checksum = False # TODO: remove this global. - + specs = spack.cmd.parse_specs(args.package, concretize=True) if len(specs) > 1: tty.die("Only 1 top-level package can be specified") topSpec = iter(specs).next() - + newInstalls = set() for spec in topSpec.traverse(): package = spack.repo.get(spec) if not package.installed: newInstalls.add(spec) - + if not args.output: bId = BuildId(topSpec) outputDir = join_path(os.getcwd(), "test-output") @@ -183,7 +183,7 @@ def test_install(parser, args): outputFpath = join_path(outputDir, "test-{0}.xml".format(bId.stringId())) else: outputFpath = args.output - + for spec in topSpec.traverse(order='post'): # Calling do_install for the top-level package would be sufficient but # this attempts to keep going if any package fails (other packages which @@ -202,7 +202,7 @@ def test_install(parser, args): pass except FetchError: pass - + jrf = JunitResultFormat() handled = {} create_test_output(topSpec, newInstalls, jrf) diff --git a/lib/spack/spack/cmd/uninstall.py b/lib/spack/spack/cmd/uninstall.py index d01aa2136b..350ef372cb 100644 --- a/lib/spack/spack/cmd/uninstall.py +++ b/lib/spack/spack/cmd/uninstall.py @@ -22,6 +22,7 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## +from __future__ import print_function import sys import argparse @@ -63,12 +64,12 @@ def uninstall(parser, args): matching_specs = spack.installed_db.query(spec) if not args.all and len(matching_specs) > 1: tty.error("%s matches multiple packages:" % spec) - print + print() display_specs(matching_specs, long=True) - print - print "You can either:" - print " a) Use a more specific spec, or" - print " b) use spack uninstall -a to uninstall ALL matching specs." + print() + print("You can either:") + print(" a) Use a more specific spec, or") + print(" b) use spack uninstall -a to uninstall ALL matching specs.") sys.exit(1) if len(matching_specs) == 0: @@ -79,7 +80,7 @@ def uninstall(parser, args): try: # should work if package is known to spack pkgs.append(s.package) - except spack.repository.UnknownPackageError, e: + except spack.repository.UnknownPackageError as e: # The package.py file has gone away -- but still # want to uninstall. spack.Package(s).do_uninstall(force=True) @@ -94,11 +95,11 @@ def uninstall(parser, args): for pkg in pkgs: try: pkg.do_uninstall(force=args.force) - except PackageStillNeededError, e: + except PackageStillNeededError as e: tty.error("Will not uninstall %s" % e.spec.format("$_$@$%@$#", color=True)) - print - print "The following packages depend on it:" + print('') + print("The following packages depend on it:") display_specs(e.dependents, long=True) - print - print "You can use spack uninstall -f to force this action." + print('') + print("You can use spack uninstall -f to force this action.") sys.exit(1) diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index a665f6062d..d38c0b00b1 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -51,7 +51,7 @@ _version_cache = {} def get_compiler_version(compiler_path, version_arg, regex='(.*)'): if not compiler_path in _version_cache: compiler = Executable(compiler_path) - output = compiler(version_arg, return_output=True, error=os.devnull) + output = compiler(version_arg, output=str, error=str) match = re.search(regex, output) _version_cache[compiler_path] = match.group(1) if match else 'unknown' @@ -256,12 +256,12 @@ class Compiler(object): def __repr__(self): - """Return a string represntation of the compiler toolchain.""" + """Return a string representation of the compiler toolchain.""" return self.__str__() def __str__(self): - """Return a string represntation of the compiler toolchain.""" + """Return a string representation of the compiler toolchain.""" return "%s(%s)" % ( self.name, '\n '.join((str(s) for s in (self.cc, self.cxx, self.f77, self.fc)))) diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index facc9c338b..3a04bc2ebc 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -51,7 +51,7 @@ _required_instance_vars = ['cc', 'cxx', 'f77', 'fc'] if platform.system() == 'Darwin': _default_order = ['clang', 'gcc', 'intel'] else: - _default_order = ['gcc', 'intel', 'pgi', 'clang', 'xlc'] + _default_order = ['gcc', 'intel', 'pgi', 'clang', 'xlc', 'nag'] def _auto_compiler_spec(function): @@ -74,28 +74,36 @@ def _to_dict(compiler): def get_compiler_config(arch=None, scope=None): """Return the compiler configuration for the specified architecture. """ - # If any configuration file has compilers, just stick with the - # ones already configured. - config = spack.config.get_config('compilers', scope=scope) - + # Check whether we're on a front-end (native) architecture. my_arch = spack.architecture.sys_type() if arch is None: arch = my_arch - if arch in config: - return config[arch] - - # Only for the current arch in *highest* scope: automatically try to - # find compilers if none are configured yet. - if arch == my_arch and scope == 'user': + def init_compiler_config(): + """Compiler search used when Spack has no compilers.""" config[arch] = {} compilers = find_compilers(*get_path('PATH')) for compiler in compilers: config[arch].update(_to_dict(compiler)) spack.config.update_config('compilers', config, scope=scope) - return config[arch] - return {} + config = spack.config.get_config('compilers', scope=scope) + + # Update the configuration if there are currently no compilers + # configured. Avoid updating automatically if there ARE site + # compilers configured but no user ones. + if arch == my_arch and arch not in config: + if scope is None: + # We know no compilers were configured in any scope. + init_compiler_config() + elif scope == 'user': + # Check the site config and update the user config if + # nothing is configured at the site level. + site_config = spack.config.get_config('compilers', scope='site') + if not site_config: + init_compiler_config() + + return config[arch] if arch in config else {} def add_compilers_to_config(compilers, arch=None, scope=None): diff --git a/lib/spack/spack/compilers/clang.py b/lib/spack/spack/compilers/clang.py index 340051019c..e406d86a24 100644 --- a/lib/spack/spack/compilers/clang.py +++ b/lib/spack/spack/compilers/clang.py @@ -22,7 +22,10 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## +import re +import spack.compiler as cpr from spack.compiler import * +from spack.util.executable import * class Clang(Compiler): # Subclasses use possible names of C compiler @@ -47,11 +50,34 @@ class Clang(Compiler): @classmethod def default_version(self, comp): """The '--version' option works for clang compilers. - Output looks like this:: + On most platforms, output looks like this:: clang version 3.1 (trunk 149096) Target: x86_64-unknown-linux-gnu Thread model: posix + + On Mac OS X, it looks like this: + + Apple LLVM version 7.0.2 (clang-700.1.81) + Target: x86_64-apple-darwin15.2.0 + Thread model: posix + """ - return get_compiler_version( - comp, '--version', r'(?:clang version|based on LLVM) ([^ )]+)') + if comp not in cpr._version_cache: + compiler = Executable(comp) + output = compiler('--version', output=str, error=str) + + ver = 'unknown' + match = re.search(r'^Apple LLVM version ([^ )]+)', output) + if match: + # Apple's LLVM compiler has its own versions, so suffix them. + ver = match.group(1) + '-apple' + else: + # Normal clang compiler versions are left as-is + match = re.search(r'^clang version ([^ )]+)', output) + if match: + ver = match.group(1) + + cpr._version_cache[comp] = ver + + return cpr._version_cache[comp] diff --git a/lib/spack/spack/compilers/gcc.py b/lib/spack/spack/compilers/gcc.py index 495b638a3a..64214db32d 100644 --- a/lib/spack/spack/compilers/gcc.py +++ b/lib/spack/spack/compilers/gcc.py @@ -40,7 +40,8 @@ class Gcc(Compiler): fc_names = ['gfortran'] # MacPorts builds gcc versions with prefixes and -mp-X.Y suffixes. - suffixes = [r'-mp-\d\.\d'] + # Homebrew and Linuxes may build gcc with -X, -X.Y suffixes + suffixes = [r'-mp-\d\.\d', r'-\d\.\d', r'-\d'] # Named wrapper links within spack.build_env_path link_paths = {'cc' : 'gcc/gcc', diff --git a/lib/spack/spack/compilers/nag.py b/lib/spack/spack/compilers/nag.py new file mode 100644 index 0000000000..527a05a090 --- /dev/null +++ b/lib/spack/spack/compilers/nag.py @@ -0,0 +1,33 @@ +from spack.compiler import * + +class Nag(Compiler): + # Subclasses use possible names of C compiler + cc_names = [] + + # Subclasses use possible names of C++ compiler + cxx_names = [] + + # Subclasses use possible names of Fortran 77 compiler + f77_names = ['nagfor'] + + # Subclasses use possible names of Fortran 90 compiler + fc_names = ['nagfor'] + + # Named wrapper links within spack.build_env_path + link_paths = { # Use default wrappers for C and C++, in case provided in compilers.yaml + 'cc' : 'cc', + 'cxx' : 'c++', + 'f77' : 'nag/nagfor', + 'fc' : 'nag/nagfor' } + + @classmethod + def default_version(self, comp): + """The '-V' option works for nag compilers. + Output looks like this:: + + NAG Fortran Compiler Release 6.0(Hibiya) Build 1037 + Product NPL6A60NA for x86-64 Linux + Copyright 1990-2015 The Numerical Algorithms Group Ltd., Oxford, U.K. + """ + return get_compiler_version( + comp, '-V', r'NAG Fortran Compiler Release ([0-9.]+)') diff --git a/lib/spack/spack/compilers/pgi.py b/lib/spack/spack/compilers/pgi.py index 9ac74cfbdb..c6a1078bd9 100644 --- a/lib/spack/spack/compilers/pgi.py +++ b/lib/spack/spack/compilers/pgi.py @@ -29,28 +29,28 @@ class Pgi(Compiler): cc_names = ['pgcc'] # Subclasses use possible names of C++ compiler - cxx_names = ['pgCC'] + cxx_names = ['pgc++', 'pgCC'] # Subclasses use possible names of Fortran 77 compiler - f77_names = ['pgf77'] + f77_names = ['pgfortran', 'pgf77'] # Subclasses use possible names of Fortran 90 compiler - fc_names = ['pgf95', 'pgf90'] + fc_names = ['pgfortran', 'pgf95', 'pgf90'] # Named wrapper links within spack.build_env_path link_paths = { 'cc' : 'pgi/pgcc', - 'cxx' : 'pgi/case-insensitive/pgCC', - 'f77' : 'pgi/pgf77', - 'fc' : 'pgi/pgf90' } + 'cxx' : 'pgi/pgc++', + 'f77' : 'pgi/pgfortran', + 'fc' : 'pgi/pgfortran' } @classmethod def default_version(cls, comp): """The '-V' option works for all the PGI compilers. Output looks like this:: - pgf95 10.2-0 64-bit target on x86-64 Linux -tp nehalem-64 - Copyright 1989-2000, The Portland Group, Inc. All Rights Reserved. - Copyright 2000-2010, STMicroelectronics, Inc. All Rights Reserved. + pgcc 15.10-0 64-bit target on x86-64 Linux -tp sandybridge + The Portland Group - PGI Compilers and Tools + Copyright (c) 2015, NVIDIA CORPORATION. All rights reserved. """ return get_compiler_version( comp, '-V', r'pg[^ ]* ([^ ]+) \d\d\d?-bit target') diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 85cdb202d5..2e576743ec 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -33,12 +33,16 @@ or user preferences. TODO: make this customizable and allow users to configure concretization policies. """ +import spack import spack.spec import spack.compilers import spack.architecture import spack.error from spack.version import * - +from functools import partial +from spec import DependencyMap +from itertools import chain +from spack.config import * class DefaultConcretizer(object): """This class doesn't have any state, it just provides some methods for @@ -46,10 +50,92 @@ class DefaultConcretizer(object): default concretization strategies, or you can override all of them. """ + def _valid_virtuals_and_externals(self, spec): + """Returns a list of candidate virtual dep providers and external + packages that coiuld be used to concretize a spec.""" + # First construct a list of concrete candidates to replace spec with. + candidates = [spec] + if spec.virtual: + providers = spack.repo.providers_for(spec) + if not providers: + raise UnsatisfiableProviderSpecError(providers[0], spec) + spec_w_preferred_providers = find_spec( + spec, lambda(x): spack.pkgsort.spec_has_preferred_provider(x.name, spec.name)) + if not spec_w_preferred_providers: + spec_w_preferred_providers = spec + provider_cmp = partial(spack.pkgsort.provider_compare, spec_w_preferred_providers.name, spec.name) + candidates = sorted(providers, cmp=provider_cmp) + + # For each candidate package, if it has externals, add those to the usable list. + # if it's not buildable, then *only* add the externals. + usable = [] + for cspec in candidates: + if is_spec_buildable(cspec): + usable.append(cspec) + externals = spec_externals(cspec) + for ext in externals: + if ext.satisfies(spec): + usable.append(ext) + + # If nothing is in the usable list now, it's because we aren't + # allowed to build anything. + if not usable: + raise NoBuildError(spec) + + def cmp_externals(a, b): + if a.name != b.name: + # We're choosing between different providers, so + # maintain order from provider sort + return candidates.index(a) - candidates.index(b) + + result = cmp_specs(a, b) + if result != 0: + return result + + # prefer external packages to internal packages. + if a.external is None or b.external is None: + return -cmp(a.external, b.external) + else: + return cmp(a.external, b.external) + + usable.sort(cmp=cmp_externals) + return usable + + + def choose_virtual_or_external(self, spec): + """Given a list of candidate virtual and external packages, try to + find one that is most ABI compatible. + """ + candidates = self._valid_virtuals_and_externals(spec) + if not candidates: + return candidates + + # Find the nearest spec in the dag that has a compiler. We'll + # use that spec to calibrate compiler compatibility. + abi_exemplar = find_spec(spec, lambda(x): x.compiler) + if not abi_exemplar: + abi_exemplar = spec.root + + # Make a list including ABI compatibility of specs with the exemplar. + strict = [spack.abi.compatible(c, abi_exemplar) for c in candidates] + loose = [spack.abi.compatible(c, abi_exemplar, loose=True) for c in candidates] + keys = zip(strict, loose, candidates) + + # Sort candidates from most to least compatibility. + # Note: + # 1. We reverse because True > False. + # 2. Sort is stable, so c's keep their order. + keys.sort(key=lambda k:k[:2], reverse=True) + + # Pull the candidates back out and return them in order + candidates = [c for s,l,c in keys] + return candidates + + def concretize_version(self, spec): """If the spec is already concrete, return. Otherwise take - the most recent available version, and default to the package's - version if there are no avaialble versions. + the preferred version from spackconfig, and default to the package's + version if there are no available versions. TODO: In many cases we probably want to look for installed versions of each package and use an installed version @@ -67,20 +153,14 @@ class DefaultConcretizer(object): # If there are known available versions, return the most recent # version that satisfies the spec pkg = spec.package - - # Key function to sort versions first by whether they were - # marked `preferred=True`, then by most recent. - def preferred_key(v): - prefer = pkg.versions[v].get('preferred', False) - return (prefer, v) - + cmp_versions = partial(spack.pkgsort.version_compare, spec.name) valid_versions = sorted( [v for v in pkg.versions if any(v.satisfies(sv) for sv in spec.versions)], - key=preferred_key) + cmp=cmp_versions) if valid_versions: - spec.versions = ver([valid_versions[-1]]) + spec.versions = ver([valid_versions[0]]) else: # We don't know of any SAFE versions that match the given # spec. Grab the spec's versions and grab the highest @@ -134,7 +214,7 @@ class DefaultConcretizer(object): the default variants from the package specification. """ changed = False - for name, variant in spec.package.variants.items(): + for name, variant in spec.package_class.variants.items(): if name not in spec.variants: spec.variants[name] = spack.spec.VariantSpec(name, variant.default) changed = True @@ -145,10 +225,10 @@ class DefaultConcretizer(object): """If the spec already has a compiler, we're done. If not, then take the compiler used for the nearest ancestor with a compiler spec and use that. If the ancestor's compiler is not - concrete, then give it a valid version. If there is no - ancestor with a compiler, use the system default compiler. + concrete, then used the preferred compiler as specified in + spackconfig. - Intuition: Use the system default if no package that depends on + Intuition: Use the spackconfig default if no package that depends on this one has a strict compiler requirement. Otherwise, try to build with the compiler that will be used by libraries that link to this one, to maximize compatibility. @@ -160,40 +240,91 @@ class DefaultConcretizer(object): spec.compiler in all_compilers): return False - try: - nearest = next(p for p in spec.traverse(direction='parents') - if p.compiler is not None).compiler + #Find the another spec that has a compiler, or the root if none do + other_spec = spec if spec.compiler else find_spec(spec, lambda(x) : x.compiler) + if not other_spec: + other_spec = spec.root + other_compiler = other_spec.compiler + assert(other_spec) + + # Check if the compiler is already fully specified + if other_compiler in all_compilers: + spec.compiler = other_compiler.copy() + return True + + # Filter the compilers into a sorted list based on the compiler_order from spackconfig + compiler_list = all_compilers if not other_compiler else spack.compilers.find(other_compiler) + cmp_compilers = partial(spack.pkgsort.compiler_compare, other_spec.name) + matches = sorted(compiler_list, cmp=cmp_compilers) + if not matches: + raise UnavailableCompilerVersionError(other_compiler) + + # copy concrete version into other_compiler + spec.compiler = matches[0].copy() + assert(spec.compiler.concrete) + return True # things changed. - if not nearest in all_compilers: - # Take the newest compiler that saisfies the spec - matches = sorted(spack.compilers.find(nearest)) - if not matches: - raise UnavailableCompilerVersionError(nearest) - # copy concrete version into nearest spec - nearest.versions = matches[-1].versions.copy() - assert(nearest.concrete) +def find_spec(spec, condition): + """Searches the dag from spec in an intelligent order and looks + for a spec that matches a condition""" + # First search parents, then search children + dagiter = chain(spec.traverse(direction='parents', root=False), + spec.traverse(direction='children', root=False)) + visited = set() + for relative in dagiter: + if condition(relative): + return relative + visited.add(id(relative)) - spec.compiler = nearest.copy() + # Then search all other relatives in the DAG *except* spec + for relative in spec.root.traverse(): + if relative is spec: continue + if id(relative) in visited: continue + if condition(relative): + return relative - except StopIteration: - spec.compiler = spack.compilers.default_compiler().copy() + # Finally search spec itself. + if condition(spec): + return spec - return True # things changed. + return None # Nothing matched the condition. - def choose_provider(self, spec, providers): - """This is invoked for virtual specs. Given a spec with a virtual name, - say "mpi", and a list of specs of possible providers of that spec, - select a provider and return it. - """ - assert(spec.virtual) - assert(providers) +def cmp_specs(lhs, rhs): + # Package name sort order is not configurable, always goes alphabetical + if lhs.name != rhs.name: + return cmp(lhs.name, rhs.name) + + # Package version is second in compare order + pkgname = lhs.name + if lhs.versions != rhs.versions: + return spack.pkgsort.version_compare( + pkgname, lhs.versions, rhs.versions) + + # Compiler is third + if lhs.compiler != rhs.compiler: + return spack.pkgsort.compiler_compare( + pkgname, lhs.compiler, rhs.compiler) + + # Variants + if lhs.variants != rhs.variants: + return spack.pkgsort.variant_compare( + pkgname, lhs.variants, rhs.variants) + + # Architecture + if lhs.architecture != rhs.architecture: + return spack.pkgsort.architecture_compare( + pkgname, lhs.architecture, rhs.architecture) + + # Dependency is not configurable + lhash, rhash = hash(lhs), hash(rhs) + if lhash != rhash: + return -1 if lhash < rhash else 1 + + # Equal specs + return 0 - index = spack.spec.index_specs(providers) - first_key = sorted(index.keys())[0] - latest_version = sorted(index[first_key])[-1] - return latest_version class UnavailableCompilerVersionError(spack.error.SpackError): @@ -211,3 +342,11 @@ class NoValidVersionError(spack.error.SpackError): def __init__(self, spec): super(NoValidVersionError, self).__init__( "There are no valid versions for %s that match '%s'" % (spec.name, spec.versions)) + + +class NoBuildError(spack.error.SpackError): + """Raised when a package is configured with the buildable option False, but + no satisfactory external versions can be found""" + def __init__(self, spec): + super(NoBuildError, self).__init__( + "The spec '%s' is configured as not buildable, and no matching external installs were found" % spec.name) diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 6fecde9980..6afd69b3ac 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -129,6 +129,7 @@ from ordereddict_backport import OrderedDict import llnl.util.tty as tty from llnl.util.filesystem import mkdirp +import copy import spack from spack.error import SpackError @@ -194,6 +195,49 @@ section_schemas = { 'default': [], 'items': { 'type': 'string'},},},}, + 'packages': { + '$schema': 'http://json-schema.org/schema#', + 'title': 'Spack package configuration file schema', + 'type': 'object', + 'additionalProperties': False, + 'patternProperties': { + r'packages:?': { + 'type': 'object', + 'default': {}, + 'additionalProperties': False, + 'patternProperties': { + r'\w[\w-]*': { # package name + 'type': 'object', + 'default': {}, + 'additionalProperties': False, + 'properties': { + 'version': { + 'type' : 'array', + 'default' : [], + 'items' : { 'anyOf' : [ { 'type' : 'string' }, + { 'type' : 'number'}]}}, #version strings + 'compiler': { + 'type' : 'array', + 'default' : [], + 'items' : { 'type' : 'string' } }, #compiler specs + 'buildable': { + 'type': 'boolean', + 'default': True, + }, + 'providers': { + 'type': 'object', + 'default': {}, + 'additionalProperties': False, + 'patternProperties': { + r'\w[\w-]*': { + 'type' : 'array', + 'default' : [], + 'items' : { 'type' : 'string' },},},}, + 'paths': { + 'type' : 'object', + 'default' : {}, + } + },},},},},} } """OrderedDict of config scopes keyed by name. @@ -205,7 +249,7 @@ config_scopes = OrderedDict() def validate_section_name(section): """Raise a ValueError if the section is not a valid section.""" if section not in section_schemas: - raise ValueError("Invalid config section: '%s'. Options are %s." + raise ValueError("Invalid config section: '%s'. Options are %s" % (section, section_schemas)) @@ -335,7 +379,7 @@ def validate_scope(scope): return config_scopes[scope] else: - raise ValueError("Invalid config scope: '%s'. Must be one of %s." + raise ValueError("Invalid config scope: '%s'. Must be one of %s" % (scope, config_scopes.keys())) @@ -350,7 +394,7 @@ def _read_config_file(filename, schema): "Invlaid configuration. %s exists but is not a file." % filename) elif not os.access(filename, os.R_OK): - raise ConfigFileError("Config file is not readable: %s." % filename) + raise ConfigFileError("Config file is not readable: %s" % filename) try: tty.debug("Reading config file %s" % filename) @@ -494,6 +538,39 @@ def print_section(section): raise ConfigError("Error reading configuration: %s" % section) +def spec_externals(spec): + """Return a list of external specs (with external directory path filled in), + one for each known external installation.""" + allpkgs = get_config('packages') + name = spec.name + + external_specs = [] + pkg_paths = allpkgs.get(name, {}).get('paths', None) + if not pkg_paths: + return [] + + for external_spec, path in pkg_paths.iteritems(): + if not path: + # skip entries without paths (avoid creating extra Specs) + continue + + external_spec = spack.spec.Spec(external_spec, external=path) + if external_spec.satisfies(spec): + external_specs.append(external_spec) + return external_specs + + +def is_spec_buildable(spec): + """Return true if the spec pkgspec is configured as buildable""" + allpkgs = get_config('packages') + name = spec.name + if not spec.name in allpkgs: + return True + if not 'buildable' in allpkgs[spec.name]: + return True + return allpkgs[spec.name]['buildable'] + + class ConfigError(SpackError): pass class ConfigFileError(ConfigError): pass @@ -509,7 +586,7 @@ class ConfigFormatError(ConfigError): # Try to get line number from erroneous instance and its parent instance_mark = getattr(validation_error.instance, '_start_mark', None) parent_mark = getattr(validation_error.parent, '_start_mark', None) - path = getattr(validation_error, 'path', None) + path = [str(s) for s in getattr(validation_error, 'path', None)] # Try really hard to get the parent (which sometimes is not # set) This digs it out of the validated structure if it's not diff --git a/lib/spack/spack/database.py b/lib/spack/spack/database.py index 0fa18db34b..089d29325e 100644 --- a/lib/spack/spack/database.py +++ b/lib/spack/spack/database.py @@ -330,7 +330,7 @@ class Database(object): found = rec.ref_count if not expected == found: raise AssertionError( - "Invalid ref_count: %s: %d (expected %d), in DB %s." + "Invalid ref_count: %s: %d (expected %d), in DB %s" % (key, found, expected, self._index_path)) @@ -489,7 +489,7 @@ class Database(object): 1. Marks the spec as not installed. 2. Removes the spec if it has no more dependents. 3. If removed, recursively updates dependencies' ref counts - and remvoes them if they are no longer needed. + and removes them if they are no longer needed. """ # Take a lock around the entire removal. diff --git a/lib/spack/spack/directives.py b/lib/spack/spack/directives.py index 0b98211cb9..61cd303012 100644 --- a/lib/spack/spack/directives.py +++ b/lib/spack/spack/directives.py @@ -125,7 +125,7 @@ class directive(object): dicts = (dicts,) elif type(dicts) not in (list, tuple): raise TypeError( - "dicts arg must be list, tuple, or string. Found %s." + "dicts arg must be list, tuple, or string. Found %s" % type(dicts)) self.dicts = dicts @@ -174,7 +174,11 @@ def version(pkg, ver, checksum=None, **kwargs): def _depends_on(pkg, spec, when=None): - if when is None: + # If when is False do nothing + if when is False: + return + # If when is None or True make sure the condition is always satisfied + if when is None or when is True: when = pkg.name when_spec = parse_anonymous_spec(when, pkg.name) @@ -296,8 +300,8 @@ def resource(pkg, **kwargs): raise RuntimeError(message) when_spec = parse_anonymous_spec(when, pkg.name) resources = pkg.resources.setdefault(when_spec, []) - fetcher = from_kwargs(**kwargs) name = kwargs.get('name') + fetcher = from_kwargs(**kwargs) resources.append(Resource(name, fetcher, destination, placement)) @@ -313,5 +317,5 @@ class CircularReferenceError(DirectiveError): def __init__(self, directive, package): super(CircularReferenceError, self).__init__( directive, - "Package '%s' cannot pass itself to %s." % (package, directive)) + "Package '%s' cannot pass itself to %s" % (package, directive)) self.package = package diff --git a/lib/spack/spack/directory_layout.py b/lib/spack/spack/directory_layout.py index 3e416a6a1f..da8f1aa1bc 100644 --- a/lib/spack/spack/directory_layout.py +++ b/lib/spack/spack/directory_layout.py @@ -85,6 +85,16 @@ class DirectoryLayout(object): raise NotImplementedError() + def check_installed(self, spec): + """Checks whether a spec is installed. + + Return the spec's prefix, if it is installed, None otherwise. + + Raise an exception if the install is inconsistent or corrupt. + """ + raise NotImplementedError() + + def extension_map(self, spec): """Get a dict of currently installed extension packages for a spec. @@ -140,7 +150,7 @@ class DirectoryLayout(object): if os.path.exists(path): try: shutil.rmtree(path) - except exceptions.OSError, e: + except exceptions.OSError as e: raise RemoveFailedError(spec, path, e) path = os.path.dirname(path) @@ -173,7 +183,9 @@ class YamlDirectoryLayout(DirectoryLayout): self.spec_file_name = 'spec.yaml' self.extension_file_name = 'extensions.yaml' - self.build_log_name = 'build.out' # TODO: use config file. + self.build_log_name = 'build.out' # build log. + self.build_env_name = 'build.env' # build environment + self.packages_dir = 'repos' # archive of package.py files # Cache of already written/read extension maps. self._extension_maps = {} @@ -186,6 +198,10 @@ class YamlDirectoryLayout(DirectoryLayout): def relative_path_for_spec(self, spec): _check_concrete(spec) + + if spec.external: + return spec.external + dir_name = "%s-%s-%s" % ( spec.name, spec.version, @@ -231,29 +247,49 @@ class YamlDirectoryLayout(DirectoryLayout): self.build_log_name) + def build_env_path(self, spec): + return join_path(self.path_for_spec(spec), self.metadata_dir, + self.build_env_name) + + + def build_packages_path(self, spec): + return join_path(self.path_for_spec(spec), self.metadata_dir, + self.packages_dir) + + def create_install_directory(self, spec): _check_concrete(spec) + prefix = self.check_installed(spec) + if prefix: + raise InstallDirectoryAlreadyExistsError(prefix) + + mkdirp(self.metadata_path(spec)) + self.write_spec(spec, self.spec_file_path(spec)) + + + def check_installed(self, spec): + _check_concrete(spec) path = self.path_for_spec(spec) spec_file_path = self.spec_file_path(spec) - if os.path.isdir(path): - if not os.path.isfile(spec_file_path): - raise InconsistentInstallDirectoryError( - 'No spec file found at path %s' % spec_file_path) + if not os.path.isdir(path): + return None - installed_spec = self.read_spec(spec_file_path) - if installed_spec == self.spec: - raise InstallDirectoryAlreadyExistsError(path) + if not os.path.isfile(spec_file_path): + raise InconsistentInstallDirectoryError( + 'Inconsistent state: install prefix exists but contains no spec.yaml:', + " " + path) - if spec.dag_hash() == installed_spec.dag_hash(): - raise SpecHashCollisionError(installed_hash, spec_hash) - else: - raise InconsistentInstallDirectoryError( - 'Spec file in %s does not match hash!' % spec_file_path) + installed_spec = self.read_spec(spec_file_path) + if installed_spec == spec: + return path - mkdirp(self.metadata_path(spec)) - self.write_spec(spec, spec_file_path) + if spec.dag_hash() == installed_spec.dag_hash(): + raise SpecHashCollisionError(installed_hash, spec_hash) + else: + raise InconsistentInstallDirectoryError( + 'Spec file in %s does not match hash!' % spec_file_path) def all_specs(self): @@ -323,7 +359,7 @@ class YamlDirectoryLayout(DirectoryLayout): if not dag_hash in by_hash: raise InvalidExtensionSpecError( - "Spec %s not found in %s." % (dag_hash, prefix)) + "Spec %s not found in %s" % (dag_hash, prefix)) ext_spec = by_hash[dag_hash] if not prefix == ext_spec.prefix: @@ -387,8 +423,8 @@ class YamlDirectoryLayout(DirectoryLayout): class DirectoryLayoutError(SpackError): """Superclass for directory layout errors.""" - def __init__(self, message): - super(DirectoryLayoutError, self).__init__(message) + def __init__(self, message, long_msg=None): + super(DirectoryLayoutError, self).__init__(message, long_msg) class SpecHashCollisionError(DirectoryLayoutError): @@ -410,8 +446,8 @@ class RemoveFailedError(DirectoryLayoutError): class InconsistentInstallDirectoryError(DirectoryLayoutError): """Raised when a package seems to be installed to the wrong place.""" - def __init__(self, message): - super(InconsistentInstallDirectoryError, self).__init__(message) + def __init__(self, message, long_msg=None): + super(InconsistentInstallDirectoryError, self).__init__(message, long_msg) class InstallDirectoryAlreadyExistsError(DirectoryLayoutError): @@ -438,7 +474,7 @@ class ExtensionConflictError(DirectoryLayoutError): """Raised when an extension is added to a package that already has it.""" def __init__(self, spec, ext_spec, conflict): super(ExtensionConflictError, self).__init__( - "%s cannot be installed in %s because it conflicts with %s."% ( + "%s cannot be installed in %s because it conflicts with %s"% ( ext_spec.short_spec, spec.short_spec, conflict.short_spec)) diff --git a/lib/spack/spack/environment.py b/lib/spack/spack/environment.py new file mode 100644 index 0000000000..72aafa4e2d --- /dev/null +++ b/lib/spack/spack/environment.py @@ -0,0 +1,252 @@ +import os +import os.path +import collections +import inspect + + +class NameModifier(object): + def __init__(self, name, **kwargs): + self.name = name + self.args = {'name': name} + self.args.update(kwargs) + + +class NameValueModifier(object): + def __init__(self, name, value, **kwargs): + self.name = name + self.value = value + self.args = {'name': name, 'value': value} + self.args.update(kwargs) + + +class SetEnv(NameValueModifier): + def execute(self): + os.environ[self.name] = str(self.value) + + +class UnsetEnv(NameModifier): + def execute(self): + os.environ.pop(self.name, None) # Avoid throwing if the variable was not set + + +class SetPath(NameValueModifier): + def execute(self): + string_path = concatenate_paths(self.value) + os.environ[self.name] = string_path + + +class AppendPath(NameValueModifier): + def execute(self): + environment_value = os.environ.get(self.name, '') + directories = environment_value.split(':') if environment_value else [] + directories.append(os.path.normpath(self.value)) + os.environ[self.name] = ':'.join(directories) + + +class PrependPath(NameValueModifier): + def execute(self): + environment_value = os.environ.get(self.name, '') + directories = environment_value.split(':') if environment_value else [] + directories = [os.path.normpath(self.value)] + directories + os.environ[self.name] = ':'.join(directories) + + +class RemovePath(NameValueModifier): + def execute(self): + environment_value = os.environ.get(self.name, '') + directories = environment_value.split(':') if environment_value else [] + directories = [os.path.normpath(x) for x in directories if x != os.path.normpath(self.value)] + os.environ[self.name] = ':'.join(directories) + + +class EnvironmentModifications(object): + """ + Keeps track of requests to modify the current environment. + + Each call to a method to modify the environment stores the extra information on the caller in the request: + - 'filename' : filename of the module where the caller is defined + - 'lineno': line number where the request occurred + - 'context' : line of code that issued the request that failed + """ + + def __init__(self, other=None): + """ + Initializes a new instance, copying commands from other if it is not None + + Args: + other: another instance of EnvironmentModifications from which (optional) + """ + self.env_modifications = [] + if other is not None: + self.extend(other) + + def __iter__(self): + return iter(self.env_modifications) + + def __len__(self): + return len(self.env_modifications) + + def extend(self, other): + self._check_other(other) + self.env_modifications.extend(other.env_modifications) + + @staticmethod + def _check_other(other): + if not isinstance(other, EnvironmentModifications): + raise TypeError('other must be an instance of EnvironmentModifications') + + def _get_outside_caller_attributes(self): + stack = inspect.stack() + try: + _, filename, lineno, _, context, index = stack[2] + context = context[index].strip() + except Exception: + filename, lineno, context = 'unknown file', 'unknown line', 'unknown context' + args = { + 'filename': filename, + 'lineno': lineno, + 'context': context + } + return args + + def set(self, name, value, **kwargs): + """ + Stores in the current object a request to set an environment variable + + Args: + name: name of the environment variable to be set + value: value of the environment variable + """ + kwargs.update(self._get_outside_caller_attributes()) + item = SetEnv(name, value, **kwargs) + self.env_modifications.append(item) + + def unset(self, name, **kwargs): + """ + Stores in the current object a request to unset an environment variable + + Args: + name: name of the environment variable to be set + """ + kwargs.update(self._get_outside_caller_attributes()) + item = UnsetEnv(name, **kwargs) + self.env_modifications.append(item) + + def set_path(self, name, elts, **kwargs): + """ + Stores a request to set a path generated from a list. + + Args: + name: name o the environment variable to be set. + elts: elements of the path to set. + """ + kwargs.update(self._get_outside_caller_attributes()) + item = SetPath(name, elts, **kwargs) + self.env_modifications.append(item) + + def append_path(self, name, path, **kwargs): + """ + Stores in the current object a request to append a path to a path list + + Args: + name: name of the path list in the environment + path: path to be appended + """ + kwargs.update(self._get_outside_caller_attributes()) + item = AppendPath(name, path, **kwargs) + self.env_modifications.append(item) + + def prepend_path(self, name, path, **kwargs): + """ + Same as `append_path`, but the path is pre-pended + + Args: + name: name of the path list in the environment + path: path to be pre-pended + """ + kwargs.update(self._get_outside_caller_attributes()) + item = PrependPath(name, path, **kwargs) + self.env_modifications.append(item) + + def remove_path(self, name, path, **kwargs): + """ + Stores in the current object a request to remove a path from a path list + + Args: + name: name of the path list in the environment + path: path to be removed + """ + kwargs.update(self._get_outside_caller_attributes()) + item = RemovePath(name, path, **kwargs) + self.env_modifications.append(item) + + def group_by_name(self): + """ + Returns a dict of the modifications grouped by variable name + + Returns: + dict mapping the environment variable name to the modifications to be done on it + """ + modifications = collections.defaultdict(list) + for item in self: + modifications[item.name].append(item) + return modifications + + def clear(self): + """ + Clears the current list of modifications + """ + self.env_modifications.clear() + + def apply_modifications(self): + """ + Applies the modifications and clears the list + """ + modifications = self.group_by_name() + # Apply the modifications to the environment variables one variable at a time + for name, actions in sorted(modifications.items()): + for x in actions: + x.execute() + + +def concatenate_paths(paths): + """ + Concatenates an iterable of paths into a string of column separated paths + + Args: + paths: iterable of paths + + Returns: + string + """ + return ':'.join(str(item) for item in paths) + + +def set_or_unset_not_first(variable, changes, errstream): + """ + Check if we are going to set or unset something after other modifications have already been requested + """ + indexes = [ii for ii, item in enumerate(changes) if ii != 0 and type(item) in [SetEnv, UnsetEnv]] + if indexes: + good = '\t \t{context} at {filename}:{lineno}' + nogood = '\t--->\t{context} at {filename}:{lineno}' + errstream('Suspicious requests to set or unset the variable \'{var}\' found'.format(var=variable)) + for ii, item in enumerate(changes): + print_format = nogood if ii in indexes else good + errstream(print_format.format(**item.args)) + + +def validate(env, errstream): + """ + Validates the environment modifications to check for the presence of suspicious patterns. Prompts a warning for + everything that was found + + Current checks: + - set or unset variables after other changes on the same variable + + Args: + env: list of environment modifications + """ + modifications = env.group_by_name() + for variable, list_of_changes in sorted(modifications.items()): + set_or_unset_not_first(variable, list_of_changes, errstream) diff --git a/lib/spack/spack/fetch_strategy.py b/lib/spack/spack/fetch_strategy.py index 0657146bf6..0d0a7db8a9 100644 --- a/lib/spack/spack/fetch_strategy.py +++ b/lib/spack/spack/fetch_strategy.py @@ -44,6 +44,7 @@ import os import sys import re import shutil +import copy from functools import wraps import llnl.util.tty as tty from llnl.util.filesystem import * @@ -55,23 +56,28 @@ from spack.util.string import * from spack.version import Version, ver from spack.util.compression import decompressor_for, extension +import spack.util.pattern as pattern + """List of all fetch strategies, created by FetchStrategy metaclass.""" all_strategies = [] + def _needs_stage(fun): """Many methods on fetch strategies require a stage to be set using set_stage(). This decorator adds a check for self.stage.""" + @wraps(fun) def wrapper(self, *args, **kwargs): if not self.stage: raise NoStageError(fun) return fun(self, *args, **kwargs) + return wrapper class FetchStrategy(object): """Superclass of all fetch strategies.""" - enabled = False # Non-abstract subclasses should be enabled. + enabled = False # Non-abstract subclasses should be enabled. required_attributes = None # Attributes required in version() args. class __metaclass__(type): @@ -80,28 +86,28 @@ class FetchStrategy(object): type.__init__(cls, name, bases, dict) if cls.enabled: all_strategies.append(cls) - def __init__(self): # The stage is initialized late, so that fetch strategies can be constructed # at package construction time. This is where things will be fetched. self.stage = None - def set_stage(self, stage): """This is called by Stage before any of the fetching methods are called on the stage.""" self.stage = stage - # Subclasses need to implement these methods def fetch(self): pass # Return True on success, False on fail. + def check(self): pass # Do checksum. + def expand(self): pass # Expand archive. + def reset(self): pass # Revert to freshly downloaded state. def archive(self, destination): pass # Used to create tarball for mirror. - def __str__(self): # Should be human readable URL. + def __str__(self): # Should be human readable URL. return "FetchStrategy.__str___" # This method is used to match fetch strategies to version() @@ -111,6 +117,15 @@ class FetchStrategy(object): return any(k in args for k in cls.required_attributes) +@pattern.composite(interface=FetchStrategy) +class FetchStrategyComposite(object): + """ + Composite for a FetchStrategy object. Implements the GoF composite pattern. + """ + matches = FetchStrategy.matches + set_stage = FetchStrategy.set_stage + + class URLFetchStrategy(FetchStrategy): """FetchStrategy that pulls source code from a URL for an archive, checks the archive against a checksum,and decompresses the archive. @@ -129,6 +144,8 @@ class URLFetchStrategy(FetchStrategy): self.digest = kwargs.get('md5', None) if not self.digest: self.digest = digest + self.expand_archive = kwargs.get('expand', True) + if not self.url: raise ValueError("URLFetchStrategy requires a url for fetching.") @@ -137,24 +154,24 @@ class URLFetchStrategy(FetchStrategy): self.stage.chdir() if self.archive_file: - tty.msg("Already downloaded %s." % self.archive_file) + tty.msg("Already downloaded %s" % self.archive_file) return tty.msg("Trying to fetch from %s" % self.url) - curl_args = ['-O', # save file to disk - '-f', # fail on >400 errors - '-D', '-', # print out HTML headers - '-L', self.url,] + curl_args = ['-O', # save file to disk + '-f', # fail on >400 errors + '-D', '-', # print out HTML headers + '-L', self.url, ] if sys.stdout.isatty(): curl_args.append('-#') # status bar when using a tty else: - curl_args.append('-sS') # just errors when not. + curl_args.append('-sS') # just errors when not. # Run curl but grab the mime type from the http headers headers = spack.curl( - *curl_args, return_output=True, fail_on_error=False) + *curl_args, output=str, fail_on_error=False) if spack.curl.returncode != 0: # clean up archive on failure. @@ -164,24 +181,23 @@ class URLFetchStrategy(FetchStrategy): if spack.curl.returncode == 22: # This is a 404. Curl will print the error. raise FailedDownloadError( - self.url, "URL %s was not found!" % self.url) + self.url, "URL %s was not found!" % self.url) elif spack.curl.returncode == 60: # This is a certificate error. Suggest spack -k raise FailedDownloadError( - self.url, - "Curl was unable to fetch due to invalid certificate. " - "This is either an attack, or your cluster's SSL configuration " - "is bad. If you believe your SSL configuration is bad, you " - "can try running spack -k, which will not check SSL certificates." - "Use this at your own risk.") + self.url, + "Curl was unable to fetch due to invalid certificate. " + "This is either an attack, or your cluster's SSL configuration " + "is bad. If you believe your SSL configuration is bad, you " + "can try running spack -k, which will not check SSL certificates." + "Use this at your own risk.") else: # This is some other curl error. Curl will print the # error, but print a spack message too raise FailedDownloadError( - self.url, "Curl failed with error %d" % spack.curl.returncode) - + self.url, "Curl failed with error %d" % spack.curl.returncode) # Check if we somehow got an HTML file rather than the archive we # asked for. We only look at the last content type, to handle @@ -196,7 +212,6 @@ class URLFetchStrategy(FetchStrategy): if not self.archive_file: raise FailedDownloadError(self.url) - @property def archive_file(self): """Path to the source archive within this stage directory.""" @@ -204,12 +219,16 @@ class URLFetchStrategy(FetchStrategy): @_needs_stage def expand(self): + if not self.expand_archive: + tty.msg("Skipping expand step for %s" % self.archive_file) + return + tty.msg("Staging archive: %s" % self.archive_file) self.stage.chdir() if not self.archive_file: raise NoArchiveFileError("URLFetchStrategy couldn't find archive file", - "Failed on expand() for URL %s" % self.url) + "Failed on expand() for URL %s" % self.url) decompress = decompressor_for(self.archive_file) @@ -241,7 +260,6 @@ class URLFetchStrategy(FetchStrategy): # Set the wd back to the stage when done. self.stage.chdir() - def archive(self, destination): """Just moves this archive to the destination.""" if not self.archive_file: @@ -252,7 +270,6 @@ class URLFetchStrategy(FetchStrategy): shutil.move(self.archive_file, destination) - @_needs_stage def check(self): """Check the downloaded archive against a checksum digest. @@ -263,9 +280,8 @@ class URLFetchStrategy(FetchStrategy): checker = crypto.Checker(self.digest) if not checker.check(self.archive_file): raise ChecksumError( - "%s checksum failed for %s." % (checker.hash_name, self.archive_file), - "Expected %s but got %s." % (self.digest, checker.sum)) - + "%s checksum failed for %s" % (checker.hash_name, self.archive_file), + "Expected %s but got %s" % (self.digest, checker.sum)) @_needs_stage def reset(self): @@ -277,12 +293,10 @@ class URLFetchStrategy(FetchStrategy): shutil.rmtree(self.stage.source_path, ignore_errors=True) self.expand() - def __repr__(self): url = self.url if self.url else "no url" return "URLFetchStrategy<%s>" % url - def __str__(self): if self.url: return self.url @@ -298,33 +312,30 @@ class VCSFetchStrategy(FetchStrategy): # Set a URL based on the type of fetch strategy. self.url = kwargs.get(name, None) if not self.url: raise ValueError( - "%s requires %s argument." % (self.__class__, name)) + "%s requires %s argument." % (self.__class__, name)) # Ensure that there's only one of the rev_types if sum(k in kwargs for k in rev_types) > 1: raise FetchStrategyError( - "Supply only one of %s to fetch with %s." % ( - comma_or(rev_types), name)) + "Supply only one of %s to fetch with %s" % ( + comma_or(rev_types), name)) # Set attributes for each rev type. for rt in rev_types: setattr(self, rt, kwargs.get(rt, None)) - @_needs_stage def check(self): - tty.msg("No checksum needed when fetching with %s." % self.name) - + tty.msg("No checksum needed when fetching with %s" % self.name) @_needs_stage def expand(self): tty.debug("Source fetched with %s is already expanded." % self.name) - @_needs_stage def archive(self, destination, **kwargs): - assert(extension(destination) == 'tar.gz') - assert(self.stage.source_path.startswith(self.stage.path)) + assert (extension(destination) == 'tar.gz') + assert (self.stage.source_path.startswith(self.stage.path)) tar = which('tar', required=True) @@ -338,16 +349,13 @@ class VCSFetchStrategy(FetchStrategy): self.stage.chdir() tar('-czf', destination, os.path.basename(self.stage.source_path)) - def __str__(self): return "VCS: %s" % self.url - def __repr__(self): return "%s<%s>" % (self.__class__, self.url) - class GitFetchStrategy(VCSFetchStrategy): """Fetch strategy that gets source code from a git repository. Use like this in a package: @@ -368,30 +376,31 @@ class GitFetchStrategy(VCSFetchStrategy): required_attributes = ('git',) def __init__(self, **kwargs): + # Discards the keywords in kwargs that may conflict with the next call to __init__ + forwarded_args = copy.copy(kwargs) + forwarded_args.pop('name', None) + super(GitFetchStrategy, self).__init__( - 'git', 'tag', 'branch', 'commit', **kwargs) + 'git', 'tag', 'branch', 'commit', **forwarded_args) self._git = None - @property def git_version(self): - vstring = self.git('--version', return_output=True).lstrip('git version ') + vstring = self.git('--version', output=str).lstrip('git version ') return Version(vstring) - @property def git(self): if not self._git: self._git = which('git', required=True) return self._git - @_needs_stage def fetch(self): self.stage.chdir() if self.stage.source_path: - tty.msg("Already fetched %s." % self.stage.source_path) + tty.msg("Already fetched %s" % self.stage.source_path) return args = [] @@ -418,7 +427,7 @@ class GitFetchStrategy(VCSFetchStrategy): if self.branch: args.extend(['--branch', self.branch]) elif self.tag and self.git_version >= ver('1.8.5.2'): - args.extend(['--branch', self.tag]) + args.extend(['--branch', self.tag]) # Try to be efficient if we're using a new enough git. # This checks out only one branch's history @@ -429,7 +438,7 @@ class GitFetchStrategy(VCSFetchStrategy): # Yet more efficiency, only download a 1-commit deep tree if self.git_version >= ver('1.7.1'): try: - self.git(*(args + ['--depth','1', self.url])) + self.git(*(args + ['--depth', '1', self.url])) cloned = True except spack.error.SpackError: # This will fail with the dumb HTTP transport @@ -452,18 +461,15 @@ class GitFetchStrategy(VCSFetchStrategy): self.git('pull', '--tags', ignore_errors=1) self.git('checkout', self.tag) - def archive(self, destination): super(GitFetchStrategy, self).archive(destination, exclude='.git') - @_needs_stage def reset(self): self.stage.chdir_to_source() self.git('checkout', '.') self.git('clean', '-f') - def __str__(self): return "[git] %s" % self.url @@ -483,26 +489,28 @@ class SvnFetchStrategy(VCSFetchStrategy): required_attributes = ['svn'] def __init__(self, **kwargs): + # Discards the keywords in kwargs that may conflict with the next call to __init__ + forwarded_args = copy.copy(kwargs) + forwarded_args.pop('name', None) + super(SvnFetchStrategy, self).__init__( - 'svn', 'revision', **kwargs) + 'svn', 'revision', **forwarded_args) self._svn = None if self.revision is not None: self.revision = str(self.revision) - @property def svn(self): if not self._svn: self._svn = which('svn', required=True) return self._svn - @_needs_stage def fetch(self): self.stage.chdir() if self.stage.source_path: - tty.msg("Already fetched %s." % self.stage.source_path) + tty.msg("Already fetched %s" % self.stage.source_path) return tty.msg("Trying to check out svn repository: %s" % self.url) @@ -515,10 +523,9 @@ class SvnFetchStrategy(VCSFetchStrategy): self.svn(*args) self.stage.chdir_to_source() - def _remove_untracked_files(self): """Removes untracked files in an svn repository.""" - status = self.svn('status', '--no-ignore', return_output=True) + status = self.svn('status', '--no-ignore', output=str) self.svn('status', '--no-ignore') for line in status.split('\n'): if not re.match('^[I?]', line): @@ -529,23 +536,19 @@ class SvnFetchStrategy(VCSFetchStrategy): elif os.path.isdir(path): shutil.rmtree(path, ignore_errors=True) - def archive(self, destination): super(SvnFetchStrategy, self).archive(destination, exclude='.svn') - @_needs_stage def reset(self): self.stage.chdir_to_source() self._remove_untracked_files() self.svn('revert', '.', '-R') - def __str__(self): return "[svn] %s" % self.url - class HgFetchStrategy(VCSFetchStrategy): """Fetch strategy that gets source code from a Mercurial repository. Use like this in a package: @@ -567,11 +570,14 @@ class HgFetchStrategy(VCSFetchStrategy): required_attributes = ['hg'] def __init__(self, **kwargs): + # Discards the keywords in kwargs that may conflict with the next call to __init__ + forwarded_args = copy.copy(kwargs) + forwarded_args.pop('name', None) + super(HgFetchStrategy, self).__init__( - 'hg', 'revision', **kwargs) + 'hg', 'revision', **forwarded_args) self._hg = None - @property def hg(self): if not self._hg: @@ -583,7 +589,7 @@ class HgFetchStrategy(VCSFetchStrategy): self.stage.chdir() if self.stage.source_path: - tty.msg("Already fetched %s." % self.stage.source_path) + tty.msg("Already fetched %s" % self.stage.source_path) return args = [] @@ -597,11 +603,9 @@ class HgFetchStrategy(VCSFetchStrategy): self.hg(*args) - def archive(self, destination): super(HgFetchStrategy, self).archive(destination, exclude='.hg') - @_needs_stage def reset(self): self.stage.chdir() @@ -619,7 +623,6 @@ class HgFetchStrategy(VCSFetchStrategy): shutil.move(scrubbed, source_path) self.stage.chdir_to_source() - def __str__(self): return "[hg] %s" % self.url @@ -693,9 +696,10 @@ class FetchError(spack.error.SpackError): class FailedDownloadError(FetchError): """Raised wen a download fails.""" + def __init__(self, url, msg=""): super(FailedDownloadError, self).__init__( - "Failed to fetch file from URL: %s" % url, msg) + "Failed to fetch file from URL: %s" % url, msg) self.url = url @@ -718,12 +722,14 @@ class InvalidArgsError(FetchError): class ChecksumError(FetchError): """Raised when archive fails to checksum.""" + def __init__(self, message, long_msg=None): super(ChecksumError, self).__init__(message, long_msg) class NoStageError(FetchError): """Raised when fetch operations are called before set_stage().""" + def __init__(self, method): super(NoStageError, self).__init__( - "Must call FetchStrategy.set_stage() before calling %s" % method.__name__) + "Must call FetchStrategy.set_stage() before calling %s" % method.__name__) diff --git a/lib/spack/spack/hooks/sbang.py b/lib/spack/spack/hooks/sbang.py new file mode 100644 index 0000000000..d78adb576e --- /dev/null +++ b/lib/spack/spack/hooks/sbang.py @@ -0,0 +1,98 @@ +############################################################################## +# Copyright (c) 2013-2015, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +import os + +from llnl.util.filesystem import * +import llnl.util.tty as tty + +import spack +import spack.modules + +# Character limit for shebang line. Using Linux's 127 characters +# here, as it is the shortest I could find on a modern OS. +shebang_limit = 127 + +def shebang_too_long(path): + """Detects whether a file has a shebang line that is too long.""" + with open(path, 'r') as script: + bytes = script.read(2) + if bytes != '#!': + return False + + line = bytes + script.readline() + return len(line) > shebang_limit + + +def filter_shebang(path): + """Adds a second shebang line, using sbang, at the beginning of a file.""" + with open(path, 'r') as original_file: + original = original_file.read() + + # This line will be prepended to file + new_sbang_line = '#!/bin/bash %s/bin/sbang\n' % spack.spack_root + + # Skip files that are already using sbang. + if original.startswith(new_sbang_line): + return + + backup = path + ".shebang.bak" + os.rename(path, backup) + + with open(path, 'w') as new_file: + new_file.write(new_sbang_line) + new_file.write(original) + + copy_mode(backup, path) + unset_executable_mode(backup) + + tty.warn("Patched overly long shebang in %s" % path) + + +def filter_shebangs_in_directory(directory): + for file in os.listdir(directory): + path = os.path.join(directory, file) + + # only handle files + if not os.path.isfile(path): + continue + + # only handle links that resolve within THIS package's prefix. + if os.path.islink(path): + real_path = os.path.realpath(path) + if not real_path.startswith(directory + os.sep): + continue + + # test the file for a long shebang, and filter + if shebang_too_long(path): + filter_shebang(path) + + +def post_install(pkg): + """This hook edits scripts so that they call /bin/bash + $spack_prefix/bin/sbang instead of something longer than the + shebang limit.""" + if not os.path.isdir(pkg.prefix.bin): + return + filter_shebangs_in_directory(pkg.prefix.bin) diff --git a/lib/spack/spack/mirror.py b/lib/spack/spack/mirror.py index 1d9b0e7ef2..6981f69ac0 100644 --- a/lib/spack/spack/mirror.py +++ b/lib/spack/spack/mirror.py @@ -45,25 +45,31 @@ from spack.version import * from spack.util.compression import extension, allowed_archive -def mirror_archive_filename(spec): +def mirror_archive_filename(spec, fetcher): """Get the name of the spec's archive in the mirror.""" if not spec.version.concrete: raise ValueError("mirror.path requires spec with concrete version.") - fetcher = spec.package.fetcher if isinstance(fetcher, fs.URLFetchStrategy): - # If we fetch this version with a URLFetchStrategy, use URL's archive type - ext = url.downloaded_file_extension(fetcher.url) + if fetcher.expand_archive: + # If we fetch this version with a URLFetchStrategy, use URL's archive type + ext = url.downloaded_file_extension(fetcher.url) + else: + # If the archive shouldn't be expanded, don't check for its extension. + ext = None else: # Otherwise we'll make a .tar.gz ourselves ext = 'tar.gz' - return "%s-%s.%s" % (spec.package.name, spec.version, ext) + filename = "%s-%s" % (spec.package.name, spec.version) + if ext: + filename += ".%s" % ext + return filename -def mirror_archive_path(spec): +def mirror_archive_path(spec, fetcher): """Get the relative path to the spec's archive within a mirror.""" - return join_path(spec.name, mirror_archive_filename(spec)) + return join_path(spec.name, mirror_archive_filename(spec, fetcher)) def get_matching_versions(specs, **kwargs): @@ -74,10 +80,11 @@ def get_matching_versions(specs, **kwargs): # Skip any package that has no known versions. if not pkg.versions: - tty.msg("No safe (checksummed) versions for package %s." % pkg.name) + tty.msg("No safe (checksummed) versions for package %s" % pkg.name) continue num_versions = kwargs.get('num_versions', 0) + matching_spec = [] for i, v in enumerate(reversed(sorted(pkg.versions))): # Generate no more than num_versions versions for each spec. if num_versions and i >= num_versions: @@ -88,7 +95,11 @@ def get_matching_versions(specs, **kwargs): s = Spec(pkg.name) s.versions = VersionList([v]) s.variants = spec.variants.copy() - matching.append(s) + matching_spec.append(s) + + if not matching_spec: + tty.warn("No known version matches spec: %s" % spec) + matching.extend(matching_spec) return matching @@ -106,7 +117,6 @@ def suggest_archive_basename(resource): return basename - def create(path, specs, **kwargs): """Create a directory to be used as a spack mirror, and fill it with package archives. @@ -147,81 +157,72 @@ def create(path, specs, **kwargs): # Get the absolute path of the root before we start jumping around. mirror_root = os.path.abspath(path) if not os.path.isdir(mirror_root): - mkdirp(mirror_root) + try: + mkdirp(mirror_root) + except OSError as e: + raise MirrorError( + "Cannot create directory '%s':" % mirror_root, str(e)) # Things to keep track of while parsing specs. - present = [] - mirrored = [] - error = [] + categories = { + 'present': [], + 'mirrored': [], + 'error': [] + } # Iterate through packages and download all the safe tarballs for each of them - everything_already_exists = True for spec in version_specs: - pkg = spec.package - - stage = None - try: - # create a subdirectory for the current package@version - archive_path = os.path.abspath(join_path(mirror_root, mirror_archive_path(spec))) - subdir = os.path.dirname(archive_path) - mkdirp(subdir) - - if os.path.exists(archive_path): - tty.msg("Already added %s" % spec.format("$_$@")) - else: - everything_already_exists = False - # Set up a stage and a fetcher for the download - unique_fetch_name = spec.format("$_$@") - fetcher = fs.for_package_version(pkg, pkg.version) - stage = Stage(fetcher, name=unique_fetch_name) - fetcher.set_stage(stage) - - # Do the fetch and checksum if necessary - fetcher.fetch() - if not kwargs.get('no_checksum', False): - fetcher.check() - tty.msg("Checksum passed for %s@%s" % (pkg.name, pkg.version)) - - # Fetchers have to know how to archive their files. Use - # that to move/copy/create an archive in the mirror. - fetcher.archive(archive_path) - tty.msg("Added %s." % spec.format("$_$@")) - - # Fetch resources if they are associated with the spec - resources = pkg._get_resources() - for resource in resources: - resource_archive_path = join_path(subdir, suggest_archive_basename(resource)) - if os.path.exists(resource_archive_path): - tty.msg("Already added resource %s (%s@%s)." % (resource.name, pkg.name, pkg.version)) - continue - everything_already_exists = False - resource_stage_folder = pkg._resource_stage(resource) - resource_stage = Stage(resource.fetcher, name=resource_stage_folder) - resource.fetcher.set_stage(resource_stage) - resource.fetcher.fetch() - if not kwargs.get('no_checksum', False): - resource.fetcher.check() - tty.msg("Checksum passed for the resource %s (%s@%s)" % (resource.name, pkg.name, pkg.version)) - resource.fetcher.archive(resource_archive_path) - tty.msg("Added resource %s (%s@%s)." % (resource.name, pkg.name, pkg.version)) - - if everything_already_exists: - present.append(spec) - else: - mirrored.append(spec) - - except Exception, e: - if spack.debug: - sys.excepthook(*sys.exc_info()) - else: - tty.warn("Error while fetching %s." % spec.format('$_$@'), e.message) - error.append(spec) - - finally: - if stage: - stage.destroy() - - return (present, mirrored, error) + add_single_spec(spec, mirror_root, categories, **kwargs) + + return categories['present'], categories['mirrored'], categories['error'] + + +def add_single_spec(spec, mirror_root, categories, **kwargs): + tty.msg("Adding package {pkg} to mirror".format(pkg=spec.format("$_$@"))) + spec_exists_in_mirror = True + try: + with spec.package.stage: + # fetcher = stage.fetcher + # fetcher.fetch() + # ... + # fetcher.archive(archive_path) + for ii, stage in enumerate(spec.package.stage): + fetcher = stage.fetcher + if ii == 0: + # create a subdirectory for the current package@version + archive_path = os.path.abspath(join_path(mirror_root, mirror_archive_path(spec, fetcher))) + name = spec.format("$_$@") + else: + resource = stage.resource + archive_path = join_path(subdir, suggest_archive_basename(resource)) + name = "{resource} ({pkg}).".format(resource=resource.name, pkg=spec.format("$_$@")) + subdir = os.path.dirname(archive_path) + mkdirp(subdir) + + if os.path.exists(archive_path): + tty.msg("{name} : already added".format(name=name)) + else: + spec_exists_in_mirror = False + fetcher.fetch() + if not kwargs.get('no_checksum', False): + fetcher.check() + tty.msg("{name} : checksum passed".format(name=name)) + + # Fetchers have to know how to archive their files. Use + # that to move/copy/create an archive in the mirror. + fetcher.archive(archive_path) + tty.msg("{name} : added".format(name=name)) + + if spec_exists_in_mirror: + categories['present'].append(spec) + else: + categories['mirrored'].append(spec) + except Exception as e: + if spack.debug: + sys.excepthook(*sys.exc_info()) + else: + tty.warn("Error while fetching %s" % spec.format('$_$@'), e.message) + categories['error'].append(spec) class MirrorError(spack.error.SpackError): diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py index 7036626e29..05c93cd3e6 100644 --- a/lib/spack/spack/modules.py +++ b/lib/spack/spack/modules.py @@ -22,42 +22,38 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -"""This module contains code for creating environment modules, which -can include dotkits, tcl modules, lmod, and others. +""" +This module contains code for creating environment modules, which can include dotkits, tcl modules, lmod, and others. -The various types of modules are installed by post-install hooks and -removed after an uninstall by post-uninstall hooks. This class -consolidates the logic for creating an abstract description of the -information that module systems need. Currently that includes a -number of directories to be appended to paths in the user's environment: +The various types of modules are installed by post-install hooks and removed after an uninstall by post-uninstall hooks. +This class consolidates the logic for creating an abstract description of the information that module systems need. +Currently that includes a number of directories to be appended to paths in the user's environment: * /bin directories to be appended to PATH * /lib* directories for LD_LIBRARY_PATH + * /include directories for CPATH * /man* and /share/man* directories for MANPATH * the package prefix for CMAKE_PREFIX_PATH -This module also includes logic for coming up with unique names for -the module files so that they can be found by the various -shell-support files in $SPACK/share/spack/setup-env.*. +This module also includes logic for coming up with unique names for the module files so that they can be found by the +various shell-support files in $SPACK/share/spack/setup-env.*. -Each hook in hooks/ implements the logic for writing its specific type -of module file. +Each hook in hooks/ implements the logic for writing its specific type of module file. """ -__all__ = ['EnvModule', 'Dotkit', 'TclModule'] - import os +import os.path import re -import textwrap import shutil -from glob import glob +import textwrap import llnl.util.tty as tty +import spack from llnl.util.filesystem import join_path, mkdirp +from spack.environment import * -import spack +__all__ = ['EnvModule', 'Dotkit', 'TclModule'] -"""Registry of all types of modules. Entries created by EnvModule's - metaclass.""" +# Registry of all types of modules. Entries created by EnvModule's metaclass module_types = {} @@ -78,8 +74,43 @@ def print_help(): "") +def inspect_path(prefix): + """ + Inspects the prefix of an installation to search for common layouts. Issues a request to modify the environment + accordingly when an item is found. + + Args: + prefix: prefix of the installation + + Returns: + instance of EnvironmentModifications containing the requested modifications + """ + env = EnvironmentModifications() + # Inspect the prefix to check for the existence of common directories + prefix_inspections = { + 'bin': ('PATH',), + 'man': ('MANPATH',), + 'lib': ('LIBRARY_PATH', 'LD_LIBRARY_PATH'), + 'lib64': ('LIBRARY_PATH', 'LD_LIBRARY_PATH'), + 'include': ('CPATH',) + } + for attribute, variables in prefix_inspections.items(): + expected = getattr(prefix, attribute) + if os.path.isdir(expected): + for variable in variables: + env.prepend_path(variable, expected) + # PKGCONFIG + for expected in (join_path(prefix.lib, 'pkgconfig'), join_path(prefix.lib64, 'pkgconfig')): + if os.path.isdir(expected): + env.prepend_path('PKG_CONFIG_PATH', expected) + # CMake related variables + env.prepend_path('CMAKE_PREFIX_PATH', prefix) + return env + + class EnvModule(object): name = 'env_module' + formats = {} class __metaclass__(type): def __init__(cls, name, bases, dict): @@ -87,65 +118,32 @@ class EnvModule(object): if cls.name != 'env_module': module_types[cls.name] = cls - def __init__(self, spec=None): - # category in the modules system - # TODO: come up with smarter category names. - self.category = "spack" + self.spec = spec + self.pkg = spec.package # Just stored for convenience - # Descriptions for the module system's UI - self.short_description = "" - self.long_description = "" + # short description default is just the package + version + # packages can provide this optional attribute + self.short_description = spec.format("$_ $@") + if hasattr(self.pkg, 'short_description'): + self.short_description = self.pkg.short_description - # dict pathname -> list of directories to be prepended to in - # the module file. - self._paths = None - self.spec = spec + # long description is the docstring with reduced whitespace. + self.long_description = None + if self.spec.package.__doc__: + self.long_description = re.sub(r'\s+', ' ', self.spec.package.__doc__) @property - def paths(self): - if self._paths is None: - self._paths = {} - - def add_path(path_name, directory): - path = self._paths.setdefault(path_name, []) - path.append(directory) - - # Add paths if they exist. - for var, directory in [ - ('PATH', self.spec.prefix.bin), - ('MANPATH', self.spec.prefix.man), - ('MANPATH', self.spec.prefix.share_man), - ('LIBRARY_PATH', self.spec.prefix.lib), - ('LIBRARY_PATH', self.spec.prefix.lib64), - ('LD_LIBRARY_PATH', self.spec.prefix.lib), - ('LD_LIBRARY_PATH', self.spec.prefix.lib64), - ('PKG_CONFIG_PATH', join_path(self.spec.prefix.lib, 'pkgconfig')), - ('PKG_CONFIG_PATH', join_path(self.spec.prefix.lib64, 'pkgconfig'))]: - - if os.path.isdir(directory): - add_path(var, directory) - - # Add python path unless it's an actual python installation - # TODO: is there a better way to do this? - if self.spec.name != 'python': - site_packages = glob(join_path(self.spec.prefix.lib, "python*/site-packages")) - if site_packages: - add_path('PYTHONPATH', site_packages[0]) - - if self.spec.package.extends(spack.spec.Spec('ruby')): - add_path('GEM_PATH', self.spec.prefix) - - # short description is just the package + version - # TODO: maybe packages can optionally provide it. - self.short_description = self.spec.format("$_ $@") - - # long description is the docstring with reduced whitespace. - if self.spec.package.__doc__: - self.long_description = re.sub(r'\s+', ' ', self.spec.package.__doc__) - - return self._paths + def category(self): + # Anything defined at the package level takes precedence + if hasattr(self.pkg, 'category'): + return self.pkg.category + # Extensions + for extendee in self.pkg.extendees: + return '{extendee} extension'.format(extendee=extendee) + # Not very descriptive fallback + return 'spack installed package' def write(self): @@ -154,18 +152,41 @@ class EnvModule(object): if not os.path.exists(module_dir): mkdirp(module_dir) - # If there are no paths, no need for a dotkit. - if not self.paths: + # Environment modifications guessed by inspecting the + # installation prefix + env = inspect_path(self.spec.prefix) + + # Let the extendee modify their extensions before asking for + # package-specific modifications + for extendee in self.pkg.extendees: + extendee_spec = self.spec[extendee] + extendee_spec.package.modify_module( + self.pkg.module, extendee_spec, self.spec) + + # Package-specific environment modifications + spack_env = EnvironmentModifications() + self.spec.package.setup_environment(spack_env, env) + + # TODO : implement site-specific modifications and filters + if not env: return with open(self.file_name, 'w') as f: - self._write(f) + self.write_header(f) + for line in self.process_environment_command(env): + f.write(line) - - def _write(self, stream): - """To be implemented by subclasses.""" + def write_header(self, stream): raise NotImplementedError() + def process_environment_command(self, env): + for command in env: + try: + yield self.formats[type(command)].format(**command.args) + except KeyError: + tty.warn('Cannot handle command of type {command} : skipping request'.format(command=type(command))) + tty.warn('{context} at {filename}:{lineno}'.format(**command.args)) + @property def file_name(self): @@ -173,14 +194,12 @@ class EnvModule(object): where this module lives.""" raise NotImplementedError() - @property def use_name(self): """Subclasses should implement this to return the name the module command uses to refer to the package.""" raise NotImplementedError() - def remove(self): mod_file = self.file_name if os.path.exists(mod_file): @@ -191,17 +210,23 @@ class Dotkit(EnvModule): name = 'dotkit' path = join_path(spack.share_path, "dotkit") + formats = { + PrependPath: 'dk_alter {name} {value}\n', + SetEnv: 'dk_setenv {name} {value}\n' + } + @property def file_name(self): - return join_path(Dotkit.path, self.spec.architecture, - self.spec.format('$_$@$%@$+$#.dk')) + return join_path(Dotkit.path, self.spec.architecture, '%s.dk' % self.use_name) @property def use_name(self): - return self.spec.format('$_$@$%@$+$#') - + return "%s-%s-%s-%s-%s" % (self.spec.name, self.spec.version, + self.spec.compiler.name, + self.spec.compiler.version, + self.spec.dag_hash()) - def _write(self, dk_file): + def write_header(self, dk_file): # Category if self.category: dk_file.write('#c %s\n' % self.category) @@ -215,47 +240,41 @@ class Dotkit(EnvModule): for line in textwrap.wrap(self.long_description, 72): dk_file.write("#h %s\n" % line) - # Path alterations - for var, dirs in self.paths.items(): - for directory in dirs: - dk_file.write("dk_alter %s %s\n" % (var, directory)) - - # Let CMake find this package. - dk_file.write("dk_alter CMAKE_PREFIX_PATH %s\n" % self.spec.prefix) - class TclModule(EnvModule): name = 'tcl' path = join_path(spack.share_path, "modules") + formats = { + PrependPath: 'prepend-path {name} \"{value}\"\n', + AppendPath: 'append-path {name} \"{value}\"\n', + RemovePath: 'remove-path {name} \"{value}\"\n', + SetEnv: 'setenv {name} \"{value}\"\n', + UnsetEnv: 'unsetenv {name}\n' + } + @property def file_name(self): return join_path(TclModule.path, self.spec.architecture, self.use_name) - @property def use_name(self): - return self.spec.format('$_$@$%@$+$#') - - - def _write(self, m_file): - # TODO: cateogry? - m_file.write('#%Module1.0\n') - + return "%s-%s-%s-%s-%s" % (self.spec.name, self.spec.version, + self.spec.compiler.name, + self.spec.compiler.version, + self.spec.dag_hash()) + + def write_header(self, module_file): + # TCL Modulefile header + module_file.write('#%Module1.0\n') + # TODO : category ? # Short description if self.short_description: - m_file.write('module-whatis \"%s\"\n\n' % self.short_description) + module_file.write('module-whatis \"%s\"\n\n' % self.short_description) # Long description if self.long_description: - m_file.write('proc ModulesHelp { } {\n') + module_file.write('proc ModulesHelp { } {\n') doc = re.sub(r'"', '\"', self.long_description) - m_file.write("puts stderr \"%s\"\n" % doc) - m_file.write('}\n\n') - - # Path alterations - for var, dirs in self.paths.items(): - for directory in dirs: - m_file.write("prepend-path %s \"%s\"\n" % (var, directory)) - - m_file.write("prepend-path CMAKE_PREFIX_PATH \"%s\"\n" % self.spec.prefix) + module_file.write("puts stderr \"%s\"\n" % doc) + module_file.write('}\n\n') diff --git a/lib/spack/spack/multimethod.py b/lib/spack/spack/multimethod.py index df9b9b2ab1..3cd17e796a 100644 --- a/lib/spack/spack/multimethod.py +++ b/lib/spack/spack/multimethod.py @@ -138,7 +138,7 @@ class when(object): methods like install() that depend on the package's spec. For example: - .. code-block:: + .. code-block:: python class SomePackage(Package): ... @@ -163,26 +163,28 @@ class when(object): if you only have part of the install that is platform specific, you could do this: - class SomePackage(Package): - ... - # virtual dependence on MPI. - # could resolve to mpich, mpich2, OpenMPI - depends_on('mpi') + .. code-block:: python - def setup(self): - # do nothing in the default case - pass + class SomePackage(Package): + ... + # virtual dependence on MPI. + # could resolve to mpich, mpich2, OpenMPI + depends_on('mpi') + + def setup(self): + # do nothing in the default case + pass - @when('^openmpi') - def setup(self): - # do something special when this is built with OpenMPI for - # its MPI implementations. + @when('^openmpi') + def setup(self): + # do something special when this is built with OpenMPI for + # its MPI implementations. - def install(self, prefix): - # Do common install stuff - self.setup() - # Do more common install stuff + def install(self, prefix): + # Do common install stuff + self.setup() + # Do more common install stuff There must be one (and only one) @when clause that matches the package's spec. If there is more than one, or if none match, @@ -193,10 +195,11 @@ class when(object): platform-specific versions. There's not much we can do to get around this because of the way decorators work. """ -class when(object): def __init__(self, spec): pkg = get_calling_module_name() - self.spec = parse_anonymous_spec(spec, pkg) + if spec is True: + spec = pkg + self.spec = parse_anonymous_spec(spec, pkg) if spec is not False else None def __call__(self, method): # Get the first definition of the method in the calling scope @@ -207,7 +210,9 @@ class when(object): if not type(original_method) == SpecMultiMethod: original_method = SpecMultiMethod(original_method) - original_method.register(self.spec, method) + if self.spec is not None: + original_method.register(self.spec, method) + return original_method diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index fba2912b75..9af3221837 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -34,38 +34,34 @@ rundown on spack and how it differs from homebrew, look at the README. """ import os -import errno import re -import shutil -import time -import itertools -import subprocess -import platform as py_platform -import multiprocessing -from urlparse import urlparse, urljoin import textwrap -from StringIO import StringIO +import time +import glob import llnl.util.tty as tty -from llnl.util.tty.log import log_output -from llnl.util.link_tree import LinkTree -from llnl.util.filesystem import * -from llnl.util.lang import * - import spack -import spack.error +import spack.build_environment import spack.compilers -import spack.mirror -import spack.hooks import spack.directives -import spack.build_environment +import spack.error +import spack.fetch_strategy as fs +import spack.hooks +import spack.mirror +import spack.repository import spack.url import spack.util.web -import spack.fetch_strategy as fs -from spack.version import * -from spack.stage import Stage -from spack.util.compression import allowed_archive, extension +from StringIO import StringIO +from llnl.util.filesystem import * +from llnl.util.lang import * +from llnl.util.link_tree import LinkTree +from llnl.util.tty.log import log_output +from spack.stage import Stage, ResourceStage, StageComposite +from spack.util.compression import allowed_archive +from spack.util.environment import dump_environment from spack.util.executable import ProcessError +from spack.version import * +from urlparse import urlparse """Allowed URL schemes for spack packages.""" _ALLOWED_URL_SCHEMES = ["http", "https", "ftp", "file", "git"] @@ -316,6 +312,18 @@ class Package(object): """Most packages are NOT extendable. Set to True if you want extensions.""" extendable = False + """List of prefix-relative file paths (or a single path). If these do + not exist after install, or if they exist but are not files, + sanity checks fail. + """ + sanity_check_is_file = [] + + """List of prefix-relative directory paths (or a single path). If + these do not exist after install, or if they exist but are not + directories, sanity checks will fail. + """ + sanity_check_is_dir = [] + def __init__(self, spec): # this determines how the package should be built. @@ -433,23 +441,51 @@ class Package(object): return spack.url.substitute_version(self.nearest_url(version), self.url_version(version)) + def _make_resource_stage(self, root_stage, fetcher, resource): + resource_stage_folder = self._resource_stage(resource) + resource_mirror = join_path(self.name, os.path.basename(fetcher.url)) + stage = ResourceStage(resource.fetcher, root=root_stage, resource=resource, + name=resource_stage_folder, mirror_path=resource_mirror) + return stage + + def _make_root_stage(self, fetcher): + # Construct a mirror path (TODO: get this out of package.py) + mp = spack.mirror.mirror_archive_path(self.spec, fetcher) + # Construct a path where the stage should build.. + s = self.spec + stage_name = "%s-%s-%s" % (s.name, s.version, s.dag_hash()) + # Build the composite stage + stage = Stage(fetcher, mirror_path=mp, name=stage_name) + return stage + + def _make_stage(self): + # Construct a composite stage on top of the composite FetchStrategy + composite_fetcher = self.fetcher + composite_stage = StageComposite() + resources = self._get_needed_resources() + for ii, fetcher in enumerate(composite_fetcher): + if ii == 0: + # Construct root stage first + stage = self._make_root_stage(fetcher) + else: + # Construct resource stage + resource = resources[ii - 1] # ii == 0 is root! + stage = self._make_resource_stage(composite_stage[0], fetcher, resource) + # Append the item to the composite + composite_stage.append(stage) + + # Create stage on first access. Needed because fetch, stage, + # patch, and install can be called independently of each + # other, so `with self.stage:` in do_install isn't sufficient. + composite_stage.create() + return composite_stage @property def stage(self): if not self.spec.concrete: raise ValueError("Can only get a stage for a concrete package.") - if self._stage is None: - # Construct a mirror path (TODO: get this out of package.py) - mp = spack.mirror.mirror_archive_path(self.spec) - - # Construct a path where the stage should build.. - s = self.spec - stage_name = "%s-%s-%s" % (s.name, s.version, s.dag_hash()) - - # Build the stage - self._stage = Stage(self.fetcher, mirror_path=mp, name=stage_name) - + self._stage = self._make_stage() return self._stage @@ -459,17 +495,27 @@ class Package(object): self._stage = stage + def _make_fetcher(self): + # Construct a composite fetcher that always contains at least + # one element (the root package). In case there are resources + # associated with the package, append their fetcher to the + # composite. + root_fetcher = fs.for_package_version(self, self.version) + fetcher = fs.FetchStrategyComposite() # Composite fetcher + fetcher.append(root_fetcher) # Root fetcher is always present + resources = self._get_needed_resources() + for resource in resources: + fetcher.append(resource.fetcher) + return fetcher + @property def fetcher(self): if not self.spec.versions.concrete: - raise ValueError( - "Can only get a fetcher for a package with concrete versions.") - + raise ValueError("Can only get a fetcher for a package with concrete versions.") if not self._fetcher: - self._fetcher = fs.for_package_version(self, self.version) + self._fetcher = self._make_fetcher() return self._fetcher - @fetcher.setter def fetcher(self, f): self._fetcher = f @@ -631,8 +677,8 @@ class Package(object): spack.install_layout.remove_install_directory(self.spec) - def do_fetch(self): - """Creates a stage directory and downloads the taball for this package. + def do_fetch(self, mirror_only=False): + """Creates a stage directory and downloads the tarball for this package. Working directory will be set to the stage directory. """ if not self.spec.concrete: @@ -654,79 +700,24 @@ class Package(object): if not ignore_checksum: raise FetchError( - "Will not fetch %s." % self.spec.format('$_$@'), checksum_msg) - - self.stage.fetch() + "Will not fetch %s" % self.spec.format('$_$@'), checksum_msg) - ########## - # Fetch resources - resources = self._get_resources() - for resource in resources: - resource_stage_folder = self._resource_stage(resource) - # FIXME : works only for URLFetchStrategy - resource_mirror = join_path(self.name, os.path.basename(resource.fetcher.url)) - resource_stage = Stage(resource.fetcher, name=resource_stage_folder, mirror_path=resource_mirror) - resource.fetcher.set_stage(resource_stage) - # Delegate to stage object to trigger mirror logic - resource_stage.fetch() - resource_stage.check() - ########## + self.stage.fetch(mirror_only) self._fetch_time = time.time() - start_time if spack.do_checksum and self.version in self.versions: self.stage.check() - def do_stage(self): + + def do_stage(self, mirror_only=False): """Unpacks the fetched tarball, then changes into the expanded tarball directory.""" if not self.spec.concrete: raise ValueError("Can only stage concrete packages.") - def _expand_archive(stage, name=self.name): - archive_dir = stage.source_path - if not archive_dir: - stage.expand_archive() - tty.msg("Created stage in %s." % stage.path) - else: - tty.msg("Already staged %s in %s." % (name, stage.path)) - - - self.do_fetch() - _expand_archive(self.stage) - - ########## - # Stage resources in appropriate path - resources = self._get_resources() - # TODO: this is to allow nested resources, a better solution would be - # good - for resource in sorted(resources, key=lambda res: len(res.destination)): - stage = resource.fetcher.stage - _expand_archive(stage, resource.name) - # Turn placement into a dict with relative paths - placement = os.path.basename(stage.source_path) if resource.placement is None else resource.placement - if not isinstance(placement, dict): - placement = {'': placement} - # Make the paths in the dictionary absolute and link - for key, value in placement.iteritems(): - target_path = join_path(self.stage.source_path, resource.destination) - link_path = join_path(target_path, value) - source_path = join_path(stage.source_path, key) - - try: - os.makedirs(target_path) - except OSError as err: - if err.errno == errno.EEXIST and os.path.isdir(target_path): - pass - else: raise - - # NOTE: a reasonable fix for the TODO above might be to have - # these expand in place, but expand_archive does not offer - # this - - if not os.path.exists(link_path): - shutil.move(source_path, link_path) - ########## + self.do_fetch(mirror_only) + self.stage.expand_archive() self.stage.chdir_to_source() @@ -744,7 +735,7 @@ class Package(object): # If there are no patches, note it. if not self.patches and not has_patch_fun: - tty.msg("No patches needed for %s." % self.name) + tty.msg("No patches needed for %s" % self.name) return # Construct paths to special files in the archive dir used to @@ -757,7 +748,7 @@ class Package(object): # If we encounter an archive that failed to patch, restage it # so that we can apply all the patches again. if os.path.isfile(bad_file): - tty.msg("Patching failed last time. Restaging.") + tty.msg("Patching failed last time. Restaging.") self.stage.restage() self.stage.chdir_to_source() @@ -767,7 +758,7 @@ class Package(object): tty.msg("Already patched %s" % self.name) return elif os.path.isfile(no_patches_file): - tty.msg("No patches needed for %s." % self.name) + tty.msg("No patches needed for %s" % self.name) return # Apply all the patches for specs that match this one @@ -788,10 +779,10 @@ class Package(object): if has_patch_fun: try: self.patch() - tty.msg("Ran patch() for %s." % self.name) + tty.msg("Ran patch() for %s" % self.name) patched = True except: - tty.msg("patch() function failed for %s." % self.name) + tty.msg("patch() function failed for %s" % self.name) touch(bad_file) raise @@ -822,12 +813,15 @@ class Package(object): mkdirp(self.prefix.man1) - def _get_resources(self): + def _get_needed_resources(self): resources = [] # Select the resources that are needed for this build for when_spec, resource_list in self.resources.items(): if when_spec in self.spec: resources.extend(resource_list) + # Sorts the resources by the length of the string representing their destination. Since any nested resource + # must contain another resource's name in its path, it seems that should work + resources = sorted(resources, key=lambda res: len(res.destination)) return resources def _resource_stage(self, resource): @@ -835,10 +829,6 @@ class Package(object): resource_stage_folder = '-'.join(pieces) return resource_stage_folder - def _build_logger(self, log_path): - """Create a context manager to log build output.""" - - def do_install(self, keep_prefix=False, keep_stage=False, ignore_deps=False, @@ -850,7 +840,9 @@ class Package(object): Args: keep_prefix -- Keep install prefix on failure. By default, destroys it. - keep_stage -- Keep stage on successful build. By default, destroys it. + keep_stage -- By default, stage is destroyed only if there are no + exceptions during build. Set to True to keep the stage + even with exceptions. ignore_deps -- Do not install dependencies before installing this package. fake -- Don't really build -- install fake stub files instead. skip_patch -- Skip patch stage of build if True. @@ -860,99 +852,103 @@ class Package(object): if not self.spec.concrete: raise ValueError("Can only install concrete packages.") - if os.path.exists(self.prefix): - tty.msg("%s is already installed in %s." % (self.name, self.prefix)) + # No installation needed if package is external + if self.spec.external: + tty.msg("%s is externally installed in %s" % (self.name, self.spec.external)) + return + + # Ensure package is not already installed + if spack.install_layout.check_installed(self.spec): + tty.msg("%s is already installed in %s" % (self.name, self.prefix)) return tty.msg("Installing %s" % self.name) + # First, install dependencies recursively. if not ignore_deps: self.do_install_dependencies( keep_prefix=keep_prefix, keep_stage=keep_stage, ignore_deps=ignore_deps, - fake=fake, skip_patch=skip_patch, verbose=verbose, - make_jobs=make_jobs) - - start_time = time.time() - if not fake: - if not skip_patch: - self.do_patch() - else: - self.do_stage() + fake=fake, skip_patch=skip_patch, verbose=verbose, make_jobs=make_jobs) - # create the install directory. The install layout - # handles this in case so that it can use whatever - # package naming scheme it likes. - spack.install_layout.create_install_directory(self.spec) - - def cleanup(): - if not keep_prefix: - # If anything goes wrong, remove the install prefix - self.remove_prefix() - else: - tty.warn("Keeping install prefix in place despite error.", - "Spack will think this package is installed." + - "Manually remove this directory to fix:", - self.prefix, wrap=True) + # Set parallelism before starting build. + self.make_jobs = make_jobs + # Then install the package itself. + def build_process(): + """Forked for each build. Has its own process and python + module space set up by build_environment.fork().""" + start_time = time.time() + if not fake: + if not skip_patch: + self.do_patch() + else: + self.do_stage() - def real_work(): - try: - tty.msg("Building %s." % self.name) + tty.msg("Building %s" % self.name) + self.stage.keep = keep_stage + with self.stage: # Run the pre-install hook in the child process after # the directory is created. spack.hooks.pre_install(self) - # Set up process's build environment before running install. if fake: self.do_fake_install() else: # Do the real install in the source directory. - self.stage.chdir_to_source() - - # This redirects I/O to a build log (and optionally to the terminal) - log_path = join_path(os.getcwd(), 'spack-build.out') - log_file = open(log_path, 'w') - with log_output(log_file, verbose, sys.stdout.isatty(), True): - self.install(self.spec, self.prefix) + self.stage.chdir_to_source() - # Ensure that something was actually installed. - self._sanity_check_install() + # Save the build environment in a file before building. + env_path = join_path(os.getcwd(), 'spack-build.env') - # Move build log into install directory on success - if not fake: - log_install_path = spack.install_layout.build_log_path(self.spec) - install(log_path, log_install_path) + try: + # Redirect I/O to a build log (and optionally to the terminal) + log_path = join_path(os.getcwd(), 'spack-build.out') + log_file = open(log_path, 'w') + with log_output(log_file, verbose, sys.stdout.isatty(), True): + dump_environment(env_path) + self.install(self.spec, self.prefix) - # On successful install, remove the stage. - if not keep_stage: - self.stage.destroy() + except ProcessError as e: + # Annotate ProcessErrors with the location of the build log. + e.build_log = log_path + raise e - # Stop timer. - self._total_time = time.time() - start_time - build_time = self._total_time - self._fetch_time + # Ensure that something was actually installed. + self.sanity_check_prefix() - tty.msg("Successfully installed %s." % self.name, - "Fetch: %s. Build: %s. Total: %s." - % (_hms(self._fetch_time), _hms(build_time), _hms(self._total_time))) - print_pkg(self.prefix) + # Copy provenance into the install directory on success + log_install_path = spack.install_layout.build_log_path(self.spec) + env_install_path = spack.install_layout.build_env_path(self.spec) + packages_dir = spack.install_layout.build_packages_path(self.spec) - except ProcessError, e: - # Annotate with location of build log. - e.build_log = log_path - cleanup() - raise e + install(log_path, log_install_path) + install(env_path, env_install_path) + dump_packages(self.spec, packages_dir) - except: - # other exceptions just clean up and raise. - cleanup() - raise + # Stop timer. + self._total_time = time.time() - start_time + build_time = self._total_time - self._fetch_time - # Set parallelism before starting build. - self.make_jobs = make_jobs + tty.msg("Successfully installed %s" % self.name, + "Fetch: %s. Build: %s. Total: %s." + % (_hms(self._fetch_time), _hms(build_time), _hms(self._total_time))) + print_pkg(self.prefix) - # Do the build. - spack.build_environment.fork(self, real_work) + try: + # Create the install prefix and fork the build process. + spack.install_layout.create_install_directory(self.spec) + spack.build_environment.fork(self, build_process) + except: + # remove the install prefix if anything went wrong during install. + if not keep_prefix: + self.remove_prefix() + else: + tty.warn("Keeping install prefix in place despite error.", + "Spack will think this package is installed. " + + "Manually remove this directory to fix:", + self.prefix, wrap=True) + raise # note: PARENT of the build process adds the new package to # the database, so that we don't need to re-read from file. @@ -962,7 +958,21 @@ class Package(object): spack.hooks.post_install(self) - def _sanity_check_install(self): + def sanity_check_prefix(self): + """This function checks whether install succeeded.""" + def check_paths(path_list, filetype, predicate): + if isinstance(path_list, basestring): + path_list = [path_list] + + for path in path_list: + abs_path = os.path.join(self.prefix, path) + if not predicate(abs_path): + raise InstallError("Install failed for %s. No such %s in prefix: %s" + % (self.name, filetype, path)) + + check_paths(self.sanity_check_is_file, 'file', os.path.isfile) + check_paths(self.sanity_check_is_dir, 'directory', os.path.isdir) + installed = set(os.listdir(self.prefix)) installed.difference_update(spack.install_layout.hidden_file_paths) if not installed: @@ -992,38 +1002,127 @@ class Package(object): return __import__(self.__class__.__module__, fromlist=[self.__class__.__name__]) + def setup_environment(self, spack_env, run_env): + """Set up the compile and runtime environemnts for a package. + + `spack_env` and `run_env` are `EnvironmentModifications` + objects. Package authors can call methods on them to alter + the environment within Spack and at runtime. + + Both `spack_env` and `run_env` are applied within the build + process, before this package's `install()` method is called. - def setup_dependent_environment(self, module, spec, dependent_spec): - """Called before the install() method of dependents. + Modifications in `run_env` will *also* be added to the + generated environment modules for this package. Default implementation does nothing, but this can be - overridden by an extendable package to set up the install - environment for its extensions. This is useful if there are - some common steps to installing all extensions for a - certain package. + overridden if the package needs a particular environment. - Some examples: + Examples: - 1. Installing python modules generally requires PYTHONPATH to - point to the lib/pythonX.Y/site-packages directory in the - module's install prefix. This could set that variable. + 1. Qt extensions need `QTDIR` set. - 2. Extensions often need to invoke the 'python' interpreter - from the Python installation being extended. This routine can - put a 'python' Execuable object in the module scope for the - extension package to simplify extension installs. + Args: + spack_env (EnvironmentModifications): list of + modifications to be applied when this package is built + within Spack. - 3. A lot of Qt extensions need QTDIR set. This can be used to do that. + run_env (EnvironmentModifications): list of environment + changes to be applied when this package is run outside + of Spack. """ pass + def setup_dependent_environment(self, spack_env, run_env, dependent_spec): + """Set up the environment of packages that depend on this one. + + This is similar to `setup_environment`, but it is used to + modify the compile and runtime environments of packages that + *depend* on this one. This gives packages like Python and + others that follow the extension model a way to implement + common environment or compile-time settings for dependencies. + + By default, this delegates to self.setup_environment() + + Example : + + 1. Installing python modules generally requires + `PYTHONPATH` to point to the lib/pythonX.Y/site-packages + directory in the module's install prefix. This could + set that variable. + + Args: + + spack_env (EnvironmentModifications): list of + modifications to be applied when the dependent package + is bulit within Spack. + + run_env (EnvironmentModifications): list of environment + changes to be applied when the dependent package is + run outside of Spack. + + dependent_spec (Spec): The spec of the dependent package + about to be built. This allows the extendee (self) to + query the dependent's state. Note that *this* + package's spec is available as `self.spec`. + + This is useful if there are some common steps to installing + all extensions for a certain package. + + """ + self.setup_environment(spack_env, run_env) + + + def setup_dependent_package(self, module, dependent_spec): + """Set up Python module-scope variables for dependent packages. + + Called before the install() method of dependents. + + Default implementation does nothing, but this can be + overridden by an extendable package to set up the module of + its extensions. This is useful if there are some common steps + to installing all extensions for a certain package. + + Example : + + 1. Extensions often need to invoke the `python` + interpreter from the Python installation being + extended. This routine can put a 'python' Executable + object in the module scope for the extension package to + simplify extension installs. + + 2. MPI compilers could set some variables in the + dependent's scope that point to `mpicc`, `mpicxx`, + etc., allowing them to be called by common names + regardless of which MPI is used. + + 3. BLAS/LAPACK implementations can set some variables + indicating the path to their libraries, since these + paths differ by BLAS/LAPACK implementation. + + Args: + + module (module): The Python `module` object of the + dependent package. Packages can use this to set + module-scope variables for the dependent to use. + + dependent_spec (Spec): The spec of the dependent package + about to be built. This allows the extendee (self) to + query the dependent's state. Note that *this* + package's spec is available as `self.spec`. + + This is useful if there are some common steps to installing + all extensions for a certain package. + + """ + pass + def install(self, spec, prefix): """Package implementations override this with their own build configuration.""" raise InstallError("Package %s provides no install method!" % self.name) - def do_uninstall(self, force=False): if not self.installed: raise InstallError(str(self.spec) + " is not installed.") @@ -1039,7 +1138,7 @@ class Package(object): # Uninstalling in Spack only requires removing the prefix. self.remove_prefix() spack.installed_db.remove(self.spec) - tty.msg("Successfully uninstalled %s." % self.spec.short_spec) + tty.msg("Successfully uninstalled %s" % self.spec.short_spec) # Once everything else is done, run post install hooks spack.hooks.post_uninstall(self) @@ -1086,7 +1185,7 @@ class Package(object): self.extendee_spec.package.activate(self, **self.extendee_args) spack.install_layout.add_extension(self.extendee_spec, self.spec) - tty.msg("Activated extension %s for %s." + tty.msg("Activated extension %s for %s" % (self.spec.short_spec, self.extendee_spec.format("$_$@$+$%@"))) @@ -1138,7 +1237,7 @@ class Package(object): if self.activated: spack.install_layout.remove_extension(self.extendee_spec, self.spec) - tty.msg("Deactivated extension %s for %s." + tty.msg("Deactivated extension %s for %s" % (self.spec.short_spec, self.extendee_spec.format("$_$@$+$%@"))) @@ -1166,8 +1265,7 @@ class Package(object): def do_clean(self): """Removes the package's build stage and source tarball.""" - if os.path.exists(self.stage.path): - self.stage.destroy() + self.stage.destroy() def format_doc(self, **kwargs): @@ -1206,7 +1304,7 @@ class Package(object): try: return spack.util.web.find_versions_of_archive( *self.all_urls, list_url=self.list_url, list_depth=self.list_depth) - except spack.error.NoNetworkConnectionError, e: + except spack.error.NoNetworkConnectionError as e: tty.die("Package.fetch_versions couldn't connect to:", e.url, e.message) @@ -1224,8 +1322,29 @@ class Package(object): @property def rpath_args(self): - """Get the rpath args as a string, with -Wl,-rpath= for each element.""" - return " ".join("-Wl,-rpath=%s" % p for p in self.rpath) + """Get the rpath args as a string, with -Wl,-rpath, for each element.""" + return " ".join("-Wl,-rpath,%s" % p for p in self.rpath) + + +def install_dependency_symlinks(pkg, spec, prefix): + """Execute a dummy install and flatten dependencies""" + flatten_dependencies(spec, prefix) + +def flatten_dependencies(spec, flat_dir): + """Make each dependency of spec present in dir via symlink.""" + for dep in spec.traverse(root=False): + name = dep.name + + dep_path = spack.install_layout.path_for_spec(dep) + dep_files = LinkTree(dep_path) + + os.mkdir(flat_dir+'/'+name) + + conflict = dep_files.find_conflict(flat_dir+'/'+name) + if conflict: + raise DependencyConflictError(conflict) + + dep_files.merge(flat_dir+'/'+name) def validate_package_url(url_string): @@ -1238,6 +1357,52 @@ def validate_package_url(url_string): tty.die("Invalid file type in URL: '%s'" % url_string) +def dump_packages(spec, path): + """Dump all package information for a spec and its dependencies. + + This creates a package repository within path for every + namespace in the spec DAG, and fills the repos wtih package + files and patch files for every node in the DAG. + """ + mkdirp(path) + + # Copy in package.py files from any dependencies. + # Note that we copy them in as they are in the *install* directory + # NOT as they are in the repository, because we want a snapshot of + # how *this* particular build was done. + for node in spec.traverse(): + if node is not spec: + # Locate the dependency package in the install tree and find + # its provenance information. + source = spack.install_layout.build_packages_path(node) + source_repo_root = join_path(source, node.namespace) + + # There's no provenance installed for the source package. Skip it. + # User can always get something current from the builtin repo. + if not os.path.isdir(source_repo_root): + continue + + # Create a source repo and get the pkg directory out of it. + try: + source_repo = spack.repository.Repo(source_repo_root) + source_pkg_dir = source_repo.dirname_for_package_name(node.name) + except RepoError as e: + tty.warn("Warning: Couldn't copy in provenance for %s" % node.name) + + # Create a destination repository + dest_repo_root = join_path(path, node.namespace) + if not os.path.exists(dest_repo_root): + spack.repository.create_repo(dest_repo_root) + repo = spack.repository.Repo(dest_repo_root) + + # Get the location of the package in the dest repo. + dest_pkg_dir = repo.dirname_for_package_name(node.name) + if node is not spec: + install_tree(source_pkg_dir, dest_pkg_dir) + else: + spack.repo.dump_provenance(node, dest_pkg_dir) + + def print_pkg(message): """Outputs a message with a package icon.""" from llnl.util.tty.color import cwrite @@ -1269,6 +1434,10 @@ class InstallError(spack.error.SpackError): super(InstallError, self).__init__(message, long_msg) +class ExternalPackageError(InstallError): + """Raised by install() when a package is only for external use.""" + + class PackageStillNeededError(InstallError): """Raised when package is still needed by another on uninstall.""" def __init__(self, spec, dependents): @@ -1288,7 +1457,7 @@ class PackageVersionError(PackageError): """Raised when a version URL cannot automatically be determined.""" def __init__(self, version): super(PackageVersionError, self).__init__( - "Cannot determine a URL automatically for version %s." % version, + "Cannot determine a URL automatically for version %s" % version, "Please provide a url for this version in the package.py file.") @@ -1319,3 +1488,11 @@ class ExtensionConflictError(ExtensionError): class ActivationError(ExtensionError): def __init__(self, msg, long_msg=None): super(ActivationError, self).__init__(msg, long_msg) + + +class DependencyConflictError(spack.error.SpackError): + """Raised when the dependencies cannot be flattened as asked for.""" + def __init__(self, conflict): + super(DependencyConflictError, self).__init__( + "%s conflicts with another file in the flattened directory." %( + conflict)) diff --git a/lib/spack/spack/preferred_packages.py b/lib/spack/spack/preferred_packages.py new file mode 100644 index 0000000000..4d8526c75f --- /dev/null +++ b/lib/spack/spack/preferred_packages.py @@ -0,0 +1,175 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## + +import spack +from spack.version import * + +class PreferredPackages(object): + _default_order = {'compiler' : [ 'gcc', 'intel', 'clang', 'pgi', 'xlc' ] } # Arbitrary, but consistent + + def __init__(self): + self.preferred = spack.config.get_config('packages') + self._spec_for_pkgname_cache = {} + + # Given a package name, sort component (e.g, version, compiler, ...), and + # a second_key (used by providers), return the list + def _order_for_package(self, pkgname, component, second_key, test_all=True): + pkglist = [pkgname] + if test_all: + pkglist.append('all') + for pkg in pkglist: + order = self.preferred.get(pkg, {}).get(component, {}) + if type(order) is dict: + order = order.get(second_key, {}) + if not order: + continue + return [str(s).strip() for s in order] + return [] + + + # A generic sorting function. Given a package name and sort + # component, return less-than-0, 0, or greater-than-0 if + # a is respectively less-than, equal to, or greater than b. + def _component_compare(self, pkgname, component, a, b, reverse_natural_compare, second_key): + if a is None: + return -1 + if b is None: + return 1 + orderlist = self._order_for_package(pkgname, component, second_key) + a_in_list = str(a) in orderlist + b_in_list = str(b) in orderlist + if a_in_list and not b_in_list: + return -1 + elif b_in_list and not a_in_list: + return 1 + + cmp_a = None + cmp_b = None + reverse = None + if not a_in_list and not b_in_list: + cmp_a = a + cmp_b = b + reverse = -1 if reverse_natural_compare else 1 + else: + cmp_a = orderlist.index(str(a)) + cmp_b = orderlist.index(str(b)) + reverse = 1 + + if cmp_a < cmp_b: + return -1 * reverse + elif cmp_a > cmp_b: + return 1 * reverse + else: + return 0 + + + # A sorting function for specs. Similar to component_compare, but + # a and b are considered to match entries in the sorting list if they + # satisfy the list component. + def _spec_compare(self, pkgname, component, a, b, reverse_natural_compare, second_key): + if not a or not a.concrete: + return -1 + if not b or not b.concrete: + return 1 + specs = self._spec_for_pkgname(pkgname, component, second_key) + a_index = None + b_index = None + reverse = -1 if reverse_natural_compare else 1 + for i, cspec in enumerate(specs): + if a_index == None and (cspec.satisfies(a) or a.satisfies(cspec)): + a_index = i + if b_index: + break + if b_index == None and (cspec.satisfies(b) or b.satisfies(cspec)): + b_index = i + if a_index: + break + + if a_index != None and b_index == None: return -1 + elif a_index == None and b_index != None: return 1 + elif a_index != None and b_index == a_index: return -1 * cmp(a, b) + elif a_index != None and b_index != None and a_index != b_index: return cmp(a_index, b_index) + else: return cmp(a, b) * reverse + + + + # Given a sort order specified by the pkgname/component/second_key, return + # a list of CompilerSpecs, VersionLists, or Specs for that sorting list. + def _spec_for_pkgname(self, pkgname, component, second_key): + key = (pkgname, component, second_key) + if not key in self._spec_for_pkgname_cache: + pkglist = self._order_for_package(pkgname, component, second_key) + if not pkglist: + if component in self._default_order: + pkglist = self._default_order[component] + if component == 'compiler': + self._spec_for_pkgname_cache[key] = [spack.spec.CompilerSpec(s) for s in pkglist] + elif component == 'version': + self._spec_for_pkgname_cache[key] = [VersionList(s) for s in pkglist] + else: + self._spec_for_pkgname_cache[key] = [spack.spec.Spec(s) for s in pkglist] + return self._spec_for_pkgname_cache[key] + + + def provider_compare(self, pkgname, provider_str, a, b): + """Return less-than-0, 0, or greater than 0 if a is respecively less-than, equal-to, or + greater-than b. A and b are possible implementations of provider_str. + One provider is less-than another if it is preferred over the other. + For example, provider_compare('scorep', 'mpi', 'mvapich', 'openmpi') would return -1 if + mvapich should be preferred over openmpi for scorep.""" + return self._spec_compare(pkgname, 'providers', a, b, False, provider_str) + + + def spec_has_preferred_provider(self, pkgname, provider_str): + """Return True iff the named package has a list of preferred provider""" + return bool(self._order_for_package(pkgname, 'providers', provider_str, False)) + + + def version_compare(self, pkgname, a, b): + """Return less-than-0, 0, or greater than 0 if version a of pkgname is + respecively less-than, equal-to, or greater-than version b of pkgname. + One version is less-than another if it is preferred over the other.""" + return self._spec_compare(pkgname, 'version', a, b, True, None) + + + def variant_compare(self, pkgname, a, b): + """Return less-than-0, 0, or greater than 0 if variant a of pkgname is + respecively less-than, equal-to, or greater-than variant b of pkgname. + One variant is less-than another if it is preferred over the other.""" + return self._component_compare(pkgname, 'variant', a, b, False, None) + + + def architecture_compare(self, pkgname, a, b): + """Return less-than-0, 0, or greater than 0 if architecture a of pkgname is + respecively less-than, equal-to, or greater-than architecture b of pkgname. + One architecture is less-than another if it is preferred over the other.""" + return self._component_compare(pkgname, 'architecture', a, b, False, None) + + + def compiler_compare(self, pkgname, a, b): + """Return less-than-0, 0, or greater than 0 if compiler a of pkgname is + respecively less-than, equal-to, or greater-than compiler b of pkgname. + One compiler is less-than another if it is preferred over the other.""" + return self._spec_compare(pkgname, 'compiler', a, b, False, None) diff --git a/lib/spack/spack/repository.py b/lib/spack/spack/repository.py index 31596cee7a..d2fdc937f7 100644 --- a/lib/spack/spack/repository.py +++ b/lib/spack/spack/repository.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://llnl.github.io/spack +# For details, see https://software.llnl.gov/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify @@ -33,7 +33,7 @@ from bisect import bisect_left from external import yaml import llnl.util.tty as tty -from llnl.util.filesystem import join_path +from llnl.util.filesystem import * import spack.error import spack.config @@ -156,7 +156,7 @@ class RepoPath(object): if repo.namespace in self.by_namespace: raise DuplicateRepoError( - "Package repos '%s' and '%s' both provide namespace %s." + "Package repos '%s' and '%s' both provide namespace %s" % (repo.root, self.by_namespace[repo.namespace].root, repo.namespace)) # Add repo to the pkg indexes @@ -233,6 +233,11 @@ class RepoPath(object): return providers + @_autospec + def extensions_for(self, extendee_spec): + return [p for p in self.all_packages() if p.extends(extendee_spec)] + + def find_module(self, fullname, path=None): """Implements precedence for overlaid namespaces. @@ -295,8 +300,11 @@ class RepoPath(object): for repo in self.repos: if spec.name in repo: return repo - else: - raise UnknownPackageError(spec.name) + + # If the package isn't in any repo, return the one with + # highest precedence. This is for commands like `spack edit` + # that can operate on packages that don't exist yet. + return self.first_repo() @_autospec @@ -308,6 +316,21 @@ class RepoPath(object): return self.repo_for_pkg(spec).get(spec) + def get_pkg_class(self, pkg_name): + """Find a class for the spec's package and return the class object.""" + return self.repo_for_pkg(pkg_name).get_pkg_class(pkg_name) + + + @_autospec + def dump_provenance(self, spec, path): + """Dump provenance information for a spec to a particular path. + + This dumps the package file and any associated patch files. + Raises UnknownPackageError if not found. + """ + return self.repo_for_pkg(spec).dump_provenance(spec, path) + + def dirname_for_package_name(self, pkg_name): return self.repo_for_pkg(pkg_name).dirname_for_package_name(pkg_name) @@ -527,12 +550,12 @@ class Repo(object): raise UnknownPackageError(spec.name) if spec.namespace and spec.namespace != self.namespace: - raise UnknownPackageError("Repository %s does not contain package %s." + raise UnknownPackageError("Repository %s does not contain package %s" % (self.namespace, spec.fullname)) key = hash(spec) if new or key not in self._instances: - package_class = self._get_pkg_class(spec.name) + package_class = self.get_pkg_class(spec.name) try: copy = spec.copy() # defensive copy. Package owns its spec. self._instances[key] = package_class(copy) @@ -544,6 +567,35 @@ class Repo(object): return self._instances[key] + @_autospec + def dump_provenance(self, spec, path): + """Dump provenance information for a spec to a particular path. + + This dumps the package file and any associated patch files. + Raises UnknownPackageError if not found. + """ + # Some preliminary checks. + if spec.virtual: + raise UnknownPackageError(spec.name) + + if spec.namespace and spec.namespace != self.namespace: + raise UnknownPackageError("Repository %s does not contain package %s." + % (self.namespace, spec.fullname)) + + # Install any patch files needed by packages. + mkdirp(path) + for spec, patches in spec.package.patches.items(): + for patch in patches: + if patch.path: + if os.path.exists(patch.path): + install(patch.path, path) + else: + tty.warn("Patch file did not exist: %s" % patch.path) + + # Install the package.py file itself. + install(self.filename_for_package_name(spec), path) + + def purge(self): """Clear entire package instance cache.""" self._instances.clear() @@ -668,7 +720,7 @@ class Repo(object): return self._modules[pkg_name] - def _get_pkg_class(self, pkg_name): + def get_pkg_class(self, pkg_name): """Get the class for the package out of its module. First loads (or fetches from cache) a module for the @@ -697,6 +749,58 @@ class Repo(object): return self.exists(pkg_name) +def create_repo(root, namespace=None): + """Create a new repository in root with the specified namespace. + + If the namespace is not provided, use basename of root. + Return the canonicalized path and the namespace of the created repository. + """ + root = canonicalize_path(root) + if not namespace: + namespace = os.path.basename(root) + + if not re.match(r'\w[\.\w-]*', namespace): + raise InvalidNamespaceError("'%s' is not a valid namespace." % namespace) + + existed = False + if os.path.exists(root): + if os.path.isfile(root): + raise BadRepoError('File %s already exists and is not a directory' % root) + elif os.path.isdir(root): + if not os.access(root, os.R_OK | os.W_OK): + raise BadRepoError('Cannot create new repo in %s: cannot access directory.' % root) + if os.listdir(root): + raise BadRepoError('Cannot create new repo in %s: directory is not empty.' % root) + existed = True + + full_path = os.path.realpath(root) + parent = os.path.dirname(full_path) + if not os.access(parent, os.R_OK | os.W_OK): + raise BadRepoError("Cannot create repository in %s: can't access parent!" % root) + + try: + config_path = os.path.join(root, repo_config_name) + packages_path = os.path.join(root, packages_dir_name) + + mkdirp(packages_path) + with open(config_path, 'w') as config: + config.write("repo:\n") + config.write(" namespace: '%s'\n" % namespace) + + except (IOError, OSError) as e: + raise BadRepoError('Failed to create new repository in %s.' % root, + "Caused by %s: %s" % (type(e), e)) + + # try to clean up. + if existed: + shutil.rmtree(config_path, ignore_errors=True) + shutil.rmtree(packages_path, ignore_errors=True) + else: + shutil.rmtree(root, ignore_errors=True) + + return full_path, namespace + + class RepoError(spack.error.SpackError): """Superclass for repository-related errors.""" @@ -705,6 +809,10 @@ class NoRepoConfiguredError(RepoError): """Raised when there are no repositories configured.""" +class InvalidNamespaceError(RepoError): + """Raised when an invalid namespace is encountered.""" + + class BadRepoError(RepoError): """Raised when repo layout is invalid.""" @@ -722,7 +830,7 @@ class UnknownPackageError(PackageLoadError): def __init__(self, name, repo=None): msg = None if repo: - msg = "Package %s not found in repository %s." % (name, repo) + msg = "Package %s not found in repository %s" % (name, repo) else: msg = "Package %s not found." % name super(UnknownPackageError, self).__init__(msg) diff --git a/lib/spack/spack/resource.py b/lib/spack/spack/resource.py index 2bf92947fd..ddfaaf4cb0 100644 --- a/lib/spack/spack/resource.py +++ b/lib/spack/spack/resource.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://llnl.github.io/spack +# For details, see https://software.llnl.gov/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 10e246bf2e..d04135860e 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -353,7 +353,7 @@ class VariantMap(HashableMap): @property def concrete(self): return self.spec._concrete or all( - v in self for v in self.spec.package.variants) + v in self for v in self.spec.package_class.variants) def copy(self): @@ -418,9 +418,12 @@ class Spec(object): # cases we've read them from a file want to assume normal. # This allows us to manipulate specs that Spack doesn't have # package.py files for. - self._normal = kwargs.get('normal', False) + self._normal = kwargs.get('normal', False) self._concrete = kwargs.get('concrete', False) + # Allow a spec to be constructed with an external path. + self.external = kwargs.get('external', None) + # This allows users to construct a spec DAG with literals. # Note that given two specs a and b, Spec(a) copies a, but # Spec(a, b) will copy a but just add b as a dep. @@ -498,6 +501,14 @@ class Spec(object): @property + def package_class(self): + """Internal package call gets only the class object for a package. + Use this to just get package metadata. + """ + return spack.repo.get_pkg_class(self.name) + + + @property def virtual(self): """Right now, a spec is virtual if no package exists with its name. @@ -770,12 +781,11 @@ class Spec(object): # Concretize virtual dependencies last. Because they're added # to presets below, their constraints will all be merged, but we'll # still need to select a concrete package later. - if not self.virtual: - changed |= any( - (spack.concretizer.concretize_architecture(self), - spack.concretizer.concretize_compiler(self), - spack.concretizer.concretize_version(self), - spack.concretizer.concretize_variants(self))) + changed |= any( + (spack.concretizer.concretize_architecture(self), + spack.concretizer.concretize_compiler(self), + spack.concretizer.concretize_version(self), + spack.concretizer.concretize_variants(self))) presets[self.name] = self visited.add(self.name) @@ -786,8 +796,30 @@ class Spec(object): """Replace this virtual spec with a concrete spec.""" assert(self.virtual) for name, dependent in self.dependents.items(): + # remove self from all dependents. + del dependent.dependencies[self.name] + + # add the replacement, unless it is already a dep of dependent. + if concrete.name not in dependent.dependencies: + dependent._add_dependency(concrete) + + + def _replace_node(self, replacement): + """Replace this spec with another. + + Connects all dependents of this spec to its replacement, and + disconnects this spec from any dependencies it has. New spec + will have any dependencies the replacement had, and may need + to be normalized. + + """ + for name, dependent in self.dependents.items(): del dependent.dependencies[self.name] - dependent._add_dependency(concrete) + dependent._add_dependency(replacement) + + for name, dep in self.dependencies.items(): + del dep.dependents[self.name] + del self.dependencies[dep.name] def _expand_virtual_packages(self): @@ -807,22 +839,81 @@ class Spec(object): this are infrequent, but should implement this before it is a problem. """ + # Make an index of stuff this spec already provides + self_index = ProviderIndex(self.traverse(), restrict=True) + changed = False - while True: - virtuals =[v for v in self.traverse() if v.virtual] - if not virtuals: - return changed + done = False + while not done: + done = True + for spec in list(self.traverse()): + replacement = None + if spec.virtual: + replacement = self._find_provider(spec, self_index) + if replacement: + # TODO: may break if in-place on self but + # shouldn't happen if root is traversed first. + spec._replace_with(replacement) + done=False + break + + if not replacement: + # Get a list of possible replacements in order of preference. + candidates = spack.concretizer.choose_virtual_or_external(spec) + + # Try the replacements in order, skipping any that cause + # satisfiability problems. + for replacement in candidates: + if replacement is spec: + break + + # Replace spec with the candidate and normalize + copy = self.copy() + copy[spec.name]._dup(replacement.copy(deps=False)) + + try: + # If there are duplicate providers or duplicate provider + # deps, consolidate them and merge constraints. + copy.normalize(force=True) + break + except SpecError as e: + # On error, we'll try the next replacement. + continue + + # If replacement is external then trim the dependencies + if replacement.external: + if (spec.dependencies): + changed = True + spec.dependencies = DependencyMap() + replacement.dependencies = DependencyMap() + + # TODO: could this and the stuff in _dup be cleaned up? + def feq(cfield, sfield): + return (not cfield) or (cfield == sfield) + + if replacement is spec or (feq(replacement.name, spec.name) and + feq(replacement.versions, spec.versions) and + feq(replacement.compiler, spec.compiler) and + feq(replacement.architecture, spec.architecture) and + feq(replacement.dependencies, spec.dependencies) and + feq(replacement.variants, spec.variants) and + feq(replacement.external, spec.external)): + continue + + # Refine this spec to the candidate. This uses + # replace_with AND dup so that it can work in + # place. TODO: make this more efficient. + if spec.virtual: + spec._replace_with(replacement) + changed = True + if spec._dup(replacement, deps=False, cleardeps=False): + changed = True - for spec in virtuals: - providers = spack.repo.providers_for(spec) - concrete = spack.concretizer.choose_provider(spec, providers) - concrete = concrete.copy() - spec._replace_with(concrete) - changed = True + self_index.update(spec) + done=False + break - # If there are duplicate providers or duplicate provider deps, this - # consolidates them and merge constraints. - changed |= self.normalize(force=True) + return changed def concretize(self): @@ -837,6 +928,7 @@ class Spec(object): with requirements of its pacakges. See flatten() and normalize() for more details on this. """ + if self._concrete: return @@ -844,7 +936,7 @@ class Spec(object): force = False while changed: - changes = (self.normalize(force=force), + changes = (self.normalize(force), self._expand_virtual_packages(), self._concretize_helper()) changed = any(changes) @@ -970,8 +1062,8 @@ class Spec(object): def _find_provider(self, vdep, provider_index): """Find provider for a virtual spec in the provider index. - Raise an exception if there is a conflicting virtual - dependency already in this spec. + Raise an exception if there is a conflicting virtual + dependency already in this spec. """ assert(vdep.virtual) providers = provider_index.providers_for(vdep) @@ -1012,17 +1104,14 @@ class Spec(object): """ changed = False - # If it's a virtual dependency, try to find a provider and - # merge that. + # If it's a virtual dependency, try to find an existing + # provider in the spec, and merge that. if dep.virtual: visited.add(dep.name) provider = self._find_provider(dep, provider_index) if provider: dep = provider - else: - # if it's a real dependency, check whether it provides - # something already required in the spec. index = ProviderIndex([dep], restrict=True) for vspec in (v for v in spec_deps.values() if v.virtual): if index.providers_for(vspec): @@ -1069,7 +1158,7 @@ class Spec(object): # if we descend into a virtual spec, there's nothing more # to normalize. Concretize will finish resolving it later. - if self.virtual: + if self.virtual or self.external: return False # Combine constraints from package deps with constraints from @@ -1119,13 +1208,14 @@ class Spec(object): # Get all the dependencies into one DependencyMap spec_deps = self.flat_dependencies(copy=False) - # Initialize index of virtual dependency providers - index = ProviderIndex(spec_deps.values(), restrict=True) + # Initialize index of virtual dependency providers if + # concretize didn't pass us one already + provider_index = ProviderIndex(spec_deps.values(), restrict=True) # traverse the package DAG and fill out dependencies according # to package files & their 'when' specs visited = set() - any_change = self._normalize_helper(visited, spec_deps, index) + any_change = self._normalize_helper(visited, spec_deps, provider_index) # If there are deps specified but not visited, they're not # actually deps of this package. Raise an error. @@ -1163,7 +1253,7 @@ class Spec(object): # Ensure that variants all exist. for vname, variant in spec.variants.items(): - if vname not in spec.package.variants: + if vname not in spec.package_class.variants: raise UnknownVariantError(spec.name, vname) @@ -1404,15 +1494,25 @@ class Spec(object): Whether deps should be copied too. Set to false to copy a spec but not its dependencies. """ + # We don't count dependencies as changes here + changed = True + if hasattr(self, 'name'): + changed = (self.name != other.name and self.versions != other.versions and + self.architecture != other.architecture and self.compiler != other.compiler and + self.variants != other.variants and self._normal != other._normal and + self.concrete != other.concrete and self.external != other.external) + # Local node attributes get copied first. self.name = other.name self.versions = other.versions.copy() self.architecture = other.architecture self.compiler = other.compiler.copy() if other.compiler else None - self.dependents = DependencyMap() - self.dependencies = DependencyMap() + if kwargs.get('cleardeps', True): + self.dependents = DependencyMap() + self.dependencies = DependencyMap() self.variants = other.variants.copy() self.variants.spec = self + self.external = other.external self.namespace = other.namespace # If we copy dependencies, preserve DAG structure in the new spec @@ -1431,6 +1531,8 @@ class Spec(object): # Since we preserved structure, we can copy _normal safely. self._normal = other._normal self._concrete = other._concrete + self.external = other.external + return changed def copy(self, **kwargs): @@ -1571,14 +1673,28 @@ class Spec(object): $_ Package name $. Full package name (with namespace) - $@ Version - $% Compiler - $%@ Compiler & compiler version + $@ Version with '@' prefix + $% Compiler with '%' prefix + $%@ Compiler with '%' prefix & compiler version with '@' prefix $+ Options - $= Architecture - $# 7-char prefix of DAG hash + $= Architecture with '=' prefix + $# 7-char prefix of DAG hash with '-' prefix $$ $ + You can also use full-string versions, which leave off the prefixes: + + ${PACKAGE} Package name + ${VERSION} Version + ${COMPILER} Full compiler string + ${COMPILERNAME} Compiler name + ${COMPILERVER} Compiler version + ${OPTIONS} Options + ${ARCHITECTURE} Architecture + ${SHA1} Dependencies 8-char sha1 prefix + + ${SPACK_ROOT} The spack root directory + ${SPACK_INSTALL} The default spack install directory, ${SPACK_PREFIX}/opt + Optionally you can provide a width, e.g. $20_ for a 20-wide name. Like printf, you can provide '-' for left justification, e.g. $-20_ for a left-justified name. @@ -1594,7 +1710,8 @@ class Spec(object): color = kwargs.get('color', False) length = len(format_string) out = StringIO() - escape = compiler = False + named = escape = compiler = False + named_str = fmt = '' def write(s, c): if color: @@ -1636,9 +1753,12 @@ class Spec(object): elif c == '#': out.write('-' + fmt % (self.dag_hash(7))) elif c == '$': - if fmt != '': + if fmt != '%s': raise ValueError("Can't use format width with $$.") out.write('$') + elif c == '{': + named = True + named_str = '' escape = False elif compiler: @@ -1652,6 +1772,43 @@ class Spec(object): out.write(c) compiler = False + elif named: + if not c == '}': + if i == length - 1: + raise ValueError("Error: unterminated ${ in format: '%s'" + % format_string) + named_str += c + continue; + if named_str == 'PACKAGE': + write(fmt % self.name, '@') + if named_str == 'VERSION': + if self.versions and self.versions != _any_version: + write(fmt % str(self.versions), '@') + elif named_str == 'COMPILER': + if self.compiler: + write(fmt % self.compiler, '%') + elif named_str == 'COMPILERNAME': + if self.compiler: + write(fmt % self.compiler.name, '%') + elif named_str == 'COMPILERVER': + if self.compiler: + write(fmt % self.compiler.versions, '%') + elif named_str == 'OPTIONS': + if self.variants: + write(fmt % str(self.variants), '+') + elif named_str == 'ARCHITECTURE': + if self.architecture: + write(fmt % str(self.architecture), '=') + elif named_str == 'SHA1': + if self.dependencies: + out.write(fmt % str(self.dag_hash(7))) + elif named_str == 'SPACK_ROOT': + out.write(fmt % spack.prefix) + elif named_str == 'SPACK_INSTALL': + out.write(fmt % spack.install_path) + + named = False + elif c == '$': escape = True if i == length - 1: @@ -1782,6 +1939,7 @@ class SpecParser(spack.parse.Parser): spec.variants = VariantMap(spec) spec.architecture = None spec.compiler = None + spec.external = None spec.dependents = DependencyMap() spec.dependencies = DependencyMap() spec.namespace = spec_namespace diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index 31635a854a..f88f82fc2d 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -23,7 +23,7 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import os -import re +import errno import shutil import tempfile from urlparse import urljoin @@ -31,46 +31,65 @@ from urlparse import urljoin import llnl.util.tty as tty from llnl.util.filesystem import * +import spack.util.pattern as pattern + import spack import spack.config import spack.fetch_strategy as fs import spack.error - STAGE_PREFIX = 'spack-stage-' class Stage(object): - """A Stage object manaages a directory where some source code is - downloaded and built before being installed. It handles - fetching the source code, either as an archive to be expanded - or by checking it out of a repository. A stage's lifecycle - looks like this: - - Stage() - Constructor creates the stage directory. - fetch() - Fetch a source archive into the stage. - expand_archive() - Expand the source archive. - <install> - Build and install the archive. This is handled by the Package class. - destroy() - Remove the stage once the package has been installed. - - If spack.use_tmp_stage is True, spack will attempt to create stages - in a tmp directory. Otherwise, stages are created directly in - spack.stage_path. - - There are two kinds of stages: named and unnamed. Named stages can - persist between runs of spack, e.g. if you fetched a tarball but - didn't finish building it, you won't have to fetch it again. - - Unnamed stages are created using standard mkdtemp mechanisms or - similar, and are intended to persist for only one run of spack. + """Manages a temporary stage directory for building. + + A Stage object is a context manager that handles a directory where + some source code is downloaded and built before being installed. + It handles fetching the source code, either as an archive to be + expanded or by checking it out of a repository. A stage's + lifecycle looks like this: + + ``` + with Stage() as stage: # Context manager creates and destroys the stage directory + stage.fetch() # Fetch a source archive into the stage. + stage.expand_archive() # Expand the source archive. + <install> # Build and install the archive. (handled by user of Stage) + ``` + + When used as a context manager, the stage is automatically + destroyed if no exception is raised by the context. If an + excpetion is raised, the stage is left in the filesystem and NOT + destroyed, for potential reuse later. + + You can also use the stage's create/destroy functions manually, + like this: + + ``` + stage = Stage() + try: + stage.create() # Explicitly create the stage directory. + stage.fetch() # Fetch a source archive into the stage. + stage.expand_archive() # Expand the source archive. + <install> # Build and install the archive. (handled by user of Stage) + finally: + stage.destroy() # Explicitly destroy the stage directory. + ``` + + If spack.use_tmp_stage is True, spack will attempt to create + stages in a tmp directory. Otherwise, stages are created directly + in spack.stage_path. + + There are two kinds of stages: named and unnamed. Named stages + can persist between runs of spack, e.g. if you fetched a tarball + but didn't finish building it, you won't have to fetch it again. + + Unnamed stages are created using standard mkdtemp mechanisms or + similar, and are intended to persist for only one run of spack. """ - def __init__(self, url_or_fetch_strategy, **kwargs): + def __init__(self, url_or_fetch_strategy, + name=None, mirror_path=None, keep=False): """Create a stage object. Parameters: url_or_fetch_strategy @@ -82,6 +101,17 @@ class Stage(object): and will persist between runs (or if you construct another stage object later). If name is not provided, then this stage will be given a unique name automatically. + + mirror_path + If provided, Stage will search Spack's mirrors for + this archive at the mirror_path, before using the + default fetch strategy. + + keep + By default, when used as a context manager, the Stage + is deleted on exit when no exceptions are raised. + Pass True to keep the stage intact even if no + exceptions are raised. """ # TODO: fetch/stage coupling needs to be reworked -- the logic # TODO: here is convoluted and not modular enough. @@ -93,25 +123,52 @@ class Stage(object): raise ValueError("Can't construct Stage without url or fetch strategy") self.fetcher.set_stage(self) self.default_fetcher = self.fetcher # self.fetcher can change with mirrors. - self.skip_checksum_for_mirror = True # used for mirrored archives of repositories. + self.skip_checksum_for_mirror = True # used for mirrored archives of repositories. + + # TODO : this uses a protected member of tempfile, but seemed the only way to get a temporary name + # TODO : besides, the temporary link name won't be the same as the temporary stage area in tmp_root + self.name = name + if name is None: + self.name = STAGE_PREFIX + next(tempfile._get_candidate_names()) + self.mirror_path = mirror_path + self.tmp_root = find_tmp_root() - self.name = kwargs.get('name') - self.mirror_path = kwargs.get('mirror_path') + # Try to construct here a temporary name for the stage directory + # If this is a named stage, then construct a named path. + self.path = join_path(spack.stage_path, self.name) - self.tmp_root = find_tmp_root() + # Flag to decide whether to delete the stage folder on exit or not + self.keep = keep - self.path = None - self._setup() + def __enter__(self): + """ + Entering a stage context will create the stage directory - def _cleanup_dead_links(self): - """Remove any dead links in the stage directory.""" - for file in os.listdir(spack.stage_path): - path = join_path(spack.stage_path, file) - if os.path.islink(path): - real_path = os.path.realpath(path) - if not os.path.exists(path): - os.unlink(path) + Returns: + self + """ + self.create() + return self + + + def __exit__(self, exc_type, exc_val, exc_tb): + """ + Exiting from a stage context will delete the stage directory unless: + - it was explicitly requested not to do so + - an exception has been raised + + Args: + exc_type: exception type + exc_val: exception value + exc_tb: exception traceback + + Returns: + Boolean + """ + # Delete when there are no exceptions, unless asked to keep. + if exc_type is None and not self.keep: + self.destroy() def _need_to_create_path(self): @@ -131,11 +188,11 @@ class Stage(object): # Path looks ok, but need to check the target of the link. if os.path.islink(self.path): real_path = os.path.realpath(self.path) - real_tmp = os.path.realpath(self.tmp_root) + real_tmp = os.path.realpath(self.tmp_root) if spack.use_tmp_stage: - # If we're using a tmp dir, it's a link, and it points at the right spot, - # then keep it. + # If we're using a tmp dir, it's a link, and it points at the + # right spot, then keep it. if (real_path.startswith(real_tmp) and os.path.exists(real_path)): return False else: @@ -150,56 +207,6 @@ class Stage(object): return False - - def _setup(self): - """Creates the stage directory. - If spack.use_tmp_stage is False, the stage directory is created - directly under spack.stage_path. - - If spack.use_tmp_stage is True, this will attempt to create a - stage in a temporary directory and link it into spack.stage_path. - Spack will use the first writable location in spack.tmp_dirs to - create a stage. If there is no valid location in tmp_dirs, fall - back to making the stage inside spack.stage_path. - """ - # Create the top-level stage directory - mkdirp(spack.stage_path) - self._cleanup_dead_links() - - # If this is a named stage, then construct a named path. - if self.name is not None: - self.path = join_path(spack.stage_path, self.name) - - # If this is a temporary stage, them make the temp directory - tmp_dir = None - if self.tmp_root: - if self.name is None: - # Unnamed tmp root. Link the path in - tmp_dir = tempfile.mkdtemp('', STAGE_PREFIX, self.tmp_root) - self.name = os.path.basename(tmp_dir) - self.path = join_path(spack.stage_path, self.name) - if self._need_to_create_path(): - os.symlink(tmp_dir, self.path) - - else: - if self._need_to_create_path(): - tmp_dir = tempfile.mkdtemp('', STAGE_PREFIX, self.tmp_root) - os.symlink(tmp_dir, self.path) - - # if we're not using a tmp dir, create the stage directly in the - # stage dir, rather than linking to it. - else: - if self.name is None: - self.path = tempfile.mkdtemp('', STAGE_PREFIX, spack.stage_path) - self.name = os.path.basename(self.path) - else: - if self._need_to_create_path(): - mkdirp(self.path) - - # Make sure we can actually do something with the stage we made. - ensure_access(self.path) - - @property def archive_file(self): """Path to the source archive within this stage directory.""" @@ -216,35 +223,43 @@ class Stage(object): else: return None - @property def source_path(self): - """Returns the path to the expanded/checked out source code - within this fetch strategy's path. + """Returns the path to the expanded/checked out source code. + + To find the source code, this method searches for the first + subdirectory of the stage that it can find, and returns it. + This assumes nothing besides the archive file will be in the + stage path, but it has the advantage that we don't need to + know the name of the archive or its contents. - This assumes nothing else is going ot be put in the - FetchStrategy's path. It searches for the first - subdirectory of the path it can find, then returns that. + If the fetch strategy is not supposed to expand the downloaded + file, it will just return the stage path. If the archive needs + to be expanded, it will return None when no archive is found. """ + if isinstance(self.fetcher, fs.URLFetchStrategy): + if not self.fetcher.expand_archive: + return self.path + for p in [os.path.join(self.path, f) for f in os.listdir(self.path)]: if os.path.isdir(p): return p return None - def chdir(self): """Changes directory to the stage path. Or dies if it is not set up.""" if os.path.isdir(self.path): os.chdir(self.path) else: - tty.die("Setup failed: no such directory: " + self.path) - + raise ChdirError("Setup failed: no such directory: " + self.path) - def fetch(self): + def fetch(self, mirror_only=False): """Downloads an archive or checks out code from a repository.""" self.chdir() - fetchers = [self.default_fetcher] + fetchers = [] + if not mirror_only: + fetchers.append(self.default_fetcher) # TODO: move mirror logic out of here and clean it up! # TODO: Or @alalazo may have some ideas about how to use a @@ -252,7 +267,13 @@ class Stage(object): self.skip_checksum_for_mirror = True if self.mirror_path: mirrors = spack.config.get_config('mirrors') - urls = [urljoin(u, self.mirror_path) for name, u in mirrors.items()] + + # Join URLs of mirror roots with mirror paths. Because + # urljoin() will strip everything past the final '/' in + # the root, so we add a '/' if it is not present. + mirror_roots = [root if root.endswith('/') else root + '/' + for root in mirrors.values()] + urls = [urljoin(root, self.mirror_path) for root in mirror_roots] # If this archive is normally fetched from a tarball URL, # then use the same digest. `spack mirror` ensures that @@ -261,10 +282,11 @@ class Stage(object): if isinstance(self.default_fetcher, fs.URLFetchStrategy): digest = self.default_fetcher.digest - # Have to skip the checkesum for things archived from + # Have to skip the checksum for things archived from # repositories. How can this be made safer? self.skip_checksum_for_mirror = not bool(digest) + # Add URL strategies for all the mirrors with the digest for url in urls: fetchers.insert(0, fs.URLFetchStrategy(url, digest)) @@ -274,7 +296,7 @@ class Stage(object): self.fetcher = fetcher self.fetcher.fetch() break - except spack.error.SpackError, e: + except spack.error.SpackError as e: tty.msg("Fetching from %s failed." % fetcher) tty.debug(e) continue @@ -283,7 +305,6 @@ class Stage(object): self.fetcher = self.default_fetcher raise fs.FetchError(errMessage, None) - def check(self): """Check the downloaded archive against a checksum digest. No-op if this stage checks code out of a repository.""" @@ -297,14 +318,17 @@ class Stage(object): else: self.fetcher.check() - def expand_archive(self): """Changes to the stage directory and attempt to expand the downloaded archive. Fail if the stage is not set up or if the archive is not yet downloaded. """ - self.fetcher.expand() - + archive_dir = self.source_path + if not archive_dir: + self.fetcher.expand() + tty.msg("Created stage in %s" % self.path) + else: + tty.msg("Already staged %s in %s" % (self.name, self.path)) def chdir_to_source(self): """Changes directory to the expanded archive directory. @@ -318,16 +342,41 @@ class Stage(object): if not os.listdir(path): tty.die("Archive was empty for %s" % self.name) - def restage(self): """Removes the expanded archive path if it exists, then re-expands the archive. """ self.fetcher.reset() + def create(self): + """ + Creates the stage directory + + If self.tmp_root evaluates to False, the stage directory is + created directly under spack.stage_path, otherwise this will + attempt to create a stage in a temporary directory and link it + into spack.stage_path. + + Spack will use the first writable location in spack.tmp_dirs + to create a stage. If there is no valid location in tmp_dirs, + fall back to making the stage inside spack.stage_path. + """ + # Create the top-level stage directory + mkdirp(spack.stage_path) + remove_dead_links(spack.stage_path) + # If a tmp_root exists then create a directory there and then link it in the stage area, + # otherwise create the stage directory in self.path + if self._need_to_create_path(): + if self.tmp_root: + tmp_dir = tempfile.mkdtemp('', STAGE_PREFIX, self.tmp_root) + os.symlink(tmp_dir, self.path) + else: + mkdirp(self.path) + # Make sure we can actually do something with the stage we made. + ensure_access(self.path) def destroy(self): - """Remove this stage directory.""" + """Removes this stage directory.""" remove_linked_tree(self.path) # Make sure we don't end up in a removed directory @@ -337,8 +386,82 @@ class Stage(object): os.chdir(os.path.dirname(self.path)) +class ResourceStage(Stage): + def __init__(self, url_or_fetch_strategy, root, resource, **kwargs): + super(ResourceStage, self).__init__(url_or_fetch_strategy, **kwargs) + self.root_stage = root + self.resource = resource + + def expand_archive(self): + super(ResourceStage, self).expand_archive() + root_stage = self.root_stage + resource = self.resource + placement = os.path.basename(self.source_path) if resource.placement is None else resource.placement + if not isinstance(placement, dict): + placement = {'': placement} + # Make the paths in the dictionary absolute and link + for key, value in placement.iteritems(): + target_path = join_path(root_stage.source_path, resource.destination) + destination_path = join_path(target_path, value) + source_path = join_path(self.source_path, key) + + try: + os.makedirs(target_path) + except OSError as err: + if err.errno == errno.EEXIST and os.path.isdir(target_path): + pass + else: + raise + + if not os.path.exists(destination_path): + # Create a symlink + tty.info('Moving resource stage\n\tsource : {stage}\n\tdestination : {destination}'.format( + stage=source_path, destination=destination_path + )) + shutil.move(source_path, destination_path) + + +@pattern.composite(method_list=['fetch', 'create', 'check', 'expand_archive', 'restage', 'destroy']) +class StageComposite: + """ + Composite for Stage type objects. The first item in this composite is considered to be the root package, and + operations that return a value are forwarded to it. + """ + # + # __enter__ and __exit__ delegate to all stages in the composite. + # + def __enter__(self): + for item in self: + item.__enter__() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + for item in reversed(self): + item.keep = getattr(self, 'keep', False) + item.__exit__(exc_type, exc_val, exc_tb) + + # + # Below functions act only on the *first* stage in the composite. + # + @property + def source_path(self): + return self[0].source_path + + @property + def path(self): + return self[0].path + + def chdir_to_source(self): + return self[0].chdir_to_source() + + @property + def archive_file(self): + return self[0].archive_file + + class DIYStage(object): """Simple class that allows any directory to be a spack stage.""" + def __init__(self, path): self.archive_file = None self.path = path @@ -348,12 +471,16 @@ class DIYStage(object): if os.path.isdir(self.path): os.chdir(self.path) else: - tty.die("Setup failed: no such directory: " + self.path) + raise ChdirError("Setup failed: no such directory: " + self.path) + + # DIY stages do nothing as context managers. + def __enter__(self): pass + def __exit__(self, exc_type, exc_val, exc_tb): pass def chdir_to_source(self): self.chdir() - def fetch(self): + def fetch(self, mirror_only): tty.msg("No need to fetch for DIY.") def check(self): @@ -376,26 +503,12 @@ def _get_mirrors(): return [val for name, val in config.iteritems()] - def ensure_access(file=spack.stage_path): """Ensure we can access a directory and die with an error if we can't.""" if not can_access(file): tty.die("Insufficient permissions for %s" % file) -def remove_linked_tree(path): - """Removes a directory and its contents. If the directory is a symlink, - follows the link and reamoves the real directory before removing the - link. - """ - if os.path.exists(path): - if os.path.islink(path): - shutil.rmtree(os.path.realpath(path), True) - os.unlink(path) - else: - shutil.rmtree(path, True) - - def purge(): """Remove all build directories in the top-level stage path.""" if os.path.isdir(spack.stage_path): @@ -424,19 +537,15 @@ def find_tmp_root(): class StageError(spack.error.SpackError): - def __init__(self, message, long_message=None): - super(self, StageError).__init__(message, long_message) + """"Superclass for all errors encountered during staging.""" class RestageError(StageError): - def __init__(self, message, long_msg=None): - super(RestageError, self).__init__(message, long_msg) + """"Error encountered during restaging.""" class ChdirError(StageError): - def __init__(self, message, long_msg=None): - super(ChdirError, self).__init__(message, long_msg) - + """Raised when Spack can't change directories.""" # Keep this in namespace for convenience FailedDownloadError = fs.FailedDownloadError diff --git a/lib/spack/spack/test/__init__.py b/lib/spack/spack/test/__init__.py index a569cbbf35..cd842561e6 100644 --- a/lib/spack/spack/test/__init__.py +++ b/lib/spack/spack/test/__init__.py @@ -48,6 +48,7 @@ test_names = ['versions', 'package_sanity', 'config', 'directory_layout', + 'pattern', 'python_version', 'git_fetch', 'svn_fetch', @@ -64,7 +65,9 @@ test_names = ['versions', 'lock', 'database', 'namespace_trie', - 'yaml'] + 'yaml', + 'sbang', + 'environment'] def list_tests(): @@ -86,20 +89,20 @@ def run(names, outputDir, verbose=False): "Valid names are:") colify(sorted(test_names), indent=4) sys.exit(1) - + tally = Tally() for test in names: module = 'spack.test.' + test print module - + tty.msg("Running test: %s" % test) - + runOpts = ["--with-%s" % spack.test.tally_plugin.Tally.name] - + if outputDir: xmlOutputFname = "unittests-{0}.xml".format(test) xmlOutputPath = join_path(outputDir, xmlOutputFname) - runOpts += ["--with-xunit", + runOpts += ["--with-xunit", "--xunit-file={0}".format(xmlOutputPath)] argv = [""] + runOpts + [module] result = nose.run(argv=argv, addplugins=[tally]) diff --git a/lib/spack/spack/test/cc.py b/lib/spack/spack/test/cc.py index 4188b8d550..f3f6d4a22e 100644 --- a/lib/spack/spack/test/cc.py +++ b/lib/spack/spack/test/cc.py @@ -39,11 +39,11 @@ test_command = [ 'arg1', '-Wl,--start-group', 'arg2', - '-Wl,-rpath=/first/rpath', 'arg3', '-Wl,-rpath', '-Wl,/second/rpath', + '-Wl,-rpath,/first/rpath', 'arg3', '-Wl,-rpath', '-Wl,/second/rpath', '-llib1', '-llib2', 'arg4', '-Wl,--end-group', - '-Xlinker,-rpath', '-Xlinker,/third/rpath', '-Xlinker,-rpath=/fourth/rpath', + '-Xlinker', '-rpath', '-Xlinker', '/third/rpath', '-Xlinker', '-rpath', '-Xlinker', '/fourth/rpath', '-llib3', '-llib4', 'arg5', 'arg6'] @@ -65,17 +65,17 @@ class CompilerTest(unittest.TestCase): def check_cc(self, command, args, expected): os.environ['SPACK_TEST_COMMAND'] = command - self.assertEqual(self.cc(*args, return_output=True).strip(), expected) + self.assertEqual(self.cc(*args, output=str).strip(), expected) def check_ld(self, command, args, expected): os.environ['SPACK_TEST_COMMAND'] = command - self.assertEqual(self.ld(*args, return_output=True).strip(), expected) + self.assertEqual(self.ld(*args, output=str).strip(), expected) def check_cpp(self, command, args, expected): os.environ['SPACK_TEST_COMMAND'] = command - self.assertEqual(self.cpp(*args, return_output=True).strip(), expected) + self.assertEqual(self.cpp(*args, output=str).strip(), expected) def test_vcheck_mode(self): @@ -95,13 +95,13 @@ class CompilerTest(unittest.TestCase): def test_ccld_mode(self): self.check_cc('dump-mode', [], "ccld") self.check_cc('dump-mode', ['foo.c', '-o', 'foo'], "ccld") - self.check_cc('dump-mode', ['foo.c', '-o', 'foo', '-Wl,-rpath=foo'], "ccld") - self.check_cc('dump-mode', ['foo.o', 'bar.o', 'baz.o', '-o', 'foo', '-Wl,-rpath=foo'], "ccld") + self.check_cc('dump-mode', ['foo.c', '-o', 'foo', '-Wl,-rpath,foo'], "ccld") + self.check_cc('dump-mode', ['foo.o', 'bar.o', 'baz.o', '-o', 'foo', '-Wl,-rpath,foo'], "ccld") def test_ld_mode(self): self.check_ld('dump-mode', [], "ld") - self.check_ld('dump-mode', ['foo.o', 'bar.o', 'baz.o', '-o', 'foo', '-Wl,-rpath=foo'], "ld") + self.check_ld('dump-mode', ['foo.o', 'bar.o', 'baz.o', '-o', 'foo', '-Wl,-rpath,foo'], "ld") def test_includes(self): diff --git a/lib/spack/spack/test/concretize.py b/lib/spack/spack/test/concretize.py index 7f2938aec5..08cce09674 100644 --- a/lib/spack/spack/test/concretize.py +++ b/lib/spack/spack/test/concretize.py @@ -22,10 +22,9 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import unittest - import spack from spack.spec import Spec, CompilerSpec +from spack.concretize import find_spec from spack.test.mock_packages_test import * class ConcretizeTest(MockPackagesTest): @@ -143,6 +142,34 @@ class ConcretizeTest(MockPackagesTest): for spec in spack.repo.providers_for('mpi@3'))) + def test_concretize_two_virtuals(self): + """Test a package with multiple virtual dependencies.""" + s = Spec('hypre').concretize() + + + def test_concretize_two_virtuals_with_one_bound(self): + """Test a package with multiple virtual dependencies and one preset.""" + s = Spec('hypre ^openblas').concretize() + + + def test_concretize_two_virtuals_with_two_bound(self): + """Test a package with multiple virtual dependencies and two of them preset.""" + s = Spec('hypre ^openblas ^netlib-lapack').concretize() + + + def test_concretize_two_virtuals_with_dual_provider(self): + """Test a package with multiple virtual dependencies and force a provider + that provides both.""" + s = Spec('hypre ^openblas-with-lapack').concretize() + + + def test_concretize_two_virtuals_with_dual_provider_and_a_conflict(self): + """Test a package with multiple virtual dependencies and force a provider + that provides both, and another conflicting package that provides one.""" + s = Spec('hypre ^openblas-with-lapack ^netlib-lapack') + self.assertRaises(spack.spec.MultipleProviderError, s.concretize) + + def test_virtual_is_fully_expanded_for_callpath(self): # force dependence on fake "zmpi" by asking for MPI 10.0 spec = Spec('callpath ^mpi@10.0') @@ -192,3 +219,100 @@ class ConcretizeTest(MockPackagesTest): # TODO: not exactly the syntax I would like. self.assertTrue(spec['libdwarf'].compiler.satisfies('clang')) self.assertTrue(spec['libelf'].compiler.satisfies('clang')) + + + def test_external_package(self): + spec = Spec('externaltool') + spec.concretize() + + self.assertEqual(spec['externaltool'].external, '/path/to/external_tool') + self.assertFalse('externalprereq' in spec) + self.assertTrue(spec['externaltool'].compiler.satisfies('gcc')) + + + def test_nobuild_package(self): + got_error = False + spec = Spec('externaltool%clang') + try: + spec.concretize() + except spack.concretize.NoBuildError: + got_error = True + self.assertTrue(got_error) + + + def test_external_and_virtual(self): + spec = Spec('externaltest') + spec.concretize() + self.assertEqual(spec['externaltool'].external, '/path/to/external_tool') + self.assertEqual(spec['stuff'].external, '/path/to/external_virtual_gcc') + self.assertTrue(spec['externaltool'].compiler.satisfies('gcc')) + self.assertTrue(spec['stuff'].compiler.satisfies('gcc')) + + + def test_find_spec_parents(self): + """Tests the spec finding logic used by concretization. """ + s = Spec('a +foo', + Spec('b +foo', + Spec('c'), + Spec('d +foo')), + Spec('e +foo')) + + self.assertEqual('a', find_spec(s['b'], lambda s: '+foo' in s).name) + + + def test_find_spec_children(self): + s = Spec('a', + Spec('b +foo', + Spec('c'), + Spec('d +foo')), + Spec('e +foo')) + self.assertEqual('d', find_spec(s['b'], lambda s: '+foo' in s).name) + s = Spec('a', + Spec('b +foo', + Spec('c +foo'), + Spec('d')), + Spec('e +foo')) + self.assertEqual('c', find_spec(s['b'], lambda s: '+foo' in s).name) + + + def test_find_spec_sibling(self): + s = Spec('a', + Spec('b +foo', + Spec('c'), + Spec('d')), + Spec('e +foo')) + self.assertEqual('e', find_spec(s['b'], lambda s: '+foo' in s).name) + self.assertEqual('b', find_spec(s['e'], lambda s: '+foo' in s).name) + + s = Spec('a', + Spec('b +foo', + Spec('c'), + Spec('d')), + Spec('e', + Spec('f +foo'))) + self.assertEqual('f', find_spec(s['b'], lambda s: '+foo' in s).name) + + + def test_find_spec_self(self): + s = Spec('a', + Spec('b +foo', + Spec('c'), + Spec('d')), + Spec('e')) + self.assertEqual('b', find_spec(s['b'], lambda s: '+foo' in s).name) + + + def test_find_spec_none(self): + s = Spec('a', + Spec('b', + Spec('c'), + Spec('d')), + Spec('e')) + self.assertEqual(None, find_spec(s['b'], lambda s: '+foo' in s)) + + + def test_compiler_child(self): + s = Spec('mpileaks%clang ^dyninst%gcc') + s.concretize() + self.assertTrue(s['mpileaks'].satisfies('%clang')) + self.assertTrue(s['dyninst'].satisfies('%gcc')) diff --git a/lib/spack/spack/test/config.py b/lib/spack/spack/test/config.py index d8be5a855b..0562d2d620 100644 --- a/lib/spack/spack/test/config.py +++ b/lib/spack/spack/test/config.py @@ -22,13 +22,13 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import unittest -import shutil import os +import shutil from tempfile import mkdtemp -from ordereddict_backport import OrderedDict + import spack import spack.config +from ordereddict_backport import OrderedDict from spack.test.mock_packages_test import * # Some sample compiler config data diff --git a/lib/spack/spack/test/configure_guess.py b/lib/spack/spack/test/configure_guess.py index a4e8565b62..2440d120e5 100644 --- a/lib/spack/spack/test/configure_guess.py +++ b/lib/spack/spack/test/configure_guess.py @@ -23,20 +23,15 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import os -import unittest import shutil import tempfile +import unittest from llnl.util.filesystem import * - from spack.cmd.create import ConfigureGuesser from spack.stage import Stage - -from spack.fetch_strategy import URLFetchStrategy -from spack.directory_layout import YamlDirectoryLayout -from spack.util.executable import which from spack.test.mock_packages_test import * -from spack.test.mock_repo import MockArchive +from spack.util.executable import which class InstallTest(unittest.TestCase): @@ -52,8 +47,6 @@ class InstallTest(unittest.TestCase): def tearDown(self): shutil.rmtree(self.tmpdir, ignore_errors=True) - if self.stage: - self.stage.destroy() os.chdir(self.orig_dir) @@ -64,12 +57,12 @@ class InstallTest(unittest.TestCase): url = 'file://' + join_path(os.getcwd(), 'archive.tar.gz') print url - self.stage = Stage(url) - self.stage.fetch() + with Stage(url) as stage: + stage.fetch() - guesser = ConfigureGuesser() - guesser(self.stage) - self.assertEqual(system, guesser.build_system) + guesser = ConfigureGuesser() + guesser(stage) + self.assertEqual(system, guesser.build_system) def test_python(self): diff --git a/lib/spack/spack/test/database.py b/lib/spack/spack/test/database.py index 0205f4b8ce..ce6e8a0552 100644 --- a/lib/spack/spack/test/database.py +++ b/lib/spack/spack/test/database.py @@ -26,19 +26,19 @@ These tests check the database is functioning properly, both in memory and in its file """ -import tempfile -import shutil +import os.path import multiprocessing - -from llnl.util.lock import * -from llnl.util.filesystem import join_path +import shutil +import tempfile import spack +from llnl.util.filesystem import join_path +from llnl.util.lock import * +from llnl.util.tty.colify import colify from spack.database import Database from spack.directory_layout import YamlDirectoryLayout from spack.test.mock_packages_test import * -from llnl.util.tty.colify import colify def _print_ref_counts(): """Print out all ref counts for the graph used here, for debugging""" diff --git a/lib/spack/spack/test/directory_layout.py b/lib/spack/spack/test/directory_layout.py index 925cb648ed..8ad8f1a360 100644 --- a/lib/spack/spack/test/directory_layout.py +++ b/lib/spack/spack/test/directory_layout.py @@ -25,20 +25,17 @@ """\ This test verifies that the Spack directory layout works properly. """ -import unittest -import tempfile -import shutil import os - -from llnl.util.filesystem import * +import shutil +import tempfile import spack -from spack.spec import Spec -from spack.repository import RepoPath +from llnl.util.filesystem import * from spack.directory_layout import YamlDirectoryLayout +from spack.repository import RepoPath +from spack.spec import Spec from spack.test.mock_packages_test import * - # number of packages to test (to reduce test time) max_packages = 10 @@ -69,6 +66,9 @@ class DirectoryLayoutTest(MockPackagesTest): packages = list(spack.repo.all_packages())[:max_packages] for pkg in packages: + if pkg.name.startswith('external'): + #External package tests cannot be installed + continue spec = pkg.spec # If a spec fails to concretize, just skip it. If it is a @@ -174,6 +174,9 @@ class DirectoryLayoutTest(MockPackagesTest): # Create install prefixes for all packages in the list installed_specs = {} for pkg in packages: + if pkg.name.startswith('external'): + #External package tests cannot be installed + continue spec = pkg.spec.concretized() installed_specs[spec.name] = spec self.layout.create_install_directory(spec) diff --git a/lib/spack/spack/test/environment.py b/lib/spack/spack/test/environment.py new file mode 100644 index 0000000000..6c8f5ea43c --- /dev/null +++ b/lib/spack/spack/test/environment.py @@ -0,0 +1,73 @@ +import unittest +import os +from spack.environment import EnvironmentModifications + + +class EnvironmentTest(unittest.TestCase): + def setUp(self): + os.environ.clear() + os.environ['UNSET_ME'] = 'foo' + os.environ['EMPTY_PATH_LIST'] = '' + os.environ['PATH_LIST'] = '/path/second:/path/third' + os.environ['REMOVE_PATH_LIST'] = '/a/b:/duplicate:/a/c:/remove/this:/a/d:/duplicate/:/f/g' + + def test_set(self): + env = EnvironmentModifications() + env.set('A', 'dummy value') + env.set('B', 3) + env.apply_modifications() + self.assertEqual('dummy value', os.environ['A']) + self.assertEqual(str(3), os.environ['B']) + + def test_unset(self): + env = EnvironmentModifications() + self.assertEqual('foo', os.environ['UNSET_ME']) + env.unset('UNSET_ME') + env.apply_modifications() + self.assertRaises(KeyError, os.environ.__getitem__, 'UNSET_ME') + + def test_set_path(self): + env = EnvironmentModifications() + env.set_path('A', ['foo', 'bar', 'baz']) + env.apply_modifications() + self.assertEqual('foo:bar:baz', os.environ['A']) + + def test_path_manipulation(self): + env = EnvironmentModifications() + + env.append_path('PATH_LIST', '/path/last') + env.prepend_path('PATH_LIST', '/path/first') + + env.append_path('EMPTY_PATH_LIST', '/path/middle') + env.append_path('EMPTY_PATH_LIST', '/path/last') + env.prepend_path('EMPTY_PATH_LIST', '/path/first') + + env.append_path('NEWLY_CREATED_PATH_LIST', '/path/middle') + env.append_path('NEWLY_CREATED_PATH_LIST', '/path/last') + env.prepend_path('NEWLY_CREATED_PATH_LIST', '/path/first') + + env.remove_path('REMOVE_PATH_LIST', '/remove/this') + env.remove_path('REMOVE_PATH_LIST', '/duplicate/') + + env.apply_modifications() + self.assertEqual('/path/first:/path/second:/path/third:/path/last', os.environ['PATH_LIST']) + self.assertEqual('/path/first:/path/middle:/path/last', os.environ['EMPTY_PATH_LIST']) + self.assertEqual('/path/first:/path/middle:/path/last', os.environ['NEWLY_CREATED_PATH_LIST']) + self.assertEqual('/a/b:/a/c:/a/d:/f/g', os.environ['REMOVE_PATH_LIST']) + + def test_extra_arguments(self): + env = EnvironmentModifications() + env.set('A', 'dummy value', who='Pkg1') + for x in env: + assert 'who' in x.args + env.apply_modifications() + self.assertEqual('dummy value', os.environ['A']) + + def test_extend(self): + env = EnvironmentModifications() + env.set('A', 'dummy value') + env.set('B', 3) + copy_construct = EnvironmentModifications(env) + self.assertEqual(len(copy_construct), 2) + for x, y in zip(env, copy_construct): + assert x is y diff --git a/lib/spack/spack/test/git_fetch.py b/lib/spack/spack/test/git_fetch.py index 3813079065..3578044116 100644 --- a/lib/spack/spack/test/git_fetch.py +++ b/lib/spack/spack/test/git_fetch.py @@ -23,19 +23,12 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import os -import unittest -import shutil -import tempfile - -from llnl.util.filesystem import * import spack -from spack.version import ver -from spack.stage import Stage -from spack.util.executable import which - +from llnl.util.filesystem import * from spack.test.mock_packages_test import * from spack.test.mock_repo import MockGitRepo +from spack.version import ver class GitFetchTest(MockPackagesTest): @@ -52,22 +45,15 @@ class GitFetchTest(MockPackagesTest): spec.concretize() self.pkg = spack.repo.get(spec, new=True) - def tearDown(self): """Destroy the stage space used by this test.""" super(GitFetchTest, self).tearDown() - - if self.repo.stage is not None: - self.repo.stage.destroy() - - self.pkg.do_clean() - + self.repo.destroy() def assert_rev(self, rev): """Check that the current git revision is equal to the supplied rev.""" self.assertEqual(self.repo.rev_hash('HEAD'), self.repo.rev_hash(rev)) - def try_fetch(self, rev, test_file, args): """Tries to: 1. Fetch the repo using a fetch strategy constructed with @@ -79,26 +65,27 @@ class GitFetchTest(MockPackagesTest): """ self.pkg.versions[ver('git')] = args - self.pkg.do_stage() - self.assert_rev(rev) + with self.pkg.stage: + self.pkg.do_stage() + self.assert_rev(rev) - file_path = join_path(self.pkg.stage.source_path, test_file) - self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) - self.assertTrue(os.path.isfile(file_path)) + file_path = join_path(self.pkg.stage.source_path, test_file) + self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) + self.assertTrue(os.path.isfile(file_path)) - os.unlink(file_path) - self.assertFalse(os.path.isfile(file_path)) + os.unlink(file_path) + self.assertFalse(os.path.isfile(file_path)) - untracked_file = 'foobarbaz' - touch(untracked_file) - self.assertTrue(os.path.isfile(untracked_file)) - self.pkg.do_restage() - self.assertFalse(os.path.isfile(untracked_file)) + untracked_file = 'foobarbaz' + touch(untracked_file) + self.assertTrue(os.path.isfile(untracked_file)) + self.pkg.do_restage() + self.assertFalse(os.path.isfile(untracked_file)) - self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) - self.assertTrue(os.path.isfile(file_path)) + self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) + self.assertTrue(os.path.isfile(file_path)) - self.assert_rev(rev) + self.assert_rev(rev) def test_fetch_master(self): diff --git a/lib/spack/spack/test/hg_fetch.py b/lib/spack/spack/test/hg_fetch.py index ee8327aec8..b8a0c1ec46 100644 --- a/lib/spack/spack/test/hg_fetch.py +++ b/lib/spack/spack/test/hg_fetch.py @@ -23,16 +23,12 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import os -import unittest - -from llnl.util.filesystem import * - import spack + from spack.version import ver -from spack.stage import Stage -from spack.util.executable import which -from spack.test.mock_packages_test import * from spack.test.mock_repo import MockHgRepo +from llnl.util.filesystem import * +from spack.test.mock_packages_test import * class HgFetchTest(MockPackagesTest): @@ -49,16 +45,10 @@ class HgFetchTest(MockPackagesTest): spec.concretize() self.pkg = spack.repo.get(spec, new=True) - def tearDown(self): """Destroy the stage space used by this test.""" super(HgFetchTest, self).tearDown() - - if self.repo.stage is not None: - self.repo.stage.destroy() - - self.pkg.do_clean() - + self.repo.destroy() def try_fetch(self, rev, test_file, args): """Tries to: @@ -71,26 +61,27 @@ class HgFetchTest(MockPackagesTest): """ self.pkg.versions[ver('hg')] = args - self.pkg.do_stage() - self.assertEqual(self.repo.get_rev(), rev) + with self.pkg.stage: + self.pkg.do_stage() + self.assertEqual(self.repo.get_rev(), rev) - file_path = join_path(self.pkg.stage.source_path, test_file) - self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) - self.assertTrue(os.path.isfile(file_path)) + file_path = join_path(self.pkg.stage.source_path, test_file) + self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) + self.assertTrue(os.path.isfile(file_path)) - os.unlink(file_path) - self.assertFalse(os.path.isfile(file_path)) + os.unlink(file_path) + self.assertFalse(os.path.isfile(file_path)) - untracked = 'foobarbaz' - touch(untracked) - self.assertTrue(os.path.isfile(untracked)) - self.pkg.do_restage() - self.assertFalse(os.path.isfile(untracked)) + untracked = 'foobarbaz' + touch(untracked) + self.assertTrue(os.path.isfile(untracked)) + self.pkg.do_restage() + self.assertFalse(os.path.isfile(untracked)) - self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) - self.assertTrue(os.path.isfile(file_path)) + self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) + self.assertTrue(os.path.isfile(file_path)) - self.assertEqual(self.repo.get_rev(), rev) + self.assertEqual(self.repo.get_rev(), rev) def test_fetch_default(self): diff --git a/lib/spack/spack/test/install.py b/lib/spack/spack/test/install.py index 628329a423..8297893f01 100644 --- a/lib/spack/spack/test/install.py +++ b/lib/spack/spack/test/install.py @@ -22,18 +22,13 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import os -import unittest import shutil import tempfile -from llnl.util.filesystem import * - import spack -from spack.stage import Stage -from spack.fetch_strategy import URLFetchStrategy +from llnl.util.filesystem import * from spack.directory_layout import YamlDirectoryLayout -from spack.util.executable import which +from spack.fetch_strategy import URLFetchStrategy, FetchStrategyComposite from spack.test.mock_packages_test import * from spack.test.mock_repo import MockArchive @@ -59,9 +54,7 @@ class InstallTest(MockPackagesTest): def tearDown(self): super(InstallTest, self).tearDown() - - if self.repo.stage is not None: - self.repo.stage.destroy() + self.repo.destroy() # Turn checksumming back on spack.do_checksum = True @@ -81,7 +74,10 @@ class InstallTest(MockPackagesTest): pkg = spack.repo.get(spec) # Fake the URL for the package so it downloads from a file. - pkg.fetcher = URLFetchStrategy(self.repo.url) + + fetcher = FetchStrategyComposite() + fetcher.append(URLFetchStrategy(self.repo.url)) + pkg.fetcher = fetcher try: pkg.do_install() diff --git a/lib/spack/spack/test/link_tree.py b/lib/spack/spack/test/link_tree.py index 886b7ef4c5..ee37e765c7 100644 --- a/lib/spack/spack/test/link_tree.py +++ b/lib/spack/spack/test/link_tree.py @@ -24,8 +24,6 @@ ############################################################################## import os import unittest -import shutil -import tempfile from llnl.util.filesystem import * from llnl.util.link_tree import LinkTree @@ -38,6 +36,7 @@ class LinkTreeTest(unittest.TestCase): def setUp(self): self.stage = Stage('link-tree-test') + self.stage.create() with working_dir(self.stage.path): touchp('source/1') @@ -51,10 +50,8 @@ class LinkTreeTest(unittest.TestCase): source_path = os.path.join(self.stage.path, 'source') self.link_tree = LinkTree(source_path) - def tearDown(self): - if self.stage: - self.stage.destroy() + self.stage.destroy() def check_file_link(self, filename): diff --git a/lib/spack/spack/test/lock.py b/lib/spack/spack/test/lock.py index bc68df01db..3b11d18da4 100644 --- a/lib/spack/spack/test/lock.py +++ b/lib/spack/spack/test/lock.py @@ -25,15 +25,13 @@ """ These tests ensure that our lock works correctly. """ -import unittest -import os -import tempfile import shutil +import tempfile +import unittest from multiprocessing import Process -from llnl.util.lock import * from llnl.util.filesystem import join_path, touch - +from llnl.util.lock import * from spack.util.multiproc import Barrier # This is the longest a failed test will take, as the barriers will diff --git a/lib/spack/spack/test/make_executable.py b/lib/spack/spack/test/make_executable.py index 09efec8580..a2606acf19 100644 --- a/lib/spack/spack/test/make_executable.py +++ b/lib/spack/spack/test/make_executable.py @@ -28,13 +28,13 @@ Tests for Spack's built-in parallel make support. This just tests whether the right args are getting passed to make. """ import os -import unittest -import tempfile import shutil +import tempfile +import unittest from llnl.util.filesystem import * -from spack.util.environment import path_put_first from spack.build_environment import MakeExecutable +from spack.util.environment import path_put_first class MakeExecutableTest(unittest.TestCase): @@ -56,47 +56,47 @@ class MakeExecutableTest(unittest.TestCase): def test_make_normal(self): make = MakeExecutable('make', 8) - self.assertEqual(make(return_output=True).strip(), '-j8') - self.assertEqual(make('install', return_output=True).strip(), '-j8 install') + self.assertEqual(make(output=str).strip(), '-j8') + self.assertEqual(make('install', output=str).strip(), '-j8 install') def test_make_explicit(self): make = MakeExecutable('make', 8) - self.assertEqual(make(parallel=True, return_output=True).strip(), '-j8') - self.assertEqual(make('install', parallel=True, return_output=True).strip(), '-j8 install') + self.assertEqual(make(parallel=True, output=str).strip(), '-j8') + self.assertEqual(make('install', parallel=True, output=str).strip(), '-j8 install') def test_make_one_job(self): make = MakeExecutable('make', 1) - self.assertEqual(make(return_output=True).strip(), '') - self.assertEqual(make('install', return_output=True).strip(), 'install') + self.assertEqual(make(output=str).strip(), '') + self.assertEqual(make('install', output=str).strip(), 'install') def test_make_parallel_false(self): make = MakeExecutable('make', 8) - self.assertEqual(make(parallel=False, return_output=True).strip(), '') - self.assertEqual(make('install', parallel=False, return_output=True).strip(), 'install') + self.assertEqual(make(parallel=False, output=str).strip(), '') + self.assertEqual(make('install', parallel=False, output=str).strip(), 'install') def test_make_parallel_disabled(self): make = MakeExecutable('make', 8) os.environ['SPACK_NO_PARALLEL_MAKE'] = 'true' - self.assertEqual(make(return_output=True).strip(), '') - self.assertEqual(make('install', return_output=True).strip(), 'install') + self.assertEqual(make(output=str).strip(), '') + self.assertEqual(make('install', output=str).strip(), 'install') os.environ['SPACK_NO_PARALLEL_MAKE'] = '1' - self.assertEqual(make(return_output=True).strip(), '') - self.assertEqual(make('install', return_output=True).strip(), 'install') + self.assertEqual(make(output=str).strip(), '') + self.assertEqual(make('install', output=str).strip(), 'install') # These don't disable (false and random string) os.environ['SPACK_NO_PARALLEL_MAKE'] = 'false' - self.assertEqual(make(return_output=True).strip(), '-j8') - self.assertEqual(make('install', return_output=True).strip(), '-j8 install') + self.assertEqual(make(output=str).strip(), '-j8') + self.assertEqual(make('install', output=str).strip(), '-j8 install') os.environ['SPACK_NO_PARALLEL_MAKE'] = 'foobar' - self.assertEqual(make(return_output=True).strip(), '-j8') - self.assertEqual(make('install', return_output=True).strip(), '-j8 install') + self.assertEqual(make(output=str).strip(), '-j8') + self.assertEqual(make('install', output=str).strip(), '-j8 install') del os.environ['SPACK_NO_PARALLEL_MAKE'] @@ -106,20 +106,20 @@ class MakeExecutableTest(unittest.TestCase): # These should work os.environ['SPACK_NO_PARALLEL_MAKE'] = 'true' - self.assertEqual(make(parallel=True, return_output=True).strip(), '') - self.assertEqual(make('install', parallel=True, return_output=True).strip(), 'install') + self.assertEqual(make(parallel=True, output=str).strip(), '') + self.assertEqual(make('install', parallel=True, output=str).strip(), 'install') os.environ['SPACK_NO_PARALLEL_MAKE'] = '1' - self.assertEqual(make(parallel=True, return_output=True).strip(), '') - self.assertEqual(make('install', parallel=True, return_output=True).strip(), 'install') + self.assertEqual(make(parallel=True, output=str).strip(), '') + self.assertEqual(make('install', parallel=True, output=str).strip(), 'install') # These don't disable (false and random string) os.environ['SPACK_NO_PARALLEL_MAKE'] = 'false' - self.assertEqual(make(parallel=True, return_output=True).strip(), '-j8') - self.assertEqual(make('install', parallel=True, return_output=True).strip(), '-j8 install') + self.assertEqual(make(parallel=True, output=str).strip(), '-j8') + self.assertEqual(make('install', parallel=True, output=str).strip(), '-j8 install') os.environ['SPACK_NO_PARALLEL_MAKE'] = 'foobar' - self.assertEqual(make(parallel=True, return_output=True).strip(), '-j8') - self.assertEqual(make('install', parallel=True, return_output=True).strip(), '-j8 install') + self.assertEqual(make(parallel=True, output=str).strip(), '-j8') + self.assertEqual(make('install', parallel=True, output=str).strip(), '-j8 install') del os.environ['SPACK_NO_PARALLEL_MAKE'] diff --git a/lib/spack/spack/test/mirror.py b/lib/spack/spack/test/mirror.py index 04e9e3db2e..e707adfe9d 100644 --- a/lib/spack/spack/test/mirror.py +++ b/lib/spack/spack/test/mirror.py @@ -23,11 +23,10 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import os -from filecmp import dircmp - import spack import spack.mirror -from spack.util.compression import decompressor_for + +from filecmp import dircmp from spack.test.mock_packages_test import * from spack.test.mock_repo import * @@ -44,8 +43,16 @@ class MirrorTest(MockPackagesTest): self.repos = {} + def tearDown(self): + """Destroy all the stages created by the repos in setup.""" + super(MirrorTest, self).tearDown() + for repo in self.repos.values(): + repo.destroy() + self.repos.clear() + + def set_up_package(self, name, MockRepoClass, url_attr): - """Use this to set up a mock package to be mirrored. + """Set up a mock package to be mirrored. Each package needs us to: 1. Set up a mock repo/archive to fetch from. 2. Point the package's version args at that repo. @@ -65,22 +72,15 @@ class MirrorTest(MockPackagesTest): pkg.versions[v][url_attr] = repo.url - def tearDown(self): - """Destroy all the stages created by the repos in setup.""" - super(MirrorTest, self).tearDown() - - for name, repo in self.repos.items(): - if repo.stage: - pass #repo.stage.destroy() - - self.repos.clear() + def check_mirror(self): + with Stage('spack-mirror-test') as stage: + mirror_root = join_path(stage.path, 'test-mirror') + # register mirror with spack config + mirrors = { 'spack-mirror-test' : 'file://' + mirror_root } + spack.config.update_config('mirrors', mirrors) - def check_mirror(self): - stage = Stage('spack-mirror-test') - mirror_root = join_path(stage.path, 'test-mirror') - try: os.chdir(stage.path) spack.mirror.create( mirror_root, self.repos, no_checksum=True) @@ -88,7 +88,7 @@ class MirrorTest(MockPackagesTest): # Stage directory exists self.assertTrue(os.path.isdir(mirror_root)) - # subdirs for each package + # check that there are subdirs for each package for name in self.repos: subdir = join_path(mirror_root, name) self.assertTrue(os.path.isdir(subdir)) @@ -96,40 +96,28 @@ class MirrorTest(MockPackagesTest): files = os.listdir(subdir) self.assertEqual(len(files), 1) - # Decompress archive in the mirror - archive = files[0] - archive_path = join_path(subdir, archive) - decomp = decompressor_for(archive_path) - - with working_dir(subdir): - decomp(archive_path) - - # Find the untarred archive directory. - files = os.listdir(subdir) - self.assertEqual(len(files), 2) - self.assertTrue(archive in files) - files.remove(archive) - - expanded_archive = join_path(subdir, files[0]) - self.assertTrue(os.path.isdir(expanded_archive)) - - # Compare the original repo with the expanded archive - repo = self.repos[name] - if not 'svn' in name: - original_path = repo.path - else: - co = 'checked_out' - svn('checkout', repo.url, co) - original_path = join_path(subdir, co) - - dcmp = dircmp(original_path, expanded_archive) - - # make sure there are no new files in the expanded tarball - self.assertFalse(dcmp.right_only) - self.assertTrue(all(l in exclude for l in dcmp.left_only)) - - finally: - pass #stage.destroy() + # Now try to fetch each package. + for name, mock_repo in self.repos.items(): + spec = Spec(name).concretized() + pkg = spec.package + + saved_checksum_setting = spack.do_checksum + with pkg.stage: + # Stage the archive from the mirror and cd to it. + spack.do_checksum = False + pkg.do_stage(mirror_only=True) + # Compare the original repo with the expanded archive + original_path = mock_repo.path + if 'svn' in name: + # have to check out the svn repo to compare. + original_path = join_path(mock_repo.path, 'checked_out') + svn('checkout', mock_repo.url, original_path) + dcmp = dircmp(original_path, pkg.stage.source_path) + # make sure there are no new files in the expanded tarball + self.assertFalse(dcmp.right_only) + # and that all original files are present. + self.assertTrue(all(l in exclude for l in dcmp.left_only)) + spack.do_checksum = saved_checksum_setting def test_git_mirror(self): diff --git a/lib/spack/spack/test/mock_packages_test.py b/lib/spack/spack/test/mock_packages_test.py index e9f1f95df5..6d24a84150 100644 --- a/lib/spack/spack/test/mock_packages_test.py +++ b/lib/spack/spack/test/mock_packages_test.py @@ -22,17 +22,15 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import sys import os import shutil -import unittest import tempfile -from ordereddict_backport import OrderedDict - -from llnl.util.filesystem import mkdirp +import unittest import spack import spack.config +from llnl.util.filesystem import mkdirp +from ordereddict_backport import OrderedDict from spack.repository import RepoPath from spack.spec import Spec @@ -51,6 +49,19 @@ compilers: fc: /path/to/gfortran """ +mock_packages_config = """\ +packages: + externaltool: + buildable: False + paths: + externaltool@1.0%gcc@4.5.0: /path/to/external_tool + externalvirtual: + buildable: False + paths: + externalvirtual@2.0%clang@3.3: /path/to/external_virtual_clang + externalvirtual@1.0%gcc@4.5.0: /path/to/external_virtual_gcc +""" + class MockPackagesTest(unittest.TestCase): def initmock(self): # Use the mock packages database for these tests. This allows @@ -68,9 +79,10 @@ class MockPackagesTest(unittest.TestCase): self.mock_user_config = os.path.join(self.temp_config, 'user') mkdirp(self.mock_site_config) mkdirp(self.mock_user_config) - comp_yaml = os.path.join(self.mock_site_config, 'compilers.yaml') - with open(comp_yaml, 'w') as f: - f.write(mock_compiler_config) + for confs in [('compilers.yaml', mock_compiler_config), ('packages.yaml', mock_packages_config)]: + conf_yaml = os.path.join(self.mock_site_config, confs[0]) + with open(conf_yaml, 'w') as f: + f.write(confs[1]) # TODO: Mocking this up is kind of brittle b/c ConfigScope # TODO: constructor modifies config_scopes. Make it cleaner. diff --git a/lib/spack/spack/test/mock_repo.py b/lib/spack/spack/test/mock_repo.py index c454b1f106..a8bdfb5571 100644 --- a/lib/spack/spack/test/mock_repo.py +++ b/lib/spack/spack/test/mock_repo.py @@ -26,13 +26,9 @@ import os import shutil from llnl.util.filesystem import * - -import spack -from spack.version import ver from spack.stage import Stage from spack.util.executable import which - # # VCS Systems used by mock repo code. # @@ -55,6 +51,12 @@ class MockRepo(object): mkdirp(self.path) + def destroy(self): + """Destroy resources associated with this mock repo.""" + if self.stage: + self.stage.destroy() + + class MockArchive(MockRepo): """Creates a very simple archive directory with a configure script and a makefile that installs to a prefix. Tars it up into an archive.""" @@ -141,7 +143,7 @@ class MockGitRepo(MockVCSRepo): self.url = self.path def rev_hash(self, rev): - return git('rev-parse', rev, return_output=True).strip() + return git('rev-parse', rev, output=str).strip() class MockSvnRepo(MockVCSRepo): @@ -193,4 +195,4 @@ class MockHgRepo(MockVCSRepo): def get_rev(self): """Get current mercurial revision.""" - return hg('id', '-i', return_output=True).strip() + return hg('id', '-i', output=str).strip() diff --git a/lib/spack/spack/test/multimethod.py b/lib/spack/spack/test/multimethod.py index 7bf4ff0a0a..2d4b8cd584 100644 --- a/lib/spack/spack/test/multimethod.py +++ b/lib/spack/spack/test/multimethod.py @@ -25,14 +25,11 @@ """ Test for multi_method dispatch. """ -import unittest import spack from spack.multimethod import * -from spack.version import * -from spack.spec import Spec -from spack.multimethod import when from spack.test.mock_packages_test import * +from spack.version import * class MultiMethodTest(MockPackagesTest): diff --git a/lib/spack/spack/test/namespace_trie.py b/lib/spack/spack/test/namespace_trie.py index d0d809004d..2023ba6d96 100644 --- a/lib/spack/spack/test/namespace_trie.py +++ b/lib/spack/spack/test/namespace_trie.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://llnl.github.io/spack +# For details, see https://software.llnl.gov/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify @@ -23,6 +23,7 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import unittest + from spack.util.naming import NamespaceTrie diff --git a/lib/spack/spack/test/optional_deps.py b/lib/spack/spack/test/optional_deps.py index ebd7281999..55f35ea4c9 100644 --- a/lib/spack/spack/test/optional_deps.py +++ b/lib/spack/spack/test/optional_deps.py @@ -22,10 +22,8 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import unittest -import spack -from spack.spec import Spec, CompilerSpec +from spack.spec import Spec from spack.test.mock_packages_test import * class ConcretizeTest(MockPackagesTest): diff --git a/lib/spack/spack/test/packages.py b/lib/spack/spack/test/packages.py index 83984dc5f6..f0b5e05f3b 100644 --- a/lib/spack/spack/test/packages.py +++ b/lib/spack/spack/test/packages.py @@ -22,14 +22,12 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import unittest - -from llnl.util.filesystem import join_path import spack +from llnl.util.filesystem import join_path from spack.repository import Repo -from spack.util.naming import mod_to_class from spack.test.mock_packages_test import * +from spack.util.naming import mod_to_class class PackagesTest(MockPackagesTest): diff --git a/lib/spack/spack/test/pattern.py b/lib/spack/spack/test/pattern.py new file mode 100644 index 0000000000..6c783c6a5f --- /dev/null +++ b/lib/spack/spack/test/pattern.py @@ -0,0 +1,104 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## + +import unittest + +import spack.util.pattern as pattern + + +class CompositeTest(unittest.TestCase): + + def setUp(self): + class Base: + counter = 0 + + def add(self): + raise NotImplemented('add not implemented') + + def subtract(self): + raise NotImplemented('subtract not implemented') + + class One(Base): + def add(self): + Base.counter += 1 + + def subtract(self): + Base.counter -= 1 + + class Two(Base): + def add(self): + Base.counter += 2 + + def subtract(self): + Base.counter -= 2 + + self.Base = Base + self.One = One + self.Two = Two + + def test_composite_from_method_list(self): + + @pattern.composite(method_list=['add', 'subtract']) + class CompositeFromMethodList: + pass + + composite = CompositeFromMethodList() + composite.append(self.One()) + composite.append(self.Two()) + composite.add() + self.assertEqual(self.Base.counter, 3) + composite.pop() + composite.subtract() + self.assertEqual(self.Base.counter, 2) + + def test_composite_from_interface(self): + + @pattern.composite(interface=self.Base) + class CompositeFromInterface: + pass + + composite = CompositeFromInterface() + composite.append(self.One()) + composite.append(self.Two()) + composite.add() + self.assertEqual(self.Base.counter, 3) + composite.pop() + composite.subtract() + self.assertEqual(self.Base.counter, 2) + + def test_error_conditions(self): + + def wrong_container(): + @pattern.composite(interface=self.Base, container=2) + class CompositeFromInterface: + pass + + def no_methods(): + @pattern.composite() + class CompositeFromInterface: + pass + + self.assertRaises(TypeError, wrong_container) + self.assertRaises(TypeError, no_methods) diff --git a/lib/spack/spack/test/python_version.py b/lib/spack/spack/test/python_version.py index d74d3b9b7d..4294975304 100644 --- a/lib/spack/spack/test/python_version.py +++ b/lib/spack/spack/test/python_version.py @@ -28,12 +28,11 @@ This test ensures that all Spack files are Python version 2.6 or less. Spack was originally 2.7, but enough systems in 2014 are still using 2.6 on their frontend nodes that we need 2.6 to get adopted. """ -import unittest import os import re +import unittest import llnl.util.tty as tty - import pyqver2 import spack diff --git a/lib/spack/spack/test/sbang.py b/lib/spack/spack/test/sbang.py new file mode 100644 index 0000000000..825bc4be98 --- /dev/null +++ b/lib/spack/spack/test/sbang.py @@ -0,0 +1,93 @@ +############################################################################## +# Copyright (c) 2013-2015, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +"""\ +Test that Spack's shebang filtering works correctly. +""" +import os +import unittest +import tempfile +import shutil + +from llnl.util.filesystem import * +from spack.hooks.sbang import filter_shebangs_in_directory +import spack + +short_line = "#!/this/is/short/bin/bash\n" +long_line = "#!/this/" + ('x' * 200) + "/is/long\n" +sbang_line = '#!/bin/bash %s/bin/sbang\n' % spack.spack_root +last_line = "last!\n" + +class SbangTest(unittest.TestCase): + def setUp(self): + self.tempdir = tempfile.mkdtemp() + + # make sure we can ignore non-files + directory = os.path.join(self.tempdir, 'dir') + mkdirp(directory) + + # Script with short shebang + self.short_shebang = os.path.join(self.tempdir, 'short') + with open(self.short_shebang, 'w') as f: + f.write(short_line) + f.write(last_line) + + # Script with long shebang + self.long_shebang = os.path.join(self.tempdir, 'long') + with open(self.long_shebang, 'w') as f: + f.write(long_line) + f.write(last_line) + + # Script already using sbang. + self.has_shebang = os.path.join(self.tempdir, 'shebang') + with open(self.has_shebang, 'w') as f: + f.write(sbang_line) + f.write(long_line) + f.write(last_line) + + + def tearDown(self): + shutil.rmtree(self.tempdir, ignore_errors=True) + + + + def test_shebang_handling(self): + filter_shebangs_in_directory(self.tempdir) + + # Make sure this is untouched + with open(self.short_shebang, 'r') as f: + self.assertEqual(f.readline(), short_line) + self.assertEqual(f.readline(), last_line) + + # Make sure this got patched. + with open(self.long_shebang, 'r') as f: + self.assertEqual(f.readline(), sbang_line) + self.assertEqual(f.readline(), long_line) + self.assertEqual(f.readline(), last_line) + + # Make sure this is untouched + with open(self.has_shebang, 'r') as f: + self.assertEqual(f.readline(), sbang_line) + self.assertEqual(f.readline(), long_line) + self.assertEqual(f.readline(), last_line) diff --git a/lib/spack/spack/test/spec_dag.py b/lib/spack/spack/test/spec_dag.py index 632f777cde..5e6162b6e6 100644 --- a/lib/spack/spack/test/spec_dag.py +++ b/lib/spack/spack/test/spec_dag.py @@ -31,8 +31,6 @@ You can find the dummy packages here:: import spack import spack.package -from llnl.util.lang import list_modules - from spack.spec import Spec from spack.test.mock_packages_test import * diff --git a/lib/spack/spack/test/spec_semantics.py b/lib/spack/spack/test/spec_semantics.py index 44a09cbd7f..8c33d1ff6e 100644 --- a/lib/spack/spack/test/spec_semantics.py +++ b/lib/spack/spack/test/spec_semantics.py @@ -22,7 +22,6 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import unittest from spack.spec import * from spack.test.mock_packages_test import * diff --git a/lib/spack/spack/test/spec_syntax.py b/lib/spack/spack/test/spec_syntax.py index 1daaa4be8f..6e08e30e13 100644 --- a/lib/spack/spack/test/spec_syntax.py +++ b/lib/spack/spack/test/spec_syntax.py @@ -23,9 +23,10 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import unittest + import spack.spec -from spack.spec import * from spack.parse import Token +from spack.spec import * # Sample output for a complex lexing. complex_lex = [Token(ID, 'mvapich_foo'), diff --git a/lib/spack/spack/test/stage.py b/lib/spack/spack/test/stage.py index c1b2a2a573..ea425127c4 100644 --- a/lib/spack/spack/test/stage.py +++ b/lib/spack/spack/test/stage.py @@ -25,15 +25,13 @@ """\ Test that the Stage class works correctly. """ -import unittest -import shutil import os -import getpass +import shutil +import unittest from contextlib import * -from llnl.util.filesystem import * - import spack +from llnl.util.filesystem import * from spack.stage import Stage from spack.util.executable import which @@ -192,116 +190,125 @@ class StageTest(unittest.TestCase): def test_setup_and_destroy_name_with_tmp(self): with use_tmp(True): - stage = Stage(archive_url, name=stage_name) - self.check_setup(stage, stage_name) - - stage.destroy() + with Stage(archive_url, name=stage_name) as stage: + self.check_setup(stage, stage_name) self.check_destroy(stage, stage_name) def test_setup_and_destroy_name_without_tmp(self): with use_tmp(False): - stage = Stage(archive_url, name=stage_name) - self.check_setup(stage, stage_name) - - stage.destroy() + with Stage(archive_url, name=stage_name) as stage: + self.check_setup(stage, stage_name) self.check_destroy(stage, stage_name) def test_setup_and_destroy_no_name_with_tmp(self): with use_tmp(True): - stage = Stage(archive_url) - self.check_setup(stage, None) - - stage.destroy() + with Stage(archive_url) as stage: + self.check_setup(stage, None) self.check_destroy(stage, None) def test_setup_and_destroy_no_name_without_tmp(self): with use_tmp(False): - stage = Stage(archive_url) - self.check_setup(stage, None) - - stage.destroy() + with Stage(archive_url) as stage: + self.check_setup(stage, None) self.check_destroy(stage, None) def test_chdir(self): - stage = Stage(archive_url, name=stage_name) - - stage.chdir() - self.check_setup(stage, stage_name) - self.check_chdir(stage, stage_name) - - stage.destroy() + with Stage(archive_url, name=stage_name) as stage: + stage.chdir() + self.check_setup(stage, stage_name) + self.check_chdir(stage, stage_name) self.check_destroy(stage, stage_name) def test_fetch(self): - stage = Stage(archive_url, name=stage_name) - - stage.fetch() - self.check_setup(stage, stage_name) - self.check_chdir(stage, stage_name) - self.check_fetch(stage, stage_name) - - stage.destroy() + with Stage(archive_url, name=stage_name) as stage: + stage.fetch() + self.check_setup(stage, stage_name) + self.check_chdir(stage, stage_name) + self.check_fetch(stage, stage_name) self.check_destroy(stage, stage_name) def test_expand_archive(self): - stage = Stage(archive_url, name=stage_name) - - stage.fetch() - self.check_setup(stage, stage_name) - self.check_fetch(stage, stage_name) - - stage.expand_archive() - self.check_expand_archive(stage, stage_name) - - stage.destroy() + with Stage(archive_url, name=stage_name) as stage: + stage.fetch() + self.check_setup(stage, stage_name) + self.check_fetch(stage, stage_name) + stage.expand_archive() + self.check_expand_archive(stage, stage_name) self.check_destroy(stage, stage_name) def test_expand_archive(self): - stage = Stage(archive_url, name=stage_name) + with Stage(archive_url, name=stage_name) as stage: + stage.fetch() + self.check_setup(stage, stage_name) + self.check_fetch(stage, stage_name) + stage.expand_archive() + stage.chdir_to_source() + self.check_expand_archive(stage, stage_name) + self.check_chdir_to_source(stage, stage_name) + self.check_destroy(stage, stage_name) - stage.fetch() - self.check_setup(stage, stage_name) - self.check_fetch(stage, stage_name) - stage.expand_archive() - stage.chdir_to_source() - self.check_expand_archive(stage, stage_name) - self.check_chdir_to_source(stage, stage_name) + def test_restage(self): + with Stage(archive_url, name=stage_name) as stage: + stage.fetch() + stage.expand_archive() + stage.chdir_to_source() + self.check_expand_archive(stage, stage_name) + self.check_chdir_to_source(stage, stage_name) + + # Try to make a file in the old archive dir + with open('foobar', 'w') as file: + file.write("this file is to be destroyed.") + + self.assertTrue('foobar' in os.listdir(stage.source_path)) + + # Make sure the file is not there after restage. + stage.restage() + self.check_chdir(stage, stage_name) + self.check_fetch(stage, stage_name) + stage.chdir_to_source() + self.check_chdir_to_source(stage, stage_name) + self.assertFalse('foobar' in os.listdir(stage.source_path)) + self.check_destroy(stage, stage_name) - stage.destroy() + + def test_no_keep_without_exceptions(self): + with Stage(archive_url, name=stage_name, keep=False) as stage: + pass self.check_destroy(stage, stage_name) - def test_restage(self): - stage = Stage(archive_url, name=stage_name) + def test_keep_without_exceptions(self): + with Stage(archive_url, name=stage_name, keep=True) as stage: + pass + path = self.get_stage_path(stage, stage_name) + self.assertTrue(os.path.isdir(path)) - stage.fetch() - stage.expand_archive() - stage.chdir_to_source() - self.check_expand_archive(stage, stage_name) - self.check_chdir_to_source(stage, stage_name) - # Try to make a file in the old archive dir - with open('foobar', 'w') as file: - file.write("this file is to be destroyed.") + def test_no_keep_with_exceptions(self): + try: + with Stage(archive_url, name=stage_name, keep=False) as stage: + raise Exception() - self.assertTrue('foobar' in os.listdir(stage.source_path)) + path = self.get_stage_path(stage, stage_name) + self.assertTrue(os.path.isdir(path)) + except: + pass # ignore here. - # Make sure the file is not there after restage. - stage.restage() - self.check_chdir(stage, stage_name) - self.check_fetch(stage, stage_name) - stage.chdir_to_source() - self.check_chdir_to_source(stage, stage_name) - self.assertFalse('foobar' in os.listdir(stage.source_path)) + def test_keep_exceptions(self): + try: + with Stage(archive_url, name=stage_name, keep=True) as stage: + raise Exception() - stage.destroy() - self.check_destroy(stage, stage_name) + path = self.get_stage_path(stage, stage_name) + self.assertTrue(os.path.isdir(path)) + except: + pass # ignore here. diff --git a/lib/spack/spack/test/svn_fetch.py b/lib/spack/spack/test/svn_fetch.py index 2ee4748fdb..1ee4ee700e 100644 --- a/lib/spack/spack/test/svn_fetch.py +++ b/lib/spack/spack/test/svn_fetch.py @@ -24,18 +24,12 @@ ############################################################################## import os import re -import unittest -import shutil -import tempfile - -from llnl.util.filesystem import * - import spack + +from spack.test.mock_repo import svn, MockSvnRepo from spack.version import ver -from spack.stage import Stage -from spack.util.executable import which from spack.test.mock_packages_test import * -from spack.test.mock_repo import svn, MockSvnRepo +from llnl.util.filesystem import * class SvnFetchTest(MockPackagesTest): @@ -51,21 +45,15 @@ class SvnFetchTest(MockPackagesTest): spec.concretize() self.pkg = spack.repo.get(spec, new=True) - def tearDown(self): """Destroy the stage space used by this test.""" super(SvnFetchTest, self).tearDown() - - if self.repo.stage is not None: - self.repo.stage.destroy() - - self.pkg.do_clean() - + self.repo.destroy() def assert_rev(self, rev): """Check that the current revision is equal to the supplied rev.""" def get_rev(): - output = svn('info', return_output=True) + output = svn('info', output=str) self.assertTrue("Revision" in output) for line in output.split('\n'): match = re.match(r'Revision: (\d+)', line) @@ -73,7 +61,6 @@ class SvnFetchTest(MockPackagesTest): return match.group(1) self.assertEqual(get_rev(), rev) - def try_fetch(self, rev, test_file, args): """Tries to: 1. Fetch the repo using a fetch strategy constructed with @@ -85,26 +72,27 @@ class SvnFetchTest(MockPackagesTest): """ self.pkg.versions[ver('svn')] = args - self.pkg.do_stage() - self.assert_rev(rev) + with self.pkg.stage: + self.pkg.do_stage() + self.assert_rev(rev) - file_path = join_path(self.pkg.stage.source_path, test_file) - self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) - self.assertTrue(os.path.isfile(file_path)) + file_path = join_path(self.pkg.stage.source_path, test_file) + self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) + self.assertTrue(os.path.isfile(file_path)) - os.unlink(file_path) - self.assertFalse(os.path.isfile(file_path)) + os.unlink(file_path) + self.assertFalse(os.path.isfile(file_path)) - untracked = 'foobarbaz' - touch(untracked) - self.assertTrue(os.path.isfile(untracked)) - self.pkg.do_restage() - self.assertFalse(os.path.isfile(untracked)) + untracked = 'foobarbaz' + touch(untracked) + self.assertTrue(os.path.isfile(untracked)) + self.pkg.do_restage() + self.assertFalse(os.path.isfile(untracked)) - self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) - self.assertTrue(os.path.isfile(file_path)) + self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) + self.assertTrue(os.path.isfile(file_path)) - self.assert_rev(rev) + self.assert_rev(rev) def test_fetch_default(self): diff --git a/lib/spack/spack/test/tally_plugin.py b/lib/spack/spack/test/tally_plugin.py index 9ca898c47c..4163ab95dd 100644 --- a/lib/spack/spack/test/tally_plugin.py +++ b/lib/spack/spack/test/tally_plugin.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://scalability-software.llnl.gov/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify @@ -22,10 +22,10 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -from nose.plugins import Plugin - import os +from nose.plugins import Plugin + class Tally(Plugin): name = 'tally' @@ -34,7 +34,7 @@ class Tally(Plugin): self.successCount = 0 self.failCount = 0 self.errorCount = 0 - + @property def numberOfTestsRun(self): """Excludes skipped tests""" @@ -48,10 +48,10 @@ class Tally(Plugin): def addSuccess(self, test): self.successCount += 1 - + def addError(self, test, err): self.errorCount += 1 - + def addFailure(self, test, err): self.failCount += 1 diff --git a/lib/spack/spack/test/unit_install.py b/lib/spack/spack/test/unit_install.py index ccc409dd60..18615b7efe 100644 --- a/lib/spack/spack/test/unit_install.py +++ b/lib/spack/spack/test/unit_install.py @@ -22,10 +22,11 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import unittest import itertools +import unittest import spack + test_install = __import__("spack.cmd.test-install", fromlist=["BuildId", "create_test_output", "TestResult"]) diff --git a/lib/spack/spack/test/url_extrapolate.py b/lib/spack/spack/test/url_extrapolate.py index 87adf89401..068a335b49 100644 --- a/lib/spack/spack/test/url_extrapolate.py +++ b/lib/spack/spack/test/url_extrapolate.py @@ -25,10 +25,7 @@ """\ Tests ability of spack to extrapolate URL versions from existing versions. """ -import spack import spack.url as url -from spack.spec import Spec -from spack.version import ver from spack.test.mock_packages_test import * diff --git a/lib/spack/spack/test/url_parse.py b/lib/spack/spack/test/url_parse.py index efde7c0c73..561d4658a1 100644 --- a/lib/spack/spack/test/url_parse.py +++ b/lib/spack/spack/test/url_parse.py @@ -27,8 +27,8 @@ This file has a bunch of versions tests taken from the excellent version detection in Homebrew. """ import unittest + import spack.url as url -from pprint import pprint class UrlParseTest(unittest.TestCase): diff --git a/lib/spack/spack/test/url_substitution.py b/lib/spack/spack/test/url_substitution.py index aec8baf4ea..2be38af0d3 100644 --- a/lib/spack/spack/test/url_substitution.py +++ b/lib/spack/spack/test/url_substitution.py @@ -27,7 +27,6 @@ This test does sanity checks on substituting new versions into URLs """ import unittest -import spack import spack.url as url diff --git a/lib/spack/spack/test/versions.py b/lib/spack/spack/test/versions.py index 108450e098..2732006eb3 100644 --- a/lib/spack/spack/test/versions.py +++ b/lib/spack/spack/test/versions.py @@ -28,6 +28,7 @@ We try to maintain compatibility with RPM's version semantics where it makes sense. """ import unittest + from spack.version import * diff --git a/lib/spack/spack/test/yaml.py b/lib/spack/spack/test/yaml.py index 5a357b8e69..b930c022f2 100644 --- a/lib/spack/spack/test/yaml.py +++ b/lib/spack/spack/test/yaml.py @@ -26,6 +26,7 @@ Test Spack's custom YAML format. """ import unittest + import spack.util.spack_yaml as syaml test_file = """\ diff --git a/lib/spack/spack/url.py b/lib/spack/spack/url.py index 02c0b83e26..f51f05cad7 100644 --- a/lib/spack/spack/url.py +++ b/lib/spack/spack/url.py @@ -142,7 +142,7 @@ def split_url_extension(path): def downloaded_file_extension(path): """This returns the type of archive a URL refers to. This is - sometimes confusing becasue of URLs like: + sometimes confusing because of URLs like: (1) https://github.com/petdance/ack/tarball/1.93_02 @@ -225,7 +225,7 @@ def parse_version_offset(path): (r'_((\d+\.)+\d+[a-z]?)[.]orig$', stem), # e.g. http://www.openssl.org/source/openssl-0.9.8s.tar.gz - (r'-([^-]+(-alpha|-beta)?)', stem), + (r'-v?([^-]+(-alpha|-beta)?)', stem), # e.g. astyle_1.23_macosx.tar.gz (r'_([^_]+(_alpha|_beta)?)', stem), diff --git a/lib/spack/spack/util/compression.py b/lib/spack/spack/util/compression.py index ea1f233bce..5ae5867428 100644 --- a/lib/spack/spack/util/compression.py +++ b/lib/spack/spack/util/compression.py @@ -27,13 +27,12 @@ import os from itertools import product from spack.util.executable import which -# Supported archvie extensions. +# Supported archive extensions. PRE_EXTS = ["tar"] EXTS = ["gz", "bz2", "xz", "Z", "zip", "tgz"] -# Add EXTS last so that .tar.gz is matched *before* tar.gz -ALLOWED_ARCHIVE_TYPES = [".".join(l) for l in product(PRE_EXTS, EXTS)] + EXTS - +# Add PRE_EXTS and EXTS last so that .tar.gz is matched *before* .tar or .gz +ALLOWED_ARCHIVE_TYPES = [".".join(l) for l in product(PRE_EXTS, EXTS)] + PRE_EXTS + EXTS def allowed_archive(path): return any(path.endswith(t) for t in ALLOWED_ARCHIVE_TYPES) diff --git a/lib/spack/spack/util/environment.py b/lib/spack/spack/util/environment.py index cd413dcfbc..55e653fd2f 100644 --- a/lib/spack/spack/util/environment.py +++ b/lib/spack/spack/util/environment.py @@ -59,7 +59,8 @@ def path_put_first(var_name, directories): path_set(var_name, new_path) -def pop_keys(dictionary, *keys): - for key in keys: - if key in dictionary: - dictionary.pop(key) +def dump_environment(path): + """Dump the current environment out to a file.""" + with open(path, 'w') as env_file: + for key, val in sorted(os.environ.items()): + env_file.write("%s=%s\n" % (key, val)) diff --git a/lib/spack/spack/util/executable.py b/lib/spack/spack/util/executable.py index ba765eb662..fc27b789d0 100644 --- a/lib/spack/spack/util/executable.py +++ b/lib/spack/spack/util/executable.py @@ -55,24 +55,80 @@ class Executable(object): def __call__(self, *args, **kwargs): - """Run the executable with subprocess.check_output, return output.""" - return_output = kwargs.get("return_output", False) - fail_on_error = kwargs.get("fail_on_error", True) - ignore_errors = kwargs.get("ignore_errors", ()) + """Run this executable in a subprocess. + + Arguments + args + command line arguments to the executable to run. + + Optional arguments + + fail_on_error + + Raise an exception if the subprocess returns an + error. Default is True. When not set, the return code is + avaiale as `exe.returncode`. + + ignore_errors + + An optional list/tuple of error codes that can be + *ignored*. i.e., if these codes are returned, this will + not raise an exception when `fail_on_error` is `True`. + + output, error + + These arguments allow you to specify new stdout and stderr + values. They default to `None`, which means the + subprocess will inherit the parent's file descriptors. + + You can set these to: + - python streams, e.g. open Python file objects, or os.devnull; + - filenames, which will be automatically opened for writing; or + - `str`, as in the Python string type. If you set these to `str`, + output and error will be written to pipes and returned as + a string. If both `output` and `error` are set to `str`, + then one string is returned containing output concatenated + with error. + + input + + Same as output, error, but `str` is not an allowed value. + + Deprecated arguments + + return_output[=False] + + Setting this to True is the same as setting output=str. + This argument may be removed in future Spack versions. + + """ + fail_on_error = kwargs.pop("fail_on_error", True) + ignore_errors = kwargs.pop("ignore_errors", ()) + + # TODO: This is deprecated. Remove in a future version. + return_output = kwargs.pop("return_output", False) # Default values of None says to keep parent's file descriptors. - output = kwargs.get("output", None) - error = kwargs.get("error", None) - input = kwargs.get("input", None) + if return_output: + output = str + else: + output = kwargs.pop("output", None) + + error = kwargs.pop("error", None) + input = kwargs.pop("input", None) + if input is str: + raise ValueError("Cannot use `str` as input stream.") def streamify(arg, mode): if isinstance(arg, basestring): return open(arg, mode), True + elif arg is str: + return subprocess.PIPE, False else: return arg, False - output, ostream = streamify(output, 'w') - error, estream = streamify(error, 'w') - input, istream = streamify(input, 'r') + ostream, close_ostream = streamify(output, 'w') + estream, close_estream = streamify(error, 'w') + istream, close_istream = streamify(input, 'r') # if they just want to ignore one error code, make it a tuple. if isinstance(ignore_errors, int): @@ -92,19 +148,20 @@ class Executable(object): tty.debug(cmd_line) try: - if return_output: - output = subprocess.PIPE - proc = subprocess.Popen( - cmd, stdin=input, stderr=error, stdout=output) + cmd, stdin=istream, stderr=estream, stdout=ostream) out, err = proc.communicate() rc = self.returncode = proc.returncode if fail_on_error and rc != 0 and (rc not in ignore_errors): raise ProcessError("Command exited with status %d:" % proc.returncode, cmd_line) - if return_output: - return out + + if output is str or error is str: + result = '' + if output is str: result += out + if error is str: result += err + return result except OSError, e: raise ProcessError( @@ -119,9 +176,9 @@ class Executable(object): % (proc.returncode, cmd_line)) finally: - if ostream: output.close() - if estream: error.close() - if istream: input.close() + if close_ostream: output.close() + if close_estream: error.close() + if close_istream: input.close() def __eq__(self, other): diff --git a/lib/spack/spack/util/pattern.py b/lib/spack/spack/util/pattern.py new file mode 100644 index 0000000000..17a126498b --- /dev/null +++ b/lib/spack/spack/util/pattern.py @@ -0,0 +1,116 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +import inspect +import collections +import functools + + +def composite(interface=None, method_list=None, container=list): + """ + Returns a class decorator that patches a class adding all the methods it needs to be a composite for a given + interface. + + :param interface: class exposing the interface to which the composite object must conform. Only non-private and + non-special methods will be taken into account + + :param method_list: names of methods that should be part of the composite + + :param container: container for the composite object (default = list). Must fulfill the MutableSequence contract. + The composite class will expose the container API to manage object composition + + :return: class decorator + """ + # Check if container fulfills the MutableSequence contract and raise an exception if it doesn't + # The patched class returned by the decorator will inherit from the container class to expose the + # interface needed to manage objects composition + if not issubclass(container, collections.MutableSequence): + raise TypeError("Container must fulfill the MutableSequence contract") + + # Check if at least one of the 'interface' or the 'method_list' arguments are defined + if interface is None and method_list is None: + raise TypeError("Either 'interface' or 'method_list' must be defined on a call to composite") + + def cls_decorator(cls): + # Retrieve the base class of the composite. Inspect its methods and decide which ones will be overridden + def no_special_no_private(x): + return inspect.ismethod(x) and not x.__name__.startswith('_') + + # Patch the behavior of each of the methods in the previous list. This is done associating an instance of the + # descriptor below to any method that needs to be patched. + class IterateOver(object): + """ + Decorator used to patch methods in a composite. It iterates over all the items in the instance containing the + associated attribute and calls for each of them an attribute with the same name + """ + def __init__(self, name, func=None): + self.name = name + self.func = func + + def __get__(self, instance, owner): + def getter(*args, **kwargs): + for item in instance: + getattr(item, self.name)(*args, **kwargs) + # If we are using this descriptor to wrap a method from an interface, then we must conditionally + # use the `functools.wraps` decorator to set the appropriate fields. + if self.func is not None: + getter = functools.wraps(self.func)(getter) + return getter + + dictionary_for_type_call = {} + # Construct a dictionary with the methods explicitly passed as name + if method_list is not None: + # python@2.7: method_list_dict = {name: IterateOver(name) for name in method_list} + method_list_dict = {} + for name in method_list: + method_list_dict[name] = IterateOver(name) + dictionary_for_type_call.update(method_list_dict) + # Construct a dictionary with the methods inspected from the interface + if interface is not None: + ########## + # python@2.7: interface_methods = {name: method for name, method in inspect.getmembers(interface, predicate=no_special_no_private)} + interface_methods = {} + for name, method in inspect.getmembers(interface, predicate=no_special_no_private): + interface_methods[name] = method + ########## + # python@2.7: interface_methods_dict = {name: IterateOver(name, method) for name, method in interface_methods.iteritems()} + interface_methods_dict = {} + for name, method in interface_methods.iteritems(): + interface_methods_dict[name] = IterateOver(name, method) + ########## + dictionary_for_type_call.update(interface_methods_dict) + # Get the methods that are defined in the scope of the composite class and override any previous definition + ########## + # python@2.7: cls_method = {name: method for name, method in inspect.getmembers(cls, predicate=inspect.ismethod)} + cls_method = {} + for name, method in inspect.getmembers(cls, predicate=inspect.ismethod): + cls_method[name] = method + ########## + dictionary_for_type_call.update(cls_method) + # Generate the new class on the fly and return it + # FIXME : inherit from interface if we start to use ABC classes? + wrapper_class = type(cls.__name__, (cls, container), dictionary_for_type_call) + return wrapper_class + + return cls_decorator diff --git a/lib/spack/spack/util/web.py b/lib/spack/spack/util/web.py index e26daef296..73f4858b02 100644 --- a/lib/spack/spack/util/web.py +++ b/lib/spack/spack/util/web.py @@ -86,12 +86,12 @@ def _spider(args): if not "Content-type" in resp.headers: tty.debug("ignoring page " + url) - return pages + return pages, links if not resp.headers["Content-type"].startswith('text/html'): tty.debug("ignoring page " + url + " with content type " + resp.headers["Content-type"]) - return pages + return pages, links # Do the real GET request when we know it's just HTML. req.get_method = lambda: "GET" @@ -173,7 +173,7 @@ def spider(root_url, **kwargs): performance over a sequential fetch. """ max_depth = kwargs.setdefault('depth', 1) - pages, links = _spider((root_url, set(), root_url, None, 1, max_depth, False)) + pages, links = _spider((root_url, set(), root_url, None, 1, max_depth, False)) return pages, links diff --git a/share/spack/setup-env.sh b/share/spack/setup-env.sh index 586a5b836b..764af68400 100755 --- a/share/spack/setup-env.sh +++ b/share/spack/setup-env.sh @@ -141,7 +141,7 @@ function _spack_pathadd { fi # Do the actual prepending here. - eval "_pa_oldvalue=\$${_pa_varname}" + eval "_pa_oldvalue=\${${_pa_varname}:-}" if [ -d "$_pa_new_path" ] && [[ ":$_pa_oldvalue:" != *":$_pa_new_path:"* ]]; then if [ -n "$_pa_oldvalue" ]; then diff --git a/var/spack/repos/builtin.mock/packages/externalprereq/package.py b/var/spack/repos/builtin.mock/packages/externalprereq/package.py new file mode 100644 index 0000000000..7d63925693 --- /dev/null +++ b/var/spack/repos/builtin.mock/packages/externalprereq/package.py @@ -0,0 +1,34 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + +class Externalprereq(Package): + homepage = "http://somewhere.com" + url = "http://somewhere.com/prereq-1.0.tar.gz" + + version('1.4', 'f1234567890abcdef1234567890abcde') + + def install(self, spec, prefix): + pass diff --git a/var/spack/repos/builtin.mock/packages/externaltest/package.py b/var/spack/repos/builtin.mock/packages/externaltest/package.py new file mode 100644 index 0000000000..c546922f87 --- /dev/null +++ b/var/spack/repos/builtin.mock/packages/externaltest/package.py @@ -0,0 +1,37 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + +class Externaltest(Package): + homepage = "http://somewhere.com" + url = "http://somewhere.com/test-1.0.tar.gz" + + version('1.0', '1234567890abcdef1234567890abcdef') + + depends_on('stuff') + depends_on('externaltool') + + def install(self, spec, prefix): + pass diff --git a/var/spack/repos/builtin.mock/packages/externaltool/package.py b/var/spack/repos/builtin.mock/packages/externaltool/package.py new file mode 100644 index 0000000000..af902bd70e --- /dev/null +++ b/var/spack/repos/builtin.mock/packages/externaltool/package.py @@ -0,0 +1,36 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + +class Externaltool(Package): + homepage = "http://somewhere.com" + url = "http://somewhere.com/tool-1.0.tar.gz" + + version('1.0', '1234567890abcdef1234567890abcdef') + + depends_on('externalprereq') + + def install(self, spec, prefix): + pass diff --git a/var/spack/repos/builtin.mock/packages/externalvirtual/package.py b/var/spack/repos/builtin.mock/packages/externalvirtual/package.py new file mode 100644 index 0000000000..722c1e1c53 --- /dev/null +++ b/var/spack/repos/builtin.mock/packages/externalvirtual/package.py @@ -0,0 +1,37 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + +class Externalvirtual(Package): + homepage = "http://somewhere.com" + url = "http://somewhere.com/stuff-1.0.tar.gz" + + version('1.0', '1234567890abcdef1234567890abcdef') + version('2.0', '234567890abcdef1234567890abcdef1') + + provides('stuff') + + def install(self, spec, prefix): + pass diff --git a/var/spack/repos/builtin.mock/packages/hypre/package.py b/var/spack/repos/builtin.mock/packages/hypre/package.py new file mode 100644 index 0000000000..f69f16d2cc --- /dev/null +++ b/var/spack/repos/builtin.mock/packages/hypre/package.py @@ -0,0 +1,39 @@ +############################################################################## +# Copyright (c) 2013-2015, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + +class Hypre(Package): + """Hypre is included here as an example of a package that depends on + both LAPACK and BLAS.""" + homepage = "http://www.openblas.net" + url = "http://github.com/xianyi/OpenBLAS/archive/v0.2.15.tar.gz" + + version('0.2.15', 'b1190f3d3471685f17cfd1ec1d252ac9') + + depends_on('lapack') + depends_on('blas') + + def install(self, spec, prefix): + pass diff --git a/var/spack/repos/builtin.mock/packages/mpich/package.py b/var/spack/repos/builtin.mock/packages/mpich/package.py index 55bf97f2cf..836d9c4a9f 100644 --- a/var/spack/repos/builtin.mock/packages/mpich/package.py +++ b/var/spack/repos/builtin.mock/packages/mpich/package.py @@ -38,6 +38,7 @@ class Mpich(Package): version('3.0.2', 'foobarbaz') version('3.0.1', 'foobarbaz') version('3.0', 'foobarbaz') + version('1.0', 'foobarbas') provides('mpi@:3', when='@3:') provides('mpi@:1', when='@:1') diff --git a/var/spack/repos/builtin.mock/packages/openblas-with-lapack/package.py b/var/spack/repos/builtin.mock/packages/openblas-with-lapack/package.py new file mode 100644 index 0000000000..509bfb71e5 --- /dev/null +++ b/var/spack/repos/builtin.mock/packages/openblas-with-lapack/package.py @@ -0,0 +1,38 @@ +############################################################################## +# Copyright (c) 2013-2015, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + +class OpenblasWithLapack(Package): + """Dummy version of OpenBLAS that also provides LAPACK, for testing.""" + homepage = "http://www.openblas.net" + url = "http://github.com/xianyi/OpenBLAS/archive/v0.2.15.tar.gz" + + version('0.2.15', 'b1190f3d3471685f17cfd1ec1d252ac9') + + provides('lapack') + provides('blas') + + def install(self, spec, prefix): + pass diff --git a/var/spack/repos/builtin/packages/arpack-ng/package.py b/var/spack/repos/builtin/packages/arpack-ng/package.py new file mode 100644 index 0000000000..614071cf53 --- /dev/null +++ b/var/spack/repos/builtin/packages/arpack-ng/package.py @@ -0,0 +1,61 @@ +from spack import * + + +class ArpackNg(Package): + """ + ARPACK-NG is a collection of Fortran77 subroutines designed to solve large scale eigenvalue problems. + + Important Features: + + * Reverse Communication Interface. + * Single and Double Precision Real Arithmetic Versions for Symmetric, + Non-symmetric, Standard or Generalized Problems. + * Single and Double Precision Complex Arithmetic Versions for Standard or + Generalized Problems. + * Routines for Banded Matrices - Standard or Generalized Problems. + * Routines for The Singular Value Decomposition. + * Example driver routines that may be used as templates to implement numerous + Shift-Invert strategies for all problem types, data types and precision. + + This project is a joint project between Debian, Octave and Scilab in order to + provide a common and maintained version of arpack. + + Indeed, no single release has been published by Rice university for the last + few years and since many software (Octave, Scilab, R, Matlab...) forked it and + implemented their own modifications, arpack-ng aims to tackle this by providing + a common repository and maintained versions. + + arpack-ng is replacing arpack almost everywhere. + """ + homepage = 'https://github.com/opencollab/arpack-ng' + url = 'https://github.com/opencollab/arpack-ng/archive/3.3.0.tar.gz' + + version('3.3.0', 'ed3648a23f0a868a43ef44c97a21bad5') + + variant('shared', default=True, description='Enables the build of shared libraries') + variant('mpi', default=False, description='Activates MPI support') + + # The function pdlamch10 does not set the return variable. This is fixed upstream + # see https://github.com/opencollab/arpack-ng/issues/34 + patch('pdlamch10.patch', when='@3.3:') + + depends_on('blas') + depends_on('lapack') + depends_on('mpi', when='+mpi') + + def install(self, spec, prefix): + # Apparently autotools are not bootstrapped + bootstrap = Executable('./bootstrap') + + options = ['--prefix=%s' % prefix] + + if '+mpi' in spec: + options.append('--enable-mpi') + + if '~shared' in spec: + options.append('--enable-shared=no') + + bootstrap() + configure(*options) + make() + make('install') diff --git a/var/spack/repos/builtin/packages/arpack-ng/pdlamch10.patch b/var/spack/repos/builtin/packages/arpack-ng/pdlamch10.patch new file mode 100644 index 0000000000..922828909f --- /dev/null +++ b/var/spack/repos/builtin/packages/arpack-ng/pdlamch10.patch @@ -0,0 +1,15 @@ +diff --git a/PARPACK/SRC/MPI/pdlamch10.f b/PARPACK/SRC/MPI/pdlamch10.f +index 6571da9..2882c2e 100644 +--- a/PARPACK/SRC/MPI/pdlamch10.f ++++ b/PARPACK/SRC/MPI/pdlamch10.f +@@ -86,8 +86,8 @@ + TEMP = TEMP1 + END IF + * +- PDLAMCH = TEMP ++ PDLAMCH10 = TEMP + * +-* End of PDLAMCH ++* End of PDLAMCH10 + * + END diff --git a/var/spack/repos/builtin/packages/autoconf/package.py b/var/spack/repos/builtin/packages/autoconf/package.py index 5189faf054..6412e810a6 100644 --- a/var/spack/repos/builtin/packages/autoconf/package.py +++ b/var/spack/repos/builtin/packages/autoconf/package.py @@ -6,6 +6,7 @@ class Autoconf(Package): url = "http://ftp.gnu.org/gnu/autoconf/autoconf-2.69.tar.gz" version('2.69', '82d05e03b93e45f5a39b828dc9c6c29b') + version('2.62', '6c1f3b3734999035d77da5024aab4fbd') def install(self, spec, prefix): configure("--prefix=%s" % prefix) diff --git a/var/spack/repos/builtin/packages/automake/package.py b/var/spack/repos/builtin/packages/automake/package.py index 9115822730..2172a42030 100644 --- a/var/spack/repos/builtin/packages/automake/package.py +++ b/var/spack/repos/builtin/packages/automake/package.py @@ -5,7 +5,9 @@ class Automake(Package): homepage = "http://www.gnu.org/software/automake/" url = "http://ftp.gnu.org/gnu/automake/automake-1.14.tar.gz" + version('1.15', '716946a105ca228ab545fc37a70df3a3') version('1.14.1', 'd052a3e884631b9c7892f2efce542d75') + version('1.11.6', '0286dc30295b62985ca51919202ecfcc') depends_on('autoconf') diff --git a/var/spack/repos/builtin/packages/binutils/package.py b/var/spack/repos/builtin/packages/binutils/package.py index de04221e33..897539a439 100644 --- a/var/spack/repos/builtin/packages/binutils/package.py +++ b/var/spack/repos/builtin/packages/binutils/package.py @@ -4,10 +4,13 @@ class Binutils(Package): """GNU binutils, which contain the linker, assembler, objdump and others""" homepage = "http://www.gnu.org/software/binutils/" - version('2.25', 'd9f3303f802a5b6b0bb73a335ab89d66',url="ftp://ftp.gnu.org/gnu/binutils/binutils-2.25.tar.bz2") - version('2.24', 'e0f71a7b2ddab0f8612336ac81d9636b',url="ftp://ftp.gnu.org/gnu/binutils/binutils-2.24.tar.bz2") - version('2.23.2', '4f8fa651e35ef262edc01d60fb45702e',url="ftp://ftp.gnu.org/gnu/binutils/binutils-2.23.2.tar.bz2") - version('2.20.1', '2b9dc8f2b7dbd5ec5992c6e29de0b764',url="ftp://ftp.gnu.org/gnu/binutils/binutils-2.20.1.tar.bz2") + url="https://ftp.gnu.org/gnu/binutils/binutils-2.25.tar.bz2" + + version('2.26', '64146a0faa3b411ba774f47d41de239f') + version('2.25', 'd9f3303f802a5b6b0bb73a335ab89d66') + version('2.24', 'e0f71a7b2ddab0f8612336ac81d9636b') + version('2.23.2', '4f8fa651e35ef262edc01d60fb45702e') + version('2.20.1', '2b9dc8f2b7dbd5ec5992c6e29de0b764') # Add a patch that creates binutils libiberty_pic.a which is preferred by OpenSpeedShop and cbtf-krell variant('krellpatch', default=False, description="build with openspeedshop based patch.") diff --git a/var/spack/repos/builtin/packages/blitz/package.py b/var/spack/repos/builtin/packages/blitz/package.py new file mode 100644 index 0000000000..9413b276fe --- /dev/null +++ b/var/spack/repos/builtin/packages/blitz/package.py @@ -0,0 +1,15 @@ +from spack import * + +class Blitz(Package): + """N-dimensional arrays for C++""" + homepage = "http://github.com/blitzpp/blitz" + url = "https://github.com/blitzpp/blitz/tarball/1.0.0" + + version('1.0.0', '9f040b9827fe22228a892603671a77af') + + # No dependencies + + def install(self, spec, prefix): + configure('--prefix=%s' % prefix) + make() + make("install") diff --git a/var/spack/repos/builtin/packages/boost/boost_11856.patch b/var/spack/repos/builtin/packages/boost/boost_11856.patch new file mode 100644 index 0000000000..3b4052ca18 --- /dev/null +++ b/var/spack/repos/builtin/packages/boost/boost_11856.patch @@ -0,0 +1,34 @@ +--- a/libs/container/src/pool_resource.cpp 2015-11-06 12:49:55.000000000 -0800 ++++ b/libs/container/src/pool_resource.cpp 2015-12-22 07:54:36.202131121 -0800 +@@ -32,11 +32,11 @@ + class pool_data_t + : public block_slist_base<> + { +- typedef block_slist_base<> block_slist_base; ++ typedef block_slist_base<> block_slist_base_t; + + public: + explicit pool_data_t(std::size_t initial_blocks_per_chunk) +- : block_slist_base(), next_blocks_per_chunk(initial_blocks_per_chunk) ++ : block_slist_base_t(), next_blocks_per_chunk(initial_blocks_per_chunk) + { slist_algo::init_header(&free_slist); } + + void *allocate_block() BOOST_NOEXCEPT +@@ -59,7 +59,7 @@ + void release(memory_resource &upstream) + { + slist_algo::init_header(&free_slist); +- this->block_slist_base::release(upstream); ++ this->block_slist_base_t::release(upstream); + next_blocks_per_chunk = pool_options_minimum_max_blocks_per_chunk; + } + +@@ -72,7 +72,7 @@ + + //Minimum block size is at least max_align, so all pools allocate sizes that are multiple of max_align, + //meaning that all blocks are max_align-aligned. +- char *p = static_cast<char *>(block_slist_base::allocate(blocks_per_chunk*pool_block, mr)); ++ char *p = static_cast<char *>(block_slist_base_t::allocate(blocks_per_chunk*pool_block, mr)); + + //Create header types. This is no-throw + for(std::size_t i = 0, max = blocks_per_chunk; i != max; ++i){ diff --git a/var/spack/repos/builtin/packages/boost/package.py b/var/spack/repos/builtin/packages/boost/package.py index 3427b74ad6..fb1f5daee7 100644 --- a/var/spack/repos/builtin/packages/boost/package.py +++ b/var/spack/repos/builtin/packages/boost/package.py @@ -1,4 +1,5 @@ from spack import * +import spack class Boost(Package): """Boost provides free peer-reviewed portable C++ source @@ -44,15 +45,50 @@ class Boost(Package): version('1.34.1', '2d938467e8a448a2c9763e0a9f8ca7e5') version('1.34.0', 'ed5b9291ffad776f8757a916e1726ad0') + default_install_libs = set(['atomic', + 'chrono', + 'date_time', + 'filesystem', + 'graph', + 'iostreams', + 'locale', + 'log', + 'math', + 'program_options', + 'random', + 'regex', + 'serialization', + 'signals', + 'system', + 'test', + 'thread', + 'wave']) + + # mpi/python are not installed by default because they pull in many + # dependencies and/or because there is a great deal of customization + # possible (and it would be difficult to choose sensible defaults) + default_noinstall_libs = set(['mpi', 'python']) + + all_libs = default_install_libs | default_noinstall_libs + + for lib in all_libs: + variant(lib, default=(lib not in default_noinstall_libs), + description="Compile with {0} library".format(lib)) + variant('debug', default=False, description='Switch to the debug version of Boost') - variant('python', default=False, description='Activate the component Boost.Python') - variant('mpi', default=False, description='Activate the component Boost.MPI') - variant('compression', default=True, description='Activate the compression Boost.iostreams') + variant('shared', default=True, description="Additionally build shared libraries") + variant('multithreaded', default=True, description="Build multi-threaded versions of libraries") + variant('singlethreaded', default=True, description="Build single-threaded versions of libraries") + variant('icu_support', default=False, description="Include ICU support (for regex/locale libraries)") + depends_on('icu', when='+icu_support') depends_on('python', when='+python') depends_on('mpi', when='+mpi') - depends_on('bzip2', when='+compression') - depends_on('zlib', when='+compression') + depends_on('bzip2', when='+iostreams') + depends_on('zlib', when='+iostreams') + + # Patch fix from https://svn.boost.org/trac/boost/ticket/11856 + patch('boost_11856.patch', when='@1.60.0%gcc@4.4.7') def url_for_version(self, version): """Handle Boost's weird URLs, which write the version two different ways.""" @@ -77,22 +113,20 @@ class Boost(Package): # fallback to gcc if no toolset found return 'gcc' - def determine_bootstrap_options(self, spec, options): - options.append('--with-toolset=%s' % self.determine_toolset(spec)) + def determine_bootstrap_options(self, spec, withLibs, options): + boostToolsetId = self.determine_toolset(spec) + options.append('--with-toolset=%s' % boostToolsetId) + options.append("--with-libraries=%s" % ','.join(withLibs)) - without_libs = [] - if '~mpi' in spec: - without_libs.append('mpi') - if '~python' in spec: - without_libs.append('python') - else: + if '+python' in spec: options.append('--with-python=%s' % join_path(spec['python'].prefix.bin, 'python')) - if without_libs: - options.append('--without-libraries=%s' % ','.join(without_libs)) - with open('user-config.jam', 'w') as f: + compiler_wrapper = join_path(spack.build_env_path, 'c++') + f.write("using {0} : : {1} ;\n".format(boostToolsetId, + compiler_wrapper)) + if '+mpi' in spec: f.write('using mpi : %s ;\n' % join_path(spec['mpi'].prefix.bin, 'mpicxx')) @@ -107,12 +141,10 @@ class Boost(Package): else: options.append('variant=release') - if '~compression' in spec: - options.extend([ - '-s', 'NO_BZIP2=1', - '-s', 'NO_ZLIB=1']) + if '+icu_support' in spec: + options.extend(['-s', 'ICU_PATH=%s' % spec['icu'].prefix]) - if '+compression' in spec: + if '+iostreams' in spec: options.extend([ '-s', 'BZIP2_INCLUDE=%s' % spec['bzip2'].prefix.include, '-s', 'BZIP2_LIBPATH=%s' % spec['bzip2'].prefix.lib, @@ -120,20 +152,46 @@ class Boost(Package): '-s', 'ZLIB_LIBPATH=%s' % spec['zlib'].prefix.lib, ]) + linkTypes = ['static'] + if '+shared' in spec: + linkTypes.append('shared') + + threadingOpts = [] + if '+multithreaded' in spec: + threadingOpts.append('multi') + if '+singlethreaded' in spec: + threadingOpts.append('single') + if not threadingOpts: + raise RuntimeError("At least one of {singlethreaded, multithreaded} must be enabled") + options.extend([ 'toolset=%s' % self.determine_toolset(spec), - 'link=static,shared', - 'threading=single,multi', + 'link=%s' % ','.join(linkTypes), '--layout=tagged']) + + return threadingOpts def install(self, spec, prefix): + withLibs = list() + for lib in Boost.all_libs: + if "+{0}".format(lib) in spec: + withLibs.append(lib) + if not withLibs: + # if no libraries are specified for compilation, then you dont have + # to configure/build anything, just copy over to the prefix directory. + src = join_path(self.stage.source_path, 'boost') + mkdirp(join_path(prefix, 'include')) + dst = join_path(prefix, 'include', 'boost') + install_tree(src, dst) + return + # to make Boost find the user-config.jam env['BOOST_BUILD_PATH'] = './' bootstrap = Executable('./bootstrap.sh') bootstrap_options = ['--prefix=%s' % prefix] - self.determine_bootstrap_options(spec, bootstrap_options) + self.determine_bootstrap_options(spec, withLibs, bootstrap_options) bootstrap(*bootstrap_options) @@ -143,6 +201,10 @@ class Boost(Package): b2 = Executable(b2name) b2_options = ['-j', '%s' % make_jobs] - self.determine_b2_options(spec, b2_options) + threadingOpts = self.determine_b2_options(spec, b2_options) - b2('install', *b2_options) + # In theory it could be done on one call but it fails on + # Boost.MPI if the threading options are not separated. + for threadingOpt in threadingOpts: + b2('install', 'threading=%s' % threadingOpt, *b2_options) + diff --git a/var/spack/repos/builtin/packages/caliper/package.py b/var/spack/repos/builtin/packages/caliper/package.py new file mode 100644 index 0000000000..d51b4a4dd5 --- /dev/null +++ b/var/spack/repos/builtin/packages/caliper/package.py @@ -0,0 +1,25 @@ +from spack import * + +class Caliper(Package): + """ + Caliper is a generic context annotation system. It gives programmers the + ability to provide arbitrary program context information to (performance) + tools at runtime. + """ + + homepage = "https://github.com/LLNL/Caliper" + url = "" + + version('master', git='ssh://git@github.com:LLNL/Caliper.git') + + variant('mpi', default=False, description='Enable MPI function wrappers.') + + depends_on('libunwind') + depends_on('papi') + depends_on('mpi', when='+mpi') + + def install(self, spec, prefix): + with working_dir('build', create=True): + cmake('..', *std_cmake_args) + make() + make("install") diff --git a/var/spack/repos/builtin/packages/cgal/package.py b/var/spack/repos/builtin/packages/cgal/package.py index 97356433be..ef4a2736db 100644 --- a/var/spack/repos/builtin/packages/cgal/package.py +++ b/var/spack/repos/builtin/packages/cgal/package.py @@ -46,6 +46,7 @@ class Cgal(Package): depends_on('mpfr') depends_on('gmp') depends_on('zlib') + depends_on('cmake') # FIXME : Qt5 dependency missing (needs Qt5 and OpenGL) # FIXME : Optional third party libraries missing diff --git a/var/spack/repos/builtin/packages/cmake/package.py b/var/spack/repos/builtin/packages/cmake/package.py index f67ae21ebd..cc93c7067c 100644 --- a/var/spack/repos/builtin/packages/cmake/package.py +++ b/var/spack/repos/builtin/packages/cmake/package.py @@ -28,23 +28,58 @@ class Cmake(Package): """A cross-platform, open-source build system. CMake is a family of tools designed to build, test and package software.""" homepage = 'https://www.cmake.org' + url = 'https://cmake.org/files/v3.4/cmake-3.4.3.tar.gz' - version('2.8.10.2', '097278785da7182ec0aea8769d06860c', - url = 'http://www.cmake.org/files/v2.8/cmake-2.8.10.2.tar.gz') - - version('3.0.2', 'db4c687a31444a929d2fdc36c4dfb95f', - url = 'http://www.cmake.org/files/v3.0/cmake-3.0.2.tar.gz') - - version('3.4.0', 'cd3034e0a44256a0917e254167217fc8', - url = 'http://cmake.org/files/v3.4/cmake-3.4.0.tar.gz') + version('3.5.0', '33c5d09d4c33d4ffcc63578a6ba8777e') + version('3.4.3', '4cb3ff35b2472aae70f542116d616e63') + version('3.4.0', 'cd3034e0a44256a0917e254167217fc8') + version('3.3.1', '52638576f4e1e621fed6c3410d3a1b12') + version('3.0.2', 'db4c687a31444a929d2fdc36c4dfb95f') + version('2.8.10.2', '097278785da7182ec0aea8769d06860c') variant('ncurses', default=True, description='Enables the build of the ncurses gui') + variant('qt', default=False, description='Enables the build of cmake-gui') + variant('doc', default=False, description='Enables the generation of html and man page documentation') depends_on('ncurses', when='+ncurses') + depends_on('qt', when='+qt') + depends_on('python@2.7.11:', when='+doc') + depends_on('py-sphinx', when='+doc') + + def url_for_version(self, version): + """Handle CMake's version-based custom URLs.""" + return 'https://cmake.org/files/v%s/cmake-%s.tar.gz' % (version.up_to(2), version) + + def validate(self, spec): + """ + Checks if incompatible versions of qt were specified + + :param spec: spec of the package + :raises RuntimeError: in case of inconsistencies + """ + + if '+qt' in spec and spec.satisfies('^qt@5.4.0'): + msg = 'qt-5.4.0 has broken CMake modules.' + raise RuntimeError(msg) def install(self, spec, prefix): - configure('--prefix=' + prefix, - '--parallel=' + str(make_jobs), - '--', '-DCMAKE_USE_OPENSSL=ON') + # Consistency check + self.validate(spec) + + # configure, build, install: + options = ['--prefix=%s' % prefix] + options.append('--parallel=%s' % str(make_jobs)) + + if '+qt' in spec: + options.append('--qt-gui') + + if '+doc' in spec: + options.append('--sphinx-html') + options.append('--sphinx-man') + + options.append('--') + options.append('-DCMAKE_USE_OPENSSL=ON') + + configure(*options) make() make('install') diff --git a/var/spack/repos/builtin/packages/cmocka/package.py b/var/spack/repos/builtin/packages/cmocka/package.py new file mode 100644 index 0000000000..7377016a6b --- /dev/null +++ b/var/spack/repos/builtin/packages/cmocka/package.py @@ -0,0 +1,16 @@ +from spack import * + +class Cmocka(Package): + """Unit-testing framework in pure C""" + homepage = "https://cmocka.org/" + url = "https://cmocka.org/files/1.0/cmocka-1.0.1.tar.xz" + + version('1.0.1', 'ed861e501a21a92b2af63e466df2015e') + parallel = False + + def install(self, spec, prefix): + with working_dir('spack-build', create=True): + cmake('..', *std_cmake_args) + + make() + make("install") diff --git a/var/spack/repos/builtin/packages/cryptopp/package.py b/var/spack/repos/builtin/packages/cryptopp/package.py new file mode 100644 index 0000000000..1693c4b160 --- /dev/null +++ b/var/spack/repos/builtin/packages/cryptopp/package.py @@ -0,0 +1,31 @@ +import glob +from spack import * + +class Cryptopp(Package): + """Crypto++ is an open-source C++ library of cryptographic schemes. The + library supports a number of different cryptography algorithms, including + authenticated encryption schemes (GCM, CCM), hash functions (SHA-1, SHA2), + public-key encryption (RSA, DSA), and a few obsolete/historical encryption + algorithms (MD5, Panama).""" + + homepage = "http://www.cryptopp.com/" + url = "http://www.cryptopp.com/cryptopp563.zip" + + version('5.6.3', '3c5b70e2ec98b7a24988734446242d07') + version('5.6.2', '7ed022585698df48e65ce9218f6c6a67') + + def install(self, spec, prefix): + make() + + mkdirp(prefix.include) + for hfile in glob.glob('*.h*'): + install(hfile, prefix.include) + + mkdirp(prefix.lib) + install('libcryptopp.a', prefix.lib) + + def url_for_version(self, version): + version_tuple = tuple(v for v in iter(version)) + version_string = reduce(lambda vs, nv: vs + str(nv), version_tuple, "") + + return "%scryptopp%s.zip" % (Cryptopp.homepage, version_string) diff --git a/var/spack/repos/builtin/packages/curl/package.py b/var/spack/repos/builtin/packages/curl/package.py index 9e684445c7..ab6305fc08 100644 --- a/var/spack/repos/builtin/packages/curl/package.py +++ b/var/spack/repos/builtin/packages/curl/package.py @@ -7,6 +7,7 @@ class Curl(Package): homepage = "http://curl.haxx.se" url = "http://curl.haxx.se/download/curl-7.46.0.tar.bz2" + version('7.47.1', '9ea3123449439bbd960cd25cf98796fb') version('7.46.0', '9979f989a2a9930d10f1b3deeabc2148') version('7.45.0', '62c1a352b28558f25ba6209214beadc8') version('7.44.0', '6b952ca00e5473b16a11f05f06aa8dae') diff --git a/var/spack/repos/builtin/packages/dakota/package.py b/var/spack/repos/builtin/packages/dakota/package.py new file mode 100644 index 0000000000..5ec82ef83d --- /dev/null +++ b/var/spack/repos/builtin/packages/dakota/package.py @@ -0,0 +1,55 @@ +from spack import * + + +class Dakota(Package): + """ + The Dakota toolkit provides a flexible, extensible interface between analysis codes and iterative systems + analysis methods. Dakota contains algorithms for: + + - optimization with gradient and non gradient-based methods; + - uncertainty quantification with sampling, reliability, stochastic expansion, and epistemic methods; + - parameter estimation with nonlinear least squares methods; + - sensitivity/variance analysis with design of experiments and parameter study methods. + + These capabilities may be used on their own or as components within advanced strategies such as hybrid optimization, + surrogate-based optimization, mixed integer nonlinear programming, or optimization under uncertainty. + """ + + homepage = 'https://dakota.sandia.gov/' + url = 'https://dakota.sandia.gov/sites/default/files/distributions/public/dakota-6.3-public.src.tar.gz' + _url_str = 'https://dakota.sandia.gov/sites/default/files/distributions/public/dakota-{version}-public.src.tar.gz' + + version('6.3', '05a58d209fae604af234c894c3f73f6d') + + variant('debug', default=False, description='Builds a debug version of the libraries') + variant('shared', default=True, description='Enables the build of shared libraries') + variant('mpi', default=True, description='Activates MPI support') + + depends_on('blas') + depends_on('lapack') + depends_on('mpi', when='+mpi') + + depends_on('python') + depends_on('boost') + + def url_for_version(self, version): + return Dakota._url_str.format(version=version) + + def install(self, spec, prefix): + options = [] + options.extend(std_cmake_args) + + options.extend(['-DCMAKE_BUILD_TYPE:STRING=%s' % ('Debug' if '+debug' in spec else 'Release'), + '-DBUILD_SHARED_LIBS:BOOL=%s' % ('ON' if '+shared' in spec else 'OFF')]) + + if '+mpi' in spec: + options.extend(['-DDAKOTA_HAVE_MPI:BOOL=ON', + '-DMPI_CXX_COMPILER:STRING=%s' % join_path(spec['mpi'].prefix.bin, 'mpicxx')]) + + build_directory = join_path(self.stage.path, 'spack-build') + source_directory = self.stage.source_path + + with working_dir(build_directory, create=True): + cmake(source_directory, *options) + make() + make("install") diff --git a/var/spack/repos/builtin/packages/eigen/package.py b/var/spack/repos/builtin/packages/eigen/package.py new file mode 100644 index 0000000000..8d6e672f86 --- /dev/null +++ b/var/spack/repos/builtin/packages/eigen/package.py @@ -0,0 +1,69 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by David Beckingsale, david@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## + +from spack import * + + +class Eigen(Package): + """ + Eigen is a C++ template library for linear algebra: matrices, vectors, numerical solvers, and related algorithms + """ + + homepage = 'http://eigen.tuxfamily.org/' + url = 'http://bitbucket.org/eigen/eigen/get/3.2.7.tar.bz2' + + version('3.2.7', 'cc1bacbad97558b97da6b77c9644f184', url='http://bitbucket.org/eigen/eigen/get/3.2.7.tar.bz2') + + variant('debug', default=False, description='Builds the library in debug mode') + + variant('metis', default=True, description='Enables metis backend') + variant('scotch', default=True, description='Enables scotch backend') + variant('fftw', default=True, description='Enables FFTW backend') + variant('suitesparse', default=True, description='Enables SuiteSparse support') + + # TODO : dependency on googlehash, superlu, adolc missing + + depends_on('metis', when='+metis') + depends_on('scotch', when='+scotch') + depends_on('fftw', when='+fftw') + depends_on('suite-sparse', when='+suitesparse') + depends_on('mpfr@2.3.0:') # Eigen 3.2.7 requires at least 2.3.0 + depends_on('gmp') + + def install(self, spec, prefix): + + options = [] + options.extend(std_cmake_args) + + build_directory = join_path(self.stage.path, 'spack-build') + source_directory = self.stage.source_path + + if '+debug' in spec: + options.append('-DCMAKE_BUILD_TYPE:STRING=Debug') + + with working_dir(build_directory, create=True): + cmake(source_directory, *options) + make() + make("install") diff --git a/var/spack/repos/builtin/packages/elpa/package.py b/var/spack/repos/builtin/packages/elpa/package.py new file mode 100644 index 0000000000..2ade5b0b37 --- /dev/null +++ b/var/spack/repos/builtin/packages/elpa/package.py @@ -0,0 +1,55 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## + +from spack import * + + +class Elpa(Package): + """ + Eigenvalue solvers for Petaflop-Applications (ELPA) + """ + + homepage = 'http://elpa.mpcdf.mpg.de/' + url = 'http://elpa.mpcdf.mpg.de/elpa-2015.11.001.tar.gz' + + version('2015.11.001', 'de0f35b7ee7c971fd0dca35c900b87e6', url='http://elpa.mpcdf.mpg.de/elpa-2015.11.001.tar.gz') + + variant('openmp', default=False, description='Activates OpenMP support') + + depends_on('mpi') + depends_on('blas') + depends_on('lapack') + depends_on('scalapack') + + def install(self, spec, prefix): + + options = ["--prefix=%s" % prefix] + + if '+openmp' in spec: + options.append("--enable-openmp") + + configure(*options) + make() + make("install") diff --git a/var/spack/repos/builtin/packages/emacs/package.py b/var/spack/repos/builtin/packages/emacs/package.py new file mode 100644 index 0000000000..caa264857e --- /dev/null +++ b/var/spack/repos/builtin/packages/emacs/package.py @@ -0,0 +1,21 @@ +from spack import * + +class Emacs(Package): + """The Emacs programmable text editor.""" + homepage = "https://www.gnu.org/software/emacs" + url = "http://ftp.gnu.org/gnu/emacs/emacs-24.5.tar.gz" + + version('24.5', 'd74b597503a68105e61b5b9f6d065b44') + + depends_on('ncurses') + # Emacs also depends on: + # GTK or other widget library + # libtiff, png, etc. + # For now, we assume the system provides all that stuff. + # For Ubuntu 14.04 LTS: + # sudo apt-get install libgtk-3-dev libxpm-dev libtiff5-dev libjpeg8-dev libgif-dev libpng12-dev + + def install(self, spec, prefix): + configure('--prefix=%s' % prefix) + make() + make("install") diff --git a/var/spack/repos/builtin/packages/espresso/package.py b/var/spack/repos/builtin/packages/espresso/package.py new file mode 100644 index 0000000000..a2bf58f585 --- /dev/null +++ b/var/spack/repos/builtin/packages/espresso/package.py @@ -0,0 +1,65 @@ +from spack import * + +import os + +class Espresso(Package): + """ + QE is an integrated suite of Open-Source computer codes for electronic-structure calculations and materials + modeling at the nanoscale. It is based on density-functional theory, plane waves, and pseudopotentials. + """ + homepage = 'http://quantum-espresso.org' + url = 'http://www.qe-forge.org/gf/download/frsrelease/204/912/espresso-5.3.0.tar.gz' + + version('5.3.0', '6848fcfaeb118587d6be36bd10b7f2c3') + + variant('mpi', default=True, description='Build Quantum-ESPRESSO with mpi support') + variant('openmp', default=False, description='Enables openMP support') + variant('scalapack', default=True, description='Enables scalapack support') + variant('elpa', default=True, description='Use elpa as an eigenvalue solver') + + depends_on('blas') + depends_on('lapack') + + depends_on('mpi', when='+mpi') + depends_on('fftw~mpi', when='~mpi') + depends_on('fftw+mpi', when='+mpi') + depends_on('scalapack', when='+scalapack+mpi') # TODO : + mpi needed to avoid false dependencies installation + + def check_variants(self, spec): + error = 'you cannot ask for \'+{variant}\' when \'+mpi\' is not active' + if '+scalapack' in spec and '~mpi' in spec: + raise RuntimeError(error.format(variant='scalapack')) + if '+elpa' in spec and ('~mpi' in spec or '~scalapack' in spec): + raise RuntimeError(error.format(variant='elpa')) + + def install(self, spec, prefix): + self.check_variants(spec) + + options = ['-prefix=%s' % prefix] + + if '+mpi' in spec: + options.append('--enable-parallel') + + if '+openmp' in spec: + options.append('--enable-openmp') + + if '+scalapack' in spec: + options.append('--with-scalapack=yes') + + if '+elpa' in spec: + options.append('--with-elpa=yes') + + # Add a list of directories to search + search_list = [] + for name, dependency_spec in spec.dependencies.iteritems(): + search_list.extend([dependency_spec.prefix.lib, + dependency_spec.prefix.lib64]) + + search_list = " ".join(search_list) + options.append('LIBDIRS=%s' % search_list) + options.append('F90=%s' % os.environ['FC']) + + configure(*options) + make('all') + make('install') + diff --git a/var/spack/repos/builtin/packages/exodusii/exodus-cmake.patch b/var/spack/repos/builtin/packages/exodusii/exodus-cmake.patch new file mode 100644 index 0000000000..25355269ca --- /dev/null +++ b/var/spack/repos/builtin/packages/exodusii/exodus-cmake.patch @@ -0,0 +1,12 @@ +diff --git a/cmake-exodus b/cmake-exodus +index 787fd9d..ed073a2 100755 +--- a/cmake-exodus ++++ b/cmake-exodus +@@ -1,4 +1,6 @@ +-EXTRA_ARGS=$@ ++#!/bin/bash ++ ++EXTRA_ARGS=-DSEACASProj_ENABLE_CXX11=OFF + + ### Change this to point to the compilers you want to use + CC=gcc diff --git a/var/spack/repos/builtin/packages/exodusii/package.py b/var/spack/repos/builtin/packages/exodusii/package.py new file mode 100644 index 0000000000..af258b7e6e --- /dev/null +++ b/var/spack/repos/builtin/packages/exodusii/package.py @@ -0,0 +1,49 @@ +from spack import * + +# TODO: Add support for a C++11 enabled installation that filters out the +# TODO: "C++11-Disabled" flag (but only if the spec compiler supports C++11). + +# TODO: Add support for parallel installation that uses MPI. + +# TODO: Create installation options for NetCDF that support larger page size +# TODO: suggested by Exodus (see the repository "README" file). + +class Exodusii(Package): + """Exodus II is a C++/Fortran library developed to store and retrieve data for + finite element analyses. It's used for preprocessing (problem definition), + postprocessing (results visualization), and data transfer between codes. + An Exodus II data file is a random access, machine independent, binary + file that is written and read via C, C++, or Fortran API routines.""" + + homepage = "https://github.com/gsjaardema/seacas" + url = "https://github.com/gsjaardema/seacas/archive/master.zip" + + version('2016-02-08', git='https://github.com/gsjaardema/seacas.git', commit='dcf3529') + + # TODO: Make this a build dependency once build dependencies are supported + # (see: https://github.com/LLNL/spack/pull/378). + depends_on('cmake@2.8.7:') + depends_on('hdf5~shared~mpi') + depends_on('netcdf~mpi') + + patch('exodus-cmake.patch') + + def patch(self): + ff = FileFilter('cmake-exodus') + + ff.filter('CMAKE_INSTALL_PREFIX:PATH=${ACCESS}', + 'CMAKE_INSTALL_PREFIX:PATH=%s' % self.spec.prefix, string=True) + ff.filter('NetCDF_DIR:PATH=${TPL}', + 'NetCDF_DIR:PATH=%s' % self.spec['netcdf'].prefix, string=True) + ff.filter('HDF5_ROOT:PATH=${TPL}', + 'HDF5_ROOT:PATH=%s' % self.spec['hdf5'].prefix, string=True) + + def install(self, spec, prefix): + mkdirp('build') + cd('build') + + cmake_exodus = Executable('../cmake-exodus') + cmake_exodus() + + make() + make('install') diff --git a/var/spack/repos/builtin/packages/expat/package.py b/var/spack/repos/builtin/packages/expat/package.py index 082da5bf0b..3f925c6546 100644 --- a/var/spack/repos/builtin/packages/expat/package.py +++ b/var/spack/repos/builtin/packages/expat/package.py @@ -7,6 +7,7 @@ class Expat(Package): version('2.1.0', 'dd7dab7a5fea97d2a6a43f511449b7cd') + depends_on('cmake') def install(self, spec, prefix): diff --git a/var/spack/repos/builtin/packages/fftw/package.py b/var/spack/repos/builtin/packages/fftw/package.py index 4d2b964242..bc129aaf1a 100644 --- a/var/spack/repos/builtin/packages/fftw/package.py +++ b/var/spack/repos/builtin/packages/fftw/package.py @@ -47,6 +47,8 @@ class Fftw(Package): depends_on('mpi', when='+mpi') + # TODO : add support for architecture specific optimizations as soon as targets are supported + def install(self, spec, prefix): options = ['--prefix=%s' % prefix, '--enable-shared', diff --git a/var/spack/repos/builtin/packages/fish/package.py b/var/spack/repos/builtin/packages/fish/package.py index 1225558705..b5a4a2d209 100644 --- a/var/spack/repos/builtin/packages/fish/package.py +++ b/var/spack/repos/builtin/packages/fish/package.py @@ -7,7 +7,8 @@ class Fish(Package): homepage = "http://fishshell.com/" url = "http://fishshell.com/files/2.2.0/fish-2.2.0.tar.gz" - list_url = homepage + list_url = "http://fishshell.com/files/" + list_depth = 2 version('2.2.0', 'a76339fd14ce2ec229283c53e805faac48c3e99d9e3ede9d82c0554acfc7b77a') diff --git a/var/spack/repos/builtin/packages/gcc/package.py b/var/spack/repos/builtin/packages/gcc/package.py index 61b16f3fd8..f8958ee290 100644 --- a/var/spack/repos/builtin/packages/gcc/package.py +++ b/var/spack/repos/builtin/packages/gcc/package.py @@ -36,8 +36,6 @@ class Gcc(Package): list_url = 'http://open-source-box.org/gcc/' list_depth = 2 - DEPENDS_ON_ISL_PREDICATE = '@5.0:' - version('5.3.0', 'c9616fd448f980259c31de613e575719') version('5.2.0', 'a51bcfeb3da7dd4c623e27207ed43467') version('4.9.3', '6f831b4d251872736e8e9cc09746f327') @@ -50,15 +48,14 @@ class Gcc(Package): version('4.5.4', '27e459c2566b8209ab064570e1b378f7') variant('gold', default=True, description="Build the gold linker plugin for ld-based LTO") - + depends_on("mpfr") depends_on("gmp") - depends_on("mpc") # when @4.5: + depends_on("mpc", when='@4.5:') + depends_on("isl", when='@5.0:') depends_on("binutils~libiberty", when='~gold') depends_on("binutils~libiberty+gold", when='+gold') - # Save these until we can do optional deps. - depends_on("isl", when=DEPENDS_ON_ISL_PREDICATE) #depends_on("ppl") #depends_on("cloog") @@ -91,7 +88,7 @@ class Gcc(Package): "--with-as=%s/bin/as" % spec['binutils'].prefix] options.extend(binutils_options) # Isl - if spec.satisfies(Gcc.DEPENDS_ON_ISL_PREDICATE): + if 'isl' in spec: isl_options = ["--with-isl=%s" % spec['isl'].prefix] options.extend(isl_options) @@ -102,7 +99,7 @@ class Gcc(Package): configure(*options) make() make("install") - + self.write_rpath_specs() @@ -121,7 +118,7 @@ class Gcc(Package): return gcc = Executable(join_path(self.prefix.bin, 'gcc')) - lines = gcc('-dumpspecs', return_output=True).strip().split("\n") + lines = gcc('-dumpspecs', output=str).strip().split("\n") specs_file = join_path(self.spec_dir, 'specs') with closing(open(specs_file, 'w')) as out: for line in lines: diff --git a/var/spack/repos/builtin/packages/gdb/package.py b/var/spack/repos/builtin/packages/gdb/package.py index dd02b426b9..b346fe80c2 100644 --- a/var/spack/repos/builtin/packages/gdb/package.py +++ b/var/spack/repos/builtin/packages/gdb/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://llnl.github.io/spack +# For details, see https://software.llnl.gov/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/repos/builtin/packages/git/package.py b/var/spack/repos/builtin/packages/git/package.py index ddc5078c4d..586b6ce3c3 100644 --- a/var/spack/repos/builtin/packages/git/package.py +++ b/var/spack/repos/builtin/packages/git/package.py @@ -5,14 +5,22 @@ class Git(Package): system designed to handle everything from small to very large projects with speed and efficiency.""" homepage = "http://git-scm.com" - url = "https://www.kernel.org/pub/software/scm/git/git-2.2.1.tar.gz" + url = "https://github.com/git/git/tarball/v2.7.1" - version('2.6.3', 'b711be7628a4a2c25f38d859ee81b423') - version('2.6.2', 'da293290da69f45a86a311ad3cd43dc8') - version('2.6.1', '4c62ee9c5991fe93d99cf2a6b68397fd') - version('2.6.0', 'eb76a07148d94802a1745d759716a57e') - version('2.5.4', '3eca2390cf1fa698b48e2a233563a76b') - version('2.2.1', 'ff41fdb094eed1ec430aed8ee9b9849c') + version('2.8.0-rc2', 'c2cf9f2cc70e35f2fafbaf9258f82e4c') + version('2.7.3', 'fa1c008b56618c355a32ba4a678305f6') + version('2.7.1', 'bf0706b433a8dedd27a63a72f9a66060') + + + # See here for info on vulnerable Git versions: + # http://www.theregister.co.uk/2016/03/16/git_server_client_patch_now/ + # All the following are vulnerable + #version('2.6.3', 'b711be7628a4a2c25f38d859ee81b423') + #version('2.6.2', 'da293290da69f45a86a311ad3cd43dc8') + #version('2.6.1', '4c62ee9c5991fe93d99cf2a6b68397fd') + #version('2.6.0', 'eb76a07148d94802a1745d759716a57e') + #version('2.5.4', '3eca2390cf1fa698b48e2a233563a76b') + #version('2.2.1', 'ff41fdb094eed1ec430aed8ee9b9849c') # Git compiles with curl support by default on but if your system @@ -24,6 +32,7 @@ class Git(Package): variant("expat", default=False, description="Add the internal support of expat for https push") depends_on("openssl") + depends_on("autoconf") depends_on("curl", when="+curl") depends_on("expat", when="+expat") @@ -47,6 +56,7 @@ class Git(Package): if '+expat' in spec: configure_args.append("--with-expat=%s" % spec['expat'].prefix) + which('autoreconf')('-i') configure(*configure_args) make() make("install") diff --git a/var/spack/repos/builtin/packages/gl2ps/package.py b/var/spack/repos/builtin/packages/gl2ps/package.py new file mode 100644 index 0000000000..cb376b3f03 --- /dev/null +++ b/var/spack/repos/builtin/packages/gl2ps/package.py @@ -0,0 +1,18 @@ +from spack import * + +class Gl2ps(Package): + """GL2PS is a C library providing high quality vector output for any + OpenGL application.""" + + homepage = "http://www.geuz.org/gl2ps/" + url = "http://geuz.org/gl2ps/src/gl2ps-1.3.9.tgz" + + version('1.3.9', '377b2bcad62d528e7096e76358f41140') + + depends_on("libpng") + + def install(self, spec, prefix): + cmake('.', *std_cmake_args) + + make() + make("install") diff --git a/var/spack/repos/builtin/packages/gromacs/package.py b/var/spack/repos/builtin/packages/gromacs/package.py new file mode 100644 index 0000000000..5fe8399308 --- /dev/null +++ b/var/spack/repos/builtin/packages/gromacs/package.py @@ -0,0 +1,56 @@ +from spack import * + + +class Gromacs(Package): + """ + GROMACS (GROningen MAchine for Chemical Simulations) is a molecular dynamics package primarily designed for + simulations of proteins, lipids and nucleic acids. It was originally developed in the Biophysical Chemistry + department of University of Groningen, and is now maintained by contributors in universities and research centers + across the world. + + GROMACS is one of the fastest and most popular software packages available and can run on CPUs as well as GPUs. + It is free, open source released under the GNU General Public License. Starting from version 4.6, GROMACS is + released under the GNU Lesser General Public License. + """ + + homepage = 'http://www.gromacs.org' + url = 'ftp://ftp.gromacs.org/pub/gromacs/gromacs-5.1.2.tar.gz' + + version('5.1.2', '614d0be372f1a6f1f36382b7a6fcab98') + + variant('mpi', default=True, description='Activate MPI support') + variant('shared', default=True, description='Enables the build of shared libraries') + variant('debug', default=False, description='Enables debug mode') + variant('double', default=False, description='Produces a double precision version of the executables') + + depends_on('mpi', when='+mpi') + + depends_on('fftw') + + # TODO : add GPU support + + def install(self, spec, prefix): + + options = [] + + if '+mpi' in spec: + options.append('-DGMX_MPI:BOOL=ON') + + if '+double' in spec: + options.append('-DGMX_DOUBLE:BOOL=ON') + + if '~shared' in spec: + options.append('-DBUILD_SHARED_LIBS:BOOL=OFF') + + if '+debug' in spec: + options.append('-DCMAKE_BUILD_TYPE:STRING=Debug') + else: + options.append('-DCMAKE_BUILD_TYPE:STRING=Release') + + options.extend(std_cmake_args) + + with working_dir('spack-build', create=True): + + cmake('..', *options) + make() + make('install') diff --git a/var/spack/repos/builtin/packages/hdf/package.py b/var/spack/repos/builtin/packages/hdf/package.py new file mode 100644 index 0000000000..ac6435f2a2 --- /dev/null +++ b/var/spack/repos/builtin/packages/hdf/package.py @@ -0,0 +1,45 @@ +from spack import * + +class Hdf(Package): + """HDF4 (also known as HDF) is a library and multi-object + file format for storing and managing data between machines.""" + + homepage = "https://www.hdfgroup.org/products/hdf4/" + url = "https://www.hdfgroup.org/ftp/HDF/releases/HDF4.2.11/src/hdf-4.2.11.tar.gz" + list_url = "https://www.hdfgroup.org/ftp/HDF/releases/" + list_depth = 3 + + version('4.2.11', '063f9928f3a19cc21367b71c3b8bbf19') + + variant('szip', default=False, description="Enable szip support") + + depends_on("jpeg") + depends_on("szip", when='+szip') + depends_on("zlib") + + + def url_for_version(self, version): + return "https://www.hdfgroup.org/ftp/HDF/releases/HDF" + str(version) + "/src/hdf-" + str(version) + ".tar.gz" + + + def install(self, spec, prefix): + config_args = [ + 'CFLAGS=-fPIC', + '--prefix=%s' % prefix, + '--with-jpeg=%s' % spec['jpeg'].prefix, + '--with-zlib=%s' % spec['zlib'].prefix, + '--disable-netcdf', # must be disabled to build NetCDF with HDF4 support + '--enable-fortran', + '--disable-shared', # fortran and shared libraries are not compatible + '--enable-static', + '--enable-production' + ] + + # SZip support + if '+szip' in spec: + config_args.append('--with-szlib=%s' % spec['szip'].prefix) + + configure(*config_args) + + make() + make("install") diff --git a/var/spack/repos/builtin/packages/hdf5/package.py b/var/spack/repos/builtin/packages/hdf5/package.py index 9a40164341..513a38ee8a 100644 --- a/var/spack/repos/builtin/packages/hdf5/package.py +++ b/var/spack/repos/builtin/packages/hdf5/package.py @@ -1,5 +1,31 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## + from spack import * + class Hdf5(Package): """HDF5 is a data model, library, and file format for storing and managing data. It supports an unlimited variety of datatypes, and is designed for @@ -7,7 +33,7 @@ class Hdf5(Package): """ homepage = "http://www.hdfgroup.org/HDF5/" - url = "http://www.hdfgroup.org/ftp/HDF5/releases/hdf5-1.8.13/src/hdf5-1.8.13.tar.gz" + url = "http://www.hdfgroup.org/ftp/HDF5/releases/hdf5-1.8.13/src/hdf5-1.8.13.tar.gz" list_url = "http://www.hdfgroup.org/ftp/HDF5/releases" list_depth = 3 @@ -15,26 +41,64 @@ class Hdf5(Package): version('1.8.15', '03cccb5b33dbe975fdcd8ae9dc021f24') version('1.8.13', 'c03426e9e77d7766944654280b467289') + variant('debug', default=False, description='Builds a debug version of the library') + variant('shared', default=True, description='Builds a shared version of the library') + variant('cxx', default=True, description='Enable C++ support') variant('fortran', default=True, description='Enable Fortran support') + variant('mpi', default=False, description='Enable MPI support') - variant('threadsafe', default=False, description='Enable multithreading') + variant('szip', default=False, description='Enable szip support') + variant('threadsafe', default=False, description='Enable thread-safe capabilities') depends_on("mpi", when='+mpi') + depends_on("szip", when='+szip') depends_on("zlib") - # TODO: currently hard-coded to use OpenMPI + def validate(self, spec): + """ + Checks if incompatible variants have been activated at the same time + + :param spec: spec of the package + :raises RuntimeError: in case of inconsistencies + """ + if '+fortran' in spec and not self.compiler.fc: + msg = 'cannot build a fortran variant without a fortran compiler' + raise RuntimeError(msg) + + if '+threadsafe' in spec and ('+cxx' in spec or '+fortran' in spec): + raise RuntimeError("cannot use variant +threadsafe with either +cxx or +fortran") + def install(self, spec, prefix): + self.validate(spec) + # Handle compilation after spec validation extra_args = [] + + # Always enable this option. This does not actually enable any + # features: it only *allows* the user to specify certain + # combinations of other arguments. Enabling it just skips a + # sanity check in configure, so this doesn't merit a variant. + extra_args.append("--enable-unsupported") + + if '+debug' in spec: + extra_args.append('--enable-debug=all') + else: + extra_args.append('--enable-production') + + if '+shared' in spec: + extra_args.append('--enable-shared') + else: + extra_args.append('--enable-static-exec') + if '+cxx' in spec: - extra_args.extend([ - '--enable-cxx' - ]) + extra_args.append('--enable-cxx') + if '+fortran' in spec: extra_args.extend([ '--enable-fortran', '--enable-fortran2003' ]) + if '+mpi' in spec: # The HDF5 configure script warns if cxx and mpi are enabled # together. There doesn't seem to be a real reason for this, except @@ -43,27 +107,28 @@ class Hdf5(Package): # this is not actually a problem. extra_args.extend([ "--enable-parallel", - "--enable-unsupported", "CC=%s" % spec['mpi'].prefix.bin + "/mpicc", - "CXX=%s" % spec['mpi'].prefix.bin + "/mpic++", - "FC=%s" % spec['mpi'].prefix.bin + "/mpifort", ]) - if '+threads' in spec: - if '+cxx' in spec or '+fortran' in spec: - die("Cannot use variant +threads with either +cxx or +fortran") + + if '+cxx' in spec: + extra_args.append("CXX=%s" % spec['mpi'].prefix.bin + "/mpic++") + + if '+fortran' in spec: + extra_args.append("FC=%s" % spec['mpi'].prefix.bin + "/mpifort") + + if '+szip' in spec: + extra_args.append("--with-szlib=%s" % spec['szip'].prefix) + + if '+threadsafe' in spec: extra_args.extend([ '--enable-threadsafe', '--disable-hl', - 'CPPFLAGS=-DHDatexit=""', - 'CFLAGS=-DHDatexit=""' ]) configure( "--prefix=%s" % prefix, "--with-zlib=%s" % spec['zlib'].prefix, - "--enable-shared", *extra_args) - make() make("install") diff --git a/var/spack/repos/builtin/packages/hpx5/package.py b/var/spack/repos/builtin/packages/hpx5/package.py new file mode 100644 index 0000000000..3dae3c4170 --- /dev/null +++ b/var/spack/repos/builtin/packages/hpx5/package.py @@ -0,0 +1,52 @@ +from spack import * +import os + +class Hpx5(Package): + """The HPX-5 Runtime System. HPX-5 (High Performance ParalleX) is an + open source, portable, performance-oriented runtime developed at + CREST (Indiana University). HPX-5 provides a distributed + programming model allowing programs to run unmodified on systems + from a single SMP to large clusters and supercomputers with + thousands of nodes. HPX-5 supports a wide variety of Intel and ARM + platforms. It is being used by a broad range of scientific + applications enabling scientists to write code that performs and + scales better than contemporary runtimes.""" + homepage = "http://hpx.crest.iu.edu" + url = "http://hpx.crest.iu.edu/release/hpx-2.0.0.tar.gz" + + version('2.0.0', '3d2ff3aab6c46481f9ec65c5b2bfe7a6') + version('1.3.0', '2260ecc7f850e71a4d365a43017d8cee') + version('1.2.0', '4972005f85566af4afe8b71afbf1480f') + version('1.1.0', '646afb460ecb7e0eea713a634933ce4f') + version('1.0.0', '8020822adf6090bd59ed7fe465f6c6cb') + + variant('debug', default=False, description='Build a debug version of HPX-5') + variant('photon', default=False, description='Enable Photon support') + variant('mpi', default=False, description='Enable MPI support') + + depends_on("mpi", when='+mpi') + depends_on("mpi", when='+photon') + + def install(self, spec, prefix): + extra_args = [] + if '+debug' in spec: + extra_args.extend([ + '--enable-debug', + 'CFLAGS=-g -O0' + ]) + else: + extra_args.append('CFLAGS=-O3') + + if '+mpi' in spec: + extra_args.append('--enable-mpi') + + if '+photon' in spec: + extra_args.extend([ + '--enable-mpi', + '--enable-photon' + ]) + + os.chdir("./hpx/") + configure('--prefix=%s' % prefix, *extra_args) + make() + make("install") diff --git a/var/spack/repos/builtin/packages/hwloc/package.py b/var/spack/repos/builtin/packages/hwloc/package.py index 7ebede76a3..ab7205646e 100644 --- a/var/spack/repos/builtin/packages/hwloc/package.py +++ b/var/spack/repos/builtin/packages/hwloc/package.py @@ -14,15 +14,18 @@ class Hwloc(Package): efficiently.""" homepage = "http://www.open-mpi.org/projects/hwloc/" url = "http://www.open-mpi.org/software/hwloc/v1.9/downloads/hwloc-1.9.tar.gz" + list_url = "http://www.open-mpi.org/software/hwloc/" + list_depth = 3 - version('1.11.2', '486169cbe111cdea57be12638828ebbf', - url='http://www.open-mpi.org/software/hwloc/v1.11/downloads/hwloc-1.11.2.tar.bz2') - version('1.11.1', '002742efd3a8431f98d6315365a2b543', - url='http://www.open-mpi.org/software/hwloc/v1.11/downloads/hwloc-1.11.1.tar.bz2') - version('1.9', '1f9f9155682fe8946a97c08896109508') + version('1.11.2', 'e4ca55c2a5c5656da4a4e37c8fc51b23') + version('1.11.1', 'feb4e416a1b25963ed565d8b42252fdc') + version('1.9', '1f9f9155682fe8946a97c08896109508') depends_on('libpciaccess') + def url_for_version(self, version): + return "http://www.open-mpi.org/software/hwloc/v%s/downloads/hwloc-%s.tar.gz" % (version.up_to(2), version) + def install(self, spec, prefix): configure("--prefix=%s" % prefix) diff --git a/var/spack/repos/builtin/packages/hypre/package.py b/var/spack/repos/builtin/packages/hypre/package.py index 0f7f14dd89..242ee100d7 100644 --- a/var/spack/repos/builtin/packages/hypre/package.py +++ b/var/spack/repos/builtin/packages/hypre/package.py @@ -1,4 +1,5 @@ from spack import * +import os class Hypre(Package): """Hypre is a library of high performance preconditioners that @@ -8,8 +9,11 @@ class Hypre(Package): homepage = "http://computation.llnl.gov/project/linear_solvers/software.php" url = "http://computation.llnl.gov/project/linear_solvers/download/hypre-2.10.0b.tar.gz" + version('2.10.1', 'dc048c4cabb3cd549af72591474ad674') version('2.10.0b', '768be38793a35bb5d055905b271f5b8e') + variant('shared', default=True, description="Build shared library version (disables static library)") + depends_on("mpi") depends_on("blas") depends_on("lapack") @@ -17,16 +21,26 @@ class Hypre(Package): def install(self, spec, prefix): blas_dir = spec['blas'].prefix lapack_dir = spec['lapack'].prefix + mpi_dir = spec['mpi'].prefix + + os.environ['CC'] = os.path.join(mpi_dir, 'bin', 'mpicc') + os.environ['CXX'] = os.path.join(mpi_dir, 'bin', 'mpicxx') + os.environ['F77'] = os.path.join(mpi_dir, 'bin', 'mpif77') + + + configure_args = [ + "--prefix=%s" % prefix, + "--with-lapack-libs=lapack", + "--with-lapack-lib-dirs=%s/lib" % lapack_dir, + "--with-blas-libs=blas", + "--with-blas-lib-dirs=%s/lib" % blas_dir] + if '+shared' in self.spec: + configure_args.append("--enable-shared") # Hypre's source is staged under ./src so we'll have to manually # cd into it. with working_dir("src"): - configure( - "--prefix=%s" % prefix, - "--with-blas-libs=blas", - "--with-blas-lib-dirs=%s/lib" % blas_dir, - "--with-lapack-libs=\"lapack blas\"", - "--with-lapack-lib-dirs=%s/lib" % lapack_dir, - "--with-MPI") + configure(*configure_args) + make() make("install") diff --git a/var/spack/repos/builtin/packages/jdk/package.py b/var/spack/repos/builtin/packages/jdk/package.py index f8f5fc21bd..cbcc53ac0a 100644 --- a/var/spack/repos/builtin/packages/jdk/package.py +++ b/var/spack/repos/builtin/packages/jdk/package.py @@ -28,7 +28,7 @@ class Jdk(Package): '-H', # specify required License Agreement cookie 'Cookie: oraclelicense=accept-securebackup-cookie'] - def do_fetch(self): + def do_fetch(self, mirror_only=False): # Add our custom curl commandline options tty.msg( "[Jdk] Adding required commandline options to curl " + @@ -39,7 +39,7 @@ class Jdk(Package): spack.curl.add_default_arg(option) # Now perform the actual fetch - super(Jdk, self).do_fetch() + super(Jdk, self).do_fetch(mirror_only) def install(self, spec, prefix): diff --git a/var/spack/repos/builtin/packages/libedit/package.py b/var/spack/repos/builtin/packages/libedit/package.py index bcd5212b9e..faed8bad37 100644 --- a/var/spack/repos/builtin/packages/libedit/package.py +++ b/var/spack/repos/builtin/packages/libedit/package.py @@ -7,6 +7,8 @@ class Libedit(Package): version('3.1', '43cdb5df3061d78b5e9d59109871b4f6', url="http://thrysoee.dk/editline/libedit-20150325-3.1.tar.gz") + depends_on('ncurses') + def install(self, spec, prefix): configure('--prefix=%s' % prefix) diff --git a/var/spack/repos/builtin/packages/libelf/package.py b/var/spack/repos/builtin/packages/libelf/package.py index 29bc21b65c..9f16708af5 100644 --- a/var/spack/repos/builtin/packages/libelf/package.py +++ b/var/spack/repos/builtin/packages/libelf/package.py @@ -38,6 +38,8 @@ class Libelf(Package): provides('elf') + sanity_check_is_file = 'include/libelf.h' + def install(self, spec, prefix): configure("--prefix=" + prefix, "--enable-shared", diff --git a/var/spack/repos/builtin/packages/libevent/package.py b/var/spack/repos/builtin/packages/libevent/package.py index 11b1083d67..714a155dc0 100644 --- a/var/spack/repos/builtin/packages/libevent/package.py +++ b/var/spack/repos/builtin/packages/libevent/package.py @@ -22,9 +22,16 @@ class Libevent(Package): version('2.0.13', 'af786b4b3f790c9d3279792edf7867fc') version('2.0.12', '42986228baf95e325778ed328a93e070') + variant('openssl', default=True, description="Build with encryption enabled at the libevent level.") + depends_on('openssl', when='+openssl') def install(self, spec, prefix): - configure("--prefix=%s" % prefix) + configure_args = [] + if '+openssl' in spec: + configure_args.append('--enable-openssl') + else: + configure_args.append('--enable-openssl') + configure("--prefix=%s" % prefix, *configure_args) make() make("install") diff --git a/var/spack/repos/builtin/packages/libgpg-error/package.py b/var/spack/repos/builtin/packages/libgpg-error/package.py index 6c1d1a10a7..dd5fc2408a 100644 --- a/var/spack/repos/builtin/packages/libgpg-error/package.py +++ b/var/spack/repos/builtin/packages/libgpg-error/package.py @@ -9,6 +9,7 @@ class LibgpgError(Package): homepage = "https://www.gnupg.org/related_software/libgpg-error" url = "ftp://ftp.gnupg.org/gcrypt/libgpg-error/libgpg-error-1.18.tar.bz2" + version('1.21', 'ab0b5aba6d0a185b41d07bda804fd8b2') version('1.18', '12312802d2065774b787cbfc22cc04e9') def install(self, spec, prefix): diff --git a/var/spack/repos/builtin/packages/libsigsegv/package.py b/var/spack/repos/builtin/packages/libsigsegv/package.py new file mode 100644 index 0000000000..4b486198ec --- /dev/null +++ b/var/spack/repos/builtin/packages/libsigsegv/package.py @@ -0,0 +1,15 @@ +from spack import * + +class Libsigsegv(Package): + """GNU libsigsegv is a library for handling page faults in user mode.""" + homepage = "https://www.gnu.org/software/libsigsegv/" + url = "ftp://ftp.gnu.org/gnu/libsigsegv/libsigsegv-2.10.tar.gz" + + version('2.10', '7f96fb1f65b3b8cbc1582fb7be774f0f') + + def install(self, spec, prefix): + configure('--prefix=%s' % prefix, + '--enable-shared') + + make() + make("install") diff --git a/var/spack/repos/builtin/packages/llvm/package.py b/var/spack/repos/builtin/packages/llvm/package.py index a2b2c6eccc..1d25d59e50 100644 --- a/var/spack/repos/builtin/packages/llvm/package.py +++ b/var/spack/repos/builtin/packages/llvm/package.py @@ -1,5 +1,5 @@ ############################################################################## -# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Copyright (c) 2016, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory. # # This file is part of Spack. @@ -34,7 +34,7 @@ class Llvm(Package): it is the full name of the project. """ homepage = 'http://llvm.org/' - url = 'http://llvm.org/releases/3.7.0/llvm-3.7.0.src.tar.xz' + url = 'http://llvm.org/releases/3.7.1/llvm-3.7.1.src.tar.xz' version('3.0', 'a8e5f5f1c1adebae7b4a654c376a6005', url='http://llvm.org/releases/3.0/llvm-3.0.tar.gz') # currently required by mesa package @@ -52,7 +52,7 @@ class Llvm(Package): depends_on('cmake @2.8.12.2:') # Universal dependency - depends_on('python@2.7:') + depends_on('python@2.7:2.8') # Seems not to support python 3.X.Y # lldb dependencies depends_on('ncurses', when='+lldb') @@ -118,6 +118,51 @@ class Llvm(Package): } releases = [ { + 'version' : 'trunk', + 'repo' : 'http://llvm.org/svn/llvm-project/llvm/trunk', + 'resources' : { + 'compiler-rt' : 'http://llvm.org/svn/llvm-project/compiler-rt/trunk', + 'openmp' : 'http://llvm.org/svn/llvm-project/openmp/trunk', + 'polly' : 'http://llvm.org/svn/llvm-project/polly/trunk', + 'libcxx' : 'http://llvm.org/svn/llvm-project/libcxx/trunk', + 'libcxxabi' : 'http://llvm.org/svn/llvm-project/libcxxabi/trunk', + 'clang' : 'http://llvm.org/svn/llvm-project/cfe/trunk', + 'clang-tools-extra' : 'http://llvm.org/svn/llvm-project/clang-tools-extra/trunk', + 'lldb' : 'http://llvm.org/svn/llvm-project/lldb/trunk', + 'llvm-libunwind' : 'http://llvm.org/svn/llvm-project/libunwind/trunk', + } + }, + { + 'version' : '3.8.0', + 'md5':'07a7a74f3c6bd65de4702bf941b511a0', + 'resources' : { + 'compiler-rt' : 'd6fcbe14352ffb708e4d1ac2e48bb025', + 'openmp' : '8fd7cc35d48051613cf1e750e9f22e40', + 'polly' : '1b3b20f52d34a4024e21a4ea7112caa7', + 'libcxx' : 'd6e0bdbbee39f7907ad74fd56d03b88a', + 'libcxxabi' : 'bbe6b4d72c7c5978550d370af529bcf7', + 'clang' : 'cc99e7019bb74e6459e80863606250c5', + 'clang-tools-extra' : 'c2344f50e0eea0b402f0092a80ddc036', + 'lldb' : 'a5da35ed9cc8c8817ee854e3dbfba00e', + 'llvm-libunwind' : '162ade468607f153cca12be90b5194fa', + } + }, + { + 'version' : '3.7.1', + 'md5':'bf8b3a2c79e61212c5409041dfdbd319', + 'resources' : { + 'compiler-rt' : '1c6975daf30bb3b0473b53c3a1a6ff01', + 'openmp' : 'b4ad08cda4e5c22e42b66062b140438e', + 'polly' : '3a2a7367002740881637f4d47bca4dc3', + 'libcxx' : 'f9c43fa552a10e14ff53b94d04bea140', + 'libcxxabi' : '52d925afac9f97e9dcac90745255c169', + 'clang' : '0acd026b5529164197563d135a8fd83e', + 'clang-tools-extra' : '5d49ff745037f061a7c86aeb6a24c3d2', + 'lldb' : 'a106d8a0d21fc84d76953822fbaf3398', + 'llvm-libunwind' : '814bd52c9247c5d04629658fbcb3ab8c', + } + }, + { 'version' : '3.7.0', 'md5':'b98b9495e5655a672d6cb83e1a180f8e', 'resources' : { @@ -161,15 +206,25 @@ class Llvm(Package): ] for release in releases: - version(release['version'], release['md5'], url=llvm_url % release) - - for name, md5 in release['resources'].items(): - resource(name=name, - url=resources[name]['url'] % release, - md5=md5, - destination=resources[name]['destination'], - when='@%(version)s' % release, - placement=resources[name].get('placement', None)) + if release['version'] == 'trunk' : + version(release['version'], svn=release['repo']) + + for name, repo in release['resources'].items(): + resource(name=name, + svn=repo, + destination=resources[name]['destination'], + when='@%(version)s' % release, + placement=resources[name].get('placement', None)) + else: + version(release['version'], release['md5'], url=llvm_url % release) + + for name, md5 in release['resources'].items(): + resource(name=name, + url=resources[name]['url'] % release, + md5=md5, + destination=resources[name]['destination'], + when='@%(version)s' % release, + placement=resources[name].get('placement', None)) def install(self, spec, prefix): env['CXXFLAGS'] = self.compiler.cxx11_flag diff --git a/var/spack/repos/builtin/packages/m4/package.py b/var/spack/repos/builtin/packages/m4/package.py new file mode 100644 index 0000000000..a4b9dcb623 --- /dev/null +++ b/var/spack/repos/builtin/packages/m4/package.py @@ -0,0 +1,25 @@ +from spack import * + +class M4(Package): + """GNU M4 is an implementation of the traditional Unix macro processor.""" + homepage = "https://www.gnu.org/software/m4/m4.html" + url = "ftp://ftp.gnu.org/gnu/m4/m4-1.4.17.tar.gz" + + version('1.4.17', 'a5e9954b1dae036762f7b13673a2cf76') + + patch('pgi.patch', when='@1.4.17') + + variant('sigsegv', default=True, description="Build the libsigsegv dependency") + + depends_on('libsigsegv', when='+sigsegv') + + def install(self, spec, prefix): + configure_args = [] + if 'libsigsegv' in spec: + configure_args.append('--with-libsigsegv-prefix=%s' % spec['libsigsegv'].prefix) + else: + configure_args.append('--without-libsigsegv-prefix') + + configure("--prefix=%s" % prefix, *configure_args) + make() + make("install") diff --git a/var/spack/repos/builtin/packages/m4/pgi.patch b/var/spack/repos/builtin/packages/m4/pgi.patch new file mode 100644 index 0000000000..1ad63e2cf1 --- /dev/null +++ b/var/spack/repos/builtin/packages/m4/pgi.patch @@ -0,0 +1,10 @@ +--- a/lib/config.hin ++++ b/lib/config.hin +@@ -1510,6 +1510,7 @@ + ? defined __GNUC_STDC_INLINE__ && __GNUC_STDC_INLINE__ \ + : (199901L <= __STDC_VERSION__ \ + && !defined __HP_cc \ ++ && !defined __PGI \ + && !(defined __SUNPRO_C && __STDC__))) \ + && !defined _GL_EXTERN_INLINE_APPLE_BUG) + # define _GL_INLINE inline diff --git a/var/spack/repos/builtin/packages/mpc/package.py b/var/spack/repos/builtin/packages/mpc/package.py index 50477a0ccb..108fec678f 100644 --- a/var/spack/repos/builtin/packages/mpc/package.py +++ b/var/spack/repos/builtin/packages/mpc/package.py @@ -37,6 +37,12 @@ class Mpc(Package): depends_on("gmp") depends_on("mpfr") + def url_for_version(self, version): + if version < Version("1.0.1"): + return "http://www.multiprecision.org/mpc/download/mpc-%s.tar.gz" % version + else: + return "ftp://ftp.gnu.org/gnu/mpc/mpc-%s.tar.gz" % version + def install(self, spec, prefix): configure("--prefix=%s" % prefix) make() diff --git a/var/spack/repos/builtin/packages/mpfr/package.py b/var/spack/repos/builtin/packages/mpfr/package.py index 2758b0da2e..7e6e7d5bb6 100644 --- a/var/spack/repos/builtin/packages/mpfr/package.py +++ b/var/spack/repos/builtin/packages/mpfr/package.py @@ -28,12 +28,13 @@ class Mpfr(Package): """The MPFR library is a C library for multiple-precision floating-point computations with correct rounding.""" homepage = "http://www.mpfr.org" - url = "http://www.mpfr.org/mpfr-current/mpfr-3.1.3.tar.bz2" + url = "https://gforge.inria.fr/frs/download.php/latestfile/159/mpfr-3.1.2.tar.bz2" + version('3.1.4', 'b8a2f6b0e68bef46e53da2ac439e1cf4') version('3.1.3', '5fdfa3cfa5c86514ee4a241a1affa138') version('3.1.2', 'ee2c3ac63bf0c2359bf08fc3ee094c19') - depends_on('gmp@4.1.0:') + depends_on('gmp') def install(self, spec, prefix): configure("--prefix=%s" % prefix) diff --git a/var/spack/repos/builtin/packages/mpich/package.py b/var/spack/repos/builtin/packages/mpich/package.py index 00b7dfda75..b20dc8dd60 100644 --- a/var/spack/repos/builtin/packages/mpich/package.py +++ b/var/spack/repos/builtin/packages/mpich/package.py @@ -25,6 +25,7 @@ from spack import * import os + class Mpich(Package): """MPICH is a high performance and widely portable implementation of the Message Passing Interface (MPI) standard.""" @@ -41,21 +42,33 @@ class Mpich(Package): version('3.1', '5643dd176499bfb7d25079aaff25f2ec') version('3.0.4', '9c5d5d4fe1e17dd12153f40bc5b6dbc0') + variant('verbs', default=False, description='Build support for OpenFabrics verbs.') + provides('mpi@:3.0', when='@3:') provides('mpi@:1.3', when='@1:') - def setup_dependent_environment(self, module, spec, dep_spec): - """For dependencies, make mpicc's use spack wrapper.""" - os.environ['MPICH_CC'] = 'cc' - os.environ['MPICH_CXX'] = 'c++' - os.environ['MPICH_F77'] = 'f77' - os.environ['MPICH_F90'] = 'f90' + def setup_dependent_environment(self, env, dependent_spec): + env.set('MPICH_CC', spack_cc) + env.set('MPICH_CXX', spack_cxx) + env.set('MPICH_F77', spack_f77) + env.set('MPICH_F90', spack_f90) + env.set('MPICH_FC', spack_fc) + def setup_dependent_package(self, module, dep_spec): + """For dependencies, make mpicc's use spack wrapper.""" + # FIXME : is this necessary ? Shouldn't this be part of a contract with MPI providers? + module.mpicc = join_path(self.prefix.bin, 'mpicc') def install(self, spec, prefix): config_args = ["--prefix=" + prefix, "--enable-shared"] + # Variants + if '+verbs' in spec: + config_args.append("--with-ibverbs") + else: + config_args.append("--without-ibverbs") + # TODO: Spack should make it so that you can't actually find # these compilers if they're "disabled" for the current # compiler configuration. diff --git a/var/spack/repos/builtin/packages/mumps/Makefile.inc b/var/spack/repos/builtin/packages/mumps/Makefile.inc new file mode 100644 index 0000000000..2e6a041878 --- /dev/null +++ b/var/spack/repos/builtin/packages/mumps/Makefile.inc @@ -0,0 +1,38 @@ +LPORDDIR = $(topdir)/PORD/lib/ +IPORD = -I$(topdir)/PORD/include/ +LPORD = -L$(LPORDDIR) -lpord + +ORDERINGSC = $(ORDERINGSF) +LORDERINGS = $(LMETIS) $(LPORD) $(LSCOTCH) +IORDERINGSF = $(ISCOTCH) +IORDERINGSC = $(IMETIS) $(IPORD) $(ISCOTCH) + +PLAT = +LIBEXT = .a +OUTC = -o +OUTF = -o +RM = /bin/rm -f +AR = ar vr +RANLIB = ranlib + +INCSEQ = -I$(topdir)/libseq +LIBSEQ = -L$(topdir)/libseq -lmpiseq + +INCPAR = +LIBPAR = $(SCALAP) + +LIBOTHERS = -lpthread + +#Sequential: +ifeq ($(MUMPS_TYPE),seq) +INCS = $(INCSEQ) +LIBS = $(LIBSEQ) +LIBSEQNEEDED = libseqneeded +endif + +#Parallel: +ifeq ($(MUMPS_TYPE),par) +INCS = $(INCPAR) +LIBS = $(LIBPAR) +LIBSEQNEEDED = +endif diff --git a/var/spack/repos/builtin/packages/mumps/package.py b/var/spack/repos/builtin/packages/mumps/package.py new file mode 100644 index 0000000000..5c120c37df --- /dev/null +++ b/var/spack/repos/builtin/packages/mumps/package.py @@ -0,0 +1,143 @@ +from spack import * +import os + + +class Mumps(Package): + """MUMPS: a MUltifrontal Massively Parallel sparse direct Solver""" + + homepage = "http://mumps.enseeiht.fr" + url = "http://mumps.enseeiht.fr/MUMPS_5.0.1.tar.gz" + + version('5.0.1', 'b477573fdcc87babe861f62316833db0') + + variant('mpi', default=True, description='Activate the compilation of MUMPS with the MPI support') + variant('scotch', default=False, description='Activate Scotch as a possible ordering library') + variant('ptscotch', default=False, description='Activate PT-Scotch as a possible ordering library') + variant('metis', default=False, description='Activate Metis as a possible ordering library') + variant('parmetis', default=False, description='Activate Parmetis as a possible ordering library') + variant('double', default=True, description='Activate the compilation of dmumps') + variant('float', default=True, description='Activate the compilation of smumps') + variant('complex', default=True, description='Activate the compilation of cmumps and/or zmumps') + variant('idx64', default=False, description='Use int64_t/integer*8 as default index type') + + + depends_on('scotch + esmumps', when='~ptscotch+scotch') + depends_on('scotch + esmumps + mpi', when='+ptscotch') + depends_on('metis', when='+metis') + depends_on('parmetis', when="+parmetis") + depends_on('blas') + depends_on('lapack') + depends_on('scalapack', when='+mpi') + depends_on('mpi', when='+mpi') + + # this function is not a patch function because in case scalapack + # is needed it uses self.spec['scalapack'].fc_link set by the + # setup_dependent_environment in scalapck. This happen after patch + # end before install + # def patch(self): + def write_makefile_inc(self): + if ('+parmetis' in self.spec or '+ptscotch' in self.spec) and '+mpi' not in self.spec: + raise RuntimeError('You cannot use the variants parmetis or ptscotch without mpi') + + makefile_conf = ["LIBBLAS = -L%s -lblas" % self.spec['blas'].prefix.lib] + + orderings = ['-Dpord'] + + if '+ptscotch' in self.spec or '+scotch' in self.spec: + join_lib = ' -l%s' % ('pt' if '+ptscotch' in self.spec else '') + makefile_conf.extend( + ["ISCOTCH = -I%s" % self.spec['scotch'].prefix.include, + "LSCOTCH = -L%s %s%s" % (self.spec['scotch'].prefix.lib, + join_lib, + join_lib.join(['esmumps', 'scotch', 'scotcherr']))]) + orderings.append('-Dscotch') + if '+ptscotch' in self.spec: + orderings.append('-Dptscotch') + + if '+parmetis' in self.spec and '+metis' in self.spec: + libname = 'parmetis' if '+parmetis' in self.spec else 'metis' + makefile_conf.extend( + ["IMETIS = -I%s" % self.spec['parmetis'].prefix.include, + "LMETIS = -L%s -l%s -L%s -l%s" % (self.spec['parmetis'].prefix.lib, 'parmetis',self.spec['metis'].prefix.lib, 'metis')]) + + orderings.append('-Dparmetis') + elif '+metis' in self.spec: + makefile_conf.extend( + ["IMETIS = -I%s" % self.spec['metis'].prefix.include, + "LMETIS = -L%s -l%s" % (self.spec['metis'].prefix.lib, 'metis')]) + + orderings.append('-Dmetis') + + makefile_conf.append("ORDERINGSF = %s" % (' '.join(orderings))) + + # TODO: test this part, it needs a full blas, scalapack and + # partitionning environment with 64bit integers + if '+idx64' in self.spec: + makefile_conf.extend( + # the fortran compilation flags most probably are + # working only for intel and gnu compilers this is + # perhaps something the compiler should provide + ['OPTF = -O -DALLOW_NON_INIT %s' % '-fdefault-integer-8' if self.compiler.name == "gcc" else '-i8', + 'OPTL = -O ', + 'OPTC = -O -DINTSIZE64']) + else: + makefile_conf.extend( + ['OPTF = -O -DALLOW_NON_INIT', + 'OPTL = -O ', + 'OPTC = -O ']) + + + if '+mpi' in self.spec: + makefile_conf.extend( + ["CC = %s" % join_path(self.spec['mpi'].prefix.bin, 'mpicc'), + "FC = %s" % join_path(self.spec['mpi'].prefix.bin, 'mpif90'), + "FL = %s" % join_path(self.spec['mpi'].prefix.bin, 'mpif90'), + "SCALAP = %s" % self.spec['scalapack'].fc_link, + "MUMPS_TYPE = par"]) + else: + makefile_conf.extend( + ["CC = cc", + "FC = fc", + "FL = fc", + "MUMPS_TYPE = seq"]) + + # TODO: change the value to the correct one according to the + # compiler possible values are -DAdd_, -DAdd__ and/or -DUPPER + makefile_conf.append("CDEFS = -DAdd_") + + + makefile_inc_template = join_path(os.path.dirname(self.module.__file__), + 'Makefile.inc') + with open(makefile_inc_template, "r") as fh: + makefile_conf.extend(fh.read().split('\n')) + + with working_dir('.'): + with open("Makefile.inc", "w") as fh: + makefile_inc = '\n'.join(makefile_conf) + fh.write(makefile_inc) + + + + def install(self, spec, prefix): + make_libs = [] + + # the coice to compile ?examples is to have kind of a sanity + # check on the libraries generated. + if '+float' in spec: + make_libs.append('sexamples') + if '+complex' in spec: + make_libs.append('cexamples') + + if '+double' in spec: + make_libs.append('dexamples') + if '+complex' in spec: + make_libs.append('zexamples') + + self.write_makefile_inc() + + make(*make_libs) + + install_tree('lib', prefix.lib) + install_tree('include', prefix.include) + if '~mpi' in spec: + install('libseq/libmpiseq.a', prefix.lib) diff --git a/var/spack/repos/builtin/packages/mvapich2/package.py b/var/spack/repos/builtin/packages/mvapich2/package.py index 23a11b3171..e4e95f92af 100644 --- a/var/spack/repos/builtin/packages/mvapich2/package.py +++ b/var/spack/repos/builtin/packages/mvapich2/package.py @@ -4,15 +4,13 @@ import os class Mvapich2(Package): """MVAPICH2 is an MPI implementation for Infiniband networks.""" homepage = "http://mvapich.cse.ohio-state.edu/" + url = "http://mvapich.cse.ohio-state.edu/download/mvapich/mv2/mvapich2-2.2b.tar.gz" - version('2.2a', 'b8ceb4fc5f5a97add9b3ff1b9cbe39d2', - url='http://mvapich.cse.ohio-state.edu/download/mvapich/mv2/mvapich2-2.2a.tar.gz') + version('2.2b', '5651e8b7a72d7c77ca68da48f3a5d108') + version('2.2a', 'b8ceb4fc5f5a97add9b3ff1b9cbe39d2') + version('2.0', '9fbb68a4111a8b6338e476dc657388b4') + version('1.9', '5dc58ed08fd3142c260b70fe297e127c') - version('2.0', '9fbb68a4111a8b6338e476dc657388b4', - url='http://mvapich.cse.ohio-state.edu/download/mvapich/mv2/mvapich2-2.0.tar.gz') - - version('1.9', '5dc58ed08fd3142c260b70fe297e127c', - url="http://mvapich.cse.ohio-state.edu/download/mvapich2/mv2/mvapich2-1.9.tgz") patch('ad_lustre_rwcontig_open_source.patch', when='@1.9') provides('mpi@:2.2', when='@1.9') # MVAPICH2-1.9 supports MPI 2.2 @@ -41,16 +39,25 @@ class Mvapich2(Package): NEMESISIBTCP = 'nemesisibtcp' NEMESISIB = 'nemesisib' NEMESIS = 'nemesis' + MRAIL = 'mrail' SUPPORTED_NETWORKS = (PSM, SOCK, NEMESIS, NEMESISIB, NEMESISIBTCP) variant(PSM, default=False, description='Configures a build for QLogic PSM-CH3') variant(SOCK, default=False, description='Configures a build for TCP/IP-CH3') variant(NEMESISIBTCP, default=False, description='Configures a build for both OFA-IB-Nemesis and TCP/IP-Nemesis') variant(NEMESISIB, default=False, description='Configures a build for OFA-IB-Nemesis') variant(NEMESIS, default=False, description='Configures a build for TCP/IP-Nemesis') + variant(MRAIL, default=False, description='Configures a build for OFA-IB-CH3') ########## # FIXME : CUDA support is missing + def url_for_version(self, version): + base_url = "http://mvapich.cse.ohio-state.edu/download" + if version < Version('2.0'): + return "%s/mvapich2/mv2/mvapich2-%s.tar.gz" % (base_url, version) + else: + return "%s/mvapich/mv2/mvapich2-%s.tar.gz" % (base_url, version) + @staticmethod def enabled(x): """ @@ -116,8 +123,8 @@ class Mvapich2(Package): count += 1 if count > 1: raise RuntimeError('network variants are mutually exclusive (only one can be selected at a time)') - - # From here on I can suppose that ony one variant has been selected + network_options = [] + # From here on I can suppose that only one variant has been selected if self.enabled(Mvapich2.PSM) in spec: network_options = ["--with-device=ch3:psm"] elif self.enabled(Mvapich2.SOCK) in spec: @@ -128,7 +135,7 @@ class Mvapich2(Package): network_options = ["--with-device=ch3:nemesis:ib"] elif self.enabled(Mvapich2.NEMESIS) in spec: network_options = ["--with-device=ch3:nemesis"] - else: + elif self.enabled(Mvapich2.MRAIL) in spec: network_options = ["--with-device=ch3:mrail", "--with-rdma=gen2"] configure_args.extend(network_options) @@ -141,7 +148,14 @@ class Mvapich2(Package): "--enable-romio", "--disable-silent-rules", ] - if not self.compiler.f77 and not self.compiler.fc: + + if self.compiler.f77 and self.compiler.fc: + configure_args.append("--enable-fortran=all") + elif self.compiler.f77: + configure_args.append("--enable-fortran=f77") + elif self.compiler.fc: + configure_args.append("--enable-fortran=fc") + else: configure_args.append("--enable-fortran=none") # Set the type of the build (debug, release) diff --git a/var/spack/repos/builtin/packages/ndiff/package.py b/var/spack/repos/builtin/packages/ndiff/package.py new file mode 100644 index 0000000000..10e445c81e --- /dev/null +++ b/var/spack/repos/builtin/packages/ndiff/package.py @@ -0,0 +1,21 @@ +from spack import * + +class Ndiff(Package): + """The ndiff tool is a binary utility that compares putatively similar files + while ignoring small numeric differernces. This utility is most often used + to compare files containing a lot of floating-point numeric data that + may be slightly different due to numeric error.""" + + homepage = "http://ftp.math.utah.edu/pub/ndiff/" + url = "http://ftp.math.utah.edu/pub/ndiff/ndiff-2.00.tar.gz" + + version('2.00', '885548b4dc26e72c5455bebb5ba6c16d') + version('1.00', 'f41ffe5d12f36cd36b6311acf46eccdc') + + def install(self, spec, prefix): + configure('--prefix=%s' % prefix) + + mkdirp(prefix.bin) + mkdirp('%s/lib' % prefix.share) + + make('install-exe', 'install-shrlib') diff --git a/var/spack/repos/builtin/packages/netcdf-cxx4/package.py b/var/spack/repos/builtin/packages/netcdf-cxx4/package.py new file mode 100644 index 0000000000..b83e964b00 --- /dev/null +++ b/var/spack/repos/builtin/packages/netcdf-cxx4/package.py @@ -0,0 +1,15 @@ +from spack import * + +class NetcdfCxx4(Package): + """C++ interface for NetCDF4""" + homepage = "http://www.unidata.ucar.edu/software/netcdf" + url = "http://www.unidata.ucar.edu/downloads/netcdf/ftp/netcdf-cxx4-4.2.tar.gz" + + version('4.2', 'd019853802092cf686254aaba165fc81') + + depends_on('netcdf') + + def install(self, spec, prefix): + configure('--prefix=%s' % prefix) + make() + make("install") diff --git a/var/spack/repos/builtin/packages/netcdf-fortran/package.py b/var/spack/repos/builtin/packages/netcdf-fortran/package.py new file mode 100644 index 0000000000..e4e33445e5 --- /dev/null +++ b/var/spack/repos/builtin/packages/netcdf-fortran/package.py @@ -0,0 +1,16 @@ +from spack import * + +class NetcdfFortran(Package): + """Fortran interface for NetCDF4""" + + homepage = "http://www.unidata.ucar.edu/software/netcdf" + url = "http://www.unidata.ucar.edu/downloads/netcdf/ftp/netcdf-fortran-4.4.3.tar.gz" + + version('4.4.3', 'bfd4ae23a34635b273d3eb0d91cbde9e') + + depends_on('netcdf') + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix) + make() + make("install") diff --git a/var/spack/repos/builtin/packages/netcdf/netcdf-4.3.3-mpi.patch b/var/spack/repos/builtin/packages/netcdf/netcdf-4.3.3-mpi.patch deleted file mode 100644 index 46dda5fc9d..0000000000 --- a/var/spack/repos/builtin/packages/netcdf/netcdf-4.3.3-mpi.patch +++ /dev/null @@ -1,25 +0,0 @@ -diff -Nur netcdf-4.3.3/CMakeLists.txt netcdf-4.3.3.mpi/CMakeLists.txt ---- netcdf-4.3.3/CMakeLists.txt 2015-02-12 16:44:35.000000000 -0500 -+++ netcdf-4.3.3.mpi/CMakeLists.txt 2015-10-14 16:44:41.176300658 -0400 -@@ -753,6 +753,7 @@ - SET(USE_PARALLEL OFF CACHE BOOL "") - MESSAGE(STATUS "Cannot find HDF5 library built with parallel support. Disabling parallel build.") - ELSE() -+ FIND_PACKAGE(MPI REQUIRED) - SET(USE_PARALLEL ON CACHE BOOL "") - SET(STATUS_PARALLEL "ON") - ENDIF() -diff -Nur netcdf-4.3.3/liblib/CMakeLists.txt netcdf-4.3.3.mpi/liblib/CMakeLists.txt ---- netcdf-4.3.3/liblib/CMakeLists.txt 2015-02-12 16:44:35.000000000 -0500 -+++ netcdf-4.3.3.mpi/liblib/CMakeLists.txt 2015-10-14 16:44:57.757793634 -0400 -@@ -71,6 +71,10 @@ - SET(TLL_LIBS ${TLL_LIBS} ${CURL_LIBRARY}) - ENDIF() - -+IF(USE_PARALLEL) -+ SET(TLL_LIBS ${TLL_LIBS} ${MPI_C_LIBRARIES}) -+ENDIF() -+ - IF(USE_HDF4) - SET(TLL_LIBS ${TLL_LIBS} ${HDF4_LIBRARIES}) - ENDIF() diff --git a/var/spack/repos/builtin/packages/netcdf/package.py b/var/spack/repos/builtin/packages/netcdf/package.py index 239644d894..227362399a 100644 --- a/var/spack/repos/builtin/packages/netcdf/package.py +++ b/var/spack/repos/builtin/packages/netcdf/package.py @@ -1,28 +1,76 @@ from spack import * + class Netcdf(Package): """NetCDF is a set of software libraries and self-describing, machine-independent - data formats that support the creation, access, and sharing of array-oriented - scientific data.""" + data formats that support the creation, access, and sharing of array-oriented + scientific data.""" - homepage = "http://www.unidata.ucar.edu/software/netcdf/" + homepage = "http://www.unidata.ucar.edu/software/netcdf" url = "ftp://ftp.unidata.ucar.edu/pub/netcdf/netcdf-4.3.3.tar.gz" + version('4.4.0', 'cffda0cbd97fdb3a06e9274f7aef438e') version('4.3.3', '5fbd0e108a54bd82cb5702a73f56d2ae') - patch('netcdf-4.3.3-mpi.patch') + variant('mpi', default=True, description='Enables MPI parallelism') + variant('hdf4', default=False, description="Enable HDF4 support") # Dependencies: - depends_on("cmake @2.8.12:") - # >HDF5 - depends_on("hdf5") + depends_on("curl") # required for DAP support + depends_on("hdf", when='+hdf4') + depends_on("hdf5+mpi~cxx", when='+mpi') # required for NetCDF-4 support + depends_on("hdf5~mpi", when='~mpi') # required for NetCDF-4 support + depends_on("zlib") # required for NetCDF-4 support def install(self, spec, prefix): - with working_dir('spack-build', create=True): - cmake('..', - "-DCMAKE_INSTALL_PREFIX:PATH=%s" % prefix, - "-DENABLE_DAP:BOOL=OFF", # Disable DAP. - "-DBUILD_SHARED_LIBS:BOOL=OFF") # Don't build shared libraries (use static libs). - - make() - make("install") + # Environment variables + CPPFLAGS = [] + LDFLAGS = [] + LIBS = [] + + config_args = [ + "--prefix=%s" % prefix, + "--enable-fsync", + "--enable-v2", + "--enable-utilities", + "--enable-shared", + "--enable-static", + "--enable-largefile", + # necessary for HDF5 support + "--enable-netcdf-4", + "--enable-dynamic-loading", + # necessary for DAP support + "--enable-dap" + ] + + if '+mpi' in spec: + config_args.append('--enable-parallel4') + + CPPFLAGS.append("-I%s/include" % spec['hdf5'].prefix) + LDFLAGS.append( "-L%s/lib" % spec['hdf5'].prefix) + + # HDF4 support + # As of NetCDF 4.1.3, "--with-hdf4=..." is no longer a valid option + # You must use the environment variables CPPFLAGS and LDFLAGS + if '+hdf4' in spec: + config_args.append("--enable-hdf4") + CPPFLAGS.append("-I%s/include" % spec['hdf'].prefix) + LDFLAGS.append( "-L%s/lib" % spec['hdf'].prefix) + LIBS.append( "-l%s" % "jpeg") + + if 'szip' in spec: + CPPFLAGS.append("-I%s/include" % spec['szip'].prefix) + LDFLAGS.append( "-L%s/lib" % spec['szip'].prefix) + LIBS.append( "-l%s" % "sz") + + # Fortran support + # In version 4.2+, NetCDF-C and NetCDF-Fortran have split. + # Use the netcdf-fortran package to install Fortran support. + + config_args.append('CPPFLAGS=%s' % ' '.join(CPPFLAGS)) + config_args.append('LDFLAGS=%s' % ' '.join(LDFLAGS)) + config_args.append('LIBS=%s' % ' '.join(LIBS)) + + configure(*config_args) + make() + make("install") diff --git a/var/spack/repos/builtin/packages/netlib-lapack/package.py b/var/spack/repos/builtin/packages/netlib-lapack/package.py index fb6b99e27c..78c5a053fe 100644 --- a/var/spack/repos/builtin/packages/netlib-lapack/package.py +++ b/var/spack/repos/builtin/packages/netlib-lapack/package.py @@ -12,6 +12,7 @@ class NetlibLapack(Package): homepage = "http://www.netlib.org/lapack/" url = "http://www.netlib.org/lapack/lapack-3.5.0.tgz" + version('3.6.0', 'f2f6c67134e851fe189bb3ca1fbb5101') version('3.5.0', 'b1d3e3e425b2e44a06760ff173104bdf') version('3.4.2', '61bf1a8a4469d4bdb7604f5897179478') version('3.4.1', '44c3869c38c8335c2b9c2a8bb276eb55') @@ -19,13 +20,13 @@ class NetlibLapack(Package): version('3.3.1', 'd0d533ec9a5b74933c2a1e84eedc58b4') variant('shared', default=False, description="Build shared library version") + variant('fpic', default=False, description="Build with -fpic compiler option") # virtual dependency provides('lapack') # blas is a virtual dependency. depends_on('blas') - depends_on('cmake') # Doesn't always build correctly in parallel @@ -36,24 +37,23 @@ class NetlibLapack(Package): blas = self.spec['netlib-blas'] return [join_path(blas.prefix.lib, 'blas.a')] - @when('^atlas') def get_blas_libs(self): blas = self.spec['atlas'] return [join_path(blas.prefix.lib, l) for l in ('libf77blas.a', 'libatlas.a')] - def install(self, spec, prefix): blas_libs = ";".join(self.get_blas_libs()) cmake_args = [".", '-DBLAS_LIBRARIES=' + blas_libs] if '+shared' in spec: cmake_args.append('-DBUILD_SHARED_LIBS=ON') + if '+fpic' in spec: + cmake_args.append('-DCMAKE_POSITION_INDEPENDENT_CODE=ON') cmake_args += std_cmake_args cmake(*cmake_args) make() make("install") - diff --git a/var/spack/repos/builtin/packages/netlib-scalapack/package.py b/var/spack/repos/builtin/packages/netlib-scalapack/package.py new file mode 100644 index 0000000000..c3e6822cdf --- /dev/null +++ b/var/spack/repos/builtin/packages/netlib-scalapack/package.py @@ -0,0 +1,51 @@ +from spack import * +import sys + +class NetlibScalapack(Package): + """ScaLAPACK is a library of high-performance linear algebra routines for parallel distributed memory machines""" + + homepage = "http://www.netlib.org/scalapack/" + url = "http://www.netlib.org/scalapack/scalapack-2.0.2.tgz" + + version('2.0.2', '2f75e600a2ba155ed9ce974a1c4b536f') + version('2.0.1', '17b8cde589ea0423afe1ec43e7499161') + version('2.0.0', '9e76ae7b291be27faaad47cfc256cbfe') + # versions before 2.0.0 are not using cmake and requires blacs as + # a separated package + + variant('shared', default=True, description='Build the shared library version') + variant('fpic', default=False, description="Build with -fpic compiler option") + + provides('scalapack') + + depends_on('mpi') + depends_on('lapack') + + def install(self, spec, prefix): + options = [ + "-DBUILD_SHARED_LIBS:BOOL=%s" % ('ON' if '+shared' in spec else 'OFF'), + "-DBUILD_STATIC_LIBS:BOOL=%s" % ('OFF' if '+shared' in spec else 'ON'), + "-DUSE_OPTIMIZED_LAPACK_BLAS:BOOL=ON", # forces scalapack to use find_package(LAPACK) + ] + + if '+fpic' in spec: + options.extend([ + "-DCMAKE_C_FLAGS=-fPIC", + "-DCMAKE_Fortran_FLAGS=-fPIC" + ]) + + options.extend(std_cmake_args) + + with working_dir('spack-build', create=True): + cmake('..', *options) + make() + make("install") + + def setup_dependent_package(self, module, dependent_spec): + spec = self.spec + lib_dsuffix = '.dylib' if sys.platform == 'darwin' else '.so' + lib_suffix = lib_dsuffix if '+shared' in spec else '.a' + + spec.fc_link = '-L%s -lscalapack' % spec.prefix.lib + spec.cc_link = spec.fc_link + spec.libraries = [join_path(spec.prefix.lib, 'libscalapack%s' % lib_suffix)] diff --git a/var/spack/repos/builtin/packages/octave/package.py b/var/spack/repos/builtin/packages/octave/package.py new file mode 100644 index 0000000000..6e99c23652 --- /dev/null +++ b/var/spack/repos/builtin/packages/octave/package.py @@ -0,0 +1,183 @@ +from spack import * + +class Octave(Package): + """GNU Octave is a high-level language, primarily intended for numerical + computations. It provides a convenient command line interface for solving + linear and nonlinear problems numerically, and for performing other + numerical experiments using a language that is mostly compatible with + Matlab. It may also be used as a batch-oriented language.""" + + homepage = "https://www.gnu.org/software/octave/" + url = "ftp://ftp.gnu.org/gnu/octave/octave-4.0.0.tar.gz" + + version('4.0.0' , 'a69f8320a4f20a8480c1b278b1adb799') + + # Variants + variant('readline', default=True) + variant('arpack', default=False) + variant('curl', default=False) + variant('fftw', default=False) + variant('fltk', default=False) + variant('fontconfig', default=False) + variant('freetype', default=False) + variant('glpk', default=False) + variant('gl2ps', default=False) + variant('gnuplot', default=False) + variant('magick', default=False) + variant('hdf5', default=False) + variant('jdk', default=False) + variant('llvm', default=False) + variant('opengl', default=False) + variant('qhull', default=False) + variant('qrupdate', default=False) + variant('qscintilla', default=False) + variant('qt', default=False) + variant('suiteparse', default=False) + variant('zlib', default=False) + + # Required dependencies + depends_on('blas') + depends_on('lapack') + depends_on('pcre') + + # Strongly recommended dependencies + depends_on('readline', when='+readline') + + # Optional dependencies + depends_on('arpack', when='+arpack') + depends_on('curl', when='+curl') + depends_on('fftw', when='+fftw') + depends_on('fltk', when='+fltk') + depends_on('fontconfig', when='+fontconfig') + depends_on('freetype', when='+freetype') + depends_on('glpk', when='+glpk') + depends_on('gl2ps', when='+gl2ps') + depends_on('gnuplot', when='+gnuplot') + depends_on('ImageMagick', when='+magick') + depends_on('hdf5', when='+hdf5') + depends_on('jdk', when='+jdk') + depends_on('llvm', when='+llvm') + #depends_on('opengl', when='+opengl') # TODO: add package + depends_on('qhull', when='+qhull') + depends_on('qrupdate', when='+qrupdate') + #depends_on('qscintilla', when='+qscintilla) # TODO: add package + depends_on('qt', when='+qt') + depends_on('suite-sparse',when='+suitesparse') + depends_on('zlib', when='+zlib') + + + def install(self, spec, prefix): + config_args = [ + "--prefix=%s" % prefix + ] + + # Required dependencies + config_args.extend([ + "--with-blas=%s" % spec['blas'].prefix.lib, + "--with-lapack=%s" % spec['lapack'].prefix.lib + ]) + + # Strongly recommended dependencies + if '+readline' in spec: + config_args.append('--enable-readline') + else: + config_args.append('--disable-readline') + + # Optional dependencies + if '+arpack' in spec: + config_args.extend([ + "--with-arpack-includedir=%s" % spec['arpack'].prefix.include, + "--with-arpack-libdir=%s" % spec['arpack'].prefix.lib + ]) + else: + config_args.append("--without-arpack") + + if '+curl' in spec: + config_args.extend([ + "--with-curl-includedir=%s" % spec['curl'].prefix.include, + "--with-curl-libdir=%s" % spec['curl'].prefix.lib + ]) + else: + config_args.append("--without-curl") + + if '+fftw' in spec: + config_args.extend([ + "--with-fftw3-includedir=%s" % spec['fftw'].prefix.include, + "--with-fftw3-libdir=%s" % spec['fftw'].prefix.lib, + "--with-fftw3f-includedir=%s" % spec['fftw'].prefix.include, + "--with-fftw3f-libdir=%s" % spec['fftw'].prefix.lib + ]) + else: + config_args.extend([ + "--without-fftw3", + "--without-fftw3f" + ]) + + if '+fltk' in spec: + config_args.extend([ + "--with-fltk-prefix=%s" % spec['fltk'].prefix, + "--with-fltk-exec-prefix=%s" % spec['fltk'].prefix + ]) + else: + config_args.append("--without-fltk") + + if '+glpk' in spec: + config_args.extend([ + "--with-glpk-includedir=%s" % spec['glpk'].prefix.include, + "--with-glpk-libdir=%s" % spec['glpk'].prefix.lib + ]) + else: + config_args.append("--without-glpk") + + if '+magick' in spec: + config_args.append("--with-magick=%s" % spec['ImageMagick'].prefix.lib) + + if '+hdf5' in spec: + config_args.extend([ + "--with-hdf5-includedir=%s" % spec['hdf5'].prefix.include, + "--with-hdf5-libdir=%s" % spec['hdf5'].prefix.lib + ]) + else: + config_args.append("--without-hdf5") + + if '+jdk' in spec: + config_args.extend([ + "--with-java-homedir=%s" % spec['jdk'].prefix, + "--with-java-includedir=%s" % spec['jdk'].prefix.include, + "--with-java-libdir=%s" % spec['jdk'].prefix.lib + ]) + + if '~opengl' in spec: + config_args.extend([ + "--without-opengl", + "--without-framework-opengl" + ]) + + if '+qhull' in spec: + config_args.extend([ + "--with-qhull-includedir=%s" % spec['qhull'].prefix.include, + "--with-qhull-libdir=%s" % spec['qhull'].prefix.lib + ]) + else: + config_args.append("--without-qhull") + + if '+qrupdate' in spec: + config_args.extend([ + "--with-qrupdate-includedir=%s" % spec['qrupdate'].prefix.include, + "--with-qrupdate-libdir=%s" % spec['qrupdate'].prefix.lib + ]) + else: + config_args.append("--without-qrupdate") + + if '+zlib' in spec: + config_args.extend([ + "--with-z-includedir=%s" % spec['zlib'].prefix.include, + "--with-z-libdir=%s" % spec['zlib'].prefix.lib + ]) + else: + config_args.append("--without-z") + + configure(*config_args) + + make() + make("install") diff --git a/var/spack/repos/builtin/packages/openblas/package.py b/var/spack/repos/builtin/packages/openblas/package.py index 9c8fa1c694..781a1e2ec8 100644 --- a/var/spack/repos/builtin/packages/openblas/package.py +++ b/var/spack/repos/builtin/packages/openblas/package.py @@ -1,10 +1,12 @@ from spack import * +import sys class Openblas(Package): """OpenBLAS: An optimized BLAS library""" homepage = "http://www.openblas.net" url = "http://github.com/xianyi/OpenBLAS/archive/v0.2.15.tar.gz" + version('0.2.16', 'fef46ab92463bdbb1479dcec594ef6dc') version('0.2.15', 'b1190f3d3471685f17cfd1ec1d252ac9') # virtual dependency @@ -15,11 +17,14 @@ class Openblas(Package): make('libs', 'netlib', 'shared', 'CC=cc', 'FC=f77') make('install', "PREFIX='%s'" % prefix) + lib_dsuffix = 'dylib' if sys.platform == 'darwin' else 'so' # Blas virtual package should provide blas.a and libblas.a with working_dir(prefix.lib): symlink('libopenblas.a', 'blas.a') symlink('libopenblas.a', 'libblas.a') + symlink('libopenblas.%s' % lib_dsuffix, 'libblas.%s' % lib_dsuffix) # Lapack virtual package should provide liblapack.a with working_dir(prefix.lib): symlink('libopenblas.a', 'liblapack.a') + symlink('libopenblas.%s' % lib_dsuffix, 'liblapack.%s' % lib_dsuffix) diff --git a/var/spack/repos/builtin/packages/opencv/package.py b/var/spack/repos/builtin/packages/opencv/package.py new file mode 100644 index 0000000000..99b555323f --- /dev/null +++ b/var/spack/repos/builtin/packages/opencv/package.py @@ -0,0 +1,50 @@ +from spack import * + + +class Opencv(Package): + """ + OpenCV is released under a BSD license and hence it's free for both academic and commercial use. It has C++, C, + Python and Java interfaces and supports Windows, Linux, Mac OS, iOS and Android. OpenCV was designed for + computational efficiency and with a strong focus on real-time applications. Written in optimized C/C++, the library + can take advantage of multi-core processing. Enabled with OpenCL, it can take advantage of the hardware + acceleration of the underlying heterogeneous compute platform. Adopted all around the world, OpenCV has more than + 47 thousand people of user community and estimated number of downloads exceeding 9 million. Usage ranges from + interactive art, to mines inspection, stitching maps on the web or through advanced robotics. + """ + homepage = 'http://opencv.org/' + url = 'https://github.com/Itseez/opencv/archive/3.1.0.tar.gz' + + version('3.1.0', '70e1dd07f0aa06606f1bc0e3fa15abd3') + + variant('shared', default=True, description='Enables the build of shared libraries') + variant('debug', default=False, description='Builds a debug version of the libraries') + + variant('eigen', default=True, description='Activates support for eigen') + variant('ipp', default=True, description='Activates support for IPP') + + depends_on('zlib') + depends_on('libpng') + depends_on('libjpeg-turbo') + depends_on('libtiff') + + depends_on('python') + depends_on('py-numpy') + + depends_on('eigen', when='+eigen') + + # FIXME : GUI extensions missing + # FIXME : CUDA extensions still missing + + def install(self, spec, prefix): + cmake_options = [] + cmake_options.extend(std_cmake_args) + + cmake_options.extend(['-DCMAKE_BUILD_TYPE:STRING=%s' % ('Debug' if '+debug' in spec else 'Release'), + '-DBUILD_SHARED_LIBS:BOOL=%s' % ('ON' if '+shared' in spec else 'OFF'), + '-DENABLE_PRECOMPILED_HEADERS:BOOL=OFF', + '-DWITH_IPP:BOOL=%s' % ('ON' if '+ipp' in spec else 'OFF')]) + + with working_dir('spack_build', create=True): + cmake('..', *cmake_options) + make('VERBOSE=1') + make("install") diff --git a/var/spack/repos/builtin/packages/openmpi/package.py b/var/spack/repos/builtin/packages/openmpi/package.py index 463719f9db..9a127f1812 100644 --- a/var/spack/repos/builtin/packages/openmpi/package.py +++ b/var/spack/repos/builtin/packages/openmpi/package.py @@ -17,45 +17,48 @@ class Openmpi(Package): list_url = "http://www.open-mpi.org/software/ompi/" list_depth = 3 + version('1.10.2', 'b2f43d9635d2d52826e5ef9feb97fd4c') version('1.10.1', 'f0fcd77ed345b7eafb431968124ba16e') version('1.10.0', '280cf952de68369cebaca886c5ce0304') - version('1.8.8', '0dab8e602372da1425e9242ae37faf8c') - version('1.6.5', '03aed2a4aa4d0b27196962a2a65fc475') + version('1.8.8', '0dab8e602372da1425e9242ae37faf8c') + version('1.6.5', '03aed2a4aa4d0b27196962a2a65fc475') patch('ad_lustre_rwcontig_open_source.patch', when="@1.6.5") patch('llnl-platforms.patch', when="@1.6.5") - patch('configure.patch', when="@1.10.0:") + patch('configure.patch', when="@1.10.0:1.10.1") - variant('psm', default=False, description='Build support for the PSM library.') + variant('psm', default=False, description='Build support for the PSM library.') variant('verbs', default=False, description='Build support for OpenFabrics verbs.') + # TODO : variant support for other schedulers is missing + variant('tm', default=False, description='Build TM (Torque, PBSPro, and compatible) support') + provides('mpi@:2.2', when='@1.6.5') provides('mpi@:3.0', when='@1.7.5:') - depends_on('hwloc') - def url_for_version(self, version): return "http://www.open-mpi.org/software/ompi/v%s/downloads/openmpi-%s.tar.bz2" % (version.up_to(2), version) - def setup_dependent_environment(self, module, spec, dep_spec): - """For dependencies, make mpicc's use spack wrapper.""" - os.environ['OMPI_CC'] = 'cc' - os.environ['OMPI_CXX'] = 'c++' - os.environ['OMPI_FC'] = 'f90' - os.environ['OMPI_F77'] = 'f77' + def setup_dependent_environment(self, spack_env, run_env, dependent_spec): + spack_env.set('OMPI_CC', spack_cc) + spack_env.set('OMPI_CXX', spack_cxx) + spack_env.set('OMPI_FC', spack_fc) + spack_env.set('OMPI_F77', spack_f77) def install(self, spec, prefix): config_args = ["--prefix=%s" % prefix, "--with-hwloc=%s" % spec['hwloc'].prefix, - "--with-tm", # necessary for Torque support "--enable-shared", "--enable-static"] # Variants + if '+tm' in spec: + config_args.append("--with-tm") # necessary for Torque support + if '+psm' in spec: config_args.append("--with-psm") @@ -85,7 +88,6 @@ class Openmpi(Package): self.filter_compilers() - def filter_compilers(self): """Run after install to make the MPI compilers use the compilers that Spack built the package with. @@ -94,7 +96,7 @@ class Openmpi(Package): to Spack's generic cc, c++ and f90. We want them to be bound to whatever compiler they were built with. """ - kwargs = { 'ignore_absent' : True, 'backup' : False, 'string' : False } + kwargs = {'ignore_absent': True, 'backup': False, 'string': False} dir = os.path.join(self.prefix, 'share/openmpi/') cc_wrappers = ['mpicc-vt-wrapper-data.txt', 'mpicc-wrapper-data.txt', @@ -132,5 +134,3 @@ class Openmpi(Package): if not os.path.islink(path): filter_file('compiler=.*', 'compiler=%s' % self.compiler.fc, path, **kwargs) - - diff --git a/var/spack/repos/builtin/packages/openssl/package.py b/var/spack/repos/builtin/packages/openssl/package.py index bbb169ec6b..70afaf4038 100644 --- a/var/spack/repos/builtin/packages/openssl/package.py +++ b/var/spack/repos/builtin/packages/openssl/package.py @@ -1,3 +1,6 @@ +import urllib +import llnl.util.tty as tty + from spack import * class Openssl(Package): @@ -10,12 +13,51 @@ class Openssl(Package): url = "http://www.openssl.org/source/openssl-1.0.1h.tar.gz" version('1.0.1h', '8d6d684a9430d5cc98a62a5d8fbda8cf') + version('1.0.1r', '1abd905e079542ccae948af37e393d28') version('1.0.2d', '38dd619b2e77cbac69b99f52a053d25a') version('1.0.2e', '5262bfa25b60ed9de9f28d5d52d77fc5') + version('1.0.2f', 'b3bf73f507172be9292ea2a8c28b659d') + version('1.0.2g', 'f3c710c045cdee5fd114feb69feba7aa') depends_on("zlib") parallel = False + def url_for_version(self, version): + # This URL is computed pinging the place where the latest version is stored. To avoid slowdown + # due to repeated pinging, we store the URL in a private class attribute to do the job only once per version + openssl_urls = getattr(Openssl, '_openssl_url', {}) + openssl_url = openssl_urls.get(version, None) + # Same idea, but just to avoid issuing the same message multiple times + warnings_given_to_user = getattr(Openssl, '_warnings_given', {}) + if openssl_url is None: + latest = 'http://www.openssl.org/source/openssl-{version}.tar.gz' + older = 'http://www.openssl.org/source/old/{version_number}/openssl-{version_full}.tar.gz' + # Try to use the url where the latest tarballs are stored. If the url does not exist (404), then + # return the url for older format + version_number = '.'.join([str(x) for x in version[:-1]]) + older_url = older.format(version_number=version_number, version_full=version) + latest_url = latest.format(version=version) + response = urllib.urlopen(latest.format(version=version)) + if response.getcode() == 404: + openssl_url = older_url + # Checks if we already warned the user for this particular version of OpenSSL. + # If not we display a warning message and mark this version + if not warnings_given_to_user.get(version, False): + tty.warn('This installation depends on an old version of OpenSSL, which may have known security issues. ') + tty.warn('Consider updating to the latest version of this package.') + tty.warn('More details at {homepage}'.format(homepage=Openssl.homepage)) + warnings_given_to_user[version] = True + else: + openssl_url = latest_url + # Store the computed URL + openssl_urls[version] = openssl_url + # Store the updated dictionary of URLS + Openssl._openssl_url = openssl_urls + # Store the updated dictionary of warnings + Openssl._warnings_given = warnings_given_to_user + + return openssl_url + def install(self, spec, prefix): # OpenSSL uses a variable APPS in its Makefile. If it happens to be set # in the environment, then this will override what is set in the diff --git a/var/spack/repos/builtin/packages/pango/package.py b/var/spack/repos/builtin/packages/pango/package.py index df43625bf5..79dad3a3d2 100644 --- a/var/spack/repos/builtin/packages/pango/package.py +++ b/var/spack/repos/builtin/packages/pango/package.py @@ -16,4 +16,4 @@ class Pango(Package): def install(self, spec, prefix): configure("--prefix=%s" % prefix) make() - make("install") + make("install", parallel=False) diff --git a/var/spack/repos/builtin/packages/papi/package.py b/var/spack/repos/builtin/packages/papi/package.py index 910e0aa9f9..53d69e28d9 100644 --- a/var/spack/repos/builtin/packages/papi/package.py +++ b/var/spack/repos/builtin/packages/papi/package.py @@ -13,6 +13,7 @@ class Papi(Package): homepage = "http://icl.cs.utk.edu/papi/index.html" url = "http://icl.cs.utk.edu/projects/papi/downloads/papi-5.4.1.tar.gz" + version('5.4.3', '3211b5a5bb389fe692370f5cf4cc2412') version('5.4.1', '9134a99219c79767a11463a76b0b01a2') version('5.3.0', '367961dd0ab426e5ae367c2713924ffb') diff --git a/var/spack/repos/builtin/packages/parallel-netcdf/package.py b/var/spack/repos/builtin/packages/parallel-netcdf/package.py new file mode 100644 index 0000000000..62a8f7ca0b --- /dev/null +++ b/var/spack/repos/builtin/packages/parallel-netcdf/package.py @@ -0,0 +1,20 @@ +from spack import * + +class ParallelNetcdf(Package): + """Parallel netCDF (PnetCDF) is a library providing high-performance + parallel I/O while still maintaining file-format compatibility with + Unidata's NetCDF.""" + + homepage = "https://trac.mcs.anl.gov/projects/parallel-netcdf" + url = "http://cucis.ece.northwestern.edu/projects/PnetCDF/Release/parallel-netcdf-1.6.1.tar.gz" + + version('1.6.1', '62a094eb952f9d1e15f07d56e535052604f1ac34') + + depends_on("m4") + depends_on("mpi") + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix, + "--with-mpi=%s" % spec['mpi'].prefix) + make() + make("install") diff --git a/var/spack/repos/builtin/packages/paraview/package.py b/var/spack/repos/builtin/packages/paraview/package.py index aaab352e66..c16054816c 100644 --- a/var/spack/repos/builtin/packages/paraview/package.py +++ b/var/spack/repos/builtin/packages/paraview/package.py @@ -2,9 +2,11 @@ from spack import * class Paraview(Package): homepage = 'http://www.paraview.org' - url = 'http://www.paraview.org/files/v4.4/ParaView-v4.4.0-source.tar.gz' + url = 'http://www.paraview.org/files/v5.0/ParaView-v' + _url_str = 'http://www.paraview.org/files/v%s/ParaView-v%s-source.tar.gz' - version('4.4.0', 'fa1569857dd680ebb4d7ff89c2227378', url='http://www.paraview.org/files/v4.4/ParaView-v4.4.0-source.tar.gz') + version('4.4.0', 'fa1569857dd680ebb4d7ff89c2227378') + version('5.0.0', '4598f0b421460c8bbc635c9a1c3bdbee') variant('python', default=False, description='Enable Python support') @@ -14,8 +16,9 @@ class Paraview(Package): variant('osmesa', default=False, description='Enable OSMesa support') variant('qt', default=False, description='Enable Qt support') + variant('opengl2', default=False, description='Enable OpenGL2 backend') - depends_on('python', when='+python') + depends_on('python@2:2.7', when='+python') depends_on('py-numpy', when='+python') depends_on('py-matplotlib', when='+python') depends_on('tcl', when='+tcl') @@ -24,8 +27,8 @@ class Paraview(Package): depends_on('bzip2') depends_on('freetype') - depends_on('hdf5') depends_on('hdf5+mpi', when='+mpi') + depends_on('hdf5~mpi', when='~mpi') depends_on('jpeg') depends_on('libpng') depends_on('libtiff') @@ -35,6 +38,11 @@ class Paraview(Package): #depends_on('sqlite') # external version not supported depends_on('zlib') + def url_for_version(self, version): + """Handle ParaView version-based custom URLs.""" + return self._url_str % (version.up_to(2), version) + + def install(self, spec, prefix): with working_dir('spack-build', create=True): def feature_to_bool(feature, on='ON', off='OFF'): diff --git a/var/spack/repos/builtin/packages/parmetis/enable_external_metis.patch b/var/spack/repos/builtin/packages/parmetis/enable_external_metis.patch new file mode 100644 index 0000000000..514781b8b8 --- /dev/null +++ b/var/spack/repos/builtin/packages/parmetis/enable_external_metis.patch @@ -0,0 +1,13 @@ +diff --git a/CMakeLists.txt b/CMakeLists.txt +index ca945dd..1bf94e9 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -33,7 +33,7 @@ include_directories(${GKLIB_PATH}) + include_directories(${METIS_PATH}/include) + + # List of directories that cmake will look for CMakeLists.txt +-add_subdirectory(${METIS_PATH}/libmetis ${CMAKE_BINARY_DIR}/libmetis) ++#add_subdirectory(${METIS_PATH}/libmetis ${CMAKE_BINARY_DIR}/libmetis) + add_subdirectory(include) + add_subdirectory(libparmetis) + add_subdirectory(programs) diff --git a/var/spack/repos/builtin/packages/parmetis/package.py b/var/spack/repos/builtin/packages/parmetis/package.py index c897dec7e4..c691cf4191 100644 --- a/var/spack/repos/builtin/packages/parmetis/package.py +++ b/var/spack/repos/builtin/packages/parmetis/package.py @@ -25,9 +25,6 @@ from spack import * -# FIXME : lot of code is duplicated from packages/metis/package.py . Inheriting from there may reduce -# FIXME : the installation rules to just a few lines - class Parmetis(Package): """ @@ -43,13 +40,17 @@ class Parmetis(Package): variant('debug', default=False, description='Builds the library in debug mode') variant('gdb', default=False, description='Enables gdb support') - variant('idx64', default=False, description='Use int64_t as default index type') - variant('double', default=False, description='Use double precision floating point types') - depends_on('cmake @2.8:') # build dependency depends_on('mpi') - # FIXME : this should conflict with metis as it builds its own version internally + patch('enable_external_metis.patch') + depends_on('metis') + + # bug fixes from PETSc developers + # https://bitbucket.org/petsc/pkg-parmetis/commits/1c1a9fd0f408dc4d42c57f5c3ee6ace411eb222b/raw/ + patch('pkg-parmetis-1c1a9fd0f408dc4d42c57f5c3ee6ace411eb222b.patch') + # https://bitbucket.org/petsc/pkg-parmetis/commits/82409d68aa1d6cbc70740d0f35024aae17f7d5cb/raw/ + patch('pkg-parmetis-82409d68aa1d6cbc70740d0f35024aae17f7d5cb.patch') depends_on('gdb', when='+gdb') @@ -63,8 +64,8 @@ class Parmetis(Package): # FIXME : Once a contract is defined, MPI compilers should be retrieved indirectly via spec['mpi'] in case # FIXME : they use a non-standard name - options.extend(['-DGKLIB_PATH:PATH={metis_source}/GKlib'.format(metis_source=metis_source), - '-DMETIS_PATH:PATH={metis_source}'.format(metis_source=metis_source), + options.extend(['-DGKLIB_PATH:PATH={metis_source}/GKlib'.format(metis_source=metis_source), # still need headers from METIS source, and they are not installed with METIS. shame... + '-DMETIS_PATH:PATH={metis_source}'.format(metis_source=spec['metis'].prefix), '-DCMAKE_C_COMPILER:STRING=mpicc', '-DCMAKE_CXX_COMPILER:STRING=mpicxx']) @@ -78,18 +79,7 @@ class Parmetis(Package): if '+gdb' in spec: options.append('-DGDB:BOOL=ON') - metis_header = join_path(metis_source, 'include', 'metis.h') - - if '+idx64' in spec: - filter_file('IDXTYPEWIDTH 32', 'IDXTYPEWIDTH 64', metis_header) - - if '+double' in spec: - filter_file('REALTYPEWIDTH 32', 'REALTYPEWIDTH 64', metis_header) - with working_dir(build_directory, create=True): cmake(source_directory, *options) make() make("install") - # Parmetis build system doesn't allow for an external metis to be used, but doesn't copy the required - # metis header either - install(metis_header, self.prefix.include) diff --git a/var/spack/repos/builtin/packages/parmetis/pkg-parmetis-1c1a9fd0f408dc4d42c57f5c3ee6ace411eb222b.patch b/var/spack/repos/builtin/packages/parmetis/pkg-parmetis-1c1a9fd0f408dc4d42c57f5c3ee6ace411eb222b.patch new file mode 100644 index 0000000000..e6b8056c21 --- /dev/null +++ b/var/spack/repos/builtin/packages/parmetis/pkg-parmetis-1c1a9fd0f408dc4d42c57f5c3ee6ace411eb222b.patch @@ -0,0 +1,77 @@ +From 1c1a9fd0f408dc4d42c57f5c3ee6ace411eb222b Mon Sep 17 00:00:00 2001 +From: Jed Brown <jed@59A2.org> +Date: Fri, 12 Oct 2012 15:45:10 -0500 +Subject: [PATCH] ParMetis bug fixes reported by John Fettig [petsc-maint + #133631] + +''' +I have also reported to to Karypis but have received zero +response and he hasn't released any updates to the original release +either. At least he approved my forum posting so that other people +can see the bug and the fix. +http://glaros.dtc.umn.edu/gkhome/node/837 +''' + +Hg-commit: 1c2b9fe39201d404b493885093b5992028b9b8d4 +--- + libparmetis/xyzpart.c | 12 ++++++------ + 1 file changed, 6 insertions(+), 6 deletions(-) + +diff --git a/libparmetis/xyzpart.c b/libparmetis/xyzpart.c +index 3a2c289..63abfcb 100644 +--- a/libparmetis/xyzpart.c ++++ b/libparmetis/xyzpart.c +@@ -104,7 +104,7 @@ void IRBinCoordinates(ctrl_t *ctrl, graph_t *graph, idx_t ndims, real_t *xyz, + + for (i=0; i<nbins; i++) + emarkers[i] = gmin + (gmax-gmin)*i/nbins; +- emarkers[nbins] = gmax*(1.0+2.0*REAL_EPSILON); ++ emarkers[nbins] = gmax*(1.0+copysign(1.0,gmax)*2.0*REAL_EPSILON); + + /* get into a iterative backet boundary refinement */ + for (l=0; l<5; l++) { +@@ -152,7 +152,7 @@ void IRBinCoordinates(ctrl_t *ctrl, graph_t *graph, idx_t ndims, real_t *xyz, + } + } + nemarkers[0] = gmin; +- nemarkers[nbins] = gmax*(1.0+2.0*REAL_EPSILON); ++ nemarkers[nbins] = gmax*(1.0+copysign(1.0,gmax)*2.0*REAL_EPSILON); + rcopy(nbins+1, nemarkers, emarkers); + } + +@@ -218,7 +218,7 @@ void RBBinCoordinates(ctrl_t *ctrl, graph_t *graph, idx_t ndims, real_t *xyz, + + emarkers[0] = gmin; + emarkers[1] = gsum/gnvtxs; +- emarkers[2] = gmax*(1.0+2.0*REAL_EPSILON); ++ emarkers[2] = gmax*(1.0+(gmax < 0 ? -1. : 1.)*2.0*REAL_EPSILON); + cnbins = 2; + + /* get into a iterative backet boundary refinement */ +@@ -227,7 +227,7 @@ void RBBinCoordinates(ctrl_t *ctrl, graph_t *graph, idx_t ndims, real_t *xyz, + iset(cnbins, 0, lcounts); + rset(cnbins, 0, lsums); + for (j=0, i=0; i<nvtxs;) { +- if (cand[i].key < emarkers[j+1]) { ++ if (cand[i].key <= emarkers[j+1]) { + lcounts[j]++; + lsums[j] += cand[i].key; + i++; +@@ -272,12 +272,12 @@ void RBBinCoordinates(ctrl_t *ctrl, graph_t *graph, idx_t ndims, real_t *xyz, + + rsorti(cnbins, nemarkers); + rcopy(cnbins, nemarkers, emarkers); +- emarkers[cnbins] = gmax*(1.0+2.0*REAL_EPSILON); ++ emarkers[cnbins] = gmax*(1.0+(gmax < 0 ? -1. : 1.)*2.0*REAL_EPSILON); + } + + /* assign the coordinate to the appropriate bin */ + for (j=0, i=0; i<nvtxs;) { +- if (cand[i].key < emarkers[j+1]) { ++ if (cand[i].key <= emarkers[j+1]) { + bxyz[cand[i].val*ndims+k] = j; + i++; + } +-- +2.1.1.1.g1fb337f + diff --git a/var/spack/repos/builtin/packages/parmetis/pkg-parmetis-82409d68aa1d6cbc70740d0f35024aae17f7d5cb.patch b/var/spack/repos/builtin/packages/parmetis/pkg-parmetis-82409d68aa1d6cbc70740d0f35024aae17f7d5cb.patch new file mode 100644 index 0000000000..9651d55347 --- /dev/null +++ b/var/spack/repos/builtin/packages/parmetis/pkg-parmetis-82409d68aa1d6cbc70740d0f35024aae17f7d5cb.patch @@ -0,0 +1,35 @@ +From 82409d68aa1d6cbc70740d0f35024aae17f7d5cb Mon Sep 17 00:00:00 2001 +From: Sean Farley <sean@mcs.anl.gov> +Date: Tue, 20 Mar 2012 11:59:44 -0500 +Subject: [PATCH] parmetis: fix bug reported by jfettig; '<' to '<=' in xyzpart + +Hg-commit: 2dd2eae596acaabbc80e0ef875182616f868dbc2 +--- + libparmetis/xyzpart.c | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/libparmetis/xyzpart.c b/libparmetis/xyzpart.c +index 307aed9..3a2c289 100644 +--- a/libparmetis/xyzpart.c ++++ b/libparmetis/xyzpart.c +@@ -111,7 +111,7 @@ void IRBinCoordinates(ctrl_t *ctrl, graph_t *graph, idx_t ndims, real_t *xyz, + /* determine bucket counts */ + iset(nbins, 0, lcounts); + for (j=0, i=0; i<nvtxs;) { +- if (cand[i].key < emarkers[j+1]) { ++ if (cand[i].key <= emarkers[j+1]) { + lcounts[j]++; + i++; + } +@@ -158,7 +158,7 @@ void IRBinCoordinates(ctrl_t *ctrl, graph_t *graph, idx_t ndims, real_t *xyz, + + /* assign the coordinate to the appropriate bin */ + for (j=0, i=0; i<nvtxs;) { +- if (cand[i].key < emarkers[j+1]) { ++ if (cand[i].key <= emarkers[j+1]) { + bxyz[cand[i].val*ndims+k] = j; + i++; + } +-- +2.1.1.1.g1fb337f + diff --git a/var/spack/repos/builtin/packages/petsc/package.py b/var/spack/repos/builtin/packages/petsc/package.py index 87f700629d..7239baaf7f 100644 --- a/var/spack/repos/builtin/packages/petsc/package.py +++ b/var/spack/repos/builtin/packages/petsc/package.py @@ -1,39 +1,101 @@ +import os from spack import * + class Petsc(Package): - """PETSc is a suite of data structures and routines for the - scalable (parallel) solution of scientific applications modeled by - partial differential equations.""" + """ + PETSc is a suite of data structures and routines for the scalable (parallel) solution of scientific applications + modeled by partial differential equations. + """ homepage = "http://www.mcs.anl.gov/petsc/index.html" - url = "http://ftp.mcs.anl.gov/pub/petsc/release-snapshots/petsc-3.5.3.tar.gz" + url = "http://ftp.mcs.anl.gov/pub/petsc/release-snapshots/petsc-3.5.3.tar.gz" + version('3.6.3', '91dd3522de5a5ef039ff8f50800db606') version('3.5.3', 'd4fd2734661e89f18ac6014b5dd1ef2f') version('3.5.2', 'ad170802b3b058b5deb9cd1f968e7e13') version('3.5.1', 'a557e029711ebf425544e117ffa44d8f') + version('3.4.4', '7edbc68aa6d8d6a3295dd5f6c2f6979d') + + variant('shared', default=True, description='Enables the build of shared libraries') + variant('mpi', default=True, description='Activates MPI support') + variant('double', default=True, description='Switches between single and double precision') + + variant('metis', default=True, description='Activates support for metis and parmetis') + variant('hdf5', default=True, description='Activates support for HDF5 (only parallel)') + variant('boost', default=True, description='Activates support for Boost') + variant('hypre', default=True, description='Activates support for Hypre') + + # Virtual dependencies + depends_on('blas') + depends_on('lapack') + depends_on('mpi', when='+mpi') - depends_on("python @2.6:2.9") # requires Python for building + # Build dependencies + depends_on('python @2.6:2.7') - depends_on("boost") - depends_on("blas") - depends_on("lapack") - depends_on("hypre") - depends_on("parmetis") - depends_on("metis") - depends_on("hdf5+mpi") - depends_on("mpi") + # Other dependencies + depends_on('boost', when='+boost') + depends_on('metis', when='+metis') + + depends_on('hdf5+mpi', when='+hdf5+mpi') + depends_on('parmetis', when='+metis+mpi') + depends_on('hypre', when='+hypre+mpi') + + def mpi_dependent_options(self): + if '~mpi' in self.spec: + compiler_opts = [ + '--with-cc=%s' % os.environ['CC'], + '--with-cxx=%s' % (os.environ['CXX'] if self.compiler.cxx is not None else '0'), + '--with-fc=%s' % (os.environ['FC'] if self.compiler.fc is not None else '0'), + '--with-mpi=0' + ] + error_message_fmt = '\t{library} support requires "+mpi" to be activated' + + # If mpi is disabled (~mpi), it's an error to have any of these enabled. + # This generates a list of any such errors. + errors = [error_message_fmt.format(library=x) + for x in ('hdf5', 'hypre', 'parmetis') + if ('+'+x) in self.spec] + if errors: + errors = ['incompatible variants given'] + errors + raise RuntimeError('\n'.join(errors)) + else: + if self.compiler.name == "clang": + compiler_opts = [ + '--with-mpi=1', + '--with-cc=%s -Qunused-arguments' % join_path(self.spec['mpi'].prefix.bin, 'mpicc'), # Avoid confusing PETSc config by clang: warning: argument unused during compilation + '--with-cxx=%s -Qunused-arguments' % join_path(self.spec['mpi'].prefix.bin, 'mpic++'), + '--with-fc=%s' % join_path(self.spec['mpi'].prefix.bin, 'mpif90'), + '--with-f77=%s' % join_path(self.spec['mpi'].prefix.bin, 'mpif77'), + ] + else: + compiler_opts = [ + '--with-mpi=1', + '--with-mpi-dir=%s' % self.spec['mpi'].prefix, + ] + return compiler_opts def install(self, spec, prefix): - configure("--prefix=%s" % prefix, - "--with-blas-lib=%s/libblas.a" % spec['blas'].prefix.lib, - "--with-lapack-lib=%s/liblapack.a" % spec['lapack'].prefix.lib, - "--with-boost-dir=%s" % spec['boost'].prefix, - "--with-hypre-dir=%s" % spec['hypre'].prefix, - "--with-parmetis-dir=%s" % spec['parmetis'].prefix, - "--with-metis-dir=%s" % spec['metis'].prefix, - "--with-hdf5-dir=%s" % spec['hdf5'].prefix, - "--with-mpi-dir=%s" % spec['mpi'].prefix, - "--with-shared-libraries=0") + options = ['--with-debugging=0', + '--with-ssl=0'] + options.extend(self.mpi_dependent_options()) + options.extend([ + '--with-precision=%s' % ('double' if '+double' in spec else 'single'), + '--with-shared-libraries=%s' % ('1' if '+shared' in spec else '0'), + '--with-blas-lapack-dir=%s' % spec['lapack'].prefix + ]) + # Activates library support if needed + for library in ('metis', 'boost', 'hdf5', 'hypre', 'parmetis'): + options.append( + '--with-{library}={value}'.format(library=library, value=('1' if library in spec else '0')) + ) + if library in spec: + options.append( + '--with-{library}-dir={path}'.format(library=library, path=spec[library].prefix) + ) + + configure('--prefix=%s' % prefix, *options) # PETSc has its own way of doing parallel make. make('MAKE_NP=%s' % make_jobs, parallel=False) diff --git a/var/spack/repos/builtin/packages/proj/package.py b/var/spack/repos/builtin/packages/proj/package.py new file mode 100644 index 0000000000..797772f4f6 --- /dev/null +++ b/var/spack/repos/builtin/packages/proj/package.py @@ -0,0 +1,20 @@ +from spack import * + +class Proj(Package): + """Cartographic Projections""" + homepage = "https://github.com/OSGeo/proj.4/wiki" + url = "http://download.osgeo.org/proj/proj-4.9.2.tar.gz" + + version('4.9.2', '9843131676e31bbd903d60ae7dc76cf9') + version('4.9.1', '3cbb2a964fd19a496f5f4265a717d31c') + version('4.8.0', 'd815838c92a29179298c126effbb1537') + version('4.7.0', '927d34623b52e0209ba2bfcca18fe8cd') + version('4.6.1', '7dbaab8431ad50c25669fd3fb28dc493') + + # No dependencies + + def install(self, spec, prefix): + configure('--prefix=%s' % prefix) + + make() + make("install") diff --git a/var/spack/repos/builtin/packages/py-mpi4py/package.py b/var/spack/repos/builtin/packages/py-mpi4py/package.py index 8001689a18..f599205644 100644 --- a/var/spack/repos/builtin/packages/py-mpi4py/package.py +++ b/var/spack/repos/builtin/packages/py-mpi4py/package.py @@ -5,7 +5,9 @@ class PyMpi4py(Package): homepage = "https://pypi.python.org/pypi/mpi4py" url = "https://pypi.python.org/packages/source/m/mpi4py/mpi4py-1.3.1.tar.gz" + version('2.0.0', '4f7d8126d7367c239fd67615680990e3') version('1.3.1', 'dbe9d22bdc8ed965c23a7ceb6f32fc3c') + extends('python') depends_on('py-setuptools') depends_on('mpi') diff --git a/var/spack/repos/builtin/packages/py-nose/package.py b/var/spack/repos/builtin/packages/py-nose/package.py index e7c6cf0264..4fee99098e 100644 --- a/var/spack/repos/builtin/packages/py-nose/package.py +++ b/var/spack/repos/builtin/packages/py-nose/package.py @@ -1,11 +1,12 @@ from spack import * + class PyNose(Package): """nose extends the test loading and running features of unittest, making it easier to write, find and run tests.""" homepage = "https://pypi.python.org/pypi/nose" - url = "https://pypi.python.org/packages/source/n/nose/nose-1.3.4.tar.gz" + url = "https://pypi.python.org/packages/source/n/nose/nose-1.3.4.tar.gz" version('1.3.4', '6ed7169887580ddc9a8e16048d38274d') version('1.3.6', '0ca546d81ca8309080fc80cb389e7a16') diff --git a/var/spack/repos/builtin/packages/py-phonopy/package.py b/var/spack/repos/builtin/packages/py-phonopy/package.py new file mode 100644 index 0000000000..6d10fea74f --- /dev/null +++ b/var/spack/repos/builtin/packages/py-phonopy/package.py @@ -0,0 +1,18 @@ +from spack import * + +class PyPhonopy(Package): + """Phonopy is an open source package for phonon + calculations at harmonic and quasi-harmonic levels.""" + homepage = "http://atztogo.github.io/phonopy/index.html" + url = "http://sourceforge.net/projects/phonopy/files/phonopy/phonopy-1.10/phonopy-1.10.0.tar.gz" + + version('1.10.0', '973ed1bcea46e21b9bf747aab9061ff6') + + extends('python') + depends_on('py-numpy') + depends_on('py-scipy') + depends_on('py-matplotlib') + depends_on('py-pyyaml') + + def install(self, spec, prefix): + python('setup.py', 'install', '--home=%s' % prefix) diff --git a/var/spack/repos/builtin/packages/py-pyyaml/package.py b/var/spack/repos/builtin/packages/py-pyyaml/package.py new file mode 100644 index 0000000000..cae42f6e59 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-pyyaml/package.py @@ -0,0 +1,13 @@ +from spack import * + +class PyPyyaml(Package): + """PyYAML is a YAML parser and emitter for Python.""" + homepage = "http://pyyaml.org/wiki/PyYAML" + url = "http://pyyaml.org/download/pyyaml/PyYAML-3.11.tar.gz" + + version('3.11', 'f50e08ef0fe55178479d3a618efe21db') + + extends('python') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/repos/builtin/packages/py-wheel/package.py b/var/spack/repos/builtin/packages/py-wheel/package.py new file mode 100644 index 0000000000..3118e74519 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-wheel/package.py @@ -0,0 +1,15 @@ +from spack import * + +class PyWheel(Package): + """A built-package format for Python.""" + + homepage = "https://pypi.python.org/pypi/wheel" + url = "https://pypi.python.org/packages/source/w/wheel/wheel-0.26.0.tar.gz" + + version('0.26.0', '4cfc6e7e3dc7377d0164914623922a10') + + extends('python') + depends_on('py-setuptools') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/repos/builtin/packages/python/package.py b/var/spack/repos/builtin/packages/python/package.py index a1ce06feb0..4f55bc803e 100644 --- a/var/spack/repos/builtin/packages/python/package.py +++ b/var/spack/repos/builtin/packages/python/package.py @@ -1,11 +1,14 @@ +import functools +import glob +import inspect import os import re from contextlib import closing -from llnl.util.lang import match_predicate -from spack.util.environment import * -from spack import * import spack +from llnl.util.lang import match_predicate +from spack import * +from spack.util.environment import * class Python(Package): @@ -34,8 +37,9 @@ class Python(Package): env['PYTHONHOME'] = prefix env['MACOSX_DEPLOYMENT_TARGET'] = '10.6' - # Rest of install is pretty standard except setup.py needs to be able to read the CPPFLAGS - # and LDFLAGS as it scans for the library and headers to build + # Rest of install is pretty standard except setup.py needs to + # be able to read the CPPFLAGS and LDFLAGS as it scans for the + # library and headers to build configure_args= [ "--prefix=%s" % prefix, "--with-threads", @@ -55,6 +59,20 @@ class Python(Package): make() make("install") + # Modify compiler paths in configuration files. This is necessary for + # building site packages outside of spack + filter_file(r'([/s]=?)([\S=]*)/lib/spack/env(/[^\s/]*)?/(\S*)(\s)', + (r'\4\5'), + join_path(prefix.lib, 'python%d.%d' % self.version[:2], '_sysconfigdata.py')) + + python3_version = '' + if spec.satisfies('@3:'): + python3_version = '-%d.%dm' % self.version[:2] + makefile_filepath = join_path(prefix.lib, 'python%d.%d' % self.version[:2], 'config%s' % python3_version, 'Makefile') + filter_file(r'([/s]=?)([\S=]*)/lib/spack/env(/[^\s/]*)?/(\S*)(\s)', + (r'\4\5'), + makefile_filepath) + # ======================================================================== # Set up environment to make install easy for python extensions. @@ -75,12 +93,28 @@ class Python(Package): return os.path.join(self.python_lib_dir, 'site-packages') - def setup_dependent_environment(self, module, spec, ext_spec): - """Called before python modules' install() methods. + def setup_dependent_environment(self, spack_env, run_env, extension_spec): + # TODO: do this only for actual extensions. + + # Set PYTHONPATH to include site-packages dir for the + # extension and any other python extensions it depends on. + python_paths = [] + for d in extension_spec.traverse(): + if d.package.extends(self.spec): + python_paths.append(os.path.join(d.prefix, self.site_packages_dir)) + + pythonpath = ':'.join(python_paths) + spack_env.set('PYTHONPATH', pythonpath) + run_env.set('PYTHONPATH', pythonpath) + + + def modify_module(self, module, spec, ext_spec): + """ + Called before python modules' install() methods. In most cases, extensions will only need to have one line:: - python('setup.py', 'install', '--prefix=%s' % prefix) + python('setup.py', 'install', '--prefix=%s' % prefix) """ # Python extension builds can have a global python executable function if self.version >= Version("3.0.0") and self.version < Version("4.0.0"): @@ -96,15 +130,6 @@ class Python(Package): # Make the site packages directory if it does not exist already. mkdirp(module.site_packages_dir) - # Set PYTHONPATH to include site-packages dir for the - # extension and any other python extensions it depends on. - python_paths = [] - for d in ext_spec.traverse(): - if d.package.extends(self.spec): - python_paths.append(os.path.join(d.prefix, self.site_packages_dir)) - os.environ['PYTHONPATH'] = ':'.join(python_paths) - - # ======================================================================== # Handle specifics of activating and deactivating python modules. # ======================================================================== diff --git a/var/spack/repos/builtin/packages/qhull/package.py b/var/spack/repos/builtin/packages/qhull/package.py index f6712ced38..8f7c2f31b1 100644 --- a/var/spack/repos/builtin/packages/qhull/package.py +++ b/var/spack/repos/builtin/packages/qhull/package.py @@ -8,20 +8,20 @@ class Qhull(Package): implements the Quickhull algorithm for computing the convex hull. It handles roundoff errors from floating point arithmetic. It computes volumes, surface areas, and - approximations to the convex hull. - - Qhull does not support triangulation of non-convex surfaces, - mesh generation of non-convex objects, medium-sized inputs in - 9-D and higher, alpha shapes, weighted Voronoi diagrams, - Voronoi volumes, or constrained Delaunay triangulations.""" + approximations to the convex hull.""" homepage = "http://www.qhull.org" + version('7.2.0', 'e6270733a826a6a7c32b796e005ec3dc', + url="http://www.qhull.org/download/qhull-2015-src-7.2.0.tgz") + version('1.0', 'd0f978c0d8dfb2e919caefa56ea2953c', url="http://www.qhull.org/download/qhull-2012.1-src.tgz") # https://github.com/qhull/qhull/pull/5 - patch('qhull-iterator.patch') + patch('qhull-iterator.patch', when='@1.0') + + depends_on('cmake') def install(self, spec, prefix): with working_dir('spack-build', create=True): diff --git a/var/spack/repos/builtin/packages/qrupdate/package.py b/var/spack/repos/builtin/packages/qrupdate/package.py new file mode 100644 index 0000000000..5374d02c97 --- /dev/null +++ b/var/spack/repos/builtin/packages/qrupdate/package.py @@ -0,0 +1,18 @@ +from spack import * + +class Qrupdate(Package): + """qrupdate is a Fortran library for fast updates of QR and + Cholesky decompositions.""" + + homepage = "http://sourceforge.net/projects/qrupdate/" + url = "https://downloads.sourceforge.net/qrupdate/qrupdate-1.1.2.tar.gz" + + version('1.1.2', '6d073887c6e858c24aeda5b54c57a8c4') + + depends_on("blas") + depends_on("lapack") + + def install(self, spec, prefix): + # Build static and dynamic libraries + make("lib", "solib") + make("install", "PREFIX=%s" % prefix) diff --git a/var/spack/repos/builtin/packages/qt/package.py b/var/spack/repos/builtin/packages/qt/package.py index e8d843519d..d08e8e81e1 100644 --- a/var/spack/repos/builtin/packages/qt/package.py +++ b/var/spack/repos/builtin/packages/qt/package.py @@ -8,6 +8,9 @@ class Qt(Package): list_url = 'http://download.qt-project.org/official_releases/qt/' list_depth = 2 + version('5.4.2', 'fa1c4d819b401b267eb246a543a63ea5', + url='http://download.qt-project.org/official_releases/qt/5.4/5.4.2/single/qt-everywhere-opensource-src-5.4.2.tar.gz') + version('5.4.0', 'e8654e4b37dd98039ba20da7a53877e6', url='http://download.qt-project.org/official_releases/qt/5.4/5.4.0/single/qt-everywhere-opensource-src-5.4.0.tar.gz') @@ -23,6 +26,7 @@ class Qt(Package): version('3.3.8b', '9f05b4125cfe477cc52c9742c3c09009', url="http://download.qt.io/archive/qt/3/qt-x11-free-3.3.8b.tar.gz") + variant('mesa', default=False, description='depend on mesa') # Add patch for compile issues with qt3 found with use in the OpenSpeedShop project variant('krellpatch', default=False, description="build with openspeedshop based patch.") patch('qt3krell.patch', when='@3.3.8b+krellpatch') @@ -48,13 +52,16 @@ class Qt(Package): # depends_on("icu4c") # OpenGL hardware acceleration - depends_on("mesa", when='@4:') + depends_on("mesa", when='@4:+mesa') depends_on("libxcb") - def setup_dependent_environment(self, module, spec, dep_spec): - """Dependencies of Qt find it using the QTDIR environment variable.""" - os.environ['QTDIR'] = self.prefix + def setup_environment(self, spack_env, env): + env.set('QTDIR', self.prefix) + + + def setup_dependent_environment(self, spack_env, run_env, dspec): + spack_env.set('QTDIR', self.prefix) def patch(self): diff --git a/var/spack/repos/builtin/packages/ruby/package.py b/var/spack/repos/builtin/packages/ruby/package.py index 6b6242362c..7ff1898ce9 100644 --- a/var/spack/repos/builtin/packages/ruby/package.py +++ b/var/spack/repos/builtin/packages/ruby/package.py @@ -1,9 +1,8 @@ from spack import * -import spack -import os + class Ruby(Package): - """A dynamic, open source programming language with a focus on + """A dynamic, open source programming language with a focus on simplicity and productivity.""" homepage = "https://www.ruby-lang.org/" @@ -15,11 +14,23 @@ class Ruby(Package): def install(self, spec, prefix): configure("--prefix=%s" % prefix) - make() make("install") - def setup_dependent_environment(self, module, spec, ext_spec): + def setup_dependent_environment(self, spack_env, run_env, extension_spec): + # TODO: do this only for actual extensions. + # Set GEM_PATH to include dependent gem directories + ruby_paths = [] + for d in extension_spec.traverse(): + if d.package.extends(self.spec): + ruby_paths.append(d.prefix) + + spack_env.set_path('GEM_PATH', ruby_paths) + + # The actual installation path for this gem + spack_env.set('GEM_HOME', extension_spec.prefix) + + def modify_module(self, module, spec, ext_spec): """Called before ruby modules' install() methods. Sets GEM_HOME and GEM_PATH to values appropriate for the package being built. @@ -30,12 +41,3 @@ class Ruby(Package): # Ruby extension builds have global ruby and gem functions module.ruby = Executable(join_path(spec.prefix.bin, 'ruby')) module.gem = Executable(join_path(spec.prefix.bin, 'gem')) - - # Set GEM_PATH to include dependent gem directories - ruby_paths = [] - for d in ext_spec.traverse(): - if d.package.extends(self.spec): - ruby_paths.append(d.prefix) - os.environ['GEM_PATH'] = ':'.join(ruby_paths) - # The actual installation path for this gem - os.environ['GEM_HOME'] = ext_spec.prefix diff --git a/var/spack/repos/builtin/packages/silo/package.py b/var/spack/repos/builtin/packages/silo/package.py index 9eda11df15..d1aed78e0e 100644 --- a/var/spack/repos/builtin/packages/silo/package.py +++ b/var/spack/repos/builtin/packages/silo/package.py @@ -1,19 +1,28 @@ from spack import * class Silo(Package): - """Silo is a library for reading and writing a wide variety of scientific data to binary, disk files.""" + """Silo is a library for reading and writing a wide variety of scientific + data to binary, disk files.""" homepage = "http://wci.llnl.gov/simulation/computer-codes/silo" url = "https://wci.llnl.gov/content/assets/docs/simulation/computer-codes/silo/silo-4.8/silo-4.8.tar.gz" - #version('4.9', 'a83eda4f06761a86726e918fc55e782a') version('4.8', 'b1cbc0e7ec435eb656dc4b53a23663c9') - depends_on("hdf5@:1.8.12") + variant('fortran', default=True, description='Enable Fortran support') + + depends_on("hdf5") def install(self, spec, prefix): - configure("--prefix=%s" % prefix, - "--with-hdf5=%s" %spec['hdf5'].prefix) + config_args = [ + '--enable-fortran' if '+fortran' in spec else '--disable-fortran', + ] + + configure( + "--prefix=%s" % prefix, + "--with-hdf5=%s,%s" % (spec['hdf5'].prefix.include, spec['hdf5'].prefix.lib), + "--with-zlib=%s,%s" % (spec['zlib'].prefix.include, spec['zlib'].prefix.lib), + *config_args) make() make("install") diff --git a/var/spack/repos/builtin/packages/suite-sparse/package.py b/var/spack/repos/builtin/packages/suite-sparse/package.py new file mode 100644 index 0000000000..b57f9967c3 --- /dev/null +++ b/var/spack/repos/builtin/packages/suite-sparse/package.py @@ -0,0 +1,26 @@ +from spack import * + + +class SuiteSparse(Package): + """ + SuiteSparse is a suite of sparse matrix algorithms + """ + homepage = 'http://faculty.cse.tamu.edu/davis/suitesparse.html' + url = 'http://faculty.cse.tamu.edu/davis/SuiteSparse/SuiteSparse-4.5.1.tar.gz' + + version('4.5.1', 'f0ea9aad8d2d1ffec66a5b6bfeff5319') + + depends_on('blas') + depends_on('lapack') + + depends_on('metis@5.1.0', when='@4.5.1') + + def install(self, spec, prefix): + # The build system of SuiteSparse is quite old-fashioned + # It's basically a plain Makefile which include an header (SuiteSparse_config/SuiteSparse_config.mk) + # with a lot of convoluted logic in it. + # Any kind of customization will need to go through filtering of that file + + # FIXME : this actually uses the current workaround + # FIXME : (blas / lapack always provide libblas and liblapack as aliases) + make('install', 'INSTALL=%s' % prefix, 'BLAS=-lblas', 'LAPACK=-llapack') diff --git a/var/spack/repos/builtin/packages/superlu-dist/package.py b/var/spack/repos/builtin/packages/superlu-dist/package.py new file mode 100644 index 0000000000..c4c76909b3 --- /dev/null +++ b/var/spack/repos/builtin/packages/superlu-dist/package.py @@ -0,0 +1,63 @@ +from spack import * + +class SuperluDist(Package): + """A general purpose library for the direct solution of large, sparse, nonsymmetric systems of linear equations on high performance machines.""" + homepage = "http://crd-legacy.lbl.gov/~xiaoye/SuperLU/" + url = "http://crd-legacy.lbl.gov/~xiaoye/SuperLU/superlu_dist_4.1.tar.gz" + + version('4.3', 'ee66c84e37b4f7cc557771ccc3dc43ae') + version('4.2', 'ae9fafae161f775fbac6eba11e530a65') + version('4.1', '4edee38cc29f687bd0c8eb361096a455') + version('4.0', 'c0b98b611df227ae050bc1635c6940e0') + + depends_on ('mpi') + depends_on ('blas') + depends_on ('lapack') + depends_on ('parmetis') + depends_on ('metis') + + def install(self, spec, prefix): + makefile_inc = [] + makefile_inc.extend([ + 'PLAT = _mac_x', + 'DSuperLUroot = %s' % self.stage.source_path, #self.stage.path, prefix + 'DSUPERLULIB = $(DSuperLUroot)/lib/libsuperlu_dist.a', + 'BLASDEF = -DUSE_VENDOR_BLAS', + 'BLASLIB = -L%s -llapack %s -lblas' % (spec['lapack'].prefix.lib, spec['blas'].prefix.lib), # FIXME: avoid hardcoding blas/lapack lib names + 'METISLIB = -L%s -lmetis' % spec['metis'].prefix.lib, + 'PARMETISLIB = -L%s -lparmetis' % spec['parmetis'].prefix.lib, + 'FLIBS =', + 'LIBS = $(DSUPERLULIB) $(BLASLIB) $(PARMETISLIB) $(METISLIB)', + 'ARCH = ar', + 'ARCHFLAGS = cr', + 'RANLIB = true', + 'CC = mpicc', # FIXME avoid hardcoding MPI compiler names + 'CFLAGS = -fPIC -std=c99 -O2 -I%s -I%s' %(spec['parmetis'].prefix.include, spec['metis'].prefix.include), + 'NOOPTS = -fPIC -std=c99', + 'FORTRAN = mpif77', + 'F90FLAGS = -O2', + 'LOADER = mpif77', + 'LOADOPTS =', + 'CDEFS = -DAdd_' + ]) + + #with working_dir('src'): + with open('make.inc', 'w') as fh: + fh.write('\n'.join(makefile_inc)) + + make("lib", parallel=False) + + # FIXME: + # cd "EXAMPLE" do + # system "make" + + # need to install by hand + headers_location = join_path(self.prefix.include,'superlu_dist') + mkdirp(headers_location) + # FIXME: fetch all headers in the folder automatically + for header in ['Cnames.h','cublas_utils.h','dcomplex.h','html_mainpage.h','machines.h','old_colamd.h','psymbfact.h','superlu_ddefs.h','superlu_defs.h','superlu_enum_consts.h','superlu_zdefs.h','supermatrix.h','util_dist.h']: + superludist_header = join_path(self.stage.source_path, 'SRC/',header) + install(superludist_header, headers_location) + + superludist_lib = join_path(self.stage.source_path, 'lib/libsuperlu_dist.a') + install(superludist_lib,self.prefix.lib) diff --git a/var/spack/repos/builtin/packages/tetgen/package.py b/var/spack/repos/builtin/packages/tetgen/package.py new file mode 100644 index 0000000000..30c2b76655 --- /dev/null +++ b/var/spack/repos/builtin/packages/tetgen/package.py @@ -0,0 +1,28 @@ +from spack import * + +class Tetgen(Package): + """TetGen is a program and library that can be used to generate tetrahedral + meshes for given 3D polyhedral domains. TetGen generates exact constrained + Delaunay tetrahedralizations, boundary conforming Delaunay meshes, and + Voronoi paritions.""" + + homepage = "http://www.tetgen.org" + url = "http://www.tetgen.org/files/tetgen1.4.3.tar.gz" + + version('1.4.3', 'd6a4bcdde2ac804f7ec66c29dcb63c18') + + # TODO: Make this a build dependency once build dependencies are supported + # (see: https://github.com/LLNL/spack/pull/378). + depends_on('cmake@2.8.7:', when='@1.5.0:') + + def install(self, spec, prefix): + make('tetgen', 'tetlib') + + mkdirp(prefix.bin) + install('tetgen', prefix.bin) + + mkdirp(prefix.include) + install('tetgen.h', prefix.include) + + mkdirp(prefix.lib) + install('libtet.a', prefix.lib) diff --git a/var/spack/repos/builtin/packages/texinfo/package.py b/var/spack/repos/builtin/packages/texinfo/package.py index a83c10c0c1..6cf8d79072 100644 --- a/var/spack/repos/builtin/packages/texinfo/package.py +++ b/var/spack/repos/builtin/packages/texinfo/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://llnl.github.io/spack +# For details, see https://software.llnl.gov/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/repos/builtin/packages/thrift/package.py b/var/spack/repos/builtin/packages/thrift/package.py index 0e15052f64..6430f40e80 100644 --- a/var/spack/repos/builtin/packages/thrift/package.py +++ b/var/spack/repos/builtin/packages/thrift/package.py @@ -12,33 +12,44 @@ class Thrift(Package): version('0.9.2', '89f63cc4d0100912f4a1f8a9dee63678') - extends("python") - - depends_on("autoconf") - depends_on("automake") - depends_on("bison") - depends_on("boost") - depends_on("flex") - depends_on("jdk") - depends_on("libtool") - depends_on("openssl") - depends_on("python") - - # Compilation fails for most languages, fortunately cpp installs fine - # All other languages (yes, including C) are omitted until someone needs them + # Currently only support for c-family and python + variant('c', default=True, description="Build support for C-family languages") + variant('python', default=True, description="Build support for python") + + depends_on('jdk') + depends_on('autoconf') + depends_on('automake') + depends_on('libtool') + depends_on('boost@1.53:') + depends_on('bison') + depends_on('flex') + depends_on('openssl') + + # Variant dependencies + extends('python', when='+python') + + depends_on('zlib', when='+c') + depends_on('libevent', when='+c') + def install(self, spec, prefix): - env["PY_PREFIX"] = prefix - env["JAVA_PREFIX"] = prefix - - configure("--prefix=%s" % prefix, - "--with-boost=%s" % spec['boost'].prefix, - "--with-c=no", - "--with-go=no", - "--with-python=yes", - "--with-lua=no", - "--with-php=no", - "--with-qt4=no", - "--enable-tests=no") + env['PY_PREFIX'] = prefix + env['JAVA_HOME'] = spec['jdk'].prefix + + # configure options + options = ['--prefix=%s' % prefix] + + options.append('--with-boost=%s' % spec['boost'].prefix) + options.append('--enable-tests=no') + + options.append('--with-c=%s' % ('yes' if '+c' in spec else 'no')) + options.append('--with-python=%s' % ('yes' if '+python' in spec else 'no')) + options.append('--with-java=%s' % ('yes' if '+java' in spec else 'no')) + options.append('--with-go=%s' % ('yes' if '+go' in spec else 'no')) + options.append('--with-lua=%s' % ('yes' if '+lua' in spec else 'no')) + options.append('--with-php=%s' % ('yes' if '+php' in spec else 'no')) + options.append('--with-qt4=%s' % ('yes' if '+qt4' in spec else 'no')) + + configure(*options) make() make("install") diff --git a/var/spack/repos/builtin/packages/tmux/package.py b/var/spack/repos/builtin/packages/tmux/package.py index 23d36db427..f2067d1366 100644 --- a/var/spack/repos/builtin/packages/tmux/package.py +++ b/var/spack/repos/builtin/packages/tmux/package.py @@ -7,10 +7,11 @@ class Tmux(Package): do a lot more. """ - homepage = "http://tmux.sourceforge.net" - url = "http://downloads.sourceforge.net/project/tmux/tmux/tmux-1.9/tmux-1.9a.tar.gz" + homepage = "http://tmux.github.io" + url = "https://github.com/tmux/tmux/releases/download/2.1/tmux-2.1.tar.gz" version('1.9a', 'b07601711f96f1d260b390513b509a2d') + version('2.1', '74a2855695bccb51b6e301383ad4818c') depends_on('libevent') depends_on('ncurses') diff --git a/var/spack/repos/builtin/packages/triangle/package.py b/var/spack/repos/builtin/packages/triangle/package.py new file mode 100644 index 0000000000..f65d93776d --- /dev/null +++ b/var/spack/repos/builtin/packages/triangle/package.py @@ -0,0 +1,20 @@ +from spack import * + +class Triangle(Package): + """Triangle is a two-dimensional mesh generator and Delaunay + triangulator. Triangle generates exact Delaunay triangulations, + constrained Delaunay triangulations, conforming Delaunay + triangulations, Voronoi diagrams, and high-quality triangular + meshes.""" + + homepage = "http://www.cs.cmu.edu/~quake/triangle.html" + url = "http://www.netlib.org/voronoi/triangle.zip" + + version('1.6', '10aff8d7950f5e0e2fb6dd2e340be2c9') + + def install(self, spec, prefix): + make() + mkdirp(prefix.bin) + + install('triangle', prefix.bin) + install('showme', prefix.bin) diff --git a/var/spack/repos/builtin/packages/trilinos/package.py b/var/spack/repos/builtin/packages/trilinos/package.py index 7c43f796a4..edc40476e3 100644 --- a/var/spack/repos/builtin/packages/trilinos/package.py +++ b/var/spack/repos/builtin/packages/trilinos/package.py @@ -10,40 +10,44 @@ class Trilinos(Package): homepage = "https://trilinos.org/" url = "http://trilinos.csbsju.edu/download/files/trilinos-12.2.1-Source.tar.gz" + version('12.4.2', '7c830f7f0f68b8ad324690603baf404e') version('12.2.1', '6161926ea247863c690e927687f83be9') version('12.0.1', 'bd99741d047471e127b8296b2ec08017') version('11.14.3', '2f4f83f8333e4233c57d0f01c4b57426') version('11.14.2', 'a43590cf896c677890d75bfe75bc6254') version('11.14.1', '40febc57f76668be8b6a77b7607bb67f') - variant('mpi', default=True, description='Add a dependency on MPI and enables MPI dependent packages') + variant('shared', default=True, description='Enables the build of shared libraries') + variant('debug', default=False, description='Builds a debug version of the libraries') # Everything should be compiled with -fpic depends_on('blas') depends_on('lapack') depends_on('boost') - depends_on('netcdf') depends_on('matio') depends_on('glm') depends_on('swig') - depends_on('mpi', when='+mpi') - def install(self, spec, prefix): + # MPI related dependencies + depends_on('mpi') + depends_on('netcdf+mpi') + + depends_on('python') # Needs py-numpy activated - options = [ - '-DTrilinos_ENABLE_ALL_PACKAGES:BOOL=ON', - '-DTrilinos_ENABLE_TESTS:BOOL=OFF', - '-DTrilinos_ENABLE_EXAMPLES:BOOL=OFF', - '-DBUILD_SHARED_LIBS:BOOL=ON', - '-DBLAS_LIBRARY_DIRS:PATH=%s' % spec['blas'].prefix, - '-DLAPACK_LIBRARY_DIRS:PATH=%s' % spec['lapack'].prefix - ] - if '+mpi' in spec: - mpi_options = ['-DTPL_ENABLE_MPI:BOOL=ON'] - options.extend(mpi_options) - - # -DCMAKE_INSTALL_PREFIX and all the likes... + def install(self, spec, prefix): + options = [] options.extend(std_cmake_args) + + options.extend(['-DTrilinos_ENABLE_ALL_PACKAGES:BOOL=ON', + '-DTrilinos_ENABLE_TESTS:BOOL=OFF', + '-DTrilinos_ENABLE_EXAMPLES:BOOL=OFF', + '-DCMAKE_BUILD_TYPE:STRING=%s' % ('Debug' if '+debug' in spec else 'Release'), + '-DBUILD_SHARED_LIBS:BOOL=%s' % ('ON' if '+shared' in spec else 'OFF'), + '-DTPL_ENABLE_MPI:STRING=ON', + '-DBLAS_LIBRARY_DIRS:PATH=%s' % spec['blas'].prefix, + '-DLAPACK_LIBRARY_DIRS:PATH=%s' % spec['lapack'].prefix + ]) + with working_dir('spack-build', create=True): cmake('..', *options) make() diff --git a/var/spack/repos/builtin/packages/udunits2/package.py b/var/spack/repos/builtin/packages/udunits2/package.py new file mode 100644 index 0000000000..9954a733bb --- /dev/null +++ b/var/spack/repos/builtin/packages/udunits2/package.py @@ -0,0 +1,16 @@ +from spack import * + +class Udunits2(Package): + """Automated units conversion""" + + homepage = "http://www.unidata.ucar.edu/software/udunits" + url = "ftp://ftp.unidata.ucar.edu/pub/udunits/udunits-2.2.20.tar.gz" + + version('2.2.20', '1586b70a49dfe05da5fcc29ef239dce0') + + depends_on('expat') + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix) + make() + make("install") diff --git a/var/spack/repos/builtin/packages/xz/package.py b/var/spack/repos/builtin/packages/xz/package.py index ba6c9733a7..fdcac95345 100644 --- a/var/spack/repos/builtin/packages/xz/package.py +++ b/var/spack/repos/builtin/packages/xz/package.py @@ -8,11 +8,9 @@ class Xz(Package): homepage = "http://tukaani.org/xz/" url = "http://tukaani.org/xz/xz-5.2.0.tar.bz2" - version('5.2.0', '867cc8611760240ebf3440bd6e170bb9', - url = 'http://tukaani.org/xz/xz-5.2.0.tar.bz2') - version('5.2.2', 'f90c9a0c8b259aee2234c4e0d7fd70af', - url = 'http://tukaani.org/xz/xz-5.2.2.tar.bz2') - + version('5.2.0', '867cc8611760240ebf3440bd6e170bb9') + version('5.2.2', 'f90c9a0c8b259aee2234c4e0d7fd70af') + def install(self, spec, prefix): configure("--prefix=%s" % prefix) make() diff --git a/var/spack/repos/builtin/packages/zfp/package.py b/var/spack/repos/builtin/packages/zfp/package.py new file mode 100644 index 0000000000..620fe9d456 --- /dev/null +++ b/var/spack/repos/builtin/packages/zfp/package.py @@ -0,0 +1,26 @@ +from spack import * + +class Zfp(Package): + """zfp is an open source C library for compressed floating-point arrays that supports + very high throughput read and write random acces, target error bounds or bit rates. + Although bit-for-bit lossless compression is not always possible, zfp is usually + accurate to within machine epsilon in near-lossless mode, and is often orders of + magnitude more accurate than other lossy compressors. + """ + + homepage = "http://computation.llnl.gov/projects/floating-point-compression" + url = "http://computation.llnl.gov/projects/floating-point-compression/download/zfp-0.5.0.tar.gz" + + version('0.5.0', '2ab29a852e65ad85aae38925c5003654') + + def install(self, spec, prefix): + make("shared") + + # No install provided + mkdirp(prefix.lib) + mkdirp(prefix.include) + install('lib/libzfp.so', prefix.lib) + install('inc/zfp.h', prefix.include) + install('inc/types.h', prefix.include) + install('inc/bitstream.h', prefix.include) + install('inc/system.h', prefix.include) |