diff options
242 files changed, 3269 insertions, 626 deletions
diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000000..ab379be486 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,26 @@ +language: python +python: + - "2.6" + - "2.7" + +# Use new Travis infrastructure (Docker can't sudo yet) +sudo: false + +# No need to install any deps. +install: true + +before_install: + # Need this for the git tests to succeed. + - git config --global user.email "spack@example.com" + - git config --global user.name "Test User" + +script: + - . share/spack/setup-env.sh + - spack test + +notifications: + email: + recipients: + - tgamblin@llnl.gov + on_success: change + on_failure: always @@ -5,7 +5,7 @@ This file is part of Spack. Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. LLNL-CODE-647188 -For details, see https://scalability-llnl.github.io/spack +For details, see https://github.com/llnl/spack This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License (as published by @@ -1,6 +1,8 @@  ============ +[](https://travis-ci.org/LLNL/spack) + Spack is a package management tool designed to support multiple versions and configurations of software on a wide variety of platforms and environments. It was designed for large supercomputing centers, @@ -17,20 +19,26 @@ written in pure Python, and specs allow package authors to write a single build script for many different builds of the same package. See the -[Feature Overview](http://scalability-llnl.github.io/spack/features.html) +[Feature Overview](http://llnl.github.io/spack/features.html) for examples and highlights. To install spack and install your first package: - $ git clone https://github.com/scalability-llnl/spack.git + $ git clone https://github.com/llnl/spack.git $ cd spack/bin $ ./spack install libelf Documentation ---------------- -[Full documentation](http://scalability-llnl.github.io/spack) -for Spack is also available. +[**Full documentation**](http://llnl.github.io/spack) for Spack is +the first place to look. + +See also: + * [Technical paper](http://www.computer.org/csdl/proceedings/sc/2015/3723/00/2807623.pdf) and + [slides](https://tgamblin.github.io/files/Gamblin-Spack-SC15-Talk.pdf) on Spack's design and implementation. + * [Short presentation](https://tgamblin.github.io/files/Gamblin-Spack-Lightning-Talk-BOF-SC15.pdf) from the *Getting Scientific Software Installed* BOF session at Supercomputing 2015. + Get Involved! ------------------------ @@ -62,10 +70,19 @@ latest stable release. Authors ---------------- -Many thanks go to Spack's [contributors](https://github.com/scalability-llnl/spack/graphs/contributors). +Many thanks go to Spack's [contributors](https://github.com/llnl/spack/graphs/contributors). Spack was originally written by Todd Gamblin, tgamblin@llnl.gov. +### Citing Spack + +If you are referencing Spack in a publication, please cite the following paper: + + * Todd Gamblin, Matthew P. LeGendre, Michael R. Collette, Gregory L. Lee, + Adam Moody, Bronis R. de Supinski, and W. Scott Futral. + [**The Spack Package Manager: Bringing Order to HPC Software Chaos**](http://www.computer.org/csdl/proceedings/sc/2015/3723/00/2807623.pdf). + In *Supercomputing 2015 (SC’15)*, Austin, Texas, November 15-20 2015. LLNL-CONF-669890. + Release ---------------- Spack is released under an LGPL license. For more details see the @@ -7,7 +7,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify @@ -38,6 +38,23 @@ SPACK_PREFIX = os.path.dirname(os.path.dirname(SPACK_FILE)) # Allow spack libs to be imported in our scripts SPACK_LIB_PATH = os.path.join(SPACK_PREFIX, "lib", "spack") sys.path.insert(0, SPACK_LIB_PATH) +SPACK_EXTERNAL_LIBS = os.path.join(SPACK_LIB_PATH, "external") +sys.path.insert(0, SPACK_EXTERNAL_LIBS) + +# Quick and dirty check to clean orphaned .pyc files left over from +# previous revisions. These files were present in earlier versions of +# Spack, were removed, but shadow system modules that Spack still +# imports. If we leave them, Spack will fail in mysterious ways. +# TODO: more elegant solution for orphaned pyc files. +orphaned_pyc_files = [os.path.join(SPACK_EXTERNAL_LIBS, n) + for n in ('functools.pyc', 'ordereddict.pyc')] +for pyc_file in orphaned_pyc_files: + if not os.path.exists(pyc_file): + continue + try: + os.remove(pyc_file) + except OSError as e: + print "WARNING: Spack may fail mysteriously. Couldn't remove orphaned .pyc file: %s" % pyc # If there is no working directory, use the spack prefix. try: diff --git a/bin/spack-python b/bin/spack-python index 8a4b9c175d..e0745e8c58 100755 --- a/bin/spack-python +++ b/bin/spack-python @@ -7,7 +7,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/docs/conf.py b/lib/spack/docs/conf.py index bce9ef0e94..3d2a8251aa 100644 --- a/lib/spack/docs/conf.py +++ b/lib/spack/docs/conf.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify @@ -43,6 +43,7 @@ import subprocess # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. sys.path.insert(0, os.path.abspath('exts')) +sys.path.insert(0, os.path.abspath('../external')) # Add the Spack bin directory to the path so that we can use its output in docs. spack_root = '../../..' diff --git a/lib/spack/docs/exts/sphinxcontrib/__init__.py b/lib/spack/docs/exts/sphinxcontrib/__init__.py index 838d616eb4..298856746c 100644 --- a/lib/spack/docs/exts/sphinxcontrib/__init__.py +++ b/lib/spack/docs/exts/sphinxcontrib/__init__.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/docs/exts/sphinxcontrib/programoutput.py b/lib/spack/docs/exts/sphinxcontrib/programoutput.py index ff006acf72..f0fa045c86 100644 --- a/lib/spack/docs/exts/sphinxcontrib/programoutput.py +++ b/lib/spack/docs/exts/sphinxcontrib/programoutput.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/docs/getting_started.rst b/lib/spack/docs/getting_started.rst index d958d9e74a..67ca18e71a 100644 --- a/lib/spack/docs/getting_started.rst +++ b/lib/spack/docs/getting_started.rst @@ -5,11 +5,11 @@ Download -------------------- Getting spack is easy. You can clone it from the `github repository -<https://github.com/scalability-llnl/spack>`_ using this command: +<https://github.com/llnl/spack>`_ using this command: .. code-block:: sh - $ git clone https://github.com/scalability-llnl/spack.git + $ git clone https://github.com/llnl/spack.git This will create a directory called ``spack``. We'll assume that the full path to this directory is in the ``SPACK_ROOT`` environment @@ -22,7 +22,7 @@ go: $ spack install libelf For a richer experience, use Spack's `shell support -<http://scalability-llnl.github.io/spack/basic_usage.html#environment-modules>`_: +<http://llnl.github.io/spack/basic_usage.html#environment-modules>`_: .. code-block:: sh diff --git a/lib/spack/docs/index.rst b/lib/spack/docs/index.rst index 97c8361421..79757208c9 100644 --- a/lib/spack/docs/index.rst +++ b/lib/spack/docs/index.rst @@ -24,12 +24,12 @@ maintain a single file for many different builds of the same package. See the :doc:`features` for examples and highlights. Get spack from the `github repository -<https://github.com/scalability-llnl/spack>`_ and install your first +<https://github.com/llnl/spack>`_ and install your first package: .. code-block:: sh - $ git clone https://github.com/scalability-llnl/spack.git + $ git clone https://github.com/llnl/spack.git $ cd spack/bin $ ./spack install libelf diff --git a/lib/spack/env/cc b/lib/spack/env/cc index fa85bb595e..75a63f6fac 100755 --- a/lib/spack/env/cc +++ b/lib/spack/env/cc @@ -7,7 +7,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/external/__init__.py b/lib/spack/external/__init__.py index 0578022210..7a89a1ac67 100644 --- a/lib/spack/external/__init__.py +++ b/lib/spack/external/__init__.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/external/functools.py b/lib/spack/external/functools_backport.py index 19f0903c82..19f0903c82 100644 --- a/lib/spack/external/functools.py +++ b/lib/spack/external/functools_backport.py diff --git a/lib/spack/external/ordereddict.py b/lib/spack/external/ordereddict_backport.py index 8ddad1477e..8ddad1477e 100644 --- a/lib/spack/external/ordereddict.py +++ b/lib/spack/external/ordereddict_backport.py diff --git a/lib/spack/llnl/util/filesystem.py b/lib/spack/llnl/util/filesystem.py index 029a7536df..24cfbfde71 100644 --- a/lib/spack/llnl/util/filesystem.py +++ b/lib/spack/llnl/util/filesystem.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify @@ -222,7 +222,7 @@ def working_dir(dirname, **kwargs): def touch(path): """Creates an empty file at the specified path.""" - with closing(open(path, 'a')) as file: + with open(path, 'a') as file: os.utime(path, None) diff --git a/lib/spack/llnl/util/lang.py b/lib/spack/llnl/util/lang.py index 9e1bef18ca..cc87b7eac2 100644 --- a/lib/spack/llnl/util/lang.py +++ b/lib/spack/llnl/util/lang.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify @@ -87,10 +87,7 @@ def index_by(objects, *funcs): result = {} for o in objects: key = f(o) - if key not in result: - result[key] = [o] - else: - result[key].append(o) + result.setdefault(key, []).append(o) for key, objects in result.items(): result[key] = index_by(objects, *funcs[1:]) diff --git a/lib/spack/llnl/util/link_tree.py b/lib/spack/llnl/util/link_tree.py index 583f077b79..6ae8aff75c 100644 --- a/lib/spack/llnl/util/link_tree.py +++ b/lib/spack/llnl/util/link_tree.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/llnl/util/lock.py b/lib/spack/llnl/util/lock.py new file mode 100644 index 0000000000..a7a9bf6b19 --- /dev/null +++ b/lib/spack/llnl/util/lock.py @@ -0,0 +1,175 @@ +############################################################################## +# Copyright (c) 2013-2015, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +import os +import fcntl +import errno +import time +import socket + +# Default timeout in seconds, after which locks will raise exceptions. +_default_timeout = 60 + +# Sleep time per iteration in spin loop (in seconds) +_sleep_time = 1e-5 + + +class Lock(object): + def __init__(self,file_path): + self._file_path = file_path + self._fd = None + self._reads = 0 + self._writes = 0 + + + def _lock(self, op, timeout): + """This takes a lock using POSIX locks (``fnctl.lockf``). + + The lock is implemented as a spin lock using a nonblocking + call to lockf(). + + On acquiring an exclusive lock, the lock writes this process's + pid and host to the lock file, in case the holding process + needs to be killed later. + + If the lock times out, it raises a ``LockError``. + """ + start_time = time.time() + while (time.time() - start_time) < timeout: + try: + if self._fd is None: + self._fd = os.open(self._file_path, os.O_RDWR) + + fcntl.lockf(self._fd, op | fcntl.LOCK_NB) + if op == fcntl.LOCK_EX: + os.write(self._fd, "pid=%s,host=%s" % (os.getpid(), socket.getfqdn())) + return + + except IOError as error: + if error.errno == errno.EAGAIN or error.errno == errno.EACCES: + pass + else: + raise + time.sleep(_sleep_time) + + raise LockError("Timed out waiting for lock.") + + + def _unlock(self): + """Releases a lock using POSIX locks (``fcntl.lockf``) + + Releases the lock regardless of mode. Note that read locks may + be masquerading as write locks, but this removes either. + + """ + fcntl.lockf(self._fd,fcntl.LOCK_UN) + os.close(self._fd) + self._fd = None + + + def acquire_read(self, timeout=_default_timeout): + """Acquires a recursive, shared lock for reading. + + Read and write locks can be acquired and released in arbitrary + order, but the POSIX lock is held until all local read and + write locks are released. + + Returns True if it is the first acquire and actually acquires + the POSIX lock, False if it is a nested transaction. + + """ + if self._reads == 0 and self._writes == 0: + self._lock(fcntl.LOCK_SH, timeout) # can raise LockError. + self._reads += 1 + return True + else: + self._reads += 1 + return False + + + def acquire_write(self, timeout=_default_timeout): + """Acquires a recursive, exclusive lock for writing. + + Read and write locks can be acquired and released in arbitrary + order, but the POSIX lock is held until all local read and + write locks are released. + + Returns True if it is the first acquire and actually acquires + the POSIX lock, False if it is a nested transaction. + + """ + if self._writes == 0: + self._lock(fcntl.LOCK_EX, timeout) # can raise LockError. + self._writes += 1 + return True + else: + self._writes += 1 + return False + + + def release_read(self): + """Releases a read lock. + + Returns True if the last recursive lock was released, False if + there are still outstanding locks. + + Does limited correctness checking: if a read lock is released + when none are held, this will raise an assertion error. + + """ + assert self._reads > 0 + + if self._reads == 1 and self._writes == 0: + self._unlock() # can raise LockError. + self._reads -= 1 + return True + else: + self._reads -= 1 + return False + + + def release_write(self): + """Releases a write lock. + + Returns True if the last recursive lock was released, False if + there are still outstanding locks. + + Does limited correctness checking: if a read lock is released + when none are held, this will raise an assertion error. + + """ + assert self._writes > 0 + + if self._writes == 1 and self._reads == 0: + self._unlock() # can raise LockError. + self._writes -= 1 + return True + else: + self._writes -= 1 + return False + + +class LockError(Exception): + """Raised when an attempt to acquire a lock times out.""" + pass diff --git a/lib/spack/llnl/util/tty/__init__.py b/lib/spack/llnl/util/tty/__init__.py index 48368543ff..203f429a48 100644 --- a/lib/spack/llnl/util/tty/__init__.py +++ b/lib/spack/llnl/util/tty/__init__.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/llnl/util/tty/colify.py b/lib/spack/llnl/util/tty/colify.py index db928444c7..5545cf0311 100644 --- a/lib/spack/llnl/util/tty/colify.py +++ b/lib/spack/llnl/util/tty/colify.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/llnl/util/tty/color.py b/lib/spack/llnl/util/tty/color.py index 0d09303da0..167a99d3c2 100644 --- a/lib/spack/llnl/util/tty/color.py +++ b/lib/spack/llnl/util/tty/color.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/llnl/util/tty/log.py b/lib/spack/llnl/util/tty/log.py index 5a52d45bc7..2819cd40df 100644 --- a/lib/spack/llnl/util/tty/log.py +++ b/lib/spack/llnl/util/tty/log.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index 6e8d41895f..02eeed01fa 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify @@ -56,6 +56,12 @@ packages_path = join_path(var_path, "packages") db = PackageDB(packages_path) # +# Set up the installed packages database +# +from spack.database import Database +installed_db = Database(install_path) + +# # Paths to mock files for testing. # mock_packages_path = join_path(var_path, "mock_packages") diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 0c4b605e91..2701fab90c 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify @@ -23,13 +23,13 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import os -import platform as py_platform +import re +import platform from llnl.util.lang import memoized import spack import spack.error as serr -from spack.version import Version class InvalidSysTypeError(serr.SpackError): @@ -59,14 +59,11 @@ def get_sys_type_from_environment(): return os.environ.get('SYS_TYPE') -def get_mac_sys_type(): - """Return a Mac OS SYS_TYPE or None if this isn't a mac.""" - mac_ver = py_platform.mac_ver()[0] - if not mac_ver: - return None - - return "macosx_%s_%s" % ( - Version(mac_ver).up_to(2), py_platform.machine()) +def get_sys_type_from_platform(): + """Return the architecture from Python's platform module.""" + sys_type = platform.system() + '-' + platform.machine() + sys_type = re.sub(r'[^\w-]', '_', sys_type) + return sys_type.lower() @memoized @@ -74,7 +71,7 @@ def sys_type(): """Returns a SysType for the current machine.""" methods = [get_sys_type_from_spack_globals, get_sys_type_from_environment, - get_mac_sys_type] + get_sys_type_from_platform] # search for a method that doesn't return None sys_type = None diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index a133faa629..dac25d9940 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -296,4 +296,9 @@ def fork(pkg, function): # message. Just make the parent exit with an error code. pid, returncode = os.waitpid(pid, 0) if returncode != 0: - sys.exit(1) + raise InstallError("Installation process had nonzero exit code." + .format(str(returncode))) + + +class InstallError(spack.error.SpackError): + """Raised when a package fails to install""" diff --git a/lib/spack/spack/cmd/__init__.py b/lib/spack/spack/cmd/__init__.py index b96ac5af51..926e7ac14a 100644 --- a/lib/spack/spack/cmd/__init__.py +++ b/lib/spack/spack/cmd/__init__.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify @@ -124,7 +124,7 @@ def elide_list(line_list, max_num=10): def disambiguate_spec(spec): - matching_specs = spack.db.get_installed(spec) + matching_specs = spack.installed_db.query(spec) if not matching_specs: tty.die("Spec '%s' matches no installed packages." % spec) diff --git a/lib/spack/spack/cmd/activate.py b/lib/spack/spack/cmd/activate.py index 1004f1f8e6..bcd01f2a28 100644 --- a/lib/spack/spack/cmd/activate.py +++ b/lib/spack/spack/cmd/activate.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify @@ -22,7 +22,7 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -from external import argparse +import argparse import llnl.util.tty as tty import spack import spack.cmd diff --git a/lib/spack/spack/cmd/arch.py b/lib/spack/spack/cmd/arch.py index f0e88d1849..db27544ffd 100644 --- a/lib/spack/spack/cmd/arch.py +++ b/lib/spack/spack/cmd/arch.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/bootstrap.py b/lib/spack/spack/cmd/bootstrap.py index f75b68b00a..e4ec7da35d 100644 --- a/lib/spack/spack/cmd/bootstrap.py +++ b/lib/spack/spack/cmd/bootstrap.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/cd.py b/lib/spack/spack/cmd/cd.py index 24d56db7d0..16cbe6555a 100644 --- a/lib/spack/spack/cmd/cd.py +++ b/lib/spack/spack/cmd/cd.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/checksum.py b/lib/spack/spack/cmd/checksum.py index 8a448450c2..6b7022a7a1 100644 --- a/lib/spack/spack/cmd/checksum.py +++ b/lib/spack/spack/cmd/checksum.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify @@ -24,7 +24,7 @@ ############################################################################## import os import re -from external import argparse +import argparse import hashlib from pprint import pprint from subprocess import CalledProcessError diff --git a/lib/spack/spack/cmd/clean.py b/lib/spack/spack/cmd/clean.py index c20136ebe5..c3409887fb 100644 --- a/lib/spack/spack/cmd/clean.py +++ b/lib/spack/spack/cmd/clean.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify @@ -22,7 +22,7 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -from external import argparse +import argparse import llnl.util.tty as tty diff --git a/lib/spack/spack/cmd/compiler.py b/lib/spack/spack/cmd/compiler.py index 2a64dc914e..589ca87fb5 100644 --- a/lib/spack/spack/cmd/compiler.py +++ b/lib/spack/spack/cmd/compiler.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify @@ -22,7 +22,7 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -from external import argparse +import argparse import llnl.util.tty as tty from llnl.util.tty.color import colorize diff --git a/lib/spack/spack/cmd/compilers.py b/lib/spack/spack/cmd/compilers.py index 8d046bfd7c..c485a910eb 100644 --- a/lib/spack/spack/cmd/compilers.py +++ b/lib/spack/spack/cmd/compilers.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/config.py b/lib/spack/spack/cmd/config.py index 8c18f88b64..a6e914131e 100644 --- a/lib/spack/spack/cmd/config.py +++ b/lib/spack/spack/cmd/config.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify @@ -23,7 +23,7 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import sys -from external import argparse +import argparse import llnl.util.tty as tty diff --git a/lib/spack/spack/cmd/create.py b/lib/spack/spack/cmd/create.py index 46e6bcec14..5e42860f3e 100644 --- a/lib/spack/spack/cmd/create.py +++ b/lib/spack/spack/cmd/create.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify @@ -27,7 +27,7 @@ import os import hashlib import re -from external.ordereddict import OrderedDict +from ordereddict_backport import OrderedDict import llnl.util.tty as tty from llnl.util.filesystem import mkdirp diff --git a/lib/spack/spack/cmd/deactivate.py b/lib/spack/spack/cmd/deactivate.py index e44be41029..a0c78bf755 100644 --- a/lib/spack/spack/cmd/deactivate.py +++ b/lib/spack/spack/cmd/deactivate.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify @@ -22,7 +22,7 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -from external import argparse +import argparse import llnl.util.tty as tty import spack @@ -54,7 +54,7 @@ def deactivate(parser, args): if args.all: if pkg.extendable: tty.msg("Deactivating all extensions of %s" % pkg.spec.short_spec) - ext_pkgs = spack.db.installed_extensions_for(spec) + ext_pkgs = spack.installed_db.installed_extensions_for(spec) for ext_pkg in ext_pkgs: ext_pkg.spec.normalize() diff --git a/lib/spack/spack/cmd/dependents.py b/lib/spack/spack/cmd/dependents.py index 652f243b98..de76098d2f 100644 --- a/lib/spack/spack/cmd/dependents.py +++ b/lib/spack/spack/cmd/dependents.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify @@ -22,7 +22,7 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -from external import argparse +import argparse import llnl.util.tty as tty diff --git a/lib/spack/spack/cmd/diy.py b/lib/spack/spack/cmd/diy.py index 6e7f10fba6..ebe8424f09 100644 --- a/lib/spack/spack/cmd/diy.py +++ b/lib/spack/spack/cmd/diy.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify @@ -24,7 +24,7 @@ ############################################################################## import sys import os -from external import argparse +import argparse import llnl.util.tty as tty @@ -58,36 +58,38 @@ def diy(self, args): if len(specs) > 1: tty.die("spack diy only takes one spec.") - spec = specs[0] - if not spack.db.exists(spec.name): - tty.warn("No such package: %s" % spec.name) - create = tty.get_yes_or_no("Create this package?", default=False) - if not create: - tty.msg("Exiting without creating.") - sys.exit(1) - else: - tty.msg("Running 'spack edit -f %s'" % spec.name) - edit_package(spec.name, True) - return + # Take a write lock before checking for existence. + with spack.installed_db.write_transaction(): + spec = specs[0] + if not spack.db.exists(spec.name): + tty.warn("No such package: %s" % spec.name) + create = tty.get_yes_or_no("Create this package?", default=False) + if not create: + tty.msg("Exiting without creating.") + sys.exit(1) + else: + tty.msg("Running 'spack edit -f %s'" % spec.name) + edit_package(spec.name, True) + return - if not spec.version.concrete: - tty.die("spack diy spec must have a single, concrete version.") + if not spec.version.concrete: + tty.die("spack diy spec must have a single, concrete version.") - spec.concretize() - package = spack.db.get(spec) + spec.concretize() + package = spack.db.get(spec) - if package.installed: - tty.error("Already installed in %s" % package.prefix) - tty.msg("Uninstall or try adding a version suffix for this DIY build.") - sys.exit(1) + if package.installed: + tty.error("Already installed in %s" % package.prefix) + tty.msg("Uninstall or try adding a version suffix for this DIY build.") + sys.exit(1) - # Forces the build to run out of the current directory. - package.stage = DIYStage(os.getcwd()) + # Forces the build to run out of the current directory. + package.stage = DIYStage(os.getcwd()) - # TODO: make this an argument, not a global. - spack.do_checksum = False + # TODO: make this an argument, not a global. + spack.do_checksum = False - package.do_install( - keep_prefix=args.keep_prefix, - ignore_deps=args.ignore_deps, - keep_stage=True) # don't remove source dir for DIY. + package.do_install( + keep_prefix=args.keep_prefix, + ignore_deps=args.ignore_deps, + keep_stage=True) # don't remove source dir for DIY. diff --git a/lib/spack/spack/cmd/doc.py b/lib/spack/spack/cmd/doc.py index 601ae26e5e..29cadec94f 100644 --- a/lib/spack/spack/cmd/doc.py +++ b/lib/spack/spack/cmd/doc.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/edit.py b/lib/spack/spack/cmd/edit.py index b8764ba391..b168d967b9 100644 --- a/lib/spack/spack/cmd/edit.py +++ b/lib/spack/spack/cmd/edit.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/env.py b/lib/spack/spack/cmd/env.py index bde76b5daf..525e955a00 100644 --- a/lib/spack/spack/cmd/env.py +++ b/lib/spack/spack/cmd/env.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify @@ -23,7 +23,7 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import os -from external import argparse +import argparse import llnl.util.tty as tty import spack.cmd import spack.build_environment as build_env diff --git a/lib/spack/spack/cmd/extensions.py b/lib/spack/spack/cmd/extensions.py index fc8e6842c3..c2cf288877 100644 --- a/lib/spack/spack/cmd/extensions.py +++ b/lib/spack/spack/cmd/extensions.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify @@ -23,7 +23,7 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import sys -from external import argparse +import argparse import llnl.util.tty as tty from llnl.util.tty.colify import colify @@ -37,7 +37,7 @@ description = "List extensions for package." def setup_parser(subparser): format_group = subparser.add_mutually_exclusive_group() format_group.add_argument( - '-l', '--long', action='store_const', dest='mode', const='long', + '-l', '--long', action='store_true', dest='long', help='Show dependency hashes as well as versions.') format_group.add_argument( '-p', '--paths', action='store_const', dest='mode', const='paths', @@ -80,7 +80,7 @@ def extensions(parser, args): colify(ext.name for ext in extensions) # List specs of installed extensions. - installed = [s.spec for s in spack.db.installed_extensions_for(spec)] + installed = [s.spec for s in spack.installed_db.installed_extensions_for(spec)] print if not installed: tty.msg("None installed.") @@ -95,4 +95,4 @@ def extensions(parser, args): tty.msg("None activated.") return tty.msg("%d currently activated:" % len(activated)) - spack.cmd.find.display_specs(activated.values(), mode=args.mode) + spack.cmd.find.display_specs(activated.values(), mode=args.mode, long=args.long) diff --git a/lib/spack/spack/cmd/fetch.py b/lib/spack/spack/cmd/fetch.py index 0ccebd9486..6f9e7ab5e2 100644 --- a/lib/spack/spack/cmd/fetch.py +++ b/lib/spack/spack/cmd/fetch.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify @@ -22,7 +22,7 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -from external import argparse +import argparse import spack import spack.cmd diff --git a/lib/spack/spack/cmd/find.py b/lib/spack/spack/cmd/find.py index 3c993990b1..c7a376fd8d 100644 --- a/lib/spack/spack/cmd/find.py +++ b/lib/spack/spack/cmd/find.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify @@ -25,7 +25,7 @@ import sys import collections import itertools -from external import argparse +import argparse from StringIO import StringIO import llnl.util.tty as tty @@ -55,6 +55,16 @@ def setup_parser(subparser): help='Show dependency hashes as well as versions.') subparser.add_argument( + '-u', '--unknown', action='store_true', dest='unknown', + help='Show only specs Spack does not have a package for.') + subparser.add_argument( + '-m', '--missing', action='store_true', dest='missing', + help='Show missing dependencies as well as installed specs.') + subparser.add_argument( + '-M', '--only-missing', action='store_true', dest='only_missing', + help='Show only missing dependencies.') + + subparser.add_argument( 'query_specs', nargs=argparse.REMAINDER, help='optional specs to filter results') @@ -113,6 +123,7 @@ def display_specs(specs, **kwargs): if hashes: string += gray_hash(s, hlen) + ' ' string += s.format('$-_$@$+', color=True) + return string colify(fmt(s) for s in specs) @@ -136,11 +147,21 @@ def find(parser, args): if not query_specs: return + # Set up query arguments. + installed, known = True, any + if args.only_missing: + installed = False + elif args.missing: + installed = any + if args.unknown: + known = False + q_args = { 'installed' : installed, 'known' : known } + # Get all the specs the user asked for if not query_specs: - specs = set(spack.db.installed_package_specs()) + specs = set(spack.installed_db.query(**q_args)) else: - results = [set(spack.db.get_installed(qs)) for qs in query_specs] + results = [set(spack.installed_db.query(qs, **q_args)) for qs in query_specs] specs = set.union(*results) if not args.mode: diff --git a/lib/spack/spack/cmd/graph.py b/lib/spack/spack/cmd/graph.py index cb93a1b543..586a852351 100644 --- a/lib/spack/spack/cmd/graph.py +++ b/lib/spack/spack/cmd/graph.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify @@ -22,7 +22,7 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -from external import argparse +import argparse import spack import spack.cmd diff --git a/lib/spack/spack/cmd/help.py b/lib/spack/spack/cmd/help.py index eae3aabd97..841a0d5bcb 100644 --- a/lib/spack/spack/cmd/help.py +++ b/lib/spack/spack/cmd/help.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/info.py b/lib/spack/spack/cmd/info.py index 085e4db44d..8040e23936 100644 --- a/lib/spack/spack/cmd/info.py +++ b/lib/spack/spack/cmd/info.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/install.py b/lib/spack/spack/cmd/install.py index acb688a092..5ee7bc01b7 100644 --- a/lib/spack/spack/cmd/install.py +++ b/lib/spack/spack/cmd/install.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify @@ -22,7 +22,7 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -from external import argparse +import argparse import llnl.util.tty as tty @@ -71,10 +71,11 @@ def install(parser, args): specs = spack.cmd.parse_specs(args.packages, concretize=True) for spec in specs: package = spack.db.get(spec) - package.do_install( - keep_prefix=args.keep_prefix, - keep_stage=args.keep_stage, - ignore_deps=args.ignore_deps, - make_jobs=args.jobs, - verbose=args.verbose, - fake=args.fake) + with spack.installed_db.write_transaction(): + package.do_install( + keep_prefix=args.keep_prefix, + keep_stage=args.keep_stage, + ignore_deps=args.ignore_deps, + make_jobs=args.jobs, + verbose=args.verbose, + fake=args.fake) diff --git a/lib/spack/spack/cmd/list.py b/lib/spack/spack/cmd/list.py index 1f0978a18e..b51d5b429a 100644 --- a/lib/spack/spack/cmd/list.py +++ b/lib/spack/spack/cmd/list.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify @@ -24,7 +24,7 @@ ############################################################################## import sys import llnl.util.tty as tty -from external import argparse +import argparse from llnl.util.tty.colify import colify import spack diff --git a/lib/spack/spack/cmd/load.py b/lib/spack/spack/cmd/load.py index 06574d9725..30d86c3b01 100644 --- a/lib/spack/spack/cmd/load.py +++ b/lib/spack/spack/cmd/load.py @@ -6,7 +6,7 @@ # Written by David Beckingsale, david@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify @@ -22,7 +22,7 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -from external import argparse +import argparse import spack.modules description ="Add package to environment using modules." diff --git a/lib/spack/spack/cmd/location.py b/lib/spack/spack/cmd/location.py index e8e9c3f277..e805cc4012 100644 --- a/lib/spack/spack/cmd/location.py +++ b/lib/spack/spack/cmd/location.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify @@ -24,7 +24,7 @@ ############################################################################## import os import sys -from external import argparse +import argparse import llnl.util.tty as tty from llnl.util.filesystem import join_path diff --git a/lib/spack/spack/cmd/md5.py b/lib/spack/spack/cmd/md5.py index dfa1be412b..ef1e4f3475 100644 --- a/lib/spack/spack/cmd/md5.py +++ b/lib/spack/spack/cmd/md5.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify @@ -24,7 +24,7 @@ ############################################################################## import os import hashlib -from external import argparse +import argparse import llnl.util.tty as tty from llnl.util.filesystem import * diff --git a/lib/spack/spack/cmd/mirror.py b/lib/spack/spack/cmd/mirror.py index 2356170a9a..4a1ce00b75 100644 --- a/lib/spack/spack/cmd/mirror.py +++ b/lib/spack/spack/cmd/mirror.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify @@ -26,7 +26,7 @@ import os import sys from datetime import datetime -from external import argparse +import argparse import llnl.util.tty as tty from llnl.util.tty.colify import colify diff --git a/lib/spack/spack/cmd/module.py b/lib/spack/spack/cmd/module.py index 34f0855a50..a5a9570eb5 100644 --- a/lib/spack/spack/cmd/module.py +++ b/lib/spack/spack/cmd/module.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify @@ -25,7 +25,7 @@ import sys import os import shutil -from external import argparse +import argparse import llnl.util.tty as tty from llnl.util.lang import partition_list @@ -65,7 +65,7 @@ def module_find(mtype, spec_array): tty.die("You can only pass one spec.") spec = specs[0] - specs = [s for s in spack.db.installed_package_specs() if s.satisfies(spec)] + specs = spack.installed_db.query(spec) if len(specs) == 0: tty.die("No installed packages match spec %s" % spec) @@ -86,7 +86,7 @@ def module_find(mtype, spec_array): def module_refresh(): """Regenerate all module files for installed packages known to spack (some packages may no longer exist).""" - specs = [s for s in spack.db.installed_known_package_specs()] + specs = [s for s in spack.installed_db.query(installed=True, known=True)] for name, cls in module_types.items(): tty.msg("Regenerating %s module files." % name) diff --git a/lib/spack/spack/cmd/package-list.py b/lib/spack/spack/cmd/package-list.py index f048482845..eca9f918f1 100644 --- a/lib/spack/spack/cmd/package-list.py +++ b/lib/spack/spack/cmd/package-list.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify @@ -34,7 +34,7 @@ description = "Print a list of all packages in reStructuredText." def github_url(pkg): """Link to a package file on github.""" - return ("https://github.com/scalability-llnl/spack/blob/master/var/spack/packages/%s/package.py" % + return ("https://github.com/llnl/spack/blob/master/var/spack/packages/%s/package.py" % pkg.name) diff --git a/lib/spack/spack/cmd/patch.py b/lib/spack/spack/cmd/patch.py index a6556c4828..8fc6f1383e 100644 --- a/lib/spack/spack/cmd/patch.py +++ b/lib/spack/spack/cmd/patch.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify @@ -22,7 +22,7 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -from external import argparse +import argparse import spack.cmd import spack diff --git a/lib/spack/spack/cmd/pkg.py b/lib/spack/spack/cmd/pkg.py index 055b7c2062..1ebc5ce3ee 100644 --- a/lib/spack/spack/cmd/pkg.py +++ b/lib/spack/spack/cmd/pkg.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify @@ -24,7 +24,7 @@ ############################################################################## import os -from external import argparse +import argparse import llnl.util.tty as tty from llnl.util.tty.colify import colify diff --git a/lib/spack/spack/cmd/providers.py b/lib/spack/spack/cmd/providers.py index 2bcdc9fba2..0472f9bbe4 100644 --- a/lib/spack/spack/cmd/providers.py +++ b/lib/spack/spack/cmd/providers.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify @@ -23,7 +23,7 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import os -from external import argparse +import argparse from llnl.util.tty.colify import colify diff --git a/lib/spack/spack/cmd/purge.py b/lib/spack/spack/cmd/purge.py index 9b96937149..d5d7513c46 100644 --- a/lib/spack/spack/cmd/purge.py +++ b/lib/spack/spack/cmd/purge.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/python.py b/lib/spack/spack/cmd/python.py index 7bd2e45ce0..e26b8d3e79 100644 --- a/lib/spack/spack/cmd/python.py +++ b/lib/spack/spack/cmd/python.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify @@ -25,7 +25,7 @@ import os import sys import code -from external import argparse +import argparse import platform import spack diff --git a/lib/spack/spack/cmd/reindex.py b/lib/spack/spack/cmd/reindex.py new file mode 100644 index 0000000000..2b30ef8814 --- /dev/null +++ b/lib/spack/spack/cmd/reindex.py @@ -0,0 +1,31 @@ +############################################################################## +# Copyright (c) 2013-2015, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +import argparse +import spack + +description = "Rebuild Spack's package database." + +def reindex(parser, args): + spack.installed_db.reindex(spack.install_layout) diff --git a/lib/spack/spack/cmd/restage.py b/lib/spack/spack/cmd/restage.py index e735a12c32..703ae30a04 100644 --- a/lib/spack/spack/cmd/restage.py +++ b/lib/spack/spack/cmd/restage.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify @@ -22,7 +22,7 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -from external import argparse +import argparse import llnl.util.tty as tty diff --git a/lib/spack/spack/cmd/spec.py b/lib/spack/spack/cmd/spec.py index 407519313c..43a106ea37 100644 --- a/lib/spack/spack/cmd/spec.py +++ b/lib/spack/spack/cmd/spec.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify @@ -22,7 +22,7 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -from external import argparse +import argparse import spack.cmd import llnl.util.tty as tty diff --git a/lib/spack/spack/cmd/stage.py b/lib/spack/spack/cmd/stage.py index f3dc97be17..7638cf31c4 100644 --- a/lib/spack/spack/cmd/stage.py +++ b/lib/spack/spack/cmd/stage.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify @@ -23,7 +23,7 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import os -from external import argparse +import argparse import llnl.util.tty as tty import spack diff --git a/lib/spack/spack/cmd/test-install.py b/lib/spack/spack/cmd/test-install.py new file mode 100644 index 0000000000..e37554155f --- /dev/null +++ b/lib/spack/spack/cmd/test-install.py @@ -0,0 +1,211 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +import argparse +import xml.etree.ElementTree as ET +import itertools +import re +import os +import codecs + +import llnl.util.tty as tty +from llnl.util.filesystem import * + +import spack +from spack.build_environment import InstallError +from spack.fetch_strategy import FetchError +import spack.cmd + +description = "Treat package installations as unit tests and output formatted test results" + +def setup_parser(subparser): + subparser.add_argument( + '-j', '--jobs', action='store', type=int, + help="Explicitly set number of make jobs. Default is #cpus.") + + subparser.add_argument( + '-n', '--no-checksum', action='store_true', dest='no_checksum', + help="Do not check packages against checksum") + + subparser.add_argument( + '-o', '--output', action='store', help="test output goes in this file") + + subparser.add_argument( + 'package', nargs=argparse.REMAINDER, help="spec of package to install") + + +class JunitResultFormat(object): + def __init__(self): + self.root = ET.Element('testsuite') + self.tests = [] + + def add_test(self, buildId, testResult, buildInfo=None): + self.tests.append((buildId, testResult, buildInfo)) + + def write_to(self, stream): + self.root.set('tests', '{0}'.format(len(self.tests))) + for buildId, testResult, buildInfo in self.tests: + testcase = ET.SubElement(self.root, 'testcase') + testcase.set('classname', buildId.name) + testcase.set('name', buildId.stringId()) + if testResult == TestResult.FAILED: + failure = ET.SubElement(testcase, 'failure') + failure.set('type', "Build Error") + failure.text = buildInfo + elif testResult == TestResult.SKIPPED: + skipped = ET.SubElement(testcase, 'skipped') + skipped.set('type', "Skipped Build") + skipped.text = buildInfo + ET.ElementTree(self.root).write(stream) + + +class TestResult(object): + PASSED = 0 + FAILED = 1 + SKIPPED = 2 + + +class BuildId(object): + def __init__(self, spec): + self.name = spec.name + self.version = spec.version + self.hashId = spec.dag_hash() + + def stringId(self): + return "-".join(str(x) for x in (self.name, self.version, self.hashId)) + + def __hash__(self): + return hash((self.name, self.version, self.hashId)) + + def __eq__(self, other): + if not isinstance(other, BuildId): + return False + + return ((self.name, self.version, self.hashId) == + (other.name, other.version, other.hashId)) + + +def fetch_log(path): + if not os.path.exists(path): + return list() + with codecs.open(path, 'rb', 'utf-8') as F: + return list(line.strip() for line in F.readlines()) + + +def failed_dependencies(spec): + return set(childSpec for childSpec in spec.dependencies.itervalues() if not + spack.db.get(childSpec).installed) + + +def create_test_output(topSpec, newInstalls, output, getLogFunc=fetch_log): + # Post-order traversal is not strictly required but it makes sense to output + # tests for dependencies first. + for spec in topSpec.traverse(order='post'): + if spec not in newInstalls: + continue + + failedDeps = failed_dependencies(spec) + package = spack.db.get(spec) + if failedDeps: + result = TestResult.SKIPPED + dep = iter(failedDeps).next() + depBID = BuildId(dep) + errOutput = "Skipped due to failed dependency: {0}".format( + depBID.stringId()) + elif (not package.installed) and (not package.stage.source_path): + result = TestResult.FAILED + errOutput = "Failure to fetch package resources." + elif not package.installed: + result = TestResult.FAILED + lines = getLogFunc(package.build_log_path) + errMessages = list(line for line in lines if + re.search('error:', line, re.IGNORECASE)) + errOutput = errMessages if errMessages else lines[-10:] + errOutput = '\n'.join(itertools.chain( + [spec.to_yaml(), "Errors:"], errOutput, + ["Build Log:", package.build_log_path])) + else: + result = TestResult.PASSED + errOutput = None + + bId = BuildId(spec) + output.add_test(bId, result, errOutput) + + +def test_install(parser, args): + if not args.package: + tty.die("install requires a package argument") + + if args.jobs is not None: + if args.jobs <= 0: + tty.die("The -j option must be a positive integer!") + + if args.no_checksum: + spack.do_checksum = False # TODO: remove this global. + + specs = spack.cmd.parse_specs(args.package, concretize=True) + if len(specs) > 1: + tty.die("Only 1 top-level package can be specified") + topSpec = iter(specs).next() + + newInstalls = set() + for spec in topSpec.traverse(): + package = spack.db.get(spec) + if not package.installed: + newInstalls.add(spec) + + if not args.output: + bId = BuildId(topSpec) + outputDir = join_path(os.getcwd(), "test-output") + if not os.path.exists(outputDir): + os.mkdir(outputDir) + outputFpath = join_path(outputDir, "test-{0}.xml".format(bId.stringId())) + else: + outputFpath = args.output + + for spec in topSpec.traverse(order='post'): + # Calling do_install for the top-level package would be sufficient but + # this attempts to keep going if any package fails (other packages which + # are not dependents may succeed) + package = spack.db.get(spec) + if (not failed_dependencies(spec)) and (not package.installed): + try: + package.do_install( + keep_prefix=False, + keep_stage=True, + ignore_deps=False, + make_jobs=args.jobs, + verbose=True, + fake=False) + except InstallError: + pass + except FetchError: + pass + + jrf = JunitResultFormat() + handled = {} + create_test_output(topSpec, newInstalls, jrf) + + with open(outputFpath, 'wb') as F: + jrf.write_to(F) diff --git a/lib/spack/spack/cmd/test.py b/lib/spack/spack/cmd/test.py index b1418ac2f1..1669ec4cc9 100644 --- a/lib/spack/spack/cmd/test.py +++ b/lib/spack/spack/cmd/test.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/uninstall.py b/lib/spack/spack/cmd/uninstall.py index aa62510fed..191d9d88e8 100644 --- a/lib/spack/spack/cmd/uninstall.py +++ b/lib/spack/spack/cmd/uninstall.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify @@ -23,7 +23,7 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import sys -from external import argparse +import argparse import llnl.util.tty as tty from llnl.util.tty.colify import colify @@ -53,51 +53,52 @@ def uninstall(parser, args): if not args.packages: tty.die("uninstall requires at least one package argument.") - specs = spack.cmd.parse_specs(args.packages) + with spack.installed_db.write_transaction(): + specs = spack.cmd.parse_specs(args.packages) - # For each spec provided, make sure it refers to only one package. - # Fail and ask user to be unambiguous if it doesn't - pkgs = [] - for spec in specs: - matching_specs = spack.db.get_installed(spec) - if not args.all and len(matching_specs) > 1: - tty.error("%s matches multiple packages:" % spec) - print - display_specs(matching_specs, long=True) - print - print "You can either:" - print " a) Use a more specific spec, or" - print " b) use spack uninstall -a to uninstall ALL matching specs." - sys.exit(1) + # For each spec provided, make sure it refers to only one package. + # Fail and ask user to be unambiguous if it doesn't + pkgs = [] + for spec in specs: + matching_specs = spack.installed_db.query(spec) + if not args.all and len(matching_specs) > 1: + tty.error("%s matches multiple packages:" % spec) + print + display_specs(matching_specs, long=True) + print + print "You can either:" + print " a) Use a more specific spec, or" + print " b) use spack uninstall -a to uninstall ALL matching specs." + sys.exit(1) - if len(matching_specs) == 0: - if args.force: continue - tty.die("%s does not match any installed packages." % spec) + if len(matching_specs) == 0: + if args.force: continue + tty.die("%s does not match any installed packages." % spec) - for s in matching_specs: - try: - # should work if package is known to spack - pkgs.append(s.package) + for s in matching_specs: + try: + # should work if package is known to spack + pkgs.append(s.package) - except spack.packages.UnknownPackageError, e: - # The package.py file has gone away -- but still want to uninstall. - spack.Package(s).do_uninstall(force=True) + except spack.packages.UnknownPackageError, e: + # The package.py file has gone away -- but still want to uninstall. + spack.Package(s).do_uninstall(force=True) - # Sort packages to be uninstalled by the number of installed dependents - # This ensures we do things in the right order - def num_installed_deps(pkg): - return len(pkg.installed_dependents) - pkgs.sort(key=num_installed_deps) + # Sort packages to be uninstalled by the number of installed dependents + # This ensures we do things in the right order + def num_installed_deps(pkg): + return len(pkg.installed_dependents) + pkgs.sort(key=num_installed_deps) - # Uninstall packages in order now. - for pkg in pkgs: - try: - pkg.do_uninstall(force=args.force) - except PackageStillNeededError, e: - tty.error("Will not uninstall %s" % e.spec.format("$_$@$%@$#", color=True)) - print - print "The following packages depend on it:" - display_specs(e.dependents, long=True) - print - print "You can use spack uninstall -f to force this action." - sys.exit(1) + # Uninstall packages in order now. + for pkg in pkgs: + try: + pkg.do_uninstall(force=args.force) + except PackageStillNeededError, e: + tty.error("Will not uninstall %s" % e.spec.format("$_$@$%@$#", color=True)) + print + print "The following packages depend on it:" + display_specs(e.dependents, long=True) + print + print "You can use spack uninstall -f to force this action." + sys.exit(1) diff --git a/lib/spack/spack/cmd/unload.py b/lib/spack/spack/cmd/unload.py index 6442c48cb1..cfb640ee6f 100644 --- a/lib/spack/spack/cmd/unload.py +++ b/lib/spack/spack/cmd/unload.py @@ -6,7 +6,7 @@ # Written by David Beckingsale, david@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify @@ -22,7 +22,7 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -from external import argparse +import argparse import spack.modules description ="Remove package from environment using module." diff --git a/lib/spack/spack/cmd/unuse.py b/lib/spack/spack/cmd/unuse.py index 2a7229a3a0..06176a976b 100644 --- a/lib/spack/spack/cmd/unuse.py +++ b/lib/spack/spack/cmd/unuse.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify @@ -22,7 +22,7 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -from external import argparse +import argparse import spack.modules description ="Remove package from environment using dotkit." diff --git a/lib/spack/spack/cmd/urls.py b/lib/spack/spack/cmd/urls.py index 417ce3ab68..a544b6153b 100644 --- a/lib/spack/spack/cmd/urls.py +++ b/lib/spack/spack/cmd/urls.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/use.py b/lib/spack/spack/cmd/use.py index e34c194739..c09695cfd3 100644 --- a/lib/spack/spack/cmd/use.py +++ b/lib/spack/spack/cmd/use.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify @@ -22,7 +22,7 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -from external import argparse +import argparse import spack.modules description ="Add package to environment using dotkit." diff --git a/lib/spack/spack/cmd/versions.py b/lib/spack/spack/cmd/versions.py index ed16728261..494f13d36d 100644 --- a/lib/spack/spack/cmd/versions.py +++ b/lib/spack/spack/cmd/versions.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index 1e800a8979..b9abf943e8 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index b7b021a1ac..66e608cf79 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/compilers/clang.py b/lib/spack/spack/compilers/clang.py index 790901c86e..b34d1f2f9c 100644 --- a/lib/spack/spack/compilers/clang.py +++ b/lib/spack/spack/compilers/clang.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/compilers/gcc.py b/lib/spack/spack/compilers/gcc.py index f0d27d590e..2886888d57 100644 --- a/lib/spack/spack/compilers/gcc.py +++ b/lib/spack/spack/compilers/gcc.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/compilers/intel.py b/lib/spack/spack/compilers/intel.py index 2a72c4eaea..1298429974 100644 --- a/lib/spack/spack/compilers/intel.py +++ b/lib/spack/spack/compilers/intel.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/compilers/pgi.py b/lib/spack/spack/compilers/pgi.py index d97f24c12e..6999eb50de 100644 --- a/lib/spack/spack/compilers/pgi.py +++ b/lib/spack/spack/compilers/pgi.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/compilers/xl.py b/lib/spack/spack/compilers/xl.py index 562186b865..308f811eb4 100644 --- a/lib/spack/spack/compilers/xl.py +++ b/lib/spack/spack/compilers/xl.py @@ -7,7 +7,7 @@ # Written by François Bissey, francois.bissey@canterbury.ac.nz, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 66002492cb..d9419da784 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 3e91958c2c..c127f6a28f 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify @@ -90,12 +90,12 @@ import os import exceptions import sys -from external.ordereddict import OrderedDict +from ordereddict_backport import OrderedDict from llnl.util.lang import memoized import spack.error -from external import yaml -from external.yaml.error import MarkedYAMLError +import yaml +from yaml.error import MarkedYAMLError import llnl.util.tty as tty from llnl.util.filesystem import mkdirp diff --git a/lib/spack/spack/database.py b/lib/spack/spack/database.py new file mode 100644 index 0000000000..d62a47547d --- /dev/null +++ b/lib/spack/spack/database.py @@ -0,0 +1,628 @@ +############################################################################## +# Copyright (c) 2013-2015, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +"""Spack's installation tracking database. + +The database serves two purposes: + + 1. It implements a cache on top of a potentially very large Spack + directory hierarchy, speeding up many operations that would + otherwise require filesystem access. + + 2. It will allow us to track external installations as well as lost + packages and their dependencies. + +Prior ot the implementation of this store, a direcotry layout served +as the authoritative database of packages in Spack. This module +provides a cache and a sanity checking mechanism for what is in the +filesystem. + +""" +import os +import time +import socket + +import yaml +from yaml.error import MarkedYAMLError, YAMLError + +import llnl.util.tty as tty +from llnl.util.filesystem import * +from llnl.util.lock import * + +import spack.spec +from spack.version import Version +from spack.spec import Spec +from spack.error import SpackError + +# DB goes in this directory underneath the root +_db_dirname = '.spack-db' + +# DB version. This is stuck in the DB file to track changes in format. +_db_version = Version('0.9') + +# Default timeout for spack database locks is 5 min. +_db_lock_timeout = 60 + + +def _autospec(function): + """Decorator that automatically converts the argument of a single-arg + function to a Spec.""" + def converter(self, spec_like, *args, **kwargs): + if not isinstance(spec_like, spack.spec.Spec): + spec_like = spack.spec.Spec(spec_like) + return function(self, spec_like, *args, **kwargs) + return converter + + +class InstallRecord(object): + """A record represents one installation in the DB. + + The record keeps track of the spec for the installation, its + install path, AND whether or not it is installed. We need the + installed flag in case a user either: + + a) blew away a directory, or + b) used spack uninstall -f to get rid of it + + If, in either case, the package was removed but others still + depend on it, we still need to track its spec, so we don't + actually remove from the database until a spec has no installed + dependents left. + + """ + def __init__(self, spec, path, installed, ref_count=0): + self.spec = spec + self.path = str(path) + self.installed = bool(installed) + self.ref_count = ref_count + + def to_dict(self): + return { 'spec' : self.spec.to_node_dict(), + 'path' : self.path, + 'installed' : self.installed, + 'ref_count' : self.ref_count } + + @classmethod + def from_dict(cls, spec, dictionary): + d = dictionary + return InstallRecord(spec, d['path'], d['installed'], d['ref_count']) + + +class Database(object): + def __init__(self, root, db_dir=None): + """Create a Database for Spack installations under ``root``. + + A Database is a cache of Specs data from ``$prefix/spec.yaml`` + files in Spack installation directories. + + By default, Database files (data and lock files) are stored + under ``root/.spack-db``, which is created if it does not + exist. This is the ``db_dir``. + + The Database will attempt to read an ``index.yaml`` file in + ``db_dir``. If it does not find one, it will be created when + needed by scanning the entire Database root for ``spec.yaml`` + files according to Spack's ``DirectoryLayout``. + + Caller may optionally provide a custom ``db_dir`` parameter + where data will be stored. This is intended to be used for + testing the Database class. + + """ + self.root = root + + if db_dir is None: + # If the db_dir is not provided, default to within the db root. + self._db_dir = join_path(self.root, _db_dirname) + else: + # Allow customizing the database directory location for testing. + self._db_dir = db_dir + + # Set up layout of database files within the db dir + self._index_path = join_path(self._db_dir, 'index.yaml') + self._lock_path = join_path(self._db_dir, 'lock') + + # Create needed directories and files + if not os.path.exists(self._db_dir): + mkdirp(self._db_dir) + + if not os.path.exists(self._lock_path): + touch(self._lock_path) + + # initialize rest of state. + self.lock = Lock(self._lock_path) + self._data = {} + + + def write_transaction(self, timeout=_db_lock_timeout): + """Get a write lock context manager for use in a `with` block.""" + return WriteTransaction(self, self._read, self._write, timeout) + + + def read_transaction(self, timeout=_db_lock_timeout): + """Get a read lock context manager for use in a `with` block.""" + return ReadTransaction(self, self._read, None, timeout) + + + def _write_to_yaml(self, stream): + """Write out the databsae to a YAML file. + + This function does not do any locking or transactions. + """ + # map from per-spec hash code to installation record. + installs = dict((k, v.to_dict()) for k, v in self._data.items()) + + # database includes installation list and version. + + # NOTE: this DB version does not handle multiple installs of + # the same spec well. If there are 2 identical specs with + # different paths, it can't differentiate. + # TODO: fix this before we support multiple install locations. + database = { + 'database' : { + 'installs' : installs, + 'version' : str(_db_version) + } + } + + try: + return yaml.dump(database, stream=stream, default_flow_style=False) + except YAMLError as e: + raise SpackYAMLError("error writing YAML database:", str(e)) + + + def _read_spec_from_yaml(self, hash_key, installs, parent_key=None): + """Recursively construct a spec from a hash in a YAML database. + + Does not do any locking. + """ + if hash_key not in installs: + parent = read_spec(installs[parent_key]['path']) + + spec_dict = installs[hash_key]['spec'] + + # Build spec from dict first. + spec = Spec.from_node_dict(spec_dict) + + # Add dependencies from other records in the install DB to + # form a full spec. + for dep_hash in spec_dict[spec.name]['dependencies'].values(): + child = self._read_spec_from_yaml(dep_hash, installs, hash_key) + spec._add_dependency(child) + + return spec + + + def _read_from_yaml(self, stream): + """ + Fill database from YAML, do not maintain old data + Translate the spec portions from node-dict form to spec form + + Does not do any locking. + """ + try: + if isinstance(stream, basestring): + with open(stream, 'r') as f: + yfile = yaml.load(f) + else: + yfile = yaml.load(stream) + + except MarkedYAMLError as e: + raise SpackYAMLError("error parsing YAML database:", str(e)) + + if yfile is None: + return + + def check(cond, msg): + if not cond: raise CorruptDatabaseError(self._index_path, msg) + + check('database' in yfile, "No 'database' attribute in YAML.") + + # High-level file checks + db = yfile['database'] + check('installs' in db, "No 'installs' in YAML DB.") + check('version' in db, "No 'version' in YAML DB.") + + # TODO: better version checking semantics. + version = Version(db['version']) + if version != _db_version: + raise InvalidDatabaseVersionError(_db_version, version) + + # Iterate through database and check each record. + installs = db['installs'] + data = {} + for hash_key, rec in installs.items(): + try: + # This constructs a spec DAG from the list of all installs + spec = self._read_spec_from_yaml(hash_key, installs) + + # Validate the spec by ensuring the stored and actual + # hashes are the same. + spec_hash = spec.dag_hash() + if not spec_hash == hash_key: + tty.warn("Hash mismatch in database: %s -> spec with hash %s" + % (hash_key, spec_hash)) + continue # TODO: is skipping the right thing to do? + + # Insert the brand new spec in the database. Each + # spec has its own copies of its dependency specs. + # TODO: would a more immmutable spec implementation simplify this? + data[hash_key] = InstallRecord.from_dict(spec, rec) + + except Exception as e: + tty.warn("Invalid database reecord:", + "file: %s" % self._index_path, + "hash: %s" % hash_key, + "cause: %s" % str(e)) + raise + + self._data = data + + + def reindex(self, directory_layout): + """Build database index from scratch based from a directory layout. + + Locks the DB if it isn't locked already. + + """ + with self.write_transaction(): + old_data = self._data + try: + self._data = {} + + # Ask the directory layout to traverse the filesystem. + for spec in directory_layout.all_specs(): + # Create a spec for each known package and add it. + path = directory_layout.path_for_spec(spec) + self._add(spec, path, directory_layout) + + self._check_ref_counts() + + except: + # If anything explodes, restore old data, skip write. + self._data = old_data + raise + + + def _check_ref_counts(self): + """Ensure consistency of reference counts in the DB. + + Raise an AssertionError if something is amiss. + + Does no locking. + """ + counts = {} + for key, rec in self._data.items(): + counts.setdefault(key, 0) + for dep in rec.spec.dependencies.values(): + dep_key = dep.dag_hash() + counts.setdefault(dep_key, 0) + counts[dep_key] += 1 + + for rec in self._data.values(): + key = rec.spec.dag_hash() + expected = counts[key] + found = rec.ref_count + if not expected == found: + raise AssertionError( + "Invalid ref_count: %s: %d (expected %d), in DB %s." + % (key, found, expected, self._index_path)) + + + def _write(self): + """Write the in-memory database index to its file path. + + Does no locking. + + """ + temp_file = self._index_path + ( + '.%s.%s.temp' % (socket.getfqdn(), os.getpid())) + + # Write a temporary database file them move it into place + try: + with open(temp_file, 'w') as f: + self._write_to_yaml(f) + os.rename(temp_file, self._index_path) + except: + # Clean up temp file if something goes wrong. + if os.path.exists(temp_file): + os.remove(temp_file) + raise + + + def _read(self): + """Re-read Database from the data in the set location. + + This does no locking. + """ + if os.path.isfile(self._index_path): + # Read from YAML file if a database exists + self._read_from_yaml(self._index_path) + + else: + # The file doesn't exist, try to traverse the directory. + # reindex() takes its own write lock, so no lock here. + self.reindex(spack.install_layout) + + + def _add(self, spec, path, directory_layout=None): + """Add an install record for spec at path to the database. + + This assumes that the spec is not already installed. It + updates the ref counts on dependencies of the spec in the DB. + + This operation is in-memory, and does not lock the DB. + + """ + key = spec.dag_hash() + if key in self._data: + rec = self._data[key] + rec.installed = True + + # TODO: this overwrites a previous install path (when path != + # self._data[key].path), and the old path still has a + # dependent in the DB. We could consider re-RPATH-ing the + # dependents. This case is probably infrequent and may not be + # worth fixing, but this is where we can discover it. + rec.path = path + + else: + self._data[key] = InstallRecord(spec, path, True) + for dep in spec.dependencies.values(): + self._increment_ref_count(dep, directory_layout) + + + def _increment_ref_count(self, spec, directory_layout=None): + """Recursively examine dependencies and update their DB entries.""" + key = spec.dag_hash() + if key not in self._data: + installed = False + path = None + if directory_layout: + path = directory_layout.path_for_spec(spec) + installed = os.path.isdir(path) + + self._data[key] = InstallRecord(spec.copy(), path, installed) + + for dep in spec.dependencies.values(): + self._increment_ref_count(dep) + + self._data[key].ref_count += 1 + + @_autospec + def add(self, spec, path): + """Add spec at path to database, locking and reading DB to sync. + + ``add()`` will lock and read from the DB on disk. + + """ + # TODO: ensure that spec is concrete? + # Entire add is transactional. + with self.write_transaction(): + self._add(spec, path) + + + def _get_matching_spec_key(self, spec, **kwargs): + """Get the exact spec OR get a single spec that matches.""" + key = spec.dag_hash() + if not key in self._data: + match = self.query_one(spec, **kwargs) + if match: + return match.dag_hash() + raise KeyError("No such spec in database! %s" % spec) + return key + + + @_autospec + def get_record(self, spec, **kwargs): + key = self._get_matching_spec_key(spec, **kwargs) + return self._data[key] + + + def _decrement_ref_count(self, spec): + key = spec.dag_hash() + + if not key in self._data: + # TODO: print something here? DB is corrupt, but + # not much we can do. + return + + rec = self._data[key] + rec.ref_count -= 1 + + if rec.ref_count == 0 and not rec.installed: + del self._data[key] + for dep in spec.dependencies.values(): + self._decrement_ref_count(dep) + + + def _remove(self, spec): + """Non-locking version of remove(); does real work. + """ + key = self._get_matching_spec_key(spec) + rec = self._data[key] + + if rec.ref_count > 0: + rec.installed = False + return rec.spec + + del self._data[key] + for dep in rec.spec.dependencies.values(): + self._decrement_ref_count(dep) + + # Returns the concrete spec so we know it in the case where a + # query spec was passed in. + return rec.spec + + + @_autospec + def remove(self, spec): + """Removes a spec from the database. To be called on uninstall. + + Reads the database, then: + + 1. Marks the spec as not installed. + 2. Removes the spec if it has no more dependents. + 3. If removed, recursively updates dependencies' ref counts + and remvoes them if they are no longer needed. + + """ + # Take a lock around the entire removal. + with self.write_transaction(): + return self._remove(spec) + + + @_autospec + def installed_extensions_for(self, extendee_spec): + """ + Return the specs of all packages that extend + the given spec + """ + for s in self.query(): + try: + if s.package.extends(extendee_spec): + yield s.package + except UnknownPackageError as e: + continue + # skips unknown packages + # TODO: conditional way to do this instead of catching exceptions + + + def query(self, query_spec=any, known=any, installed=True): + """Run a query on the database. + + ``query_spec`` + Queries iterate through specs in the database and return + those that satisfy the supplied ``query_spec``. If + query_spec is `any`, This will match all specs in the + database. If it is a spec, we'll evaluate + ``spec.satisfies(query_spec)``. + + The query can be constrained by two additional attributes: + + ``known`` + Possible values: True, False, any + + Specs that are "known" are those for which Spack can + locate a ``package.py`` file -- i.e., Spack "knows" how to + install them. Specs that are unknown may represent + packages that existed in a previous version of Spack, but + have since either changed their name or been removed. + + ``installed`` + Possible values: True, False, any + + Specs for which a prefix exists are "installed". A spec + that is NOT installed will be in the database if some + other spec depends on it but its installation has gone + away since Spack installed it. + + TODO: Specs are a lot like queries. Should there be a + wildcard spec object, and should specs have attributes + like installed and known that can be queried? Or are + these really special cases that only belong here? + + """ + with self.read_transaction(): + results = [] + for key, rec in self._data.items(): + if installed is not any and rec.installed != installed: + continue + if known is not any and spack.db.exists(rec.spec.name) != known: + continue + if query_spec is any or rec.spec.satisfies(query_spec): + results.append(rec.spec) + + return sorted(results) + + + def query_one(self, query_spec, known=any, installed=True): + """Query for exactly one spec that matches the query spec. + + Raises an assertion error if more than one spec matches the + query. Returns None if no installed package matches. + + """ + concrete_specs = self.query(query_spec, known, installed) + assert len(concrete_specs) <= 1 + return concrete_specs[0] if concrete_specs else None + + + def missing(self, spec): + with self.read_transaction(): + key = spec.dag_hash() + return key in self._data and not self._data[key].installed + + +class _Transaction(object): + """Simple nested transaction context manager that uses a file lock. + + This class can trigger actions when the lock is acquired for the + first time and released for the last. + + Timeout for lock is customizable. + """ + def __init__(self, db, acquire_fn=None, release_fn=None, + timeout=_db_lock_timeout): + self._db = db + self._timeout = timeout + self._acquire_fn = acquire_fn + self._release_fn = release_fn + + def __enter__(self): + if self._enter() and self._acquire_fn: + self._acquire_fn() + + def __exit__(self, type, value, traceback): + if self._exit() and self._release_fn: + self._release_fn() + + +class ReadTransaction(_Transaction): + def _enter(self): + return self._db.lock.acquire_read(self._timeout) + + def _exit(self): + return self._db.lock.release_read() + + +class WriteTransaction(_Transaction): + def _enter(self): + return self._db.lock.acquire_write(self._timeout) + + def _exit(self): + return self._db.lock.release_write() + + +class CorruptDatabaseError(SpackError): + def __init__(self, path, msg=''): + super(CorruptDatabaseError, self).__init__( + "Spack database is corrupt: %s. %s" %(path, msg)) + + +class InvalidDatabaseVersionError(SpackError): + def __init__(self, expected, found): + super(InvalidDatabaseVersionError, self).__init__( + "Expected database version %s but found version %s" + % (expected, found)) diff --git a/lib/spack/spack/directives.py b/lib/spack/spack/directives.py index 9297d6dac3..2a818e8d0c 100644 --- a/lib/spack/spack/directives.py +++ b/lib/spack/spack/directives.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify @@ -239,12 +239,10 @@ def patch(pkg, url_or_filename, level=1, when=None): when = pkg.name when_spec = parse_anonymous_spec(when, pkg.name) - if when_spec not in pkg.patches: - pkg.patches[when_spec] = [Patch(pkg.name, url_or_filename, level)] - else: - # if this spec is identical to some other, then append this - # patch to the existing list. - pkg.patches[when_spec].append(Patch(pkg.name, url_or_filename, level)) + cur_patches = pkg.patches.setdefault(when_spec, []) + # if this spec is identical to some other, then append this + # patch to the existing list. + cur_patches.append(Patch(pkg.name, url_or_filename, level)) @directive('variants') diff --git a/lib/spack/spack/directory_layout.py b/lib/spack/spack/directory_layout.py index 85ecc1ce2b..056606b429 100644 --- a/lib/spack/spack/directory_layout.py +++ b/lib/spack/spack/directory_layout.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify @@ -29,10 +29,9 @@ import hashlib import shutil import glob import tempfile -from external import yaml +import yaml import llnl.util.tty as tty -from llnl.util.lang import memoized from llnl.util.filesystem import join_path, mkdirp from spack.spec import Spec @@ -258,7 +257,6 @@ class YamlDirectoryLayout(DirectoryLayout): self.write_spec(spec, spec_file_path) - @memoized def all_specs(self): if not os.path.isdir(self.root): return [] @@ -269,7 +267,6 @@ class YamlDirectoryLayout(DirectoryLayout): return [self.read_spec(s) for s in spec_files] - @memoized def specs_by_hash(self): by_hash = {} for spec in self.all_specs(): diff --git a/lib/spack/spack/error.py b/lib/spack/spack/error.py index bfa7951a47..0c2e7eb53c 100644 --- a/lib/spack/spack/error.py +++ b/lib/spack/spack/error.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify @@ -55,8 +55,8 @@ class SpackError(Exception): def __str__(self): msg = self.message - if self.long_message: - msg += "\n %s" % self.long_message + if self._long_message: + msg += "\n %s" % self._long_message return msg class UnsupportedPlatformError(SpackError): diff --git a/lib/spack/spack/fetch_strategy.py b/lib/spack/spack/fetch_strategy.py index b810023c5a..8c4bd4ed8a 100644 --- a/lib/spack/spack/fetch_strategy.py +++ b/lib/spack/spack/fetch_strategy.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify @@ -190,7 +190,7 @@ class URLFetchStrategy(FetchStrategy): if content_types and 'text/html' in content_types[-1]: tty.warn("The contents of " + self.archive_file + " look like HTML.", "The checksum will likely be bad. If it is, you can use", - "'spack clean --dist' to remove the bad archive, then fix", + "'spack clean <package>' to remove the bad archive, then fix", "your internet gateway issue and install again.") if not self.archive_file: diff --git a/lib/spack/spack/graph.py b/lib/spack/spack/graph.py index 5fb6a9cd23..e8c5cfb080 100644 --- a/lib/spack/spack/graph.py +++ b/lib/spack/spack/graph.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/hooks/__init__.py b/lib/spack/spack/hooks/__init__.py index 1c44e8abaa..2765f7be39 100644 --- a/lib/spack/spack/hooks/__init__.py +++ b/lib/spack/spack/hooks/__init__.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/hooks/dotkit.py b/lib/spack/spack/hooks/dotkit.py index 4e748ff80a..9123637356 100644 --- a/lib/spack/spack/hooks/dotkit.py +++ b/lib/spack/spack/hooks/dotkit.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/hooks/extensions.py b/lib/spack/spack/hooks/extensions.py index cf87a78c8c..b4847d697f 100644 --- a/lib/spack/spack/hooks/extensions.py +++ b/lib/spack/spack/hooks/extensions.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/hooks/tclmodule.py b/lib/spack/spack/hooks/tclmodule.py index 0b9fd5a67c..8b315f27a2 100644 --- a/lib/spack/spack/hooks/tclmodule.py +++ b/lib/spack/spack/hooks/tclmodule.py @@ -6,7 +6,7 @@ # Written by David Beckingsale, david@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/mirror.py b/lib/spack/spack/mirror.py index 306c8085aa..ee0bf6de11 100644 --- a/lib/spack/spack/mirror.py +++ b/lib/spack/spack/mirror.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py index 56a61adefb..59ed9f893e 100644 --- a/lib/spack/spack/modules.py +++ b/lib/spack/spack/modules.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/multimethod.py b/lib/spack/spack/multimethod.py index 892619c6ac..df9b9b2ab1 100644 --- a/lib/spack/spack/multimethod.py +++ b/lib/spack/spack/multimethod.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 61606d0590..daba5cd352 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify @@ -570,9 +570,12 @@ class Package(object): @property def installed_dependents(self): """Return a list of the specs of all installed packages that depend - on this one.""" + on this one. + + TODO: move this method to database.py? + """ dependents = [] - for spec in spack.db.installed_package_specs(): + for spec in spack.installed_db.query(): if self.name == spec.name: continue for dep in spec.traverse(): @@ -786,6 +789,7 @@ class Package(object): "Manually remove this directory to fix:", self.prefix) + def real_work(): try: tty.msg("Building %s." % self.name) @@ -845,6 +849,10 @@ class Package(object): # Do the build. spack.build_environment.fork(self, real_work) + # note: PARENT of the build process adds the new package to + # the database, so that we don't need to re-read from file. + spack.installed_db.add(self.spec, self.prefix) + # Once everything else is done, run post install hooks spack.hooks.post_install(self) @@ -864,6 +872,14 @@ class Package(object): @property + def build_log_path(self): + if self.installed: + return spack.install_layout.build_log_path(self.spec) + else: + return join_path(self.stage.source_path, 'spack-build.out') + + + @property def module(self): """Use this to add variables to the class's module's scope. This lets us use custom syntax in the install method. @@ -917,6 +933,7 @@ class Package(object): # Uninstalling in Spack only requires removing the prefix. self.remove_prefix() + spack.installed_db.remove(self.spec) tty.msg("Successfully uninstalled %s." % self.spec.short_spec) # Once everything else is done, run post install hooks diff --git a/lib/spack/spack/packages.py b/lib/spack/spack/packages.py index adfbc26c1d..f6f4cbf025 100644 --- a/lib/spack/spack/packages.py +++ b/lib/spack/spack/packages.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify @@ -96,12 +96,6 @@ class PackageDB(object): @_autospec - def get_installed(self, spec): - """Get all the installed specs that satisfy the provided spec constraint.""" - return [s for s in self.installed_package_specs() if s.satisfies(spec)] - - - @_autospec def providers_for(self, vpkg_spec): if self.provider_index is None: self.provider_index = ProviderIndex(self.all_package_names()) @@ -117,19 +111,6 @@ class PackageDB(object): return [p for p in self.all_packages() if p.extends(extendee_spec)] - @_autospec - def installed_extensions_for(self, extendee_spec): - for s in self.installed_package_specs(): - try: - if s.package.extends(extendee_spec): - yield s.package - except UnknownPackageError, e: - # Skip packages we know nothing about - continue - # TODO: add some conditional way to do this instead of - # catching exceptions. - - def dirname_for_package_name(self, pkg_name): """Get the directory name for a particular package. This is the directory that contains its package.py file.""" @@ -150,29 +131,6 @@ class PackageDB(object): return join_path(pkg_dir, _package_file_name) - def installed_package_specs(self): - """Read installed package names straight from the install directory - layout. - """ - # Get specs from the directory layout but ensure that they're - # all normalized properly. - installed = [] - for spec in spack.install_layout.all_specs(): - spec.normalize() - installed.append(spec) - return installed - - - def installed_known_package_specs(self): - """Read installed package names straight from the install - directory layout, but return only specs for which the - package is known to this version of spack. - """ - for spec in spack.install_layout.all_specs(): - if self.exists(spec.name): - yield spec - - @memoized def all_package_names(self): """Generator function for all packages. This looks for diff --git a/lib/spack/spack/parse.py b/lib/spack/spack/parse.py index bc12ec258c..e9467aa685 100644 --- a/lib/spack/spack/parse.py +++ b/lib/spack/spack/parse.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/patch.py b/lib/spack/spack/patch.py index b1b6e07738..42f2105f52 100644 --- a/lib/spack/spack/patch.py +++ b/lib/spack/spack/patch.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index e1fbb84423..f62182ce76 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify @@ -96,8 +96,8 @@ import hashlib import base64 from StringIO import StringIO from operator import attrgetter -from external import yaml -from external.yaml.error import MarkedYAMLError +import yaml +from yaml.error import MarkedYAMLError import llnl.util.tty as tty from llnl.util.lang import * @@ -640,7 +640,9 @@ class Spec(object): def dag_hash(self, length=None): - """Return a hash of the entire spec DAG, including connectivity.""" + """ + Return a hash of the entire spec DAG, including connectivity. + """ yaml_text = yaml.dump( self.to_node_dict(), default_flow_style=True, width=sys.maxint) sha = hashlib.sha1(yaml_text) @@ -710,7 +712,7 @@ class Spec(object): try: yfile = yaml.load(stream) except MarkedYAMLError, e: - raise SpackYAMLError("error parsing YMAL spec:", str(e)) + raise SpackYAMLError("error parsing YAML spec:", str(e)) for node in yfile['spec']: name = next(iter(node)) @@ -1205,6 +1207,13 @@ class Spec(object): return common + def constrained(self, other, deps=True): + """Return a constrained copy without modifying this spec.""" + clone = self.copy(deps=deps) + clone.constrain(other, deps) + return clone + + def dep_difference(self, other): """Returns dependencies in self that are not in other.""" mine = set(s.name for s in self.traverse(root=False)) @@ -1998,4 +2007,4 @@ class UnsatisfiableDependencySpecError(UnsatisfiableSpecError): class SpackYAMLError(spack.error.SpackError): def __init__(self, msg, yaml_error): - super(SpackError, self).__init__(msg, str(yaml_error)) + super(SpackYAMLError, self).__init__(msg, str(yaml_error)) diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index 008c5f0429..754344fc01 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify @@ -261,7 +261,8 @@ class Stage(object): tty.debug(e) continue else: - tty.die("All fetchers failed for %s" % self.name) + errMessage = "All fetchers failed for %s" % self.name + raise fs.FetchError(errMessage, None) def check(self): diff --git a/lib/spack/spack/test/__init__.py b/lib/spack/spack/test/__init__.py index 6b3715be6f..13cb1d2b78 100644 --- a/lib/spack/spack/test/__init__.py +++ b/lib/spack/spack/test/__init__.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify @@ -56,7 +56,10 @@ test_names = ['versions', 'spec_yaml', 'optional_deps', 'make_executable', - 'configure_guess'] + 'configure_guess', + 'unit_install', + 'lock', + 'database'] def list_tests(): @@ -76,7 +79,7 @@ def run(names, verbose=False): if test not in test_names: tty.error("%s is not a valid spack test name." % test, "Valid names are:") - colify(test_names, indent=4) + colify(sorted(test_names), indent=4) sys.exit(1) runner = unittest.TextTestRunner(verbosity=verbosity) diff --git a/lib/spack/spack/test/cc.py b/lib/spack/spack/test/cc.py index aa16f9b351..4188b8d550 100644 --- a/lib/spack/spack/test/cc.py +++ b/lib/spack/spack/test/cc.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/test/concretize.py b/lib/spack/spack/test/concretize.py index b3a77d076a..2f8e0c7ec0 100644 --- a/lib/spack/spack/test/concretize.py +++ b/lib/spack/spack/test/concretize.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/test/config.py b/lib/spack/spack/test/config.py index 790b22f3b0..ed11e34c69 100644 --- a/lib/spack/spack/test/config.py +++ b/lib/spack/spack/test/config.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/test/configure_guess.py b/lib/spack/spack/test/configure_guess.py index 766dd51d52..a4e8565b62 100644 --- a/lib/spack/spack/test/configure_guess.py +++ b/lib/spack/spack/test/configure_guess.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/test/database.py b/lib/spack/spack/test/database.py new file mode 100644 index 0000000000..c07d32686e --- /dev/null +++ b/lib/spack/spack/test/database.py @@ -0,0 +1,352 @@ +############################################################################## +# Copyright (c) 2013-2015, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +""" +These tests check the database is functioning properly, +both in memory and in its file +""" +import tempfile +import shutil +import multiprocessing + +from llnl.util.lock import * +from llnl.util.filesystem import join_path + +import spack +from spack.database import Database +from spack.directory_layout import YamlDirectoryLayout +from spack.test.mock_packages_test import * + +from llnl.util.tty.colify import colify + +def _print_ref_counts(): + """Print out all ref counts for the graph used here, for debugging""" + recs = [] + + def add_rec(spec): + cspecs = spack.installed_db.query(spec, installed=any) + + if not cspecs: + recs.append("[ %-7s ] %-20s-" % ('', spec)) + else: + key = cspecs[0].dag_hash() + rec = spack.installed_db.get_record(cspecs[0]) + recs.append("[ %-7s ] %-20s%d" % (key[:7], spec, rec.ref_count)) + + with spack.installed_db.read_transaction(): + add_rec('mpileaks ^mpich') + add_rec('callpath ^mpich') + add_rec('mpich') + + add_rec('mpileaks ^mpich2') + add_rec('callpath ^mpich2') + add_rec('mpich2') + + add_rec('mpileaks ^zmpi') + add_rec('callpath ^zmpi') + add_rec('zmpi') + add_rec('fake') + + add_rec('dyninst') + add_rec('libdwarf') + add_rec('libelf') + + colify(recs, cols=3) + + +class DatabaseTest(MockPackagesTest): + + def _mock_install(self, spec): + s = Spec(spec) + pkg = spack.db.get(s.concretized()) + pkg.do_install(fake=True) + + + def _mock_remove(self, spec): + specs = spack.installed_db.query(spec) + assert(len(specs) == 1) + spec = specs[0] + spec.package.do_uninstall(spec) + + + def setUp(self): + super(DatabaseTest, self).setUp() + # + # TODO: make the mockup below easier. + # + + # Make a fake install directory + self.install_path = tempfile.mkdtemp() + self.spack_install_path = spack.install_path + spack.install_path = self.install_path + + self.install_layout = YamlDirectoryLayout(self.install_path) + self.spack_install_layout = spack.install_layout + spack.install_layout = self.install_layout + + # Make fake database and fake install directory. + self.installed_db = Database(self.install_path) + self.spack_installed_db = spack.installed_db + spack.installed_db = self.installed_db + + # make a mock database with some packages installed note that + # the ref count for dyninst here will be 3, as it's recycled + # across each install. + # + # Here is what the mock DB looks like: + # + # o mpileaks o mpileaks' o mpileaks'' + # |\ |\ |\ + # | o callpath | o callpath' | o callpath'' + # |/| |/| |/| + # o | mpich o | mpich2 o | zmpi + # | | o | fake + # | | | + # | |______________/ + # | .____________/ + # |/ + # o dyninst + # |\ + # | o libdwarf + # |/ + # o libelf + # + + # Transaction used to avoid repeated writes. + with spack.installed_db.write_transaction(): + self._mock_install('mpileaks ^mpich') + self._mock_install('mpileaks ^mpich2') + self._mock_install('mpileaks ^zmpi') + + + def tearDown(self): + super(DatabaseTest, self).tearDown() + shutil.rmtree(self.install_path) + spack.install_path = self.spack_install_path + spack.install_layout = self.spack_install_layout + spack.installed_db = self.spack_installed_db + + + def test_005_db_exists(self): + """Make sure db cache file exists after creating.""" + index_file = join_path(self.install_path, '.spack-db', 'index.yaml') + lock_file = join_path(self.install_path, '.spack-db', 'lock') + + self.assertTrue(os.path.exists(index_file)) + self.assertTrue(os.path.exists(lock_file)) + + + def test_010_all_install_sanity(self): + """Ensure that the install layout reflects what we think it does.""" + all_specs = spack.install_layout.all_specs() + self.assertEqual(len(all_specs), 13) + + # query specs with multiple configurations + mpileaks_specs = [s for s in all_specs if s.satisfies('mpileaks')] + callpath_specs = [s for s in all_specs if s.satisfies('callpath')] + mpi_specs = [s for s in all_specs if s.satisfies('mpi')] + + self.assertEqual(len(mpileaks_specs), 3) + self.assertEqual(len(callpath_specs), 3) + self.assertEqual(len(mpi_specs), 3) + + # query specs with single configurations + dyninst_specs = [s for s in all_specs if s.satisfies('dyninst')] + libdwarf_specs = [s for s in all_specs if s.satisfies('libdwarf')] + libelf_specs = [s for s in all_specs if s.satisfies('libelf')] + + self.assertEqual(len(dyninst_specs), 1) + self.assertEqual(len(libdwarf_specs), 1) + self.assertEqual(len(libelf_specs), 1) + + # Query by dependency + self.assertEqual(len([s for s in all_specs if s.satisfies('mpileaks ^mpich')]), 1) + self.assertEqual(len([s for s in all_specs if s.satisfies('mpileaks ^mpich2')]), 1) + self.assertEqual(len([s for s in all_specs if s.satisfies('mpileaks ^zmpi')]), 1) + + + def test_015_write_and_read(self): + # write and read DB + with spack.installed_db.write_transaction(): + specs = spack.installed_db.query() + recs = [spack.installed_db.get_record(s) for s in specs] + + for spec, rec in zip(specs, recs): + new_rec = spack.installed_db.get_record(spec) + self.assertEqual(new_rec.ref_count, rec.ref_count) + self.assertEqual(new_rec.spec, rec.spec) + self.assertEqual(new_rec.path, rec.path) + self.assertEqual(new_rec.installed, rec.installed) + + + def _check_db_sanity(self): + """Utiilty function to check db against install layout.""" + expected = sorted(spack.install_layout.all_specs()) + actual = sorted(self.installed_db.query()) + + self.assertEqual(len(expected), len(actual)) + for e, a in zip(expected, actual): + self.assertEqual(e, a) + + + def test_020_db_sanity(self): + """Make sure query() returns what's actually in the db.""" + self._check_db_sanity() + + + def test_030_db_sanity_from_another_process(self): + def read_and_modify(): + self._check_db_sanity() # check that other process can read DB + with self.installed_db.write_transaction(): + self._mock_remove('mpileaks ^zmpi') + + p = multiprocessing.Process(target=read_and_modify, args=()) + p.start() + p.join() + + # ensure child process change is visible in parent process + with self.installed_db.read_transaction(): + self.assertEqual(len(self.installed_db.query('mpileaks ^zmpi')), 0) + + + def test_040_ref_counts(self): + """Ensure that we got ref counts right when we read the DB.""" + self.installed_db._check_ref_counts() + + + def test_050_basic_query(self): + """Ensure that querying the database is consistent with what is installed.""" + # query everything + self.assertEqual(len(spack.installed_db.query()), 13) + + # query specs with multiple configurations + mpileaks_specs = self.installed_db.query('mpileaks') + callpath_specs = self.installed_db.query('callpath') + mpi_specs = self.installed_db.query('mpi') + + self.assertEqual(len(mpileaks_specs), 3) + self.assertEqual(len(callpath_specs), 3) + self.assertEqual(len(mpi_specs), 3) + + # query specs with single configurations + dyninst_specs = self.installed_db.query('dyninst') + libdwarf_specs = self.installed_db.query('libdwarf') + libelf_specs = self.installed_db.query('libelf') + + self.assertEqual(len(dyninst_specs), 1) + self.assertEqual(len(libdwarf_specs), 1) + self.assertEqual(len(libelf_specs), 1) + + # Query by dependency + self.assertEqual(len(self.installed_db.query('mpileaks ^mpich')), 1) + self.assertEqual(len(self.installed_db.query('mpileaks ^mpich2')), 1) + self.assertEqual(len(self.installed_db.query('mpileaks ^zmpi')), 1) + + + def _check_remove_and_add_package(self, spec): + """Remove a spec from the DB, then add it and make sure everything's + still ok once it is added. This checks that it was + removed, that it's back when added again, and that ref + counts are consistent. + """ + original = self.installed_db.query() + self.installed_db._check_ref_counts() + + # Remove spec + concrete_spec = self.installed_db.remove(spec) + self.installed_db._check_ref_counts() + remaining = self.installed_db.query() + + # ensure spec we removed is gone + self.assertEqual(len(original) - 1, len(remaining)) + self.assertTrue(all(s in original for s in remaining)) + self.assertTrue(concrete_spec not in remaining) + + # add it back and make sure everything is ok. + self.installed_db.add(concrete_spec, "") + installed = self.installed_db.query() + self.assertEqual(len(installed), len(original)) + + # sanity check against direcory layout and check ref counts. + self._check_db_sanity() + self.installed_db._check_ref_counts() + + + def test_060_remove_and_add_root_package(self): + self._check_remove_and_add_package('mpileaks ^mpich') + + + def test_070_remove_and_add_dependency_package(self): + self._check_remove_and_add_package('dyninst') + + + def test_080_root_ref_counts(self): + rec = self.installed_db.get_record('mpileaks ^mpich') + + # Remove a top-level spec from the DB + self.installed_db.remove('mpileaks ^mpich') + + # record no longer in DB + self.assertEqual(self.installed_db.query('mpileaks ^mpich', installed=any), []) + + # record's deps have updated ref_counts + self.assertEqual(self.installed_db.get_record('callpath ^mpich').ref_count, 0) + self.assertEqual(self.installed_db.get_record('mpich').ref_count, 1) + + # put the spec back + self.installed_db.add(rec.spec, rec.path) + + # record is present again + self.assertEqual(len(self.installed_db.query('mpileaks ^mpich', installed=any)), 1) + + # dependencies have ref counts updated + self.assertEqual(self.installed_db.get_record('callpath ^mpich').ref_count, 1) + self.assertEqual(self.installed_db.get_record('mpich').ref_count, 2) + + + def test_090_non_root_ref_counts(self): + mpileaks_mpich_rec = self.installed_db.get_record('mpileaks ^mpich') + callpath_mpich_rec = self.installed_db.get_record('callpath ^mpich') + + # "force remove" a non-root spec from the DB + self.installed_db.remove('callpath ^mpich') + + # record still in DB but marked uninstalled + self.assertEqual(self.installed_db.query('callpath ^mpich', installed=True), []) + self.assertEqual(len(self.installed_db.query('callpath ^mpich', installed=any)), 1) + + # record and its deps have same ref_counts + self.assertEqual(self.installed_db.get_record('callpath ^mpich', installed=any).ref_count, 1) + self.assertEqual(self.installed_db.get_record('mpich').ref_count, 2) + + # remove only dependent of uninstalled callpath record + self.installed_db.remove('mpileaks ^mpich') + + # record and parent are completely gone. + self.assertEqual(self.installed_db.query('mpileaks ^mpich', installed=any), []) + self.assertEqual(self.installed_db.query('callpath ^mpich', installed=any), []) + + # mpich ref count updated properly. + mpich_rec = self.installed_db.get_record('mpich') + self.assertEqual(mpich_rec.ref_count, 0) diff --git a/lib/spack/spack/test/directory_layout.py b/lib/spack/spack/test/directory_layout.py index b3ad8efec4..703ac1b867 100644 --- a/lib/spack/spack/test/directory_layout.py +++ b/lib/spack/spack/test/directory_layout.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/test/git_fetch.py b/lib/spack/spack/test/git_fetch.py index 9700bd7533..244680b5d0 100644 --- a/lib/spack/spack/test/git_fetch.py +++ b/lib/spack/spack/test/git_fetch.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/test/hg_fetch.py b/lib/spack/spack/test/hg_fetch.py index 531dfabaa1..f8c6571bda 100644 --- a/lib/spack/spack/test/hg_fetch.py +++ b/lib/spack/spack/test/hg_fetch.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/test/install.py b/lib/spack/spack/test/install.py index 5659e97a4d..1ef4171fb2 100644 --- a/lib/spack/spack/test/install.py +++ b/lib/spack/spack/test/install.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/test/link_tree.py b/lib/spack/spack/test/link_tree.py index 9e887ecc49..886b7ef4c5 100644 --- a/lib/spack/spack/test/link_tree.py +++ b/lib/spack/spack/test/link_tree.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/test/lock.py b/lib/spack/spack/test/lock.py new file mode 100644 index 0000000000..bc68df01db --- /dev/null +++ b/lib/spack/spack/test/lock.py @@ -0,0 +1,266 @@ +############################################################################## +# Copyright (c) 2013-2015, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +""" +These tests ensure that our lock works correctly. +""" +import unittest +import os +import tempfile +import shutil +from multiprocessing import Process + +from llnl.util.lock import * +from llnl.util.filesystem import join_path, touch + +from spack.util.multiproc import Barrier + +# This is the longest a failed test will take, as the barriers will +# time out and raise an exception. +barrier_timeout = 5 + + +class LockTest(unittest.TestCase): + + def setUp(self): + self.tempdir = tempfile.mkdtemp() + self.lock_path = join_path(self.tempdir, 'lockfile') + touch(self.lock_path) + + + def tearDown(self): + shutil.rmtree(self.tempdir, ignore_errors=True) + + + def multiproc_test(self, *functions): + """Order some processes using simple barrier synchronization.""" + b = Barrier(len(functions), timeout=barrier_timeout) + procs = [Process(target=f, args=(b,)) for f in functions] + for p in procs: p.start() + for p in procs: + p.join() + self.assertEqual(p.exitcode, 0) + + + # + # Process snippets below can be composed into tests. + # + def acquire_write(self, barrier): + lock = Lock(self.lock_path) + lock.acquire_write() # grab exclusive lock + barrier.wait() + barrier.wait() # hold the lock until exception raises in other procs. + + def acquire_read(self, barrier): + lock = Lock(self.lock_path) + lock.acquire_read() # grab shared lock + barrier.wait() + barrier.wait() # hold the lock until exception raises in other procs. + + def timeout_write(self, barrier): + lock = Lock(self.lock_path) + barrier.wait() # wait for lock acquire in first process + self.assertRaises(LockError, lock.acquire_write, 0.1) + barrier.wait() + + def timeout_read(self, barrier): + lock = Lock(self.lock_path) + barrier.wait() # wait for lock acquire in first process + self.assertRaises(LockError, lock.acquire_read, 0.1) + barrier.wait() + + + # + # Test that exclusive locks on other processes time out when an + # exclusive lock is held. + # + def test_write_lock_timeout_on_write(self): + self.multiproc_test(self.acquire_write, self.timeout_write) + + def test_write_lock_timeout_on_write_2(self): + self.multiproc_test(self.acquire_write, self.timeout_write, self.timeout_write) + + def test_write_lock_timeout_on_write_3(self): + self.multiproc_test(self.acquire_write, self.timeout_write, self.timeout_write, self.timeout_write) + + + # + # Test that shared locks on other processes time out when an + # exclusive lock is held. + # + def test_read_lock_timeout_on_write(self): + self.multiproc_test(self.acquire_write, self.timeout_read) + + def test_read_lock_timeout_on_write_2(self): + self.multiproc_test(self.acquire_write, self.timeout_read, self.timeout_read) + + def test_read_lock_timeout_on_write_3(self): + self.multiproc_test(self.acquire_write, self.timeout_read, self.timeout_read, self.timeout_read) + + + # + # Test that exclusive locks time out when shared locks are held. + # + def test_write_lock_timeout_on_read(self): + self.multiproc_test(self.acquire_read, self.timeout_write) + + def test_write_lock_timeout_on_read_2(self): + self.multiproc_test(self.acquire_read, self.timeout_write, self.timeout_write) + + def test_write_lock_timeout_on_read_3(self): + self.multiproc_test(self.acquire_read, self.timeout_write, self.timeout_write, self.timeout_write) + + + # + # Test that exclusive locks time while lots of shared locks are held. + # + def test_write_lock_timeout_with_multiple_readers_2_1(self): + self.multiproc_test(self.acquire_read, self.acquire_read, self.timeout_write) + + def test_write_lock_timeout_with_multiple_readers_2_2(self): + self.multiproc_test(self.acquire_read, self.acquire_read, self.timeout_write, self.timeout_write) + + def test_write_lock_timeout_with_multiple_readers_3_1(self): + self.multiproc_test(self.acquire_read, self.acquire_read, self.acquire_read, self.timeout_write) + + def test_write_lock_timeout_with_multiple_readers_3_2(self): + self.multiproc_test(self.acquire_read, self.acquire_read, self.acquire_read, self.timeout_write, self.timeout_write) + + + # + # Longer test case that ensures locks are reusable. Ordering is + # enforced by barriers throughout -- steps are shown with numbers. + # + def test_complex_acquire_and_release_chain(self): + def p1(barrier): + lock = Lock(self.lock_path) + + lock.acquire_write() + barrier.wait() # ---------------------------------------- 1 + # others test timeout + barrier.wait() # ---------------------------------------- 2 + lock.release_write() # release and others acquire read + barrier.wait() # ---------------------------------------- 3 + self.assertRaises(LockError, lock.acquire_write, 0.1) + lock.acquire_read() + barrier.wait() # ---------------------------------------- 4 + lock.release_read() + barrier.wait() # ---------------------------------------- 5 + + # p2 upgrades read to write + barrier.wait() # ---------------------------------------- 6 + self.assertRaises(LockError, lock.acquire_write, 0.1) + self.assertRaises(LockError, lock.acquire_read, 0.1) + barrier.wait() # ---------------------------------------- 7 + # p2 releases write and read + barrier.wait() # ---------------------------------------- 8 + + # p3 acquires read + barrier.wait() # ---------------------------------------- 9 + # p3 upgrades read to write + barrier.wait() # ---------------------------------------- 10 + self.assertRaises(LockError, lock.acquire_write, 0.1) + self.assertRaises(LockError, lock.acquire_read, 0.1) + barrier.wait() # ---------------------------------------- 11 + # p3 releases locks + barrier.wait() # ---------------------------------------- 12 + lock.acquire_read() + barrier.wait() # ---------------------------------------- 13 + lock.release_read() + + + def p2(barrier): + lock = Lock(self.lock_path) + + # p1 acquires write + barrier.wait() # ---------------------------------------- 1 + self.assertRaises(LockError, lock.acquire_write, 0.1) + self.assertRaises(LockError, lock.acquire_read, 0.1) + barrier.wait() # ---------------------------------------- 2 + lock.acquire_read() + barrier.wait() # ---------------------------------------- 3 + # p1 tests shared read + barrier.wait() # ---------------------------------------- 4 + # others release reads + barrier.wait() # ---------------------------------------- 5 + + lock.acquire_write() # upgrade read to write + barrier.wait() # ---------------------------------------- 6 + # others test timeout + barrier.wait() # ---------------------------------------- 7 + lock.release_write() # release read AND write (need both) + lock.release_read() + barrier.wait() # ---------------------------------------- 8 + + # p3 acquires read + barrier.wait() # ---------------------------------------- 9 + # p3 upgrades read to write + barrier.wait() # ---------------------------------------- 10 + self.assertRaises(LockError, lock.acquire_write, 0.1) + self.assertRaises(LockError, lock.acquire_read, 0.1) + barrier.wait() # ---------------------------------------- 11 + # p3 releases locks + barrier.wait() # ---------------------------------------- 12 + lock.acquire_read() + barrier.wait() # ---------------------------------------- 13 + lock.release_read() + + + def p3(barrier): + lock = Lock(self.lock_path) + + # p1 acquires write + barrier.wait() # ---------------------------------------- 1 + self.assertRaises(LockError, lock.acquire_write, 0.1) + self.assertRaises(LockError, lock.acquire_read, 0.1) + barrier.wait() # ---------------------------------------- 2 + lock.acquire_read() + barrier.wait() # ---------------------------------------- 3 + # p1 tests shared read + barrier.wait() # ---------------------------------------- 4 + lock.release_read() + barrier.wait() # ---------------------------------------- 5 + + # p2 upgrades read to write + barrier.wait() # ---------------------------------------- 6 + self.assertRaises(LockError, lock.acquire_write, 0.1) + self.assertRaises(LockError, lock.acquire_read, 0.1) + barrier.wait() # ---------------------------------------- 7 + # p2 releases write & read + barrier.wait() # ---------------------------------------- 8 + + lock.acquire_read() + barrier.wait() # ---------------------------------------- 9 + lock.acquire_write() + barrier.wait() # ---------------------------------------- 10 + # others test timeout + barrier.wait() # ---------------------------------------- 11 + lock.release_read() # release read AND write in opposite + lock.release_write() # order from before on p2 + barrier.wait() # ---------------------------------------- 12 + lock.acquire_read() + barrier.wait() # ---------------------------------------- 13 + lock.release_read() + + self.multiproc_test(p1, p2, p3) diff --git a/lib/spack/spack/test/make_executable.py b/lib/spack/spack/test/make_executable.py index c4bfeb2a03..09efec8580 100644 --- a/lib/spack/spack/test/make_executable.py +++ b/lib/spack/spack/test/make_executable.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/test/mirror.py b/lib/spack/spack/test/mirror.py index 89ab14359e..189a85fb1a 100644 --- a/lib/spack/spack/test/mirror.py +++ b/lib/spack/spack/test/mirror.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/test/mock_packages_test.py b/lib/spack/spack/test/mock_packages_test.py index 00f81114af..e4e1b21b53 100644 --- a/lib/spack/spack/test/mock_packages_test.py +++ b/lib/spack/spack/test/mock_packages_test.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/test/mock_repo.py b/lib/spack/spack/test/mock_repo.py index fd184e64bc..c454b1f106 100644 --- a/lib/spack/spack/test/mock_repo.py +++ b/lib/spack/spack/test/mock_repo.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/test/multimethod.py b/lib/spack/spack/test/multimethod.py index cd5d9e625e..d8d61d14c8 100644 --- a/lib/spack/spack/test/multimethod.py +++ b/lib/spack/spack/test/multimethod.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/test/optional_deps.py b/lib/spack/spack/test/optional_deps.py index fbee0cfa8f..ebd7281999 100644 --- a/lib/spack/spack/test/optional_deps.py +++ b/lib/spack/spack/test/optional_deps.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/test/package_sanity.py b/lib/spack/spack/test/package_sanity.py index 6222e7b5f8..370cf676ef 100644 --- a/lib/spack/spack/test/package_sanity.py +++ b/lib/spack/spack/test/package_sanity.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/test/packages.py b/lib/spack/spack/test/packages.py index a8183cf6a6..b2daea7b7b 100644 --- a/lib/spack/spack/test/packages.py +++ b/lib/spack/spack/test/packages.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/test/python_version.py b/lib/spack/spack/test/python_version.py index 5779d31ed2..ba570b416f 100644 --- a/lib/spack/spack/test/python_version.py +++ b/lib/spack/spack/test/python_version.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify @@ -34,7 +34,7 @@ import re import llnl.util.tty as tty -from external import pyqver2 +import pyqver2 import spack spack_max_version = (2,6) diff --git a/lib/spack/spack/test/spec_dag.py b/lib/spack/spack/test/spec_dag.py index 549f829d3e..94438b6a0c 100644 --- a/lib/spack/spack/test/spec_dag.py +++ b/lib/spack/spack/test/spec_dag.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/test/spec_semantics.py b/lib/spack/spack/test/spec_semantics.py index 6666dbbb52..1381556aad 100644 --- a/lib/spack/spack/test/spec_semantics.py +++ b/lib/spack/spack/test/spec_semantics.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify @@ -190,11 +190,23 @@ class SpecSematicsTest(MockPackagesTest): def test_satisfies_virtual(self): + # Don't use check_satisfies: it checks constrain() too, and + # you can't constrain a non-virtual by a virtual. self.assertTrue(Spec('mpich').satisfies(Spec('mpi'))) self.assertTrue(Spec('mpich2').satisfies(Spec('mpi'))) self.assertTrue(Spec('zmpi').satisfies(Spec('mpi'))) + def test_satisfies_virtual_dep_with_virtual_constraint(self): + """Ensure we can satisfy virtual constraints when there are multiple + vdep providers in the specs.""" + self.assertTrue(Spec('netlib-lapack ^openblas').satisfies('netlib-lapack ^openblas')) + self.assertFalse(Spec('netlib-lapack ^netlib-blas').satisfies('netlib-lapack ^openblas')) + + self.assertFalse(Spec('netlib-lapack ^openblas').satisfies('netlib-lapack ^netlib-blas')) + self.assertTrue(Spec('netlib-lapack ^netlib-blas').satisfies('netlib-lapack ^netlib-blas')) + + # ================================================================================ # Indexing specs # ================================================================================ @@ -327,4 +339,3 @@ class SpecSematicsTest(MockPackagesTest): self.check_constrain_not_changed('libelf^foo+debug', 'libelf^foo+debug') self.check_constrain_not_changed('libelf^foo~debug', 'libelf^foo~debug') self.check_constrain_not_changed('libelf^foo=bgqos_0', 'libelf^foo=bgqos_0') - diff --git a/lib/spack/spack/test/spec_syntax.py b/lib/spack/spack/test/spec_syntax.py index 404f38906e..1daaa4be8f 100644 --- a/lib/spack/spack/test/spec_syntax.py +++ b/lib/spack/spack/test/spec_syntax.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/test/spec_yaml.py b/lib/spack/spack/test/spec_yaml.py index 869befc02a..11987ea1b3 100644 --- a/lib/spack/spack/test/spec_yaml.py +++ b/lib/spack/spack/test/spec_yaml.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/test/stage.py b/lib/spack/spack/test/stage.py index 8cff8f7960..c1b2a2a573 100644 --- a/lib/spack/spack/test/stage.py +++ b/lib/spack/spack/test/stage.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/test/svn_fetch.py b/lib/spack/spack/test/svn_fetch.py index 184fe8faa1..9229af76d4 100644 --- a/lib/spack/spack/test/svn_fetch.py +++ b/lib/spack/spack/test/svn_fetch.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/test/unit_install.py b/lib/spack/spack/test/unit_install.py new file mode 100644 index 0000000000..41c76a6dfa --- /dev/null +++ b/lib/spack/spack/test/unit_install.py @@ -0,0 +1,121 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +import unittest +import itertools + +import spack +test_install = __import__("spack.cmd.test-install", + fromlist=["BuildId", "create_test_output", "TestResult"]) + +class MockOutput(object): + def __init__(self): + self.results = {} + + def add_test(self, buildId, passed=True, buildInfo=None): + self.results[buildId] = passed + + def write_to(self, stream): + pass + +class MockSpec(object): + def __init__(self, name, version, hashStr=None): + self.dependencies = {} + self.name = name + self.version = version + self.hash = hashStr if hashStr else hash((name, version)) + + def traverse(self, order=None): + allDeps = itertools.chain.from_iterable(i.traverse() for i in + self.dependencies.itervalues()) + return set(itertools.chain([self], allDeps)) + + def dag_hash(self): + return self.hash + + def to_yaml(self): + return "<<<MOCK YAML {0}>>>".format(test_install.BuildId(self).stringId()) + +class MockPackage(object): + def __init__(self, buildLogPath): + self.installed = False + self.build_log_path = buildLogPath + +specX = MockSpec("X", "1.2.0") +specY = MockSpec("Y", "2.3.8") +specX.dependencies['Y'] = specY +pkgX = MockPackage('logX') +pkgY = MockPackage('logY') +bIdX = test_install.BuildId(specX) +bIdY = test_install.BuildId(specY) + +class UnitInstallTest(unittest.TestCase): + """Tests test-install where X->Y""" + + def setUp(self): + super(UnitInstallTest, self).setUp() + + pkgX.installed = False + pkgY.installed = False + + pkgDb = MockPackageDb({specX:pkgX, specY:pkgY}) + spack.db = pkgDb + + def tearDown(self): + super(UnitInstallTest, self).tearDown() + + def test_installing_both(self): + mo = MockOutput() + + pkgX.installed = True + pkgY.installed = True + test_install.create_test_output(specX, [specX, specY], mo, getLogFunc=test_fetch_log) + + self.assertEqual(mo.results, + {bIdX:test_install.TestResult.PASSED, + bIdY:test_install.TestResult.PASSED}) + + def test_dependency_already_installed(self): + mo = MockOutput() + + pkgX.installed = True + pkgY.installed = True + test_install.create_test_output(specX, [specX], mo, getLogFunc=test_fetch_log) + + self.assertEqual(mo.results, {bIdX:test_install.TestResult.PASSED}) + + #TODO: add test(s) where Y fails to install + +class MockPackageDb(object): + def __init__(self, init=None): + self.specToPkg = {} + if init: + self.specToPkg.update(init) + + def get(self, spec): + return self.specToPkg[spec] + +def test_fetch_log(path): + return [] + diff --git a/lib/spack/spack/test/url_extrapolate.py b/lib/spack/spack/test/url_extrapolate.py index 00d8216020..87adf89401 100644 --- a/lib/spack/spack/test/url_extrapolate.py +++ b/lib/spack/spack/test/url_extrapolate.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/test/url_parse.py b/lib/spack/spack/test/url_parse.py index ae1d559f7c..efde7c0c73 100644 --- a/lib/spack/spack/test/url_parse.py +++ b/lib/spack/spack/test/url_parse.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/test/url_substitution.py b/lib/spack/spack/test/url_substitution.py index db7ddd251d..8b90ee086a 100644 --- a/lib/spack/spack/test/url_substitution.py +++ b/lib/spack/spack/test/url_substitution.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/test/versions.py b/lib/spack/spack/test/versions.py index 20e946e90e..108450e098 100644 --- a/lib/spack/spack/test/versions.py +++ b/lib/spack/spack/test/versions.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/url.py b/lib/spack/spack/url.py index 58838306af..02c0b83e26 100644 --- a/lib/spack/spack/url.py +++ b/lib/spack/spack/url.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify @@ -70,7 +70,7 @@ def find_list_url(url): """ url_types = [ - # e.g. https://github.com/scalability-llnl/callpath/archive/v1.0.1.tar.gz + # e.g. https://github.com/llnl/callpath/archive/v1.0.1.tar.gz (r'^(https://github.com/[^/]+/[^/]+)/archive/', lambda m: m.group(1) + '/releases') ] @@ -209,8 +209,8 @@ def parse_version_offset(path): # e.g. foobar-4.5.1 (r'-((\d+\.)*\d+)$', stem), - # e.g. foobar-4.5.1b - (r'-((\d+\.)*\d+\-?([a-z]|rc|RC|tp|TP)\d*)$', stem), + # e.g. foobar-4.5.1b, foobar4.5RC, foobar.v4.5.1b + (r'[-._]?v?((\d+\.)*\d+[-._]?([a-z]|rc|RC|tp|TP?)\d*)$', stem), # e.g. foobar-4.5.0-beta1, or foobar-4.50-beta (r'-((\d+\.)*\d+-beta(\d+)?)$', stem), diff --git a/lib/spack/spack/util/__init__.py b/lib/spack/spack/util/__init__.py index 1c388f31c5..b54691b67c 100644 --- a/lib/spack/spack/util/__init__.py +++ b/lib/spack/spack/util/__init__.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/util/compression.py b/lib/spack/spack/util/compression.py index fd17785ad0..ea1f233bce 100644 --- a/lib/spack/spack/util/compression.py +++ b/lib/spack/spack/util/compression.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/util/crypto.py b/lib/spack/spack/util/crypto.py index 8a8574cd3d..5269260284 100644 --- a/lib/spack/spack/util/crypto.py +++ b/lib/spack/spack/util/crypto.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/util/debug.py b/lib/spack/spack/util/debug.py index 37985eccdd..7930753f6f 100644 --- a/lib/spack/spack/util/debug.py +++ b/lib/spack/spack/util/debug.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/util/environment.py b/lib/spack/spack/util/environment.py index 7a4ff919ad..cd413dcfbc 100644 --- a/lib/spack/spack/util/environment.py +++ b/lib/spack/spack/util/environment.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/util/executable.py b/lib/spack/spack/util/executable.py index d1dfb62ffb..4f9958062b 100644 --- a/lib/spack/spack/util/executable.py +++ b/lib/spack/spack/util/executable.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/util/multiproc.py b/lib/spack/spack/util/multiproc.py index 9e045a090f..8ca82df011 100644 --- a/lib/spack/spack/util/multiproc.py +++ b/lib/spack/spack/util/multiproc.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify @@ -27,9 +27,11 @@ This implements a parallel map operation but it can accept more values than multiprocessing.Pool.apply() can. For example, apply() will fail to pickle functions if they're passed indirectly as parameters. """ -from multiprocessing import Process, Pipe +from multiprocessing import Process, Pipe, Semaphore, Value from itertools import izip +__all__ = ['spawn', 'parmap', 'Barrier'] + def spawn(f): def fun(pipe,x): pipe.send(f(x)) @@ -43,3 +45,49 @@ def parmap(f,X): [p.join() for p in proc] return [p.recv() for (p,c) in pipe] + +class Barrier: + """Simple reusable semaphore barrier. + + Python 2.6 doesn't have multiprocessing barriers so we implement this. + + See http://greenteapress.com/semaphores/downey08semaphores.pdf, p. 41. + """ + def __init__(self, n, timeout=None): + self.n = n + self.to = timeout + self.count = Value('i', 0) + self.mutex = Semaphore(1) + self.turnstile1 = Semaphore(0) + self.turnstile2 = Semaphore(1) + + + def wait(self): + if not self.mutex.acquire(timeout=self.to): + raise BarrierTimeoutError() + self.count.value += 1 + if self.count.value == self.n: + if not self.turnstile2.acquire(timeout=self.to): + raise BarrierTimeoutError() + self.turnstile1.release() + self.mutex.release() + + if not self.turnstile1.acquire(timeout=self.to): + raise BarrierTimeoutError() + self.turnstile1.release() + + if not self.mutex.acquire(timeout=self.to): + raise BarrierTimeoutError() + self.count.value -= 1 + if self.count.value == 0: + if not self.turnstile1.acquire(timeout=self.to): + raise BarrierTimeoutError() + self.turnstile2.release() + self.mutex.release() + + if not self.turnstile2.acquire(timeout=self.to): + raise BarrierTimeoutError() + self.turnstile2.release() + + +class BarrierTimeoutError: pass diff --git a/lib/spack/spack/util/prefix.py b/lib/spack/spack/util/prefix.py index 7bd63c16ca..c613ca5182 100644 --- a/lib/spack/spack/util/prefix.py +++ b/lib/spack/spack/util/prefix.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/util/string.py b/lib/spack/spack/util/string.py index 234163bf52..1556ce6d29 100644 --- a/lib/spack/spack/util/string.py +++ b/lib/spack/spack/util/string.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/util/web.py b/lib/spack/spack/util/web.py index 1420d62a77..94384e9c86 100644 --- a/lib/spack/spack/util/web.py +++ b/lib/spack/spack/util/web.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/variant.py b/lib/spack/spack/variant.py index 3d3e2b0f6d..8959e76684 100644 --- a/lib/spack/spack/variant.py +++ b/lib/spack/spack/variant.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/version.py b/lib/spack/spack/version.py index 35db05e018..e8a0a261c9 100644 --- a/lib/spack/spack/version.py +++ b/lib/spack/spack/version.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify @@ -48,7 +48,7 @@ import sys import re from bisect import bisect_left from functools import wraps -from external.functools import total_ordering +from functools_backport import total_ordering # Valid version characters VALID_VERSION = r'[A-Za-z0-9_.-]' diff --git a/lib/spack/spack/virtual.py b/lib/spack/spack/virtual.py index fa070e6bd5..d16aea8642 100644 --- a/lib/spack/spack/virtual.py +++ b/lib/spack/spack/virtual.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify @@ -73,10 +73,8 @@ class ProviderIndex(object): for provided_spec, provider_spec in pkg.provided.iteritems(): if provider_spec.satisfies(spec, deps=False): provided_name = provided_spec.name - if provided_name not in self.providers: - self.providers[provided_name] = {} - provider_map = self.providers[provided_name] + provider_map = self.providers.setdefault(provided_name, {}) if not provided_spec in provider_map: provider_map[provided_spec] = set() @@ -119,12 +117,13 @@ class ProviderIndex(object): return sorted(providers) - # TODO: this is pretty darned nasty, and inefficient. + # TODO: this is pretty darned nasty, and inefficient, but there + # are not that many vdeps in most specs. def _cross_provider_maps(self, lmap, rmap): result = {} for lspec, rspec in itertools.product(lmap, rmap): try: - constrained = lspec.copy().constrain(rspec) + constrained = lspec.constrained(rspec) except spack.spec.UnsatisfiableSpecError: continue @@ -132,10 +131,8 @@ class ProviderIndex(object): for lp_spec, rp_spec in itertools.product(lmap[lspec], rmap[rspec]): if lp_spec.name == rp_spec.name: try: - const = lp_spec.copy().constrain(rp_spec,deps=False) - if constrained not in result: - result[constrained] = set() - result[constrained].add(const) + const = lp_spec.constrained(rp_spec, deps=False) + result.setdefault(constrained, set()).add(const) except spack.spec.UnsatisfiableSpecError: continue return result @@ -161,4 +158,4 @@ class ProviderIndex(object): if crossed: result[name] = crossed - return bool(result) + return all(c in result for c in common) diff --git a/share/spack/csh/spack.csh b/share/spack/csh/spack.csh index b21da23836..d64ce8935b 100644 --- a/share/spack/csh/spack.csh +++ b/share/spack/csh/spack.csh @@ -101,7 +101,7 @@ case unload: breaksw default: - \spack $_sp_args + \spack $_sp_flags $_sp_args breaksw endsw diff --git a/share/spack/setup-env.csh b/share/spack/setup-env.csh index 5f91670a60..42d8c42726 100755 --- a/share/spack/setup-env.csh +++ b/share/spack/setup-env.csh @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/share/spack/setup-env.sh b/share/spack/setup-env.sh index b90846d28f..c96a94195d 100755 --- a/share/spack/setup-env.sh +++ b/share/spack/setup-env.sh @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/mock_packages/callpath/package.py b/var/spack/mock_packages/callpath/package.py index 5b6b70ba2a..abc576f78f 100644 --- a/var/spack/mock_packages/callpath/package.py +++ b/var/spack/mock_packages/callpath/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/mock_packages/direct_mpich/package.py b/var/spack/mock_packages/direct_mpich/package.py index 2ced82521b..efe7fc2afc 100644 --- a/var/spack/mock_packages/direct_mpich/package.py +++ b/var/spack/mock_packages/direct_mpich/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/mock_packages/dyninst/package.py b/var/spack/mock_packages/dyninst/package.py index 7998578da1..ea57950865 100644 --- a/var/spack/mock_packages/dyninst/package.py +++ b/var/spack/mock_packages/dyninst/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/mock_packages/fake/package.py b/var/spack/mock_packages/fake/package.py index fb3c2bdd2e..5f81ef20fc 100644 --- a/var/spack/mock_packages/fake/package.py +++ b/var/spack/mock_packages/fake/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/mock_packages/indirect_mpich/package.py b/var/spack/mock_packages/indirect_mpich/package.py index daf8b4b166..0b1773a27b 100644 --- a/var/spack/mock_packages/indirect_mpich/package.py +++ b/var/spack/mock_packages/indirect_mpich/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/mock_packages/libdwarf/package.py b/var/spack/mock_packages/libdwarf/package.py index 0b8df04cfb..e486a5de03 100644 --- a/var/spack/mock_packages/libdwarf/package.py +++ b/var/spack/mock_packages/libdwarf/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/mock_packages/libelf/package.py b/var/spack/mock_packages/libelf/package.py index 94c8f942cd..5e5b0b7143 100644 --- a/var/spack/mock_packages/libelf/package.py +++ b/var/spack/mock_packages/libelf/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/mock_packages/mpich/package.py b/var/spack/mock_packages/mpich/package.py index f77d3efc5d..55bf97f2cf 100644 --- a/var/spack/mock_packages/mpich/package.py +++ b/var/spack/mock_packages/mpich/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/mock_packages/mpich2/package.py b/var/spack/mock_packages/mpich2/package.py index 827b94c8a4..90f969b898 100644 --- a/var/spack/mock_packages/mpich2/package.py +++ b/var/spack/mock_packages/mpich2/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/mock_packages/mpileaks/package.py b/var/spack/mock_packages/mpileaks/package.py index 3989f1b452..9a18c5e1f2 100644 --- a/var/spack/mock_packages/mpileaks/package.py +++ b/var/spack/mock_packages/mpileaks/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/mock_packages/multimethod/package.py b/var/spack/mock_packages/multimethod/package.py index 75b1606ffc..ea103fe175 100644 --- a/var/spack/mock_packages/multimethod/package.py +++ b/var/spack/mock_packages/multimethod/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/mock_packages/netlib-blas/package.py b/var/spack/mock_packages/netlib-blas/package.py new file mode 100644 index 0000000000..39f2c92ae5 --- /dev/null +++ b/var/spack/mock_packages/netlib-blas/package.py @@ -0,0 +1,36 @@ +############################################################################## +# Copyright (c) 2013-2015, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + +class NetlibBlas(Package): + homepage = "http://www.netlib.org/lapack/" + url = "http://www.netlib.org/lapack/lapack-3.5.0.tgz" + + version('3.5.0', 'b1d3e3e425b2e44a06760ff173104bdf') + + provides('blas') + + def install(self, spec, prefix): + pass diff --git a/var/spack/mock_packages/netlib-lapack/package.py b/var/spack/mock_packages/netlib-lapack/package.py new file mode 100644 index 0000000000..331844e544 --- /dev/null +++ b/var/spack/mock_packages/netlib-lapack/package.py @@ -0,0 +1,37 @@ +############################################################################## +# Copyright (c) 2013-2015, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + +class NetlibLapack(Package): + homepage = "http://www.netlib.org/lapack/" + url = "http://www.netlib.org/lapack/lapack-3.5.0.tgz" + + version('3.5.0', 'b1d3e3e425b2e44a06760ff173104bdf') + + provides('lapack') + depends_on('blas') + + def install(self, spec, prefix): + pass diff --git a/var/spack/mock_packages/openblas/package.py b/var/spack/mock_packages/openblas/package.py new file mode 100644 index 0000000000..c7771b92a3 --- /dev/null +++ b/var/spack/mock_packages/openblas/package.py @@ -0,0 +1,37 @@ +############################################################################## +# Copyright (c) 2013-2015, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + +class Openblas(Package): + """OpenBLAS: An optimized BLAS library""" + homepage = "http://www.openblas.net" + url = "http://github.com/xianyi/OpenBLAS/archive/v0.2.15.tar.gz" + + version('0.2.15', 'b1190f3d3471685f17cfd1ec1d252ac9') + + provides('blas') + + def install(self, spec, prefix): + pass diff --git a/var/spack/mock_packages/trivial_install_test_package/package.py b/var/spack/mock_packages/trivial_install_test_package/package.py index c4db9f5f07..fec5849e67 100644 --- a/var/spack/mock_packages/trivial_install_test_package/package.py +++ b/var/spack/mock_packages/trivial_install_test_package/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/mock_packages/zmpi/package.py b/var/spack/mock_packages/zmpi/package.py index 8c6ceda6d3..201fac2fbf 100644 --- a/var/spack/mock_packages/zmpi/package.py +++ b/var/spack/mock_packages/zmpi/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/packages/Mitos/package.py b/var/spack/packages/Mitos/package.py index e312da3ffc..ea131872dd 100644 --- a/var/spack/packages/Mitos/package.py +++ b/var/spack/packages/Mitos/package.py @@ -4,13 +4,20 @@ class Mitos(Package): """Mitos is a library and a tool for collecting sampled memory performance data to view with MemAxes""" - homepage = "https://github.com/scalability-llnl/Mitos" - url = "https://github.com/scalability-llnl/Mitos" + homepage = "https://github.com/llnl/Mitos" + url = "https://github.com/llnl/Mitos" - version('0.9.1', 'c6cb57f3cae54f5157affd97ef7ef79e', git='https://github.com/scalability-llnl/Mitos.git', tag='v0.9.1') + version('0.9.2', + git='https://github.com/llnl/Mitos.git', + commit='8cb143a2e8c00353ff531a781a9ca0992b0aaa3d') + + version('0.9.1', + git='https://github.com/llnl/Mitos.git', + tag='v0.9.1') depends_on('dyninst@8.2.1:') depends_on('hwloc') + depends_on('mpi') def install(self, spec, prefix): with working_dir('spack-build', create=True): diff --git a/var/spack/packages/SAMRAI/package.py b/var/spack/packages/SAMRAI/package.py index eef041f0d5..a17aea9c99 100644 --- a/var/spack/packages/SAMRAI/package.py +++ b/var/spack/packages/SAMRAI/package.py @@ -12,6 +12,7 @@ class Samrai(Package): list_url = homepage version('3.9.1', '232d04d0c995f5abf20d94350befd0b2') + version('3.8.0', 'c18fcffa706346bfa5828b36787ce5fe') version('3.7.3', '12d574eacadf8c9a70f1bb4cd1a69df6') version('3.7.2', 'f6a716f171c9fdbf3cb12f71fa6e2737') version('3.6.3-beta', 'ef0510bf2893042daedaca434e5ec6ce') @@ -32,13 +33,10 @@ class Samrai(Package): # TODO: currently hard-coded to use openmpi - be careful! def install(self, spec, prefix): - mpi = next(m for m in ('openmpi', 'mpich', 'mvapich') - if m in spec) - configure( "--prefix=%s" % prefix, - "--with-CXX=%s" % spec[mpi].prefix.bin + "/mpic++", - "--with-CC=%s" % spec[mpi].prefix.bin + "/mpicc", + "--with-CXX=%s" % spec['mpi'].prefix.bin + "/mpic++", + "--with-CC=%s" % spec['mpi'].prefix.bin + "/mpicc", "--with-hdf5=%s" % spec['hdf5'].prefix, "--with-boost=%s" % spec['boost'].prefix, "--with-zlib=%s" % spec['zlib'].prefix, diff --git a/var/spack/packages/adept-utils/package.py b/var/spack/packages/adept-utils/package.py index e4a2e1523f..fb59576c21 100644 --- a/var/spack/packages/adept-utils/package.py +++ b/var/spack/packages/adept-utils/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify @@ -27,8 +27,8 @@ from spack import * class AdeptUtils(Package): """Utility libraries for LLNL performance tools.""" - homepage = "https://github.com/scalability-llnl/adept-utils" - url = "https://github.com/scalability-llnl/adept-utils/archive/v1.0.tar.gz" + homepage = "https://github.com/llnl/adept-utils" + url = "https://github.com/llnl/adept-utils/archive/v1.0.tar.gz" version('1.0.1', '731a310717adcb004d9d195130efee7d') version('1.0', '5c6cd9badce56c945ac8551e34804397') diff --git a/var/spack/packages/apex/package.py b/var/spack/packages/apex/package.py index 6404d5208a..8769d97056 100644 --- a/var/spack/packages/apex/package.py +++ b/var/spack/packages/apex/package.py @@ -3,12 +3,9 @@ from spack.util.environment import * class Apex(Package): homepage = "http://github.com/khuck/xpress-apex" - #url = "http://github.com/khuck/xpress-apex/archive/v0.1-release-candidate.tar.gz" - url = "http://github.com/khuck/xpress-apex" + url = "http://github.com/khuck/xpress-apex/archive/v0.1.tar.gz" - #version('0.1', '6e039c224387348296739f6bf360d081') - #version('master', branch='master', git='https://github.com/khuck/xpress-apex.git') - version('2015-10-21', git='https://github.com/khuck/xpress-apex.git', commit='d2e66ddde689120472fc57fc546d8cd80aab745c') + version('0.1', '8b95f0c0313da1575960d3ad69f18e75') depends_on("binutils+libiberty") depends_on("boost@1.54:") diff --git a/var/spack/packages/automaded/package.py b/var/spack/packages/automaded/package.py index 9fbd93e3b3..e0bc7efb8b 100644 --- a/var/spack/packages/automaded/package.py +++ b/var/spack/packages/automaded/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify @@ -36,8 +36,8 @@ class Automaded(Package): finding the process (or group of processes) that caused the hang. """ - homepage = "https://github.com/scalability-llnl/AutomaDeD" - url = "https://github.com/scalability-llnl/AutomaDeD/archive/v1.0.tar.gz" + homepage = "https://github.com/llnl/AutomaDeD" + url = "https://github.com/llnl/AutomaDeD/archive/v1.0.tar.gz" version('1.0', '16a3d4def2c4c77d0bc4b21de8b3ab03') diff --git a/var/spack/packages/binutils/package.py b/var/spack/packages/binutils/package.py index cac0a0407f..be5a9b714b 100644 --- a/var/spack/packages/binutils/package.py +++ b/var/spack/packages/binutils/package.py @@ -20,7 +20,8 @@ class Binutils(Package): '--enable-multilib', '--enable-shared', '--enable-64-bit-bfd', - '--enable-targets=all'] + '--enable-targets=all', + '--with-sysroot=/'] if '+libiberty' in spec: configure_args.append('--enable-install-libiberty') diff --git a/var/spack/packages/boost/package.py b/var/spack/packages/boost/package.py index 35824d53a2..81dadbbf61 100644 --- a/var/spack/packages/boost/package.py +++ b/var/spack/packages/boost/package.py @@ -43,7 +43,16 @@ class Boost(Package): version('1.34.1', '2d938467e8a448a2c9763e0a9f8ca7e5') version('1.34.0', 'ed5b9291ffad776f8757a916e1726ad0') + variant('debug', default=False, description='Switch to the debug version of Boost') + variant('python', default=False, description='Activate the component Boost.Python') + variant('mpi', default=False, description='Activate the component Boost.MPI') + variant('compression', default=True, description='Activate the compression Boost.iostreams') + depends_on('mpi', when='+mpi') + depends_on('python', when='+python') + depends_on('zlib', when='+compression') + depends_on('bzip2', when='+compression') + def url_for_version(self, version): """Handle Boost's weird URLs, which write the version two different ways.""" parts = [str(p) for p in Version(version)] @@ -52,15 +61,78 @@ class Boost(Package): return "http://downloads.sourceforge.net/project/boost/boost/%s/boost_%s.tar.bz2" % ( dots, underscores) + def determine_toolset(self): + toolsets = {'gcc': 'gcc', + 'icpc': 'intel', + 'clang++': 'clang'} + + for cc, toolset in toolsets.iteritems(): + if(cc in self.compiler.cxx_names): + return toolset + + # fallback to gcc if no toolset found + return 'gcc' + + def determine_bootstrap_options(self, spec, options): + options.append('--with-toolset=%s' % self.determine_toolset()) + + without_libs = [] + if '~mpi' in spec: + without_libs.append('mpi') + if '~python' in spec: + without_libs.append('python') + else: + options.append('--with-python=%s' % (spec['python'].prefix.bin + '/python')) + + if without_libs: + options.append('--without-libraries=%s' % ','.join(without_libs)) + + with open('user-config.jam', 'w') as f: + if '+mpi' in spec: + f.write('using mpi : %s ;\n' % (spec['mpi'].prefix.bin + '/mpicxx')) + if '+python' in spec: + f.write('using python : %s : %s ;\n' % (spec['python'].version, + (spec['python'].prefix.bin + '/python'))) + + def determine_b2_options(self, spec, options): + if '+debug' in spec: + options.append('variant=debug') + else: + options.append('variant=release') + + if '~compression' in spec: + options.extend(['-s NO_BZIP2=1', + '-s NO_ZLIB=1', + ]) + + if '+compression' in spec: + options.extend(['-s BZIP2_INCLUDE=%s' % spec['bzip2'].prefix.include, + '-s BZIP2_LIBPATH=%s' % spec['bzip2'].prefix.lib, + '-s ZLIB_INCLUDE=%s' % spec['zlib'].prefix.include, + '-s ZLIB_LIBPATH=%s' % spec['zlib'].prefix.lib]) + + options.extend(['toolset=%s' % self.determine_toolset(), + 'link=static,shared', + '--layout=tagged']) def install(self, spec, prefix): + # to make him find the user-config.jam + env['BOOST_BUILD_PATH'] = './' + bootstrap = Executable('./bootstrap.sh') - bootstrap() + + bootstrap_options = ['--prefix=%s' % prefix] + self.determine_bootstrap_options(spec, bootstrap_options) + + bootstrap(*bootstrap_options) # b2 used to be called bjam, before 1.47 (sigh) b2name = './b2' if spec.satisfies('@1.47:') else './bjam' b2 = Executable(b2name) - b2('install', - '-j %s' % make_jobs, - '--prefix=%s' % prefix) + b2_options = ['-j %s' % make_jobs] + + self.determine_b2_options(spec, b2_options) + + b2('install', 'threading=single', *b2_options) + b2('install', 'threading=multi', *b2_options) diff --git a/var/spack/packages/bzip2/package.py b/var/spack/packages/bzip2/package.py index d88336664d..638ba1fa4d 100644 --- a/var/spack/packages/bzip2/package.py +++ b/var/spack/packages/bzip2/package.py @@ -1,36 +1,60 @@ from spack import * -from glob import glob class Bzip2(Package): """bzip2 is a freely available, patent free high-quality data compressor. It typically compresses files to within 10% to 15% of the best available techniques (the PPM family of statistical compressors), whilst being around twice as fast at compression - and six times faster at decompression.""" + and six times faster at decompression. + + """ homepage = "http://www.bzip.org" url = "http://www.bzip.org/1.0.6/bzip2-1.0.6.tar.gz" version('1.0.6', '00b516f4704d4a7cb50a1d97e6e8e15b') - def install(self, spec, prefix): - # No configure system -- have to filter the makefile for this package. - filter_file(r'CC=gcc', 'CC=cc', 'Makefile', string=True) + def patch(self): + mf = FileFilter('Makefile-libbz2_so') + mf.filter(r'^CC=gcc', 'CC=cc') + + # Below stuff patches the link line to use RPATHs on Mac OS X. + if 'darwin' in self.spec.architecture: + v = self.spec.version + v1, v2, v3 = (v.up_to(i) for i in (1,2,3)) + + mf.filter('$(CC) -shared -Wl,-soname -Wl,libbz2.so.{0} -o libbz2.so.{1} $(OBJS)'.format(v2, v3), + '$(CC) -dynamiclib -Wl,-install_name -Wl,@rpath/libbz2.{0}.dylib -current_version {1} -compatibility_version {2} -o libbz2.{3}.dylib $(OBJS)'.format(v1, v2, v3, v3), string=True) + + mf.filter('$(CC) $(CFLAGS) -o bzip2-shared bzip2.c libbz2.so.{0}'.format(v3), + '$(CC) $(CFLAGS) -o bzip2-shared bzip2.c libbz2.{0}.dylib'.format(v3), string=True) + mf.filter('rm -f libbz2.so.{0}'.format(v2), + 'rm -f libbz2.{0}.dylib'.format(v2), string=True) + mf.filter('ln -s libbz2.so.{0} libbz2.so.{1}'.format(v3, v2), + 'ln -s libbz2.{0}.dylib libbz2.{1}.dylib'.format(v3, v2), string=True) + + + def install(self, spec, prefix): make('-f', 'Makefile-libbz2_so') make('clean') make("install", "PREFIX=%s" % prefix) - bzip2_exe = join_path(prefix.bin, 'bzip2') - install('bzip2-shared', bzip2_exe) - for i, libfile in enumerate(glob('libbz2.so*')): - install(libfile, prefix.lib) - if i == 0: - symlink(join_path(prefix.lib, libfile), join_path(prefix.lib, 'libbz2.so')) + install('bzip2-shared', join_path(prefix.bin, 'bzip2')) + + v1, v2, v3 = (self.spec.version.up_to(i) for i in (1,2,3)) + if 'darwin' in self.spec.architecture: + lib = 'libbz2.dylib' + lib1, lib2, lib3 = ('libbz2.{0}.dylib'.format(v) for v in (v1, v2, v3)) + else: + lib = 'libbz2.so' + lib1, lib2, lib3 = ('libbz2.so.{0}'.format(v) for v in (v1, v2, v3)) - bunzip2 = join_path(prefix.bin, 'bunzip2') - remove(bunzip2) - symlink(bzip2_exe, bunzip2) + install(lib3, join_path(prefix.lib, lib3)) + with working_dir(prefix.lib): + for l in (lib, lib1, lib2): + symlink(lib3, l) - bzcat = join_path(prefix.bin, 'bzcat') - remove(bzcat) - symlink(bzip2_exe, bzcat) + with working_dir(prefix.bin): + force_remove('bunzip2', 'bzcat') + symlink('bzip2', 'bunzip2') + symlink('bzip2', 'bzcat') diff --git a/var/spack/packages/callpath/package.py b/var/spack/packages/callpath/package.py index f8a1eab9f7..3d2d96249e 100644 --- a/var/spack/packages/callpath/package.py +++ b/var/spack/packages/callpath/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify @@ -28,8 +28,8 @@ class Callpath(Package): """Library for representing callpaths consistently in distributed-memory performance tools.""" - homepage = "https://github.com/scalability-llnl/callpath" - url = "https://github.com/scalability-llnl/callpath/archive/v1.0.1.tar.gz" + homepage = "https://github.com/llnl/callpath" + url = "https://github.com/llnl/callpath/archive/v1.0.1.tar.gz" version('1.0.2', 'b1994d5ee7c7db9d27586fc2dcf8f373') version('1.0.1', '0047983d2a52c5c335f8ba7f5bab2325') diff --git a/var/spack/packages/cityhash/package.py b/var/spack/packages/cityhash/package.py new file mode 100644 index 0000000000..1643cc3b42 --- /dev/null +++ b/var/spack/packages/cityhash/package.py @@ -0,0 +1,16 @@ +from spack import * +from spack.util.environment import * + +class Cityhash(Package): + homepage = "https://github.com/google/cityhash" + url = "https://github.com/google/cityhash" + + version('2013-07-31', git='https://github.com/google/cityhash.git', commit='8af9b8c2b889d80c22d6bc26ba0df1afb79a30db') + version('master', branch='master', git='https://github.com/google/cityhash.git') + + def install(self, spec, prefix): + configure('--enable-sse4.2', '--prefix=%s' % prefix) + + make() + make("install") + diff --git a/var/spack/packages/clang/package.py b/var/spack/packages/clang/package.py index 4f10385dbd..20a5ac2c94 100644 --- a/var/spack/packages/clang/package.py +++ b/var/spack/packages/clang/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify @@ -28,11 +28,16 @@ class Clang(Package): """The goal of the Clang project is to create a new C, C++, Objective C and Objective C++ front-end for the LLVM compiler. """ - homepage = "http://clang.llvm.org" - list_url = "http://llvm.org/releases/download.html" + homepage = 'http://clang.llvm.org' + url = 'http://llvm.org/releases/3.7.0/cfe-3.7.0.src.tar.xz' - depends_on("llvm") - version('3.4.2', '87945973b7c73038871c5f849a818588', url='http://llvm.org/releases/3.4.2/cfe-3.4.2.src.tar.xz') + depends_on('llvm@3.7.0', when='@3.7.0') + depends_on('llvm@3.6.2', when='@3.6.2') + depends_on('llvm@3.5.1', when='@3.5.1') + + version('3.7.0', '8f9d27335e7331cf0a4711e952f21f01', url='http://llvm.org/releases/3.7.0/cfe-3.7.0.src.tar.xz') + version('3.6.2', 'ff862793682f714bb7862325b9c06e20', url='http://llvm.org/releases/3.6.2/cfe-3.6.2.src.tar.xz') + version('3.5.1', '93f9532f8f7e6f1d8e5c1116907051cb', url='http://llvm.org/releases/3.5.1/cfe-3.5.1.src.tar.xz') def install(self, spec, prefix): env['CXXFLAGS'] = self.compiler.cxx11_flag diff --git a/var/spack/packages/cleverleaf/package.py b/var/spack/packages/cleverleaf/package.py new file mode 100644 index 0000000000..4e7e6a855a --- /dev/null +++ b/var/spack/packages/cleverleaf/package.py @@ -0,0 +1,23 @@ +from spack import * + +class Cleverleaf(Package): + """ + CleverLeaf is a hydrodynamics mini-app that extends CloverLeaf with Adaptive + Mesh Refinement using the SAMRAI toolkit from Lawrence Livermore National + Laboratory. The primary goal of CleverLeaf is to evaluate the application of + AMR to the Lagrangian-Eulerian hydrodynamics scheme used by CloverLeaf. + """ + + homepage = "http://uk-mac.github.io/CleverLeaf/" + url = "https://github.com/UK-MAC/CleverLeaf/tarball/master" + + version('develop', git='https://github.com/UK-MAC/CleverLeaf_ref.git', branch='develop') + + depends_on("SAMRAI@3.8.0:") + depends_on("hdf5") + depends_on("boost") + + def install(self, spec, prefix): + cmake(*std_cmake_args) + make() + make("install") diff --git a/var/spack/packages/cmake/package.py b/var/spack/packages/cmake/package.py index 9efa370c8b..c24a80748c 100644 --- a/var/spack/packages/cmake/package.py +++ b/var/spack/packages/cmake/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify @@ -34,6 +34,9 @@ class Cmake(Package): version('3.0.2', 'db4c687a31444a929d2fdc36c4dfb95f', url = 'http://www.cmake.org/files/v3.0/cmake-3.0.2.tar.gz') + + version('3.4.0', 'cd3034e0a44256a0917e254167217fc8', + url = 'https://cmake.org/files/v3.4/cmake-3.4.0.tar.gz') # version('3.0.1', 'e2e05d84cb44a42f1371d9995631dcf5') # version('3.0.0', '21a1c85e1a3b803c4b48e7ff915a863e') diff --git a/var/spack/packages/cram/package.py b/var/spack/packages/cram/package.py index 4b8ec56f25..b19422b8a8 100644 --- a/var/spack/packages/cram/package.py +++ b/var/spack/packages/cram/package.py @@ -2,11 +2,12 @@ from spack import * class Cram(Package): """Cram runs many small MPI jobs inside one large MPI job.""" - homepage = "https://github.com/scalability-llnl/cram" - url = "http://github.com/scalability-llnl/cram/archive/v1.0.1.tar.gz" + homepage = "https://github.com/llnl/cram" + url = "http://github.com/llnl/cram/archive/v1.0.1.tar.gz" version('1.0.1', 'c73711e945cf5dc603e44395f6647f5e') + extends('python') depends_on("mpi") def install(self, spec, prefix): diff --git a/var/spack/packages/czmq/package.py b/var/spack/packages/czmq/package.py index a2f1947554..0665332179 100644 --- a/var/spack/packages/czmq/package.py +++ b/var/spack/packages/czmq/package.py @@ -1,4 +1,5 @@ from spack import * +import os class Czmq(Package): """ A C interface to the ZMQ library """ @@ -7,11 +8,24 @@ class Czmq(Package): version('3.0.2', '23e9885f7ee3ce88d99d0425f52e9be1', url='https://github.com/zeromq/czmq/archive/v3.0.2.tar.gz') + depends_on('libtool') + depends_on('automake') + depends_on('autoconf') + depends_on('pkg-config') depends_on('zeromq') def install(self, spec, prefix): bash = which("bash") - bash("./autogen.sh") + # Work around autogen.sh oddities + # bash("./autogen.sh") + mkdirp("config") + autoreconf = which("autoreconf") + autoreconf("--install", "--verbose", "--force", + "-I", "config", + "-I", os.path.join(spec['pkg-config'].prefix, "share", "aclocal"), + "-I", os.path.join(spec['automake'].prefix, "share", "aclocal"), + "-I", os.path.join(spec['libtool'].prefix, "share", "aclocal"), + ) configure("--prefix=%s" % prefix) make() diff --git a/var/spack/packages/damselfly/package.py b/var/spack/packages/damselfly/package.py new file mode 100644 index 0000000000..96666d1abe --- /dev/null +++ b/var/spack/packages/damselfly/package.py @@ -0,0 +1,14 @@ +from spack import * + +class Damselfly(Package): + """Damselfly is a model-based parallel network simulator.""" + homepage = "https://github.com/llnl/damselfly" + url = "https://github.com/llnl/damselfly" + + version('1.0', '05cf7e2d8ece4408c0f2abb7ab63fd74c0d62895', git='https://github.com/llnl/damselfly.git', tag='v1.0') + + def install(self, spec, prefix): + with working_dir('spack-build', create=True): + cmake('-DCMAKE_BUILD_TYPE=release', '..', *std_cmake_args) + make() + make('install') diff --git a/var/spack/packages/dbus/package.py b/var/spack/packages/dbus/package.py index f7f394498c..294b0de54e 100644 --- a/var/spack/packages/dbus/package.py +++ b/var/spack/packages/dbus/package.py @@ -20,7 +20,10 @@ class Dbus(Package): version('1.8.2', 'd6f709bbec0a022a1847c7caec9d6068') def install(self, spec, prefix): - configure("--prefix=%s" % prefix) + configure( + "--prefix=%s" % prefix, + "--disable-systemd", + "--disable-launchd") make() make("install") diff --git a/var/spack/packages/dyninst/package.py b/var/spack/packages/dyninst/package.py index 41ec57dd2f..7f8598e0e5 100644 --- a/var/spack/packages/dyninst/package.py +++ b/var/spack/packages/dyninst/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/packages/flux/package.py b/var/spack/packages/flux/package.py index c128f46be8..dc4e0594c5 100644 --- a/var/spack/packages/flux/package.py +++ b/var/spack/packages/flux/package.py @@ -12,20 +12,23 @@ class Flux(Package): # Also needs autotools, but should use the system version if available depends_on("zeromq@4.0.4:") depends_on("czmq@2.2:") + depends_on("hwloc") depends_on("lua@5.1:5.1.99") depends_on("munge") depends_on("libjson-c") depends_on("libxslt") + depends_on("python") + depends_on("py-cffi") + # TODO: This provides a catalog, hacked with environment below for now depends_on("docbook-xml") depends_on("asciidoc") - depends_on("python") - depends_on("py-cffi") def install(self, spec, prefix): # Bootstrap with autotools bash = which('bash') bash('./autogen.sh') + bash('./autogen.sh') #yes, twice, intentionally # Fix asciidoc dependency on xml style sheets and whatnot os.environ['XML_CATALOG_FILES'] = os.path.join(spec['docbook-xml'].prefix, diff --git a/var/spack/packages/gcc/package.py b/var/spack/packages/gcc/package.py index 2fc6794b70..7ec160d595 100644 --- a/var/spack/packages/gcc/package.py +++ b/var/spack/packages/gcc/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify @@ -36,21 +36,26 @@ class Gcc(Package): list_url = 'http://open-source-box.org/gcc/' list_depth = 2 + DEPENDS_ON_ISL_PREDICATE = '@5.0:' + + version('5.3.0', 'c9616fd448f980259c31de613e575719') + version('5.2.0', 'a51bcfeb3da7dd4c623e27207ed43467') + version('4.9.3', '6f831b4d251872736e8e9cc09746f327') version('4.9.2', '4df8ee253b7f3863ad0b86359cd39c43') version('4.9.1', 'fddf71348546af523353bd43d34919c1') + version('4.8.5', '80d2c2982a3392bb0b89673ff136e223') version('4.8.4', '5a84a30839b2aca22a2d723de2a626ec') version('4.7.4', '4c696da46297de6ae77a82797d2abe28') version('4.6.4', 'b407a3d1480c11667f293bfb1f17d1a4') version('4.5.4', '27e459c2566b8209ab064570e1b378f7') - + depends_on("mpfr") depends_on("gmp") depends_on("mpc") # when @4.5: - depends_on("libelf") depends_on("binutils~libiberty") # Save these until we can do optional deps. - #depends_on("isl") + depends_on("isl", when=DEPENDS_ON_ISL_PREDICATE) #depends_on("ppl") #depends_on("cloog") @@ -62,26 +67,39 @@ class Gcc(Package): if spec.satisfies("@4.7.1:"): enabled_languages.add('go') - # Rest of install is straightforward. - configure("--prefix=%s" % prefix, - "--libdir=%s/lib64" % prefix, - "--disable-multilib", - "--enable-languages=" + ','.join(enabled_languages), - "--with-mpc=%s" % spec['mpc'].prefix, - "--with-mpfr=%s" % spec['mpfr'].prefix, - "--with-gmp=%s" % spec['gmp'].prefix, - "--with-libelf=%s" % spec['libelf'].prefix, - "--with-stage1-ldflags=%s" % self.rpath_args, - "--with-boot-ldflags=%s" % self.rpath_args, - "--enable-lto", - "--with-gnu-ld", - "--with-ld=%s/bin/ld" % spec['binutils'].prefix, - "--with-gnu-as", - "--with-as=%s/bin/as" % spec['binutils'].prefix, - "--with-quad") - make() - make("install") + # Generic options to compile GCC + options = ["--prefix=%s" % prefix, + "--libdir=%s/lib64" % prefix, + "--disable-multilib", + "--enable-languages=" + ','.join(enabled_languages), + "--with-mpc=%s" % spec['mpc'].prefix, + "--with-mpfr=%s" % spec['mpfr'].prefix, + "--with-gmp=%s" % spec['gmp'].prefix, + "--enable-lto", + "--with-gnu-ld", + "--with-gnu-as", + "--with-quad"] + # Binutils + static_bootstrap_flags = "-static-libstdc++ -static-libgcc" + binutils_options = ["--with-sysroot=/", + "--with-stage1-ldflags=%s %s" % (self.rpath_args, static_bootstrap_flags), + "--with-boot-ldflags=%s %s" % (self.rpath_args, static_bootstrap_flags), + "--with-ld=%s/bin/ld" % spec['binutils'].prefix, + "--with-as=%s/bin/as" % spec['binutils'].prefix] + options.extend(binutils_options) + # Isl + if spec.satisfies(Gcc.DEPENDS_ON_ISL_PREDICATE): + isl_options = ["--with-isl=%s" % spec['isl'].prefix] + options.extend(isl_options) + build_dir = join_path(self.stage.path, 'spack-build') + configure = Executable( join_path(self.stage.source_path, 'configure') ) + with working_dir(build_dir, create=True): + # Rest of install is straightforward. + configure(*options) + make() + make("install") + self.write_rpath_specs() @@ -100,13 +118,11 @@ class Gcc(Package): return gcc = Executable(join_path(self.prefix.bin, 'gcc')) - lines = gcc('-dumpspecs', return_output=True).split("\n") - for i, line in enumerate(lines): - if line.startswith("*link:"): - specs_file = join_path(self.spec_dir, 'specs') - with closing(open(specs_file, 'w')) as out: - out.write(lines[i] + "\n") - out.write("-rpath %s/lib:%s/lib64 \\\n" - % (self.prefix, self.prefix)) - out.write(lines[i+1] + "\n") - set_install_permissions(specs_file) + lines = gcc('-dumpspecs', return_output=True).strip().split("\n") + specs_file = join_path(self.spec_dir, 'specs') + with closing(open(specs_file, 'w')) as out: + for line in lines: + out.write(line + "\n") + if line.startswith("*link:"): + out.write("-rpath %s/lib:%s/lib64 \\\n"% (self.prefix, self.prefix)) + set_install_permissions(specs_file) diff --git a/var/spack/packages/gdb/package.py b/var/spack/packages/gdb/package.py new file mode 100644 index 0000000000..fd567f346b --- /dev/null +++ b/var/spack/packages/gdb/package.py @@ -0,0 +1,48 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## + +from spack import * + + +class Gdb(Package): + """ + GDB, the GNU Project debugger, allows you to see what is going on `inside' another program while it executes + -- or what another program was doing at the moment it crashed. + """ + homepage = "https://www.gnu.org/software/gdb" + url = "http://ftp.gnu.org/gnu/gdb/gdb-7.10.tar.xz" + + version('7.10.1', '39e654460c9cdd80200a29ac020cfe11') + version('7.10', '2a35bac41fa8e10bf04f3a0dd7f7f363') + version('7.9.1', '35374c77a70884eb430c97061053a36e') + version('7.9', 'e6279f26559d839f0b4218a482bcb43e') + version('7.8.2', 'a80cf252ed2e775d4e4533341bbf2459') + + depends_on('texinfo') + + def install(self, spec, prefix): + configure('--prefix=%s' % prefix) + make() + make("install") diff --git a/var/spack/packages/gmp/package.py b/var/spack/packages/gmp/package.py index d6af821b34..fe13de3b95 100644 --- a/var/spack/packages/gmp/package.py +++ b/var/spack/packages/gmp/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify @@ -31,6 +31,7 @@ class Gmp(Package): homepage = "https://gmplib.org" url = "https://gmplib.org/download/gmp/gmp-6.0.0a.tar.bz2" + version('6.1.0' , '86ee6e54ebfc4a90b643a65e402c4048') version('6.0.0a', 'b7ff2d88cae7f8085bd5006096eed470') version('6.0.0' , '6ef5869ae735db9995619135bd856b84') diff --git a/var/spack/packages/gperftools/package.py b/var/spack/packages/gperftools/package.py index 8900462324..0ba44c9329 100644 --- a/var/spack/packages/gperftools/package.py +++ b/var/spack/packages/gperftools/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/packages/hdf5/package.py b/var/spack/packages/hdf5/package.py index 15e0ef9338..48997425cd 100644 --- a/var/spack/packages/hdf5/package.py +++ b/var/spack/packages/hdf5/package.py @@ -11,6 +11,7 @@ class Hdf5(Package): list_url = "http://www.hdfgroup.org/ftp/HDF5/releases" list_depth = 3 + version('1.8.16', 'b8ed9a36ae142317f88b0c7ef4b9c618') version('1.8.15', '03cccb5b33dbe975fdcd8ae9dc021f24') version('1.8.13', 'c03426e9e77d7766944654280b467289') @@ -25,8 +26,8 @@ class Hdf5(Package): "--with-zlib=%s" % spec['zlib'].prefix, "--enable-parallel", "--enable-shared", - "CC=%s" % spec['mpich'].prefix.bin + "/mpicc", - "CXX=%s" % spec['mpich'].prefix.bin + "/mpic++") + "CC=%s" % spec['mpi'].prefix.bin + "/mpicc", + "CXX=%s" % spec['mpi'].prefix.bin + "/mpic++") make() make("install") diff --git a/var/spack/packages/hwloc/package.py b/var/spack/packages/hwloc/package.py index 31a31f376a..452a7d7ce3 100644 --- a/var/spack/packages/hwloc/package.py +++ b/var/spack/packages/hwloc/package.py @@ -15,8 +15,12 @@ class Hwloc(Package): homepage = "http://www.open-mpi.org/projects/hwloc/" url = "http://www.open-mpi.org/software/hwloc/v1.9/downloads/hwloc-1.9.tar.gz" + version('1.11.1', '002742efd3a8431f98d6315365a2b543', + url='http://www.open-mpi.org/software/hwloc/v1.11/downloads/hwloc-1.11.1.tar.bz2') version('1.9', '1f9f9155682fe8946a97c08896109508') + depends_on('libpciaccess') + def install(self, spec, prefix): configure("--prefix=%s" % prefix) diff --git a/var/spack/packages/hypre/package.py b/var/spack/packages/hypre/package.py index 198b3f00dc..0f7f14dd89 100644 --- a/var/spack/packages/hypre/package.py +++ b/var/spack/packages/hypre/package.py @@ -5,8 +5,8 @@ class Hypre(Package): features parallel multigrid methods for both structured and unstructured grid problems.""" - homepage = "https://computation.llnl.gov/project/linear_solvers/software.php" - url = "https://computation.llnl.gov/project/linear_solvers/download/hypre-2.10.0b.tar.gz" + homepage = "http://computation.llnl.gov/project/linear_solvers/software.php" + url = "http://computation.llnl.gov/project/linear_solvers/download/hypre-2.10.0b.tar.gz" version('2.10.0b', '768be38793a35bb5d055905b271f5b8e') diff --git a/var/spack/packages/jdk/package.py b/var/spack/packages/jdk/package.py index 8f8076dd14..f8f5fc21bd 100644 --- a/var/spack/packages/jdk/package.py +++ b/var/spack/packages/jdk/package.py @@ -14,8 +14,8 @@ class Jdk(Package): in the form of a binary product aimed at Java developers.""" homepage = "http://www.oracle.com/technetwork/java/javase/downloads/index.html" - version('8u25-linux-x64', 'e145c03a7edc845215092786bcfba77e', - url="http://download.oracle.com/otn-pub/java/jdk/8u25-b17/jdk-8u25-linux-x64.tar.gz") + version('8u66-linux-x64', '88f31f3d642c3287134297b8c10e61bf', + url="http://download.oracle.com/otn-pub/java/jdk/8u66-b17/jdk-8u66-linux-x64.tar.gz") # Oracle requires that you accept their License Agreement in order # to access the Java packages in download.oracle.com. In order to diff --git a/var/spack/packages/jemalloc/package.py b/var/spack/packages/jemalloc/package.py new file mode 100644 index 0000000000..8cec9ea75b --- /dev/null +++ b/var/spack/packages/jemalloc/package.py @@ -0,0 +1,24 @@ +from spack import * + +class Jemalloc(Package): + """jemalloc is a general purpose malloc(3) implementation that emphasizes fragmentation avoidance and scalable concurrency support.""" + homepage = "http://www.canonware.com/jemalloc/" + url = "https://github.com/jemalloc/jemalloc/releases/download/4.0.4/jemalloc-4.0.4.tar.bz2" + + version('4.0.4', '687c5cc53b9a7ab711ccd680351ff988') + + variant('stats', default=False, description='Enable heap statistics') + variant('prof', default=False, description='Enable heap profiling') + + def install(self, spec, prefix): + configure_args = ['--prefix=%s' % prefix,] + + if '+stats' in spec: + configure_args.append('--enable-stats') + if '+prof' in spec: + configure_args.append('--enable-prof') + + configure(*configure_args) + + make() + make("install") diff --git a/var/spack/packages/judy/package.py b/var/spack/packages/judy/package.py new file mode 100644 index 0000000000..b8d8701383 --- /dev/null +++ b/var/spack/packages/judy/package.py @@ -0,0 +1,15 @@ +from spack import * + +class Judy(Package): + """A general-purpose dynamic array, associative array and hash-trie - Judy""" + homepage = "http://judy.sourceforge.net/" + url = "http://downloads.sourceforge.net/project/judy/judy/Judy-1.0.5/Judy-1.0.5.tar.gz" + + version('1.0.5', '115a0d26302676e962ae2f70ec484a54') + parallel = False + + def install(self, spec, prefix): + configure('--prefix=%s' % prefix) + + make() + make("install") diff --git a/var/spack/packages/launchmon/package.py b/var/spack/packages/launchmon/package.py index 6fbe6a68d0..f97384a249 100644 --- a/var/spack/packages/launchmon/package.py +++ b/var/spack/packages/launchmon/package.py @@ -6,7 +6,7 @@ # Written by Matthew LeGendre, legendre1@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify @@ -31,7 +31,7 @@ class Launchmon(Package): url = "http://downloads.sourceforge.net/project/launchmon/launchmon/1.0.1%20release/launchmon-1.0.1.tar.gz" version('1.0.1', '2f12465803409fd07f91174a4389eb2b') - version('1.0.1-2', git='https://github.com/scalability-llnl/launchmon.git', commit='ff7e22424b8f375318951eb1c9282fcbbfa8aadf') + version('1.0.1-2', git='https://github.com/llnl/launchmon.git', commit='ff7e22424b8f375318951eb1c9282fcbbfa8aadf') depends_on('autoconf') depends_on('automake') diff --git a/var/spack/packages/libNBC/package.py b/var/spack/packages/libNBC/package.py index 6d08f3219c..550568e97d 100644 --- a/var/spack/packages/libNBC/package.py +++ b/var/spack/packages/libNBC/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/packages/libdwarf/package.py b/var/spack/packages/libdwarf/package.py index 099a974e93..addb557519 100644 --- a/var/spack/packages/libdwarf/package.py +++ b/var/spack/packages/libdwarf/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/packages/libelf/package.py b/var/spack/packages/libelf/package.py index 9338b8f393..29bc21b65c 100644 --- a/var/spack/packages/libelf/package.py +++ b/var/spack/packages/libelf/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/packages/libmonitor/package.py b/var/spack/packages/libmonitor/package.py index 3b95b86ddf..c75e1a7947 100644 --- a/var/spack/packages/libmonitor/package.py +++ b/var/spack/packages/libmonitor/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/packages/libunwind/package.py b/var/spack/packages/libunwind/package.py index 239fcbcfd5..6f162f7b08 100644 --- a/var/spack/packages/libunwind/package.py +++ b/var/spack/packages/libunwind/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/packages/libxml2/package.py b/var/spack/packages/libxml2/package.py index 3a0af6b368..5d84fad22d 100644 --- a/var/spack/packages/libxml2/package.py +++ b/var/spack/packages/libxml2/package.py @@ -9,12 +9,20 @@ class Libxml2(Package): version('2.9.2', '9e6a9aca9d155737868b3dc5fd82f788') - extends('python') + variant('python', default=False, description='Enable Python support') + + extends('python', when='+python') depends_on('zlib') depends_on('xz') def install(self, spec, prefix): - configure("--prefix=%s" % prefix) + if '+python' in spec: + python_arg = "--with-python=%s" % spec['python'].prefix + else: + python_arg = "--without-python" + + configure("--prefix=%s" % prefix, + python_arg) make() make("install") diff --git a/var/spack/packages/llvm-lld/package.py b/var/spack/packages/llvm-lld/package.py index f229211396..cb91aa22a5 100644 --- a/var/spack/packages/llvm-lld/package.py +++ b/var/spack/packages/llvm-lld/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/packages/llvm/package.py b/var/spack/packages/llvm/package.py index 9d2be690bb..b3ca488809 100644 --- a/var/spack/packages/llvm/package.py +++ b/var/spack/packages/llvm/package.py @@ -6,7 +6,7 @@ # Written by David Beckingsale, david@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify @@ -24,6 +24,7 @@ ############################################################################## from spack import * + class Llvm(Package): """The LLVM Project is a collection of modular and reusable compiler and toolchain technologies. Despite its name, LLVM has little to do with @@ -31,14 +32,15 @@ class Llvm(Package): that can be used to build them. The name "LLVM" itself is not an acronym; it is the full name of the project. """ - homepage = "http://llvm.org/" - list_url = "http://llvm.org/releases/download.html" + homepage = 'http://llvm.org/' + url = 'http://llvm.org/releases/3.7.0/llvm-3.7.0.src.tar.xz' + version('3.7.0', 'b98b9495e5655a672d6cb83e1a180f8e', url='http://llvm.org/releases/3.7.0/llvm-3.7.0.src.tar.xz') + version('3.6.2', '0c1ee3597d75280dee603bae9cbf5cc2', url='http://llvm.org/releases/3.6.2/llvm-3.6.2.src.tar.xz') version('3.5.1', '2d3d8004f38852aa679e5945b8ce0b14', url='http://llvm.org/releases/3.5.1/llvm-3.5.1.src.tar.xz') - version('3.4.2', 'a20669f75967440de949ac3b1bad439c', url='http://llvm.org/releases/3.4.2/llvm-3.4.2.src.tar.gz') - version('3.0', 'a8e5f5f1c1adebae7b4a654c376a6005', url='http://llvm.org/releases/3.0/llvm-3.0.tar.gz') - version('2.9', '793138412d2af2c7c7f54615f8943771', url='http://llvm.org/releases/2.9/llvm-2.9.tgz') - version('2.8', '220d361b4d17051ff4bb21c64abe05ba', url='http://llvm.org/releases/2.8/llvm-2.8.tgz') + version('3.0', 'a8e5f5f1c1adebae7b4a654c376a6005', url='http://llvm.org/releases/3.0/llvm-3.0.tar.gz') # currently required by mesa package + + depends_on('python@2.7:') def install(self, spec, prefix): env['CXXFLAGS'] = self.compiler.cxx11_flag @@ -46,9 +48,7 @@ class Llvm(Package): with working_dir('spack-build', create=True): cmake('..', '-DLLVM_REQUIRES_RTTI=1', - '-DPYTHON_EXECUTABLE=/usr/bin/python', - '-DPYTHON_INCLUDE_DIR=/usr/include/python2.6', - '-DPYTHON_LIBRARY=/usr/lib64/libpython2.6.so', + '-DPYTHON_EXECUTABLE=%s/bin/python' % spec['python'].prefix, *std_cmake_args) make() make("install") diff --git a/var/spack/packages/lua/package.py b/var/spack/packages/lua/package.py index 57c443cc2d..6d8f7806d9 100644 --- a/var/spack/packages/lua/package.py +++ b/var/spack/packages/lua/package.py @@ -21,6 +21,8 @@ class Lua(Package): def install(self, spec, prefix): make('INSTALL_TOP=%s' % prefix, - 'MYLDFLAGS=-L%s/lib' % spec['ncurses'].prefix, - 'linux', + 'MYLDFLAGS="-L%s/lib -Wl,-rpath,%s"' % (spec['ncurses'].prefix,spec['ncurses'].prefix), + 'linux') + make('INSTALL_TOP=%s' % prefix, + 'MYLDFLAGS="-L%s/lib -Wl,-rpath,%s"' % (spec['ncurses'].prefix,spec['ncurses'].prefix), 'install') diff --git a/var/spack/packages/memaxes/package.py b/var/spack/packages/memaxes/package.py index 76d5d3f831..4b1da558a2 100644 --- a/var/spack/packages/memaxes/package.py +++ b/var/spack/packages/memaxes/package.py @@ -3,10 +3,10 @@ from spack import * class Memaxes(Package): """MemAxes is a visualizer for sampled memory trace data.""" - homepage = "https://github.com/scalability-llnl/MemAxes" + homepage = "https://github.com/llnl/MemAxes" version('0.5', '5874f3fda9fd2d313c0ff9684f915ab5', - url='https://github.com/scalability-llnl/MemAxes/archive/v0.5.tar.gz') + url='https://github.com/llnl/MemAxes/archive/v0.5.tar.gz') depends_on("cmake@2.8.9:") depends_on("qt@5:") @@ -16,4 +16,3 @@ class Memaxes(Package): cmake('..', *std_cmake_args) make() make("install") - diff --git a/var/spack/packages/mesa/package.py b/var/spack/packages/mesa/package.py index 2a04a8fd51..62da8c993b 100644 --- a/var/spack/packages/mesa/package.py +++ b/var/spack/packages/mesa/package.py @@ -14,7 +14,7 @@ class Mesa(Package): # mesa 7.x, 8.x, 9.x depends_on("libdrm@2.4.33") depends_on("llvm@3.0") - depends_on("libxml2") + depends_on("libxml2+python") # patch("llvm-fixes.patch") # using newer llvm diff --git a/var/spack/packages/mpc/package.py b/var/spack/packages/mpc/package.py index 6fbfca3007..50477a0ccb 100644 --- a/var/spack/packages/mpc/package.py +++ b/var/spack/packages/mpc/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify @@ -31,6 +31,7 @@ class Mpc(Package): homepage = "http://www.multiprecision.org" url = "ftp://ftp.gnu.org/gnu/mpc/mpc-1.0.2.tar.gz" + version('1.0.3', 'd6a1d5f8ddea3abd2cc3e98f58352d26') version('1.0.2', '68fadff3358fb3e7976c7a398a0af4c3') depends_on("gmp") diff --git a/var/spack/packages/mpfr/package.py b/var/spack/packages/mpfr/package.py index 9c744a22df..0f2baac004 100644 --- a/var/spack/packages/mpfr/package.py +++ b/var/spack/packages/mpfr/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/packages/mpich/package.py b/var/spack/packages/mpich/package.py index d48bf878f6..e018a08201 100644 --- a/var/spack/packages/mpich/package.py +++ b/var/spack/packages/mpich/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/packages/mpileaks/package.py b/var/spack/packages/mpileaks/package.py index 4ef866588c..661d9d66bf 100644 --- a/var/spack/packages/mpileaks/package.py +++ b/var/spack/packages/mpileaks/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/packages/muster/package.py b/var/spack/packages/muster/package.py index 722daf3d7f..0dc2e5e086 100644 --- a/var/spack/packages/muster/package.py +++ b/var/spack/packages/muster/package.py @@ -7,8 +7,8 @@ class Muster(Package): for performance data analysis on systems with very large numbers of processes. """ - homepage = "https://github.com/scalability-llnl/muster" - url = "https://github.com/scalability-llnl/muster/archive/v1.0.tar.gz" + homepage = "https://github.com/llnl/muster" + url = "https://github.com/llnl/muster/archive/v1.0.tar.gz" version('1.0.1', 'd709787db7e080447afb6571ac17723c') version('1.0', '2eec6979a4a36d3a65a792d12969be16') diff --git a/var/spack/packages/mvapich2/package.py b/var/spack/packages/mvapich2/package.py index ca0b1287c1..dc2b2cb23f 100644 --- a/var/spack/packages/mvapich2/package.py +++ b/var/spack/packages/mvapich2/package.py @@ -1,104 +1,156 @@ -import os from spack import * + class Mvapich2(Package): - """mvapich2 is an MPI implmenetation for infiniband networks.""" + """MVAPICH2 is an MPI implementation for Infiniband networks.""" homepage = "http://mvapich.cse.ohio-state.edu/" - version('1.9', '5dc58ed08fd3142c260b70fe297e127c', - url="http://mvapich.cse.ohio-state.edu/download/mvapich2/mv2/mvapich2-1.9.tgz") - patch('ad_lustre_rwcontig_open_source.patch', when='@1.9') + version('2.2a', 'b8ceb4fc5f5a97add9b3ff1b9cbe39d2', + url='http://mvapich.cse.ohio-state.edu/download/mvapich/mv2/mvapich2-2.2a.tar.gz') version('2.0', '9fbb68a4111a8b6338e476dc657388b4', url='http://mvapich.cse.ohio-state.edu/download/mvapich/mv2/mvapich2-2.0.tar.gz') - provides('mpi@:2.2', when='@1.9') # MVAPICH2-1.9 supports MPI 2.2 - provides('mpi@:3.0', when='@2.0') # MVAPICH2-2.0 supports MPI 3.0 - - - def install(self, spec, prefix): - # we'll set different configure flags depending on our environment - configure_args = [] - - # TODO: The MPICH*_FLAGS have a different name for 1.9 + version('1.9', '5dc58ed08fd3142c260b70fe297e127c', + url="http://mvapich.cse.ohio-state.edu/download/mvapich2/mv2/mvapich2-1.9.tgz") + patch('ad_lustre_rwcontig_open_source.patch', when='@1.9') + provides('mpi@:2.2', when='@1.9') # MVAPICH2-1.9 supports MPI 2.2 + provides('mpi@:3.0', when='@2.0:') # MVAPICH2-2.0 supports MPI 3.0 + + variant('debug', default=False, description='Enables debug information and error messages at run-time') + + ########## + # TODO : Process managers should be grouped into the same variant, as soon as variant capabilities will be extended + # See https://groups.google.com/forum/#!topic/spack/F8-f8B4_0so + SLURM = 'slurm' + HYDRA = 'hydra' + GFORKER = 'gforker' + REMSHELL = 'remshell' + SLURM_INCOMPATIBLE_PMS = (HYDRA, GFORKER, REMSHELL) + variant(SLURM, default=False, description='Sets slurm as the only process manager') + variant(HYDRA, default=False, description='Sets hydra as one of the process managers') + variant(GFORKER, default=False, description='Sets gforker as one of the process managers') + variant(REMSHELL, default=False, description='Sets remshell as one of the process managers') + ########## + + ########## + # TODO : Network types should be grouped into the same variant, as soon as variant capabilities will be extended + PSM = 'psm' + SOCK = 'sock' + NEMESISIBTCP = 'nemesisibtcp' + NEMESISIB = 'nemesisib' + NEMESIS = 'nemesis' + SUPPORTED_NETWORKS = (PSM, SOCK, NEMESIS, NEMESISIB, NEMESISIBTCP) + variant(PSM, default=False, description='Configures a build for QLogic PSM-CH3') + variant(SOCK, default=False, description='Configures a build for TCP/IP-CH3') + variant(NEMESISIBTCP, default=False, description='Configures a build for both OFA-IB-Nemesis and TCP/IP-Nemesis') + variant(NEMESISIB, default=False, description='Configures a build for OFA-IB-Nemesis') + variant(NEMESIS, default=False, description='Configures a build for TCP/IP-Nemesis') + ########## + + # FIXME : CUDA support is missing + + @staticmethod + def enabled(x): + """ + Given a variant name returns the string that means the variant is enabled + + :param x: variant name + :return: + """ + return '+' + x + + def set_build_type(self, spec, configure_args): + """ + Appends to configure_args the flags that depends only on the build type (i.e. release or debug) + + :param spec: spec + :param configure_args: list of current configure arguments + """ if '+debug' in spec: - # set configure flags for debug build - configure_args.append("--disable-fast") - configure_args.append("--enable-g=dbg") - configure_args.append("--enable-error-checking=runtime") - configure_args.append("--enable-error-messages=all") - configure_args.append("--enable-nmpi-as-mpi") - - if "%gnu" in spec: - # set variables for GNU compilers - os.environ['MPICHLIB_CFLAGS'] = "-g -O0" - os.environ['MPICHLIB_CXXFLAGS'] = "-g -O0" - os.environ['MPICHLIB_FFLAGS'] = "-g -O0 -fno-second-underscore" - os.environ['MPICHLIB_F90FLAGS'] = "-g -O0 -fno-second-underscore" - elif "%intel" in spec: - # set variables for Inel compilers - os.environ['MPICHLIB_CFLAGS'] = "-g -O0" - os.environ['MPICHLIB_CXXFLAGS'] = "-g -O0" - os.environ['MPICHLIB_FFLAGS'] = "-g -O0" - os.environ['MPICHLIB_F90FLAGS'] = "-g -O0" - elif "%pgi" in spec: - # set variables for PGI compilers - os.environ['MPICHLIB_CFLAGS'] = "-g -O0 -fPIC" - os.environ['MPICHLIB_CXXFLAGS'] = "-g -O0 -fPIC" - os.environ['MPICHLIB_FFLAGS'] = "-g -O0 -fPIC" - os.environ['MPICHLIB_F90FLAGS'] = "-g -O0 -fPIC" - + build_type_options = [ + "--disable-fast", + "--enable-error-checking=runtime", + "--enable-error-messages=all", + "--enable-g=dbg", "--enable-debuginfo" # Permits debugging with TotalView + ] else: - # set configure flags for normal optimizations - configure_args.append("--enable-fast=all") - configure_args.append("--enable-g=dbg") - configure_args.append("--enable-nmpi-as-mpi") - - if "%gnu" in spec: - # set variables for what compilers - os.environ['MPICHLIB_CFLAGS'] = "-g -O2" - os.environ['MPICHLIB_CXXFLAGS'] = "-g -O2" - os.environ['MPICHLIB_FFLAGS'] = "-g -O2 -fno-second-underscore" - os.environ['MPICHLIB_F90FLAGS'] = "-g -O2 -fno-second-underscore" - elif "%intel" in spec: - # set variables for Inel compilers - os.environ['MPICHLIB_CFLAGS'] = "-g -O2" - os.environ['MPICHLIB_CXXFLAGS'] = "-g -O2" - os.environ['MPICHLIB_FFLAGS'] = "-g -O2" - os.environ['MPICHLIB_F90FLAGS'] = "-g -O2" - elif "%pgi" in spec: - # set variables for PGI compilers - os.environ['MPICHLIB_CFLAGS'] = "-g -O2 -fPIC" - os.environ['MPICHLIB_CXXFLAGS'] = "-g -O2 -fPIC" - os.environ['MPICHLIB_FFLAGS'] = "-g -O2 -fPIC" - os.environ['MPICHLIB_F90FLAGS'] = "-g -O2 -fPIC" - - # determine network type by variant - if "+psm" in spec: - # throw this flag on QLogic systems to use PSM - configure_args.append("--with-device=ch3:psm") + build_type_options = ["--enable-fast=all"] + + configure_args.extend(build_type_options) + + def set_process_manager(self, spec, configure_args): + """ + Appends to configure_args the flags that will enable the appropriate process managers + + :param spec: spec + :param configure_args: list of current configure arguments + """ + # Check that slurm variant is not activated together with other pm variants + has_slurm_incompatible_variants = any(self.enabled(x) in spec for x in Mvapich2.SLURM_INCOMPATIBLE_PMS) + if self.enabled(Mvapich2.SLURM) in spec and has_slurm_incompatible_variants: + raise RuntimeError(" %s : 'slurm' cannot be activated together with other process managers" % self.name) + + process_manager_options = [] + if self.enabled(Mvapich2.SLURM) in spec: + process_manager_options = [ + "--with-pm=slurm" + ] + elif has_slurm_incompatible_variants: + pms = [] + # The variant name is equal to the process manager name in the configuration options + for x in Mvapich2.SLURM_INCOMPATIBLE_PMS: + if self.enabled(x) in spec: + pms.append(x) + process_manager_options = [ + "--with-pm=%s" % ':'.join(pms) + ] + configure_args.extend(process_manager_options) + + def set_network_type(self, spec, configure_args): + # Check that at most one variant has been activated + count = 0 + for x in Mvapich2.SUPPORTED_NETWORKS: + if self.enabled(x) in spec: + count += 1 + if count > 1: + raise RuntimeError('network variants are mutually exclusive (only one can be selected at a time)') + + # From here on I can suppose that ony one variant has been selected + if self.enabled(Mvapich2.PSM) in spec: + network_options = ["--with-device=ch3:psm"] + elif self.enabled(Mvapich2.SOCK) in spec: + network_options = ["--with-device=ch3:sock"] + elif self.enabled(Mvapich2.NEMESISIBTCP) in spec: + network_options = ["--with-device=ch3:nemesis:ib,tcp"] + elif self.enabled(Mvapich2.NEMESISIB) in spec: + network_options = ["--with-device=ch3:nemesis:ib"] + elif self.enabled(Mvapich2.NEMESIS) in spec: + network_options = ["--with-device=ch3:nemesis"] else: - # throw this flag on IB systems - configure_args.append("--with-device=ch3:mrail", "--with-rdma=gen2") - - # TODO: shared-memory build - - # TODO: CUDA + network_options = ["--with-device=ch3:mrail", "--with-rdma=gen2"] - # TODO: other file systems like panasis + configure_args.extend(network_options) - configure( - "--prefix=" + prefix, - "--enable-f77", "--enable-fc", "--enable-cxx", - "--enable-shared", "--enable-sharedlibs=gcc", - "--enable-debuginfo", - "--with-pm=no", "--with-pmi=slurm", - "--enable-romio", "--with-file-system=lustre+nfs+ufs", - "--disable-mpe", "--without-mpe", + def install(self, spec, prefix): + # we'll set different configure flags depending on our environment + configure_args = [ + "--prefix=%s" % prefix, + "--enable-shared", + "--enable-romio", "--disable-silent-rules", - *configure_args) - + ] + if not self.compiler.f77 and not self.compiler.fc: + configure_args.append("--enable-fortran=none") + + # Set the type of the build (debug, release) + self.set_build_type(spec, configure_args) + # Set the process manager + self.set_process_manager(spec, configure_args) + # Determine network type by variant + self.set_network_type(spec, configure_args) + + configure(*configure_args) make() - make("install") diff --git a/var/spack/packages/ncurses/package.py b/var/spack/packages/ncurses/package.py index 8f5763bfdd..cc180bbae1 100644 --- a/var/spack/packages/ncurses/package.py +++ b/var/spack/packages/ncurses/package.py @@ -11,6 +11,8 @@ class Ncurses(Package): version('5.9', '8cb9c412e5f2d96bc6f459aa8c6282a1', url='http://ftp.gnu.org/pub/gnu/ncurses/ncurses-5.9.tar.gz') + version('6.0', 'ee13d052e1ead260d7c28071f46eefb1', + url='http://ftp.gnu.org/pub/gnu/ncurses/ncurses-6.0.tar.gz') def install(self, spec, prefix): configure("--prefix=%s" % prefix, diff --git a/var/spack/packages/netgauge/package.py b/var/spack/packages/netgauge/package.py index c2378b0718..0ea42175c6 100644 --- a/var/spack/packages/netgauge/package.py +++ b/var/spack/packages/netgauge/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/packages/ompt-openmp/package.py b/var/spack/packages/ompt-openmp/package.py index 5d380ebd77..e5bcfb51f0 100644 --- a/var/spack/packages/ompt-openmp/package.py +++ b/var/spack/packages/ompt-openmp/package.py @@ -3,9 +3,9 @@ from spack import * class OmptOpenmp(Package): """LLVM/Clang OpenMP runtime with OMPT support. This is a fork of the OpenMPToolsInterface/LLVM-openmp fork of the official LLVM OpenMP mirror. This library provides a drop-in replacement of the OpenMP runtimes for GCC, Intel and LLVM/Clang.""" homepage = "https://github.com/OpenMPToolsInterface/LLVM-openmp" - url = "http://github.com/khuck/LLVM-openmp/archive/v0.1-spack.tar.gz" + url = "http://github.com/khuck/LLVM-openmp/archive/v0.1.tar.gz" - version('spack', '35227b2726e377faa433fc841226e036') + version('0.1', '2334e6a84b52da41b27afd9831ed5370') # depends_on("foo") diff --git a/var/spack/packages/openblas/package.py b/var/spack/packages/openblas/package.py new file mode 100644 index 0000000000..e01467c05a --- /dev/null +++ b/var/spack/packages/openblas/package.py @@ -0,0 +1,21 @@ +from spack import * + +class Openblas(Package): + """OpenBLAS: An optimized BLAS library""" + homepage = "http://www.openblas.net" + url = "http://github.com/xianyi/OpenBLAS/archive/v0.2.15.tar.gz" + + version('0.2.15', 'b1190f3d3471685f17cfd1ec1d252ac9') + + # virtual dependency + provides('blas') + provides('lapack') + + def install(self, spec, prefix): + make('libs', 'netlib', 'shared', 'CC=cc', 'FC=f77') + make('install', "PREFIX='%s'" % prefix) + + # Blas virtual package should provide blas.a and libblas.a + with working_dir(prefix.lib): + symlink('libopenblas.a', 'blas.a') + symlink('libopenblas.a', 'libblas.a') diff --git a/var/spack/packages/openmpi/configure.patch b/var/spack/packages/openmpi/configure.patch new file mode 100644 index 0000000000..18fb42c1d1 --- /dev/null +++ b/var/spack/packages/openmpi/configure.patch @@ -0,0 +1,31 @@ +This patch addresses <https://github.com/open-mpi/ompi/issues/576>. +--- a/configure ++++ b/configure +@@ -301130,10 +301130,11 @@ + case ${prev}${p} in + + -L* | -R* | -l*) +- # Some compilers place space between "-{L,R}" and the path. ++ # Some compilers place space between "-{L,R,l}" and the path. + # Remove the space. + if test $p = "-L" || +- test $p = "-R"; then ++ test $p = "-R" || ++ test $p = "-l"; then + prev=$p + continue + fi +@@ -303036,10 +303037,11 @@ + case ${prev}${p} in + + -L* | -R* | -l*) +- # Some compilers place space between "-{L,R}" and the path. ++ # Some compilers place space between "-{L,R,l}" and the path. + # Remove the space. + if test $p = "-L" || +- test $p = "-R"; then ++ test $p = "-R" || ++ test $p = "-l"; then + prev=$p + continue + fi diff --git a/var/spack/packages/openmpi/package.py b/var/spack/packages/openmpi/package.py index 5e429dedf5..be2202fbbd 100644 --- a/var/spack/packages/openmpi/package.py +++ b/var/spack/packages/openmpi/package.py @@ -14,6 +14,8 @@ class Openmpi(Package): homepage = "http://www.open-mpi.org" + version('1.10.1', 'f0fcd77ed345b7eafb431968124ba16e', + url = "http://www.open-mpi.org/software/ompi/v1.10/downloads/openmpi-1.10.1.tar.bz2") version('1.10.0', '280cf952de68369cebaca886c5ce0304', url = "http://www.open-mpi.org/software/ompi/v1.10/downloads/openmpi-1.10.0.tar.bz2") version('1.8.8', '0dab8e602372da1425e9242ae37faf8c', @@ -23,10 +25,15 @@ class Openmpi(Package): patch('ad_lustre_rwcontig_open_source.patch', when="@1.6.5") patch('llnl-platforms.patch', when="@1.6.5") + patch('configure.patch', when="@1.10.0:") provides('mpi@:2.2', when='@1.6.5') # Open MPI 1.6.5 supports MPI-2.2 provides('mpi@:3.0', when='@1.8.8') # Open MPI 1.8.8 supports MPI-3.0 provides('mpi@:3.0', when='@1.10.0') # Open MPI 1.10.0 supports MPI-3.0 + provides('mpi@:3.0', when='@1.10.1') # Open MPI 1.10.1 supports MPI-3.0 + + + depends_on('hwloc') def setup_dependent_environment(self, module, spec, dep_spec): @@ -40,6 +47,8 @@ class Openmpi(Package): def install(self, spec, prefix): config_args = ["--prefix=%s" % prefix] + config_args.append("--with-hwloc=%s" % spec['hwloc'].prefix) + # TODO: use variants for this, e.g. +lanl, +llnl, etc. # use this for LANL builds, but for LLNL builds, we need: # "--with-platform=contrib/platform/llnl/optimized" diff --git a/var/spack/packages/openssl/package.py b/var/spack/packages/openssl/package.py index c5a8aeb9dc..40648fca49 100644 --- a/var/spack/packages/openssl/package.py +++ b/var/spack/packages/openssl/package.py @@ -10,6 +10,8 @@ class Openssl(Package): url = "http://www.openssl.org/source/openssl-1.0.1h.tar.gz" version('1.0.1h', '8d6d684a9430d5cc98a62a5d8fbda8cf') + version('1.0.2d', '38dd619b2e77cbac69b99f52a053d25a') + version('1.0.2e', '5262bfa25b60ed9de9f28d5d52d77fc5') depends_on("zlib") parallel = False diff --git a/var/spack/packages/papi/package.py b/var/spack/packages/papi/package.py index 596f7114d6..097b8c41ec 100644 --- a/var/spack/packages/papi/package.py +++ b/var/spack/packages/papi/package.py @@ -13,6 +13,7 @@ class Papi(Package): homepage = "http://icl.cs.utk.edu/papi/index.html" url = "http://icl.cs.utk.edu/projects/papi/downloads/papi-5.3.0.tar.gz" + version('5.4.1', '9134a99219c79767a11463a76b0b01a2') version('5.3.0', '367961dd0ab426e5ae367c2713924ffb') def install(self, spec, prefix): @@ -20,13 +21,10 @@ class Papi(Package): configure_args=["--prefix=%s" % prefix] - # need to force consistency in the use of compilers - if spec.satisfies('%gcc'): - configure_args.append('CC=gcc') - configure_args.append('MPICH_CC=gcc') - if spec.satisfies('%intel'): - configure_args.append('CC=icc') - configure_args.append('MPICH_CC=icc') + # PAPI uses MPI if MPI is present; since we don't require an + # MPI package, we ensure that all attempts to use MPI fail, so + # that PAPI does not get confused + configure_args.append('MPICC=:') configure(*configure_args) diff --git a/var/spack/packages/paraview/package.py b/var/spack/packages/paraview/package.py new file mode 100644 index 0000000000..b0893237e9 --- /dev/null +++ b/var/spack/packages/paraview/package.py @@ -0,0 +1,74 @@ +from spack import * + +class Paraview(Package): + homepage = 'http://www.paraview.org' + url = 'http://www.paraview.org/files/v4.4/ParaView-v4.4.0-source.tar.gz' + + version('4.4.0', 'fa1569857dd680ebb4d7ff89c2227378', url='http://www.paraview.org/files/v4.4/ParaView-v4.4.0-source.tar.gz') + + variant('python', default=False, description='Enable Python support') + + variant('tcl', default=False, description='Enable TCL support') + + variant('mpi', default=False, description='Enable MPI support') + + variant('osmesa', default=False, description='Enable OSMesa support') + variant('qt', default=False, description='Enable Qt support') + + depends_on('python', when='+python') + depends_on('py-numpy', when='+python') + depends_on('py-matplotlib', when='+python') + depends_on('tcl', when='+tcl') + depends_on('mpi', when='+mpi') + depends_on('qt@:4', when='+qt') + + depends_on('bzip2') + depends_on('freetype') + depends_on('hdf5') # drags in mpi + depends_on('jpeg') + depends_on('libpng') + depends_on('libtiff') + depends_on('libxml2') + depends_on('netcdf') + #depends_on('protobuf') # version mismatches? + #depends_on('sqlite') # external version not supported + depends_on('zlib') + + def install(self, spec, prefix): + with working_dir('spack-build', create=True): + def feature_to_bool(feature, on='ON', off='OFF'): + if feature in spec: + return on + return off + + def nfeature_to_bool(feature): + return feature_to_bool(feature, on='OFF', off='ON') + + feature_args = std_cmake_args[:] + feature_args.append('-DPARAVIEW_BUILD_QT_GUI:BOOL=%s' % feature_to_bool('+qt')) + feature_args.append('-DPARAVIEW_ENABLE_PYTHON:BOOL=%s' % feature_to_bool('+python')) + if '+python' in spec: + feature_args.append('-DPYTHON_EXECUTABLE:FILEPATH=%s/bin/python' % spec['python'].prefix) + feature_args.append('-DPARAVIEW_USE_MPI:BOOL=%s' % feature_to_bool('+mpi')) + if '+mpi' in spec: + feature_args.append('-DMPIEXEC:FILEPATH=%s/bin/mpiexec' % spec['mpi'].prefix) + feature_args.append('-DVTK_ENABLE_TCL_WRAPPING:BOOL=%s' % feature_to_bool('+tcl')) + feature_args.append('-DVTK_OPENGL_HAS_OSMESA:BOOL=%s' % feature_to_bool('+osmesa')) + feature_args.append('-DVTK_USE_X:BOOL=%s' % nfeature_to_bool('+osmesa')) + feature_args.append('-DVTK_RENDERING_BACKEND:STRING=%s' % feature_to_bool('+opengl2', 'OpenGL2', 'OpenGL')) + + feature_args.extend(std_cmake_args) + + cmake('..', + '-DCMAKE_INSTALL_PREFIX:PATH=%s' % prefix, + '-DBUILD_TESTING:BOOL=OFF', + '-DVTK_USER_SYSTEM_FREETYPE:BOOL=ON', + '-DVTK_USER_SYSTEM_HDF5:BOOL=ON', + '-DVTK_USER_SYSTEM_JPEG:BOOL=ON', + '-DVTK_USER_SYSTEM_LIBXML2:BOOL=ON', + '-DVTK_USER_SYSTEM_NETCDF:BOOL=ON', + '-DVTK_USER_SYSTEM_TIFF:BOOL=ON', + '-DVTK_USER_SYSTEM_ZLIB:BOOL=ON', + *feature_args) + make() + make('install') diff --git a/var/spack/packages/pcre/package.py b/var/spack/packages/pcre/package.py index 3424048a6c..e38d337e3d 100644 --- a/var/spack/packages/pcre/package.py +++ b/var/spack/packages/pcre/package.py @@ -8,6 +8,7 @@ class Pcre(Package): url = "ftp://ftp.csx.cam.ac.uk/pub/software/programming/pcre/pcre-8.36.tar.bz2" version('8.36', 'b767bc9af0c20bc9c1fe403b0d41ad97') + version('8.38', '00aabbfe56d5a48b270f999b508c5ad2') def install(self, spec, prefix): configure("--prefix=%s" % prefix) diff --git a/var/spack/packages/pmgr_collective/package.py b/var/spack/packages/pmgr_collective/package.py index 5d9b02acc3..1fc47c658f 100644 --- a/var/spack/packages/pmgr_collective/package.py +++ b/var/spack/packages/pmgr_collective/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/packages/py-basemap/package.py b/var/spack/packages/py-basemap/package.py index 45f1085ba1..8dfc99b28d 100644 --- a/var/spack/packages/py-basemap/package.py +++ b/var/spack/packages/py-basemap/package.py @@ -11,8 +11,8 @@ class PyBasemap(Package): extends('python') depends_on('py-setuptools') depends_on('py-numpy') - depends_on('py-matplotlib') - depends_on('py-pil') + depends_on('py-matplotlib+gui') + depends_on('py-pillow') depends_on("geos") def install(self, spec, prefix): diff --git a/var/spack/packages/py-matplotlib/package.py b/var/spack/packages/py-matplotlib/package.py index e7ce3dfd24..a5fee39d42 100644 --- a/var/spack/packages/py-matplotlib/package.py +++ b/var/spack/packages/py-matplotlib/package.py @@ -9,10 +9,13 @@ class PyMatplotlib(Package): version('1.4.2', '7d22efb6cce475025733c50487bd8898') version('1.4.3', '86af2e3e3c61849ac7576a6f5ca44267') + variant('gui', default=False, description='Enable GUI') + variant('ipython', default=False, description='Enable ipython support') + extends('python', ignore=r'bin/nosetests.*$') - depends_on('py-pyside') - depends_on('py-ipython') + depends_on('py-pyside', when='+gui') + depends_on('py-ipython', when='+ipython') depends_on('py-pyparsing') depends_on('py-six') depends_on('py-dateutil') @@ -20,10 +23,10 @@ class PyMatplotlib(Package): depends_on('py-nose') depends_on('py-numpy') - depends_on('qt') + depends_on('qt', when='+gui') depends_on('bzip2') - depends_on('tcl') - depends_on('tk') + depends_on('tcl', when='+gui') + depends_on('tk', when='+gui') depends_on('qhull') def install(self, spec, prefix): diff --git a/var/spack/packages/py-pillow/package.py b/var/spack/packages/py-pillow/package.py new file mode 100644 index 0000000000..adc8507bd5 --- /dev/null +++ b/var/spack/packages/py-pillow/package.py @@ -0,0 +1,14 @@ +from spack import * + +class PyPillow(Package): + """Pillow is the friendly PIL fork by Alex Clark and Contributors. PIL is the Python Imaging Library by Fredrik Lundh and Contributors. The Python Imaging Library (PIL) adds image processing capabilities to your Python interpreter. This library supports many file formats, and provides powerful image processing and graphics capabilities.""" + + homepage = "https://python-pillow.github.io/" + url = "https://pypi.python.org/packages/source/P/Pillow/Pillow-3.0.0.tar.gz" + + version('3.0.0', 'fc8ac44e93da09678eac7e30c9b7377d') + extends('python') + depends_on('py-setuptools') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-pyside/package.py b/var/spack/packages/py-pyside/package.py index bb5da44d02..ffa433e18e 100644 --- a/var/spack/packages/py-pyside/package.py +++ b/var/spack/packages/py-pyside/package.py @@ -2,7 +2,7 @@ from spack import * import os class PyPyside(Package): - """array processing for numbers, strings, records, and objects.""" + """Python bindings for Qt.""" homepage = "https://pypi.python.org/pypi/pyside" url = "https://pypi.python.org/packages/source/P/PySide/PySide-1.2.2.tar.gz" diff --git a/var/spack/packages/py-sphinx/package.py b/var/spack/packages/py-sphinx/package.py new file mode 100644 index 0000000000..ec2e89a098 --- /dev/null +++ b/var/spack/packages/py-sphinx/package.py @@ -0,0 +1,13 @@ +from spack import * + +class PySphinx(Package): + """Sphinx Documentation Generator.""" + homepage = "http://sphinx-doc.org" + url = "https://pypi.python.org/packages/source/S/Sphinx/Sphinx-1.3.1.tar.gz" + + version('1.3.1', '8786a194acf9673464c5455b11fd4332') + + extends('python') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-twisted/package.py b/var/spack/packages/py-twisted/package.py new file mode 100644 index 0000000000..2fdebb6cb9 --- /dev/null +++ b/var/spack/packages/py-twisted/package.py @@ -0,0 +1,16 @@ +from spack import * + +class PyTwisted(Package): + """An asynchronous networking framework written in Python""" + homepage = "https://twistedmatrix.com/" + url = "https://pypi.python.org/packages/source/T/Twisted/Twisted-15.3.0.tar.bz2" + + version('15.4.0', '5337ffb6aeeff3790981a2cd56db9655') + version('15.3.0', 'b58e83da2f00b3352afad74d0c5c4599') + + depends_on('py-setuptools') + + extends('python') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/python/package.py b/var/spack/packages/python/package.py index 000881a846..31fb8b3d5a 100644 --- a/var/spack/packages/python/package.py +++ b/var/spack/packages/python/package.py @@ -26,6 +26,7 @@ class Python(Package): def install(self, spec, prefix): # Need this to allow python build to find the Python installation. env['PYTHONHOME'] = prefix + env['MACOSX_DEPLOYMENT_TARGET'] = '10.6' # Rest of install is pretty standard. configure("--prefix=%s" % prefix, diff --git a/var/spack/packages/ravel/package.py b/var/spack/packages/ravel/package.py index 01fa941cfe..d774a0ab86 100644 --- a/var/spack/packages/ravel/package.py +++ b/var/spack/packages/ravel/package.py @@ -4,8 +4,8 @@ class Ravel(Package): """Ravel is a parallel communication trace visualization tool that orders events according to logical time.""" - homepage = "https://github.com/scalability-llnl/ravel" - url = 'https://github.com/scalability-llnl/ravel/archive/v1.0.0.tar.gz' + homepage = "https://github.com/llnl/ravel" + url = 'https://github.com/llnl/ravel/archive/v1.0.0.tar.gz' version('1.0.0', 'b25fece58331c2adfcce76c5036485c2') diff --git a/var/spack/packages/rsync/package.py b/var/spack/packages/rsync/package.py new file mode 100644 index 0000000000..8ae21b1cb9 --- /dev/null +++ b/var/spack/packages/rsync/package.py @@ -0,0 +1,14 @@ +from spack import * + +class Rsync(Package): + """rsync is an open source utility that provides fast incremental file transfer.""" + homepage = "https://rsync.samba.org" + url = "https://download.samba.org/pub/rsync/rsync-3.1.1.tar.gz" + + version('3.1.1', '43bd6676f0b404326eee2d63be3cdcfe') + + def install(self, spec, prefix): + configure('--prefix=%s' % prefix) + + make() + make("install") diff --git a/var/spack/packages/scr/package.py b/var/spack/packages/scr/package.py index 9fb758f072..1408dce678 100644 --- a/var/spack/packages/scr/package.py +++ b/var/spack/packages/scr/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/packages/sparsehash/package.py b/var/spack/packages/sparsehash/package.py new file mode 100644 index 0000000000..7decaeb89b --- /dev/null +++ b/var/spack/packages/sparsehash/package.py @@ -0,0 +1,14 @@ +from spack import * + +class Sparsehash(Package): + """Sparse and dense hash-tables for C++ by Google""" + homepage = "https://github.com/sparsehash/sparsehash" + url = "https://github.com/sparsehash/sparsehash/archive/sparsehash-2.0.3.tar.gz" + + version('2.0.3', 'd8d5e2538c1c25577b3f066d7a55e99e') + + def install(self, spec, prefix): + configure('--prefix=%s' % prefix) + + make() + make("install") diff --git a/var/spack/packages/spindle/package.py b/var/spack/packages/spindle/package.py index 06a1e14284..a20753458a 100644 --- a/var/spack/packages/spindle/package.py +++ b/var/spack/packages/spindle/package.py @@ -6,7 +6,7 @@ # Written by Matthew LeGendre, legendre1@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/packages/sqlite/package.py b/var/spack/packages/sqlite/package.py index 734b0b6cb6..1cf2d30239 100644 --- a/var/spack/packages/sqlite/package.py +++ b/var/spack/packages/sqlite/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/packages/sundials/package.py b/var/spack/packages/sundials/package.py index 8b784c8c3c..7e025a8244 100644 --- a/var/spack/packages/sundials/package.py +++ b/var/spack/packages/sundials/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/packages/swig/package.py b/var/spack/packages/swig/package.py index ee536d7063..8d46c4fe46 100644 --- a/var/spack/packages/swig/package.py +++ b/var/spack/packages/swig/package.py @@ -6,7 +6,7 @@ # Written by Matthew LeGendre, legendre1@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/packages/texinfo/package.py b/var/spack/packages/texinfo/package.py new file mode 100644 index 0000000000..460db65b3e --- /dev/null +++ b/var/spack/packages/texinfo/package.py @@ -0,0 +1,46 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## + +from spack import * + + +class Texinfo(Package): + """ + Texinfo is the official documentation format of the GNU project. It was invented by Richard Stallman and Bob + Chassell many years ago, loosely based on Brian Reid's Scribe and other formatting languages of the time. It is + used by many non-GNU projects as well.FIXME: put a proper description of your package here. + """ + homepage = "https://www.gnu.org/software/texinfo/" + url = "http://ftp.gnu.org/gnu/texinfo/texinfo-6.0.tar.xz" + + version('6.0', '02818e62a5b8ae0213a7ff572991bb50') + version('5.2', 'cb489df8a7ee9d10a236197aefdb32c5') + version('5.1', '52ee905a3b705020d2a1b6ec36d53ca6') + version('5.0', 'ef2fad34c71ddc95b20c7d6a08c0d7a6') + + def install(self, spec, prefix): + configure('--prefix=%s' % prefix) + make() + make("install") diff --git a/var/spack/packages/xz/package.py b/var/spack/packages/xz/package.py index 88c5793018..ba6c9733a7 100644 --- a/var/spack/packages/xz/package.py +++ b/var/spack/packages/xz/package.py @@ -8,9 +8,13 @@ class Xz(Package): homepage = "http://tukaani.org/xz/" url = "http://tukaani.org/xz/xz-5.2.0.tar.bz2" - version('5.2.0', '867cc8611760240ebf3440bd6e170bb9') - + version('5.2.0', '867cc8611760240ebf3440bd6e170bb9', + url = 'http://tukaani.org/xz/xz-5.2.0.tar.bz2') + version('5.2.2', 'f90c9a0c8b259aee2234c4e0d7fd70af', + url = 'http://tukaani.org/xz/xz-5.2.2.tar.bz2') + def install(self, spec, prefix): configure("--prefix=%s" % prefix) make() make("install") + diff --git a/var/spack/packages/zsh/package.py b/var/spack/packages/zsh/package.py index 99ef9de2e5..06665f0c83 100644 --- a/var/spack/packages/zsh/package.py +++ b/var/spack/packages/zsh/package.py @@ -1,11 +1,15 @@ from spack import * class Zsh(Package): - """ The ZSH shell """ + """ + Zsh is a shell designed for interactive use, although it is also a powerful + scripting language. Many of the useful features of bash, ksh, and tcsh were + incorporated into zsh; many original features were added. + """ homepage = "http://www.zsh.org" - url = "http://www.zsh.org/pub/zsh-5.0.8.tar.bz2" + url = "http://downloads.sourceforge.net/project/zsh/zsh/5.1.1/zsh-5.1.1.tar.gz" - version('5.0.8', 'e6759e8dd7b714d624feffd0a73ba0fe') + version('5.1.1', checksum='8ba28a9ef82e40c3a271602f18343b2f') depends_on("pcre") |