From c7b8d09c7f180da5922801450fe0ae6a0f802377 Mon Sep 17 00:00:00 2001 From: Matthew LeGendre Date: Mon, 20 Apr 2015 10:38:18 -0700 Subject: Add packagerepos to spack, allowing for creating multiple package repositories. --- lib/spack/spack/__init__.py | 6 ++ lib/spack/spack/build_environment.py | 25 +++++- lib/spack/spack/cmd/create.py | 12 +++ lib/spack/spack/cmd/packagerepo.py | 85 ++++++++++++++++++++ lib/spack/spack/packages.py | 147 ++++++++++++++++++++++++----------- lib/spack/spack/repo_loader.py | 115 +++++++++++++++++++++++++++ 6 files changed, 342 insertions(+), 48 deletions(-) create mode 100644 lib/spack/spack/cmd/packagerepo.py create mode 100644 lib/spack/spack/repo_loader.py (limited to 'lib') diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index caa09eb6e0..1d67b45341 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -62,6 +62,12 @@ mock_config_path = join_path(var_path, "mock_configs") mock_site_config = join_path(mock_config_path, "site_spackconfig") mock_user_config = join_path(mock_config_path, "user_spackconfig") +# +# Setup the spack.repos namespace +# +from spack.repo_loader import RepoNamespace +repos = RepoNamespace() + # # This controls how spack lays out install prefixes and # stage directories. diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index a133faa629..03a4930259 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -157,7 +157,7 @@ def set_build_environment_variables(pkg): path_set("PKG_CONFIG_PATH", pkg_config_dirs) -def set_module_variables_for_package(pkg): +def set_module_variables_for_package(pkg, m): """Populate the module scope of install() with some useful functions. This makes things easier for package writers. """ @@ -228,11 +228,32 @@ def get_rpaths(pkg): return rpaths +def parent_class_modules(cls): + """Get list of super class modules that are all descend from spack.Package""" + if not issubclass(cls, spack.Package) or issubclass(spack.Package, cls): + return [] + result = [] + module = sys.modules.get(cls.__module__) + if module: + result = [ module ] + for c in cls.__bases__: + result.extend(parent_class_modules(c)) + return result + + def setup_package(pkg): """Execute all environment setup routines.""" set_compiler_environment_variables(pkg) set_build_environment_variables(pkg) - set_module_variables_for_package(pkg) + + # If a user makes their own package repo, e.g. + # spack.repos.mystuff.libelf.Libelf, and they inherit from + # an existing class like spack.repos.original.libelf.Libelf, + # then set the module variables for both classes so the + # parent class can still use them if it gets called. + modules = parent_class_modules(pkg.__class__) + for mod in modules: + set_module_variables_for_package(pkg, mod) # Allow dependencies to set up environment as well. for dep_spec in pkg.spec.traverse(root=False): diff --git a/lib/spack/spack/cmd/create.py b/lib/spack/spack/cmd/create.py index 46e6bcec14..1502942f2c 100644 --- a/lib/spack/spack/cmd/create.py +++ b/lib/spack/spack/cmd/create.py @@ -93,6 +93,9 @@ def setup_parser(subparser): subparser.add_argument( '-n', '--name', dest='alternate_name', default=None, help="Override the autodetected name for the created package.") + subparser.add_argument( + '-p', '--package-repo', dest='package_repo', default=None, + help="Create the package in the specified packagerepo.") subparser.add_argument( '-f', '--force', action='store_true', dest='force', help="Overwrite any existing package file with the same name.") @@ -160,12 +163,21 @@ def create(parser, args): tty.die("Couldn't guess a name for this package. Try running:", "", "spack create --name ") + package_repo = args.package_repo + if not valid_module_name(name): tty.die("Package name can only contain A-Z, a-z, 0-9, '_' and '-'") tty.msg("This looks like a URL for %s version %s." % (name, version)) tty.msg("Creating template for package %s" % name) + # Create a directory for the new package. + pkg_path = spack.db.filename_for_package_name(name, package_repo) + if os.path.exists(pkg_path) and not args.force: + tty.die("%s already exists." % pkg_path) + else: + mkdirp(os.path.dirname(pkg_path)) + versions = spack.package.find_versions_of_archive(url) rkeys = sorted(versions.keys(), reverse=True) versions = OrderedDict(zip(rkeys, (versions[v] for v in rkeys))) diff --git a/lib/spack/spack/cmd/packagerepo.py b/lib/spack/spack/cmd/packagerepo.py new file mode 100644 index 0000000000..66bad0ecbf --- /dev/null +++ b/lib/spack/spack/cmd/packagerepo.py @@ -0,0 +1,85 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from external import argparse + +import llnl.util.tty as tty +from llnl.util.tty.color import colorize +from llnl.util.tty.colify import colify +from llnl.util.lang import index_by + +import spack.spec +import spack.config +from spack.util.environment import get_path + +description = "Manage package sources" + +def setup_parser(subparser): + sp = subparser.add_subparsers( + metavar='SUBCOMMAND', dest='packagerepo_command') + + add_parser = sp.add_parser('add', help=packagerepo_add.__doc__) + add_parser.add_argument('directory', help="Directory containing the packages.") + + remove_parser = sp.add_parser('remove', help=packagerepo_remove.__doc__) + remove_parser.add_argument('name') + + list_parser = sp.add_parser('list', help=packagerepo_list.__doc__) + + +def packagerepo_add(args): + """Add package sources to the Spack configuration.""" + config = spack.config.get_config() + user_config = spack.config.get_config('user') + orig = None + if config.has_value('packagerepo', '', 'directories'): + orig = config.get_value('packagerepo', '', 'directories') + if orig and args.directory in orig.split(':'): + tty.die('Repo directory %s already exists in the repo list' % args.directory) + + newsetting = orig + ':' + args.directory if orig else args.directory + user_config.set_value('packagerepo', '', 'directories', newsetting) + user_config.write() + + +def packagerepo_remove(args): + """Remove a package source from the Spack configuration""" + pass + + +def packagerepo_list(args): + """List package sources and their mnemoics""" + root_names = spack.db.repos + max_len = max(len(s[0]) for s in root_names) + fmt = "%%-%ds%%s" % (max_len + 4) + for root in root_names: + print fmt % (root[0], root[1]) + + + +def packagerepo(parser, args): + action = { 'add' : packagerepo_add, + 'remove' : packagerepo_remove, + 'list' : packagerepo_list } + action[args.packagerepo_command](args) diff --git a/lib/spack/spack/packages.py b/lib/spack/spack/packages.py index adfbc26c1d..79dbd60703 100644 --- a/lib/spack/spack/packages.py +++ b/lib/spack/spack/packages.py @@ -23,10 +23,14 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import os +import exceptions import sys import inspect import glob import imp +import spack.config +import re +from contextlib import closing import llnl.util.tty as tty from llnl.util.filesystem import join_path @@ -36,13 +40,11 @@ import spack.error import spack.spec from spack.virtual import ProviderIndex from spack.util.naming import mod_to_class, validate_module_name +from sets import Set +from spack.repo_loader import RepoLoader, imported_packages_module, package_file_name -# Name of module under which packages are imported -_imported_packages_module = 'spack.packages' - -# Name of the package file inside a package directory -_package_file_name = 'package.py' - +# Filename for package repo names +_packagerepo_filename = 'reponame' def _autospec(function): """Decorator that automatically converts the argument of a single-arg @@ -55,13 +57,57 @@ def _autospec(function): class PackageDB(object): - def __init__(self, root): + def __init__(self, default_root): """Construct a new package database from a root directory.""" - self.root = root + + #Collect the repos from the config file and read their names from the file system + repo_dirs = self._repo_list_from_config() + repo_dirs.append(default_root) + self.repos = [(self._read_reponame_from_directory(dir), dir) for dir in repo_dirs] + + # Check for duplicate repo names + s = set() + dups = set(r for r in self.repos if r[0] in s or s.add(r[0])) + if dups: + reponame = list(dups)[0][0] + dir1 = list(dups)[0][1] + dir2 = dict(s)[reponame] + tty.die("Package repo %s in directory %s has the same name as the " + "repo in directory %s" % + (reponame, dir1, dir2)) + + # For each repo, create a RepoLoader + self.repo_loaders = dict([(r[0], RepoLoader(r[0], r[1])) for r in self.repos]) + self.instances = {} self.provider_index = None + def _read_reponame_from_directory(self, dir): + """For a packagerepo directory, read the repo name from the dir/reponame file""" + path = os.path.join(dir, 'reponame') + + try: + with closing(open(path, 'r')) as reponame_file: + name = reponame_file.read().lstrip().rstrip() + if not re.match(r'[a-zA-Z][a-zA-Z0-9]+', name): + tty.die("Package repo name '%s', read from %s, is an invalid name. " + "Repo names must began with a letter and only contain letters " + "and numbers." % (name, path)) + return name + except exceptions.IOError, e: + tty.die("Could not read from package repo name file %s" % path) + + + + def _repo_list_from_config(self): + """Read through the spackconfig and return the list of packagerepo directories""" + config = spack.config.get_config() + if not config.has_option('packagerepo', 'directories'): return [] + dir_string = config.get('packagerepo', 'directories') + return dir_string.split(':') + + @_autospec def get(self, spec, **kwargs): if spec.virtual: @@ -130,13 +176,33 @@ class PackageDB(object): # catching exceptions. - def dirname_for_package_name(self, pkg_name): + def repo_for_package_name(self, pkg_name, packagerepo_name=None): + """Find the dirname for a package and the packagerepo it came from + if packagerepo_name is not None, then search for the package in the + specified packagerepo""" + #Look for an existing package under any matching packagerepos + roots = [pkgrepo for pkgrepo in self.repos + if not packagerepo_name or packagerepo_name == pkgrepo[0]] + + if not roots: + tty.die("Package repo %s does not exist" % packagerepo_name) + + for pkgrepo in roots: + path = join_path(pkgrepo[1], pkg_name) + if os.path.exists(path): + return (pkgrepo[0], path) + + repo_to_add_to = roots[-1] + return (repo_to_add_to[0], join_path(repo_to_add_to[1], pkg_name)) + + + def dirname_for_package_name(self, pkg_name, packagerepo_name=None): """Get the directory name for a particular package. This is the directory that contains its package.py file.""" - return join_path(self.root, pkg_name) + return self.repo_for_package_name(pkg_name, packagerepo_name)[1] - def filename_for_package_name(self, pkg_name): + def filename_for_package_name(self, pkg_name, packagerepo_name=None): """Get the filename for the module we should load for a particular package. Packages for a pacakge DB live in ``$root//package.py`` @@ -144,10 +210,15 @@ class PackageDB(object): This will return a proper package.py path even if the package doesn't exist yet, so callers will need to ensure the package exists before importing. + + If a packagerepo is specified, then return existing + or new paths in the specified packagerepo directory. If no + package repo is supplied, return an existing path from any + package repo, and new paths in the default package repo. """ validate_module_name(pkg_name) - pkg_dir = self.dirname_for_package_name(pkg_name) - return join_path(pkg_dir, _package_file_name) + pkg_dir = self.dirname_for_package_name(pkg_name, packagerepo_name) + return join_path(pkg_dir, package_file_name) def installed_package_specs(self): @@ -176,14 +247,19 @@ class PackageDB(object): @memoized def all_package_names(self): """Generator function for all packages. This looks for - ``/package.py`` files within the root direcotry""" - all_package_names = [] - for pkg_name in os.listdir(self.root): - pkg_dir = join_path(self.root, pkg_name) - pkg_file = join_path(pkg_dir, _package_file_name) - if os.path.isfile(pkg_file): - all_package_names.append(pkg_name) - all_package_names.sort() + ``/package.py`` files within the repo direcotories""" + all_packages = Set() + for repo in self.repos: + dir = repo[1] + if not os.path.isdir(dir): + continue + for pkg_name in os.listdir(dir): + pkg_dir = join_path(dir, pkg_name) + pkg_file = join_path(pkg_dir, package_file_name) + if os.path.isfile(pkg_file): + all_packages.add(pkg_name) + all_package_names = list(all_packages) + all_package_names.sort() return all_package_names @@ -200,34 +276,13 @@ class PackageDB(object): @memoized def get_class_for_package_name(self, pkg_name): - """Get an instance of the class for a particular package. - - This method uses Python's ``imp`` package to load python - source from a Spack package's ``package.py`` file. A - normal python import would only load each package once, but - because we do this dynamically, the method needs to be - memoized to ensure there is only ONE package class - instance, per package, per database. - """ - file_path = self.filename_for_package_name(pkg_name) + """Get an instance of the class for a particular package.""" + repo = self.repo_for_package_name(pkg_name) + module_name = imported_packages_module + '.' + repo[0] + '.' + pkg_name - if os.path.exists(file_path): - if not os.path.isfile(file_path): - tty.die("Something's wrong. '%s' is not a file!" % file_path) - if not os.access(file_path, os.R_OK): - tty.die("Cannot read '%s'!" % file_path) - else: - raise UnknownPackageError(pkg_name) + module = self.repo_loaders[repo[0]].get_module(pkg_name) class_name = mod_to_class(pkg_name) - try: - module_name = _imported_packages_module + '.' + pkg_name - module = imp.load_source(module_name, file_path) - - except ImportError, e: - tty.die("Error while importing %s from %s:\n%s" % ( - pkg_name, file_path, e.message)) - cls = getattr(module, class_name) if not inspect.isclass(cls): tty.die("%s.%s is not a class" % (pkg_name, class_name)) diff --git a/lib/spack/spack/repo_loader.py b/lib/spack/spack/repo_loader.py new file mode 100644 index 0000000000..57c19a6c28 --- /dev/null +++ b/lib/spack/spack/repo_loader.py @@ -0,0 +1,115 @@ +import spack +import spack.repos +import re +import types +from llnl.util.lang import * + +# Name of module under which packages are imported +imported_packages_module = 'spack.repos' + +# Name of the package file inside a package directory +package_file_name = 'package.py' + +import sys +class LazyLoader: + """The LazyLoader handles cases when repo modules or classes + are imported. It watches for 'spack.repos.*' loads, then + redirects the load to the appropriate module.""" + def find_module(self, fullname, pathname): + if not fullname.startswith(imported_packages_module): + return None + partial_name = fullname[len(imported_packages_module)+1:] + repo = partial_name.split('.')[0] + module = partial_name.split('.')[1] + repo_loader = spack.db.repo_loaders.get(repo) + if repo_loader: + try: + self.mod = repo_loader.get_module(module) + return self + except (ImportError, spack.packages.UnknownPackageError): + return None + + def load_module(self, fullname): + return self.mod + +sys.meta_path.append(LazyLoader()) + +_reponames = {} +class RepoNamespace(types.ModuleType): + """The RepoNamespace holds the repository namespaces under + spack.repos. For example, when accessing spack.repos.original + this class will use __getattr__ to translate the 'original' + into one of spack's known repositories""" + def __init__(self): + import sys + sys.modules[imported_packages_module] = self + + def __getattr__(self, name): + if name in _reponames: + return _reponames[name] + raise AttributeError + + @property + def __file__(self): + return None + + @property + def __path__(self): + return [] + + +class RepoLoader(types.ModuleType): + """Each RepoLoader is associated with a repository, and the RepoLoader is + responsible for loading packages out of that repository. For example, + a RepoLoader may be responsible for spack.repos.original, and when someone + references spack.repos.original.libelf that RepoLoader will load the + libelf package.""" + def __init__(self, reponame, repopath): + self.path = repopath + self.reponame = reponame + self.module_name = imported_packages_module + '.' + reponame + if not reponame in _reponames: + _reponames[reponame] = self + spack.repos.add_repo(reponame, self) + + import sys + sys.modules[self.module_name] = self + + + @property + def __path__(self): + return [ self.path ] + + + def __getattr__(self, name): + if name[0] == '_': + raise AttributeError + return self.get_module(name) + + + @memoized + def get_module(self, pkg_name): + import os + import imp + import llnl.util.tty as tty + + file_path = os.path.join(self.path, pkg_name, package_file_name) + if os.path.exists(file_path): + if not os.path.isfile(file_path): + tty.die("Something's wrong. '%s' is not a file!" % file_path) + if not os.access(file_path, os.R_OK): + tty.die("Cannot read '%s'!" % file_path) + else: + raise spack.packages.UnknownPackageError(pkg_name) + + try: + module_name = imported_packages_module + '.' + self.reponame + '.' + pkg_name + module = imp.load_source(module_name, file_path) + + except ImportError, e: + tty.die("Error while importing %s from %s:\n%s" % ( + pkg_name, file_path, e.message)) + + return module + + -- cgit v1.2.3-70-g09d2 From 7ea328659f6bbaff33319427ba7c6321cc33a637 Mon Sep 17 00:00:00 2001 From: Matthew LeGendre Date: Mon, 20 Apr 2015 13:27:03 -0700 Subject: Record package repo origins in .spec files --- lib/spack/spack/packages.py | 35 ++++++++++++++++++++--------------- lib/spack/spack/repo_loader.py | 2 +- lib/spack/spack/spec.py | 29 +++++++++++++++++++++++++---- 3 files changed, 46 insertions(+), 20 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/packages.py b/lib/spack/spack/packages.py index 79dbd60703..48705948b7 100644 --- a/lib/spack/spack/packages.py +++ b/lib/spack/spack/packages.py @@ -73,12 +73,12 @@ class PackageDB(object): dir1 = list(dups)[0][1] dir2 = dict(s)[reponame] tty.die("Package repo %s in directory %s has the same name as the " - "repo in directory %s" % + "repo in directory %s" % (reponame, dir1, dir2)) # For each repo, create a RepoLoader self.repo_loaders = dict([(r[0], RepoLoader(r[0], r[1])) for r in self.repos]) - + self.instances = {} self.provider_index = None @@ -87,13 +87,13 @@ class PackageDB(object): """For a packagerepo directory, read the repo name from the dir/reponame file""" path = os.path.join(dir, 'reponame') - try: + try: with closing(open(path, 'r')) as reponame_file: - name = reponame_file.read().lstrip().rstrip() + name = reponame_file.read().lstrip().rstrip() if not re.match(r'[a-zA-Z][a-zA-Z0-9]+', name): tty.die("Package repo name '%s', read from %s, is an invalid name. " "Repo names must began with a letter and only contain letters " - "and numbers." % (name, path)) + "and numbers." % (name, path)) return name except exceptions.IOError, e: tty.die("Could not read from package repo name file %s" % path) @@ -107,7 +107,7 @@ class PackageDB(object): dir_string = config.get('packagerepo', 'directories') return dir_string.split(':') - + @_autospec def get(self, spec, **kwargs): if spec.virtual: @@ -118,7 +118,7 @@ class PackageDB(object): del self.instances[spec] if not spec in self.instances: - package_class = self.get_class_for_package_name(spec.name) + package_class = self.get_class_for_package_name(spec.name, spec.repo) try: copy = spec.copy() self.instances[copy] = package_class(copy) @@ -191,7 +191,7 @@ class PackageDB(object): path = join_path(pkgrepo[1], pkg_name) if os.path.exists(path): return (pkgrepo[0], path) - + repo_to_add_to = roots[-1] return (repo_to_add_to[0], join_path(repo_to_add_to[1], pkg_name)) @@ -259,7 +259,7 @@ class PackageDB(object): if os.path.isfile(pkg_file): all_packages.add(pkg_name) all_package_names = list(all_packages) - all_package_names.sort() + all_package_names.sort() return all_package_names @@ -275,12 +275,12 @@ class PackageDB(object): @memoized - def get_class_for_package_name(self, pkg_name): + def get_class_for_package_name(self, pkg_name, reponame = None): """Get an instance of the class for a particular package.""" - repo = self.repo_for_package_name(pkg_name) - module_name = imported_packages_module + '.' + repo[0] + '.' + pkg_name + (reponame, repodir) = self.repo_for_package_name(pkg_name, reponame) + module_name = imported_packages_module + '.' + reponame + '.' + pkg_name - module = self.repo_loaders[repo[0]].get_module(pkg_name) + module = self.repo_loaders[reponame].get_module(pkg_name) class_name = mod_to_class(pkg_name) cls = getattr(module, class_name) @@ -292,8 +292,13 @@ class PackageDB(object): class UnknownPackageError(spack.error.SpackError): """Raised when we encounter a package spack doesn't have.""" - def __init__(self, name): - super(UnknownPackageError, self).__init__("Package '%s' not found." % name) + def __init__(self, name, repo=None): + msg = None + if repo: + msg = "Package %s not found in packagerepo %s." % (name, repo) + else: + msg = "Package %s not found." % name + super(UnknownPackageError, self).__init__(msg) self.name = name diff --git a/lib/spack/spack/repo_loader.py b/lib/spack/spack/repo_loader.py index 57c19a6c28..6eaa1eead2 100644 --- a/lib/spack/spack/repo_loader.py +++ b/lib/spack/spack/repo_loader.py @@ -100,7 +100,7 @@ class RepoLoader(types.ModuleType): if not os.access(file_path, os.R_OK): tty.die("Cannot read '%s'!" % file_path) else: - raise spack.packages.UnknownPackageError(pkg_name) + raise spack.packages.UnknownPackageError(pkg_name, self.reponame if self.reponame != 'original' else None) try: module_name = imported_packages_module + '.' + self.reponame + '.' + pkg_name diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index e1fbb84423..972ba9ccbb 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -112,6 +112,7 @@ from spack.version import * from spack.util.string import * from spack.util.prefix import Prefix from spack.virtual import ProviderIndex +from spack.repo_loader import imported_packages_module # Valid pattern for an identifier in Spack identifier_re = r'\w[\w-]*' @@ -412,6 +413,7 @@ class Spec(object): self.dependencies = other.dependencies self.variants = other.variants self.variants.spec = self + self.repo = other.repo # Specs are by default not assumed to be normal, but in some # cases we've read them from a file want to assume normal. @@ -1355,6 +1357,7 @@ class Spec(object): self.dependencies = DependencyMap() self.variants = other.variants.copy() self.variants.spec = self + self.repo = other.repo # If we copy dependencies, preserve DAG structure in the new spec if kwargs.get('deps', True): @@ -1503,6 +1506,7 @@ class Spec(object): in the format string. The format strings you can provide are:: $_ Package name + $. Long package name $@ Version $% Compiler $%@ Compiler & compiler version @@ -1550,6 +1554,9 @@ class Spec(object): if c == '_': out.write(fmt % self.name) + elif c == '.': + longname = '%s.%s.%s' % (imported_packages_module, self.repo, self.name) if self.repo else self.name + out.write(fmt % longname) elif c == '@': if self.versions and self.versions != _any_version: write(fmt % (c + str(self.versions)), c) @@ -1698,17 +1705,29 @@ class SpecParser(spack.parse.Parser): def spec(self): """Parse a spec out of the input. If a spec is supplied, then initialize and return it instead of creating a new one.""" - self.check_identifier() + + spec_name = None + spec_repo = None + if self.token.value.startswith(imported_packages_module): + lst = self.token.value.split('.') + spec_name = lst[-1] + spec_repo = lst[-2] + else: + spec_name = self.token.value + (spec_repo, repodir) = spack.db.repo_for_package_name(spec_name) + + self.check_identifier(spec_name) # This will init the spec without calling __init__. spec = Spec.__new__(Spec) - spec.name = self.token.value + spec.name = spec_name spec.versions = VersionList() spec.variants = VariantMap(spec) spec.architecture = None spec.compiler = None spec.dependents = DependencyMap() spec.dependencies = DependencyMap() + spec.repo = spec_repo spec._normal = False spec._concrete = False @@ -1802,12 +1821,14 @@ class SpecParser(spack.parse.Parser): return compiler - def check_identifier(self): + def check_identifier(self, id=None): """The only identifiers that can contain '.' are versions, but version ids are context-sensitive so we have to check on a case-by-case basis. Call this if we detect a version id where it shouldn't be. """ - if '.' in self.token.value: + if not id: + id = self.token.value + if '.' in id: self.last_token_error("Identifier cannot contain '.'") -- cgit v1.2.3-70-g09d2 From e58ee88a632a0855a984cba8f0faa725b8f2eddf Mon Sep 17 00:00:00 2001 From: Matthew LeGendre Date: Mon, 20 Apr 2015 13:50:09 -0700 Subject: Add 'spack packagerepo create' command --- lib/spack/spack/cmd/packagerepo.py | 51 +++++++++++++++++++++++++++++++++----- lib/spack/spack/packages.py | 4 +-- 2 files changed, 47 insertions(+), 8 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/cmd/packagerepo.py b/lib/spack/spack/cmd/packagerepo.py index 66bad0ecbf..2819d0f980 100644 --- a/lib/spack/spack/cmd/packagerepo.py +++ b/lib/spack/spack/cmd/packagerepo.py @@ -23,15 +23,20 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## from external import argparse - import llnl.util.tty as tty from llnl.util.tty.color import colorize from llnl.util.tty.colify import colify from llnl.util.lang import index_by +from llnl.util.filesystem import join_path, mkdirp import spack.spec import spack.config from spack.util.environment import get_path +from spack.packages import packagerepo_filename + +import os +import exceptions +from contextlib import closing description = "Manage package sources" @@ -41,6 +46,10 @@ def setup_parser(subparser): add_parser = sp.add_parser('add', help=packagerepo_add.__doc__) add_parser.add_argument('directory', help="Directory containing the packages.") + + create_parser = sp.add_parser('create', help=packagerepo_create.__doc__) + create_parser.add_argument('directory', help="Directory containing the packages.") + create_parser.add_argument('name', help="Name of new package repository.") remove_parser = sp.add_parser('remove', help=packagerepo_remove.__doc__) remove_parser.add_argument('name') @@ -48,19 +57,48 @@ def setup_parser(subparser): list_parser = sp.add_parser('list', help=packagerepo_list.__doc__) -def packagerepo_add(args): - """Add package sources to the Spack configuration.""" +def add_to_config(dir): config = spack.config.get_config() user_config = spack.config.get_config('user') orig = None if config.has_value('packagerepo', '', 'directories'): orig = config.get_value('packagerepo', '', 'directories') - if orig and args.directory in orig.split(':'): - tty.die('Repo directory %s already exists in the repo list' % args.directory) + if orig and dir in orig.split(':'): + return False - newsetting = orig + ':' + args.directory if orig else args.directory + newsetting = orig + ':' + dir if orig else dir user_config.set_value('packagerepo', '', 'directories', newsetting) user_config.write() + return True + + +def packagerepo_add(args): + """Add package sources to the Spack configuration.""" + if not add_to_config(args.directory): + tty.die('Repo directory %s already exists in the repo list' % dir) + + +def packagerepo_create(args): + """Create a new package repo at a directory and name""" + dir = args.directory + name = args.name + + if os.path.exists(dir) and not os.path.isdir(dir): + tty.die('File %s already exists and is not a directory' % dir) + if not os.path.exists(dir): + try: + mkdirp(dir) + except exceptions.OSError, e: + tty.die('Failed to create new directory %s' % dir) + path = os.path.join(dir, packagerepo_filename) + try: + with closing(open(path, 'w')) as repofile: + repofile.write(name + '\n') + except exceptions.IOError, e: + tty.die('Could not create new file %s' % path) + + if not add_to_config(args.directory): + tty.warn('Repo directory %s already exists in the repo list' % dir) def packagerepo_remove(args): @@ -80,6 +118,7 @@ def packagerepo_list(args): def packagerepo(parser, args): action = { 'add' : packagerepo_add, + 'create' : packagerepo_create, 'remove' : packagerepo_remove, 'list' : packagerepo_list } action[args.packagerepo_command](args) diff --git a/lib/spack/spack/packages.py b/lib/spack/spack/packages.py index 48705948b7..3b9d74dd6e 100644 --- a/lib/spack/spack/packages.py +++ b/lib/spack/spack/packages.py @@ -44,7 +44,7 @@ from sets import Set from spack.repo_loader import RepoLoader, imported_packages_module, package_file_name # Filename for package repo names -_packagerepo_filename = 'reponame' +packagerepo_filename = 'reponame' def _autospec(function): """Decorator that automatically converts the argument of a single-arg @@ -85,7 +85,7 @@ class PackageDB(object): def _read_reponame_from_directory(self, dir): """For a packagerepo directory, read the repo name from the dir/reponame file""" - path = os.path.join(dir, 'reponame') + path = os.path.join(dir, packagerepo_filename) try: with closing(open(path, 'r')) as reponame_file: -- cgit v1.2.3-70-g09d2 From da98b07624e2403807166e8a9d0dac3752f75c0f Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 6 Jul 2015 19:57:31 -0700 Subject: Add more options to `spack edit` --- lib/spack/spack/cmd/edit.py | 35 +++++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 14 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/cmd/edit.py b/lib/spack/spack/cmd/edit.py index b8764ba391..9081d12516 100644 --- a/lib/spack/spack/cmd/edit.py +++ b/lib/spack/spack/cmd/edit.py @@ -78,9 +78,18 @@ def setup_parser(subparser): subparser.add_argument( '-f', '--force', dest='force', action='store_true', help="Open a new file in $EDITOR even if package doesn't exist.") - subparser.add_argument( - '-c', '--command', dest='edit_command', action='store_true', - help="Edit the command with the supplied name instead of a package.") + + filetypes = subparser.add_mutually_exclusive_group() + filetypes.add_argument( + '-c', '--command', dest='path', action='store_const', + const=spack.cmd.command_path, help="Edit the command with the supplied name.") + filetypes.add_argument( + '-t', '--test', dest='path', action='store_const', + const=spack.test_path, help="Edit the test with the supplied name.") + filetypes.add_argument( + '-m', '--module', dest='path', action='store_const', + const=spack.module_path, help="Edit the main spack module with the supplied name.") + subparser.add_argument( 'name', nargs='?', default=None, help="name of package to edit") @@ -88,19 +97,17 @@ def setup_parser(subparser): def edit(parser, args): name = args.name - if args.edit_command: - if not name: - path = spack.cmd.command_path - else: - path = join_path(spack.cmd.command_path, name + ".py") - if not os.path.exists(path): + path = spack.packages_path + if args.path: + path = args.path + if name: + path = join_path(path, name + ".py") + if not args.force and not os.path.exists(path): tty.die("No command named '%s'." % name) spack.editor(path) + elif name: + edit_package(name, args.force) else: # By default open the directory where packages or commands live. - if not name: - path = spack.packages_path - spack.editor(path) - else: - edit_package(name, args.force) + spack.editor(path) -- cgit v1.2.3-70-g09d2 From 92f398a897949e3b586abc38a934ae9d5cf1163c Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 7 Jul 2015 00:25:20 -0700 Subject: Better `@memoized` decorator. --- lib/spack/llnl/util/lang.py | 33 +++++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 8 deletions(-) (limited to 'lib') diff --git a/lib/spack/llnl/util/lang.py b/lib/spack/llnl/util/lang.py index 9e1bef18ca..be6dad867e 100644 --- a/lib/spack/llnl/util/lang.py +++ b/lib/spack/llnl/util/lang.py @@ -26,6 +26,7 @@ import os import re import sys import functools +import collections import inspect # Ignore emacs backups when listing modules @@ -170,16 +171,32 @@ def has_method(cls, name): return False -def memoized(obj): +class memoized(object): """Decorator that caches the results of a function, storing them in an attribute of that function.""" - cache = obj.cache = {} - @functools.wraps(obj) - def memoizer(*args, **kwargs): - if args not in cache: - cache[args] = obj(*args, **kwargs) - return cache[args] - return memoizer + def __init__(self, func): + self.func = func + self.cache = {} + + + def __call__(self, *args): + if not isinstance(args, collections.Hashable): + # Not hashable, so just call the function. + return self.func(*args) + + if args not in self.cache: + self.cache[args] = self.func(*args) + return self.cache[args] + + + def __get__(self, obj, objtype): + """Support instance methods.""" + return functools.partial(self.__call__, obj) + + + def clear(self): + """Expunge cache so that self.func will be called again.""" + self.cache.clear() def list_modules(directory, **kwargs): -- cgit v1.2.3-70-g09d2 From a2f2e6a4ff60882d8a93754ee10e8e75245cf430 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 19 Jul 2015 21:55:27 -0700 Subject: Save changes to external repo integration --- lib/spack/spack/__init__.py | 16 +-- lib/spack/spack/cmd/packagerepo.py | 124 ---------------------- lib/spack/spack/cmd/repo.py | 129 +++++++++++++++++++++++ lib/spack/spack/config.py | 206 +++++++++++++++++++++++-------------- lib/spack/spack/packages.py | 117 +++++++++++++-------- lib/spack/spack/repo_loader.py | 35 +++++-- lib/spack/spack/test/config.py | 82 +++++++++++---- var/spack/mock_packages/repo.yaml | 2 + var/spack/packages/repo.yaml | 2 + var/spack/packages/reponame | 1 - 10 files changed, 430 insertions(+), 284 deletions(-) delete mode 100644 lib/spack/spack/cmd/packagerepo.py create mode 100644 lib/spack/spack/cmd/repo.py create mode 100644 var/spack/mock_packages/repo.yaml create mode 100644 var/spack/packages/repo.yaml delete mode 100644 var/spack/packages/reponame (limited to 'lib') diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index 1d67b45341..09bc9ca52a 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -47,11 +47,17 @@ install_path = join_path(opt_path, "spack") share_path = join_path(prefix, "share", "spack") # -# Set up the packages database. +# Setup the spack.repos namespace +# +from spack.repo_loader import RepoNamespace +repos = RepoNamespace() + +# +# Set up the default packages database. # from spack.packages import PackageDB packages_path = join_path(var_path, "packages") -db = PackageDB(packages_path) +db = PackageDB() # # Paths to mock files for testing. @@ -62,12 +68,6 @@ mock_config_path = join_path(var_path, "mock_configs") mock_site_config = join_path(mock_config_path, "site_spackconfig") mock_user_config = join_path(mock_config_path, "user_spackconfig") -# -# Setup the spack.repos namespace -# -from spack.repo_loader import RepoNamespace -repos = RepoNamespace() - # # This controls how spack lays out install prefixes and # stage directories. diff --git a/lib/spack/spack/cmd/packagerepo.py b/lib/spack/spack/cmd/packagerepo.py deleted file mode 100644 index 2819d0f980..0000000000 --- a/lib/spack/spack/cmd/packagerepo.py +++ /dev/null @@ -1,124 +0,0 @@ -############################################################################## -# Copyright (c) 2013, Lawrence Livermore National Security, LLC. -# Produced at the Lawrence Livermore National Laboratory. -# -# This file is part of Spack. -# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. -# LLNL-CODE-647188 -# -# For details, see https://scalability-llnl.github.io/spack -# Please also see the LICENSE file for our notice and the LGPL. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License (as published by -# the Free Software Foundation) version 2.1 dated February 1999. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and -# conditions of the GNU General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -############################################################################## -from external import argparse -import llnl.util.tty as tty -from llnl.util.tty.color import colorize -from llnl.util.tty.colify import colify -from llnl.util.lang import index_by -from llnl.util.filesystem import join_path, mkdirp - -import spack.spec -import spack.config -from spack.util.environment import get_path -from spack.packages import packagerepo_filename - -import os -import exceptions -from contextlib import closing - -description = "Manage package sources" - -def setup_parser(subparser): - sp = subparser.add_subparsers( - metavar='SUBCOMMAND', dest='packagerepo_command') - - add_parser = sp.add_parser('add', help=packagerepo_add.__doc__) - add_parser.add_argument('directory', help="Directory containing the packages.") - - create_parser = sp.add_parser('create', help=packagerepo_create.__doc__) - create_parser.add_argument('directory', help="Directory containing the packages.") - create_parser.add_argument('name', help="Name of new package repository.") - - remove_parser = sp.add_parser('remove', help=packagerepo_remove.__doc__) - remove_parser.add_argument('name') - - list_parser = sp.add_parser('list', help=packagerepo_list.__doc__) - - -def add_to_config(dir): - config = spack.config.get_config() - user_config = spack.config.get_config('user') - orig = None - if config.has_value('packagerepo', '', 'directories'): - orig = config.get_value('packagerepo', '', 'directories') - if orig and dir in orig.split(':'): - return False - - newsetting = orig + ':' + dir if orig else dir - user_config.set_value('packagerepo', '', 'directories', newsetting) - user_config.write() - return True - - -def packagerepo_add(args): - """Add package sources to the Spack configuration.""" - if not add_to_config(args.directory): - tty.die('Repo directory %s already exists in the repo list' % dir) - - -def packagerepo_create(args): - """Create a new package repo at a directory and name""" - dir = args.directory - name = args.name - - if os.path.exists(dir) and not os.path.isdir(dir): - tty.die('File %s already exists and is not a directory' % dir) - if not os.path.exists(dir): - try: - mkdirp(dir) - except exceptions.OSError, e: - tty.die('Failed to create new directory %s' % dir) - path = os.path.join(dir, packagerepo_filename) - try: - with closing(open(path, 'w')) as repofile: - repofile.write(name + '\n') - except exceptions.IOError, e: - tty.die('Could not create new file %s' % path) - - if not add_to_config(args.directory): - tty.warn('Repo directory %s already exists in the repo list' % dir) - - -def packagerepo_remove(args): - """Remove a package source from the Spack configuration""" - pass - - -def packagerepo_list(args): - """List package sources and their mnemoics""" - root_names = spack.db.repos - max_len = max(len(s[0]) for s in root_names) - fmt = "%%-%ds%%s" % (max_len + 4) - for root in root_names: - print fmt % (root[0], root[1]) - - - -def packagerepo(parser, args): - action = { 'add' : packagerepo_add, - 'create' : packagerepo_create, - 'remove' : packagerepo_remove, - 'list' : packagerepo_list } - action[args.packagerepo_command](args) diff --git a/lib/spack/spack/cmd/repo.py b/lib/spack/spack/cmd/repo.py new file mode 100644 index 0000000000..1261c7ada9 --- /dev/null +++ b/lib/spack/spack/cmd/repo.py @@ -0,0 +1,129 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from external import argparse +import llnl.util.tty as tty +from llnl.util.tty.color import colorize +from llnl.util.tty.colify import colify +from llnl.util.lang import index_by +from llnl.util.filesystem import join_path, mkdirp + +import spack.spec +import spack.config +from spack.util.environment import get_path +from spack.packages import repo_config + +import os +import exceptions +from contextlib import closing + +description = "Manage package sources" + +def setup_parser(subparser): + sp = subparser.add_subparsers( + metavar='SUBCOMMAND', dest='repo_command') + + add_parser = sp.add_parser('add', help=repo_add.__doc__) + add_parser.add_argument('directory', help="Directory containing the packages.") + + create_parser = sp.add_parser('create', help=repo_create.__doc__) + create_parser.add_argument('directory', help="Directory containing the packages.") + create_parser.add_argument('name', help="Name of new package repository.") +<<<<<<< HEAD:lib/spack/spack/cmd/packagerepo.py + + remove_parser = sp.add_parser('remove', help=packagerepo_remove.__doc__) +======= + + remove_parser = sp.add_parser('remove', help=repo_remove.__doc__) +>>>>>>> Save changes to external repo integration:lib/spack/spack/cmd/repo.py + remove_parser.add_argument('name') + + list_parser = sp.add_parser('list', help=repo_list.__doc__) + + +def add_to_config(dir): + config = spack.config.get_config() + user_config = spack.config.get_config('user') + orig = None + if config.has_value('repo', '', 'directories'): + orig = config.get_value('repo', '', 'directories') + if orig and dir in orig.split(':'): + return False + + newsetting = orig + ':' + dir if orig else dir + user_config.set_value('repo', '', 'directories', newsetting) + user_config.write() + return True + + +def repo_add(args): + """Add package sources to the Spack configuration.""" + if not add_to_config(args.directory): + tty.die('Repo directory %s already exists in the repo list' % dir) + + +def repo_create(args): + """Create a new package repo at a directory and name""" + dir = args.directory + name = args.name + + if os.path.exists(dir) and not os.path.isdir(dir): + tty.die('File %s already exists and is not a directory' % dir) + if not os.path.exists(dir): + try: + mkdirp(dir) + except exceptions.OSError, e: + tty.die('Failed to create new directory %s' % dir) + path = os.path.join(dir, repo_config) + try: + with closing(open(path, 'w')) as repofile: + repofile.write(name + '\n') + except exceptions.IOError, e: + tty.die('Could not create new file %s' % path) + + if not add_to_config(args.directory): + tty.warn('Repo directory %s already exists in the repo list' % dir) + + +def repo_remove(args): + """Remove a package source from the Spack configuration""" + pass + + +def repo_list(args): + """List package sources and their mnemoics""" + root_names = spack.db.repos + max_len = max(len(s[0]) for s in root_names) + fmt = "%%-%ds%%s" % (max_len + 4) + for root in root_names: + print fmt % (root[0], root[1]) + + + +def repo(parser, args): + action = { 'add' : repo_add, + 'create' : repo_create, + 'remove' : repo_remove, + 'list' : repo_list } + action[args.repo_command](args) diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 3e91958c2c..dc59f9a5a3 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -45,11 +45,11 @@ several configuration files, such as compilers.yaml or mirrors.yaml. Configuration file format =============================== -Configuration files are formatted using YAML syntax. -This format is implemented by Python's -yaml class, and it's easy to read and versatile. +Configuration files are formatted using YAML syntax. This format is +implemented by libyaml (included with Spack as an external module), +and it's easy to read and versatile. -The config files are structured as trees, like this ``compiler`` section:: +Config files are structured as trees, like this ``compiler`` section:: compilers: chaos_5_x86_64_ib: @@ -83,62 +83,73 @@ would looks like: } } -Some routines, like get_mirrors_config and get_compilers_config may strip -off the top-levels of the tree and return subtrees. +Some convenience functions, like get_mirrors_config and +``get_compilers_config`` may strip off the top-levels of the tree and +return subtrees. + """ import os -import exceptions import sys - -from external.ordereddict import OrderedDict -from llnl.util.lang import memoized -import spack.error - +import copy from external import yaml from external.yaml.error import MarkedYAMLError + import llnl.util.tty as tty from llnl.util.filesystem import mkdirp +from llnl.util.lang import memoized + +import spack + _config_sections = {} class _ConfigCategory: name = None filename = None merge = True - def __init__(self, n, f, m): - self.name = n - self.filename = f - self.merge = m + def __init__(self, name, filename, merge, strip): + self.name = name + self.filename = filename + self.merge = merge + self.strip = strip self.files_read_from = [] self.result_dict = {} - _config_sections[n] = self + _config_sections[name] = self -_ConfigCategory('compilers', 'compilers.yaml', True) -_ConfigCategory('mirrors', 'mirrors.yaml', True) -_ConfigCategory('view', 'views.yaml', True) -_ConfigCategory('order', 'orders.yaml', True) +_ConfigCategory('config', 'config.yaml', True, False) +_ConfigCategory('compilers', 'compilers.yaml', True, True) +_ConfigCategory('mirrors', 'mirrors.yaml', True, True) +_ConfigCategory('view', 'views.yaml', True, True) +_ConfigCategory('order', 'orders.yaml', True, True) """Names of scopes and their corresponding configuration files.""" config_scopes = [('site', os.path.join(spack.etc_path, 'spack')), ('user', os.path.expanduser('~/.spack'))] _compiler_by_arch = {} -_read_config_file_result = {} + +@memoized def _read_config_file(filename): - """Read a given YAML configuration file""" - global _read_config_file_result - if filename in _read_config_file_result: - return _read_config_file_result[filename] + """Read a YAML configuration file""" + + # Ignore nonexisting files. + if not os.path.exists(filename): + return None + + elif not os.path.isfile(filename): + tty.die("Invlaid configuration. %s exists but is not a file." % filename) + + elif not os.access(filename, os.R_OK): + tty.die("Configuration file %s is not readable." % filename) try: with open(filename) as f: - ydict = yaml.load(f) + return yaml.load(f) + except MarkedYAMLError, e: tty.die("Error parsing yaml%s: %s" % (str(e.context_mark), e.problem)) - except exceptions.IOError, e: - _read_config_file_result[filename] = None - return None - _read_config_file_result[filename] = ydict - return ydict + + except IOError, e: + tty.die("Error reading configuration file %s: %s" % (filename, str(e))) def clear_config_caches(): @@ -147,41 +158,66 @@ def clear_config_caches(): for key,s in _config_sections.iteritems(): s.files_read_from = [] s.result_dict = {} - spack.config._read_config_file_result = {} + + _read_config_file.clear() spack.config._compiler_by_arch = {} spack.compilers._cached_default_compiler = None -def _merge_dicts(d1, d2): - """Recursively merges two configuration trees, with entries - in d2 taking precedence over d1""" - if not d1: - return d2.copy() - if not d2: - return d1 +def _merge_yaml(dest, source): + """Merges source into dest; entries in source take precedence over dest. - for key2, val2 in d2.iteritems(): - if not key2 in d1: - d1[key2] = val2 - continue - val1 = d1[key2] - if isinstance(val1, dict) and isinstance(val2, dict): - d1[key2] = _merge_dicts(val1, val2) - continue - if isinstance(val1, list) and isinstance(val2, list): - val1.extend(val2) - seen = set() - d1[key2] = [ x for x in val1 if not (x in seen or seen.add(x)) ] - continue - d1[key2] = val2 - return d1 + Config file authors can optionally end any attribute in a dict + with `::` instead of `:`, and the key will override that of the + parent instead of merging. + """ + def they_are(t): + return isinstance(dest, t) and isinstance(source, t) + # If both are None, handle specially and return None. + if source is None and dest is None: + return None -def get_config(category_name): - """Get the confguration tree for the names category. Strips off the - top-level category entry from the dict""" - global config_scopes - category = _config_sections[category_name] + # If source is None, overwrite with source. + elif source is None: + return None + + # Source list is prepended (for precedence) + if they_are(list): + seen = set(source) + dest[:] = source + [x for x in dest if x not in seen] + return dest + + # Source dict is merged into dest. Extra ':' means overwrite. + elif they_are(dict): + for sk, sv in source.iteritems(): + # allow total override with, e.g., repos:: + override = sk.endswith(':') + if override: + sk = sk.rstrip(':') + + if override or not sk in dest: + dest[sk] = copy.copy(sv) + else: + dest[sk] = _merge_yaml(dest[sk], source[sk]) + return dest + + # In any other case, overwrite with a copy of the source value. + else: + return copy.copy(source) + + +def substitute_spack_prefix(path): + """Replaces instances of $spack with Spack's prefix.""" + return path.replace('$spack', spack.prefix) + + +def get_config(category='config'): + """Get the confguration tree for a category. + + Strips off the top-level category entry from the dict + """ + category = _config_sections[category] if category.result_dict: return category.result_dict @@ -191,14 +227,18 @@ def get_config(category_name): result = _read_config_file(path) if not result: continue - if not category_name in result: - continue + + if category.strip: + if not category.name in result: + continue + result = result[category.name] + category.files_read_from.insert(0, path) - result = result[category_name] if category.merge: - category.result_dict = _merge_dicts(category.result_dict, result) + category.result_dict = _merge_yaml(category.result_dict, result) else: category.result_dict = result + return category.result_dict @@ -215,7 +255,7 @@ def get_compilers_config(arch=None): cc_config = get_config('compilers') if arch in cc_config and 'all' in cc_config: arch_compiler = dict(cc_config[arch]) - _compiler_by_arch[arch] = _merge_dict(arch_compiler, cc_config['all']) + _compiler_by_arch[arch] = _merge_yaml(arch_compiler, cc_config['all']) elif arch in cc_config: _compiler_by_arch[arch] = cc_config[arch] elif 'all' in cc_config: @@ -225,6 +265,13 @@ def get_compilers_config(arch=None): return _compiler_by_arch[arch] +def get_repos_config(): + config = get_config() + if 'repos' not in config: + return [] + return config['repos'] + + def get_mirror_config(): """Get the mirror configuration from config files""" return get_config('mirrors') @@ -232,7 +279,6 @@ def get_mirror_config(): def get_config_scope_dirname(scope): """For a scope return the config directory""" - global config_scopes for s,p in config_scopes: if s == scope: return p @@ -251,16 +297,16 @@ def get_config_scope_filename(scope, category_name): def add_to_config(category_name, addition_dict, scope=None): """Merge a new dict into a configuration tree and write the new configuration to disk""" - global _read_config_file_result get_config(category_name) category = _config_sections[category_name] - #If scope is specified, use it. Otherwise use the last config scope that - #we successfully parsed data from. + # If scope is specified, use it. Otherwise use the last config scope that + # we successfully parsed data from. file = None path = None if not scope and not category.files_read_from: scope = 'user' + if scope: try: dir = get_config_scope_dirname(scope) @@ -268,32 +314,37 @@ def add_to_config(category_name, addition_dict, scope=None): mkdirp(dir) path = os.path.join(dir, category.filename) file = open(path, 'w') - except exceptions.IOError, e: + except IOError, e: pass else: for p in category.files_read_from: try: file = open(p, 'w') - except exceptions.IOError, e: + except IOError, e: pass if file: path = p break; + if not file: tty.die('Unable to write to config file %s' % path) - #Merge the new information into the existing file info, then write to disk - new_dict = _read_config_file_result[path] + # Merge the new information into the existing file info, then write to disk + new_dict = _read_config_file(path) + if new_dict and category_name in new_dict: new_dict = new_dict[category_name] - new_dict = _merge_dicts(new_dict, addition_dict) + + new_dict = _merge_yaml(new_dict, addition_dict) new_dict = { category_name : new_dict } - _read_config_file_result[path] = new_dict + + # Install new dict as memoized value, and dump to disk + _read_config_file.cache[path] = new_dict yaml.dump(new_dict, stream=file, default_flow_style=False) file.close() - #Merge the new information into the cached results - category.result_dict = _merge_dicts(category.result_dict, addition_dict) + # Merge the new information into the cached results + category.result_dict = _merge_yaml(category.result_dict, addition_dict) def add_to_mirror_config(addition_dict, scope=None): @@ -311,7 +362,6 @@ def add_to_compiler_config(addition_dict, scope=None, arch=None): def remove_from_config(category_name, key_to_rm, scope=None): """Remove a configuration key and write a new configuration to disk""" - global config_scopes get_config(category_name) scopes_to_rm_from = [scope] if scope else [s for s,p in config_scopes] category = _config_sections[category_name] diff --git a/lib/spack/spack/packages.py b/lib/spack/spack/packages.py index 3b9d74dd6e..c414234386 100644 --- a/lib/spack/spack/packages.py +++ b/lib/spack/spack/packages.py @@ -30,7 +30,9 @@ import glob import imp import spack.config import re -from contextlib import closing +import itertools +import traceback +from external import yaml import llnl.util.tty as tty from llnl.util.filesystem import join_path @@ -44,7 +46,7 @@ from sets import Set from spack.repo_loader import RepoLoader, imported_packages_module, package_file_name # Filename for package repo names -packagerepo_filename = 'reponame' +repo_config = 'repo.yaml' def _autospec(function): """Decorator that automatically converts the argument of a single-arg @@ -56,56 +58,85 @@ def _autospec(function): return converter +def sliding_window(seq, n): + it = iter(seq) + result = tuple(itertools.islice(it, n)) + if len(result) == n: + yield result + for elem in it: + result = result[1:] + (elem,) + yield result + + class PackageDB(object): - def __init__(self, default_root): - """Construct a new package database from a root directory.""" - - #Collect the repos from the config file and read their names from the file system - repo_dirs = self._repo_list_from_config() - repo_dirs.append(default_root) - self.repos = [(self._read_reponame_from_directory(dir), dir) for dir in repo_dirs] - - # Check for duplicate repo names - s = set() - dups = set(r for r in self.repos if r[0] in s or s.add(r[0])) - if dups: - reponame = list(dups)[0][0] - dir1 = list(dups)[0][1] - dir2 = dict(s)[reponame] - tty.die("Package repo %s in directory %s has the same name as the " - "repo in directory %s" % - (reponame, dir1, dir2)) + def __init__(self, *repo_dirs): + """Construct a new package database from a list of directories. + + Args: + repo_dirs List of directories containing packages. + + If ``repo_dirs`` is empty, gets repository list from Spack configuration. + """ + if not repo_dirs: + repo_dirs = spack.config.get_repos_config() + if not repo_dirs: + tty.die("Spack configuration contains no package repositories.") + + # Collect the repos from the config file and read their names + # from the file system + repo_dirs = [spack.config.substitute_spack_prefix(rd) for rd in repo_dirs] + + self.repos = [] + for rdir in repo_dirs: + rname = self._read_reponame_from_directory(rdir) + if rname: + self.repos.append((self._read_reponame_from_directory(rdir), rdir)) + + + by_path = sorted(self.repos, key=lambda r:r[1]) + by_name = sorted(self.repos, key=lambda r:r[0]) + + for r1, r2 in by_path: + if r1[1] == r2[1]: + tty.die("Package repos are the same:", + " %20s %s" % r1, " %20s %s" % r2) + + for r1, r2 in by_name: + if r1[0] == r2[0]: + tty.die("Package repos cannot have the same name:", + " %20s %s" % r1, " %20s %s" % r2) # For each repo, create a RepoLoader - self.repo_loaders = dict([(r[0], RepoLoader(r[0], r[1])) for r in self.repos]) + self.repo_loaders = dict((name, RepoLoader(name, path)) + for name, path in self.repos) self.instances = {} self.provider_index = None def _read_reponame_from_directory(self, dir): - """For a packagerepo directory, read the repo name from the dir/reponame file""" - path = os.path.join(dir, packagerepo_filename) + """For a packagerepo directory, read the repo name from the + $root/repo.yaml file""" + path = os.path.join(dir, repo_config) try: - with closing(open(path, 'r')) as reponame_file: - name = reponame_file.read().lstrip().rstrip() - if not re.match(r'[a-zA-Z][a-zA-Z0-9]+', name): - tty.die("Package repo name '%s', read from %s, is an invalid name. " - "Repo names must began with a letter and only contain letters " - "and numbers." % (name, path)) + with open(path) as reponame_file: + yaml_data = yaml.load(reponame_file) + + if (not yaml_data or + 'repo' not in yaml_data or + 'namespace' not in yaml_data['repo']): + tty.die("Invalid %s in %s" % (repo_config, dir)) + + name = yaml_data['repo']['namespace'] + if not re.match(r'[a-zA-Z][a-zA-Z0-9_.]+', name): + tty.die( + "Package repo name '%s', read from %s, is an invalid name. " + "Repo names must began with a letter and only contain " + "letters and numbers." % (name, path)) return name except exceptions.IOError, e: - tty.die("Could not read from package repo name file %s" % path) - - - - def _repo_list_from_config(self): - """Read through the spackconfig and return the list of packagerepo directories""" - config = spack.config.get_config() - if not config.has_option('packagerepo', 'directories'): return [] - dir_string = config.get('packagerepo', 'directories') - return dir_string.split(':') + tty.die("Error reading %s when opening %s" % (repo_config, dir)) @_autospec @@ -125,7 +156,7 @@ class PackageDB(object): except Exception, e: if spack.debug: sys.excepthook(*sys.exc_info()) - raise FailedConstructorError(spec.name, e) + raise FailedConstructorError(spec.name, *sys.exc_info()) return self.instances[spec] @@ -304,8 +335,10 @@ class UnknownPackageError(spack.error.SpackError): class FailedConstructorError(spack.error.SpackError): """Raised when a package's class constructor fails.""" - def __init__(self, name, reason): + def __init__(self, name, exc_type, exc_obj, exc_tb): super(FailedConstructorError, self).__init__( "Class constructor failed for package '%s'." % name, - str(reason)) + '\nCaused by:\n' + + ('%s: %s\n' % (exc_type.__name__, exc_obj)) + + ''.join(traceback.format_tb(exc_tb))) self.name = name diff --git a/lib/spack/spack/repo_loader.py b/lib/spack/spack/repo_loader.py index 6eaa1eead2..92da1cf709 100644 --- a/lib/spack/spack/repo_loader.py +++ b/lib/spack/spack/repo_loader.py @@ -1,8 +1,10 @@ -import spack -import spack.repos import re +import sys import types +import traceback + from llnl.util.lang import * +import spack # Name of module under which packages are imported imported_packages_module = 'spack.repos' @@ -13,14 +15,30 @@ package_file_name = 'package.py' import sys class LazyLoader: """The LazyLoader handles cases when repo modules or classes - are imported. It watches for 'spack.repos.*' loads, then + are imported. It watches for 'spack.repos.*' loads, then redirects the load to the appropriate module.""" def find_module(self, fullname, pathname): if not fullname.startswith(imported_packages_module): return None + + print "HERE ===" + print + for line in traceback.format_stack(): + print " ", line.strip() + print + print "full: ", fullname + print "path: ", pathname + print + partial_name = fullname[len(imported_packages_module)+1:] - repo = partial_name.split('.')[0] - module = partial_name.split('.')[1] + + print "partial: ", partial_name + print + + last_dot = partial_name.rfind('.') + repo = partial_name[:last_dot] + module = partial_name[last_dot+1:] + repo_loader = spack.db.repo_loaders.get(repo) if repo_loader: try: @@ -43,7 +61,7 @@ class RepoNamespace(types.ModuleType): def __init__(self): import sys sys.modules[imported_packages_module] = self - + def __getattr__(self, name): if name in _reponames: return _reponames[name] @@ -62,7 +80,7 @@ class RepoLoader(types.ModuleType): """Each RepoLoader is associated with a repository, and the RepoLoader is responsible for loading packages out of that repository. For example, a RepoLoader may be responsible for spack.repos.original, and when someone - references spack.repos.original.libelf that RepoLoader will load the + references spack.repos.original.libelf that RepoLoader will load the libelf package.""" def __init__(self, reponame, repopath): self.path = repopath @@ -70,7 +88,6 @@ class RepoLoader(types.ModuleType): self.module_name = imported_packages_module + '.' + reponame if not reponame in _reponames: _reponames[reponame] = self - spack.repos.add_repo(reponame, self) import sys sys.modules[self.module_name] = self @@ -111,5 +128,3 @@ class RepoLoader(types.ModuleType): pkg_name, file_path, e.message)) return module - - diff --git a/lib/spack/spack/test/config.py b/lib/spack/spack/test/config.py index 790b22f3b0..eed182a257 100644 --- a/lib/spack/spack/test/config.py +++ b/lib/spack/spack/test/config.py @@ -30,45 +30,85 @@ import spack from spack.packages import PackageDB from spack.test.mock_packages_test import * +# Some sample compiler config data +a_comps = { + "gcc@4.7.3" : { + "cc" : "/gcc473", + "cxx" : "/g++473", + "f77" : None, + "f90" : None }, + "gcc@4.5.0" : { + "cc" : "/gcc450", + "cxx" : "/g++450", + "f77" : "/gfortran", + "f90" : "/gfortran" }, + "clang@3.3" : { + "cc" : "", + "cxx" : "", + "f77" : "", + "f90" : "" } +} + +b_comps = { + "icc@10.0" : { + "cc" : "/icc100", + "cxx" : "/icc100", + "f77" : None, + "f90" : None }, + "icc@11.1" : { + "cc" : "/icc111", + "cxx" : "/icp111", + "f77" : "/ifort", + "f90" : "/ifort" }, + "clang@3.3" : { + "cc" : "/clang", + "cxx" : "/clang++", + "f77" : None, + "f90" : None} +} + class ConfigTest(MockPackagesTest): def setUp(self): - self.initmock() + super(ConfigTest, self).setUp() self.tmp_dir = mkdtemp('.tmp', 'spack-config-test-') - spack.config.config_scopes = [('test_low_priority', os.path.join(self.tmp_dir, 'low')), - ('test_high_priority', os.path.join(self.tmp_dir, 'high'))] + spack.config.config_scopes = [ + ('test_low_priority', os.path.join(self.tmp_dir, 'low')), + ('test_high_priority', os.path.join(self.tmp_dir, 'high'))] + def tearDown(self): - self.cleanmock() + super(ConfigTest, self).tearDown() shutil.rmtree(self.tmp_dir, True) - def check_config(self, comps): + + def check_config(self, comps, *compiler_names): + """Check that named compilers in comps match Spack's config.""" config = spack.config.get_compilers_config() compiler_list = ['cc', 'cxx', 'f77', 'f90'] - for key in comps: + for key in compiler_names: for c in compiler_list: - if comps[key][c] == '/bad': - continue self.assertEqual(comps[key][c], config[key][c]) - def test_write_key(self): - a_comps = {"gcc@4.7.3" : { "cc" : "/gcc473", "cxx" : "/g++473", "f77" : None, "f90" : None }, - "gcc@4.5.0" : { "cc" : "/gcc450", "cxx" : "/g++450", "f77" : "/gfortran", "f90" : "/gfortran" }, - "clang@3.3" : { "cc" : "/bad", "cxx" : "/bad", "f77" : "/bad", "f90" : "/bad" }} + def test_write_key_in_memory(self): + # Write b_comps "on top of" a_comps. + spack.config.add_to_compiler_config(a_comps, 'test_low_priority') + spack.config.add_to_compiler_config(b_comps, 'test_high_priority') + + # Make sure the config looks how we expect. + self.check_config(a_comps, 'gcc@4.7.3', 'gcc@4.5.0') + self.check_config(b_comps, 'icc@10.0', 'icc@11.1', 'clang@3.3') - b_comps = {"icc@10.0" : { "cc" : "/icc100", "cxx" : "/icc100", "f77" : None, "f90" : None }, - "icc@11.1" : { "cc" : "/icc111", "cxx" : "/icp111", "f77" : "/ifort", "f90" : "/ifort" }, - "clang@3.3" : { "cc" : "/clang", "cxx" : "/clang++", "f77" : None, "f90" : None}} + def test_write_key_to_disk(self): + # Write b_comps "on top of" a_comps. spack.config.add_to_compiler_config(a_comps, 'test_low_priority') spack.config.add_to_compiler_config(b_comps, 'test_high_priority') - self.check_config(a_comps) - self.check_config(b_comps) - + # Clear caches so we're forced to read from disk. spack.config.clear_config_caches() - self.check_config(a_comps) - self.check_config(b_comps) - + # Same check again, to ensure consistency. + self.check_config(a_comps, 'gcc@4.7.3', 'gcc@4.5.0') + self.check_config(b_comps, 'icc@10.0', 'icc@11.1', 'clang@3.3') diff --git a/var/spack/mock_packages/repo.yaml b/var/spack/mock_packages/repo.yaml new file mode 100644 index 0000000000..d065896006 --- /dev/null +++ b/var/spack/mock_packages/repo.yaml @@ -0,0 +1,2 @@ +repo: + namespace: mock diff --git a/var/spack/packages/repo.yaml b/var/spack/packages/repo.yaml new file mode 100644 index 0000000000..4a371e1cad --- /dev/null +++ b/var/spack/packages/repo.yaml @@ -0,0 +1,2 @@ +repo: + namespace: gov.llnl.spack diff --git a/var/spack/packages/reponame b/var/spack/packages/reponame deleted file mode 100644 index 4b48deed3a..0000000000 --- a/var/spack/packages/reponame +++ /dev/null @@ -1 +0,0 @@ -original -- cgit v1.2.3-70-g09d2 From 1da56e5290ba7272ea4bd0f18cdf5cae2b83f093 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Fri, 21 Aug 2015 11:32:12 -0700 Subject: Added a database of installed packages. No methods use the database so far. Also, a bug fix: Previous version did not remove the staging directory on a failed install This led to spack refusing to uninstall dependencies of the failed install Added to cleanup() to blow away the staging directory on failed install. --- lib/spack/spack/database.py | 150 ++++++++++++++++++++++++++++++++++++ lib/spack/spack/directory_layout.py | 8 ++ lib/spack/spack/package.py | 12 +++ lib/spack/spack/spec.py | 8 +- 4 files changed, 175 insertions(+), 3 deletions(-) create mode 100644 lib/spack/spack/database.py (limited to 'lib') diff --git a/lib/spack/spack/database.py b/lib/spack/spack/database.py new file mode 100644 index 0000000000..8ae71a4385 --- /dev/null +++ b/lib/spack/spack/database.py @@ -0,0 +1,150 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +import os +import sys +import inspect +import glob +import imp + +from external import yaml +from external.yaml.error import MarkedYAMLError + +import llnl.util.tty as tty +from llnl.util.filesystem import join_path +from llnl.util.lang import * + +import spack.error +import spack.spec +from spack.spec import Spec +from spack.error import SpackError +from spack.virtual import ProviderIndex +from spack.util.naming import mod_to_class, validate_module_name + + +class Database(object): + def __init__(self,file_name="specDB.yaml"): + """ + Create an empty Database + Location defaults to root/specDB.yaml + """ + self.file_name = file_name + self.data = [] + + + def from_yaml(self,stream): + """ + Fill database from YAML + Translate the spec portions from node-dict form to spec from + """ + try: + file = yaml.load(stream) + except MarkedYAMLError, e: + raise SpackYAMLError("error parsing YAML database:", str(e)) + + if file==None: + return + + for sp in file['database']: + spec = Spec.from_node_dict(sp['spec']) + path = sp['path'] + db_entry = {'spec': spec, 'path': path} + self.data.append(db_entry) + + + @staticmethod + def read_database(root): + """Create a Database from the data in the standard location""" + database = Database() + full_path = join_path(root,database.file_name) + if os.path.isfile(full_path): + with open(full_path,'r') as f: + database.from_yaml(f) + else: + with open(full_path,'w+') as f: + database.from_yaml(f) + + return database + + + def write_database_to_yaml(self,stream): + """ + Replace each spec with its dict-node form + Then stream all data to YAML + """ + node_list = [] + for sp in self.data: + node = {} + node['spec']=Spec.to_node_dict(sp['spec']) + node['spec'][sp['spec'].name]['hash']=sp['spec'].dag_hash() + node['path']=sp['path'] + node_list.append(node) + return yaml.dump({ 'database' : node_list}, + stream=stream, default_flow_style=False) + + + def write(self,root): + """Write the database to the standard location""" + full_path = join_path(root,self.file_name) + #test for file existence + with open(full_path,'w') as f: + self.write_database_to_yaml(f) + + + @staticmethod + def add(root, spec, path): + """Read the database from the standard location + Add the specified entry as a dict + Write the database back to memory + + TODO: Caching databases + """ + database = Database.read_database(root) + + spec_and_path = {} + spec_and_path['spec']=spec + spec_and_path['path']=path + + database.data.append(spec_and_path) + + database.write(root) + + + @staticmethod + def remove(root, spec): + """ + Reads the database from the standard location + Searches for and removes the specified spec + Writes the database back to memory + + TODO: Caching databases + """ + database = Database.read_database(root) + + for sp in database.data: + #This requires specs w/o dependencies, is that sustainable? + if sp['spec'] == spec: + database.data.remove(sp) + + database.write(root) diff --git a/lib/spack/spack/directory_layout.py b/lib/spack/spack/directory_layout.py index e61929d8fd..0bbe6c9d85 100644 --- a/lib/spack/spack/directory_layout.py +++ b/lib/spack/spack/directory_layout.py @@ -37,6 +37,7 @@ from llnl.util.filesystem import join_path, mkdirp from spack.spec import Spec from spack.error import SpackError +from spack.database import Database def _check_concrete(spec): @@ -152,6 +153,8 @@ class DirectoryLayout(object): os.rmdir(path) path = os.path.dirname(path) + Database.remove(self.root,spec) + class YamlDirectoryLayout(DirectoryLayout): """Lays out installation directories like this:: @@ -263,6 +266,11 @@ class YamlDirectoryLayout(DirectoryLayout): self.write_spec(spec, spec_file_path) + def add_to_database(self, spec): + """Simply adds a spec to the database""" + Database.add(self.root, spec, self.path_for_spec(spec)) + + @memoized def all_specs(self): if not os.path.isdir(self.root): diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 3507807373..a425c555a2 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -779,6 +779,15 @@ class Package(object): "Manually remove this directory to fix:", self.prefix) + if not (keep_prefix and keep_stage): + self.do_clean() + else: + tty.warn("Keeping stage in place despite error.", + "Spack will refuse to uninstall dependencies of this package." + + "Manually remove this directory to fix:", + self.stage.path) + + def real_work(): try: tty.msg("Building %s." % self.name) @@ -808,6 +817,9 @@ class Package(object): log_install_path = spack.install_layout.build_log_path(self.spec) install(log_path, log_install_path) + #Update the database once we know install successful + spack.install_layout.add_to_database(self.spec) + # On successful install, remove the stage. if not keep_stage: self.stage.destroy() diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index e1fbb84423..df74b6064e 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -427,7 +427,6 @@ class Spec(object): spec = dep if isinstance(dep, Spec) else Spec(dep) self._add_dependency(spec) - # # Private routines here are called by the parser when building a spec. # @@ -640,7 +639,10 @@ class Spec(object): def dag_hash(self, length=None): - """Return a hash of the entire spec DAG, including connectivity.""" + """ + Return a hash of the entire spec DAG, including connectivity. + Stores the hash iff the spec is concrete. + """ yaml_text = yaml.dump( self.to_node_dict(), default_flow_style=True, width=sys.maxint) sha = hashlib.sha1(yaml_text) @@ -710,7 +712,7 @@ class Spec(object): try: yfile = yaml.load(stream) except MarkedYAMLError, e: - raise SpackYAMLError("error parsing YMAL spec:", str(e)) + raise SpackYAMLError("error parsing YAML spec:", str(e)) for node in yfile['spec']: name = next(iter(node)) -- cgit v1.2.3-70-g09d2 From 55f68bb2b05322297c3fc4d9fe265870b9385d4c Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Fri, 21 Aug 2015 13:04:27 -0700 Subject: Added hashes to the database --- lib/spack/spack/database.py | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/database.py b/lib/spack/spack/database.py index 8ae71a4385..4646530d52 100644 --- a/lib/spack/spack/database.py +++ b/lib/spack/spack/database.py @@ -48,11 +48,15 @@ class Database(object): """ Create an empty Database Location defaults to root/specDB.yaml + The individual data are dicts containing + spec: the top level spec of a package + path: the path to the install of that package + dep_hash: a hash of the dependence DAG for that package """ self.file_name = file_name self.data = [] - + def from_yaml(self,stream): """ Fill database from YAML @@ -69,10 +73,11 @@ class Database(object): for sp in file['database']: spec = Spec.from_node_dict(sp['spec']) path = sp['path'] - db_entry = {'spec': spec, 'path': path} + dep_hash = sp['hash'] + db_entry = {'spec': spec, 'path': path, 'hash':dep_hash} self.data.append(db_entry) - + @staticmethod def read_database(root): """Create a Database from the data in the standard location""" @@ -87,7 +92,7 @@ class Database(object): return database - + def write_database_to_yaml(self,stream): """ Replace each spec with its dict-node form @@ -97,7 +102,8 @@ class Database(object): for sp in self.data: node = {} node['spec']=Spec.to_node_dict(sp['spec']) - node['spec'][sp['spec'].name]['hash']=sp['spec'].dag_hash() +# node['spec'][sp['spec'].name]['hash']=sp['spec'].dag_hash() + node['hash']=sp['hash'] node['path']=sp['path'] node_list.append(node) return yaml.dump({ 'database' : node_list}, @@ -121,13 +127,14 @@ class Database(object): TODO: Caching databases """ database = Database.read_database(root) - - spec_and_path = {} - spec_and_path['spec']=spec - spec_and_path['path']=path - - database.data.append(spec_and_path) - + + sph = {} + sph['spec']=spec + sph['path']=path + sph['hash']=spec.dag_hash() + + database.data.append(sph) + database.write(root) -- cgit v1.2.3-70-g09d2 From fb1874165b420c5a8866bec7ac87a98e97f2b670 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Fri, 21 Aug 2015 16:42:12 -0700 Subject: Eliminated all calls that relied on finding all packages in the opt directory Replaced them all with references to the database Implemented caching in the database. The database now only re-reads data if the database file exists and was changed since this file last wrote to it. Added the installed_db field to the spack instance Left the call to all_specs from testdirectory_layout.py for now. --- lib/spack/spack/__init__.py | 6 ++ lib/spack/spack/cmd/__init__.py | 2 +- lib/spack/spack/cmd/deactivate.py | 2 +- lib/spack/spack/cmd/extensions.py | 2 +- lib/spack/spack/cmd/find.py | 4 +- lib/spack/spack/cmd/module.py | 4 +- lib/spack/spack/cmd/uninstall.py | 2 +- lib/spack/spack/database.py | 140 ++++++++++++++++++++++++++---------- lib/spack/spack/directory_layout.py | 6 -- lib/spack/spack/package.py | 6 +- lib/spack/spack/packages.py | 42 ----------- 11 files changed, 121 insertions(+), 95 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index caa09eb6e0..4d10bc2da6 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -53,6 +53,12 @@ from spack.packages import PackageDB packages_path = join_path(var_path, "packages") db = PackageDB(packages_path) +# +# Set up the installed packages database +# +from spack.database import Database +installed_db = Database(install_path) + # # Paths to mock files for testing. # diff --git a/lib/spack/spack/cmd/__init__.py b/lib/spack/spack/cmd/__init__.py index b96ac5af51..fd3ef3ed27 100644 --- a/lib/spack/spack/cmd/__init__.py +++ b/lib/spack/spack/cmd/__init__.py @@ -124,7 +124,7 @@ def elide_list(line_list, max_num=10): def disambiguate_spec(spec): - matching_specs = spack.db.get_installed(spec) + matching_specs = spack.installed_db.get_installed(spec) if not matching_specs: tty.die("Spec '%s' matches no installed packages." % spec) diff --git a/lib/spack/spack/cmd/deactivate.py b/lib/spack/spack/cmd/deactivate.py index e44be41029..1f0e303cdf 100644 --- a/lib/spack/spack/cmd/deactivate.py +++ b/lib/spack/spack/cmd/deactivate.py @@ -54,7 +54,7 @@ def deactivate(parser, args): if args.all: if pkg.extendable: tty.msg("Deactivating all extensions of %s" % pkg.spec.short_spec) - ext_pkgs = spack.db.installed_extensions_for(spec) + ext_pkgs = spack.installed_db.installed_extensions_for(spec) for ext_pkg in ext_pkgs: ext_pkg.spec.normalize() diff --git a/lib/spack/spack/cmd/extensions.py b/lib/spack/spack/cmd/extensions.py index fc8e6842c3..e919b1c4fb 100644 --- a/lib/spack/spack/cmd/extensions.py +++ b/lib/spack/spack/cmd/extensions.py @@ -80,7 +80,7 @@ def extensions(parser, args): colify(ext.name for ext in extensions) # List specs of installed extensions. - installed = [s.spec for s in spack.db.installed_extensions_for(spec)] + installed = [s.spec for s in spack.installed_db.installed_extensions_for(spec)] print if not installed: tty.msg("None installed.") diff --git a/lib/spack/spack/cmd/find.py b/lib/spack/spack/cmd/find.py index 3c993990b1..6f9072e311 100644 --- a/lib/spack/spack/cmd/find.py +++ b/lib/spack/spack/cmd/find.py @@ -138,9 +138,9 @@ def find(parser, args): # Get all the specs the user asked for if not query_specs: - specs = set(spack.db.installed_package_specs()) + specs = set(spack.installed_db.installed_package_specs()) else: - results = [set(spack.db.get_installed(qs)) for qs in query_specs] + results = [set(spack.installed_db.get_installed(qs)) for qs in query_specs] specs = set.union(*results) if not args.mode: diff --git a/lib/spack/spack/cmd/module.py b/lib/spack/spack/cmd/module.py index 34f0855a50..215d877bd0 100644 --- a/lib/spack/spack/cmd/module.py +++ b/lib/spack/spack/cmd/module.py @@ -65,7 +65,7 @@ def module_find(mtype, spec_array): tty.die("You can only pass one spec.") spec = specs[0] - specs = [s for s in spack.db.installed_package_specs() if s.satisfies(spec)] + specs = [s for s in spack.installed_db.installed_package_specs() if s.satisfies(spec)] if len(specs) == 0: tty.die("No installed packages match spec %s" % spec) @@ -86,7 +86,7 @@ def module_find(mtype, spec_array): def module_refresh(): """Regenerate all module files for installed packages known to spack (some packages may no longer exist).""" - specs = [s for s in spack.db.installed_known_package_specs()] + specs = [s for s in spack.installed_db.installed_known_package_specs()] for name, cls in module_types.items(): tty.msg("Regenerating %s module files." % name) diff --git a/lib/spack/spack/cmd/uninstall.py b/lib/spack/spack/cmd/uninstall.py index aa62510fed..4870712eb6 100644 --- a/lib/spack/spack/cmd/uninstall.py +++ b/lib/spack/spack/cmd/uninstall.py @@ -59,7 +59,7 @@ def uninstall(parser, args): # Fail and ask user to be unambiguous if it doesn't pkgs = [] for spec in specs: - matching_specs = spack.db.get_installed(spec) + matching_specs = spack.installed_db.get_installed(spec) if not args.all and len(matching_specs) > 1: tty.error("%s matches multiple packages:" % spec) print diff --git a/lib/spack/spack/database.py b/lib/spack/spack/database.py index 4646530d52..d027db312f 100644 --- a/lib/spack/spack/database.py +++ b/lib/spack/spack/database.py @@ -28,6 +28,9 @@ import inspect import glob import imp +import time +import copy + from external import yaml from external.yaml.error import MarkedYAMLError @@ -43,8 +46,18 @@ from spack.virtual import ProviderIndex from spack.util.naming import mod_to_class, validate_module_name +def _autospec(function): + """Decorator that automatically converts the argument of a single-arg + function to a Spec.""" + def converter(self, spec_like, **kwargs): + if not isinstance(spec_like, spack.spec.Spec): + spec_like = spack.spec.Spec(spec_like) + return function(self, spec_like, **kwargs) + return converter + + class Database(object): - def __init__(self,file_name="specDB.yaml"): + def __init__(self,root,file_name="specDB.yaml"): """ Create an empty Database Location defaults to root/specDB.yaml @@ -53,13 +66,16 @@ class Database(object): path: the path to the install of that package dep_hash: a hash of the dependence DAG for that package """ + self.root = root self.file_name = file_name + self.file_path = join_path(self.root,self.file_name) self.data = [] + self.last_write_time = 0 def from_yaml(self,stream): """ - Fill database from YAML + Fill database from YAML, do not maintain old data Translate the spec portions from node-dict form to spec from """ try: @@ -70,6 +86,7 @@ class Database(object): if file==None: return + self.data = [] for sp in file['database']: spec = Spec.from_node_dict(sp['spec']) path = sp['path'] @@ -78,19 +95,14 @@ class Database(object): self.data.append(db_entry) - @staticmethod - def read_database(root): - """Create a Database from the data in the standard location""" - database = Database() - full_path = join_path(root,database.file_name) - if os.path.isfile(full_path): - with open(full_path,'r') as f: - database.from_yaml(f) + def read_database(self): + """Reread Database from the data in the set location""" + if os.path.isfile(self.file_path): + with open(self.file_path,'r') as f: + self.from_yaml(f) else: - with open(full_path,'w+') as f: - database.from_yaml(f) - - return database + #The file doesn't exist, construct empty data. + self.data = [] def write_database_to_yaml(self,stream): @@ -110,48 +122,104 @@ class Database(object): stream=stream, default_flow_style=False) - def write(self,root): + def write(self): """Write the database to the standard location""" - full_path = join_path(root,self.file_name) - #test for file existence - with open(full_path,'w') as f: + #creates file if necessary + with open(self.file_path,'w') as f: + self.last_write_time = int(time.time()) self.write_database_to_yaml(f) - @staticmethod - def add(root, spec, path): - """Read the database from the standard location + def is_dirty(self): + """ + Returns true iff the database file exists + and was most recently written to by another spack instance. + """ + return (os.path.isfile(self.file_path) and (os.path.getmtime(self.file_path) > self.last_write_time)) + + +# @_autospec + def add(self, spec, path): + """Re-read the database from the set location if data is dirty Add the specified entry as a dict Write the database back to memory - - TODO: Caching databases """ - database = Database.read_database(root) + if self.is_dirty(): + self.read_database() sph = {} sph['spec']=spec sph['path']=path sph['hash']=spec.dag_hash() - database.data.append(sph) + self.data.append(sph) - database.write(root) + self.write() - @staticmethod - def remove(root, spec): + @_autospec + def remove(self, spec): """ - Reads the database from the standard location + Re-reads the database from the set location if data is dirty Searches for and removes the specified spec Writes the database back to memory + """ + if self.is_dirty(): + self.read_database() + + for sp in self.data: + + if sp['hash'] == spec.dag_hash() and sp['spec'] == Spec.from_node_dict(spec.to_node_dict()): + self.data.remove(sp) + + self.write() - TODO: Caching databases + + @_autospec + def get_installed(self, spec): + """ + Get all the installed specs that satisfy the provided spec constraint + """ + return [s for s in self.installed_package_specs() if s.satisfies(spec)] + + + @_autospec + def installed_extensions_for(self, extendee_spec): + """ + Return the specs of all packages that extend + the given spec + """ + for s in self.installed_package_specs(): + try: + if s.package.extends(extendee_spec): + yield s.package + except UnknownPackageError, e: + continue + #skips unknown packages + #TODO: conditional way to do this instead of catching exceptions + + + def installed_package_specs(self): """ - database = Database.read_database(root) + Read installed package names from the database + and return their specs + """ + if self.is_dirty(): + self.read_database() + + installed = [] + for sph in self.data: + sph['spec'].normalize() + sph['spec'].concretize() + installed.append(sph['spec']) + return installed - for sp in database.data: - #This requires specs w/o dependencies, is that sustainable? - if sp['spec'] == spec: - database.data.remove(sp) - database.write(root) + def installed_known_package_specs(self): + """ + Read installed package names from the database. + Return only the specs for which the package is known + to this version of spack + """ + return [s for s in self.installed_package_specs() if spack.db.exists(s.name)] + diff --git a/lib/spack/spack/directory_layout.py b/lib/spack/spack/directory_layout.py index 0bbe6c9d85..6dc1b0e550 100644 --- a/lib/spack/spack/directory_layout.py +++ b/lib/spack/spack/directory_layout.py @@ -153,7 +153,6 @@ class DirectoryLayout(object): os.rmdir(path) path = os.path.dirname(path) - Database.remove(self.root,spec) class YamlDirectoryLayout(DirectoryLayout): @@ -266,11 +265,6 @@ class YamlDirectoryLayout(DirectoryLayout): self.write_spec(spec, spec_file_path) - def add_to_database(self, spec): - """Simply adds a spec to the database""" - Database.add(self.root, spec, self.path_for_spec(spec)) - - @memoized def all_specs(self): if not os.path.isdir(self.root): diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index a425c555a2..acf558d639 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -565,7 +565,7 @@ class Package(object): """Return a list of the specs of all installed packages that depend on this one.""" dependents = [] - for spec in spack.db.installed_package_specs(): + for spec in spack.installed_db.installed_package_specs(): if self.name == spec.name: continue for dep in spec.traverse(): @@ -601,7 +601,7 @@ class Package(object): def remove_prefix(self): """Removes the prefix for a package along with any empty parent directories.""" spack.install_layout.remove_install_directory(self.spec) - + spack.installed_db.remove(self.spec) def do_fetch(self): """Creates a stage directory and downloads the taball for this package. @@ -818,7 +818,7 @@ class Package(object): install(log_path, log_install_path) #Update the database once we know install successful - spack.install_layout.add_to_database(self.spec) + spack.installed_db.add(self.spec, spack.install_layout.path_for_spec(self.spec)) # On successful install, remove the stage. if not keep_stage: diff --git a/lib/spack/spack/packages.py b/lib/spack/spack/packages.py index adfbc26c1d..2e3e95ca40 100644 --- a/lib/spack/spack/packages.py +++ b/lib/spack/spack/packages.py @@ -95,12 +95,6 @@ class PackageDB(object): self.instances.clear() - @_autospec - def get_installed(self, spec): - """Get all the installed specs that satisfy the provided spec constraint.""" - return [s for s in self.installed_package_specs() if s.satisfies(spec)] - - @_autospec def providers_for(self, vpkg_spec): if self.provider_index is None: @@ -117,19 +111,6 @@ class PackageDB(object): return [p for p in self.all_packages() if p.extends(extendee_spec)] - @_autospec - def installed_extensions_for(self, extendee_spec): - for s in self.installed_package_specs(): - try: - if s.package.extends(extendee_spec): - yield s.package - except UnknownPackageError, e: - # Skip packages we know nothing about - continue - # TODO: add some conditional way to do this instead of - # catching exceptions. - - def dirname_for_package_name(self, pkg_name): """Get the directory name for a particular package. This is the directory that contains its package.py file.""" @@ -150,29 +131,6 @@ class PackageDB(object): return join_path(pkg_dir, _package_file_name) - def installed_package_specs(self): - """Read installed package names straight from the install directory - layout. - """ - # Get specs from the directory layout but ensure that they're - # all normalized properly. - installed = [] - for spec in spack.install_layout.all_specs(): - spec.normalize() - installed.append(spec) - return installed - - - def installed_known_package_specs(self): - """Read installed package names straight from the install - directory layout, but return only specs for which the - package is known to this version of spack. - """ - for spec in spack.install_layout.all_specs(): - if self.exists(spec.name): - yield spec - - @memoized def all_package_names(self): """Generator function for all packages. This looks for -- cgit v1.2.3-70-g09d2 From 360b307f683e146151a65e1d788ce1d154c47ace Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 24 Aug 2015 09:14:16 -0700 Subject: Save progress. import gov.llnl.spack.mpich works. --- lib/spack/llnl/util/tty/colify.py | 7 + lib/spack/spack/__init__.py | 12 +- lib/spack/spack/cmd/repo.py | 17 +- lib/spack/spack/config.py | 4 +- lib/spack/spack/packages.py | 530 +++++++++++++++++++---------- lib/spack/spack/repo_loader.py | 22 +- lib/spack/spack/spec.py | 2 +- lib/spack/spack/test/directory_layout.py | 9 +- lib/spack/spack/test/mock_packages_test.py | 12 +- lib/spack/spack/test/package_sanity.py | 6 +- lib/spack/spack/test/packages.py | 6 +- lib/spack/spack/util/naming.py | 31 ++ var/spack/mock_packages/_repo.yaml | 2 + var/spack/mock_packages/repo.yaml | 2 - var/spack/packages/_repo.yaml | 2 + var/spack/packages/repo.yaml | 2 - 16 files changed, 435 insertions(+), 231 deletions(-) create mode 100644 var/spack/mock_packages/_repo.yaml delete mode 100644 var/spack/mock_packages/repo.yaml create mode 100644 var/spack/packages/_repo.yaml delete mode 100644 var/spack/packages/repo.yaml (limited to 'lib') diff --git a/lib/spack/llnl/util/tty/colify.py b/lib/spack/llnl/util/tty/colify.py index 66c52c3968..acf64c1e13 100644 --- a/lib/spack/llnl/util/tty/colify.py +++ b/lib/spack/llnl/util/tty/colify.py @@ -220,6 +220,13 @@ def colify(elts, **options): def colify_table(table, **options): + """Version of colify() for data expressed in rows, (list of lists). + + Same as regular colify but takes a list of lists, where each + sub-list must be the same length, and each is interpreted as a + row in a table. Regular colify displays a sequential list of + values in columns. + """ if table is None: raise TypeError("Can't call colify_table on NoneType") elif not table or not table[0]: diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index 09bc9ca52a..71e3ac3715 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -23,8 +23,10 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import os +import sys import tempfile from llnl.util.filesystem import * +import llnl.util.tty as tty # This lives in $prefix/lib/spack/spack/__file__ prefix = ancestor(__file__, 4) @@ -42,6 +44,7 @@ test_path = join_path(module_path, "test") hooks_path = join_path(module_path, "hooks") var_path = join_path(prefix, "var", "spack") stage_path = join_path(var_path, "stage") +packages_path = join_path(var_path, "packages") opt_path = join_path(prefix, "opt") install_path = join_path(opt_path, "spack") share_path = join_path(prefix, "share", "spack") @@ -55,9 +58,12 @@ repos = RepoNamespace() # # Set up the default packages database. # -from spack.packages import PackageDB -packages_path = join_path(var_path, "packages") -db = PackageDB() +import spack.packages +_repo_paths = spack.config.get_repos_config() +if not _repo_paths: + tty.die("Spack configuration contains no package repositories.") +db = spack.packages.PackageFinder(*_repo_paths) +sys.meta_path.append(db) # # Paths to mock files for testing. diff --git a/lib/spack/spack/cmd/repo.py b/lib/spack/spack/cmd/repo.py index 1261c7ada9..e290f60b7b 100644 --- a/lib/spack/spack/cmd/repo.py +++ b/lib/spack/spack/cmd/repo.py @@ -32,7 +32,7 @@ from llnl.util.filesystem import join_path, mkdirp import spack.spec import spack.config from spack.util.environment import get_path -from spack.packages import repo_config +from spack.packages import repo_config_filename import os import exceptions @@ -50,13 +50,8 @@ def setup_parser(subparser): create_parser = sp.add_parser('create', help=repo_create.__doc__) create_parser.add_argument('directory', help="Directory containing the packages.") create_parser.add_argument('name', help="Name of new package repository.") -<<<<<<< HEAD:lib/spack/spack/cmd/packagerepo.py - - remove_parser = sp.add_parser('remove', help=packagerepo_remove.__doc__) -======= remove_parser = sp.add_parser('remove', help=repo_remove.__doc__) ->>>>>>> Save changes to external repo integration:lib/spack/spack/cmd/repo.py remove_parser.add_argument('name') list_parser = sp.add_parser('list', help=repo_list.__doc__) @@ -81,7 +76,7 @@ def repo_add(args): """Add package sources to the Spack configuration.""" if not add_to_config(args.directory): tty.die('Repo directory %s already exists in the repo list' % dir) - + def repo_create(args): """Create a new package repo at a directory and name""" @@ -95,13 +90,13 @@ def repo_create(args): mkdirp(dir) except exceptions.OSError, e: tty.die('Failed to create new directory %s' % dir) - path = os.path.join(dir, repo_config) + path = os.path.join(dir, repo_config_filename) try: with closing(open(path, 'w')) as repofile: repofile.write(name + '\n') except exceptions.IOError, e: tty.die('Could not create new file %s' % path) - + if not add_to_config(args.directory): tty.warn('Repo directory %s already exists in the repo list' % dir) @@ -118,8 +113,8 @@ def repo_list(args): fmt = "%%-%ds%%s" % (max_len + 4) for root in root_names: print fmt % (root[0], root[1]) - - + + def repo(parser, args): action = { 'add' : repo_add, diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index dc59f9a5a3..66da91f629 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -269,7 +269,9 @@ def get_repos_config(): config = get_config() if 'repos' not in config: return [] - return config['repos'] + + repo_list = config['repos'] + return [substitute_spack_prefix(repo) for repo in repo_list] def get_mirror_config(): diff --git a/lib/spack/spack/packages.py b/lib/spack/spack/packages.py index c414234386..df54b12324 100644 --- a/lib/spack/spack/packages.py +++ b/lib/spack/spack/packages.py @@ -28,7 +28,6 @@ import sys import inspect import glob import imp -import spack.config import re import itertools import traceback @@ -41,149 +40,327 @@ from llnl.util.lang import * import spack.error import spack.spec from spack.virtual import ProviderIndex -from spack.util.naming import mod_to_class, validate_module_name -from sets import Set -from spack.repo_loader import RepoLoader, imported_packages_module, package_file_name +from spack.util.naming import * # Filename for package repo names -repo_config = 'repo.yaml' +repo_config_filename = '_repo.yaml' + +# Filename for packages in repos. +package_file_name = 'package.py' def _autospec(function): """Decorator that automatically converts the argument of a single-arg function to a Spec.""" - def converter(self, spec_like, **kwargs): + def converter(self, spec_like, *args, **kwargs): if not isinstance(spec_like, spack.spec.Spec): spec_like = spack.spec.Spec(spec_like) - return function(self, spec_like, **kwargs) + return function(self, spec_like, *args, **kwargs) return converter -def sliding_window(seq, n): - it = iter(seq) - result = tuple(itertools.islice(it, n)) - if len(result) == n: - yield result - for elem in it: - result = result[1:] + (elem,) - yield result +class NamespaceTrie(object): + def __init__(self): + self._elements = {} -class PackageDB(object): + def __setitem__(self, namespace, repo): + parts = namespace.split('.') + cur = self._elements + for p in parts[:-1]: + if p not in cur: + cur[p] = {} + cur = cur[p] + + cur[parts[-1]] = repo + + + def __getitem__(self, namespace): + parts = namespace.split('.') + cur = self._elements + for p in parts: + if p not in cur: + raise KeyError("Can't find namespace %s in trie" % namespace) + cur = cur[p] + return cur + + + def __contains__(self, namespace): + parts = namespace.split('.') + cur = self._elements + for p in parts: + if not isinstance(cur, dict): + return False + if p not in cur: + return False + cur = cur[p] + return True + + + +class PackageFinder(object): + """A PackageFinder is a wrapper around a list of PackageDBs. + + It functions exactly like a PackageDB, but it operates on the + combined results of the PackageDBs in its list instead of on a + single package repository. + """ def __init__(self, *repo_dirs): - """Construct a new package database from a list of directories. + self.repos = [] + self.by_namespace = NamespaceTrie() + self.by_path = {} + + for root in repo_dirs: + repo = PackageDB(root) + self.put_last(repo) + + + def _check_repo(self, repo): + if repo.root in self.by_path: + raise DuplicateRepoError("Package repos are the same", + repo, self.by_path[repo.root]) + + if repo.namespace in self.by_namespace: + tty.error("Package repos cannot have the same name", + repo, self.by_namespace[repo.namespace]) + + + def _add(self, repo): + self._check_repo(repo) + self.by_namespace[repo.namespace] = repo + self.by_path[repo.root] = repo + + + def put_first(self, repo): + self._add(repo) + self.repos.insert(0, repo) + + + def put_last(self, repo): + self._add(repo) + self.repos.append(repo) + - Args: - repo_dirs List of directories containing packages. + def remove(self, repo): + if repo in self.repos: + self.repos.remove(repo) - If ``repo_dirs`` is empty, gets repository list from Spack configuration. + + def swap(self, other): + repos = self.repos + by_namespace = self.by_namespace + by_path = self.by_path + + self.repos = other.repos + self.by_namespace = other.by_namespace + self.by_pah = other.by_path + + other.repos = repos + other.by_namespace = by_namespace + other.by_path = by_path + + + def all_package_names(self): + all_pkgs = set() + for repo in self.repos: + all_pkgs.update(set(repo.all_package_names())) + return all_pkgs + + + def all_packages(self): + for name in self.all_package_names(): + yield self.get(name) + + + def providers_for(self, vpkg_name): + # TODO: USE MORE THAN FIRST REPO + return self.repos[0].providers_for(vpkg_name) + + + def _get_spack_pkg_name(self, repo, py_module_name): + """Allow users to import Spack packages using legal Python identifiers. + + A python identifier might map to many different Spack package + names due to hyphen/underscore ambiguity. + + Easy example: + num3proxy -> 3proxy + + Ambiguous: + foo_bar -> foo_bar, foo-bar + + More ambiguous: + foo_bar_baz -> foo_bar_baz, foo-bar-baz, foo_bar-baz, foo-bar_baz """ - if not repo_dirs: - repo_dirs = spack.config.get_repos_config() - if not repo_dirs: - tty.die("Spack configuration contains no package repositories.") + if py_module_name in repo: + return py_module_name - # Collect the repos from the config file and read their names - # from the file system - repo_dirs = [spack.config.substitute_spack_prefix(rd) for rd in repo_dirs] + options = possible_spack_module_names(py_module_name) + options.remove(py_module_name) + for name in options: + if name in repo: + return name - self.repos = [] - for rdir in repo_dirs: - rname = self._read_reponame_from_directory(rdir) - if rname: - self.repos.append((self._read_reponame_from_directory(rdir), rdir)) + return None - by_path = sorted(self.repos, key=lambda r:r[1]) - by_name = sorted(self.repos, key=lambda r:r[0]) + def find_module(self, fullname, path=None): + if fullname in self.by_namespace: + return self - for r1, r2 in by_path: - if r1[1] == r2[1]: - tty.die("Package repos are the same:", - " %20s %s" % r1, " %20s %s" % r2) + namespace, dot, module_name = fullname.rpartition('.') + if namespace not in self.by_namespace: + return None - for r1, r2 in by_name: - if r1[0] == r2[0]: - tty.die("Package repos cannot have the same name:", - " %20s %s" % r1, " %20s %s" % r2) + repo = self.by_namespace[namespace] + name = self._get_spack_pkg_name(repo, module_name) + if not name: + return None - # For each repo, create a RepoLoader - self.repo_loaders = dict((name, RepoLoader(name, path)) - for name, path in self.repos) + return self - self.instances = {} - self.provider_index = None + def load_module(self, fullname): + if fullname in sys.modules: + return sys.modules[fullname] - def _read_reponame_from_directory(self, dir): - """For a packagerepo directory, read the repo name from the - $root/repo.yaml file""" - path = os.path.join(dir, repo_config) + if fullname in self.by_namespace: + ns = self.by_namespace[fullname] + module = imp.new_module(fullname) + module.__file__ = "" + module.__path__ = [] + module.__package__ = fullname + else: + namespace, dot, module_name = fullname.rpartition('.') + if namespace not in self.by_namespace: + raise ImportError( + "No Spack repository with namespace %s" % namespace) + + repo = self.by_namespace[namespace] + name = self._get_spack_pkg_name(repo, module_name) + if not name: + raise ImportError( + "No module %s in Spack repository %s" % (module_name, repo)) + + fullname = namespace + '.' + name + file_path = os.path.join(repo.root, name, package_file_name) + module = imp.load_source(fullname, file_path) + module.__package__ = namespace + + module.__loader__ = self + sys.modules[fullname] = module + return module + + + @_autospec + def get(self, spec, new=False): + for repo in self.repos: + if spec.name in repo: + return repo.get(spec, new) + raise UnknownPackageError(spec.name) + + + def get_repo(self, namespace): + if namespace in self.by_namespace: + repo = self.by_namespace[namespace] + if isinstance(repo, PackageDB): + return repo + return None + + + def exists(self, pkg_name): + return any(repo.exists(pkg_name) for repo in self.repos) + + + def __contains__(self, pkg_name): + return self.exists(pkg_name) + + + +class PackageDB(object): + """Class representing a package repository in the filesystem. + + Each package repository must have a top-level configuration file + called `_repo.yaml`. + + Currently, `_repo.yaml` this must define: + + `namespace`: + A Python namespace where the repository's packages should live. + + """ + def __init__(self, root): + """Instantiate a package repository from a filesystem path.""" + # Root directory, containing _repo.yaml and package dirs + self.root = root + + # Config file in /_repo.yaml + self.config_file = os.path.join(self.root, repo_config_filename) + + # Read configuration from _repo.yaml + config = self._read_config() + if not 'namespace' in config: + tty.die('Package repo in %s must define a namespace in %s.' + % (self.root, repo_config_filename)) + + # Check namespace in the repository configuration. + self.namespace = config['namespace'] + if not re.match(r'[a-zA-Z][a-zA-Z0-9_.]+', self.namespace): + tty.die(("Invalid namespace '%s' in '%s'. Namespaces must be " + "valid python identifiers separated by '.'") + % (self.namespace, self.root)) + + # These are internal cache variables. + self._instances = {} + self._provider_index = None + + + def _read_config(self): + """Check for a YAML config file in this db's root directory.""" try: - with open(path) as reponame_file: + with open(self.config_file) as reponame_file: yaml_data = yaml.load(reponame_file) - if (not yaml_data or - 'repo' not in yaml_data or - 'namespace' not in yaml_data['repo']): - tty.die("Invalid %s in %s" % (repo_config, dir)) - - name = yaml_data['repo']['namespace'] - if not re.match(r'[a-zA-Z][a-zA-Z0-9_.]+', name): - tty.die( - "Package repo name '%s', read from %s, is an invalid name. " - "Repo names must began with a letter and only contain " - "letters and numbers." % (name, path)) - return name + if (not yaml_data or 'repo' not in yaml_data or + not isinstance(yaml_data['repo'], dict)): + tty.die("Invalid %s in repository %s" + % (repo_config_filename, self.root)) + + return yaml_data['repo'] + except exceptions.IOError, e: - tty.die("Error reading %s when opening %s" % (repo_config, dir)) + tty.die("Error reading %s when opening %s" + % (self.config_file, self.root)) @_autospec - def get(self, spec, **kwargs): + def get(self, spec, new=False): if spec.virtual: raise UnknownPackageError(spec.name) - if kwargs.get('new', False): - if spec in self.instances: - del self.instances[spec] + if new: + if spec in self._instances: + del self._instances[spec] - if not spec in self.instances: + if not spec in self._instances: package_class = self.get_class_for_package_name(spec.name, spec.repo) try: copy = spec.copy() - self.instances[copy] = package_class(copy) + self._instances[copy] = package_class(copy) except Exception, e: if spack.debug: sys.excepthook(*sys.exc_info()) raise FailedConstructorError(spec.name, *sys.exc_info()) - return self.instances[spec] - - - @_autospec - def delete(self, spec): - """Force a package to be recreated.""" - del self.instances[spec] - - - def purge(self): - """Clear entire package instance cache.""" - self.instances.clear() - - - @_autospec - def get_installed(self, spec): - """Get all the installed specs that satisfy the provided spec constraint.""" - return [s for s in self.installed_package_specs() if s.satisfies(spec)] + return self._instances[spec] @_autospec def providers_for(self, vpkg_spec): - if self.provider_index is None: - self.provider_index = ProviderIndex(self.all_package_names()) + if self._provider_index is None: + self._provider_index = ProviderIndex(self.all_package_names()) - providers = self.provider_index.providers_for(vpkg_spec) + providers = self._provider_index.providers_for(vpkg_spec) if not providers: raise UnknownPackageError(vpkg_spec.name) return providers @@ -194,46 +371,13 @@ class PackageDB(object): return [p for p in self.all_packages() if p.extends(extendee_spec)] - @_autospec - def installed_extensions_for(self, extendee_spec): - for s in self.installed_package_specs(): - try: - if s.package.extends(extendee_spec): - yield s.package - except UnknownPackageError, e: - # Skip packages we know nothing about - continue - # TODO: add some conditional way to do this instead of - # catching exceptions. - - - def repo_for_package_name(self, pkg_name, packagerepo_name=None): - """Find the dirname for a package and the packagerepo it came from - if packagerepo_name is not None, then search for the package in the - specified packagerepo""" - #Look for an existing package under any matching packagerepos - roots = [pkgrepo for pkgrepo in self.repos - if not packagerepo_name or packagerepo_name == pkgrepo[0]] - - if not roots: - tty.die("Package repo %s does not exist" % packagerepo_name) - - for pkgrepo in roots: - path = join_path(pkgrepo[1], pkg_name) - if os.path.exists(path): - return (pkgrepo[0], path) - - repo_to_add_to = roots[-1] - return (repo_to_add_to[0], join_path(repo_to_add_to[1], pkg_name)) - - - def dirname_for_package_name(self, pkg_name, packagerepo_name=None): + def dirname_for_package_name(self, pkg_name): """Get the directory name for a particular package. This is the directory that contains its package.py file.""" - return self.repo_for_package_name(pkg_name, packagerepo_name)[1] + return join_path(self.root, pkg_name) - def filename_for_package_name(self, pkg_name, packagerepo_name=None): + def filename_for_package_name(self, pkg_name): """Get the filename for the module we should load for a particular package. Packages for a pacakge DB live in ``$root//package.py`` @@ -241,57 +385,25 @@ class PackageDB(object): This will return a proper package.py path even if the package doesn't exist yet, so callers will need to ensure the package exists before importing. - - If a packagerepo is specified, then return existing - or new paths in the specified packagerepo directory. If no - package repo is supplied, return an existing path from any - package repo, and new paths in the default package repo. """ validate_module_name(pkg_name) - pkg_dir = self.dirname_for_package_name(pkg_name, packagerepo_name) + pkg_dir = self.dirname_for_package_name(pkg_name) return join_path(pkg_dir, package_file_name) - def installed_package_specs(self): - """Read installed package names straight from the install directory - layout. - """ - # Get specs from the directory layout but ensure that they're - # all normalized properly. - installed = [] - for spec in spack.install_layout.all_specs(): - spec.normalize() - installed.append(spec) - return installed - - - def installed_known_package_specs(self): - """Read installed package names straight from the install - directory layout, but return only specs for which the - package is known to this version of spack. - """ - for spec in spack.install_layout.all_specs(): - if self.exists(spec.name): - yield spec - - @memoized def all_package_names(self): """Generator function for all packages. This looks for ``/package.py`` files within the repo direcotories""" - all_packages = Set() - for repo in self.repos: - dir = repo[1] - if not os.path.isdir(dir): - continue - for pkg_name in os.listdir(dir): - pkg_dir = join_path(dir, pkg_name) - pkg_file = join_path(pkg_dir, package_file_name) - if os.path.isfile(pkg_file): - all_packages.add(pkg_name) - all_package_names = list(all_packages) - all_package_names.sort() - return all_package_names + all_package_names = [] + + for pkg_name in os.listdir(self.root): + pkg_dir = join_path(self.root, pkg_name) + pkg_file = join_path(pkg_dir, package_file_name) + if os.path.isfile(pkg_file): + all_package_names.append(pkg_name) + + return sorted(all_package_names) def all_packages(self): @@ -301,19 +413,25 @@ class PackageDB(object): @memoized def exists(self, pkg_name): - """Whether a package with the supplied name exists .""" + """Whether a package with the supplied name exists.""" return os.path.exists(self.filename_for_package_name(pkg_name)) @memoized def get_class_for_package_name(self, pkg_name, reponame = None): """Get an instance of the class for a particular package.""" - (reponame, repodir) = self.repo_for_package_name(pkg_name, reponame) - module_name = imported_packages_module + '.' + reponame + '.' + pkg_name + file_path = self.filename_for_package_name(pkg_name) - module = self.repo_loaders[reponame].get_module(pkg_name) + if os.path.exists(file_path): + if not os.path.isfile(file_path): + tty.die("Something's wrong. '%s' is not a file!" % file_path) + if not os.access(file_path, os.R_OK): + tty.die("Cannot read '%s'!" % file_path) + else: + raise UnknownPackageError(pkg_name, self.namespace) class_name = mod_to_class(pkg_name) + module = __import__(self.namespace + '.' + pkg_name, fromlist=[class_name]) cls = getattr(module, class_name) if not inspect.isclass(cls): tty.die("%s.%s is not a class" % (pkg_name, class_name)) @@ -321,6 +439,63 @@ class PackageDB(object): return cls + def __str__(self): + return "" % (self.namespace, self.root) + + + def __repr__(self): + return self.__str__() + + + def __contains__(self, pkg_name): + return self.exists(pkg_name) + + + # + # Below functions deal with installed packages, and should be + # moved to some other part of Spack (conbine with + # directory_layout?) + # + @_autospec + def get_installed(self, spec): + """Get all the installed specs that satisfy the provided spec constraint.""" + return [s for s in self.installed_package_specs() if s.satisfies(spec)] + + + @_autospec + def installed_extensions_for(self, extendee_spec): + for s in self.installed_package_specs(): + try: + if s.package.extends(extendee_spec): + yield s.package + except UnknownPackageError, e: + # Skip packages we know nothing about + continue + + + def installed_package_specs(self): + """Read installed package names straight from the install directory + layout. + """ + # Get specs from the directory layout but ensure that they're + # all normalized properly. + installed = [] + for spec in spack.install_layout.all_specs(): + spec.normalize() + installed.append(spec) + return installed + + + def installed_known_package_specs(self): + """Read installed package names straight from the install + directory layout, but return only specs for which the + package is known to this version of spack. + """ + for spec in spack.install_layout.all_specs(): + if self.exists(spec.name): + yield spec + + class UnknownPackageError(spack.error.SpackError): """Raised when we encounter a package spack doesn't have.""" def __init__(self, name, repo=None): @@ -333,6 +508,13 @@ class UnknownPackageError(spack.error.SpackError): self.name = name +class DuplicateRepoError(spack.error.SpackError): + """Raised when duplicate repos are added to a PackageFinder.""" + def __init__(self, msg, repo1, repo2): + super(UnknownPackageError, self).__init__( + "%s: %s, %s" % (msg, repo1, repo2)) + + class FailedConstructorError(spack.error.SpackError): """Raised when a package's class constructor fails.""" def __init__(self, name, exc_type, exc_obj, exc_tb): diff --git a/lib/spack/spack/repo_loader.py b/lib/spack/spack/repo_loader.py index 92da1cf709..441011cf98 100644 --- a/lib/spack/spack/repo_loader.py +++ b/lib/spack/spack/repo_loader.py @@ -12,7 +12,6 @@ imported_packages_module = 'spack.repos' # Name of the package file inside a package directory package_file_name = 'package.py' -import sys class LazyLoader: """The LazyLoader handles cases when repo modules or classes are imported. It watches for 'spack.repos.*' loads, then @@ -21,15 +20,6 @@ class LazyLoader: if not fullname.startswith(imported_packages_module): return None - print "HERE ===" - print - for line in traceback.format_stack(): - print " ", line.strip() - print - print "full: ", fullname - print "path: ", pathname - print - partial_name = fullname[len(imported_packages_module)+1:] print "partial: ", partial_name @@ -50,7 +40,7 @@ class LazyLoader: def load_module(self, fullname): return self.mod -sys.meta_path.append(LazyLoader()) +#sys.meta_path.append(LazyLoader()) _reponames = {} class RepoNamespace(types.ModuleType): @@ -59,7 +49,6 @@ class RepoNamespace(types.ModuleType): this class will use __getattr__ to translate the 'original' into one of spack's known repositories""" def __init__(self): - import sys sys.modules[imported_packages_module] = self def __getattr__(self, name): @@ -89,7 +78,6 @@ class RepoLoader(types.ModuleType): if not reponame in _reponames: _reponames[reponame] = self - import sys sys.modules[self.module_name] = self @@ -110,14 +98,6 @@ class RepoLoader(types.ModuleType): import imp import llnl.util.tty as tty - file_path = os.path.join(self.path, pkg_name, package_file_name) - if os.path.exists(file_path): - if not os.path.isfile(file_path): - tty.die("Something's wrong. '%s' is not a file!" % file_path) - if not os.access(file_path, os.R_OK): - tty.die("Cannot read '%s'!" % file_path) - else: - raise spack.packages.UnknownPackageError(pkg_name, self.reponame if self.reponame != 'original' else None) try: module_name = imported_packages_module + '.' + self.reponame + '.' + pkg_name diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 972ba9ccbb..1666457502 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -1714,7 +1714,7 @@ class SpecParser(spack.parse.Parser): spec_repo = lst[-2] else: spec_name = self.token.value - (spec_repo, repodir) = spack.db.repo_for_package_name(spec_name) + spec_repo = 'gov.llnl.spack' self.check_identifier(spec_name) diff --git a/lib/spack/spack/test/directory_layout.py b/lib/spack/spack/test/directory_layout.py index b3ad8efec4..55b3f0b18f 100644 --- a/lib/spack/spack/test/directory_layout.py +++ b/lib/spack/spack/test/directory_layout.py @@ -34,7 +34,7 @@ from llnl.util.filesystem import * import spack from spack.spec import Spec -from spack.packages import PackageDB +from spack.packages import PackageFinder from spack.directory_layout import YamlDirectoryLayout # number of packages to test (to reduce test time) @@ -123,7 +123,7 @@ class DirectoryLayoutTest(unittest.TestCase): information about installed packages' specs to uninstall or query them again if the package goes away. """ - mock_db = PackageDB(spack.mock_packages_path) + mock_db = PackageFinder(spack.mock_packages_path) not_in_mock = set.difference( set(spack.db.all_package_names()), @@ -145,8 +145,7 @@ class DirectoryLayoutTest(unittest.TestCase): self.layout.create_install_directory(spec) installed_specs[spec] = self.layout.path_for_spec(spec) - tmp = spack.db - spack.db = mock_db + spack.db.swap(mock_db) # Now check that even without the package files, we know # enough to read a spec from the spec file. @@ -161,7 +160,7 @@ class DirectoryLayoutTest(unittest.TestCase): self.assertTrue(spec.eq_dag(spec_from_file)) self.assertEqual(spec.dag_hash(), spec_from_file.dag_hash()) - spack.db = tmp + spack.db.swap(mock_db) def test_find(self): diff --git a/lib/spack/spack/test/mock_packages_test.py b/lib/spack/spack/test/mock_packages_test.py index 00f81114af..1f46d65790 100644 --- a/lib/spack/spack/test/mock_packages_test.py +++ b/lib/spack/spack/test/mock_packages_test.py @@ -22,11 +22,12 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## +import sys import unittest import spack import spack.config -from spack.packages import PackageDB +from spack.packages import PackageFinder from spack.spec import Spec @@ -43,8 +44,8 @@ class MockPackagesTest(unittest.TestCase): # Use the mock packages database for these tests. This allows # us to set up contrived packages that don't interfere with # real ones. - self.real_db = spack.db - spack.db = PackageDB(spack.mock_packages_path) + self.db = PackageFinder(spack.mock_packages_path) + spack.db.swap(self.db) spack.config.clear_config_caches() self.real_scopes = spack.config.config_scopes @@ -55,7 +56,8 @@ class MockPackagesTest(unittest.TestCase): def cleanmock(self): """Restore the real packages path after any test.""" - spack.db = self.real_db + spack.db.swap(self.db) + spack.config.config_scopes = self.real_scopes spack.config.clear_config_caches() @@ -66,5 +68,3 @@ class MockPackagesTest(unittest.TestCase): def tearDown(self): self.cleanmock() - - diff --git a/lib/spack/spack/test/package_sanity.py b/lib/spack/spack/test/package_sanity.py index 6222e7b5f8..70b5d6a478 100644 --- a/lib/spack/spack/test/package_sanity.py +++ b/lib/spack/spack/test/package_sanity.py @@ -47,10 +47,10 @@ class PackageSanityTest(unittest.TestCase): def test_get_all_mock_packages(self): """Get the mock packages once each too.""" - tmp = spack.db - spack.db = PackageDB(spack.mock_packages_path) + db = PackageFinder(spack.mock_packages_path) + spack.db.swap(db) self.check_db() - spack.db = tmp + spack.db.swap(db) def test_url_versions(self): diff --git a/lib/spack/spack/test/packages.py b/lib/spack/spack/test/packages.py index a8183cf6a6..42bd91ec5c 100644 --- a/lib/spack/spack/test/packages.py +++ b/lib/spack/spack/test/packages.py @@ -44,7 +44,8 @@ class PackagesTest(MockPackagesTest): def test_package_filename(self): - filename = spack.db.filename_for_package_name('mpich') + repo = spack.db.get_repo('gov.llnl.spack.mock') + filename = repo.filename_for_package_name('mpich') self.assertEqual(filename, join_path(spack.mock_packages_path, 'mpich', 'package.py')) @@ -54,7 +55,8 @@ class PackagesTest(MockPackagesTest): def test_nonexisting_package_filename(self): - filename = spack.db.filename_for_package_name('some-nonexisting-package') + repo = spack.db.get_repo('gov.llnl.spack.mock') + filename = repo.filename_for_package_name('some-nonexisting-package') self.assertEqual(filename, join_path(spack.mock_packages_path, 'some-nonexisting-package', 'package.py')) diff --git a/lib/spack/spack/util/naming.py b/lib/spack/spack/util/naming.py index 782afbd4bb..a7b6e2b436 100644 --- a/lib/spack/spack/util/naming.py +++ b/lib/spack/spack/util/naming.py @@ -1,10 +1,14 @@ # Need this because of spack.util.string from __future__ import absolute_import import string +import itertools import re import spack +__all__ = ['mod_to_class', 'spack_module_to_python_module', 'valid_module_name', + 'validate_module_name', 'possible_spack_module_names'] + # Valid module names can contain '-' but can't start with it. _valid_module_re = r'^\w[\w-]*$' @@ -42,6 +46,33 @@ def mod_to_class(mod_name): return class_name +def spack_module_to_python_module(mod_name): + """Given a Spack module name, returns the name by which it can be + imported in Python. + """ + if re.match(r'[0-9]', mod_name): + mod_name = 'num' + mod_name + + return mod_name.replace('-', '_') + + +def possible_spack_module_names(python_mod_name): + """Given a Python module name, return a list of all possible spack module + names that could correspond to it.""" + mod_name = re.sub(r'^num(\d)', r'\1', python_mod_name) + + parts = re.split(r'(_)', mod_name) + options = [['_', '-']] * mod_name.count('_') + + results = [] + for subs in itertools.product(*options): + s = list(parts) + s[1::2] = subs + results.append(''.join(s)) + + return results + + def valid_module_name(mod_name): """Return whether the mod_name is valid for use in Spack.""" return bool(re.match(_valid_module_re, mod_name)) diff --git a/var/spack/mock_packages/_repo.yaml b/var/spack/mock_packages/_repo.yaml new file mode 100644 index 0000000000..b97b978de3 --- /dev/null +++ b/var/spack/mock_packages/_repo.yaml @@ -0,0 +1,2 @@ +repo: + namespace: gov.llnl.spack.mock diff --git a/var/spack/mock_packages/repo.yaml b/var/spack/mock_packages/repo.yaml deleted file mode 100644 index d065896006..0000000000 --- a/var/spack/mock_packages/repo.yaml +++ /dev/null @@ -1,2 +0,0 @@ -repo: - namespace: mock diff --git a/var/spack/packages/_repo.yaml b/var/spack/packages/_repo.yaml new file mode 100644 index 0000000000..4a371e1cad --- /dev/null +++ b/var/spack/packages/_repo.yaml @@ -0,0 +1,2 @@ +repo: + namespace: gov.llnl.spack diff --git a/var/spack/packages/repo.yaml b/var/spack/packages/repo.yaml deleted file mode 100644 index 4a371e1cad..0000000000 --- a/var/spack/packages/repo.yaml +++ /dev/null @@ -1,2 +0,0 @@ -repo: - namespace: gov.llnl.spack -- cgit v1.2.3-70-g09d2 From 4a2bd1753a3c0b96c0b38ee9ec909cbf98308125 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Tue, 25 Aug 2015 15:11:18 -0700 Subject: Added dependency indices to database, ensuring correctly reconstructed specs from database Began work on file locking, currently commented out. --- lib/spack/spack/database.py | 102 +++++++++++++++++++++++++++++++++----------- lib/spack/spack/package.py | 1 + 2 files changed, 78 insertions(+), 25 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/database.py b/lib/spack/spack/database.py index d027db312f..7e2c3ac079 100644 --- a/lib/spack/spack/database.py +++ b/lib/spack/spack/database.py @@ -30,6 +30,7 @@ import imp import time import copy +import errno from external import yaml from external.yaml.error import MarkedYAMLError @@ -69,6 +70,10 @@ class Database(object): self.root = root self.file_name = file_name self.file_path = join_path(self.root,self.file_name) + + self.lock_name = "db_lock" + self.lock_path = join_path(self.root,self.lock_name) + self.data = [] self.last_write_time = 0 @@ -86,17 +91,38 @@ class Database(object): if file==None: return - self.data = [] - for sp in file['database']: + data = {} + for index, sp in file['database'].items(): spec = Spec.from_node_dict(sp['spec']) + deps = sp['dependency_indices'] path = sp['path'] dep_hash = sp['hash'] - db_entry = {'spec': spec, 'path': path, 'hash':dep_hash} - self.data.append(db_entry) + db_entry = {'deps':deps, 'spec': spec, 'path': path, 'hash':dep_hash} + data[index] = db_entry + + for sph in data.values(): + for idx in sph['deps']: + sph['spec'].dependencies[data[idx]['spec'].name] = data[idx]['spec'] + + self.data = data.values() def read_database(self): - """Reread Database from the data in the set location""" + """ + Re-read Database from the data in the set location + If the cache is fresh, return immediately. + Implemented with mkdir locking for the database file. + """ + if not self.is_dirty(): + return + """ + while True: + try: + os.mkdir(self.lock_path) + break + except OSError as err: + pass + """ if os.path.isfile(self.file_path): with open(self.file_path,'r') as f: self.from_yaml(f) @@ -104,6 +130,8 @@ class Database(object): #The file doesn't exist, construct empty data. self.data = [] +# os.rmdir(self.lock_path) + def write_database_to_yaml(self,stream): """ @@ -111,24 +139,54 @@ class Database(object): Then stream all data to YAML """ node_list = [] - for sp in self.data: + spec_list = [sph['spec'] for sph in self.data] + + for sph in self.data: node = {} - node['spec']=Spec.to_node_dict(sp['spec']) -# node['spec'][sp['spec'].name]['hash']=sp['spec'].dag_hash() - node['hash']=sp['hash'] - node['path']=sp['path'] + deps = [] + for name,spec in sph['spec'].dependencies.items(): + deps.append(spec_list.index(spec)) + node['spec']=Spec.to_node_dict(sph['spec']) + node['hash']=sph['hash'] + node['path']=sph['path'] + node['dependency_indices']=deps node_list.append(node) - return yaml.dump({ 'database' : node_list}, + + node_dict = dict(enumerate(node_list)) + return yaml.dump({ 'database' : node_dict}, stream=stream, default_flow_style=False) def write(self): - """Write the database to the standard location""" - #creates file if necessary + """ + Write the database to the standard location + Implements mkdir locking for the database file + """ + """ + while True: + try: + os.mkdir(self.lock_path) + break + except OSError as err: + pass + """ with open(self.file_path,'w') as f: self.last_write_time = int(time.time()) self.write_database_to_yaml(f) + # os.rmdir(self.lock_path) + + + def get_index_of(self, spec): + """ + Returns the index of a spec in the database + If unable to find the spec it returns -1 + """ + for index, sph in enumerate(self.data): + if sph['spec'] == spec: + return index + return -1 + def is_dirty(self): """ @@ -140,12 +198,11 @@ class Database(object): # @_autospec def add(self, spec, path): - """Re-read the database from the set location if data is dirty + """Read the database from the set location Add the specified entry as a dict Write the database back to memory """ - if self.is_dirty(): - self.read_database() + self.read_database() sph = {} sph['spec']=spec @@ -160,16 +217,14 @@ class Database(object): @_autospec def remove(self, spec): """ - Re-reads the database from the set location if data is dirty + Reads the database from the set location Searches for and removes the specified spec Writes the database back to memory """ - if self.is_dirty(): - self.read_database() + self.read_database() for sp in self.data: - - if sp['hash'] == spec.dag_hash() and sp['spec'] == Spec.from_node_dict(spec.to_node_dict()): + if sp['hash'] == spec.dag_hash() and sp['spec'] == spec: self.data.remove(sp) self.write() @@ -204,13 +259,10 @@ class Database(object): Read installed package names from the database and return their specs """ - if self.is_dirty(): - self.read_database() + self.read_database() installed = [] for sph in self.data: - sph['spec'].normalize() - sph['spec'].concretize() installed.append(sph['spec']) return installed diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index acf558d639..929e0c086c 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -557,6 +557,7 @@ class Package(object): @property def installed(self): + print self.prefix return os.path.isdir(self.prefix) -- cgit v1.2.3-70-g09d2 From e32c59f805c4e8d9cb23ce9fcf2edcf571ce3949 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Tue, 25 Aug 2015 15:32:45 -0700 Subject: Fixed file locking. Fix is slightly ugly (lock integer added) but it gets the job done It avoids having to spin simply on the OSError. --- lib/spack/spack/database.py | 49 ++++++++++++++++++++++++++++----------------- 1 file changed, 31 insertions(+), 18 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/database.py b/lib/spack/spack/database.py index 7e2c3ac079..5e8bb172b8 100644 --- a/lib/spack/spack/database.py +++ b/lib/spack/spack/database.py @@ -115,22 +115,29 @@ class Database(object): """ if not self.is_dirty(): return - """ - while True: + + lock=0 + while lock==0: try: os.mkdir(self.lock_path) - break + lock=1 except OSError as err: pass - """ - if os.path.isfile(self.file_path): - with open(self.file_path,'r') as f: - self.from_yaml(f) - else: + + #The try statement ensures that a failure won't leave the + #database locked to other processes. + try: + if os.path.isfile(self.file_path): + with open(self.file_path,'r') as f: + self.from_yaml(f) + else: #The file doesn't exist, construct empty data. - self.data = [] + self.data = [] + except: + os.rmdir(self.lock_path) + raise -# os.rmdir(self.lock_path) + os.rmdir(self.lock_path) def write_database_to_yaml(self,stream): @@ -162,19 +169,25 @@ class Database(object): Write the database to the standard location Implements mkdir locking for the database file """ - """ - while True: + lock=0 + while lock==0: try: os.mkdir(self.lock_path) - break + lock=1 except OSError as err: pass - """ - with open(self.file_path,'w') as f: - self.last_write_time = int(time.time()) - self.write_database_to_yaml(f) - # os.rmdir(self.lock_path) + #The try statement ensures that a failure won't leave the + #database locked to other processes. + try: + with open(self.file_path,'w') as f: + self.last_write_time = int(time.time()) + self.write_database_to_yaml(f) + except: + os.rmdir(self.lock_path) + raise + + os.rmdir(self.lock_path) def get_index_of(self, spec): -- cgit v1.2.3-70-g09d2 From ce8df65d7b9162b71be88e6268ee5172deab4cd9 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Tue, 25 Aug 2015 16:28:55 -0700 Subject: Eliminated unnecessary differences in pull request --- lib/spack/spack/directory_layout.py | 2 -- lib/spack/spack/package.py | 1 - lib/spack/spack/spec.py | 1 + 3 files changed, 1 insertion(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/directory_layout.py b/lib/spack/spack/directory_layout.py index 6dc1b0e550..e61929d8fd 100644 --- a/lib/spack/spack/directory_layout.py +++ b/lib/spack/spack/directory_layout.py @@ -37,7 +37,6 @@ from llnl.util.filesystem import join_path, mkdirp from spack.spec import Spec from spack.error import SpackError -from spack.database import Database def _check_concrete(spec): @@ -154,7 +153,6 @@ class DirectoryLayout(object): path = os.path.dirname(path) - class YamlDirectoryLayout(DirectoryLayout): """Lays out installation directories like this:: / diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 929e0c086c..acf558d639 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -557,7 +557,6 @@ class Package(object): @property def installed(self): - print self.prefix return os.path.isdir(self.prefix) diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index df74b6064e..8050b73b9e 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -427,6 +427,7 @@ class Spec(object): spec = dep if isinstance(dep, Spec) else Spec(dep) self._add_dependency(spec) + # # Private routines here are called by the parser when building a spec. # -- cgit v1.2.3-70-g09d2 From 9345e7877983d1bde8a4aefbecdc4a8cab181186 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Tue, 25 Aug 2015 16:31:09 -0700 Subject: Fixed inaccurate comment in spec.py --- lib/spack/spack/spec.py | 1 - 1 file changed, 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 8050b73b9e..cde2e168a0 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -642,7 +642,6 @@ class Spec(object): def dag_hash(self, length=None): """ Return a hash of the entire spec DAG, including connectivity. - Stores the hash iff the spec is concrete. """ yaml_text = yaml.dump( self.to_node_dict(), default_flow_style=True, width=sys.maxint) -- cgit v1.2.3-70-g09d2 From f406fcb843edcea359e3fb6103eed560dadc1355 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Mon, 31 Aug 2015 09:38:38 -0700 Subject: Fixed several issues from code review Most importantly wrote the Lock, Read_Lock_Instance, and Write_Lock_Instance classes in lock.py Updated the locking in database.py TODO: Lock on larger areas --- lib/spack/llnl/util/lock.py | 136 ++++++++++++++++++++++++++++++++++++++++++++ lib/spack/spack/database.py | 130 ++++++++++++++++-------------------------- 2 files changed, 184 insertions(+), 82 deletions(-) create mode 100644 lib/spack/llnl/util/lock.py (limited to 'lib') diff --git a/lib/spack/llnl/util/lock.py b/lib/spack/llnl/util/lock.py new file mode 100644 index 0000000000..05641475ed --- /dev/null +++ b/lib/spack/llnl/util/lock.py @@ -0,0 +1,136 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +import os +import fcntl +import errno +import time +import socket + + +class Read_Lock_Instance(object): + """ + A context manager for getting shared access to the object lock + Arguments are lock and timeout (default 5 minutes) + """ + def __init__(self,lock,timeout = 300): + self._lock = lock + self._timeout = timeout + def __enter__(self): + self._lock.acquire_read(self._timeout) + def __exit__(self,type,value,traceback): + self._lock.release_read() + + +class Write_Lock_Instance(object): + """ + A context manager for getting exclusive access to the object lock + Arguments are lock and timeout (default 5 minutes) + """ + def __init__(self,lock,timeout = 300): + self._lock = lock + self._timeout = timeout + def __enter__(self): + self._lock.acquire_write(self._timeout) + def __exit__(self,type,value,traceback): + self._lock.release_write() + + +class Lock(object): + def __init__(self,file_path): + self._file_path = file_path + self._fd = os.open(file_path,os.O_RDWR) + self._reads = 0 + self._writes = 0 + + + def acquire_read(self,timeout): + """ + Implements recursive lock. If held in both read and write mode, + the write lock will be maintained until all locks are released + """ + if self._reads == 0 and self._writes == 0: + self._lock(fcntl.LOCK_SH,timeout) + self._reads += 1 + + + def acquire_write(self,timeout): + """ + Implements recursive lock + """ + if self._writes == 0: + self._lock(fcntl.LOCK_EX,timeout) + self._writes += 1 + + + def _lock(self,op,timeout): + """ + The timeout is implemented using nonblocking flock() + to avoid using signals for timing + Write locks store pid and host information to the lock file + Read locks do not store data + """ + total_time = 0 + while total_time < timeout: + try: + fcntl.flock(self._fd, op | fcntl.LOCK_NB) + if op == fcntl.LOCK_EX: + with open(self._file_path,'w') as f: + f.write("pid = "+str(os.getpid())+", host = "+socket.getfqdn()) + return + except IOError as error: + if error.errno == errno.EAGAIN or error.errno == EACCES: + pass + else: + raise + time.sleep(0.1) + total_time += 0.1 + + + def release_read(self): + """ + Assert there is a lock of the right type to release, recursive lock + """ + assert self._reads > 0 + if self._reads == 1 and self._writes == 0: + self._unlock() + self._reads -= 1 + + + def release_write(self): + """ + Assert there is a lock of the right type to release, recursive lock + """ + assert self._writes > 0 + if self._writes == 1 and self._reads == 0: + self._unlock() + self._writes -= 1 + + + def _unlock(self): + """ + Releases the lock regardless of mode. Note that read locks may be + masquerading as write locks at times, but this removes either. + """ + fcntl.flock(self._fd,fcntl.LOCK_UN) diff --git a/lib/spack/spack/database.py b/lib/spack/spack/database.py index 5e8bb172b8..6a429b68a9 100644 --- a/lib/spack/spack/database.py +++ b/lib/spack/spack/database.py @@ -38,6 +38,7 @@ from external.yaml.error import MarkedYAMLError import llnl.util.tty as tty from llnl.util.filesystem import join_path from llnl.util.lang import * +from llnl.util.lock import * import spack.error import spack.spec @@ -58,7 +59,7 @@ def _autospec(function): class Database(object): - def __init__(self,root,file_name="specDB.yaml"): + def __init__(self,root,file_name="_index.yaml"): """ Create an empty Database Location defaults to root/specDB.yaml @@ -67,28 +68,31 @@ class Database(object): path: the path to the install of that package dep_hash: a hash of the dependence DAG for that package """ - self.root = root - self.file_name = file_name - self.file_path = join_path(self.root,self.file_name) + self._root = root + self._file_name = file_name + self._file_path = join_path(self._root,self._file_name) - self.lock_name = "db_lock" - self.lock_path = join_path(self.root,self.lock_name) + self._lock_name = "_db_lock" + self._lock_path = join_path(self._root,self._lock_name) + if not os.path.exists(self._lock_path): + open(self._lock_path,'w').close() + self.lock = Lock(self._lock_path) - self.data = [] - self.last_write_time = 0 + self._data = [] + self._last_write_time = 0 - def from_yaml(self,stream): + def _read_from_yaml(self,stream): """ Fill database from YAML, do not maintain old data - Translate the spec portions from node-dict form to spec from + Translate the spec portions from node-dict form to spec form """ try: file = yaml.load(stream) except MarkedYAMLError, e: raise SpackYAMLError("error parsing YAML database:", str(e)) - if file==None: + if file is None: return data = {} @@ -104,51 +108,34 @@ class Database(object): for idx in sph['deps']: sph['spec'].dependencies[data[idx]['spec'].name] = data[idx]['spec'] - self.data = data.values() + self._data = data.values() def read_database(self): """ Re-read Database from the data in the set location If the cache is fresh, return immediately. - Implemented with mkdir locking for the database file. """ if not self.is_dirty(): return - lock=0 - while lock==0: - try: - os.mkdir(self.lock_path) - lock=1 - except OSError as err: - pass - - #The try statement ensures that a failure won't leave the - #database locked to other processes. - try: - if os.path.isfile(self.file_path): - with open(self.file_path,'r') as f: - self.from_yaml(f) - else: + if os.path.isfile(self._file_path): + with open(self._file_path,'r') as f: + self._read_from_yaml(f) + else: #The file doesn't exist, construct empty data. - self.data = [] - except: - os.rmdir(self.lock_path) - raise - - os.rmdir(self.lock_path) + self._data = [] - def write_database_to_yaml(self,stream): + def _write_database_to_yaml(self,stream): """ Replace each spec with its dict-node form Then stream all data to YAML """ node_list = [] - spec_list = [sph['spec'] for sph in self.data] + spec_list = [sph['spec'] for sph in self._data] - for sph in self.data: + for sph in self._data: node = {} deps = [] for name,spec in sph['spec'].dependencies.items(): @@ -167,46 +154,23 @@ class Database(object): def write(self): """ Write the database to the standard location - Implements mkdir locking for the database file + Everywhere that the database is written it is read + within the same lock, so there is no need to refresh + the database within write() """ - lock=0 - while lock==0: - try: - os.mkdir(self.lock_path) - lock=1 - except OSError as err: - pass - - #The try statement ensures that a failure won't leave the - #database locked to other processes. - try: - with open(self.file_path,'w') as f: - self.last_write_time = int(time.time()) - self.write_database_to_yaml(f) - except: - os.rmdir(self.lock_path) - raise - - os.rmdir(self.lock_path) - - - def get_index_of(self, spec): - """ - Returns the index of a spec in the database - If unable to find the spec it returns -1 - """ - for index, sph in enumerate(self.data): - if sph['spec'] == spec: - return index - return -1 - + temp_name = os.getpid() + socket.getfqdn() + ".temp" + temp_file = path.join(self._root,temp_name) + with open(self.temp_path,'w') as f: + self._last_write_time = int(time.time()) + self._write_database_to_yaml(f) + os.rename(temp_name,self._file_path) def is_dirty(self): """ Returns true iff the database file exists and was most recently written to by another spack instance. """ - return (os.path.isfile(self.file_path) and (os.path.getmtime(self.file_path) > self.last_write_time)) + return (os.path.isfile(self._file_path) and (os.path.getmtime(self._file_path) > self._last_write_time)) # @_autospec @@ -215,16 +179,15 @@ class Database(object): Add the specified entry as a dict Write the database back to memory """ - self.read_database() - sph = {} sph['spec']=spec sph['path']=path sph['hash']=spec.dag_hash() - self.data.append(sph) - - self.write() + with Write_Lock_Instance(self.lock,60): + self.read_database() + self._data.append(sph) + self.write() @_autospec @@ -234,13 +197,15 @@ class Database(object): Searches for and removes the specified spec Writes the database back to memory """ - self.read_database() + with Write_Lock_Instance(self.lock,60): + self.read_database() - for sp in self.data: - if sp['hash'] == spec.dag_hash() and sp['spec'] == spec: - self.data.remove(sp) + for sp in self._data: + #Not sure the hash comparison is necessary + if sp['hash'] == spec.dag_hash() and sp['spec'] == spec: + self._data.remove(sp) - self.write() + self.write() @_autospec @@ -272,10 +237,11 @@ class Database(object): Read installed package names from the database and return their specs """ - self.read_database() + with Read_Lock_Instance(self.lock,60): + self.read_database() installed = [] - for sph in self.data: + for sph in self._data: installed.append(sph['spec']) return installed -- cgit v1.2.3-70-g09d2 From c3246ee8ba26909ba73c3587ead9e61342692957 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Mon, 31 Aug 2015 09:46:55 -0700 Subject: Removed incorrect stage removal code from cleanup() in do_install() --- lib/spack/spack/package.py | 8 -------- 1 file changed, 8 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index acf558d639..e64b427852 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -779,14 +779,6 @@ class Package(object): "Manually remove this directory to fix:", self.prefix) - if not (keep_prefix and keep_stage): - self.do_clean() - else: - tty.warn("Keeping stage in place despite error.", - "Spack will refuse to uninstall dependencies of this package." + - "Manually remove this directory to fix:", - self.stage.path) - def real_work(): try: -- cgit v1.2.3-70-g09d2 From 9c8e46dc228e096a83a892e5f53424bb3446d8f6 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Thu, 3 Sep 2015 09:21:19 -0700 Subject: Added conservative locking to the spack commands that access the database at _index --- lib/spack/spack/cmd/__init__.py | 22 +++++++------- lib/spack/spack/cmd/deactivate.py | 12 ++++---- lib/spack/spack/cmd/diy.py | 60 ++++++++++++++++++++------------------- lib/spack/spack/cmd/extensions.py | 4 ++- lib/spack/spack/cmd/find.py | 7 +++-- lib/spack/spack/cmd/install.py | 22 +++++++------- lib/spack/spack/cmd/uninstall.py | 52 +++++++++++++++++---------------- lib/spack/spack/database.py | 11 ++++--- 8 files changed, 104 insertions(+), 86 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/cmd/__init__.py b/lib/spack/spack/cmd/__init__.py index fd3ef3ed27..c62d22979a 100644 --- a/lib/spack/spack/cmd/__init__.py +++ b/lib/spack/spack/cmd/__init__.py @@ -28,6 +28,7 @@ import sys import llnl.util.tty as tty from llnl.util.lang import attr_setdefault +from llnl.util.lock import * import spack import spack.spec @@ -124,15 +125,16 @@ def elide_list(line_list, max_num=10): def disambiguate_spec(spec): - matching_specs = spack.installed_db.get_installed(spec) - if not matching_specs: - tty.die("Spec '%s' matches no installed packages." % spec) - - elif len(matching_specs) > 1: - args = ["%s matches multiple packages." % spec, - "Matching packages:"] - args += [" " + str(s) for s in matching_specs] - args += ["Use a more specific spec."] - tty.die(*args) + with Read_Lock_Instance(spack.installed_db.lock,1800): + matching_specs = spack.installed_db.get_installed(spec) + if not matching_specs: + tty.die("Spec '%s' matches no installed packages." % spec) + + elif len(matching_specs) > 1: + args = ["%s matches multiple packages." % spec, + "Matching packages:"] + args += [" " + str(s) for s in matching_specs] + args += ["Use a more specific spec."] + tty.die(*args) return matching_specs[0] diff --git a/lib/spack/spack/cmd/deactivate.py b/lib/spack/spack/cmd/deactivate.py index 1f0e303cdf..015809345a 100644 --- a/lib/spack/spack/cmd/deactivate.py +++ b/lib/spack/spack/cmd/deactivate.py @@ -24,6 +24,7 @@ ############################################################################## from external import argparse import llnl.util.tty as tty +from llnl.util.lock import * import spack import spack.cmd @@ -54,12 +55,13 @@ def deactivate(parser, args): if args.all: if pkg.extendable: tty.msg("Deactivating all extensions of %s" % pkg.spec.short_spec) - ext_pkgs = spack.installed_db.installed_extensions_for(spec) + with Read_Lock_Instance(spack.installed_db.lock,1800): + ext_pkgs = spack.installed_db.installed_extensions_for(spec) - for ext_pkg in ext_pkgs: - ext_pkg.spec.normalize() - if ext_pkg.activated: - ext_pkg.do_deactivate(force=True) + for ext_pkg in ext_pkgs: + ext_pkg.spec.normalize() + if ext_pkg.activated: + ext_pkg.do_deactivate(force=True) elif pkg.is_extension: if not args.force and not spec.package.activated: diff --git a/lib/spack/spack/cmd/diy.py b/lib/spack/spack/cmd/diy.py index 6e7f10fba6..7403b9e3e8 100644 --- a/lib/spack/spack/cmd/diy.py +++ b/lib/spack/spack/cmd/diy.py @@ -27,6 +27,7 @@ import os from external import argparse import llnl.util.tty as tty +from llnl.util.lock import * import spack import spack.cmd @@ -54,40 +55,41 @@ def diy(self, args): if not args.spec: tty.die("spack diy requires a package spec argument.") - specs = spack.cmd.parse_specs(args.spec) - if len(specs) > 1: - tty.die("spack diy only takes one spec.") + with Write_Lock_Instance(spack.installed_db.lock,1800): + specs = spack.cmd.parse_specs(args.spec) + if len(specs) > 1: + tty.die("spack diy only takes one spec.") - spec = specs[0] - if not spack.db.exists(spec.name): - tty.warn("No such package: %s" % spec.name) - create = tty.get_yes_or_no("Create this package?", default=False) - if not create: - tty.msg("Exiting without creating.") - sys.exit(1) - else: - tty.msg("Running 'spack edit -f %s'" % spec.name) - edit_package(spec.name, True) - return + spec = specs[0] + if not spack.db.exists(spec.name): + tty.warn("No such package: %s" % spec.name) + create = tty.get_yes_or_no("Create this package?", default=False) + if not create: + tty.msg("Exiting without creating.") + sys.exit(1) + else: + tty.msg("Running 'spack edit -f %s'" % spec.name) + edit_package(spec.name, True) + return - if not spec.version.concrete: - tty.die("spack diy spec must have a single, concrete version.") + if not spec.version.concrete: + tty.die("spack diy spec must have a single, concrete version.") - spec.concretize() - package = spack.db.get(spec) + spec.concretize() + package = spack.db.get(spec) - if package.installed: - tty.error("Already installed in %s" % package.prefix) - tty.msg("Uninstall or try adding a version suffix for this DIY build.") - sys.exit(1) + if package.installed: + tty.error("Already installed in %s" % package.prefix) + tty.msg("Uninstall or try adding a version suffix for this DIY build.") + sys.exit(1) - # Forces the build to run out of the current directory. - package.stage = DIYStage(os.getcwd()) + # Forces the build to run out of the current directory. + package.stage = DIYStage(os.getcwd()) # TODO: make this an argument, not a global. - spack.do_checksum = False + spack.do_checksum = False - package.do_install( - keep_prefix=args.keep_prefix, - ignore_deps=args.ignore_deps, - keep_stage=True) # don't remove source dir for DIY. + package.do_install( + keep_prefix=args.keep_prefix, + ignore_deps=args.ignore_deps, + keep_stage=True) # don't remove source dir for DIY. diff --git a/lib/spack/spack/cmd/extensions.py b/lib/spack/spack/cmd/extensions.py index e919b1c4fb..66211e29a9 100644 --- a/lib/spack/spack/cmd/extensions.py +++ b/lib/spack/spack/cmd/extensions.py @@ -27,6 +27,7 @@ from external import argparse import llnl.util.tty as tty from llnl.util.tty.colify import colify +from llnl.util.lock import * import spack import spack.cmd @@ -80,7 +81,8 @@ def extensions(parser, args): colify(ext.name for ext in extensions) # List specs of installed extensions. - installed = [s.spec for s in spack.installed_db.installed_extensions_for(spec)] + with Read_Lock_Instance(spack.installed_db.lock,1800): + installed = [s.spec for s in spack.installed_db.installed_extensions_for(spec)] print if not installed: tty.msg("None installed.") diff --git a/lib/spack/spack/cmd/find.py b/lib/spack/spack/cmd/find.py index 6f9072e311..f7fa409ebb 100644 --- a/lib/spack/spack/cmd/find.py +++ b/lib/spack/spack/cmd/find.py @@ -32,6 +32,7 @@ import llnl.util.tty as tty from llnl.util.tty.colify import * from llnl.util.tty.color import * from llnl.util.lang import * +from llnl.util.lock import * import spack import spack.spec @@ -138,9 +139,11 @@ def find(parser, args): # Get all the specs the user asked for if not query_specs: - specs = set(spack.installed_db.installed_package_specs()) + with Read_Lock_Instance(spack.installed_db.lock,1800): + specs = set(spack.installed_db.installed_package_specs()) else: - results = [set(spack.installed_db.get_installed(qs)) for qs in query_specs] + with Read_Lock_Instance(spack.installed_db.lock,1800): + results = [set(spack.installed_db.get_installed(qs)) for qs in query_specs] specs = set.union(*results) if not args.mode: diff --git a/lib/spack/spack/cmd/install.py b/lib/spack/spack/cmd/install.py index acb688a092..330774b8d9 100644 --- a/lib/spack/spack/cmd/install.py +++ b/lib/spack/spack/cmd/install.py @@ -25,6 +25,7 @@ from external import argparse import llnl.util.tty as tty +from llnl.util.lock import * import spack import spack.cmd @@ -68,13 +69,14 @@ def install(parser, args): if args.no_checksum: spack.do_checksum = False # TODO: remove this global. - specs = spack.cmd.parse_specs(args.packages, concretize=True) - for spec in specs: - package = spack.db.get(spec) - package.do_install( - keep_prefix=args.keep_prefix, - keep_stage=args.keep_stage, - ignore_deps=args.ignore_deps, - make_jobs=args.jobs, - verbose=args.verbose, - fake=args.fake) + with Write_Lock_Instance(spack.installed_db.lock,1800): + specs = spack.cmd.parse_specs(args.packages, concretize=True) + for spec in specs: + package = spack.db.get(spec) + package.do_install( + keep_prefix=args.keep_prefix, + keep_stage=args.keep_stage, + ignore_deps=args.ignore_deps, + make_jobs=args.jobs, + verbose=args.verbose, + fake=args.fake) diff --git a/lib/spack/spack/cmd/uninstall.py b/lib/spack/spack/cmd/uninstall.py index 4870712eb6..4b0267dac2 100644 --- a/lib/spack/spack/cmd/uninstall.py +++ b/lib/spack/spack/cmd/uninstall.py @@ -27,6 +27,7 @@ from external import argparse import llnl.util.tty as tty from llnl.util.tty.colify import colify +from llnl.util.lock import * import spack import spack.cmd @@ -53,35 +54,36 @@ def uninstall(parser, args): if not args.packages: tty.die("uninstall requires at least one package argument.") - specs = spack.cmd.parse_specs(args.packages) + with Write_Lock_Instance(spack.installed_db.lock,1800): + specs = spack.cmd.parse_specs(args.packages) - # For each spec provided, make sure it refers to only one package. - # Fail and ask user to be unambiguous if it doesn't - pkgs = [] - for spec in specs: - matching_specs = spack.installed_db.get_installed(spec) - if not args.all and len(matching_specs) > 1: - tty.error("%s matches multiple packages:" % spec) - print - display_specs(matching_specs, long=True) - print - print "You can either:" - print " a) Use a more specific spec, or" - print " b) use spack uninstall -a to uninstall ALL matching specs." - sys.exit(1) + # For each spec provided, make sure it refers to only one package. + # Fail and ask user to be unambiguous if it doesn't + pkgs = [] + for spec in specs: + matching_specs = spack.installed_db.get_installed(spec) + if not args.all and len(matching_specs) > 1: + tty.error("%s matches multiple packages:" % spec) + print + display_specs(matching_specs, long=True) + print + print "You can either:" + print " a) Use a more specific spec, or" + print " b) use spack uninstall -a to uninstall ALL matching specs." + sys.exit(1) - if len(matching_specs) == 0: - if args.force: continue - tty.die("%s does not match any installed packages." % spec) + if len(matching_specs) == 0: + if args.force: continue + tty.die("%s does not match any installed packages." % spec) - for s in matching_specs: - try: - # should work if package is known to spack - pkgs.append(s.package) + for s in matching_specs: + try: + # should work if package is known to spack + pkgs.append(s.package) - except spack.packages.UnknownPackageError, e: - # The package.py file has gone away -- but still want to uninstall. - spack.Package(s).do_uninstall(force=True) + except spack.packages.UnknownPackageError, e: + # The package.py file has gone away -- but still want to uninstall. + spack.Package(s).do_uninstall(force=True) # Sort packages to be uninstalled by the number of installed dependents # This ensures we do things in the right order diff --git a/lib/spack/spack/database.py b/lib/spack/spack/database.py index 6a429b68a9..9b759827d3 100644 --- a/lib/spack/spack/database.py +++ b/lib/spack/spack/database.py @@ -158,12 +158,12 @@ class Database(object): within the same lock, so there is no need to refresh the database within write() """ - temp_name = os.getpid() + socket.getfqdn() + ".temp" - temp_file = path.join(self._root,temp_name) - with open(self.temp_path,'w') as f: + temp_name = str(os.getpid()) + socket.getfqdn() + ".temp" + temp_file = join_path(self._root,temp_name) + with open(temp_file,'w') as f: self._last_write_time = int(time.time()) self._write_database_to_yaml(f) - os.rename(temp_name,self._file_path) + os.rename(temp_file,self._file_path) def is_dirty(self): """ @@ -184,6 +184,7 @@ class Database(object): sph['path']=path sph['hash']=spec.dag_hash() + #Should always already be locked with Write_Lock_Instance(self.lock,60): self.read_database() self._data.append(sph) @@ -197,6 +198,7 @@ class Database(object): Searches for and removes the specified spec Writes the database back to memory """ + #Should always already be locked with Write_Lock_Instance(self.lock,60): self.read_database() @@ -237,6 +239,7 @@ class Database(object): Read installed package names from the database and return their specs """ + #Should always already be locked with Read_Lock_Instance(self.lock,60): self.read_database() -- cgit v1.2.3-70-g09d2 From cd23d2eaa2d06e4a407a4ae88c28d13cbbd7fd1e Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Tue, 15 Sep 2015 14:20:19 -0700 Subject: Added spack fsck and re-read from glob if the database file does not exist. Allows older versions to smoothly upgrade to the database. --- lib/spack/spack/cmd/fsck.py | 43 +++++++++++++++++++++++++++++++++++++++++++ lib/spack/spack/database.py | 18 ++++++++++++++---- 2 files changed, 57 insertions(+), 4 deletions(-) create mode 100644 lib/spack/spack/cmd/fsck.py (limited to 'lib') diff --git a/lib/spack/spack/cmd/fsck.py b/lib/spack/spack/cmd/fsck.py new file mode 100644 index 0000000000..3141a7031d --- /dev/null +++ b/lib/spack/spack/cmd/fsck.py @@ -0,0 +1,43 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from external import argparse + +from llnl.util.lock import * + +import spack +import os + +description = "Correct database irregularities" + +#Very basic version of spack fsck +def fsck(parser, args): + with Write_Lock_Instance(spack.installed_db.lock,1800): + #remove database file + if os.path.exists(spack.installed_db._file_path): + os.remove(spack.installed_db._file_path) + #read database + spack.installed_db.read_database() + #write database + spack.installed_db.write() diff --git a/lib/spack/spack/database.py b/lib/spack/spack/database.py index 9b759827d3..680d184f1f 100644 --- a/lib/spack/spack/database.py +++ b/lib/spack/spack/database.py @@ -123,8 +123,15 @@ class Database(object): with open(self._file_path,'r') as f: self._read_from_yaml(f) else: - #The file doesn't exist, construct empty data. + #The file doesn't exist, construct from file paths self._data = [] + specs = spack.install_layout.all_specs() + for spec in specs: + sph = {} + sph['spec']=spec + sph['hash']=spec.dag_hash() + sph['path']=spack.install_layout.path_for_spec(spec) + self._data.append(sph) def _write_database_to_yaml(self,stream): @@ -167,10 +174,13 @@ class Database(object): def is_dirty(self): """ - Returns true iff the database file exists - and was most recently written to by another spack instance. + Returns true iff the database file does not exist + or was most recently written to by another spack instance. """ - return (os.path.isfile(self._file_path) and (os.path.getmtime(self._file_path) > self._last_write_time)) + if not os.path.isfile(self._file_path): + return True + else: + return (os.path.getmtime(self._file_path) > self._last_write_time) # @_autospec -- cgit v1.2.3-70-g09d2 From ccf311c9c6eb958c16e9621d8a2ef7665a4fa4d7 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Thu, 17 Sep 2015 00:16:12 -0700 Subject: Several changes to DB implementation. 1. Database stores a file version, so we can add to it in the future. 2. Database indexed by hashes and not numerical indexes. 3. Specs built by database have consistent hashes and it's checked. 4. minor naming and whitespace changes. --- lib/spack/llnl/util/filesystem.py | 2 +- lib/spack/llnl/util/lock.py | 27 ++-- lib/spack/spack/cmd/find.py | 5 +- lib/spack/spack/database.py | 321 +++++++++++++++++++++++--------------- lib/spack/spack/error.py | 4 +- lib/spack/spack/spec.py | 2 +- 6 files changed, 221 insertions(+), 140 deletions(-) (limited to 'lib') diff --git a/lib/spack/llnl/util/filesystem.py b/lib/spack/llnl/util/filesystem.py index 029a7536df..03f25d3dff 100644 --- a/lib/spack/llnl/util/filesystem.py +++ b/lib/spack/llnl/util/filesystem.py @@ -222,7 +222,7 @@ def working_dir(dirname, **kwargs): def touch(path): """Creates an empty file at the specified path.""" - with closing(open(path, 'a')) as file: + with open(path, 'a') as file: os.utime(path, None) diff --git a/lib/spack/llnl/util/lock.py b/lib/spack/llnl/util/lock.py index 05641475ed..bb3b15c9cf 100644 --- a/lib/spack/llnl/util/lock.py +++ b/lib/spack/llnl/util/lock.py @@ -22,6 +22,7 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## +"""Lock implementation for shared filesystems.""" import os import fcntl import errno @@ -34,11 +35,13 @@ class Read_Lock_Instance(object): A context manager for getting shared access to the object lock Arguments are lock and timeout (default 5 minutes) """ - def __init__(self,lock,timeout = 300): + def __init__(self, lock, timeout=300): self._lock = lock self._timeout = timeout + def __enter__(self): self._lock.acquire_read(self._timeout) + def __exit__(self,type,value,traceback): self._lock.release_read() @@ -48,17 +51,21 @@ class Write_Lock_Instance(object): A context manager for getting exclusive access to the object lock Arguments are lock and timeout (default 5 minutes) """ - def __init__(self,lock,timeout = 300): + def __init__(self, lock, timeout=300): self._lock = lock self._timeout = timeout + def __enter__(self): self._lock.acquire_write(self._timeout) + def __exit__(self,type,value,traceback): self._lock.release_write() class Lock(object): - def __init__(self,file_path): + """Distributed file-based lock using ``flock``.""" + + def __init__(self, file_path): self._file_path = file_path self._fd = os.open(file_path,os.O_RDWR) self._reads = 0 @@ -71,20 +78,20 @@ class Lock(object): the write lock will be maintained until all locks are released """ if self._reads == 0 and self._writes == 0: - self._lock(fcntl.LOCK_SH,timeout) + self._lock(fcntl.LOCK_SH, timeout) self._reads += 1 - def acquire_write(self,timeout): + def acquire_write(self, timeout): """ Implements recursive lock """ if self._writes == 0: - self._lock(fcntl.LOCK_EX,timeout) + self._lock(fcntl.LOCK_EX, timeout) self._writes += 1 - def _lock(self,op,timeout): + def _lock(self, op, timeout): """ The timeout is implemented using nonblocking flock() to avoid using signals for timing @@ -96,8 +103,8 @@ class Lock(object): try: fcntl.flock(self._fd, op | fcntl.LOCK_NB) if op == fcntl.LOCK_EX: - with open(self._file_path,'w') as f: - f.write("pid = "+str(os.getpid())+", host = "+socket.getfqdn()) + with open(self._file_path, 'w') as f: + f.write("pid = " + str(os.getpid()) + ", host = " + socket.getfqdn()) return except IOError as error: if error.errno == errno.EAGAIN or error.errno == EACCES: @@ -133,4 +140,4 @@ class Lock(object): Releases the lock regardless of mode. Note that read locks may be masquerading as write locks at times, but this removes either. """ - fcntl.flock(self._fd,fcntl.LOCK_UN) + fcntl.flock(self._fd, fcntl.LOCK_UN) diff --git a/lib/spack/spack/cmd/find.py b/lib/spack/spack/cmd/find.py index f7fa409ebb..2d2b884368 100644 --- a/lib/spack/spack/cmd/find.py +++ b/lib/spack/spack/cmd/find.py @@ -139,10 +139,11 @@ def find(parser, args): # Get all the specs the user asked for if not query_specs: - with Read_Lock_Instance(spack.installed_db.lock,1800): + with Read_Lock_Instance(spack.installed_db.lock, 1800): specs = set(spack.installed_db.installed_package_specs()) + else: - with Read_Lock_Instance(spack.installed_db.lock,1800): + with Read_Lock_Instance(spack.installed_db.lock, 1800): results = [set(spack.installed_db.get_installed(qs)) for qs in query_specs] specs = set.union(*results) diff --git a/lib/spack/spack/database.py b/lib/spack/spack/database.py index 680d184f1f..43ba178fed 100644 --- a/lib/spack/spack/database.py +++ b/lib/spack/spack/database.py @@ -1,5 +1,5 @@ ############################################################################## -# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Copyright (c) 2013-2015, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory. # # This file is part of Spack. @@ -23,95 +23,192 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import os -import sys -import inspect -import glob -import imp - import time -import copy -import errno from external import yaml -from external.yaml.error import MarkedYAMLError +from external.yaml.error import MarkedYAMLError, YAMLError import llnl.util.tty as tty -from llnl.util.filesystem import join_path -from llnl.util.lang import * +from llnl.util.filesystem import * from llnl.util.lock import * -import spack.error import spack.spec +from spack.version import Version from spack.spec import Spec from spack.error import SpackError -from spack.virtual import ProviderIndex -from spack.util.naming import mod_to_class, validate_module_name + +# DB goes in this directory underneath the root +_db_dirname = '.spack-db' + +# DB version. This is stuck in the DB file to track changes in format. +_db_version = Version('0.9') def _autospec(function): """Decorator that automatically converts the argument of a single-arg function to a Spec.""" - def converter(self, spec_like, **kwargs): + def converter(self, spec_like, *args, **kwargs): if not isinstance(spec_like, spack.spec.Spec): spec_like = spack.spec.Spec(spec_like) - return function(self, spec_like, **kwargs) + return function(self, spec_like, *args, **kwargs) return converter +class InstallRecord(object): + """A record represents one installation in the DB.""" + def __init__(self, spec, path): + self.spec = spec + self.path = path + + def to_dict(self): + return { 'spec' : self.spec.to_node_dict(), + 'path' : self.path } + + @classmethod + def from_dict(cls, d): + return InstallRecord(d['spec'], d['path']) + + class Database(object): - def __init__(self,root,file_name="_index.yaml"): - """ - Create an empty Database - Location defaults to root/specDB.yaml + def __init__(self, root): + """Create an empty Database. + + Location defaults to root/_index.yaml The individual data are dicts containing spec: the top level spec of a package path: the path to the install of that package dep_hash: a hash of the dependence DAG for that package """ self._root = root - self._file_name = file_name - self._file_path = join_path(self._root,self._file_name) - self._lock_name = "_db_lock" - self._lock_path = join_path(self._root,self._lock_name) + # Set up layout of database files. + self._db_dir = join_path(self._root, _db_dirname) + self._index_path = join_path(self._db_dir, 'index.yaml') + self._lock_path = join_path(self._db_dir, 'lock') + + # Create needed directories and files + if not os.path.exists(self._db_dir): + mkdirp(self._db_dir) + if not os.path.exists(self._lock_path): - open(self._lock_path,'w').close() - self.lock = Lock(self._lock_path) + touch(self._lock_path) - self._data = [] + # initialize rest of state. + self.lock = Lock(self._lock_path) + self._data = {} self._last_write_time = 0 - def _read_from_yaml(self,stream): + def _write_to_yaml(self, stream): + """Write out the databsae to a YAML file.""" + # map from per-spec hash code to installation record. + installs = dict((k, v.to_dict()) for k, v in self._data.items()) + + # databaes includes installation list and version. + + # NOTE: this DB version does not handle multiple installs of + # the same spec well. If there are 2 identical specs with + # different paths, it can't differentiate. + # TODO: fix this before we support multiple install locations. + database = { + 'database' : { + 'installs' : installs, + 'version' : str(_db_version) + } + } + + try: + return yaml.dump(database, stream=stream, default_flow_style=False) + except YAMLError as e: + raise SpackYAMLError("error writing YAML database:", str(e)) + + + def _read_spec_from_yaml(self, hash_key, installs): + """Recursively construct a spec from a hash in a YAML database.""" + # TODO: check validity of hash_key records here. + spec_dict = installs[hash_key]['spec'] + + # Build spec from dict first. + spec = Spec.from_node_dict(spec_dict) + + # Add dependencies from other records in the install DB to + # form a full spec. + for dep_hash in spec_dict[spec.name]['dependencies'].values(): + spec._add_dependency(self._read_spec_from_yaml(dep_hash, installs)) + + return spec + + + def _read_from_yaml(self, stream): """ Fill database from YAML, do not maintain old data Translate the spec portions from node-dict form to spec form """ try: - file = yaml.load(stream) - except MarkedYAMLError, e: + if isinstance(stream, basestring): + with open(stream, 'r') as f: + yfile = yaml.load(f) + else: + yfile = yaml.load(stream) + + except MarkedYAMLError as e: raise SpackYAMLError("error parsing YAML database:", str(e)) - if file is None: + if yfile is None: return - data = {} - for index, sp in file['database'].items(): - spec = Spec.from_node_dict(sp['spec']) - deps = sp['dependency_indices'] - path = sp['path'] - dep_hash = sp['hash'] - db_entry = {'deps':deps, 'spec': spec, 'path': path, 'hash':dep_hash} - data[index] = db_entry + def check(cond, msg): + if not cond: raise CorruptDatabaseError(self._index_path, msg) + + check('database' in yfile, "No 'database' attribute in YAML.") + + # High-level file checks. + db = yfile['database'] + check('installs' in db, "No 'installs' in YAML DB.") + check('version' in db, "No 'version' in YAML DB.") - for sph in data.values(): - for idx in sph['deps']: - sph['spec'].dependencies[data[idx]['spec'].name] = data[idx]['spec'] + # TODO: better version check. + version = Version(db['version']) + if version != _db_version: + raise InvalidDatabaseVersionError(_db_version, version) - self._data = data.values() + # Iterate through database and check each record. + installs = db['installs'] + data = {} + for hash_key, rec in installs.items(): + try: + spec = self._read_spec_from_yaml(hash_key, installs) + spec_hash = spec.dag_hash() + if not spec_hash == hash_key: + tty.warn("Hash mismatch in database: %s -> spec with hash %s" + % (hash_key, spec_hash)) + continue + + data[hash_key] = InstallRecord(spec, rec['path']) + + except Exception as e: + tty.warn("Invalid database reecord:", + "file: %s" % self._index_path, + "hash: %s" % hash_key, + "cause: %s" % str(e)) + raise + + self._data = data + + + def reindex(self, directory_layout): + """Build database index from scratch based from a directory layout.""" + with Write_Lock_Instance(self.lock, 60): + data = {} + for spec in directory_layout.all_specs(): + path = directory_layout.path_for_spec(spec) + hash_key = spec.dag_hash() + data[hash_key] = InstallRecord(spec, path) + self._data = data + self.write() - def read_database(self): + def read(self): """ Re-read Database from the data in the set location If the cache is fresh, return immediately. @@ -119,43 +216,12 @@ class Database(object): if not self.is_dirty(): return - if os.path.isfile(self._file_path): - with open(self._file_path,'r') as f: - self._read_from_yaml(f) + if os.path.isfile(self._index_path): + # Read from YAML file if a database exists + self._read_from_yaml(self._index_path) else: - #The file doesn't exist, construct from file paths - self._data = [] - specs = spack.install_layout.all_specs() - for spec in specs: - sph = {} - sph['spec']=spec - sph['hash']=spec.dag_hash() - sph['path']=spack.install_layout.path_for_spec(spec) - self._data.append(sph) - - - def _write_database_to_yaml(self,stream): - """ - Replace each spec with its dict-node form - Then stream all data to YAML - """ - node_list = [] - spec_list = [sph['spec'] for sph in self._data] - - for sph in self._data: - node = {} - deps = [] - for name,spec in sph['spec'].dependencies.items(): - deps.append(spec_list.index(spec)) - node['spec']=Spec.to_node_dict(sph['spec']) - node['hash']=sph['hash'] - node['path']=sph['path'] - node['dependency_indices']=deps - node_list.append(node) - - node_dict = dict(enumerate(node_list)) - return yaml.dump({ 'database' : node_dict}, - stream=stream, default_flow_style=False) + # The file doesn't exist, try to traverse the directory. + self.reindex(spack.install_layout) def write(self): @@ -165,39 +231,42 @@ class Database(object): within the same lock, so there is no need to refresh the database within write() """ - temp_name = str(os.getpid()) + socket.getfqdn() + ".temp" - temp_file = join_path(self._root,temp_name) - with open(temp_file,'w') as f: - self._last_write_time = int(time.time()) - self._write_database_to_yaml(f) - os.rename(temp_file,self._file_path) + temp_name = '%s.%s.temp' % (socket.getfqdn(), os.getpid()) + temp_file = join_path(self._db_dir, temp_name) + + # Write a temporary database file them move it into place + try: + with open(temp_file, 'w') as f: + self._last_write_time = int(time.time()) + self._write_to_yaml(f) + os.rename(temp_file, self._index_path) + + except: + # Clean up temp file if something goes wrong. + if os.path.exists(temp_file): + os.remove(temp_file) + raise + def is_dirty(self): """ Returns true iff the database file does not exist or was most recently written to by another spack instance. """ - if not os.path.isfile(self._file_path): - return True - else: - return (os.path.getmtime(self._file_path) > self._last_write_time) + return (not os.path.isfile(self._index_path) or + (os.path.getmtime(self._index_path) > self._last_write_time)) -# @_autospec + @_autospec def add(self, spec, path): """Read the database from the set location Add the specified entry as a dict Write the database back to memory """ - sph = {} - sph['spec']=spec - sph['path']=path - sph['hash']=spec.dag_hash() - - #Should always already be locked - with Write_Lock_Instance(self.lock,60): - self.read_database() - self._data.append(sph) + # Should always already be locked + with Write_Lock_Instance(self.lock, 60): + self.read() + self._data[spec.dag_hash()] = InstallRecord(spec, path) self.write() @@ -208,23 +277,18 @@ class Database(object): Searches for and removes the specified spec Writes the database back to memory """ - #Should always already be locked - with Write_Lock_Instance(self.lock,60): - self.read_database() - - for sp in self._data: - #Not sure the hash comparison is necessary - if sp['hash'] == spec.dag_hash() and sp['spec'] == spec: - self._data.remove(sp) - + # Should always already be locked + with Write_Lock_Instance(self.lock, 60): + self.read() + hash_key = spec.dag_hash() + if hash_key in self._data: + del self._data[hash_key] self.write() @_autospec def get_installed(self, spec): - """ - Get all the installed specs that satisfy the provided spec constraint - """ + """Get installed specs that satisfy the provided spec constraint.""" return [s for s in self.installed_package_specs() if s.satisfies(spec)] @@ -238,10 +302,10 @@ class Database(object): try: if s.package.extends(extendee_spec): yield s.package - except UnknownPackageError, e: + except UnknownPackageError as e: continue - #skips unknown packages - #TODO: conditional way to do this instead of catching exceptions + # skips unknown packages + # TODO: conditional way to do this instead of catching exceptions def installed_package_specs(self): @@ -249,14 +313,10 @@ class Database(object): Read installed package names from the database and return their specs """ - #Should always already be locked - with Read_Lock_Instance(self.lock,60): - self.read_database() - - installed = [] - for sph in self._data: - installed.append(sph['spec']) - return installed + # Should always already be locked + with Read_Lock_Instance(self.lock, 60): + self.read() + return sorted(rec.spec for rec in self._data.values()) def installed_known_package_specs(self): @@ -265,5 +325,18 @@ class Database(object): Return only the specs for which the package is known to this version of spack """ - return [s for s in self.installed_package_specs() if spack.db.exists(s.name)] + return [s for s in self.installed_package_specs() + if spack.db.exists(s.name)] + + +class CorruptDatabaseError(SpackError): + def __init__(self, path, msg=''): + super(CorruptDatabaseError, self).__init__( + "Spack database is corrupt: %s. %s" %(path, msg)) + +class InvalidDatabaseVersionError(SpackError): + def __init__(self, expected, found): + super(InvalidDatabaseVersionError, self).__init__( + "Expected database version %s but found version %s" + % (expected, found)) diff --git a/lib/spack/spack/error.py b/lib/spack/spack/error.py index bfa7951a47..b3b24e6105 100644 --- a/lib/spack/spack/error.py +++ b/lib/spack/spack/error.py @@ -55,8 +55,8 @@ class SpackError(Exception): def __str__(self): msg = self.message - if self.long_message: - msg += "\n %s" % self.long_message + if self._long_message: + msg += "\n %s" % self._long_message return msg class UnsupportedPlatformError(SpackError): diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index cde2e168a0..7b79feb311 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -2000,4 +2000,4 @@ class UnsatisfiableDependencySpecError(UnsatisfiableSpecError): class SpackYAMLError(spack.error.SpackError): def __init__(self, msg, yaml_error): - super(SpackError, self).__init__(msg, str(yaml_error)) + super(SpackYAMLError, self).__init__(msg, str(yaml_error)) -- cgit v1.2.3-70-g09d2 From 6dff42be0973c5949d836eab51cfdffb0eda0a69 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Thu, 17 Sep 2015 00:21:33 -0700 Subject: WIP for Matt's branch --- bin/spack | 4 ++-- lib/spack/spack/directives.py | 4 ++-- lib/spack/spack/package.py | 6 +++++ lib/spack/spack/packages.py | 18 ++++++++++----- lib/spack/spack/patch.py | 6 ++++- lib/spack/spack/spec.py | 21 ++++++------------ lib/spack/spack/test/mock_packages_test.py | 35 ++++++++++++++++++++++-------- lib/spack/spack/test/package_sanity.py | 2 +- lib/spack/spack/test/spec_dag.py | 12 +++++----- 9 files changed, 68 insertions(+), 40 deletions(-) (limited to 'lib') diff --git a/bin/spack b/bin/spack index 5c042edd2d..baf08d1481 100755 --- a/bin/spack +++ b/bin/spack @@ -113,8 +113,8 @@ def main(): spack.spack_working_dir = working_dir if args.mock: - from spack.packages import PackageDB - spack.db = PackageDB(spack.mock_packages_path) + from spack.packages import PackageFinder + spack.db.swap(PackageFinder(spack.mock_packages_path)) # If the user asked for it, don't check ssl certs. if args.insecure: diff --git a/lib/spack/spack/directives.py b/lib/spack/spack/directives.py index 9297d6dac3..aa2cfd2846 100644 --- a/lib/spack/spack/directives.py +++ b/lib/spack/spack/directives.py @@ -240,11 +240,11 @@ def patch(pkg, url_or_filename, level=1, when=None): when_spec = parse_anonymous_spec(when, pkg.name) if when_spec not in pkg.patches: - pkg.patches[when_spec] = [Patch(pkg.name, url_or_filename, level)] + pkg.patches[when_spec] = [Patch(pkg, pkg.name, url_or_filename, level)] else: # if this spec is identical to some other, then append this # patch to the existing list. - pkg.patches[when_spec].append(Patch(pkg.name, url_or_filename, level)) + pkg.patches[when_spec].append(Patch(pkg, pkg.name, url_or_filename, level)) @directive('variants') diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 3507807373..090349685b 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -712,6 +712,12 @@ class Package(object): tty.msg("Patched %s" % self.name) + @property + def namespace(self): + namespace, dot, module = self.__module__.rpartition('.') + return namespace + + def do_fake_install(self): """Make a fake install directory contaiing a 'fake' file in bin.""" mkdirp(self.prefix.bin) diff --git a/lib/spack/spack/packages.py b/lib/spack/spack/packages.py index df54b12324..b21316ebf7 100644 --- a/lib/spack/spack/packages.py +++ b/lib/spack/spack/packages.py @@ -172,7 +172,7 @@ class PackageFinder(object): def providers_for(self, vpkg_name): - # TODO: USE MORE THAN FIRST REPO + # TODO: THIS IS WRONG; shoudl use more than first repo return self.repos[0].providers_for(vpkg_name) @@ -252,14 +252,22 @@ class PackageFinder(object): return module - @_autospec - def get(self, spec, new=False): + def _find_repo_for_spec(self, spec): + """Find a repo that contains the supplied spec's package. + + Raises UnknownPackageErrorfor if not found. + """ for repo in self.repos: if spec.name in repo: - return repo.get(spec, new) + return repo raise UnknownPackageError(spec.name) + @_autospec + def get(self, spec, new=False): + return self._find_repo_for_spec(spec).get(spec, new) + + def get_repo(self, namespace): if namespace in self.by_namespace: repo = self.by_namespace[namespace] @@ -343,7 +351,7 @@ class PackageDB(object): del self._instances[spec] if not spec in self._instances: - package_class = self.get_class_for_package_name(spec.name, spec.repo) + package_class = self.get_class_for_package_name(spec.name, spec.namespace) try: copy = spec.copy() self._instances[copy] = package_class(copy) diff --git a/lib/spack/spack/patch.py b/lib/spack/spack/patch.py index b1b6e07738..e89cf11b2f 100644 --- a/lib/spack/spack/patch.py +++ b/lib/spack/spack/patch.py @@ -41,7 +41,11 @@ class Patch(object): """This class describes a patch to be applied to some expanded source code.""" - def __init__(self, pkg_name, path_or_url, level): + def __init__(self, pkg, pkg_name, path_or_url, level): + print pkg, pkg.name, type(pkg) + print "pkg:", dir(pkg.__module__) + print "NAMESPACE", pkg.namespace() + self.pkg_name = pkg_name self.path_or_url = path_or_url self.path = None diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 1666457502..0d49b1fa95 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -112,7 +112,6 @@ from spack.version import * from spack.util.string import * from spack.util.prefix import Prefix from spack.virtual import ProviderIndex -from spack.repo_loader import imported_packages_module # Valid pattern for an identifier in Spack identifier_re = r'\w[\w-]*' @@ -413,7 +412,7 @@ class Spec(object): self.dependencies = other.dependencies self.variants = other.variants self.variants.spec = self - self.repo = other.repo + self.namespace = other.namespace # Specs are by default not assumed to be normal, but in some # cases we've read them from a file want to assume normal. @@ -1357,7 +1356,7 @@ class Spec(object): self.dependencies = DependencyMap() self.variants = other.variants.copy() self.variants.spec = self - self.repo = other.repo + self.namespace = other.namespace # If we copy dependencies, preserve DAG structure in the new spec if kwargs.get('deps', True): @@ -1555,7 +1554,7 @@ class Spec(object): if c == '_': out.write(fmt % self.name) elif c == '.': - longname = '%s.%s.%s' % (imported_packages_module, self.repo, self.name) if self.repo else self.name + longname = '%s.%s.%s' % (self.namespace, self.name) if self.namespace else self.name out.write(fmt % longname) elif c == '@': if self.versions and self.versions != _any_version: @@ -1706,15 +1705,9 @@ class SpecParser(spack.parse.Parser): """Parse a spec out of the input. If a spec is supplied, then initialize and return it instead of creating a new one.""" - spec_name = None - spec_repo = None - if self.token.value.startswith(imported_packages_module): - lst = self.token.value.split('.') - spec_name = lst[-1] - spec_repo = lst[-2] - else: - spec_name = self.token.value - spec_repo = 'gov.llnl.spack' + spec_namespace, dot, spec_name = self.token.value.rpartition('.') + if not spec_namespace: + spec_namespace = None self.check_identifier(spec_name) @@ -1727,7 +1720,7 @@ class SpecParser(spack.parse.Parser): spec.compiler = None spec.dependents = DependencyMap() spec.dependencies = DependencyMap() - spec.repo = spec_repo + spec.namespace = spec_namespace spec._normal = False spec._concrete = False diff --git a/lib/spack/spack/test/mock_packages_test.py b/lib/spack/spack/test/mock_packages_test.py index 1f46d65790..071c21b7e0 100644 --- a/lib/spack/spack/test/mock_packages_test.py +++ b/lib/spack/spack/test/mock_packages_test.py @@ -31,14 +31,6 @@ from spack.packages import PackageFinder from spack.spec import Spec -def set_pkg_dep(pkg, spec): - """Alters dependence information for a package. - Use this to mock up constraints. - """ - spec = Spec(spec) - spack.db.get(pkg).dependencies[spec.name] = { Spec(pkg) : spec } - - class MockPackagesTest(unittest.TestCase): def initmock(self): # Use the mock packages database for these tests. This allows @@ -53,14 +45,39 @@ class MockPackagesTest(unittest.TestCase): ('site', spack.mock_site_config), ('user', spack.mock_user_config)] + # Store changes to the package's dependencies so we can + # restore later. + self.saved_deps = {} + + + def set_pkg_dep(self, pkg_name, spec): + """Alters dependence information for a package. + + Adds a dependency on to pkg. + Use this to mock up constraints. + """ + spec = Spec(spec) + + # Save original dependencies before making any changes. + pkg = spack.db.get(pkg_name) + if pkg_name not in self.saved_deps: + self.saved_deps[pkg_name] = (pkg, pkg.dependencies.copy()) + + # Change dep spec + pkg.dependencies[spec.name] = { Spec(pkg_name) : spec } + def cleanmock(self): """Restore the real packages path after any test.""" spack.db.swap(self.db) - spack.config.config_scopes = self.real_scopes spack.config.clear_config_caches() + # Restore dependency changes that happened during the test + for pkg_name, (pkg, deps) in self.saved_deps.items(): + pkg.dependencies.clear() + pkg.dependencies.update(deps) + def setUp(self): self.initmock() diff --git a/lib/spack/spack/test/package_sanity.py b/lib/spack/spack/test/package_sanity.py index 70b5d6a478..a5925ea066 100644 --- a/lib/spack/spack/test/package_sanity.py +++ b/lib/spack/spack/test/package_sanity.py @@ -29,7 +29,7 @@ import unittest import spack import spack.url as url -from spack.packages import PackageDB +from spack.packages import PackageFinder class PackageSanityTest(unittest.TestCase): diff --git a/lib/spack/spack/test/spec_dag.py b/lib/spack/spack/test/spec_dag.py index 549f829d3e..a71026d183 100644 --- a/lib/spack/spack/test/spec_dag.py +++ b/lib/spack/spack/test/spec_dag.py @@ -40,8 +40,8 @@ from spack.test.mock_packages_test import * class SpecDagTest(MockPackagesTest): def test_conflicting_package_constraints(self): - set_pkg_dep('mpileaks', 'mpich@1.0') - set_pkg_dep('callpath', 'mpich@2.0') + self.set_pkg_dep('mpileaks', 'mpich@1.0') + self.set_pkg_dep('callpath', 'mpich@2.0') spec = Spec('mpileaks ^mpich ^callpath ^dyninst ^libelf ^libdwarf') @@ -223,25 +223,25 @@ class SpecDagTest(MockPackagesTest): def test_unsatisfiable_version(self): - set_pkg_dep('mpileaks', 'mpich@1.0') + self.set_pkg_dep('mpileaks', 'mpich@1.0') spec = Spec('mpileaks ^mpich@2.0 ^callpath ^dyninst ^libelf ^libdwarf') self.assertRaises(spack.spec.UnsatisfiableVersionSpecError, spec.normalize) def test_unsatisfiable_compiler(self): - set_pkg_dep('mpileaks', 'mpich%gcc') + self.set_pkg_dep('mpileaks', 'mpich%gcc') spec = Spec('mpileaks ^mpich%intel ^callpath ^dyninst ^libelf ^libdwarf') self.assertRaises(spack.spec.UnsatisfiableCompilerSpecError, spec.normalize) def test_unsatisfiable_compiler_version(self): - set_pkg_dep('mpileaks', 'mpich%gcc@4.6') + self.set_pkg_dep('mpileaks', 'mpich%gcc@4.6') spec = Spec('mpileaks ^mpich%gcc@4.5 ^callpath ^dyninst ^libelf ^libdwarf') self.assertRaises(spack.spec.UnsatisfiableCompilerSpecError, spec.normalize) def test_unsatisfiable_architecture(self): - set_pkg_dep('mpileaks', 'mpich=bgqos_0') + self.set_pkg_dep('mpileaks', 'mpich=bgqos_0') spec = Spec('mpileaks ^mpich=sles_10_ppc64 ^callpath ^dyninst ^libelf ^libdwarf') self.assertRaises(spack.spec.UnsatisfiableArchitectureSpecError, spec.normalize) -- cgit v1.2.3-70-g09d2 From e17ad6a684b94211e9b4267cca68cc7fdf6ad277 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Thu, 17 Sep 2015 01:05:19 -0700 Subject: Simplify lock context managers. --- lib/spack/llnl/util/lock.py | 30 +++++++++++++++++++++--------- lib/spack/spack/cmd/__init__.py | 3 +-- lib/spack/spack/cmd/deactivate.py | 3 +-- lib/spack/spack/cmd/diy.py | 3 +-- lib/spack/spack/cmd/extensions.py | 3 +-- lib/spack/spack/cmd/find.py | 5 ++--- lib/spack/spack/cmd/fsck.py | 17 +++-------------- lib/spack/spack/cmd/install.py | 3 +-- lib/spack/spack/cmd/uninstall.py | 3 +-- lib/spack/spack/database.py | 22 +++++++++++++++++----- 10 files changed, 49 insertions(+), 43 deletions(-) (limited to 'lib') diff --git a/lib/spack/llnl/util/lock.py b/lib/spack/llnl/util/lock.py index bb3b15c9cf..3cd02befe5 100644 --- a/lib/spack/llnl/util/lock.py +++ b/lib/spack/llnl/util/lock.py @@ -29,13 +29,15 @@ import errno import time import socket +# Default timeout for locks. +DEFAULT_TIMEOUT = 60 + +class _ReadLockContext(object): + """Context manager that takes and releases a read lock. -class Read_Lock_Instance(object): - """ - A context manager for getting shared access to the object lock Arguments are lock and timeout (default 5 minutes) """ - def __init__(self, lock, timeout=300): + def __init__(self, lock, timeout=DEFAULT_TIMEOUT): self._lock = lock self._timeout = timeout @@ -46,12 +48,12 @@ class Read_Lock_Instance(object): self._lock.release_read() -class Write_Lock_Instance(object): - """ - A context manager for getting exclusive access to the object lock +class _WriteLockContext(object): + """Context manager that takes and releases a write lock. + Arguments are lock and timeout (default 5 minutes) """ - def __init__(self, lock, timeout=300): + def __init__(self, lock, timeout=DEFAULT_TIMEOUT): self._lock = lock self._timeout = timeout @@ -72,7 +74,17 @@ class Lock(object): self._writes = 0 - def acquire_read(self,timeout): + def write_lock(self, timeout=DEFAULT_TIMEOUT): + """Convenience method that returns a write lock context.""" + return _WriteLockContext(self, timeout) + + + def read_lock(self, timeout=DEFAULT_TIMEOUT): + """Convenience method that returns a read lock context.""" + return _ReadLockContext(self, timeout) + + + def acquire_read(self, timeout): """ Implements recursive lock. If held in both read and write mode, the write lock will be maintained until all locks are released diff --git a/lib/spack/spack/cmd/__init__.py b/lib/spack/spack/cmd/__init__.py index c62d22979a..a8e8b1a48b 100644 --- a/lib/spack/spack/cmd/__init__.py +++ b/lib/spack/spack/cmd/__init__.py @@ -28,7 +28,6 @@ import sys import llnl.util.tty as tty from llnl.util.lang import attr_setdefault -from llnl.util.lock import * import spack import spack.spec @@ -125,7 +124,7 @@ def elide_list(line_list, max_num=10): def disambiguate_spec(spec): - with Read_Lock_Instance(spack.installed_db.lock,1800): + with spack.installed_db.read_lock(): matching_specs = spack.installed_db.get_installed(spec) if not matching_specs: tty.die("Spec '%s' matches no installed packages." % spec) diff --git a/lib/spack/spack/cmd/deactivate.py b/lib/spack/spack/cmd/deactivate.py index 015809345a..5428e3d2de 100644 --- a/lib/spack/spack/cmd/deactivate.py +++ b/lib/spack/spack/cmd/deactivate.py @@ -24,7 +24,6 @@ ############################################################################## from external import argparse import llnl.util.tty as tty -from llnl.util.lock import * import spack import spack.cmd @@ -55,7 +54,7 @@ def deactivate(parser, args): if args.all: if pkg.extendable: tty.msg("Deactivating all extensions of %s" % pkg.spec.short_spec) - with Read_Lock_Instance(spack.installed_db.lock,1800): + with spack.installed_db.read_lock(): ext_pkgs = spack.installed_db.installed_extensions_for(spec) for ext_pkg in ext_pkgs: diff --git a/lib/spack/spack/cmd/diy.py b/lib/spack/spack/cmd/diy.py index 7403b9e3e8..6178c9c3e3 100644 --- a/lib/spack/spack/cmd/diy.py +++ b/lib/spack/spack/cmd/diy.py @@ -27,7 +27,6 @@ import os from external import argparse import llnl.util.tty as tty -from llnl.util.lock import * import spack import spack.cmd @@ -55,7 +54,7 @@ def diy(self, args): if not args.spec: tty.die("spack diy requires a package spec argument.") - with Write_Lock_Instance(spack.installed_db.lock,1800): + with spack.installed_db.write_lock(): specs = spack.cmd.parse_specs(args.spec) if len(specs) > 1: tty.die("spack diy only takes one spec.") diff --git a/lib/spack/spack/cmd/extensions.py b/lib/spack/spack/cmd/extensions.py index 66211e29a9..f0f99a2691 100644 --- a/lib/spack/spack/cmd/extensions.py +++ b/lib/spack/spack/cmd/extensions.py @@ -27,7 +27,6 @@ from external import argparse import llnl.util.tty as tty from llnl.util.tty.colify import colify -from llnl.util.lock import * import spack import spack.cmd @@ -81,7 +80,7 @@ def extensions(parser, args): colify(ext.name for ext in extensions) # List specs of installed extensions. - with Read_Lock_Instance(spack.installed_db.lock,1800): + with spack.installed_db.read_lock(): installed = [s.spec for s in spack.installed_db.installed_extensions_for(spec)] print if not installed: diff --git a/lib/spack/spack/cmd/find.py b/lib/spack/spack/cmd/find.py index 2d2b884368..e2edd454f4 100644 --- a/lib/spack/spack/cmd/find.py +++ b/lib/spack/spack/cmd/find.py @@ -32,7 +32,6 @@ import llnl.util.tty as tty from llnl.util.tty.colify import * from llnl.util.tty.color import * from llnl.util.lang import * -from llnl.util.lock import * import spack import spack.spec @@ -139,11 +138,11 @@ def find(parser, args): # Get all the specs the user asked for if not query_specs: - with Read_Lock_Instance(spack.installed_db.lock, 1800): + with spack.installed_db.read_lock(): specs = set(spack.installed_db.installed_package_specs()) else: - with Read_Lock_Instance(spack.installed_db.lock, 1800): + with spack.installed_db.read_lock(): results = [set(spack.installed_db.get_installed(qs)) for qs in query_specs] specs = set.union(*results) diff --git a/lib/spack/spack/cmd/fsck.py b/lib/spack/spack/cmd/fsck.py index 3141a7031d..9a3c450dcf 100644 --- a/lib/spack/spack/cmd/fsck.py +++ b/lib/spack/spack/cmd/fsck.py @@ -1,5 +1,5 @@ ############################################################################## -# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Copyright (c) 2013-2015, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory. # # This file is part of Spack. @@ -23,21 +23,10 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## from external import argparse - -from llnl.util.lock import * - import spack -import os description = "Correct database irregularities" -#Very basic version of spack fsck +# Very basic version of spack fsck def fsck(parser, args): - with Write_Lock_Instance(spack.installed_db.lock,1800): - #remove database file - if os.path.exists(spack.installed_db._file_path): - os.remove(spack.installed_db._file_path) - #read database - spack.installed_db.read_database() - #write database - spack.installed_db.write() + spack.installed_db.reindex(spack.install_layout) diff --git a/lib/spack/spack/cmd/install.py b/lib/spack/spack/cmd/install.py index 330774b8d9..ada655b937 100644 --- a/lib/spack/spack/cmd/install.py +++ b/lib/spack/spack/cmd/install.py @@ -25,7 +25,6 @@ from external import argparse import llnl.util.tty as tty -from llnl.util.lock import * import spack import spack.cmd @@ -69,7 +68,7 @@ def install(parser, args): if args.no_checksum: spack.do_checksum = False # TODO: remove this global. - with Write_Lock_Instance(spack.installed_db.lock,1800): + with spack.installed_db.write_lock(): specs = spack.cmd.parse_specs(args.packages, concretize=True) for spec in specs: package = spack.db.get(spec) diff --git a/lib/spack/spack/cmd/uninstall.py b/lib/spack/spack/cmd/uninstall.py index 4b0267dac2..7425db3ca3 100644 --- a/lib/spack/spack/cmd/uninstall.py +++ b/lib/spack/spack/cmd/uninstall.py @@ -27,7 +27,6 @@ from external import argparse import llnl.util.tty as tty from llnl.util.tty.colify import colify -from llnl.util.lock import * import spack import spack.cmd @@ -54,7 +53,7 @@ def uninstall(parser, args): if not args.packages: tty.die("uninstall requires at least one package argument.") - with Write_Lock_Instance(spack.installed_db.lock,1800): + with spack.installed_db.write_lock(): specs = spack.cmd.parse_specs(args.packages) # For each spec provided, make sure it refers to only one package. diff --git a/lib/spack/spack/database.py b/lib/spack/spack/database.py index 43ba178fed..cea56eb1b9 100644 --- a/lib/spack/spack/database.py +++ b/lib/spack/spack/database.py @@ -24,13 +24,14 @@ ############################################################################## import os import time +import socket from external import yaml from external.yaml.error import MarkedYAMLError, YAMLError import llnl.util.tty as tty from llnl.util.filesystem import * -from llnl.util.lock import * +from llnl.util.lock import Lock import spack.spec from spack.version import Version @@ -99,6 +100,16 @@ class Database(object): self._last_write_time = 0 + def write_lock(self): + """Get a write lock context for use in a `with` block.""" + return self.lock.write_lock() + + + def read_lock(self): + """Get a read lock context for use in a `with` block.""" + return self.lock.read_lock() + + def _write_to_yaml(self, stream): """Write out the databsae to a YAML file.""" # map from per-spec hash code to installation record. @@ -198,13 +209,14 @@ class Database(object): def reindex(self, directory_layout): """Build database index from scratch based from a directory layout.""" - with Write_Lock_Instance(self.lock, 60): + with self.write_lock(): data = {} for spec in directory_layout.all_specs(): path = directory_layout.path_for_spec(spec) hash_key = spec.dag_hash() data[hash_key] = InstallRecord(spec, path) self._data = data + self.write() @@ -264,7 +276,7 @@ class Database(object): Write the database back to memory """ # Should always already be locked - with Write_Lock_Instance(self.lock, 60): + with self.write_lock(): self.read() self._data[spec.dag_hash()] = InstallRecord(spec, path) self.write() @@ -278,7 +290,7 @@ class Database(object): Writes the database back to memory """ # Should always already be locked - with Write_Lock_Instance(self.lock, 60): + with self.write_lock(): self.read() hash_key = spec.dag_hash() if hash_key in self._data: @@ -314,7 +326,7 @@ class Database(object): and return their specs """ # Should always already be locked - with Read_Lock_Instance(self.lock, 60): + with self.read_lock(): self.read() return sorted(rec.spec for rec in self._data.values()) -- cgit v1.2.3-70-g09d2 From fb73979345d16b4912ea1f6da148d35c676a6576 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Thu, 17 Sep 2015 16:09:59 -0700 Subject: Allow custom timeout for database locking. --- lib/spack/spack/database.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/database.py b/lib/spack/spack/database.py index cea56eb1b9..e74217a262 100644 --- a/lib/spack/spack/database.py +++ b/lib/spack/spack/database.py @@ -44,6 +44,8 @@ _db_dirname = '.spack-db' # DB version. This is stuck in the DB file to track changes in format. _db_version = Version('0.9') +# Default timeout for spack database locks is 5 min. +_db_lock_timeout = 300 def _autospec(function): """Decorator that automatically converts the argument of a single-arg @@ -100,14 +102,14 @@ class Database(object): self._last_write_time = 0 - def write_lock(self): + def write_lock(self, timeout=_db_lock_timeout): """Get a write lock context for use in a `with` block.""" - return self.lock.write_lock() + return self.lock.write_lock(timeout) - def read_lock(self): + def read_lock(self, timeout=_db_lock_timeout): """Get a read lock context for use in a `with` block.""" - return self.lock.read_lock() + return self.lock.read_lock(timeout) def _write_to_yaml(self, stream): -- cgit v1.2.3-70-g09d2 From d0e22b22406c8fb064031dbd4ac887b7a9abbc95 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Fri, 18 Sep 2015 11:40:05 -0700 Subject: Add ref counting to database. This does not handle removal properly yet. --- lib/spack/spack/cmd/__init__.py | 2 +- lib/spack/spack/cmd/find.py | 35 ++++++-- lib/spack/spack/cmd/module.py | 4 +- lib/spack/spack/cmd/uninstall.py | 2 +- lib/spack/spack/database.py | 188 ++++++++++++++++++++++++++++++--------- lib/spack/spack/package.py | 7 +- 6 files changed, 183 insertions(+), 55 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/cmd/__init__.py b/lib/spack/spack/cmd/__init__.py index a8e8b1a48b..d4778b1375 100644 --- a/lib/spack/spack/cmd/__init__.py +++ b/lib/spack/spack/cmd/__init__.py @@ -125,7 +125,7 @@ def elide_list(line_list, max_num=10): def disambiguate_spec(spec): with spack.installed_db.read_lock(): - matching_specs = spack.installed_db.get_installed(spec) + matching_specs = spack.installed_db.query(spec) if not matching_specs: tty.die("Spec '%s' matches no installed packages." % spec) diff --git a/lib/spack/spack/cmd/find.py b/lib/spack/spack/cmd/find.py index e2edd454f4..6a0c3d11ff 100644 --- a/lib/spack/spack/cmd/find.py +++ b/lib/spack/spack/cmd/find.py @@ -54,6 +54,16 @@ def setup_parser(subparser): '-L', '--very-long', action='store_true', dest='very_long', help='Show dependency hashes as well as versions.') + subparser.add_argument( + '-u', '--unknown', action='store_true', dest='unknown', + help='Show only specs Spack does not have a package for.') + subparser.add_argument( + '-m', '--missing', action='store_true', dest='missing', + help='Show missing dependencies as well as installed specs.') + subparser.add_argument( + '-M', '--only-missing', action='store_true', dest='only_missing', + help='Show only missing dependencies.') + subparser.add_argument( 'query_specs', nargs=argparse.REMAINDER, help='optional specs to filter results') @@ -113,6 +123,7 @@ def display_specs(specs, **kwargs): if hashes: string += gray_hash(s, hlen) + ' ' string += s.format('$-_$@$+', color=True) + return string colify(fmt(s) for s in specs) @@ -136,15 +147,23 @@ def find(parser, args): if not query_specs: return + # Set up query arguments. + installed, known = True, any + if args.only_missing: + installed = False + elif args.missing: + installed = any + if args.unknown: + known = False + q_args = { 'installed' : installed, 'known' : known } + # Get all the specs the user asked for - if not query_specs: - with spack.installed_db.read_lock(): - specs = set(spack.installed_db.installed_package_specs()) - - else: - with spack.installed_db.read_lock(): - results = [set(spack.installed_db.get_installed(qs)) for qs in query_specs] - specs = set.union(*results) + with spack.installed_db.read_lock(): + if not query_specs: + specs = set(spack.installed_db.query(**q_args)) + else: + results = [set(spack.installed_db.query(qs, **q_args)) for qs in query_specs] + specs = set.union(*results) if not args.mode: args.mode = 'short' diff --git a/lib/spack/spack/cmd/module.py b/lib/spack/spack/cmd/module.py index 215d877bd0..654b0cb2fa 100644 --- a/lib/spack/spack/cmd/module.py +++ b/lib/spack/spack/cmd/module.py @@ -65,7 +65,7 @@ def module_find(mtype, spec_array): tty.die("You can only pass one spec.") spec = specs[0] - specs = [s for s in spack.installed_db.installed_package_specs() if s.satisfies(spec)] + specs = spack.installed_db.query(spec) if len(specs) == 0: tty.die("No installed packages match spec %s" % spec) @@ -86,7 +86,7 @@ def module_find(mtype, spec_array): def module_refresh(): """Regenerate all module files for installed packages known to spack (some packages may no longer exist).""" - specs = [s for s in spack.installed_db.installed_known_package_specs()] + specs = [s for s in spack.installed_db.query(installed=True, known=True)] for name, cls in module_types.items(): tty.msg("Regenerating %s module files." % name) diff --git a/lib/spack/spack/cmd/uninstall.py b/lib/spack/spack/cmd/uninstall.py index 7425db3ca3..7b7c32c065 100644 --- a/lib/spack/spack/cmd/uninstall.py +++ b/lib/spack/spack/cmd/uninstall.py @@ -60,7 +60,7 @@ def uninstall(parser, args): # Fail and ask user to be unambiguous if it doesn't pkgs = [] for spec in specs: - matching_specs = spack.installed_db.get_installed(spec) + matching_specs = spack.installed_db.query(spec) if not args.all and len(matching_specs) > 1: tty.error("%s matches multiple packages:" % spec) print diff --git a/lib/spack/spack/database.py b/lib/spack/spack/database.py index e74217a262..1d1c640d66 100644 --- a/lib/spack/spack/database.py +++ b/lib/spack/spack/database.py @@ -22,6 +22,23 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## +"""Spack's installation tracking database. + +The database serves two purposes: + + 1. It implements a cache on top of a potentially very large Spack + directory hierarchy, speeding up many operations that would + otherwise require filesystem access. + + 2. It will allow us to track external installations as well as lost + packages and their dependencies. + +Prior ot the implementation of this store, a direcotry layout served +as the authoritative database of packages in Spack. This module +provides a cache and a sanity checking mechanism for what is in the +filesystem. + +""" import os import time import socket @@ -58,18 +75,37 @@ def _autospec(function): class InstallRecord(object): - """A record represents one installation in the DB.""" - def __init__(self, spec, path): + """A record represents one installation in the DB. + + The record keeps track of the spec for the installation, its + install path, AND whether or not it is installed. We need the + installed flag in case a user either: + + a) blew away a directory, or + b) used spack uninstall -f to get rid of it + + If, in either case, the package was removed but others still + depend on it, we still need to track its spec, so we don't + actually remove from the database until a spec has no installed + dependents left. + + """ + def __init__(self, spec, path, installed): self.spec = spec self.path = path + self.installed = installed + self.ref_count = 0 def to_dict(self): - return { 'spec' : self.spec.to_node_dict(), - 'path' : self.path } + return { 'spec' : self.spec.to_node_dict(), + 'path' : self.path, + 'installed' : self.installed, + 'ref_count' : self.ref_count } @classmethod def from_dict(cls, d): - return InstallRecord(d['spec'], d['path']) + # TODO: check the dict more rigorously. + return InstallRecord(d['spec'], d['path'], d['installed'], d['ref_count']) class Database(object): @@ -136,9 +172,11 @@ class Database(object): raise SpackYAMLError("error writing YAML database:", str(e)) - def _read_spec_from_yaml(self, hash_key, installs): + def _read_spec_from_yaml(self, hash_key, installs, parent_key=None): """Recursively construct a spec from a hash in a YAML database.""" - # TODO: check validity of hash_key records here. + if hash_key not in installs: + parent = read_spec(installs[parent_key]['path']) + spec_dict = installs[hash_key]['spec'] # Build spec from dict first. @@ -147,7 +185,8 @@ class Database(object): # Add dependencies from other records in the install DB to # form a full spec. for dep_hash in spec_dict[spec.name]['dependencies'].values(): - spec._add_dependency(self._read_spec_from_yaml(dep_hash, installs)) + child = self._read_spec_from_yaml(dep_hash, installs, hash_key) + spec._add_dependency(child) return spec @@ -175,12 +214,12 @@ class Database(object): check('database' in yfile, "No 'database' attribute in YAML.") - # High-level file checks. + # High-level file checks db = yfile['database'] check('installs' in db, "No 'installs' in YAML DB.") check('version' in db, "No 'version' in YAML DB.") - # TODO: better version check. + # TODO: better version checking semantics. version = Version(db['version']) if version != _db_version: raise InvalidDatabaseVersionError(_db_version, version) @@ -190,14 +229,21 @@ class Database(object): data = {} for hash_key, rec in installs.items(): try: + # This constructs a spec DAG from the list of all installs spec = self._read_spec_from_yaml(hash_key, installs) + + # Validate the spec by ensuring the stored and actual + # hashes are the same. spec_hash = spec.dag_hash() if not spec_hash == hash_key: tty.warn("Hash mismatch in database: %s -> spec with hash %s" % (hash_key, spec_hash)) - continue + continue # TODO: is skipping the right thing to do? - data[hash_key] = InstallRecord(spec, rec['path']) + # Insert the brand new spec in the database. Each + # spec has its own copies of its dependency specs. + # TODO: would a more immmutable spec implementation simplify this? + data[hash_key] = InstallRecord(spec, rec['path'], rec['installed']) except Exception as e: tty.warn("Invalid database reecord:", @@ -213,12 +259,29 @@ class Database(object): """Build database index from scratch based from a directory layout.""" with self.write_lock(): data = {} + + # Ask the directory layout to traverse the filesystem. for spec in directory_layout.all_specs(): + # Create a spec for each known package and add it. path = directory_layout.path_for_spec(spec) hash_key = spec.dag_hash() - data[hash_key] = InstallRecord(spec, path) + data[hash_key] = InstallRecord(spec, path, True) + + # Recursively examine dependencies and add them, even + # if they are NOT installed. This ensures we know + # about missing dependencies. + for dep in spec.traverse(root=False): + dep_hash = dep.dag_hash() + if dep_hash not in data: + path = directory_layout.path_for_spec(dep) + installed = os.path.isdir(path) + data[dep_hash] = InstallRecord(dep.copy(), path, installed) + data[dep_hash].ref_count += 1 + + # Assuming everything went ok, replace this object's data. self._data = data + # write out, blowing away the old version if necessary self.write() @@ -274,22 +337,37 @@ class Database(object): @_autospec def add(self, spec, path): """Read the database from the set location - Add the specified entry as a dict - Write the database back to memory + + Add the specified entry as a dict, then write the database + back to memory. This assumes that ALL dependencies are already in + the database. Should not be called otherwise. + """ # Should always already be locked with self.write_lock(): self.read() - self._data[spec.dag_hash()] = InstallRecord(spec, path) + self._data[spec.dag_hash()] = InstallRecord(spec, path, True) + + # sanity check the dependencies in case something went + # wrong during install() + # TODO: ensure no races during distributed install. + for dep in spec.traverse(root=False): + assert dep.dag_hash() in self._data + self.write() @_autospec def remove(self, spec): - """ - Reads the database from the set location - Searches for and removes the specified spec - Writes the database back to memory + """Removes a spec from the database. To be called on uninstall. + + Reads the database, then: + + 1. Marks the spec as not installed. + 2. Removes the spec if it has no more dependents. + 3. If removed, recursively updates dependencies' ref counts + and remvoes them if they are no longer needed. + """ # Should always already be locked with self.write_lock(): @@ -300,19 +378,13 @@ class Database(object): self.write() - @_autospec - def get_installed(self, spec): - """Get installed specs that satisfy the provided spec constraint.""" - return [s for s in self.installed_package_specs() if s.satisfies(spec)] - - @_autospec def installed_extensions_for(self, extendee_spec): """ Return the specs of all packages that extend the given spec """ - for s in self.installed_package_specs(): + for s in self.query(): try: if s.package.extends(extendee_spec): yield s.package @@ -322,25 +394,59 @@ class Database(object): # TODO: conditional way to do this instead of catching exceptions - def installed_package_specs(self): - """ - Read installed package names from the database - and return their specs + def query(self, query_spec=any, known=any, installed=True): + """Run a query on the database. + + ``query_spec`` + Queries iterate through specs in the database and return + those that satisfy the supplied ``query_spec``. If + query_spec is `any`, This will match all specs in the + database. If it is a spec, we'll evaluate + ``spec.satisfies(query_spec)``. + + The query can be constrained by two additional attributes: + + ``known`` + Possible values: True, False, any + + Specs that are "known" are those for which Spack can + locate a ``package.py`` file -- i.e., Spack "knows" how to + install them. Specs that are unknown may represent + packages that existed in a previous version of Spack, but + have since either changed their name or been removed. + + ``installed`` + Possible values: True, False, any + + Specs for which a prefix exists are "installed". A spec + that is NOT installed will be in the database if some + other spec depends on it but its installation has gone + away since Spack installed it. + + TODO: Specs are a lot like queries. Should there be a + wildcard spec object, and should specs have attributes + like installed and known that can be queried? Or are + these really special cases that only belong here? + """ - # Should always already be locked with self.read_lock(): self.read() - return sorted(rec.spec for rec in self._data.values()) + results = [] + for key, rec in self._data.items(): + if installed is not any and rec.installed != installed: + continue + if known is not any and spack.db.exists(rec.spec.name) != known: + continue + if query_spec is any or rec.spec.satisfies(query_spec): + results.append(rec.spec) - def installed_known_package_specs(self): - """ - Read installed package names from the database. - Return only the specs for which the package is known - to this version of spack - """ - return [s for s in self.installed_package_specs() - if spack.db.exists(s.name)] + return sorted(results) + + + def missing(self, spec): + key = spec.dag_hash() + return key in self._data and not self._data[key].installed class CorruptDatabaseError(SpackError): diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index e64b427852..e6944ce40c 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -563,9 +563,12 @@ class Package(object): @property def installed_dependents(self): """Return a list of the specs of all installed packages that depend - on this one.""" + on this one. + + TODO: move this method to database.py? + """ dependents = [] - for spec in spack.installed_db.installed_package_specs(): + for spec in spack.installed_db.query(): if self.name == spec.name: continue for dep in spec.traverse(): -- cgit v1.2.3-70-g09d2 From 5fda7daf57e2362c803d4f2152da93ed270818d9 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Thu, 17 Sep 2015 11:29:27 -0700 Subject: an ordered database test --- lib/spack/spack/test/__init__.py | 3 +- lib/spack/spack/test/database.py | 104 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 106 insertions(+), 1 deletion(-) create mode 100644 lib/spack/spack/test/database.py (limited to 'lib') diff --git a/lib/spack/spack/test/__init__.py b/lib/spack/spack/test/__init__.py index 6b3715be6f..c3b39b76f8 100644 --- a/lib/spack/spack/test/__init__.py +++ b/lib/spack/spack/test/__init__.py @@ -56,7 +56,8 @@ test_names = ['versions', 'spec_yaml', 'optional_deps', 'make_executable', - 'configure_guess'] + 'configure_guess', + 'database'] def list_tests(): diff --git a/lib/spack/spack/test/database.py b/lib/spack/spack/test/database.py new file mode 100644 index 0000000000..a3386bad99 --- /dev/null +++ b/lib/spack/spack/test/database.py @@ -0,0 +1,104 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +""" +These tests check the database is functioning properly, +both in memory and in its file +""" +import unittest + +from llnl.util.lock import * +from llnl.util.filesystem import join_path + +import spack +from spack.database import Database + + +class DatabaseTest(unittest.TestCase): + def setUp(self): + self.original_db = spack.installed_db + spack.installed_db = Database(self.original_db._root,"_test_index.yaml") + self.file_path = join_path(self.original_db._root,"_test_index.yaml") + if os.path.exists(self.file_path): + os.remove(self.file_path) + + def tearDown(self): + spack.installed_db = self.original_db + os.remove(self.file_path) + + def _test_read_from_install_tree(self): + specs = spack.install_layout.all_specs() + spack.installed_db.read_database() + spack.installed_db.write() + for sph in spack.installed_db._data: + self.assertTrue(sph['spec'] in specs) + self.assertEqual(len(specs),len(spack.installed_db._data)) + + def _test_remove_and_add(self): + specs = spack.install_layout.all_specs() + spack.installed_db.remove(specs[len(specs)-1]) + for sph in spack.installed_db._data: + self.assertTrue(sph['spec'] in specs[:len(specs)-1]) + self.assertEqual(len(specs)-1,len(spack.installed_db._data)) + + spack.installed_db.add(specs[len(specs)-1],"") + for sph in spack.installed_db._data: + self.assertTrue(sph['spec'] in specs) + self.assertEqual(len(specs),len(spack.installed_db._data)) + + def _test_read_from_file(self): + spack.installed_db.read_database() + size = len(spack.installed_db._data) + spack.installed_db._data = spack.installed_db._data[1:] + os.utime(spack.installed_db._file_path,None) + spack.installed_db.read_database() + self.assertEqual(size,len(spack.installed_db._data)) + + specs = spack.install_layout.all_specs() + self.assertEqual(size,len(specs)) + for sph in spack.installed_db._data: + self.assertTrue(sph['spec'] in specs) + + + def _test_write_to_file(self): + spack.installed_db.read_database() + size = len(spack.installed_db._data) + real_data = spack.installed_db._data + spack.installed_db._data = real_data[:size-1] + spack.installed_db.write() + spack.installed_db._data = real_data + os.utime(spack.installed_db._file_path,None) + spack.installed_db.read_database() + self.assertEqual(size-1,len(spack.installed_db._data)) + + specs = spack.install_layout.all_specs() + self.assertEqual(size,len(specs)) + for sph in spack.installed_db._data: + self.assertTrue(sph['spec'] in specs[:size-1]) + + def test_ordered_test(self): + self._test_read_from_install_tree() + self._test_remove_and_add() + self._test_read_from_file() + self._test_write_to_file() -- cgit v1.2.3-70-g09d2 From 2c81875019803af7b7f08b070f37c45e51a3c7d5 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 12 Oct 2015 14:44:41 -0700 Subject: Fix bug in colify color handling. --- lib/spack/llnl/util/tty/colify.py | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) (limited to 'lib') diff --git a/lib/spack/llnl/util/tty/colify.py b/lib/spack/llnl/util/tty/colify.py index 66c52c3968..db928444c7 100644 --- a/lib/spack/llnl/util/tty/colify.py +++ b/lib/spack/llnl/util/tty/colify.py @@ -33,8 +33,7 @@ import struct from StringIO import StringIO from llnl.util.tty import terminal_size -from llnl.util.tty.color import clen - +from llnl.util.tty.color import clen, cextra class ColumnConfig: def __init__(self, cols): @@ -42,7 +41,6 @@ class ColumnConfig: self.line_length = 0 self.valid = True self.widths = [0] * cols # does not include ansi colors - self.cwidths = [0] * cols # includes ansi colors def __repr__(self): attrs = [(a,getattr(self, a)) for a in dir(self) if not a.startswith("__")] @@ -66,8 +64,6 @@ def config_variable_cols(elts, console_width, padding, cols=0): # Get a bound on the most columns we could possibly have. # 'clen' ignores length of ansi color sequences. lengths = [clen(e) for e in elts] - clengths = [len(e) for e in elts] - max_cols = max(1, console_width / (min(lengths) + padding)) max_cols = min(len(elts), max_cols) @@ -85,7 +81,6 @@ def config_variable_cols(elts, console_width, padding, cols=0): if conf.widths[col] < (length + p): conf.line_length += length + p - conf.widths[col] conf.widths[col] = length + p - conf.cwidths[col] = clengths[i] + p conf.valid = (conf.line_length < console_width) try: @@ -118,7 +113,6 @@ def config_uniform_cols(elts, console_width, padding, cols=0): config = ColumnConfig(cols) config.widths = [max_len] * cols - config.cwidths = [max_clen] * cols return config @@ -147,9 +141,6 @@ def colify(elts, **options): method= Method to use to fit columns. Options are variable or uniform. Variable-width columns are tighter, uniform columns are all the same width and fit less data on the screen. - - len= Function to use for calculating string length. - Useful for ignoring ansi color. Default is 'len'. """ # Get keyword arguments or set defaults cols = options.pop("cols", 0) @@ -199,9 +190,6 @@ def colify(elts, **options): raise ValueError("method must be one of: " + allowed_methods) cols = config.cols - formats = ["%%-%ds" % width for width in config.cwidths[:-1]] - formats.append("%s") # last column has no trailing space - rows = (len(elts) + cols - 1) / cols rows_last_col = len(elts) % rows @@ -209,7 +197,9 @@ def colify(elts, **options): output.write(" " * indent) for col in xrange(cols): elt = col * rows + row - output.write(formats[col] % elts[elt]) + width = config.widths[col] + cextra(elts[elt]) + fmt = '%%-%ds' % width + output.write(fmt % elts[elt]) output.write("\n") row += 1 -- cgit v1.2.3-70-g09d2 From b7249d66b3b47f710ddbb1719f433b507322b5a3 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 12 Oct 2015 19:09:11 -0700 Subject: Adding command testinstall. See "spack testinstall -h" for documentation. Still need to add output formatting (in a commonly parse-able format like Junit or TAP). May want to adjust how the build log is accessed in case of a build failure. --- lib/spack/spack/cmd/testinstall.py | 129 +++++++++++++++++++++++++++++++++++++ 1 file changed, 129 insertions(+) create mode 100644 lib/spack/spack/cmd/testinstall.py (limited to 'lib') diff --git a/lib/spack/spack/cmd/testinstall.py b/lib/spack/spack/cmd/testinstall.py new file mode 100644 index 0000000000..d3a2cae3c2 --- /dev/null +++ b/lib/spack/spack/cmd/testinstall.py @@ -0,0 +1,129 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from external import argparse +import xml.etree.ElementTree as ET + +import llnl.util.tty as tty +from llnl.util.filesystem import * + +import spack +import spack.cmd + +description = "Build and install packages" + +def setup_parser(subparser): + #subparser.add_argument( + # '-i', '--ignore-dependencies', action='store_true', dest='ignore_deps', + # help="Do not try to install dependencies of requested packages.") + + subparser.add_argument( + '-j', '--jobs', action='store', type=int, + help="Explicitly set number of make jobs. Default is #cpus.") + + #always false for test + #subparser.add_argument( + # '--keep-prefix', action='store_true', dest='keep_prefix', + # help="Don't remove the install prefix if installation fails.") + + #always true for test + #subparser.add_argument( + # '--keep-stage', action='store_true', dest='keep_stage', + # help="Don't remove the build stage if installation succeeds.") + + subparser.add_argument( + '-n', '--no-checksum', action='store_true', dest='no_checksum', + help="Do not check packages against checksum") + subparser.add_argument( + '-v', '--verbose', action='store_true', dest='verbose', + help="Display verbose build output while installing.") + + #subparser.add_argument( + # '--fake', action='store_true', dest='fake', + # help="Fake install. Just remove the prefix and touch a fake file in it.") + + subparser.add_argument( + 'outputdir', help="test output goes in this directory, 1 file per package") + + subparser.add_argument( + 'packages', nargs=argparse.REMAINDER, help="specs of packages to install") + + +class JunitTestResult(object): + def __init__(self): + self.root = Element('testsuite') + self.tests = [] + + def addTest(self, identifier, passed=True, output=None): + self.tests.append((identifier, passed, output)) + + def output(self): + self.root.set('tests', '{0}'.format(len(self.tests))) + + +def testinstall(parser, args): + if not args.packages: + tty.die("install requires at least one package argument") + + if args.jobs is not None: + if args.jobs <= 0: + tty.die("The -j option must be a positive integer!") + + if args.no_checksum: + spack.do_checksum = False # TODO: remove this global. + + print "Output to:", args.outputdir + + specs = spack.cmd.parse_specs(args.packages, concretize=True) + try: + for spec in specs: + #import pdb; pdb.set_trace() + package = spack.db.get(spec) + package.do_install( + keep_prefix=False, + keep_stage=False, + ignore_deps=False, + make_jobs=args.jobs, + verbose=args.verbose, + fake=False) + finally: + for spec in specs: + package = spack.db.get(spec) + #import pdb; pdb.set_trace() + + print spec.name + print spec.version + print spec.dag_hash() + + if package.installed: + installLog = spack.install_layout.build_log_path(spec) + else: + #TODO: search recursively under stage.path instead of only within + # stage.source_path + installLog = join_path(package.stage.source_path, 'spack-build.out') + + with open(installLog, 'rb') as F: + for line in F.readlines()[:10]: + print "\t{0}".format(line.strip()) + -- cgit v1.2.3-70-g09d2 From 6cd22e5786dbf40f1261b9b0410bdafbb6dd6f29 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 12 Oct 2015 20:49:23 -0700 Subject: 1. Added Junit XML format 2. Specify output to a file vs. a directory 3. Use [1] and [2] to write an XML file tracking success of package installs in Junit XML format --- lib/spack/spack/cmd/testinstall.py | 50 ++++++++++++++++++++++++++------------ 1 file changed, 34 insertions(+), 16 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/cmd/testinstall.py b/lib/spack/spack/cmd/testinstall.py index d3a2cae3c2..7ebadbd344 100644 --- a/lib/spack/spack/cmd/testinstall.py +++ b/lib/spack/spack/cmd/testinstall.py @@ -64,24 +64,43 @@ def setup_parser(subparser): # help="Fake install. Just remove the prefix and touch a fake file in it.") subparser.add_argument( - 'outputdir', help="test output goes in this directory, 1 file per package") + 'output', help="test output goes in this file") subparser.add_argument( 'packages', nargs=argparse.REMAINDER, help="specs of packages to install") -class JunitTestResult(object): +class JunitResultFormat(object): def __init__(self): - self.root = Element('testsuite') + self.root = ET.Element('testsuite') self.tests = [] - def addTest(self, identifier, passed=True, output=None): - self.tests.append((identifier, passed, output)) + def addTest(self, buildId, passed=True, buildLog=None): + self.tests.append((buildId, passed, buildLog)) - def output(self): + def writeTo(self, stream): self.root.set('tests', '{0}'.format(len(self.tests))) - - + for buildId, passed, buildLog in self.tests: + testcase = ET.SubElement(self.root, 'testcase') + testcase.set('classname', buildId.name) + testcase.set('name', buildId.stringId()) + if not passed: + failure = ET.SubElement(testcase, 'failure') + failure.set('type', "Build Error") + failure.text = buildLog + ET.ElementTree(self.root).write(stream) + + +class BuildId(object): + def __init__(self, name, version, hashId): + self.name = name + self.version = version + self.hashId = hashId + + def stringId(self): + return "-".join(str(x) for x in (self.name, self.version, self.hashId)) + + def testinstall(parser, args): if not args.packages: tty.die("install requires at least one package argument") @@ -93,8 +112,6 @@ def testinstall(parser, args): if args.no_checksum: spack.do_checksum = False # TODO: remove this global. - print "Output to:", args.outputdir - specs = spack.cmd.parse_specs(args.packages, concretize=True) try: for spec in specs: @@ -108,13 +125,12 @@ def testinstall(parser, args): verbose=args.verbose, fake=False) finally: + jrf = JunitResultFormat() for spec in specs: package = spack.db.get(spec) #import pdb; pdb.set_trace() - print spec.name - print spec.version - print spec.dag_hash() + bId = BuildId(spec.name, spec.version, spec.dag_hash()) if package.installed: installLog = spack.install_layout.build_log_path(spec) @@ -124,6 +140,8 @@ def testinstall(parser, args): installLog = join_path(package.stage.source_path, 'spack-build.out') with open(installLog, 'rb') as F: - for line in F.readlines()[:10]: - print "\t{0}".format(line.strip()) - + buildLog = F.read() #TODO: this may not return all output + jrf.addTest(bId, package.installed, buildLog) + + with open(args.output, 'wb') as F: + jrf.writeTo(F) -- cgit v1.2.3-70-g09d2 From 9f56d9c807d9d3efc7cde0591bd53d3db404dacc Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 12 Oct 2015 20:56:03 -0700 Subject: Don't create test output for any package that was already installed. --- lib/spack/spack/cmd/testinstall.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/cmd/testinstall.py b/lib/spack/spack/cmd/testinstall.py index 7ebadbd344..04e594c0b8 100644 --- a/lib/spack/spack/cmd/testinstall.py +++ b/lib/spack/spack/cmd/testinstall.py @@ -113,23 +113,23 @@ def testinstall(parser, args): spack.do_checksum = False # TODO: remove this global. specs = spack.cmd.parse_specs(args.packages, concretize=True) + newInstalls = list() try: for spec in specs: - #import pdb; pdb.set_trace() package = spack.db.get(spec) - package.do_install( - keep_prefix=False, - keep_stage=False, - ignore_deps=False, - make_jobs=args.jobs, - verbose=args.verbose, - fake=False) + if not package.installed: + newInstalls.append(spec) + package.do_install( + keep_prefix=False, + keep_stage=False, + ignore_deps=False, + make_jobs=args.jobs, + verbose=args.verbose, + fake=False) finally: jrf = JunitResultFormat() - for spec in specs: + for spec in newInstalls: package = spack.db.get(spec) - #import pdb; pdb.set_trace() - bId = BuildId(spec.name, spec.version, spec.dag_hash()) if package.installed: -- cgit v1.2.3-70-g09d2 From d16095c8560cbaae1020a8e84494b8877bfe36f5 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 13 Oct 2015 10:35:19 -0700 Subject: Add forgotten file from previous commit. --- lib/spack/llnl/util/tty/color.py | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'lib') diff --git a/lib/spack/llnl/util/tty/color.py b/lib/spack/llnl/util/tty/color.py index 22080a7b37..0d09303da0 100644 --- a/lib/spack/llnl/util/tty/color.py +++ b/lib/spack/llnl/util/tty/color.py @@ -158,6 +158,11 @@ def clen(string): return len(re.sub(r'\033[^m]*m', '', string)) +def cextra(string): + """"Length of extra color characters in a string""" + return len(''.join(re.findall(r'\033[^m]*m', string))) + + def cwrite(string, stream=sys.stdout, color=None): """Replace all color expressions in string with ANSI control codes and write the result to the stream. If color is -- cgit v1.2.3-70-g09d2 From 1ce6d8b627fac204fc4a4500ea28b4733dd172dd Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 13 Oct 2015 10:41:47 -0700 Subject: Add spec YAML format to test output. --- lib/spack/spack/cmd/testinstall.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/cmd/testinstall.py b/lib/spack/spack/cmd/testinstall.py index 04e594c0b8..3a9e1e14e1 100644 --- a/lib/spack/spack/cmd/testinstall.py +++ b/lib/spack/spack/cmd/testinstall.py @@ -75,19 +75,19 @@ class JunitResultFormat(object): self.root = ET.Element('testsuite') self.tests = [] - def addTest(self, buildId, passed=True, buildLog=None): - self.tests.append((buildId, passed, buildLog)) + def addTest(self, buildId, passed=True, buildInfo=None): + self.tests.append((buildId, passed, buildInfo)) def writeTo(self, stream): self.root.set('tests', '{0}'.format(len(self.tests))) - for buildId, passed, buildLog in self.tests: + for buildId, passed, buildInfo in self.tests: testcase = ET.SubElement(self.root, 'testcase') testcase.set('classname', buildId.name) testcase.set('name', buildId.stringId()) if not passed: failure = ET.SubElement(testcase, 'failure') failure.set('type', "Build Error") - failure.text = buildLog + failure.text = buildInfo ET.ElementTree(self.root).write(stream) @@ -133,15 +133,18 @@ def testinstall(parser, args): bId = BuildId(spec.name, spec.version, spec.dag_hash()) if package.installed: - installLog = spack.install_layout.build_log_path(spec) + buildLogPath = spack.install_layout.build_log_path(spec) else: #TODO: search recursively under stage.path instead of only within # stage.source_path - installLog = join_path(package.stage.source_path, 'spack-build.out') + buildLogPath = join_path(package.stage.source_path, 'spack-build.out') - with open(installLog, 'rb') as F: + with open(buildLogPath, 'rb') as F: buildLog = F.read() #TODO: this may not return all output - jrf.addTest(bId, package.installed, buildLog) + #TODO: add the whole build log? it could be several thousand + # lines. It may be better to look for errors. + jrf.addTest(bId, package.installed, buildLogPath + '\n' + + spec.to_yaml() + buildLog) with open(args.output, 'wb') as F: jrf.writeTo(F) -- cgit v1.2.3-70-g09d2 From 71dcf8833c263a2f87b8bfcce6f2eaf7a14014a3 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 13 Oct 2015 19:02:41 -0700 Subject: Make sure to generate output for dependencies as if they were separate tests: the original intent was to generate output as if each package was a unit test, but I noticed that I was only generating test output for top-level packages. --- lib/spack/spack/cmd/testinstall.py | 71 +++++++++++++++++++++++++++----------- 1 file changed, 51 insertions(+), 20 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/cmd/testinstall.py b/lib/spack/spack/cmd/testinstall.py index 3a9e1e14e1..6f3514bc34 100644 --- a/lib/spack/spack/cmd/testinstall.py +++ b/lib/spack/spack/cmd/testinstall.py @@ -24,6 +24,7 @@ ############################################################################## from external import argparse import xml.etree.ElementTree as ET +import itertools import llnl.util.tty as tty from llnl.util.filesystem import * @@ -100,7 +101,37 @@ class BuildId(object): def stringId(self): return "-".join(str(x) for x in (self.name, self.version, self.hashId)) - + +def createTestOutput(spec, handled, output): + if spec in handled: + return handled[spec] + + childSuccesses = list(createTestOutput(dep, handled, output) + for dep in spec.dependencies.itervalues()) + package = spack.db.get(spec) + handled[spec] = package.installed + + if all(childSuccesses): + bId = BuildId(spec.name, spec.version, spec.dag_hash()) + + if package.installed: + buildLogPath = spack.install_layout.build_log_path(spec) + else: + #TODO: search recursively under stage.path instead of only within + # stage.source_path + buildLogPath = join_path(package.stage.source_path, 'spack-build.out') + + with open(buildLogPath, 'rb') as F: + buildLog = F.read() #TODO: this may not return all output + #TODO: add the whole build log? it could be several thousand + # lines. It may be better to look for errors. + output.addTest(bId, package.installed, buildLogPath + '\n' + + spec.to_yaml() + buildLog) + #TODO: create a failed test if a dependency didn't install? + + return handled[spec] + + def testinstall(parser, args): if not args.packages: tty.die("install requires at least one package argument") @@ -113,12 +144,17 @@ def testinstall(parser, args): spack.do_checksum = False # TODO: remove this global. specs = spack.cmd.parse_specs(args.packages, concretize=True) - newInstalls = list() + newInstalls = set() + for spec in itertools.chain.from_iterable(spec.traverse() + for spec in specs): + package = spack.db.get(spec) + if not package.installed: + newInstalls.add(spec) + try: for spec in specs: package = spack.db.get(spec) if not package.installed: - newInstalls.append(spec) package.do_install( keep_prefix=False, keep_stage=False, @@ -127,24 +163,19 @@ def testinstall(parser, args): verbose=args.verbose, fake=False) finally: + #TODO: note if multiple packages are specified and a prior one fails, + # it will prevent any of the others from succeeding even if they + # don't share any dependencies. i.e. the results may be strange if + # you run testinstall with >1 top-level package + + #Find all packages that are not a dependency of another package + topLevelNewInstalls = newInstalls - set(itertools.chain.from_iterable( + spec.dependencies for spec in newInstalls)) + jrf = JunitResultFormat() - for spec in newInstalls: - package = spack.db.get(spec) - bId = BuildId(spec.name, spec.version, spec.dag_hash()) - - if package.installed: - buildLogPath = spack.install_layout.build_log_path(spec) - else: - #TODO: search recursively under stage.path instead of only within - # stage.source_path - buildLogPath = join_path(package.stage.source_path, 'spack-build.out') - - with open(buildLogPath, 'rb') as F: - buildLog = F.read() #TODO: this may not return all output - #TODO: add the whole build log? it could be several thousand - # lines. It may be better to look for errors. - jrf.addTest(bId, package.installed, buildLogPath + '\n' + - spec.to_yaml() + buildLog) + handled = {} + for spec in topLevelNewInstalls: + createTestOutput(spec, handled, jrf) with open(args.output, 'wb') as F: jrf.writeTo(F) -- cgit v1.2.3-70-g09d2 From 3ce85b22708769ff9d8b2b0ba79e4157dac46d74 Mon Sep 17 00:00:00 2001 From: Ben Boeckel Date: Thu, 15 Oct 2015 09:27:05 -0400 Subject: spack: split spack_root from prefix A foundation for allowing runtime configuring of the prefix. --- lib/spack/spack/__init__.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index caa09eb6e0..6e8d41895f 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -27,24 +27,26 @@ import tempfile from llnl.util.filesystem import * # This lives in $prefix/lib/spack/spack/__file__ -prefix = ancestor(__file__, 4) +spack_root = ancestor(__file__, 4) # The spack script itself -spack_file = join_path(prefix, "bin", "spack") +spack_file = join_path(spack_root, "bin", "spack") # spack directory hierarchy -etc_path = join_path(prefix, "etc") -lib_path = join_path(prefix, "lib", "spack") +lib_path = join_path(spack_root, "lib", "spack") build_env_path = join_path(lib_path, "env") module_path = join_path(lib_path, "spack") compilers_path = join_path(module_path, "compilers") test_path = join_path(module_path, "test") hooks_path = join_path(module_path, "hooks") -var_path = join_path(prefix, "var", "spack") +var_path = join_path(spack_root, "var", "spack") stage_path = join_path(var_path, "stage") +share_path = join_path(spack_root, "share", "spack") + +prefix = spack_root opt_path = join_path(prefix, "opt") install_path = join_path(opt_path, "spack") -share_path = join_path(prefix, "share", "spack") +etc_path = join_path(prefix, "etc") # # Set up the packages database. -- cgit v1.2.3-70-g09d2 From 0d66362cee9849db77a9f3297aa697f21fe1acdd Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 15 Oct 2015 10:17:08 -0700 Subject: Only install 1 top-level package with testinstall. Otherwise if multiple packages are specified and a prior one fails, it will prevent any of the others from succeeding (and generating test output) even if they don't share dependencies. --- lib/spack/spack/cmd/testinstall.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/cmd/testinstall.py b/lib/spack/spack/cmd/testinstall.py index 6f3514bc34..5e5288bfbd 100644 --- a/lib/spack/spack/cmd/testinstall.py +++ b/lib/spack/spack/cmd/testinstall.py @@ -68,7 +68,7 @@ def setup_parser(subparser): 'output', help="test output goes in this file") subparser.add_argument( - 'packages', nargs=argparse.REMAINDER, help="specs of packages to install") + 'package', help="spec of package to install") class JunitResultFormat(object): @@ -133,8 +133,8 @@ def createTestOutput(spec, handled, output): def testinstall(parser, args): - if not args.packages: - tty.die("install requires at least one package argument") + if not args.package: + tty.die("install requires a package argument") if args.jobs is not None: if args.jobs <= 0: @@ -143,7 +143,8 @@ def testinstall(parser, args): if args.no_checksum: spack.do_checksum = False # TODO: remove this global. - specs = spack.cmd.parse_specs(args.packages, concretize=True) + #TODO: should a single argument be wrapped in a list? + specs = spack.cmd.parse_specs(args.package, concretize=True) newInstalls = set() for spec in itertools.chain.from_iterable(spec.traverse() for spec in specs): @@ -162,12 +163,7 @@ def testinstall(parser, args): make_jobs=args.jobs, verbose=args.verbose, fake=False) - finally: - #TODO: note if multiple packages are specified and a prior one fails, - # it will prevent any of the others from succeeding even if they - # don't share any dependencies. i.e. the results may be strange if - # you run testinstall with >1 top-level package - + finally: #Find all packages that are not a dependency of another package topLevelNewInstalls = newInstalls - set(itertools.chain.from_iterable( spec.dependencies for spec in newInstalls)) -- cgit v1.2.3-70-g09d2 From 2ae7839b666592e3acaf370d52f69376b80b28a7 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 15 Oct 2015 10:26:13 -0700 Subject: Edit function names to conform to naming conventions. --- lib/spack/spack/cmd/testinstall.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/cmd/testinstall.py b/lib/spack/spack/cmd/testinstall.py index 5e5288bfbd..ea092727fe 100644 --- a/lib/spack/spack/cmd/testinstall.py +++ b/lib/spack/spack/cmd/testinstall.py @@ -76,10 +76,10 @@ class JunitResultFormat(object): self.root = ET.Element('testsuite') self.tests = [] - def addTest(self, buildId, passed=True, buildInfo=None): + def add_test(self, buildId, passed=True, buildInfo=None): self.tests.append((buildId, passed, buildInfo)) - def writeTo(self, stream): + def write_to(self, stream): self.root.set('tests', '{0}'.format(len(self.tests))) for buildId, passed, buildInfo in self.tests: testcase = ET.SubElement(self.root, 'testcase') @@ -102,11 +102,11 @@ class BuildId(object): return "-".join(str(x) for x in (self.name, self.version, self.hashId)) -def createTestOutput(spec, handled, output): +def create_test_output(spec, handled, output): if spec in handled: return handled[spec] - childSuccesses = list(createTestOutput(dep, handled, output) + childSuccesses = list(create_test_output(dep, handled, output) for dep in spec.dependencies.itervalues()) package = spack.db.get(spec) handled[spec] = package.installed @@ -125,7 +125,7 @@ def createTestOutput(spec, handled, output): buildLog = F.read() #TODO: this may not return all output #TODO: add the whole build log? it could be several thousand # lines. It may be better to look for errors. - output.addTest(bId, package.installed, buildLogPath + '\n' + + output.add_test(bId, package.installed, buildLogPath + '\n' + spec.to_yaml() + buildLog) #TODO: create a failed test if a dependency didn't install? @@ -171,7 +171,7 @@ def testinstall(parser, args): jrf = JunitResultFormat() handled = {} for spec in topLevelNewInstalls: - createTestOutput(spec, handled, jrf) + create_test_output(spec, handled, jrf) with open(args.output, 'wb') as F: - jrf.writeTo(F) + jrf.write_to(F) -- cgit v1.2.3-70-g09d2 From e3d703b80ffe442d83879c070e0bbeafa4f4ef20 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 15 Oct 2015 10:33:39 -0700 Subject: Change name of file to conform to conventions. --- lib/spack/spack/cmd/test-install.py | 177 ++++++++++++++++++++++++++++++++++++ lib/spack/spack/cmd/testinstall.py | 177 ------------------------------------ 2 files changed, 177 insertions(+), 177 deletions(-) create mode 100644 lib/spack/spack/cmd/test-install.py delete mode 100644 lib/spack/spack/cmd/testinstall.py (limited to 'lib') diff --git a/lib/spack/spack/cmd/test-install.py b/lib/spack/spack/cmd/test-install.py new file mode 100644 index 0000000000..ea092727fe --- /dev/null +++ b/lib/spack/spack/cmd/test-install.py @@ -0,0 +1,177 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from external import argparse +import xml.etree.ElementTree as ET +import itertools + +import llnl.util.tty as tty +from llnl.util.filesystem import * + +import spack +import spack.cmd + +description = "Build and install packages" + +def setup_parser(subparser): + #subparser.add_argument( + # '-i', '--ignore-dependencies', action='store_true', dest='ignore_deps', + # help="Do not try to install dependencies of requested packages.") + + subparser.add_argument( + '-j', '--jobs', action='store', type=int, + help="Explicitly set number of make jobs. Default is #cpus.") + + #always false for test + #subparser.add_argument( + # '--keep-prefix', action='store_true', dest='keep_prefix', + # help="Don't remove the install prefix if installation fails.") + + #always true for test + #subparser.add_argument( + # '--keep-stage', action='store_true', dest='keep_stage', + # help="Don't remove the build stage if installation succeeds.") + + subparser.add_argument( + '-n', '--no-checksum', action='store_true', dest='no_checksum', + help="Do not check packages against checksum") + subparser.add_argument( + '-v', '--verbose', action='store_true', dest='verbose', + help="Display verbose build output while installing.") + + #subparser.add_argument( + # '--fake', action='store_true', dest='fake', + # help="Fake install. Just remove the prefix and touch a fake file in it.") + + subparser.add_argument( + 'output', help="test output goes in this file") + + subparser.add_argument( + 'package', help="spec of package to install") + + +class JunitResultFormat(object): + def __init__(self): + self.root = ET.Element('testsuite') + self.tests = [] + + def add_test(self, buildId, passed=True, buildInfo=None): + self.tests.append((buildId, passed, buildInfo)) + + def write_to(self, stream): + self.root.set('tests', '{0}'.format(len(self.tests))) + for buildId, passed, buildInfo in self.tests: + testcase = ET.SubElement(self.root, 'testcase') + testcase.set('classname', buildId.name) + testcase.set('name', buildId.stringId()) + if not passed: + failure = ET.SubElement(testcase, 'failure') + failure.set('type', "Build Error") + failure.text = buildInfo + ET.ElementTree(self.root).write(stream) + + +class BuildId(object): + def __init__(self, name, version, hashId): + self.name = name + self.version = version + self.hashId = hashId + + def stringId(self): + return "-".join(str(x) for x in (self.name, self.version, self.hashId)) + + +def create_test_output(spec, handled, output): + if spec in handled: + return handled[spec] + + childSuccesses = list(create_test_output(dep, handled, output) + for dep in spec.dependencies.itervalues()) + package = spack.db.get(spec) + handled[spec] = package.installed + + if all(childSuccesses): + bId = BuildId(spec.name, spec.version, spec.dag_hash()) + + if package.installed: + buildLogPath = spack.install_layout.build_log_path(spec) + else: + #TODO: search recursively under stage.path instead of only within + # stage.source_path + buildLogPath = join_path(package.stage.source_path, 'spack-build.out') + + with open(buildLogPath, 'rb') as F: + buildLog = F.read() #TODO: this may not return all output + #TODO: add the whole build log? it could be several thousand + # lines. It may be better to look for errors. + output.add_test(bId, package.installed, buildLogPath + '\n' + + spec.to_yaml() + buildLog) + #TODO: create a failed test if a dependency didn't install? + + return handled[spec] + + +def testinstall(parser, args): + if not args.package: + tty.die("install requires a package argument") + + if args.jobs is not None: + if args.jobs <= 0: + tty.die("The -j option must be a positive integer!") + + if args.no_checksum: + spack.do_checksum = False # TODO: remove this global. + + #TODO: should a single argument be wrapped in a list? + specs = spack.cmd.parse_specs(args.package, concretize=True) + newInstalls = set() + for spec in itertools.chain.from_iterable(spec.traverse() + for spec in specs): + package = spack.db.get(spec) + if not package.installed: + newInstalls.add(spec) + + try: + for spec in specs: + package = spack.db.get(spec) + if not package.installed: + package.do_install( + keep_prefix=False, + keep_stage=False, + ignore_deps=False, + make_jobs=args.jobs, + verbose=args.verbose, + fake=False) + finally: + #Find all packages that are not a dependency of another package + topLevelNewInstalls = newInstalls - set(itertools.chain.from_iterable( + spec.dependencies for spec in newInstalls)) + + jrf = JunitResultFormat() + handled = {} + for spec in topLevelNewInstalls: + create_test_output(spec, handled, jrf) + + with open(args.output, 'wb') as F: + jrf.write_to(F) diff --git a/lib/spack/spack/cmd/testinstall.py b/lib/spack/spack/cmd/testinstall.py deleted file mode 100644 index ea092727fe..0000000000 --- a/lib/spack/spack/cmd/testinstall.py +++ /dev/null @@ -1,177 +0,0 @@ -############################################################################## -# Copyright (c) 2013, Lawrence Livermore National Security, LLC. -# Produced at the Lawrence Livermore National Laboratory. -# -# This file is part of Spack. -# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. -# LLNL-CODE-647188 -# -# For details, see https://scalability-llnl.github.io/spack -# Please also see the LICENSE file for our notice and the LGPL. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License (as published by -# the Free Software Foundation) version 2.1 dated February 1999. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and -# conditions of the GNU General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -############################################################################## -from external import argparse -import xml.etree.ElementTree as ET -import itertools - -import llnl.util.tty as tty -from llnl.util.filesystem import * - -import spack -import spack.cmd - -description = "Build and install packages" - -def setup_parser(subparser): - #subparser.add_argument( - # '-i', '--ignore-dependencies', action='store_true', dest='ignore_deps', - # help="Do not try to install dependencies of requested packages.") - - subparser.add_argument( - '-j', '--jobs', action='store', type=int, - help="Explicitly set number of make jobs. Default is #cpus.") - - #always false for test - #subparser.add_argument( - # '--keep-prefix', action='store_true', dest='keep_prefix', - # help="Don't remove the install prefix if installation fails.") - - #always true for test - #subparser.add_argument( - # '--keep-stage', action='store_true', dest='keep_stage', - # help="Don't remove the build stage if installation succeeds.") - - subparser.add_argument( - '-n', '--no-checksum', action='store_true', dest='no_checksum', - help="Do not check packages against checksum") - subparser.add_argument( - '-v', '--verbose', action='store_true', dest='verbose', - help="Display verbose build output while installing.") - - #subparser.add_argument( - # '--fake', action='store_true', dest='fake', - # help="Fake install. Just remove the prefix and touch a fake file in it.") - - subparser.add_argument( - 'output', help="test output goes in this file") - - subparser.add_argument( - 'package', help="spec of package to install") - - -class JunitResultFormat(object): - def __init__(self): - self.root = ET.Element('testsuite') - self.tests = [] - - def add_test(self, buildId, passed=True, buildInfo=None): - self.tests.append((buildId, passed, buildInfo)) - - def write_to(self, stream): - self.root.set('tests', '{0}'.format(len(self.tests))) - for buildId, passed, buildInfo in self.tests: - testcase = ET.SubElement(self.root, 'testcase') - testcase.set('classname', buildId.name) - testcase.set('name', buildId.stringId()) - if not passed: - failure = ET.SubElement(testcase, 'failure') - failure.set('type', "Build Error") - failure.text = buildInfo - ET.ElementTree(self.root).write(stream) - - -class BuildId(object): - def __init__(self, name, version, hashId): - self.name = name - self.version = version - self.hashId = hashId - - def stringId(self): - return "-".join(str(x) for x in (self.name, self.version, self.hashId)) - - -def create_test_output(spec, handled, output): - if spec in handled: - return handled[spec] - - childSuccesses = list(create_test_output(dep, handled, output) - for dep in spec.dependencies.itervalues()) - package = spack.db.get(spec) - handled[spec] = package.installed - - if all(childSuccesses): - bId = BuildId(spec.name, spec.version, spec.dag_hash()) - - if package.installed: - buildLogPath = spack.install_layout.build_log_path(spec) - else: - #TODO: search recursively under stage.path instead of only within - # stage.source_path - buildLogPath = join_path(package.stage.source_path, 'spack-build.out') - - with open(buildLogPath, 'rb') as F: - buildLog = F.read() #TODO: this may not return all output - #TODO: add the whole build log? it could be several thousand - # lines. It may be better to look for errors. - output.add_test(bId, package.installed, buildLogPath + '\n' + - spec.to_yaml() + buildLog) - #TODO: create a failed test if a dependency didn't install? - - return handled[spec] - - -def testinstall(parser, args): - if not args.package: - tty.die("install requires a package argument") - - if args.jobs is not None: - if args.jobs <= 0: - tty.die("The -j option must be a positive integer!") - - if args.no_checksum: - spack.do_checksum = False # TODO: remove this global. - - #TODO: should a single argument be wrapped in a list? - specs = spack.cmd.parse_specs(args.package, concretize=True) - newInstalls = set() - for spec in itertools.chain.from_iterable(spec.traverse() - for spec in specs): - package = spack.db.get(spec) - if not package.installed: - newInstalls.add(spec) - - try: - for spec in specs: - package = spack.db.get(spec) - if not package.installed: - package.do_install( - keep_prefix=False, - keep_stage=False, - ignore_deps=False, - make_jobs=args.jobs, - verbose=args.verbose, - fake=False) - finally: - #Find all packages that are not a dependency of another package - topLevelNewInstalls = newInstalls - set(itertools.chain.from_iterable( - spec.dependencies for spec in newInstalls)) - - jrf = JunitResultFormat() - handled = {} - for spec in topLevelNewInstalls: - create_test_output(spec, handled, jrf) - - with open(args.output, 'wb') as F: - jrf.write_to(F) -- cgit v1.2.3-70-g09d2 From 11861fb8d717623c4f0014194c81fa86041fbe10 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 15 Oct 2015 10:35:42 -0700 Subject: Changing name of file requires changing function name to be invoked as a command --- lib/spack/spack/cmd/test-install.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/cmd/test-install.py b/lib/spack/spack/cmd/test-install.py index ea092727fe..eff4bcc46b 100644 --- a/lib/spack/spack/cmd/test-install.py +++ b/lib/spack/spack/cmd/test-install.py @@ -132,7 +132,7 @@ def create_test_output(spec, handled, output): return handled[spec] -def testinstall(parser, args): +def test_install(parser, args): if not args.package: tty.die("install requires a package argument") -- cgit v1.2.3-70-g09d2 From f2b4341ad6d9dc925364214863440dde29d61b50 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 15 Oct 2015 10:45:03 -0700 Subject: Always run with verbose output (so eliminate it as an option). Also remove other commented options. --- lib/spack/spack/cmd/test-install.py | 23 +---------------------- 1 file changed, 1 insertion(+), 22 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/cmd/test-install.py b/lib/spack/spack/cmd/test-install.py index eff4bcc46b..ec413115c0 100644 --- a/lib/spack/spack/cmd/test-install.py +++ b/lib/spack/spack/cmd/test-install.py @@ -35,34 +35,13 @@ import spack.cmd description = "Build and install packages" def setup_parser(subparser): - #subparser.add_argument( - # '-i', '--ignore-dependencies', action='store_true', dest='ignore_deps', - # help="Do not try to install dependencies of requested packages.") - subparser.add_argument( '-j', '--jobs', action='store', type=int, help="Explicitly set number of make jobs. Default is #cpus.") - - #always false for test - #subparser.add_argument( - # '--keep-prefix', action='store_true', dest='keep_prefix', - # help="Don't remove the install prefix if installation fails.") - - #always true for test - #subparser.add_argument( - # '--keep-stage', action='store_true', dest='keep_stage', - # help="Don't remove the build stage if installation succeeds.") subparser.add_argument( '-n', '--no-checksum', action='store_true', dest='no_checksum', help="Do not check packages against checksum") - subparser.add_argument( - '-v', '--verbose', action='store_true', dest='verbose', - help="Display verbose build output while installing.") - - #subparser.add_argument( - # '--fake', action='store_true', dest='fake', - # help="Fake install. Just remove the prefix and touch a fake file in it.") subparser.add_argument( 'output', help="test output goes in this file") @@ -161,7 +140,7 @@ def test_install(parser, args): keep_stage=False, ignore_deps=False, make_jobs=args.jobs, - verbose=args.verbose, + verbose=True, fake=False) finally: #Find all packages that are not a dependency of another package -- cgit v1.2.3-70-g09d2 From b9bf0b942c4e2770de65282311876d02d00e7f5b Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 15 Oct 2015 11:52:08 -0700 Subject: Use spec.traverse vs. recursive function. Also even though I calculated which installs are new (e.g. vs. packages that have already been installed by a previous command) I forgot to make use of that in create_test_output (so I was always generating test output even if a package had been installed before running the test-install command). Note to avoid confusion: the 'handled' variable (removed in this commit) did not serve the same purpose as 'newInstalls': it was originally required because the recursive approach would visit the same dependency twice if more than one package depended on it. --- lib/spack/spack/cmd/test-install.py | 36 ++++++++++++++++-------------------- 1 file changed, 16 insertions(+), 20 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/cmd/test-install.py b/lib/spack/spack/cmd/test-install.py index ec413115c0..6652a204fb 100644 --- a/lib/spack/spack/cmd/test-install.py +++ b/lib/spack/spack/cmd/test-install.py @@ -81,18 +81,21 @@ class BuildId(object): return "-".join(str(x) for x in (self.name, self.version, self.hashId)) -def create_test_output(spec, handled, output): - if spec in handled: - return handled[spec] - - childSuccesses = list(create_test_output(dep, handled, output) - for dep in spec.dependencies.itervalues()) - package = spack.db.get(spec) - handled[spec] = package.installed - - if all(childSuccesses): +def create_test_output(topSpec, newInstalls, output): + # Post-order traversal is not strictly required but it makes sense to output + # tests for dependencies first. + for spec in topSpec.traverse(order='post'): + if spec not in newInstalls: + continue + + if not all(spack.db.get(childSpec).installed for childSpec in + spec.dependencies.itervalues()): + #TODO: create a failed test if a dependency didn't install? + continue + bId = BuildId(spec.name, spec.version, spec.dag_hash()) + package = spack.db.get(spec) if package.installed: buildLogPath = spack.install_layout.build_log_path(spec) else: @@ -106,10 +109,7 @@ def create_test_output(spec, handled, output): # lines. It may be better to look for errors. output.add_test(bId, package.installed, buildLogPath + '\n' + spec.to_yaml() + buildLog) - #TODO: create a failed test if a dependency didn't install? - - return handled[spec] - + def test_install(parser, args): if not args.package: @@ -143,14 +143,10 @@ def test_install(parser, args): verbose=True, fake=False) finally: - #Find all packages that are not a dependency of another package - topLevelNewInstalls = newInstalls - set(itertools.chain.from_iterable( - spec.dependencies for spec in newInstalls)) - jrf = JunitResultFormat() handled = {} - for spec in topLevelNewInstalls: - create_test_output(spec, handled, jrf) + for spec in specs: + create_test_output(spec, newInstalls, jrf) with open(args.output, 'wb') as F: jrf.write_to(F) -- cgit v1.2.3-70-g09d2 From c985ad7644e00fa2fd5253d6b3f761a2596c6c4b Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 15 Oct 2015 12:23:56 -0700 Subject: Update test failure output: don't include the entire build log, just lines which mention errors (or if no such lines can be found, output the last 10 lines from the log). --- lib/spack/spack/cmd/test-install.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/cmd/test-install.py b/lib/spack/spack/cmd/test-install.py index 6652a204fb..f2c40e57e4 100644 --- a/lib/spack/spack/cmd/test-install.py +++ b/lib/spack/spack/cmd/test-install.py @@ -25,6 +25,7 @@ from external import argparse import xml.etree.ElementTree as ET import itertools +import re import llnl.util.tty as tty from llnl.util.filesystem import * @@ -104,11 +105,15 @@ def create_test_output(topSpec, newInstalls, output): buildLogPath = join_path(package.stage.source_path, 'spack-build.out') with open(buildLogPath, 'rb') as F: - buildLog = F.read() #TODO: this may not return all output - #TODO: add the whole build log? it could be several thousand - # lines. It may be better to look for errors. - output.add_test(bId, package.installed, buildLogPath + '\n' + - spec.to_yaml() + buildLog) + lines = F.readlines() + errMessages = list(line for line in lines if + re.search('error:', line, re.IGNORECASE)) + errOutput = errMessages if errMessages else lines[-10:] + errOutput = '\n'.join(itertools.chain( + [spec.to_yaml(), "Errors:"], errOutput, + ["Build Log:", buildLogPath])) + + output.add_test(bId, package.installed, errOutput) def test_install(parser, args): -- cgit v1.2.3-70-g09d2 From 4997f0fe57e65002d8122da05a4f203f51ac4345 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 15 Oct 2015 12:44:02 -0700 Subject: Move logic for tracking the build log into package.py (since that is what is managing the build log) and expose as package.build_log_path. --- lib/spack/spack/cmd/test-install.py | 11 ++--------- lib/spack/spack/package.py | 8 ++++++++ 2 files changed, 10 insertions(+), 9 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/cmd/test-install.py b/lib/spack/spack/cmd/test-install.py index f2c40e57e4..8514042239 100644 --- a/lib/spack/spack/cmd/test-install.py +++ b/lib/spack/spack/cmd/test-install.py @@ -97,21 +97,14 @@ def create_test_output(topSpec, newInstalls, output): bId = BuildId(spec.name, spec.version, spec.dag_hash()) package = spack.db.get(spec) - if package.installed: - buildLogPath = spack.install_layout.build_log_path(spec) - else: - #TODO: search recursively under stage.path instead of only within - # stage.source_path - buildLogPath = join_path(package.stage.source_path, 'spack-build.out') - - with open(buildLogPath, 'rb') as F: + with open(package.build_log_path, 'rb') as F: lines = F.readlines() errMessages = list(line for line in lines if re.search('error:', line, re.IGNORECASE)) errOutput = errMessages if errMessages else lines[-10:] errOutput = '\n'.join(itertools.chain( [spec.to_yaml(), "Errors:"], errOutput, - ["Build Log:", buildLogPath])) + ["Build Log:", package.build_log_path])) output.add_test(bId, package.installed, errOutput) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 61606d0590..da19a7c398 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -863,6 +863,14 @@ class Package(object): dep.package.do_install(**kwargs) + @property + def build_log_path(self): + if self.installed: + return spack.install_layout.build_log_path(spec) + else: + return join_path(self.stage.source_path, 'spack-build.out') + + @property def module(self): """Use this to add variables to the class's module's scope. -- cgit v1.2.3-70-g09d2 From e451421db32b5e2059d9da293c70baa4b3374449 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 15 Oct 2015 19:22:36 -0700 Subject: 1. Specifying the output file path for test-install is now an option (vs. an argument). The default path is [package id].xml in the CWD where test-install is called from. 2. Fixed a bug with package.build_log_path (which was added in this branch). 3. keep_stage for package.do_install is now set. This allows uninstalling and reinstalling packages without (re) downloading them. --- lib/spack/spack/cmd/test-install.py | 32 +++++++++++++++++++++----------- lib/spack/spack/package.py | 2 +- 2 files changed, 22 insertions(+), 12 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/cmd/test-install.py b/lib/spack/spack/cmd/test-install.py index 8514042239..e207295810 100644 --- a/lib/spack/spack/cmd/test-install.py +++ b/lib/spack/spack/cmd/test-install.py @@ -26,6 +26,7 @@ from external import argparse import xml.etree.ElementTree as ET import itertools import re +import os import llnl.util.tty as tty from llnl.util.filesystem import * @@ -45,7 +46,7 @@ def setup_parser(subparser): help="Do not check packages against checksum") subparser.add_argument( - 'output', help="test output goes in this file") + '-o', '--output', action='store', help="test output goes in this file") subparser.add_argument( 'package', help="spec of package to install") @@ -73,10 +74,10 @@ class JunitResultFormat(object): class BuildId(object): - def __init__(self, name, version, hashId): - self.name = name - self.version = version - self.hashId = hashId + def __init__(self, spec): + self.name = spec.name + self.version = spec.version + self.hashId = spec.dag_hash() def stringId(self): return "-".join(str(x) for x in (self.name, self.version, self.hashId)) @@ -94,7 +95,7 @@ def create_test_output(topSpec, newInstalls, output): #TODO: create a failed test if a dependency didn't install? continue - bId = BuildId(spec.name, spec.version, spec.dag_hash()) + bId = BuildId(spec) package = spack.db.get(spec) with open(package.build_log_path, 'rb') as F: @@ -120,22 +121,31 @@ def test_install(parser, args): if args.no_checksum: spack.do_checksum = False # TODO: remove this global. - #TODO: should a single argument be wrapped in a list? + # TODO: should a single argument be wrapped in a list? specs = spack.cmd.parse_specs(args.package, concretize=True) + + # There is 1 top-level package + topSpec = iter(specs).next() + newInstalls = set() - for spec in itertools.chain.from_iterable(spec.traverse() - for spec in specs): + for spec in topSpec.traverse(): package = spack.db.get(spec) if not package.installed: newInstalls.add(spec) + if not args.output: + bId = BuildId(topSpec) + outputFpath = join_path(os.getcwd(), "{0}.xml".format(bId.stringId())) + else: + outputFpath = args.output + try: for spec in specs: package = spack.db.get(spec) if not package.installed: package.do_install( keep_prefix=False, - keep_stage=False, + keep_stage=True, ignore_deps=False, make_jobs=args.jobs, verbose=True, @@ -146,5 +156,5 @@ def test_install(parser, args): for spec in specs: create_test_output(spec, newInstalls, jrf) - with open(args.output, 'wb') as F: + with open(outputFpath, 'wb') as F: jrf.write_to(F) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index da19a7c398..b1257a092f 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -866,7 +866,7 @@ class Package(object): @property def build_log_path(self): if self.installed: - return spack.install_layout.build_log_path(spec) + return spack.install_layout.build_log_path(self.spec) else: return join_path(self.stage.source_path, 'spack-build.out') -- cgit v1.2.3-70-g09d2 From 82ed1bc34397542394ea7fc4a23f3b827546809a Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 15 Oct 2015 19:38:47 -0700 Subject: Originally I enforced specifying 1 top-level package with the test-install command by having it consume exactly 1 positional argument (i.e. by removing "nargs=argparse.REMAINDER") but this does not work when configuring dependencies of a top-level package (which show up as additional positional args). Instead now there is an explicit check to ensure there is only 1 top-level package. --- lib/spack/spack/cmd/test-install.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/cmd/test-install.py b/lib/spack/spack/cmd/test-install.py index e207295810..962af939f2 100644 --- a/lib/spack/spack/cmd/test-install.py +++ b/lib/spack/spack/cmd/test-install.py @@ -49,7 +49,7 @@ def setup_parser(subparser): '-o', '--output', action='store', help="test output goes in this file") subparser.add_argument( - 'package', help="spec of package to install") + 'package', nargs=argparse.REMAINDER, help="spec of package to install") class JunitResultFormat(object): @@ -120,11 +120,10 @@ def test_install(parser, args): if args.no_checksum: spack.do_checksum = False # TODO: remove this global. - - # TODO: should a single argument be wrapped in a list? - specs = spack.cmd.parse_specs(args.package, concretize=True) - # There is 1 top-level package + specs = spack.cmd.parse_specs(args.package, concretize=True) + if len(specs) > 1: + tty.die("Only 1 top-level package can be specified") topSpec = iter(specs).next() newInstalls = set() -- cgit v1.2.3-70-g09d2 From 49b91235bb8522a79d5a0b213718af2a6f81f501 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 15 Oct 2015 19:59:57 -0700 Subject: Minor edit for clarity (generate output for single top level spec vs. iterating through collection of size 1) --- lib/spack/spack/cmd/test-install.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/cmd/test-install.py b/lib/spack/spack/cmd/test-install.py index 962af939f2..ee9580c128 100644 --- a/lib/spack/spack/cmd/test-install.py +++ b/lib/spack/spack/cmd/test-install.py @@ -152,8 +152,7 @@ def test_install(parser, args): finally: jrf = JunitResultFormat() handled = {} - for spec in specs: - create_test_output(spec, newInstalls, jrf) + create_test_output(topSpec, newInstalls, jrf) with open(outputFpath, 'wb') as F: jrf.write_to(F) -- cgit v1.2.3-70-g09d2 From 6cd976d036cce518d899202baeebc7103a0a6f2a Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 15 Oct 2015 20:13:08 -0700 Subject: Better description for test-install command --- lib/spack/spack/cmd/test-install.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/cmd/test-install.py b/lib/spack/spack/cmd/test-install.py index ee9580c128..0facf59b94 100644 --- a/lib/spack/spack/cmd/test-install.py +++ b/lib/spack/spack/cmd/test-install.py @@ -34,7 +34,7 @@ from llnl.util.filesystem import * import spack import spack.cmd -description = "Build and install packages" +description = "Treat package installations as unit tests and output formatted test results" def setup_parser(subparser): subparser.add_argument( -- cgit v1.2.3-70-g09d2 From 39f0f000f89f40a32b9e25d9ba681d6d032d025a Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 15 Oct 2015 22:02:14 -0700 Subject: Created unit test for core logic in test-install command. --- lib/spack/spack/cmd/test-install.py | 36 +++++++---- lib/spack/spack/test/__init__.py | 3 +- lib/spack/spack/test/unit_install.py | 118 +++++++++++++++++++++++++++++++++++ 3 files changed, 145 insertions(+), 12 deletions(-) create mode 100644 lib/spack/spack/test/unit_install.py (limited to 'lib') diff --git a/lib/spack/spack/cmd/test-install.py b/lib/spack/spack/cmd/test-install.py index 0facf59b94..7b37f66967 100644 --- a/lib/spack/spack/cmd/test-install.py +++ b/lib/spack/spack/cmd/test-install.py @@ -82,8 +82,23 @@ class BuildId(object): def stringId(self): return "-".join(str(x) for x in (self.name, self.version, self.hashId)) + def __hash__(self): + return hash((self.name, self.version, self.hashId)) + + def __eq__(self, other): + if not isinstance(other, BuildId): + return False + + return ((self.name, self.version, self.hashId) == + (other.name, other.version, other.hashId)) + + +def fetch_log(path): + with open(path, 'rb') as F: + return list(F.readlines()) -def create_test_output(topSpec, newInstalls, output): + +def create_test_output(topSpec, newInstalls, output, getLogFunc=fetch_log): # Post-order traversal is not strictly required but it makes sense to output # tests for dependencies first. for spec in topSpec.traverse(order='post'): @@ -98,17 +113,16 @@ def create_test_output(topSpec, newInstalls, output): bId = BuildId(spec) package = spack.db.get(spec) - with open(package.build_log_path, 'rb') as F: - lines = F.readlines() - errMessages = list(line for line in lines if - re.search('error:', line, re.IGNORECASE)) - errOutput = errMessages if errMessages else lines[-10:] - errOutput = '\n'.join(itertools.chain( - [spec.to_yaml(), "Errors:"], errOutput, - ["Build Log:", package.build_log_path])) - - output.add_test(bId, package.installed, errOutput) + lines = getLogFunc(package.build_log_path) + errMessages = list(line for line in lines if + re.search('error:', line, re.IGNORECASE)) + errOutput = errMessages if errMessages else lines[-10:] + errOutput = '\n'.join(itertools.chain( + [spec.to_yaml(), "Errors:"], errOutput, + ["Build Log:", package.build_log_path])) + output.add_test(bId, package.installed, errOutput) + def test_install(parser, args): if not args.package: diff --git a/lib/spack/spack/test/__init__.py b/lib/spack/spack/test/__init__.py index 6b3715be6f..6fd80d1084 100644 --- a/lib/spack/spack/test/__init__.py +++ b/lib/spack/spack/test/__init__.py @@ -56,7 +56,8 @@ test_names = ['versions', 'spec_yaml', 'optional_deps', 'make_executable', - 'configure_guess'] + 'configure_guess', + 'unit_install'] def list_tests(): diff --git a/lib/spack/spack/test/unit_install.py b/lib/spack/spack/test/unit_install.py new file mode 100644 index 0000000000..ab7d4902d0 --- /dev/null +++ b/lib/spack/spack/test/unit_install.py @@ -0,0 +1,118 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +import unittest +import itertools + +import spack +test_install = __import__("spack.cmd.test-install", + fromlist=["BuildId", "create_test_output"]) + +class MockOutput(object): + def __init__(self): + self.results = {} + + def add_test(self, buildId, passed=True, buildInfo=None): + self.results[buildId] = passed + + def write_to(self, stream): + pass + +class MockSpec(object): + def __init__(self, name, version, hashStr=None): + self.dependencies = {} + self.name = name + self.version = version + self.hash = hashStr if hashStr else hash((name, version)) + + def traverse(self, order=None): + allDeps = itertools.chain.from_iterable(i.traverse() for i in + self.dependencies.itervalues()) + return set(itertools.chain([self], allDeps)) + + def dag_hash(self): + return self.hash + + def to_yaml(self): + return "<<>>".format(test_install.BuildId(self).stringId()) + +class MockPackage(object): + def __init__(self, buildLogPath): + self.installed = False + self.build_log_path = buildLogPath + +specX = MockSpec("X", "1.2.0") +specY = MockSpec("Y", "2.3.8") +specX.dependencies['Y'] = specY +pkgX = MockPackage('logX') +pkgY = MockPackage('logY') +bIdX = test_install.BuildId(specX) +bIdY = test_install.BuildId(specY) + +class UnitInstallTest(unittest.TestCase): + """Tests test-install where X->Y""" + + def setUp(self): + super(UnitInstallTest, self).setUp() + + #import pdb; pdb.set_trace() + pkgX.installed = False + pkgY.installed = False + + pkgDb = MockPackageDb({specX:pkgX, specY:pkgY}) + spack.db = pkgDb + + def tearDown(self): + super(UnitInstallTest, self).tearDown() + + def test_installing_both(self): + mo = MockOutput() + + pkgX.installed = True + pkgY.installed = True + test_install.create_test_output(specX, [specX, specY], mo, getLogFunc=test_fetch_log) + + self.assertEqual(mo.results, {bIdX:True, bIdY:True}) + + def test_dependency_already_installed(self): + mo = MockOutput() + + pkgX.installed = True + pkgY.installed = True + test_install.create_test_output(specX, [specX], mo, getLogFunc=test_fetch_log) + + self.assertEqual(mo.results, {bIdX:True}) + +class MockPackageDb(object): + def __init__(self, init=None): + self.specToPkg = {} + if init: + self.specToPkg.update(init) + + def get(self, spec): + return self.specToPkg[spec] + +def test_fetch_log(path): + return [] + -- cgit v1.2.3-70-g09d2 From 7598612b86c35c062c2b7f4ed2bcee1729e11a03 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sat, 17 Oct 2015 17:34:39 -0700 Subject: Better info variant output. --- lib/spack/spack/cmd/info.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/cmd/info.py b/lib/spack/spack/cmd/info.py index c6209523f0..085e4db44d 100644 --- a/lib/spack/spack/cmd/info.py +++ b/lib/spack/spack/cmd/info.py @@ -65,11 +65,21 @@ def print_text_info(pkg): print "None" else: pad = padder(pkg.variants, 4) + + maxv = max(len(v) for v in sorted(pkg.variants)) + fmt = "%%-%ss%%-10s%%s" % (maxv + 4) + + print " " + fmt % ('Name', 'Default', 'Description') + print for name in sorted(pkg.variants): v = pkg.variants[name] - print " %s%s" % ( - pad(('+' if v.default else '-') + name + ':'), - "\n".join(textwrap.wrap(v.description))) + default = 'on' if v.default else 'off' + + lines = textwrap.wrap(v.description) + lines[1:] = [" " + (" " * maxv) + l for l in lines[1:]] + desc = "\n".join(lines) + + print " " + fmt % (name, default, desc) print print "Dependencies:" -- cgit v1.2.3-70-g09d2 From 3c788147cacfbf48f2eec9f3ee785d0e5d67bed3 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 18 Oct 2015 18:45:32 -0700 Subject: Add Spack logo. --- README.md | 4 ++-- lib/spack/docs/conf.py | 6 +++--- share/spack/logo/favicon.ico | Bin 0 -> 1150 bytes share/spack/logo/spack-logo-text-64.png | Bin 0 -> 18644 bytes share/spack/logo/spack-logo-white-text-48.png | Bin 0 -> 12201 bytes 5 files changed, 5 insertions(+), 5 deletions(-) create mode 100755 share/spack/logo/favicon.ico create mode 100644 share/spack/logo/spack-logo-text-64.png create mode 100644 share/spack/logo/spack-logo-white-text-48.png (limited to 'lib') diff --git a/README.md b/README.md index 98d781564c..3a2c535d4e 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ -Spack -=========== +![image](share/spack/logo/spack-logo-text-64.png "Spack") +============ Spack is a package management tool designed to support multiple versions and configurations of software on a wide variety of platforms diff --git a/lib/spack/docs/conf.py b/lib/spack/docs/conf.py index 7303d7fef6..bce9ef0e94 100644 --- a/lib/spack/docs/conf.py +++ b/lib/spack/docs/conf.py @@ -149,7 +149,7 @@ html_theme = 'sphinx_rtd_theme' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. -#html_theme_options = [('show_copyright', False)] +html_theme_options = { 'logo_only' : True } # Add any paths that contain custom themes here, relative to this directory. html_theme_path = ["_themes"] @@ -163,12 +163,12 @@ html_theme_path = ["_themes"] # The name of an image file (relative to this directory) to place at the top # of the sidebar. -#html_logo = None +html_logo = '../../../share/spack/logo/spack-logo-white-text-48.png' # The name of an image file (within the static path) to use as favicon of the # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 # pixels large. -#html_favicon = None +html_favicon = '../../../share/spack/logo/favicon.ico' # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, diff --git a/share/spack/logo/favicon.ico b/share/spack/logo/favicon.ico new file mode 100755 index 0000000000..95a67ae5b1 Binary files /dev/null and b/share/spack/logo/favicon.ico differ diff --git a/share/spack/logo/spack-logo-text-64.png b/share/spack/logo/spack-logo-text-64.png new file mode 100644 index 0000000000..8dad4c519f Binary files /dev/null and b/share/spack/logo/spack-logo-text-64.png differ diff --git a/share/spack/logo/spack-logo-white-text-48.png b/share/spack/logo/spack-logo-white-text-48.png new file mode 100644 index 0000000000..9e60867e81 Binary files /dev/null and b/share/spack/logo/spack-logo-white-text-48.png differ -- cgit v1.2.3-70-g09d2 From 246423b4b472ae0d53488c33fbca9faa035402d7 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 22 Oct 2015 16:00:03 -0700 Subject: Generate test results (designated as skipped) for parents of failed dependencies --- lib/spack/spack/cmd/test-install.py | 58 ++++++++++++++++++++++++------------ lib/spack/spack/test/unit_install.py | 11 +++---- 2 files changed, 45 insertions(+), 24 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/cmd/test-install.py b/lib/spack/spack/cmd/test-install.py index 7b37f66967..406a6d7d33 100644 --- a/lib/spack/spack/cmd/test-install.py +++ b/lib/spack/spack/cmd/test-install.py @@ -57,22 +57,32 @@ class JunitResultFormat(object): self.root = ET.Element('testsuite') self.tests = [] - def add_test(self, buildId, passed=True, buildInfo=None): - self.tests.append((buildId, passed, buildInfo)) + def add_test(self, buildId, testResult, buildInfo=None): + self.tests.append((buildId, testResult, buildInfo)) def write_to(self, stream): self.root.set('tests', '{0}'.format(len(self.tests))) - for buildId, passed, buildInfo in self.tests: + for buildId, testResult, buildInfo in self.tests: testcase = ET.SubElement(self.root, 'testcase') testcase.set('classname', buildId.name) testcase.set('name', buildId.stringId()) - if not passed: + if testResult == TestResult.FAILED: failure = ET.SubElement(testcase, 'failure') failure.set('type', "Build Error") failure.text = buildInfo + elif testResult == TestResult.SKIPPED: + skipped = ET.SubElement(testcase, 'skipped') + skipped.set('type', "Skipped Build") + skipped.text = buildInfo ET.ElementTree(self.root).write(stream) +class TestResult(object): + PASSED = 0 + FAILED = 1 + SKIPPED = 2 + + class BuildId(object): def __init__(self, spec): self.name = spec.name @@ -94,6 +104,8 @@ class BuildId(object): def fetch_log(path): + if not os.path.exists(path): + return list() with open(path, 'rb') as F: return list(F.readlines()) @@ -105,23 +117,31 @@ def create_test_output(topSpec, newInstalls, output, getLogFunc=fetch_log): if spec not in newInstalls: continue - if not all(spack.db.get(childSpec).installed for childSpec in - spec.dependencies.itervalues()): - #TODO: create a failed test if a dependency didn't install? - continue - - bId = BuildId(spec) - + failedDeps = set(childSpec for childSpec in + spec.dependencies.itervalues() if not + spack.db.get(childSpec).installed) package = spack.db.get(spec) - lines = getLogFunc(package.build_log_path) - errMessages = list(line for line in lines if - re.search('error:', line, re.IGNORECASE)) - errOutput = errMessages if errMessages else lines[-10:] - errOutput = '\n'.join(itertools.chain( - [spec.to_yaml(), "Errors:"], errOutput, - ["Build Log:", package.build_log_path])) + if failedDeps: + result = TestResult.SKIPPED + dep = iter(failedDeps).next() + depBID = BuildId(dep) + errOutput = "Skipped due to failed dependency: {0}".format( + depBID.stringId()) + elif not package.installed: + result = TestResult.FAILED + lines = getLogFunc(package.build_log_path) + errMessages = list(line for line in lines if + re.search('error:', line, re.IGNORECASE)) + errOutput = errMessages if errMessages else lines[-10:] + errOutput = '\n'.join(itertools.chain( + [spec.to_yaml(), "Errors:"], errOutput, + ["Build Log:", package.build_log_path])) + else: + result = TestResult.PASSED + errOutput = None - output.add_test(bId, package.installed, errOutput) + bId = BuildId(spec) + output.add_test(bId, result, errOutput) def test_install(parser, args): diff --git a/lib/spack/spack/test/unit_install.py b/lib/spack/spack/test/unit_install.py index ab7d4902d0..2a94f8fec0 100644 --- a/lib/spack/spack/test/unit_install.py +++ b/lib/spack/spack/test/unit_install.py @@ -27,7 +27,7 @@ import itertools import spack test_install = __import__("spack.cmd.test-install", - fromlist=["BuildId", "create_test_output"]) + fromlist=["BuildId", "create_test_output", "TestResult"]) class MockOutput(object): def __init__(self): @@ -75,8 +75,7 @@ class UnitInstallTest(unittest.TestCase): def setUp(self): super(UnitInstallTest, self).setUp() - - #import pdb; pdb.set_trace() + pkgX.installed = False pkgY.installed = False @@ -93,7 +92,9 @@ class UnitInstallTest(unittest.TestCase): pkgY.installed = True test_install.create_test_output(specX, [specX, specY], mo, getLogFunc=test_fetch_log) - self.assertEqual(mo.results, {bIdX:True, bIdY:True}) + self.assertEqual(mo.results, + {bIdX:test_install.TestResult.PASSED, + bIdY:test_install.TestResult.PASSED}) def test_dependency_already_installed(self): mo = MockOutput() @@ -102,7 +103,7 @@ class UnitInstallTest(unittest.TestCase): pkgY.installed = True test_install.create_test_output(specX, [specX], mo, getLogFunc=test_fetch_log) - self.assertEqual(mo.results, {bIdX:True}) + self.assertEqual(mo.results, {bIdX:test_install.TestResult.PASSED}) class MockPackageDb(object): def __init__(self, init=None): -- cgit v1.2.3-70-g09d2 From ea872f8098af3525f0d3e9e0d2fd2efa41466e87 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 22 Oct 2015 17:44:16 -0700 Subject: 1. Added CommandError exception to build_environment 2. The parent of a failed child process in build_environment.fork no longer calls sys.exit - instead it raises a CommandError (from [1]) 3. test-install command now attempts to install all packages even if one fails --- lib/spack/spack/build_environment.py | 6 +++++- lib/spack/spack/cmd/test-install.py | 37 ++++++++++++++++++++++-------------- 2 files changed, 28 insertions(+), 15 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index a133faa629..0d179f563b 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -296,4 +296,8 @@ def fork(pkg, function): # message. Just make the parent exit with an error code. pid, returncode = os.waitpid(pid, 0) if returncode != 0: - sys.exit(1) + raise CommandError(returncode) + + +class CommandError(StandardError): + pass diff --git a/lib/spack/spack/cmd/test-install.py b/lib/spack/spack/cmd/test-install.py index 406a6d7d33..aeb90ae733 100644 --- a/lib/spack/spack/cmd/test-install.py +++ b/lib/spack/spack/cmd/test-install.py @@ -32,6 +32,7 @@ import llnl.util.tty as tty from llnl.util.filesystem import * import spack +from spack.build_environment import CommandError import spack.cmd description = "Treat package installations as unit tests and output formatted test results" @@ -107,7 +108,12 @@ def fetch_log(path): if not os.path.exists(path): return list() with open(path, 'rb') as F: - return list(F.readlines()) + return list(line.strip() for line in F.readlines()) + + +def failed_dependencies(spec): + return set(childSpec for childSpec in spec.dependencies.itervalues() if not + spack.db.get(childSpec).installed) def create_test_output(topSpec, newInstalls, output, getLogFunc=fetch_log): @@ -117,9 +123,7 @@ def create_test_output(topSpec, newInstalls, output, getLogFunc=fetch_log): if spec not in newInstalls: continue - failedDeps = set(childSpec for childSpec in - spec.dependencies.itervalues() if not - spack.db.get(childSpec).installed) + failedDeps = failed_dependencies(spec) package = spack.db.get(spec) if failedDeps: result = TestResult.SKIPPED @@ -172,10 +176,13 @@ def test_install(parser, args): else: outputFpath = args.output - try: - for spec in specs: - package = spack.db.get(spec) - if not package.installed: + for spec in topSpec.traverse(order='post'): + # Calling do_install for the top-level package would be sufficient but + # this attempts to keep going if any package fails (other packages which + # are not dependents may succeed) + package = spack.db.get(spec) + if (not failed_dependencies(spec)) and (not package.installed): + try: package.do_install( keep_prefix=False, keep_stage=True, @@ -183,10 +190,12 @@ def test_install(parser, args): make_jobs=args.jobs, verbose=True, fake=False) - finally: - jrf = JunitResultFormat() - handled = {} - create_test_output(topSpec, newInstalls, jrf) - - with open(outputFpath, 'wb') as F: + except CommandError: + pass + + jrf = JunitResultFormat() + handled = {} + create_test_output(topSpec, newInstalls, jrf) + + with open(outputFpath, 'wb') as F: jrf.write_to(F) -- cgit v1.2.3-70-g09d2 From d76c9236236747a2a19a10941b1efd497f0202e0 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 23 Oct 2015 16:18:06 -0700 Subject: 1. Rename CommandError -> InstallError 2. InstallError now subclasses SpackError vs. StandardError (so it is now handled by the spack shell script) --- lib/spack/spack/build_environment.py | 8 +++++--- lib/spack/spack/cmd/test-install.py | 4 ++-- 2 files changed, 7 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 0d179f563b..620ad5be9e 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -36,6 +36,7 @@ from llnl.util.filesystem import * import spack import spack.compilers as compilers +from spack.error import SpackError from spack.util.executable import Executable, which from spack.util.environment import * @@ -296,8 +297,9 @@ def fork(pkg, function): # message. Just make the parent exit with an error code. pid, returncode = os.waitpid(pid, 0) if returncode != 0: - raise CommandError(returncode) + raise InstallError("Installation process had nonzero exit code." + .format(str(returncode))) -class CommandError(StandardError): - pass +class InstallError(SpackError): + """Raised when a package fails to install""" diff --git a/lib/spack/spack/cmd/test-install.py b/lib/spack/spack/cmd/test-install.py index aeb90ae733..a9f9331fcb 100644 --- a/lib/spack/spack/cmd/test-install.py +++ b/lib/spack/spack/cmd/test-install.py @@ -32,7 +32,7 @@ import llnl.util.tty as tty from llnl.util.filesystem import * import spack -from spack.build_environment import CommandError +from spack.build_environment import InstallError import spack.cmd description = "Treat package installations as unit tests and output formatted test results" @@ -190,7 +190,7 @@ def test_install(parser, args): make_jobs=args.jobs, verbose=True, fake=False) - except CommandError: + except InstallError: pass jrf = JunitResultFormat() -- cgit v1.2.3-70-g09d2 From cc0ee3dc29516d6ceb49c4144fa3cb75d0120b0d Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 23 Oct 2015 20:56:06 -0700 Subject: The HTML number conversion regex operating against a byte string will only convert individual bytes, so therefore incorrectly converts utf-8 encoded characters. Decoding byte strings to unicode objects results in correct HTML number encodings. --- lib/spack/spack/cmd/test-install.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/cmd/test-install.py b/lib/spack/spack/cmd/test-install.py index a9f9331fcb..d916519227 100644 --- a/lib/spack/spack/cmd/test-install.py +++ b/lib/spack/spack/cmd/test-install.py @@ -27,6 +27,7 @@ import xml.etree.ElementTree as ET import itertools import re import os +import codecs import llnl.util.tty as tty from llnl.util.filesystem import * @@ -107,7 +108,7 @@ class BuildId(object): def fetch_log(path): if not os.path.exists(path): return list() - with open(path, 'rb') as F: + with codecs.open(path, 'rb', 'utf-8') as F: return list(line.strip() for line in F.readlines()) -- cgit v1.2.3-70-g09d2 From 908a93a470cafcb3fad8f7412e438e7f5b939d04 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sat, 24 Oct 2015 19:54:52 -0700 Subject: Add a multiprocess Barrier class to use for testing parallel code. --- lib/spack/spack/util/multiproc.py | 50 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/util/multiproc.py b/lib/spack/spack/util/multiproc.py index 9e045a090f..21cd6f543d 100644 --- a/lib/spack/spack/util/multiproc.py +++ b/lib/spack/spack/util/multiproc.py @@ -27,9 +27,11 @@ This implements a parallel map operation but it can accept more values than multiprocessing.Pool.apply() can. For example, apply() will fail to pickle functions if they're passed indirectly as parameters. """ -from multiprocessing import Process, Pipe +from multiprocessing import Process, Pipe, Semaphore, Value from itertools import izip +__all__ = ['spawn', 'parmap', 'Barrier'] + def spawn(f): def fun(pipe,x): pipe.send(f(x)) @@ -43,3 +45,49 @@ def parmap(f,X): [p.join() for p in proc] return [p.recv() for (p,c) in pipe] + +class Barrier: + """Simple reusable semaphore barrier. + + Python 2.6 doesn't have multiprocessing barriers so we implement this. + + See http://greenteapress.com/semaphores/downey08semaphores.pdf, p. 41. + """ + def __init__(self, n, timeout=None): + self.n = n + self.to = timeout + self.count = Value('i', 0) + self.mutex = Semaphore(1) + self.turnstile1 = Semaphore(0) + self.turnstile2 = Semaphore(1) + + + def wait(self): + if not self.mutex.acquire(timeout=self.to): + raise BarrierTimeoutError() + self.count.value += 1 + if self.count.value == self.n: + if not self.turnstile2.acquire(timeout=self.to): + raise BarrierTimeoutError() + self.turnstile1.release() + self.mutex.release() + + if not self.turnstile1.acquire(timeout=self.to): + raise BarrierTimeoutError() + self.turnstile1.release() + + if not self.mutex.acquire(timeout=self.to): + raise BarrierTimeoutError() + self.count.value -= 1 + if self.count.value == 0: + if not self.turnstile1.acquire(timeout=self.to): + raise BarrierTimeoutError() + self.turnstile2.release() + self.mutex.release() + + if not self.turnstile2.acquire(timeout=self.to): + raise BarrierTimeoutError() + self.turnstile2.release() + + +class BarrierTimeoutError: pass -- cgit v1.2.3-70-g09d2 From ead8ac58c6ecde1b8bd32e9f651483b57c7e3bd5 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sat, 24 Oct 2015 19:55:22 -0700 Subject: Working Lock class, now uses POSIX fcntl locks, extensive unit test. - llnl.util.lock now uses fcntl.lockf instead of flock - purported to have more NFS compatibility. - Added an extensive test case for locks. - tests acquiring, releasing, upgrading, timeouts, shared, & exclusive cases. --- lib/spack/llnl/util/lock.py | 167 +++++++++++++------------ lib/spack/spack/test/__init__.py | 3 +- lib/spack/spack/test/lock.py | 264 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 350 insertions(+), 84 deletions(-) create mode 100644 lib/spack/spack/test/lock.py (limited to 'lib') diff --git a/lib/spack/llnl/util/lock.py b/lib/spack/llnl/util/lock.py index 3cd02befe5..6e49bf74e6 100644 --- a/lib/spack/llnl/util/lock.py +++ b/lib/spack/llnl/util/lock.py @@ -1,5 +1,5 @@ ############################################################################## -# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Copyright (c) 2013-2015, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory. # # This file is part of Spack. @@ -22,134 +22,135 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -"""Lock implementation for shared filesystems.""" import os import fcntl import errno import time import socket -# Default timeout for locks. -DEFAULT_TIMEOUT = 60 +# Default timeout in seconds, after which locks will raise exceptions. +_default_timeout = 60 -class _ReadLockContext(object): - """Context manager that takes and releases a read lock. +# Sleep time per iteration in spin loop (in seconds) +_sleep_time = 1e-5 - Arguments are lock and timeout (default 5 minutes) - """ - def __init__(self, lock, timeout=DEFAULT_TIMEOUT): - self._lock = lock - self._timeout = timeout - def __enter__(self): - self._lock.acquire_read(self._timeout) +class Lock(object): + def __init__(self,file_path): + self._file_path = file_path + self._fd = None + self._reads = 0 + self._writes = 0 - def __exit__(self,type,value,traceback): - self._lock.release_read() + def _lock(self, op, timeout): + """This takes a lock using POSIX locks (``fnctl.lockf``). -class _WriteLockContext(object): - """Context manager that takes and releases a write lock. + The lock is implemented as a spin lock using a nonblocking + call to lockf(). - Arguments are lock and timeout (default 5 minutes) - """ - def __init__(self, lock, timeout=DEFAULT_TIMEOUT): - self._lock = lock - self._timeout = timeout + On acquiring an exclusive lock, the lock writes this process's + pid and host to the lock file, in case the holding process + needs to be killed later. - def __enter__(self): - self._lock.acquire_write(self._timeout) + If the lock times out, it raises a ``LockError``. + """ + start_time = time.time() + while (time.time() - start_time) < timeout: + try: + if self._fd is None: + self._fd = os.open(self._file_path, os.O_RDWR) - def __exit__(self,type,value,traceback): - self._lock.release_write() + fcntl.lockf(self._fd, op | fcntl.LOCK_NB) + if op == fcntl.LOCK_EX: + os.write(self._fd, "pid=%s,host=%s" % (os.getpid(), socket.getfqdn())) + return + + except IOError as error: + if error.errno == errno.EAGAIN or error.errno == errno.EACCES: + pass + else: + raise + time.sleep(_sleep_time) + raise LockError("Timed out waiting for lock.") -class Lock(object): - """Distributed file-based lock using ``flock``.""" - def __init__(self, file_path): - self._file_path = file_path - self._fd = os.open(file_path,os.O_RDWR) - self._reads = 0 - self._writes = 0 + def _unlock(self): + """Releases a lock using POSIX locks (``fcntl.lockf``) + Releases the lock regardless of mode. Note that read locks may + be masquerading as write locks, but this removes either. - def write_lock(self, timeout=DEFAULT_TIMEOUT): - """Convenience method that returns a write lock context.""" - return _WriteLockContext(self, timeout) + """ + fcntl.lockf(self._fd,fcntl.LOCK_UN) + os.close(self._fd) + self._fd = None - def read_lock(self, timeout=DEFAULT_TIMEOUT): - """Convenience method that returns a read lock context.""" - return _ReadLockContext(self, timeout) + def acquire_read(self, timeout=_default_timeout): + """Acquires a recursive, shared lock for reading. + Read and write locks can be acquired and released in arbitrary + order, but the POSIX lock is held until all local read and + write locks are released. - def acquire_read(self, timeout): - """ - Implements recursive lock. If held in both read and write mode, - the write lock will be maintained until all locks are released """ if self._reads == 0 and self._writes == 0: self._lock(fcntl.LOCK_SH, timeout) self._reads += 1 - def acquire_write(self, timeout): - """ - Implements recursive lock + def acquire_write(self, timeout=_default_timeout): + """Acquires a recursive, exclusive lock for writing. + + Read and write locks can be acquired and released in arbitrary + order, but the POSIX lock is held until all local read and + write locks are released. """ if self._writes == 0: self._lock(fcntl.LOCK_EX, timeout) self._writes += 1 - def _lock(self, op, timeout): - """ - The timeout is implemented using nonblocking flock() - to avoid using signals for timing - Write locks store pid and host information to the lock file - Read locks do not store data - """ - total_time = 0 - while total_time < timeout: - try: - fcntl.flock(self._fd, op | fcntl.LOCK_NB) - if op == fcntl.LOCK_EX: - with open(self._file_path, 'w') as f: - f.write("pid = " + str(os.getpid()) + ", host = " + socket.getfqdn()) - return - except IOError as error: - if error.errno == errno.EAGAIN or error.errno == EACCES: - pass - else: - raise - time.sleep(0.1) - total_time += 0.1 + def release_read(self): + """Releases a read lock. + Returns True if the last recursive lock was released, False if + there are still outstanding locks. + + Does limited correctness checking: if a read lock is released + when none are held, this will raise an assertion error. - def release_read(self): - """ - Assert there is a lock of the right type to release, recursive lock """ assert self._reads > 0 - if self._reads == 1 and self._writes == 0: - self._unlock() + self._reads -= 1 + if self._reads == 0 and self._writes == 0: + self._unlock() + return True + return False def release_write(self): - """ - Assert there is a lock of the right type to release, recursive lock + """Releases a write lock. + + Returns True if the last recursive lock was released, False if + there are still outstanding locks. + + Does limited correctness checking: if a read lock is released + when none are held, this will raise an assertion error. + """ assert self._writes > 0 - if self._writes == 1 and self._reads == 0: - self._unlock() + self._writes -= 1 + if self._writes == 0 and self._reads == 0: + self._unlock() + return True + return False - def _unlock(self): - """ - Releases the lock regardless of mode. Note that read locks may be - masquerading as write locks at times, but this removes either. - """ - fcntl.flock(self._fd, fcntl.LOCK_UN) +class LockError(Exception): + """Raised when an attempt to acquire a lock times out.""" + pass diff --git a/lib/spack/spack/test/__init__.py b/lib/spack/spack/test/__init__.py index c3b39b76f8..84419781e2 100644 --- a/lib/spack/spack/test/__init__.py +++ b/lib/spack/spack/test/__init__.py @@ -57,6 +57,7 @@ test_names = ['versions', 'optional_deps', 'make_executable', 'configure_guess', + 'lock', 'database'] @@ -77,7 +78,7 @@ def run(names, verbose=False): if test not in test_names: tty.error("%s is not a valid spack test name." % test, "Valid names are:") - colify(test_names, indent=4) + colify(sorted(test_names), indent=4) sys.exit(1) runner = unittest.TextTestRunner(verbosity=verbosity) diff --git a/lib/spack/spack/test/lock.py b/lib/spack/spack/test/lock.py new file mode 100644 index 0000000000..2e7440bbbc --- /dev/null +++ b/lib/spack/spack/test/lock.py @@ -0,0 +1,264 @@ +############################################################################## +# Copyright (c) 2013-2015, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +""" +These tests ensure that our lock works correctly. +""" +import unittest +import os +import tempfile +import shutil +from multiprocessing import Process + +from llnl.util.lock import * +from llnl.util.filesystem import join_path, touch + +from spack.util.multiproc import Barrier + +# This is the longest a failed test will take, as the barriers will +# time out and raise an exception. +barrier_timeout = 5 + + +def order_processes(*functions): + """Order some processes using simple barrier synchronization.""" + b = Barrier(len(functions), timeout=barrier_timeout) + procs = [Process(target=f, args=(b,)) for f in functions] + for p in procs: p.start() + for p in procs: p.join() + + +class LockTest(unittest.TestCase): + + def setUp(self): + self.tempdir = tempfile.mkdtemp() + self.lock_path = join_path(self.tempdir, 'lockfile') + touch(self.lock_path) + + + def tearDown(self): + shutil.rmtree(self.tempdir, ignore_errors=True) + + + # + # Process snippets below can be composed into tests. + # + def acquire_write(self, barrier): + lock = Lock(self.lock_path) + lock.acquire_write() # grab exclusive lock + barrier.wait() + barrier.wait() # hold the lock until exception raises in other procs. + + def acquire_read(self, barrier): + lock = Lock(self.lock_path) + lock.acquire_read() # grab shared lock + barrier.wait() + barrier.wait() # hold the lock until exception raises in other procs. + + def timeout_write(self, barrier): + lock = Lock(self.lock_path) + barrier.wait() # wait for lock acquire in first process + self.assertRaises(LockError, lock.acquire_write, 0.1) + barrier.wait() + + def timeout_read(self, barrier): + lock = Lock(self.lock_path) + barrier.wait() # wait for lock acquire in first process + self.assertRaises(LockError, lock.acquire_read, 0.1) + barrier.wait() + + + # + # Test that exclusive locks on other processes time out when an + # exclusive lock is held. + # + def test_write_lock_timeout_on_write(self): + order_processes(self.acquire_write, self.timeout_write) + + def test_write_lock_timeout_on_write_2(self): + order_processes(self.acquire_write, self.timeout_write, self.timeout_write) + + def test_write_lock_timeout_on_write_3(self): + order_processes(self.acquire_write, self.timeout_write, self.timeout_write, self.timeout_write) + + + # + # Test that shared locks on other processes time out when an + # exclusive lock is held. + # + def test_read_lock_timeout_on_write(self): + order_processes(self.acquire_write, self.timeout_read) + + def test_read_lock_timeout_on_write_2(self): + order_processes(self.acquire_write, self.timeout_read, self.timeout_read) + + def test_read_lock_timeout_on_write_3(self): + order_processes(self.acquire_write, self.timeout_read, self.timeout_read, self.timeout_read) + + + # + # Test that exclusive locks time out when shared locks are held. + # + def test_write_lock_timeout_on_read(self): + order_processes(self.acquire_read, self.timeout_write) + + def test_write_lock_timeout_on_read_2(self): + order_processes(self.acquire_read, self.timeout_write, self.timeout_write) + + def test_write_lock_timeout_on_read_3(self): + order_processes(self.acquire_read, self.timeout_write, self.timeout_write, self.timeout_write) + + + # + # Test that exclusive locks time while lots of shared locks are held. + # + def test_write_lock_timeout_with_multiple_readers_2_1(self): + order_processes(self.acquire_read, self.acquire_read, self.timeout_write) + + def test_write_lock_timeout_with_multiple_readers_2_2(self): + order_processes(self.acquire_read, self.acquire_read, self.timeout_write, self.timeout_write) + + def test_write_lock_timeout_with_multiple_readers_3_1(self): + order_processes(self.acquire_read, self.acquire_read, self.acquire_read, self.timeout_write) + + def test_write_lock_timeout_with_multiple_readers_3_2(self): + order_processes(self.acquire_read, self.acquire_read, self.acquire_read, self.timeout_write, self.timeout_write) + + + # + # Longer test case that ensures locks are reusable. Ordering is + # enforced by barriers throughout -- steps are shown with numbers. + # + def test_complex_acquire_and_release_chain(self): + def p1(barrier): + lock = Lock(self.lock_path) + + lock.acquire_write() + barrier.wait() # ---------------------------------------- 1 + # others test timeout + barrier.wait() # ---------------------------------------- 2 + lock.release_write() # release and others acquire read + barrier.wait() # ---------------------------------------- 3 + self.assertRaises(LockError, lock.acquire_write, 0.1) + lock.acquire_read() + barrier.wait() # ---------------------------------------- 4 + lock.release_read() + barrier.wait() # ---------------------------------------- 5 + + # p2 upgrades read to write + barrier.wait() # ---------------------------------------- 6 + self.assertRaises(LockError, lock.acquire_write, 0.1) + self.assertRaises(LockError, lock.acquire_read, 0.1) + barrier.wait() # ---------------------------------------- 7 + # p2 releases write and read + barrier.wait() # ---------------------------------------- 8 + + # p3 acquires read + barrier.wait() # ---------------------------------------- 9 + # p3 upgrades read to write + barrier.wait() # ---------------------------------------- 10 + self.assertRaises(LockError, lock.acquire_write, 0.1) + self.assertRaises(LockError, lock.acquire_read, 0.1) + barrier.wait() # ---------------------------------------- 11 + # p3 releases locks + barrier.wait() # ---------------------------------------- 12 + lock.acquire_read() + barrier.wait() # ---------------------------------------- 13 + lock.release_read() + + + def p2(barrier): + lock = Lock(self.lock_path) + + # p1 acquires write + barrier.wait() # ---------------------------------------- 1 + self.assertRaises(LockError, lock.acquire_write, 0.1) + self.assertRaises(LockError, lock.acquire_read, 0.1) + barrier.wait() # ---------------------------------------- 2 + lock.acquire_read() + barrier.wait() # ---------------------------------------- 3 + # p1 tests shared read + barrier.wait() # ---------------------------------------- 4 + # others release reads + barrier.wait() # ---------------------------------------- 5 + + lock.acquire_write() # upgrade read to write + barrier.wait() # ---------------------------------------- 6 + # others test timeout + barrier.wait() # ---------------------------------------- 7 + lock.release_write() # release read AND write (need both) + lock.release_read() + barrier.wait() # ---------------------------------------- 8 + + # p3 acquires read + barrier.wait() # ---------------------------------------- 9 + # p3 upgrades read to write + barrier.wait() # ---------------------------------------- 10 + self.assertRaises(LockError, lock.acquire_write, 0.1) + self.assertRaises(LockError, lock.acquire_read, 0.1) + barrier.wait() # ---------------------------------------- 11 + # p3 releases locks + barrier.wait() # ---------------------------------------- 12 + lock.acquire_read() + barrier.wait() # ---------------------------------------- 13 + lock.release_read() + + + def p3(barrier): + lock = Lock(self.lock_path) + + # p1 acquires write + barrier.wait() # ---------------------------------------- 1 + self.assertRaises(LockError, lock.acquire_write, 0.1) + self.assertRaises(LockError, lock.acquire_read, 0.1) + barrier.wait() # ---------------------------------------- 2 + lock.acquire_read() + barrier.wait() # ---------------------------------------- 3 + # p1 tests shared read + barrier.wait() # ---------------------------------------- 4 + lock.release_read() + barrier.wait() # ---------------------------------------- 5 + + # p2 upgrades read to write + barrier.wait() # ---------------------------------------- 6 + self.assertRaises(LockError, lock.acquire_write, 0.1) + self.assertRaises(LockError, lock.acquire_read, 0.1) + barrier.wait() # ---------------------------------------- 7 + # p2 releases write & read + barrier.wait() # ---------------------------------------- 8 + + lock.acquire_read() + barrier.wait() # ---------------------------------------- 9 + lock.acquire_write() + barrier.wait() # ---------------------------------------- 10 + # others test timeout + barrier.wait() # ---------------------------------------- 11 + lock.release_read() # release read AND write in opposite + lock.release_write() # order from before on p2 + barrier.wait() # ---------------------------------------- 12 + lock.acquire_read() + barrier.wait() # ---------------------------------------- 13 + lock.release_read() + + order_processes(p1, p2, p3) -- cgit v1.2.3-70-g09d2 From 6a16040462b49acb356236a5f8114b055d680851 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 26 Oct 2015 11:58:52 -0700 Subject: Automatically create a 'test-output' directory in the current directory if no output path is specified. Test output files are placed in this directory. Furthermore the filenames now have the prefix "test" (but otherwise are the string representation of the spec ID as before). --- lib/spack/spack/cmd/test-install.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/cmd/test-install.py b/lib/spack/spack/cmd/test-install.py index d916519227..76602474a4 100644 --- a/lib/spack/spack/cmd/test-install.py +++ b/lib/spack/spack/cmd/test-install.py @@ -173,7 +173,10 @@ def test_install(parser, args): if not args.output: bId = BuildId(topSpec) - outputFpath = join_path(os.getcwd(), "{0}.xml".format(bId.stringId())) + outputDir = join_path(os.getcwd(), "test-output") + if not os.path.exists(outputDir): + os.mkdir(outputDir) + outputFpath = join_path(outputDir, "test-{0}.xml".format(bId.stringId())) else: outputFpath = args.output -- cgit v1.2.3-70-g09d2 From 9576860f8c97360eddaffe316450ed2f3b87e876 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 26 Oct 2015 14:27:44 -0700 Subject: Making SpackError reference consistent. --- lib/spack/spack/build_environment.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 620ad5be9e..dac25d9940 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -36,7 +36,6 @@ from llnl.util.filesystem import * import spack import spack.compilers as compilers -from spack.error import SpackError from spack.util.executable import Executable, which from spack.util.environment import * @@ -301,5 +300,5 @@ def fork(pkg, function): .format(str(returncode))) -class InstallError(SpackError): +class InstallError(spack.error.SpackError): """Raised when a package fails to install""" -- cgit v1.2.3-70-g09d2 From 9d90cb69622dbc6a22eaf2fbf796dd27a3e2def2 Mon Sep 17 00:00:00 2001 From: Ben Boeckel Date: Thu, 15 Oct 2015 09:18:16 -0400 Subject: python: use the setdefault method on dict It allows more concise code and skips some key lookups. --- lib/spack/llnl/util/lang.py | 5 +---- lib/spack/spack/directives.py | 10 ++++------ lib/spack/spack/virtual.py | 8 ++------ 3 files changed, 7 insertions(+), 16 deletions(-) (limited to 'lib') diff --git a/lib/spack/llnl/util/lang.py b/lib/spack/llnl/util/lang.py index 9e1bef18ca..156ee34c9e 100644 --- a/lib/spack/llnl/util/lang.py +++ b/lib/spack/llnl/util/lang.py @@ -87,10 +87,7 @@ def index_by(objects, *funcs): result = {} for o in objects: key = f(o) - if key not in result: - result[key] = [o] - else: - result[key].append(o) + result.setdefault(key, []).append(o) for key, objects in result.items(): result[key] = index_by(objects, *funcs[1:]) diff --git a/lib/spack/spack/directives.py b/lib/spack/spack/directives.py index 9297d6dac3..78039ac6f9 100644 --- a/lib/spack/spack/directives.py +++ b/lib/spack/spack/directives.py @@ -239,12 +239,10 @@ def patch(pkg, url_or_filename, level=1, when=None): when = pkg.name when_spec = parse_anonymous_spec(when, pkg.name) - if when_spec not in pkg.patches: - pkg.patches[when_spec] = [Patch(pkg.name, url_or_filename, level)] - else: - # if this spec is identical to some other, then append this - # patch to the existing list. - pkg.patches[when_spec].append(Patch(pkg.name, url_or_filename, level)) + cur_patches = pkg.patches.setdefault(when_spec, []) + # if this spec is identical to some other, then append this + # patch to the existing list. + cur_patches.append(Patch(pkg.name, url_or_filename, level)) @directive('variants') diff --git a/lib/spack/spack/virtual.py b/lib/spack/spack/virtual.py index fa070e6bd5..c77b259d61 100644 --- a/lib/spack/spack/virtual.py +++ b/lib/spack/spack/virtual.py @@ -73,10 +73,8 @@ class ProviderIndex(object): for provided_spec, provider_spec in pkg.provided.iteritems(): if provider_spec.satisfies(spec, deps=False): provided_name = provided_spec.name - if provided_name not in self.providers: - self.providers[provided_name] = {} - provider_map = self.providers[provided_name] + provider_map = self.providers.setdefault(provided_name, {}) if not provided_spec in provider_map: provider_map[provided_spec] = set() @@ -133,9 +131,7 @@ class ProviderIndex(object): if lp_spec.name == rp_spec.name: try: const = lp_spec.copy().constrain(rp_spec,deps=False) - if constrained not in result: - result[constrained] = set() - result[constrained].add(const) + result.setdefault(constrained, set()).add(const) except spack.spec.UnsatisfiableSpecError: continue return result -- cgit v1.2.3-70-g09d2 From 3b554c709bb1dc3704939964ac1265ccf8597718 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Mon, 26 Oct 2015 15:26:08 -0700 Subject: Fetch errors were also terminating runs of test-install with system exit, so stage.fetch() was updated to raise a FetchError instead of calling tty.die(). Output is the same for spack install in case of a fetch error. --- lib/spack/spack/cmd/test-install.py | 6 ++++++ lib/spack/spack/stage.py | 3 ++- lib/spack/spack/test/unit_install.py | 2 ++ 3 files changed, 10 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/cmd/test-install.py b/lib/spack/spack/cmd/test-install.py index 76602474a4..58ab40aa7b 100644 --- a/lib/spack/spack/cmd/test-install.py +++ b/lib/spack/spack/cmd/test-install.py @@ -34,6 +34,7 @@ from llnl.util.filesystem import * import spack from spack.build_environment import InstallError +from spack.fetch_strategy import FetchError import spack.cmd description = "Treat package installations as unit tests and output formatted test results" @@ -132,6 +133,9 @@ def create_test_output(topSpec, newInstalls, output, getLogFunc=fetch_log): depBID = BuildId(dep) errOutput = "Skipped due to failed dependency: {0}".format( depBID.stringId()) + elif (not package.installed) and (not package.stage.archive_file): + result = TestResult.FAILED + errOutput = "Failure to fetch package resources." elif not package.installed: result = TestResult.FAILED lines = getLogFunc(package.build_log_path) @@ -196,6 +200,8 @@ def test_install(parser, args): fake=False) except InstallError: pass + except FetchError: + pass jrf = JunitResultFormat() handled = {} diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index 008c5f0429..78930ecb5b 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -261,7 +261,8 @@ class Stage(object): tty.debug(e) continue else: - tty.die("All fetchers failed for %s" % self.name) + errMessage = "All fetchers failed for %s" % self.name + raise fs.FetchError(errMessage, None) def check(self): diff --git a/lib/spack/spack/test/unit_install.py b/lib/spack/spack/test/unit_install.py index 2a94f8fec0..c4b9092f05 100644 --- a/lib/spack/spack/test/unit_install.py +++ b/lib/spack/spack/test/unit_install.py @@ -105,6 +105,8 @@ class UnitInstallTest(unittest.TestCase): self.assertEqual(mo.results, {bIdX:test_install.TestResult.PASSED}) + #TODO: add test(s) where Y fails to install + class MockPackageDb(object): def __init__(self, init=None): self.specToPkg = {} -- cgit v1.2.3-70-g09d2 From 17a58ee0a95ac6b15cec3731f93262a6378e7d6a Mon Sep 17 00:00:00 2001 From: Ben Boeckel Date: Mon, 26 Oct 2015 18:31:25 -0400 Subject: architecture: use uname if available --- lib/spack/spack/architecture.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 0c4b605e91..e0d42c2077 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -24,6 +24,7 @@ ############################################################################## import os import platform as py_platform +import subprocess from llnl.util.lang import memoized @@ -69,12 +70,24 @@ def get_mac_sys_type(): Version(mac_ver).up_to(2), py_platform.machine()) +def get_sys_type_from_uname(): + """Return the architecture from uname.""" + try: + arch_proc = subprocess.Popen(['uname', '-i'], + stdout=subprocess.PIPE) + arch, _ = arch_proc.communicate() + return arch.strip() + except: + return None + + @memoized def sys_type(): """Returns a SysType for the current machine.""" methods = [get_sys_type_from_spack_globals, get_sys_type_from_environment, - get_mac_sys_type] + get_mac_sys_type, + get_sys_type_from_uname] # search for a method that doesn't return None sys_type = None -- cgit v1.2.3-70-g09d2 From 6c9b10f73dc2c5bf2281d6434d3aaa5cf2ded056 Mon Sep 17 00:00:00 2001 From: Ben Boeckel Date: Mon, 26 Oct 2015 18:54:13 -0400 Subject: architecture: remove custom mac_type method --- lib/spack/spack/architecture.py | 13 ------------- 1 file changed, 13 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index e0d42c2077..05ac5d6f35 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -23,14 +23,12 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import os -import platform as py_platform import subprocess from llnl.util.lang import memoized import spack import spack.error as serr -from spack.version import Version class InvalidSysTypeError(serr.SpackError): @@ -60,16 +58,6 @@ def get_sys_type_from_environment(): return os.environ.get('SYS_TYPE') -def get_mac_sys_type(): - """Return a Mac OS SYS_TYPE or None if this isn't a mac.""" - mac_ver = py_platform.mac_ver()[0] - if not mac_ver: - return None - - return "macosx_%s_%s" % ( - Version(mac_ver).up_to(2), py_platform.machine()) - - def get_sys_type_from_uname(): """Return the architecture from uname.""" try: @@ -86,7 +74,6 @@ def sys_type(): """Returns a SysType for the current machine.""" methods = [get_sys_type_from_spack_globals, get_sys_type_from_environment, - get_mac_sys_type, get_sys_type_from_uname] # search for a method that doesn't return None -- cgit v1.2.3-70-g09d2 From af7b96c14a555805a50f13a1fa1343650db184ab Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 27 Oct 2015 00:35:06 -0700 Subject: Lock acquires return True/False depending on whether they got POSIX lock. --- lib/spack/llnl/util/lock.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/spack/llnl/util/lock.py b/lib/spack/llnl/util/lock.py index 6e49bf74e6..dcca37687e 100644 --- a/lib/spack/llnl/util/lock.py +++ b/lib/spack/llnl/util/lock.py @@ -95,10 +95,15 @@ class Lock(object): order, but the POSIX lock is held until all local read and write locks are released. + Returns True if it is the first acquire and actually acquires + the POSIX lock, False if it is a nested transaction. + """ - if self._reads == 0 and self._writes == 0: - self._lock(fcntl.LOCK_SH, timeout) self._reads += 1 + if self._reads == 1 and self._writes == 0: + self._lock(fcntl.LOCK_SH, timeout) + return True + return False def acquire_write(self, timeout=_default_timeout): @@ -107,10 +112,16 @@ class Lock(object): Read and write locks can be acquired and released in arbitrary order, but the POSIX lock is held until all local read and write locks are released. + + Returns True if it is the first acquire and actually acquires + the POSIX lock, False if it is a nested transaction. + """ - if self._writes == 0: - self._lock(fcntl.LOCK_EX, timeout) self._writes += 1 + if self._writes == 1: + self._lock(fcntl.LOCK_EX, timeout) + return True + return False def release_read(self): -- cgit v1.2.3-70-g09d2 From 50d0a2643bbef81ec2e97da209ae1974b6b77993 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 27 Oct 2015 13:34:46 -0700 Subject: Not all package stages have an archive file (e.g. source code repos) but all of them do have a source_path: use this instead to check whether the package resources were successfully retrieved. --- lib/spack/spack/cmd/test-install.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/cmd/test-install.py b/lib/spack/spack/cmd/test-install.py index 58ab40aa7b..68b761d5dc 100644 --- a/lib/spack/spack/cmd/test-install.py +++ b/lib/spack/spack/cmd/test-install.py @@ -133,7 +133,7 @@ def create_test_output(topSpec, newInstalls, output, getLogFunc=fetch_log): depBID = BuildId(dep) errOutput = "Skipped due to failed dependency: {0}".format( depBID.stringId()) - elif (not package.installed) and (not package.stage.archive_file): + elif (not package.installed) and (not package.stage.source_path): result = TestResult.FAILED errOutput = "Failure to fetch package resources." elif not package.installed: -- cgit v1.2.3-70-g09d2 From bf8479bec6311f28cd9e18c580e23794001cbf23 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 27 Oct 2015 16:29:14 -0700 Subject: Fix stupid lock bug. - Code simplification ignored case where exception was raised. - If LockError was raised, read and write counts were incremented erroneously. - updated lock test. --- lib/spack/llnl/util/lock.py | 40 +++++++++++++++++++++++--------------- lib/spack/spack/package.py | 8 +++++--- lib/spack/spack/test/lock.py | 46 +++++++++++++++++++++++--------------------- 3 files changed, 53 insertions(+), 41 deletions(-) (limited to 'lib') diff --git a/lib/spack/llnl/util/lock.py b/lib/spack/llnl/util/lock.py index dcca37687e..ac3684bd55 100644 --- a/lib/spack/llnl/util/lock.py +++ b/lib/spack/llnl/util/lock.py @@ -99,11 +99,13 @@ class Lock(object): the POSIX lock, False if it is a nested transaction. """ - self._reads += 1 - if self._reads == 1 and self._writes == 0: - self._lock(fcntl.LOCK_SH, timeout) + if self._reads == 0 and self._writes == 0: + self._lock(fcntl.LOCK_SH, timeout) # can raise LockError. + self._reads += 1 return True - return False + else: + self._reads += 1 + return False def acquire_write(self, timeout=_default_timeout): @@ -117,11 +119,13 @@ class Lock(object): the POSIX lock, False if it is a nested transaction. """ - self._writes += 1 - if self._writes == 1: - self._lock(fcntl.LOCK_EX, timeout) + if self._writes == 0: + self._lock(fcntl.LOCK_EX, timeout) # can raise LockError. + self._writes += 1 return True - return False + else: + self._writes += 1 + return False def release_read(self): @@ -136,11 +140,13 @@ class Lock(object): """ assert self._reads > 0 - self._reads -= 1 - if self._reads == 0 and self._writes == 0: - self._unlock() + if self._reads == 1 and self._writes == 0: + self._unlock() # can raise LockError. + self._reads -= 1 return True - return False + else: + self._reads -= 1 + return False def release_write(self): @@ -155,11 +161,13 @@ class Lock(object): """ assert self._writes > 0 - self._writes -= 1 - if self._writes == 0 and self._reads == 0: - self._unlock() + if self._writes == 1 and self._reads == 0: + self._unlock() # can raise LockError. + self._writes -= 1 return True - return False + else: + self._writes -= 1 + return False class LockError(Exception): diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index e6944ce40c..2957257b1a 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -606,6 +606,7 @@ class Package(object): spack.install_layout.remove_install_directory(self.spec) spack.installed_db.remove(self.spec) + def do_fetch(self): """Creates a stage directory and downloads the taball for this package. Working directory will be set to the stage directory. @@ -812,9 +813,6 @@ class Package(object): log_install_path = spack.install_layout.build_log_path(self.spec) install(log_path, log_install_path) - #Update the database once we know install successful - spack.installed_db.add(self.spec, spack.install_layout.path_for_spec(self.spec)) - # On successful install, remove the stage. if not keep_stage: self.stage.destroy() @@ -845,6 +843,10 @@ class Package(object): # Do the build. spack.build_environment.fork(self, real_work) + # note: PARENT of the build process adds the new package to + # the database, so that we don't need to re-read from file. + spack.installed_db.add(self.spec, spack.install_layout.path_for_spec(self.spec)) + # Once everything else is done, run post install hooks spack.hooks.post_install(self) diff --git a/lib/spack/spack/test/lock.py b/lib/spack/spack/test/lock.py index 2e7440bbbc..5664e71b03 100644 --- a/lib/spack/spack/test/lock.py +++ b/lib/spack/spack/test/lock.py @@ -41,14 +41,6 @@ from spack.util.multiproc import Barrier barrier_timeout = 5 -def order_processes(*functions): - """Order some processes using simple barrier synchronization.""" - b = Barrier(len(functions), timeout=barrier_timeout) - procs = [Process(target=f, args=(b,)) for f in functions] - for p in procs: p.start() - for p in procs: p.join() - - class LockTest(unittest.TestCase): def setUp(self): @@ -61,6 +53,16 @@ class LockTest(unittest.TestCase): shutil.rmtree(self.tempdir, ignore_errors=True) + def multiproc_test(self, *functions): + """Order some processes using simple barrier synchronization.""" + b = Barrier(len(functions), timeout=barrier_timeout) + procs = [Process(target=f, args=(b,)) for f in functions] + for p in procs: p.start() + for p in procs: + p.join() + self.assertEqual(p.exitcode, 0) + + # # Process snippets below can be composed into tests. # @@ -94,13 +96,13 @@ class LockTest(unittest.TestCase): # exclusive lock is held. # def test_write_lock_timeout_on_write(self): - order_processes(self.acquire_write, self.timeout_write) + self.multiproc_test(self.acquire_write, self.timeout_write) def test_write_lock_timeout_on_write_2(self): - order_processes(self.acquire_write, self.timeout_write, self.timeout_write) + self.multiproc_test(self.acquire_write, self.timeout_write, self.timeout_write) def test_write_lock_timeout_on_write_3(self): - order_processes(self.acquire_write, self.timeout_write, self.timeout_write, self.timeout_write) + self.multiproc_test(self.acquire_write, self.timeout_write, self.timeout_write, self.timeout_write) # @@ -108,42 +110,42 @@ class LockTest(unittest.TestCase): # exclusive lock is held. # def test_read_lock_timeout_on_write(self): - order_processes(self.acquire_write, self.timeout_read) + self.multiproc_test(self.acquire_write, self.timeout_read) def test_read_lock_timeout_on_write_2(self): - order_processes(self.acquire_write, self.timeout_read, self.timeout_read) + self.multiproc_test(self.acquire_write, self.timeout_read, self.timeout_read) def test_read_lock_timeout_on_write_3(self): - order_processes(self.acquire_write, self.timeout_read, self.timeout_read, self.timeout_read) + self.multiproc_test(self.acquire_write, self.timeout_read, self.timeout_read, self.timeout_read) # # Test that exclusive locks time out when shared locks are held. # def test_write_lock_timeout_on_read(self): - order_processes(self.acquire_read, self.timeout_write) + self.multiproc_test(self.acquire_read, self.timeout_write) def test_write_lock_timeout_on_read_2(self): - order_processes(self.acquire_read, self.timeout_write, self.timeout_write) + self.multiproc_test(self.acquire_read, self.timeout_write, self.timeout_write) def test_write_lock_timeout_on_read_3(self): - order_processes(self.acquire_read, self.timeout_write, self.timeout_write, self.timeout_write) + self.multiproc_test(self.acquire_read, self.timeout_write, self.timeout_write, self.timeout_write) # # Test that exclusive locks time while lots of shared locks are held. # def test_write_lock_timeout_with_multiple_readers_2_1(self): - order_processes(self.acquire_read, self.acquire_read, self.timeout_write) + self.multiproc_test(self.acquire_read, self.acquire_read, self.timeout_write) def test_write_lock_timeout_with_multiple_readers_2_2(self): - order_processes(self.acquire_read, self.acquire_read, self.timeout_write, self.timeout_write) + self.multiproc_test(self.acquire_read, self.acquire_read, self.timeout_write, self.timeout_write) def test_write_lock_timeout_with_multiple_readers_3_1(self): - order_processes(self.acquire_read, self.acquire_read, self.acquire_read, self.timeout_write) + self.multiproc_test(self.acquire_read, self.acquire_read, self.acquire_read, self.timeout_write) def test_write_lock_timeout_with_multiple_readers_3_2(self): - order_processes(self.acquire_read, self.acquire_read, self.acquire_read, self.timeout_write, self.timeout_write) + self.multiproc_test(self.acquire_read, self.acquire_read, self.acquire_read, self.timeout_write, self.timeout_write) # @@ -261,4 +263,4 @@ class LockTest(unittest.TestCase): barrier.wait() # ---------------------------------------- 13 lock.release_read() - order_processes(p1, p2, p3) + self.multiproc_test(p1, p2, p3) -- cgit v1.2.3-70-g09d2 From a58ae0c5d0002a7c6cce606b3308dbf53fc29317 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 27 Oct 2015 16:36:44 -0700 Subject: Build database working with simple transaction support; all tests passing. --- lib/spack/llnl/util/tty/color.py | 5 + lib/spack/spack/cmd/__init__.py | 21 +- lib/spack/spack/cmd/deactivate.py | 11 +- lib/spack/spack/cmd/diy.py | 11 +- lib/spack/spack/cmd/extensions.py | 3 +- lib/spack/spack/cmd/find.py | 11 +- lib/spack/spack/cmd/install.py | 8 +- lib/spack/spack/cmd/uninstall.py | 34 ++-- lib/spack/spack/database.py | 393 ++++++++++++++++++++++++++---------- lib/spack/spack/directory_layout.py | 3 - lib/spack/spack/package.py | 2 +- lib/spack/spack/test/database.py | 373 ++++++++++++++++++++++++++++------ 12 files changed, 645 insertions(+), 230 deletions(-) (limited to 'lib') diff --git a/lib/spack/llnl/util/tty/color.py b/lib/spack/llnl/util/tty/color.py index 22080a7b37..0d09303da0 100644 --- a/lib/spack/llnl/util/tty/color.py +++ b/lib/spack/llnl/util/tty/color.py @@ -158,6 +158,11 @@ def clen(string): return len(re.sub(r'\033[^m]*m', '', string)) +def cextra(string): + """"Length of extra color characters in a string""" + return len(''.join(re.findall(r'\033[^m]*m', string))) + + def cwrite(string, stream=sys.stdout, color=None): """Replace all color expressions in string with ANSI control codes and write the result to the stream. If color is diff --git a/lib/spack/spack/cmd/__init__.py b/lib/spack/spack/cmd/__init__.py index d4778b1375..6ce6fa0960 100644 --- a/lib/spack/spack/cmd/__init__.py +++ b/lib/spack/spack/cmd/__init__.py @@ -124,16 +124,15 @@ def elide_list(line_list, max_num=10): def disambiguate_spec(spec): - with spack.installed_db.read_lock(): - matching_specs = spack.installed_db.query(spec) - if not matching_specs: - tty.die("Spec '%s' matches no installed packages." % spec) - - elif len(matching_specs) > 1: - args = ["%s matches multiple packages." % spec, - "Matching packages:"] - args += [" " + str(s) for s in matching_specs] - args += ["Use a more specific spec."] - tty.die(*args) + matching_specs = spack.installed_db.query(spec) + if not matching_specs: + tty.die("Spec '%s' matches no installed packages." % spec) + + elif len(matching_specs) > 1: + args = ["%s matches multiple packages." % spec, + "Matching packages:"] + args += [" " + str(s) for s in matching_specs] + args += ["Use a more specific spec."] + tty.die(*args) return matching_specs[0] diff --git a/lib/spack/spack/cmd/deactivate.py b/lib/spack/spack/cmd/deactivate.py index 5428e3d2de..1f0e303cdf 100644 --- a/lib/spack/spack/cmd/deactivate.py +++ b/lib/spack/spack/cmd/deactivate.py @@ -54,13 +54,12 @@ def deactivate(parser, args): if args.all: if pkg.extendable: tty.msg("Deactivating all extensions of %s" % pkg.spec.short_spec) - with spack.installed_db.read_lock(): - ext_pkgs = spack.installed_db.installed_extensions_for(spec) + ext_pkgs = spack.installed_db.installed_extensions_for(spec) - for ext_pkg in ext_pkgs: - ext_pkg.spec.normalize() - if ext_pkg.activated: - ext_pkg.do_deactivate(force=True) + for ext_pkg in ext_pkgs: + ext_pkg.spec.normalize() + if ext_pkg.activated: + ext_pkg.do_deactivate(force=True) elif pkg.is_extension: if not args.force and not spec.package.activated: diff --git a/lib/spack/spack/cmd/diy.py b/lib/spack/spack/cmd/diy.py index 6178c9c3e3..f7998720ac 100644 --- a/lib/spack/spack/cmd/diy.py +++ b/lib/spack/spack/cmd/diy.py @@ -54,11 +54,12 @@ def diy(self, args): if not args.spec: tty.die("spack diy requires a package spec argument.") - with spack.installed_db.write_lock(): - specs = spack.cmd.parse_specs(args.spec) - if len(specs) > 1: - tty.die("spack diy only takes one spec.") + specs = spack.cmd.parse_specs(args.spec) + if len(specs) > 1: + tty.die("spack diy only takes one spec.") + # Take a write lock before checking for existence. + with spack.installed_db.write_lock(): spec = specs[0] if not spack.db.exists(spec.name): tty.warn("No such package: %s" % spec.name) @@ -85,7 +86,7 @@ def diy(self, args): # Forces the build to run out of the current directory. package.stage = DIYStage(os.getcwd()) - # TODO: make this an argument, not a global. + # TODO: make this an argument, not a global. spack.do_checksum = False package.do_install( diff --git a/lib/spack/spack/cmd/extensions.py b/lib/spack/spack/cmd/extensions.py index f0f99a2691..7cadc424b0 100644 --- a/lib/spack/spack/cmd/extensions.py +++ b/lib/spack/spack/cmd/extensions.py @@ -80,8 +80,7 @@ def extensions(parser, args): colify(ext.name for ext in extensions) # List specs of installed extensions. - with spack.installed_db.read_lock(): - installed = [s.spec for s in spack.installed_db.installed_extensions_for(spec)] + installed = [s.spec for s in spack.installed_db.installed_extensions_for(spec)] print if not installed: tty.msg("None installed.") diff --git a/lib/spack/spack/cmd/find.py b/lib/spack/spack/cmd/find.py index 6a0c3d11ff..0b0dd6ef6f 100644 --- a/lib/spack/spack/cmd/find.py +++ b/lib/spack/spack/cmd/find.py @@ -158,12 +158,11 @@ def find(parser, args): q_args = { 'installed' : installed, 'known' : known } # Get all the specs the user asked for - with spack.installed_db.read_lock(): - if not query_specs: - specs = set(spack.installed_db.query(**q_args)) - else: - results = [set(spack.installed_db.query(qs, **q_args)) for qs in query_specs] - specs = set.union(*results) + if not query_specs: + specs = set(spack.installed_db.query(**q_args)) + else: + results = [set(spack.installed_db.query(qs, **q_args)) for qs in query_specs] + specs = set.union(*results) if not args.mode: args.mode = 'short' diff --git a/lib/spack/spack/cmd/install.py b/lib/spack/spack/cmd/install.py index ada655b937..ba824bd658 100644 --- a/lib/spack/spack/cmd/install.py +++ b/lib/spack/spack/cmd/install.py @@ -68,10 +68,10 @@ def install(parser, args): if args.no_checksum: spack.do_checksum = False # TODO: remove this global. - with spack.installed_db.write_lock(): - specs = spack.cmd.parse_specs(args.packages, concretize=True) - for spec in specs: - package = spack.db.get(spec) + specs = spack.cmd.parse_specs(args.packages, concretize=True) + for spec in specs: + package = spack.db.get(spec) + with spack.installed_db.write_lock(): package.do_install( keep_prefix=args.keep_prefix, keep_stage=args.keep_stage, diff --git a/lib/spack/spack/cmd/uninstall.py b/lib/spack/spack/cmd/uninstall.py index 7b7c32c065..1dae84444a 100644 --- a/lib/spack/spack/cmd/uninstall.py +++ b/lib/spack/spack/cmd/uninstall.py @@ -84,21 +84,21 @@ def uninstall(parser, args): # The package.py file has gone away -- but still want to uninstall. spack.Package(s).do_uninstall(force=True) - # Sort packages to be uninstalled by the number of installed dependents - # This ensures we do things in the right order - def num_installed_deps(pkg): - return len(pkg.installed_dependents) - pkgs.sort(key=num_installed_deps) + # Sort packages to be uninstalled by the number of installed dependents + # This ensures we do things in the right order + def num_installed_deps(pkg): + return len(pkg.installed_dependents) + pkgs.sort(key=num_installed_deps) - # Uninstall packages in order now. - for pkg in pkgs: - try: - pkg.do_uninstall(force=args.force) - except PackageStillNeededError, e: - tty.error("Will not uninstall %s" % e.spec.format("$_$@$%@$#", color=True)) - print - print "The following packages depend on it:" - display_specs(e.dependents, long=True) - print - print "You can use spack uninstall -f to force this action." - sys.exit(1) + # Uninstall packages in order now. + for pkg in pkgs: + try: + pkg.do_uninstall(force=args.force) + except PackageStillNeededError, e: + tty.error("Will not uninstall %s" % e.spec.format("$_$@$%@$#", color=True)) + print + print "The following packages depend on it:" + display_specs(e.dependents, long=True) + print + print "You can use spack uninstall -f to force this action." + sys.exit(1) diff --git a/lib/spack/spack/database.py b/lib/spack/spack/database.py index 1d1c640d66..9ce00a45e9 100644 --- a/lib/spack/spack/database.py +++ b/lib/spack/spack/database.py @@ -48,7 +48,7 @@ from external.yaml.error import MarkedYAMLError, YAMLError import llnl.util.tty as tty from llnl.util.filesystem import * -from llnl.util.lock import Lock +from llnl.util.lock import * import spack.spec from spack.version import Version @@ -62,7 +62,8 @@ _db_dirname = '.spack-db' _db_version = Version('0.9') # Default timeout for spack database locks is 5 min. -_db_lock_timeout = 300 +_db_lock_timeout = 60 + def _autospec(function): """Decorator that automatically converts the argument of a single-arg @@ -90,11 +91,11 @@ class InstallRecord(object): dependents left. """ - def __init__(self, spec, path, installed): + def __init__(self, spec, path, installed, ref_count=0): self.spec = spec self.path = path self.installed = installed - self.ref_count = 0 + self.ref_count = ref_count def to_dict(self): return { 'spec' : self.spec.to_node_dict(), @@ -103,25 +104,42 @@ class InstallRecord(object): 'ref_count' : self.ref_count } @classmethod - def from_dict(cls, d): - # TODO: check the dict more rigorously. - return InstallRecord(d['spec'], d['path'], d['installed'], d['ref_count']) + def from_dict(cls, spec, dictionary): + d = dictionary + return InstallRecord(spec, d['path'], d['installed'], d['ref_count']) class Database(object): - def __init__(self, root): - """Create an empty Database. - - Location defaults to root/_index.yaml - The individual data are dicts containing - spec: the top level spec of a package - path: the path to the install of that package - dep_hash: a hash of the dependence DAG for that package + def __init__(self, root, db_dir=None): + """Create a Database for Spack installations under ``root``. + + A Database is a cache of Specs data from ``$prefix/spec.yaml`` + files in Spack installation directories. + + By default, Database files (data and lock files) are stored + under ``root/.spack-db``, which is created if it does not + exist. This is the ``db_dir``. + + The Database will attempt to read an ``index.yaml`` file in + ``db_dir``. If it does not find one, it will be created when + needed by scanning the entire Database root for ``spec.yaml`` + files according to Spack's ``DirectoryLayout``. + + Caller may optionally provide a custom ``db_dir`` parameter + where data will be stored. This is intended to be used for + testing the Database class. + """ - self._root = root + self.root = root - # Set up layout of database files. - self._db_dir = join_path(self._root, _db_dirname) + if db_dir is None: + # If the db_dir is not provided, default to within the db root. + self._db_dir = join_path(self.root, _db_dirname) + else: + # Allow customizing the database directory location for testing. + self._db_dir = db_dir + + # Set up layout of database files within the db dir self._index_path = join_path(self._db_dir, 'index.yaml') self._lock_path = join_path(self._db_dir, 'lock') @@ -135,21 +153,23 @@ class Database(object): # initialize rest of state. self.lock = Lock(self._lock_path) self._data = {} - self._last_write_time = 0 - def write_lock(self, timeout=_db_lock_timeout): - """Get a write lock context for use in a `with` block.""" - return self.lock.write_lock(timeout) + def write_transaction(self, timeout=_db_lock_timeout): + """Get a write lock context manager for use in a `with` block.""" + return WriteTransaction(self, self._read, self._write, timeout) - def read_lock(self, timeout=_db_lock_timeout): - """Get a read lock context for use in a `with` block.""" - return self.lock.read_lock(timeout) + def read_transaction(self, timeout=_db_lock_timeout): + """Get a read lock context manager for use in a `with` block.""" + return ReadTransaction(self, self._read, None, timeout) def _write_to_yaml(self, stream): - """Write out the databsae to a YAML file.""" + """Write out the databsae to a YAML file. + + This function does not do any locking or transactions. + """ # map from per-spec hash code to installation record. installs = dict((k, v.to_dict()) for k, v in self._data.items()) @@ -173,7 +193,10 @@ class Database(object): def _read_spec_from_yaml(self, hash_key, installs, parent_key=None): - """Recursively construct a spec from a hash in a YAML database.""" + """Recursively construct a spec from a hash in a YAML database. + + Does not do any locking. + """ if hash_key not in installs: parent = read_spec(installs[parent_key]['path']) @@ -195,6 +218,8 @@ class Database(object): """ Fill database from YAML, do not maintain old data Translate the spec portions from node-dict form to spec form + + Does not do any locking. """ try: if isinstance(stream, basestring): @@ -243,7 +268,7 @@ class Database(object): # Insert the brand new spec in the database. Each # spec has its own copies of its dependency specs. # TODO: would a more immmutable spec implementation simplify this? - data[hash_key] = InstallRecord(spec, rec['path'], rec['installed']) + data[hash_key] = InstallRecord.from_dict(spec, rec) except Exception as e: tty.warn("Invalid database reecord:", @@ -256,57 +281,60 @@ class Database(object): def reindex(self, directory_layout): - """Build database index from scratch based from a directory layout.""" - with self.write_lock(): - data = {} + """Build database index from scratch based from a directory layout. - # Ask the directory layout to traverse the filesystem. - for spec in directory_layout.all_specs(): - # Create a spec for each known package and add it. - path = directory_layout.path_for_spec(spec) - hash_key = spec.dag_hash() - data[hash_key] = InstallRecord(spec, path, True) + Locks the DB if it isn't locked already. - # Recursively examine dependencies and add them, even - # if they are NOT installed. This ensures we know - # about missing dependencies. - for dep in spec.traverse(root=False): - dep_hash = dep.dag_hash() - if dep_hash not in data: - path = directory_layout.path_for_spec(dep) - installed = os.path.isdir(path) - data[dep_hash] = InstallRecord(dep.copy(), path, installed) - data[dep_hash].ref_count += 1 + """ + with self.write_transaction(): + old_data = self._data + try: + self._data = {} - # Assuming everything went ok, replace this object's data. - self._data = data + # Ask the directory layout to traverse the filesystem. + for spec in directory_layout.all_specs(): + # Create a spec for each known package and add it. + path = directory_layout.path_for_spec(spec) + self._add(spec, path, directory_layout) - # write out, blowing away the old version if necessary - self.write() + self._check_ref_counts() + except: + # If anything explodes, restore old data, skip write. + self._data = old_data + raise - def read(self): - """ - Re-read Database from the data in the set location - If the cache is fresh, return immediately. - """ - if not self.is_dirty(): - return - if os.path.isfile(self._index_path): - # Read from YAML file if a database exists - self._read_from_yaml(self._index_path) - else: - # The file doesn't exist, try to traverse the directory. - self.reindex(spack.install_layout) + def _check_ref_counts(self): + """Ensure consistency of reference counts in the DB. + Raise an AssertionError if something is amiss. - def write(self): + Does no locking. """ - Write the database to the standard location - Everywhere that the database is written it is read - within the same lock, so there is no need to refresh - the database within write() + counts = {} + for key, rec in self._data.items(): + counts.setdefault(key, 0) + for dep in rec.spec.dependencies.values(): + dep_key = dep.dag_hash() + counts.setdefault(dep_key, 0) + counts[dep_key] += 1 + + for rec in self._data.values(): + key = rec.spec.dag_hash() + expected = counts[key] + found = rec.ref_count + if not expected == found: + raise AssertionError( + "Invalid ref_count: %s: %d (expected %d), in DB %s." + % (key, found, expected, self._index_path)) + + + def _write(self): + """Write the in-memory database index to its file path. + + Does no locking. + """ temp_name = '%s.%s.temp' % (socket.getfqdn(), os.getpid()) temp_file = join_path(self._db_dir, temp_name) @@ -314,7 +342,6 @@ class Database(object): # Write a temporary database file them move it into place try: with open(temp_file, 'w') as f: - self._last_write_time = int(time.time()) self._write_to_yaml(f) os.rename(temp_file, self._index_path) @@ -325,36 +352,137 @@ class Database(object): raise - def is_dirty(self): + def _read(self): + """Re-read Database from the data in the set location. + + This does no locking. """ - Returns true iff the database file does not exist - or was most recently written to by another spack instance. + if os.path.isfile(self._index_path): + # Read from YAML file if a database exists + self._read_from_yaml(self._index_path) + + else: + # The file doesn't exist, try to traverse the directory. + # reindex() takes its own write lock, so no lock here. + self.reindex(spack.install_layout) + + + def read(self): + with self.read_transaction(): pass + + + def write(self): + with self.write_transaction(): pass + + + def _add(self, spec, path, directory_layout=None): + """Add an install record for spec at path to the database. + + This assumes that the spec is not already installed. It + updates the ref counts on dependencies of the spec in the DB. + + This operation is in-memory, and does not lock the DB. + """ - return (not os.path.isfile(self._index_path) or - (os.path.getmtime(self._index_path) > self._last_write_time)) + key = spec.dag_hash() + if key in self._data: + rec = self._data[key] + rec.installed = True + + # TODO: this overwrites a previous install path (when path != + # self._data[key].path), and the old path still has a + # dependent in the DB. We could consider re-RPATH-ing the + # dependents. This case is probably infrequent and may not be + # worth fixing, but this is where we can discover it. + rec.path = path + else: + self._data[key] = InstallRecord(spec, path, True) + for dep in spec.dependencies.values(): + self._increment_ref_count(dep, directory_layout) + + + def _increment_ref_count(self, spec, directory_layout=None): + """Recursively examine dependencies and update their DB entries.""" + key = spec.dag_hash() + if key not in self._data: + installed = False + path = None + if directory_layout: + path = directory_layout.path_for_spec(spec) + installed = os.path.isdir(path) + + self._data[key] = InstallRecord(spec.copy(), path, installed) + + for dep in spec.dependencies.values(): + self._increment_ref_count(dep) + + self._data[key].ref_count += 1 @_autospec def add(self, spec, path): - """Read the database from the set location + """Add spec at path to database, locking and reading DB to sync. + + ``add()`` will lock and read from the DB on disk. + + """ + # TODO: ensure that spec is concrete? + # Entire add is transactional. + with self.write_transaction(): + self._add(spec, path) + + + def _get_matching_spec_key(self, spec, **kwargs): + """Get the exact spec OR get a single spec that matches.""" + key = spec.dag_hash() + if not key in self._data: + match = self.query_one(spec, **kwargs) + if match: + return match.dag_hash() + raise KeyError("No such spec in database! %s" % spec) + return key + + + @_autospec + def get_record(self, spec, **kwargs): + key = self._get_matching_spec_key(spec, **kwargs) + return self._data[key] + + + def _decrement_ref_count(self, spec): + key = spec.dag_hash() + + if not key in self._data: + # TODO: print something here? DB is corrupt, but + # not much we can do. + return - Add the specified entry as a dict, then write the database - back to memory. This assumes that ALL dependencies are already in - the database. Should not be called otherwise. + rec = self._data[key] + rec.ref_count -= 1 + if rec.ref_count == 0 and not rec.installed: + del self._data[key] + for dep in spec.dependencies.values(): + self._decrement_ref_count(dep) + + + def _remove(self, spec): + """Non-locking version of remove(); does real work. """ - # Should always already be locked - with self.write_lock(): - self.read() - self._data[spec.dag_hash()] = InstallRecord(spec, path, True) + key = self._get_matching_spec_key(spec) + rec = self._data[key] + + if rec.ref_count > 0: + rec.installed = False + return rec.spec - # sanity check the dependencies in case something went - # wrong during install() - # TODO: ensure no races during distributed install. - for dep in spec.traverse(root=False): - assert dep.dag_hash() in self._data + del self._data[key] + for dep in rec.spec.dependencies.values(): + self._decrement_ref_count(dep) - self.write() + # Returns the concrete spec so we know it in the case where a + # query spec was passed in. + return rec.spec @_autospec @@ -369,13 +497,9 @@ class Database(object): and remvoes them if they are no longer needed. """ - # Should always already be locked - with self.write_lock(): - self.read() - hash_key = spec.dag_hash() - if hash_key in self._data: - del self._data[hash_key] - self.write() + # Take a lock around the entire removal. + with self.write_transaction(): + return self._remove(spec) @_autospec @@ -429,24 +553,75 @@ class Database(object): these really special cases that only belong here? """ - with self.read_lock(): - self.read() + with self.read_transaction(): + results = [] + for key, rec in self._data.items(): + if installed is not any and rec.installed != installed: + continue + if known is not any and spack.db.exists(rec.spec.name) != known: + continue + if query_spec is any or rec.spec.satisfies(query_spec): + results.append(rec.spec) - results = [] - for key, rec in self._data.items(): - if installed is not any and rec.installed != installed: - continue - if known is not any and spack.db.exists(rec.spec.name) != known: - continue - if query_spec is any or rec.spec.satisfies(query_spec): - results.append(rec.spec) + return sorted(results) - return sorted(results) + + def query_one(self, query_spec, known=any, installed=True): + """Query for exactly one spec that matches the query spec. + + Raises an assertion error if more than one spec matches the + query. Returns None if no installed package matches. + + """ + concrete_specs = self.query(query_spec, known, installed) + assert len(concrete_specs) <= 1 + return concrete_specs[0] if concrete_specs else None def missing(self, spec): - key = spec.dag_hash() - return key in self._data and not self._data[key].installed + with self.read_transaction(): + key = spec.dag_hash() + return key in self._data and not self._data[key].installed + + +class _Transaction(object): + """Simple nested transaction context manager that uses a file lock. + + This class can trigger actions when the lock is acquired for the + first time and released for the last. + + Timeout for lock is customizable. + """ + def __init__(self, db, acquire_fn=None, release_fn=None, + timeout=_db_lock_timeout): + self._db = db + self._timeout = timeout + self._acquire_fn = acquire_fn + self._release_fn = release_fn + + def __enter__(self): + if self._enter() and self._acquire_fn: + self._acquire_fn() + + def __exit__(self, type, value, traceback): + if self._exit() and self._release_fn: + self._release_fn() + + +class ReadTransaction(_Transaction): + def _enter(self): + return self._db.lock.acquire_read(self._timeout) + + def _exit(self): + return self._db.lock.release_read() + + +class WriteTransaction(_Transaction): + def _enter(self): + return self._db.lock.acquire_write(self._timeout) + + def _exit(self): + return self._db.lock.release_write() class CorruptDatabaseError(SpackError): diff --git a/lib/spack/spack/directory_layout.py b/lib/spack/spack/directory_layout.py index e61929d8fd..758ec209db 100644 --- a/lib/spack/spack/directory_layout.py +++ b/lib/spack/spack/directory_layout.py @@ -32,7 +32,6 @@ import tempfile from external import yaml import llnl.util.tty as tty -from llnl.util.lang import memoized from llnl.util.filesystem import join_path, mkdirp from spack.spec import Spec @@ -263,7 +262,6 @@ class YamlDirectoryLayout(DirectoryLayout): self.write_spec(spec, spec_file_path) - @memoized def all_specs(self): if not os.path.isdir(self.root): return [] @@ -274,7 +272,6 @@ class YamlDirectoryLayout(DirectoryLayout): return [self.read_spec(s) for s in spec_files] - @memoized def specs_by_hash(self): by_hash = {} for spec in self.all_specs(): diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 2957257b1a..b87baf403e 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -845,7 +845,7 @@ class Package(object): # note: PARENT of the build process adds the new package to # the database, so that we don't need to re-read from file. - spack.installed_db.add(self.spec, spack.install_layout.path_for_spec(self.spec)) + spack.installed_db.add(self.spec, self.prefix) # Once everything else is done, run post install hooks spack.hooks.post_install(self) diff --git a/lib/spack/spack/test/database.py b/lib/spack/spack/test/database.py index a3386bad99..3c5926e840 100644 --- a/lib/spack/spack/test/database.py +++ b/lib/spack/spack/test/database.py @@ -1,5 +1,5 @@ ############################################################################## -# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Copyright (c) 2013-2015, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory. # # This file is part of Spack. @@ -26,79 +26,320 @@ These tests check the database is functioning properly, both in memory and in its file """ -import unittest +import tempfile +import shutil +import multiprocessing from llnl.util.lock import * from llnl.util.filesystem import join_path import spack from spack.database import Database +from spack.directory_layout import YamlDirectoryLayout +from spack.test.mock_packages_test import * + +from llnl.util.tty.colify import colify + +def _print_ref_counts(): + """Print out all ref counts for the graph used here, for debugging""" + recs = [] + + def add_rec(spec): + cspecs = spack.installed_db.query(spec, installed=any) + + if not cspecs: + recs.append("[ %-7s ] %-20s-" % ('', spec)) + else: + key = cspecs[0].dag_hash() + rec = spack.installed_db.get_record(cspecs[0]) + recs.append("[ %-7s ] %-20s%d" % (key[:7], spec, rec.ref_count)) + + with spack.installed_db.read_transaction(): + add_rec('mpileaks ^mpich') + add_rec('callpath ^mpich') + add_rec('mpich') + + add_rec('mpileaks ^mpich2') + add_rec('callpath ^mpich2') + add_rec('mpich2') + + add_rec('mpileaks ^zmpi') + add_rec('callpath ^zmpi') + add_rec('zmpi') + add_rec('fake') + + add_rec('dyninst') + add_rec('libdwarf') + add_rec('libelf') + + colify(recs, cols=3) + + +class DatabaseTest(MockPackagesTest): + + def _mock_install(self, spec): + s = Spec(spec) + pkg = spack.db.get(s.concretized()) + pkg.do_install(fake=True) + + + def _mock_remove(self, spec): + specs = spack.installed_db.query(spec) + assert(len(specs) == 1) + spec = specs[0] + spec.package.do_uninstall(spec) -class DatabaseTest(unittest.TestCase): def setUp(self): - self.original_db = spack.installed_db - spack.installed_db = Database(self.original_db._root,"_test_index.yaml") - self.file_path = join_path(self.original_db._root,"_test_index.yaml") - if os.path.exists(self.file_path): - os.remove(self.file_path) + super(DatabaseTest, self).setUp() + # + # TODO: make the mockup below easier. + # + + # Make a fake install directory + self.install_path = tempfile.mkdtemp() + self.spack_install_path = spack.install_path + spack.install_path = self.install_path + + self.install_layout = YamlDirectoryLayout(self.install_path) + self.spack_install_layout = spack.install_layout + spack.install_layout = self.install_layout + + # Make fake database and fake install directory. + self.installed_db = Database(self.install_path) + self.spack_installed_db = spack.installed_db + spack.installed_db = self.installed_db + + # make a mock database with some packages installed note that + # the ref count for dyninst here will be 3, as it's recycled + # across each install. + # + # Here is what the mock DB looks like: + # + # o mpileaks o mpileaks' o mpileaks'' + # |\ |\ |\ + # | o callpath | o callpath' | o callpath'' + # |/| |/| |/| + # o | mpich o | mpich2 o | zmpi + # | | o | fake + # | | | + # | |______________/ + # | .____________/ + # |/ + # o dyninst + # |\ + # | o libdwarf + # |/ + # o libelf + # + + # Transaction used to avoid repeated writes. + with spack.installed_db.write_transaction(): + self._mock_install('mpileaks ^mpich') + self._mock_install('mpileaks ^mpich2') + self._mock_install('mpileaks ^zmpi') + def tearDown(self): - spack.installed_db = self.original_db - os.remove(self.file_path) - - def _test_read_from_install_tree(self): - specs = spack.install_layout.all_specs() - spack.installed_db.read_database() - spack.installed_db.write() - for sph in spack.installed_db._data: - self.assertTrue(sph['spec'] in specs) - self.assertEqual(len(specs),len(spack.installed_db._data)) - - def _test_remove_and_add(self): - specs = spack.install_layout.all_specs() - spack.installed_db.remove(specs[len(specs)-1]) - for sph in spack.installed_db._data: - self.assertTrue(sph['spec'] in specs[:len(specs)-1]) - self.assertEqual(len(specs)-1,len(spack.installed_db._data)) - - spack.installed_db.add(specs[len(specs)-1],"") - for sph in spack.installed_db._data: - self.assertTrue(sph['spec'] in specs) - self.assertEqual(len(specs),len(spack.installed_db._data)) - - def _test_read_from_file(self): - spack.installed_db.read_database() - size = len(spack.installed_db._data) - spack.installed_db._data = spack.installed_db._data[1:] - os.utime(spack.installed_db._file_path,None) - spack.installed_db.read_database() - self.assertEqual(size,len(spack.installed_db._data)) - - specs = spack.install_layout.all_specs() - self.assertEqual(size,len(specs)) - for sph in spack.installed_db._data: - self.assertTrue(sph['spec'] in specs) - - - def _test_write_to_file(self): - spack.installed_db.read_database() - size = len(spack.installed_db._data) - real_data = spack.installed_db._data - spack.installed_db._data = real_data[:size-1] - spack.installed_db.write() - spack.installed_db._data = real_data - os.utime(spack.installed_db._file_path,None) - spack.installed_db.read_database() - self.assertEqual(size-1,len(spack.installed_db._data)) - - specs = spack.install_layout.all_specs() - self.assertEqual(size,len(specs)) - for sph in spack.installed_db._data: - self.assertTrue(sph['spec'] in specs[:size-1]) - - def test_ordered_test(self): - self._test_read_from_install_tree() - self._test_remove_and_add() - self._test_read_from_file() - self._test_write_to_file() + super(DatabaseTest, self).tearDown() + shutil.rmtree(self.install_path) + spack.install_path = self.spack_install_path + spack.install_layout = self.spack_install_layout + spack.installed_db = self.spack_installed_db + + + def test_010_all_install_sanity(self): + """Ensure that the install layout reflects what we think it does.""" + all_specs = spack.install_layout.all_specs() + self.assertEqual(len(all_specs), 13) + + # query specs with multiple configurations + mpileaks_specs = [s for s in all_specs if s.satisfies('mpileaks')] + callpath_specs = [s for s in all_specs if s.satisfies('callpath')] + mpi_specs = [s for s in all_specs if s.satisfies('mpi')] + + self.assertEqual(len(mpileaks_specs), 3) + self.assertEqual(len(callpath_specs), 3) + self.assertEqual(len(mpi_specs), 3) + + # query specs with single configurations + dyninst_specs = [s for s in all_specs if s.satisfies('dyninst')] + libdwarf_specs = [s for s in all_specs if s.satisfies('libdwarf')] + libelf_specs = [s for s in all_specs if s.satisfies('libelf')] + + self.assertEqual(len(dyninst_specs), 1) + self.assertEqual(len(libdwarf_specs), 1) + self.assertEqual(len(libelf_specs), 1) + + # Query by dependency + self.assertEqual(len([s for s in all_specs if s.satisfies('mpileaks ^mpich')]), 1) + self.assertEqual(len([s for s in all_specs if s.satisfies('mpileaks ^mpich2')]), 1) + self.assertEqual(len([s for s in all_specs if s.satisfies('mpileaks ^zmpi')]), 1) + + + def test_015_write_and_read(self): + # write and read DB + with spack.installed_db.write_transaction(): + specs = spack.installed_db.query() + recs = [spack.installed_db.get_record(s) for s in specs] + spack.installed_db.write() + spack.installed_db.read() + + for spec, rec in zip(specs, recs): + new_rec = spack.installed_db.get_record(spec) + self.assertEqual(new_rec.ref_count, rec.ref_count) + self.assertEqual(new_rec.spec, rec.spec) + self.assertEqual(new_rec.path, rec.path) + self.assertEqual(new_rec.installed, rec.installed) + + + def _check_db_sanity(self): + """Utiilty function to check db against install layout.""" + expected = sorted(spack.install_layout.all_specs()) + actual = sorted(self.installed_db.query()) + + self.assertEqual(len(expected), len(actual)) + for e, a in zip(expected, actual): + self.assertEqual(e, a) + + + def test_020_db_sanity(self): + """Make sure query() returns what's actually in the db.""" + self._check_db_sanity() + + + def test_030_db_sanity_from_another_process(self): + def read_and_modify(): + self._check_db_sanity() # check that other process can read DB + with self.installed_db.write_transaction(): + self._mock_remove('mpileaks ^zmpi') + + p = multiprocessing.Process(target=read_and_modify, args=()) + p.start() + p.join() + + # ensure child process change is visible in parent process + with self.installed_db.read_transaction(): + self.assertEqual(len(self.installed_db.query('mpileaks ^zmpi')), 0) + + + def test_040_ref_counts(self): + """Ensure that we got ref counts right when we read the DB.""" + self.installed_db._check_ref_counts() + + + def test_050_basic_query(self): + """Ensure that querying the database is consistent with what is installed.""" + # query everything + self.assertEqual(len(spack.installed_db.query()), 13) + + # query specs with multiple configurations + mpileaks_specs = self.installed_db.query('mpileaks') + callpath_specs = self.installed_db.query('callpath') + mpi_specs = self.installed_db.query('mpi') + + self.assertEqual(len(mpileaks_specs), 3) + self.assertEqual(len(callpath_specs), 3) + self.assertEqual(len(mpi_specs), 3) + + # query specs with single configurations + dyninst_specs = self.installed_db.query('dyninst') + libdwarf_specs = self.installed_db.query('libdwarf') + libelf_specs = self.installed_db.query('libelf') + + self.assertEqual(len(dyninst_specs), 1) + self.assertEqual(len(libdwarf_specs), 1) + self.assertEqual(len(libelf_specs), 1) + + # Query by dependency + self.assertEqual(len(self.installed_db.query('mpileaks ^mpich')), 1) + self.assertEqual(len(self.installed_db.query('mpileaks ^mpich2')), 1) + self.assertEqual(len(self.installed_db.query('mpileaks ^zmpi')), 1) + + + def _check_remove_and_add_package(self, spec): + """Remove a spec from the DB, then add it and make sure everything's + still ok once it is added. This checks that it was + removed, that it's back when added again, and that ref + counts are consistent. + """ + original = self.installed_db.query() + self.installed_db._check_ref_counts() + + # Remove spec + concrete_spec = self.installed_db.remove(spec) + self.installed_db._check_ref_counts() + remaining = self.installed_db.query() + + # ensure spec we removed is gone + self.assertEqual(len(original) - 1, len(remaining)) + self.assertTrue(all(s in original for s in remaining)) + self.assertTrue(concrete_spec not in remaining) + + # add it back and make sure everything is ok. + self.installed_db.add(concrete_spec, "") + installed = self.installed_db.query() + self.assertEqual(len(installed), len(original)) + + # sanity check against direcory layout and check ref counts. + self._check_db_sanity() + self.installed_db._check_ref_counts() + + + def test_060_remove_and_add_root_package(self): + self._check_remove_and_add_package('mpileaks ^mpich') + + + def test_070_remove_and_add_dependency_package(self): + self._check_remove_and_add_package('dyninst') + + + def test_080_root_ref_counts(self): + rec = self.installed_db.get_record('mpileaks ^mpich') + + # Remove a top-level spec from the DB + self.installed_db.remove('mpileaks ^mpich') + + # record no longer in DB + self.assertEqual(self.installed_db.query('mpileaks ^mpich', installed=any), []) + + # record's deps have updated ref_counts + self.assertEqual(self.installed_db.get_record('callpath ^mpich').ref_count, 0) + self.assertEqual(self.installed_db.get_record('mpich').ref_count, 1) + + # put the spec back + self.installed_db.add(rec.spec, rec.path) + + # record is present again + self.assertEqual(len(self.installed_db.query('mpileaks ^mpich', installed=any)), 1) + + # dependencies have ref counts updated + self.assertEqual(self.installed_db.get_record('callpath ^mpich').ref_count, 1) + self.assertEqual(self.installed_db.get_record('mpich').ref_count, 2) + + + def test_090_non_root_ref_counts(self): + mpileaks_mpich_rec = self.installed_db.get_record('mpileaks ^mpich') + callpath_mpich_rec = self.installed_db.get_record('callpath ^mpich') + + # "force remove" a non-root spec from the DB + self.installed_db.remove('callpath ^mpich') + + # record still in DB but marked uninstalled + self.assertEqual(self.installed_db.query('callpath ^mpich', installed=True), []) + self.assertEqual(len(self.installed_db.query('callpath ^mpich', installed=any)), 1) + + # record and its deps have same ref_counts + self.assertEqual(self.installed_db.get_record('callpath ^mpich', installed=any).ref_count, 1) + self.assertEqual(self.installed_db.get_record('mpich').ref_count, 2) + + # remove only dependent of uninstalled callpath record + self.installed_db.remove('mpileaks ^mpich') + + # record and parent are completely gone. + self.assertEqual(self.installed_db.query('mpileaks ^mpich', installed=any), []) + self.assertEqual(self.installed_db.query('callpath ^mpich', installed=any), []) + + # mpich ref count updated properly. + mpich_rec = self.installed_db.get_record('mpich') + self.assertEqual(mpich_rec.ref_count, 0) -- cgit v1.2.3-70-g09d2 From 21e9d5c5c3d9384e15339ac7cf03a5bf639f8e4d Mon Sep 17 00:00:00 2001 From: miller86 Date: Tue, 27 Oct 2015 16:51:00 -0700 Subject: generalize one of the cases to handle many more kinds of formatting options --- lib/spack/spack/url.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/url.py b/lib/spack/spack/url.py index 58838306af..6adbfe156d 100644 --- a/lib/spack/spack/url.py +++ b/lib/spack/spack/url.py @@ -209,8 +209,8 @@ def parse_version_offset(path): # e.g. foobar-4.5.1 (r'-((\d+\.)*\d+)$', stem), - # e.g. foobar-4.5.1b - (r'-((\d+\.)*\d+\-?([a-z]|rc|RC|tp|TP)\d*)$', stem), + # e.g. foobar-4.5.1b, foobar4.5RC, foobar.v4.5.1b + (r'[-._]?v?((\d+\.)*\d+[-._]?([a-z]|rc|RC|tp|TP?)\d*)$', stem), # e.g. foobar-4.5.0-beta1, or foobar-4.50-beta (r'-((\d+\.)*\d+-beta(\d+)?)$', stem), -- cgit v1.2.3-70-g09d2 From 6db2a580081f0afdacd48ed6163bfbe6baaeb62f Mon Sep 17 00:00:00 2001 From: Massimiliano Culpo Date: Fri, 30 Oct 2015 07:40:45 +0100 Subject: Fixed typo : write_lock() -> write_transaction() --- lib/spack/spack/cmd/diy.py | 2 +- lib/spack/spack/cmd/install.py | 2 +- lib/spack/spack/cmd/uninstall.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/cmd/diy.py b/lib/spack/spack/cmd/diy.py index f7998720ac..9f8a6d39db 100644 --- a/lib/spack/spack/cmd/diy.py +++ b/lib/spack/spack/cmd/diy.py @@ -59,7 +59,7 @@ def diy(self, args): tty.die("spack diy only takes one spec.") # Take a write lock before checking for existence. - with spack.installed_db.write_lock(): + with spack.installed_db.write_transaction(): spec = specs[0] if not spack.db.exists(spec.name): tty.warn("No such package: %s" % spec.name) diff --git a/lib/spack/spack/cmd/install.py b/lib/spack/spack/cmd/install.py index ba824bd658..836a6260c8 100644 --- a/lib/spack/spack/cmd/install.py +++ b/lib/spack/spack/cmd/install.py @@ -71,7 +71,7 @@ def install(parser, args): specs = spack.cmd.parse_specs(args.packages, concretize=True) for spec in specs: package = spack.db.get(spec) - with spack.installed_db.write_lock(): + with spack.installed_db.write_transaction(): package.do_install( keep_prefix=args.keep_prefix, keep_stage=args.keep_stage, diff --git a/lib/spack/spack/cmd/uninstall.py b/lib/spack/spack/cmd/uninstall.py index 1dae84444a..e80f2d2636 100644 --- a/lib/spack/spack/cmd/uninstall.py +++ b/lib/spack/spack/cmd/uninstall.py @@ -53,7 +53,7 @@ def uninstall(parser, args): if not args.packages: tty.die("uninstall requires at least one package argument.") - with spack.installed_db.write_lock(): + with spack.installed_db.write_transaction(): specs = spack.cmd.parse_specs(args.packages) # For each spec provided, make sure it refers to only one package. -- cgit v1.2.3-70-g09d2 From 339da1da3d7f034595344f72f5d95e3fea0087f5 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Wed, 4 Nov 2015 07:46:17 -0800 Subject: Make architecture reflect OS *and* machine. Use Python's platform module. --- lib/spack/spack/architecture.py | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 05ac5d6f35..6c874e30be 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -23,7 +23,8 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import os -import subprocess +import re +import platform from llnl.util.lang import memoized @@ -58,15 +59,11 @@ def get_sys_type_from_environment(): return os.environ.get('SYS_TYPE') -def get_sys_type_from_uname(): - """Return the architecture from uname.""" - try: - arch_proc = subprocess.Popen(['uname', '-i'], - stdout=subprocess.PIPE) - arch, _ = arch_proc.communicate() - return arch.strip() - except: - return None +def get_sys_type_from_platform(): + """Return the architecture from Python's platform module.""" + sys_type = platform.system() + '-' + platform.machine() + sys_type = re.sub(r'[^\w-]', '_', sys_type) + return sys_type.lower() @memoized @@ -74,7 +71,7 @@ def sys_type(): """Returns a SysType for the current machine.""" methods = [get_sys_type_from_spack_globals, get_sys_type_from_environment, - get_sys_type_from_uname] + get_sys_type_from_platform] # search for a method that doesn't return None sys_type = None -- cgit v1.2.3-70-g09d2 From 0d993947ee5a4c29451cf0ee40f47519e1c3c4d9 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Wed, 4 Nov 2015 16:44:33 -0800 Subject: Fix SPACK-93, SPACK-94, GitHub #150 - `remove_prefix` was modified to remove from the DB, but the package may not have been added to the DB yet when `remove_prefix` is called from `cleanup`. - Made `remove_prefix` a pure utility function (it just removes the prefix) - Added `installed_db.remove()` call only after the `remove_prefix` in `uninstall`. --- lib/spack/spack/cmd/fsck.py | 32 -------------------------------- lib/spack/spack/cmd/reindex.py | 31 +++++++++++++++++++++++++++++++ lib/spack/spack/database.py | 19 +++++-------------- lib/spack/spack/package.py | 4 ++-- lib/spack/spack/test/database.py | 11 +++++++++-- 5 files changed, 47 insertions(+), 50 deletions(-) delete mode 100644 lib/spack/spack/cmd/fsck.py create mode 100644 lib/spack/spack/cmd/reindex.py (limited to 'lib') diff --git a/lib/spack/spack/cmd/fsck.py b/lib/spack/spack/cmd/fsck.py deleted file mode 100644 index 9a3c450dcf..0000000000 --- a/lib/spack/spack/cmd/fsck.py +++ /dev/null @@ -1,32 +0,0 @@ -############################################################################## -# Copyright (c) 2013-2015, Lawrence Livermore National Security, LLC. -# Produced at the Lawrence Livermore National Laboratory. -# -# This file is part of Spack. -# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. -# LLNL-CODE-647188 -# -# For details, see https://scalability-llnl.github.io/spack -# Please also see the LICENSE file for our notice and the LGPL. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License (as published by -# the Free Software Foundation) version 2.1 dated February 1999. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and -# conditions of the GNU General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -############################################################################## -from external import argparse -import spack - -description = "Correct database irregularities" - -# Very basic version of spack fsck -def fsck(parser, args): - spack.installed_db.reindex(spack.install_layout) diff --git a/lib/spack/spack/cmd/reindex.py b/lib/spack/spack/cmd/reindex.py new file mode 100644 index 0000000000..b584729ea4 --- /dev/null +++ b/lib/spack/spack/cmd/reindex.py @@ -0,0 +1,31 @@ +############################################################################## +# Copyright (c) 2013-2015, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from external import argparse +import spack + +description = "Rebuild Spack's package database." + +def reindex(parser, args): + spack.installed_db.reindex(spack.install_layout) diff --git a/lib/spack/spack/database.py b/lib/spack/spack/database.py index 9ce00a45e9..e0c14a0455 100644 --- a/lib/spack/spack/database.py +++ b/lib/spack/spack/database.py @@ -93,8 +93,8 @@ class InstallRecord(object): """ def __init__(self, spec, path, installed, ref_count=0): self.spec = spec - self.path = path - self.installed = installed + self.path = str(path) + self.installed = bool(installed) self.ref_count = ref_count def to_dict(self): @@ -173,7 +173,7 @@ class Database(object): # map from per-spec hash code to installation record. installs = dict((k, v.to_dict()) for k, v in self._data.items()) - # databaes includes installation list and version. + # database includes installation list and version. # NOTE: this DB version does not handle multiple installs of # the same spec well. If there are 2 identical specs with @@ -336,15 +336,14 @@ class Database(object): Does no locking. """ - temp_name = '%s.%s.temp' % (socket.getfqdn(), os.getpid()) - temp_file = join_path(self._db_dir, temp_name) + temp_file = self._index_path + ( + '.%s.%s.temp' % (socket.getfqdn(), os.getpid())) # Write a temporary database file them move it into place try: with open(temp_file, 'w') as f: self._write_to_yaml(f) os.rename(temp_file, self._index_path) - except: # Clean up temp file if something goes wrong. if os.path.exists(temp_file): @@ -367,14 +366,6 @@ class Database(object): self.reindex(spack.install_layout) - def read(self): - with self.read_transaction(): pass - - - def write(self): - with self.write_transaction(): pass - - def _add(self, spec, path, directory_layout=None): """Add an install record for spec at path to the database. diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index b15d4b2040..c631a35bf3 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -611,7 +611,6 @@ class Package(object): def remove_prefix(self): """Removes the prefix for a package along with any empty parent directories.""" spack.install_layout.remove_install_directory(self.spec) - spack.installed_db.remove(self.spec) def do_fetch(self): @@ -877,7 +876,7 @@ class Package(object): if self.installed: return spack.install_layout.build_log_path(self.spec) else: - return join_path(self.stage.source_path, 'spack-build.out') + return join_path(self.stage.source_path, 'spack-build.out') @property @@ -934,6 +933,7 @@ class Package(object): # Uninstalling in Spack only requires removing the prefix. self.remove_prefix() + spack.installed_db.remove(self.spec) tty.msg("Successfully uninstalled %s." % self.spec.short_spec) # Once everything else is done, run post install hooks diff --git a/lib/spack/spack/test/database.py b/lib/spack/spack/test/database.py index 3c5926e840..8416143f2d 100644 --- a/lib/spack/spack/test/database.py +++ b/lib/spack/spack/test/database.py @@ -148,6 +148,15 @@ class DatabaseTest(MockPackagesTest): spack.installed_db = self.spack_installed_db + def test_005_db_exists(self): + """Make sure db cache file exists after creating.""" + index_file = join_path(self.install_path, '.spack-db', 'index.yaml') + lock_file = join_path(self.install_path, '.spack-db', 'lock') + + self.assertTrue(os.path.exists(index_file)) + self.assertTrue(os.path.exists(lock_file)) + + def test_010_all_install_sanity(self): """Ensure that the install layout reflects what we think it does.""" all_specs = spack.install_layout.all_specs() @@ -182,8 +191,6 @@ class DatabaseTest(MockPackagesTest): with spack.installed_db.write_transaction(): specs = spack.installed_db.query() recs = [spack.installed_db.get_record(s) for s in specs] - spack.installed_db.write() - spack.installed_db.read() for spec, rec in zip(specs, recs): new_rec = spack.installed_db.get_record(spec) -- cgit v1.2.3-70-g09d2 From 6d7b26d4e08c2ca29f2fe6ff32e63ebfd377d164 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Wed, 11 Nov 2015 18:04:22 -0800 Subject: Insert lib/spack/external into sys.path. This avoids cases where the system python install and lib/spack/external have the same library installed. This requires modifying the names of some modules in lib/spack/external in cases where both the system python and backported features of future python versions (i.e. after 2.6) are used (previously distinguished by "from external import X" and "import X"). --- bin/spack | 2 + lib/spack/external/functools.py | 30 ---- lib/spack/external/functools_backport.py | 30 ++++ lib/spack/external/ordereddict.py | 262 ----------------------------- lib/spack/external/ordereddict_backport.py | 262 +++++++++++++++++++++++++++++ lib/spack/spack/cmd/activate.py | 2 +- lib/spack/spack/cmd/checksum.py | 2 +- lib/spack/spack/cmd/clean.py | 2 +- lib/spack/spack/cmd/compiler.py | 2 +- lib/spack/spack/cmd/config.py | 2 +- lib/spack/spack/cmd/create.py | 2 +- lib/spack/spack/cmd/deactivate.py | 2 +- lib/spack/spack/cmd/dependents.py | 2 +- lib/spack/spack/cmd/diy.py | 2 +- lib/spack/spack/cmd/env.py | 2 +- lib/spack/spack/cmd/extensions.py | 2 +- lib/spack/spack/cmd/fetch.py | 2 +- lib/spack/spack/cmd/find.py | 2 +- lib/spack/spack/cmd/graph.py | 2 +- lib/spack/spack/cmd/install.py | 2 +- lib/spack/spack/cmd/list.py | 2 +- lib/spack/spack/cmd/load.py | 2 +- lib/spack/spack/cmd/location.py | 2 +- lib/spack/spack/cmd/md5.py | 2 +- lib/spack/spack/cmd/mirror.py | 2 +- lib/spack/spack/cmd/module.py | 2 +- lib/spack/spack/cmd/patch.py | 2 +- lib/spack/spack/cmd/pkg.py | 2 +- lib/spack/spack/cmd/providers.py | 2 +- lib/spack/spack/cmd/python.py | 2 +- lib/spack/spack/cmd/reindex.py | 2 +- lib/spack/spack/cmd/restage.py | 2 +- lib/spack/spack/cmd/spec.py | 2 +- lib/spack/spack/cmd/stage.py | 2 +- lib/spack/spack/cmd/test-install.py | 2 +- lib/spack/spack/cmd/uninstall.py | 2 +- lib/spack/spack/cmd/unload.py | 2 +- lib/spack/spack/cmd/unuse.py | 2 +- lib/spack/spack/cmd/use.py | 2 +- lib/spack/spack/config.py | 6 +- lib/spack/spack/database.py | 4 +- lib/spack/spack/directory_layout.py | 2 +- lib/spack/spack/spec.py | 4 +- lib/spack/spack/test/python_version.py | 2 +- lib/spack/spack/version.py | 2 +- 45 files changed, 338 insertions(+), 336 deletions(-) delete mode 100644 lib/spack/external/functools.py create mode 100644 lib/spack/external/functools_backport.py delete mode 100644 lib/spack/external/ordereddict.py create mode 100644 lib/spack/external/ordereddict_backport.py (limited to 'lib') diff --git a/bin/spack b/bin/spack index 127a85f6fe..f587e206db 100755 --- a/bin/spack +++ b/bin/spack @@ -38,6 +38,8 @@ SPACK_PREFIX = os.path.dirname(os.path.dirname(SPACK_FILE)) # Allow spack libs to be imported in our scripts SPACK_LIB_PATH = os.path.join(SPACK_PREFIX, "lib", "spack") sys.path.insert(0, SPACK_LIB_PATH) +SPACK_EXTERNAL_LIBS = os.path.join(SPACK_LIB_PATH, "external") +sys.path.insert(0, SPACK_EXTERNAL_LIBS) # If there is no working directory, use the spack prefix. try: diff --git a/lib/spack/external/functools.py b/lib/spack/external/functools.py deleted file mode 100644 index 19f0903c82..0000000000 --- a/lib/spack/external/functools.py +++ /dev/null @@ -1,30 +0,0 @@ -# -# Backport of Python 2.7's total_ordering. -# - -def total_ordering(cls): - """Class decorator that fills in missing ordering methods""" - convert = { - '__lt__': [('__gt__', lambda self, other: not (self < other or self == other)), - ('__le__', lambda self, other: self < other or self == other), - ('__ge__', lambda self, other: not self < other)], - '__le__': [('__ge__', lambda self, other: not self <= other or self == other), - ('__lt__', lambda self, other: self <= other and not self == other), - ('__gt__', lambda self, other: not self <= other)], - '__gt__': [('__lt__', lambda self, other: not (self > other or self == other)), - ('__ge__', lambda self, other: self > other or self == other), - ('__le__', lambda self, other: not self > other)], - '__ge__': [('__le__', lambda self, other: (not self >= other) or self == other), - ('__gt__', lambda self, other: self >= other and not self == other), - ('__lt__', lambda self, other: not self >= other)] - } - roots = set(dir(cls)) & set(convert) - if not roots: - raise ValueError('must define at least one ordering operation: < > <= >=') - root = max(roots) # prefer __lt__ to __le__ to __gt__ to __ge__ - for opname, opfunc in convert[root]: - if opname not in roots: - opfunc.__name__ = opname - opfunc.__doc__ = getattr(int, opname).__doc__ - setattr(cls, opname, opfunc) - return cls diff --git a/lib/spack/external/functools_backport.py b/lib/spack/external/functools_backport.py new file mode 100644 index 0000000000..19f0903c82 --- /dev/null +++ b/lib/spack/external/functools_backport.py @@ -0,0 +1,30 @@ +# +# Backport of Python 2.7's total_ordering. +# + +def total_ordering(cls): + """Class decorator that fills in missing ordering methods""" + convert = { + '__lt__': [('__gt__', lambda self, other: not (self < other or self == other)), + ('__le__', lambda self, other: self < other or self == other), + ('__ge__', lambda self, other: not self < other)], + '__le__': [('__ge__', lambda self, other: not self <= other or self == other), + ('__lt__', lambda self, other: self <= other and not self == other), + ('__gt__', lambda self, other: not self <= other)], + '__gt__': [('__lt__', lambda self, other: not (self > other or self == other)), + ('__ge__', lambda self, other: self > other or self == other), + ('__le__', lambda self, other: not self > other)], + '__ge__': [('__le__', lambda self, other: (not self >= other) or self == other), + ('__gt__', lambda self, other: self >= other and not self == other), + ('__lt__', lambda self, other: not self >= other)] + } + roots = set(dir(cls)) & set(convert) + if not roots: + raise ValueError('must define at least one ordering operation: < > <= >=') + root = max(roots) # prefer __lt__ to __le__ to __gt__ to __ge__ + for opname, opfunc in convert[root]: + if opname not in roots: + opfunc.__name__ = opname + opfunc.__doc__ = getattr(int, opname).__doc__ + setattr(cls, opname, opfunc) + return cls diff --git a/lib/spack/external/ordereddict.py b/lib/spack/external/ordereddict.py deleted file mode 100644 index 8ddad1477e..0000000000 --- a/lib/spack/external/ordereddict.py +++ /dev/null @@ -1,262 +0,0 @@ -# -# Backport of OrderedDict() class that runs on Python 2.4, 2.5, 2.6, 2.7 and pypy. -# Passes Python2.7's test suite and incorporates all the latest updates. -# -# From http://code.activestate.com/recipes/576693-ordered-dictionary-for-py24/ -# This file is in the public domain, and has no particular license. -# -try: - from thread import get_ident as _get_ident -except ImportError: - from dummy_thread import get_ident as _get_ident - -try: - from _abcoll import KeysView, ValuesView, ItemsView -except ImportError: - pass - - -class OrderedDict(dict): - 'Dictionary that remembers insertion order' - # An inherited dict maps keys to values. - # The inherited dict provides __getitem__, __len__, __contains__, and get. - # The remaining methods are order-aware. - # Big-O running times for all methods are the same as for regular dictionaries. - - # The internal self.__map dictionary maps keys to links in a doubly linked list. - # The circular doubly linked list starts and ends with a sentinel element. - # The sentinel element never gets deleted (this simplifies the algorithm). - # Each link is stored as a list of length three: [PREV, NEXT, KEY]. - - def __init__(self, *args, **kwds): - '''Initialize an ordered dictionary. Signature is the same as for - regular dictionaries, but keyword arguments are not recommended - because their insertion order is arbitrary. - - ''' - if len(args) > 1: - raise TypeError('expected at most 1 arguments, got %d' % len(args)) - try: - self.__root - except AttributeError: - self.__root = root = [] # sentinel node - root[:] = [root, root, None] - self.__map = {} - self.__update(*args, **kwds) - - def __setitem__(self, key, value, dict_setitem=dict.__setitem__): - 'od.__setitem__(i, y) <==> od[i]=y' - # Setting a new item creates a new link which goes at the end of the linked - # list, and the inherited dictionary is updated with the new key/value pair. - if key not in self: - root = self.__root - last = root[0] - last[1] = root[0] = self.__map[key] = [last, root, key] - dict_setitem(self, key, value) - - def __delitem__(self, key, dict_delitem=dict.__delitem__): - 'od.__delitem__(y) <==> del od[y]' - # Deleting an existing item uses self.__map to find the link which is - # then removed by updating the links in the predecessor and successor nodes. - dict_delitem(self, key) - link_prev, link_next, key = self.__map.pop(key) - link_prev[1] = link_next - link_next[0] = link_prev - - def __iter__(self): - 'od.__iter__() <==> iter(od)' - root = self.__root - curr = root[1] - while curr is not root: - yield curr[2] - curr = curr[1] - - def __reversed__(self): - 'od.__reversed__() <==> reversed(od)' - root = self.__root - curr = root[0] - while curr is not root: - yield curr[2] - curr = curr[0] - - def clear(self): - 'od.clear() -> None. Remove all items from od.' - try: - for node in self.__map.itervalues(): - del node[:] - root = self.__root - root[:] = [root, root, None] - self.__map.clear() - except AttributeError: - pass - dict.clear(self) - - def popitem(self, last=True): - '''od.popitem() -> (k, v), return and remove a (key, value) pair. - Pairs are returned in LIFO order if last is true or FIFO order if false. - - ''' - if not self: - raise KeyError('dictionary is empty') - root = self.__root - if last: - link = root[0] - link_prev = link[0] - link_prev[1] = root - root[0] = link_prev - else: - link = root[1] - link_next = link[1] - root[1] = link_next - link_next[0] = root - key = link[2] - del self.__map[key] - value = dict.pop(self, key) - return key, value - - # -- the following methods do not depend on the internal structure -- - - def keys(self): - 'od.keys() -> list of keys in od' - return list(self) - - def values(self): - 'od.values() -> list of values in od' - return [self[key] for key in self] - - def items(self): - 'od.items() -> list of (key, value) pairs in od' - return [(key, self[key]) for key in self] - - def iterkeys(self): - 'od.iterkeys() -> an iterator over the keys in od' - return iter(self) - - def itervalues(self): - 'od.itervalues -> an iterator over the values in od' - for k in self: - yield self[k] - - def iteritems(self): - 'od.iteritems -> an iterator over the (key, value) items in od' - for k in self: - yield (k, self[k]) - - def update(*args, **kwds): - '''od.update(E, **F) -> None. Update od from dict/iterable E and F. - - If E is a dict instance, does: for k in E: od[k] = E[k] - If E has a .keys() method, does: for k in E.keys(): od[k] = E[k] - Or if E is an iterable of items, does: for k, v in E: od[k] = v - In either case, this is followed by: for k, v in F.items(): od[k] = v - - ''' - if len(args) > 2: - raise TypeError('update() takes at most 2 positional ' - 'arguments (%d given)' % (len(args),)) - elif not args: - raise TypeError('update() takes at least 1 argument (0 given)') - self = args[0] - # Make progressively weaker assumptions about "other" - other = () - if len(args) == 2: - other = args[1] - if isinstance(other, dict): - for key in other: - self[key] = other[key] - elif hasattr(other, 'keys'): - for key in other.keys(): - self[key] = other[key] - else: - for key, value in other: - self[key] = value - for key, value in kwds.items(): - self[key] = value - - __update = update # let subclasses override update without breaking __init__ - - __marker = object() - - def pop(self, key, default=__marker): - '''od.pop(k[,d]) -> v, remove specified key and return the corresponding value. - If key is not found, d is returned if given, otherwise KeyError is raised. - - ''' - if key in self: - result = self[key] - del self[key] - return result - if default is self.__marker: - raise KeyError(key) - return default - - def setdefault(self, key, default=None): - 'od.setdefault(k[,d]) -> od.get(k,d), also set od[k]=d if k not in od' - if key in self: - return self[key] - self[key] = default - return default - - def __repr__(self, _repr_running={}): - 'od.__repr__() <==> repr(od)' - call_key = id(self), _get_ident() - if call_key in _repr_running: - return '...' - _repr_running[call_key] = 1 - try: - if not self: - return '%s()' % (self.__class__.__name__,) - return '%s(%r)' % (self.__class__.__name__, self.items()) - finally: - del _repr_running[call_key] - - def __reduce__(self): - 'Return state information for pickling' - items = [[k, self[k]] for k in self] - inst_dict = vars(self).copy() - for k in vars(OrderedDict()): - inst_dict.pop(k, None) - if inst_dict: - return (self.__class__, (items,), inst_dict) - return self.__class__, (items,) - - def copy(self): - 'od.copy() -> a shallow copy of od' - return self.__class__(self) - - @classmethod - def fromkeys(cls, iterable, value=None): - '''OD.fromkeys(S[, v]) -> New ordered dictionary with keys from S - and values equal to v (which defaults to None). - - ''' - d = cls() - for key in iterable: - d[key] = value - return d - - def __eq__(self, other): - '''od.__eq__(y) <==> od==y. Comparison to another OD is order-sensitive - while comparison to a regular mapping is order-insensitive. - - ''' - if isinstance(other, OrderedDict): - return len(self)==len(other) and self.items() == other.items() - return dict.__eq__(self, other) - - def __ne__(self, other): - return not self == other - - # -- the following methods are only used in Python 2.7 -- - - def viewkeys(self): - "od.viewkeys() -> a set-like object providing a view on od's keys" - return KeysView(self) - - def viewvalues(self): - "od.viewvalues() -> an object providing a view on od's values" - return ValuesView(self) - - def viewitems(self): - "od.viewitems() -> a set-like object providing a view on od's items" - return ItemsView(self) diff --git a/lib/spack/external/ordereddict_backport.py b/lib/spack/external/ordereddict_backport.py new file mode 100644 index 0000000000..8ddad1477e --- /dev/null +++ b/lib/spack/external/ordereddict_backport.py @@ -0,0 +1,262 @@ +# +# Backport of OrderedDict() class that runs on Python 2.4, 2.5, 2.6, 2.7 and pypy. +# Passes Python2.7's test suite and incorporates all the latest updates. +# +# From http://code.activestate.com/recipes/576693-ordered-dictionary-for-py24/ +# This file is in the public domain, and has no particular license. +# +try: + from thread import get_ident as _get_ident +except ImportError: + from dummy_thread import get_ident as _get_ident + +try: + from _abcoll import KeysView, ValuesView, ItemsView +except ImportError: + pass + + +class OrderedDict(dict): + 'Dictionary that remembers insertion order' + # An inherited dict maps keys to values. + # The inherited dict provides __getitem__, __len__, __contains__, and get. + # The remaining methods are order-aware. + # Big-O running times for all methods are the same as for regular dictionaries. + + # The internal self.__map dictionary maps keys to links in a doubly linked list. + # The circular doubly linked list starts and ends with a sentinel element. + # The sentinel element never gets deleted (this simplifies the algorithm). + # Each link is stored as a list of length three: [PREV, NEXT, KEY]. + + def __init__(self, *args, **kwds): + '''Initialize an ordered dictionary. Signature is the same as for + regular dictionaries, but keyword arguments are not recommended + because their insertion order is arbitrary. + + ''' + if len(args) > 1: + raise TypeError('expected at most 1 arguments, got %d' % len(args)) + try: + self.__root + except AttributeError: + self.__root = root = [] # sentinel node + root[:] = [root, root, None] + self.__map = {} + self.__update(*args, **kwds) + + def __setitem__(self, key, value, dict_setitem=dict.__setitem__): + 'od.__setitem__(i, y) <==> od[i]=y' + # Setting a new item creates a new link which goes at the end of the linked + # list, and the inherited dictionary is updated with the new key/value pair. + if key not in self: + root = self.__root + last = root[0] + last[1] = root[0] = self.__map[key] = [last, root, key] + dict_setitem(self, key, value) + + def __delitem__(self, key, dict_delitem=dict.__delitem__): + 'od.__delitem__(y) <==> del od[y]' + # Deleting an existing item uses self.__map to find the link which is + # then removed by updating the links in the predecessor and successor nodes. + dict_delitem(self, key) + link_prev, link_next, key = self.__map.pop(key) + link_prev[1] = link_next + link_next[0] = link_prev + + def __iter__(self): + 'od.__iter__() <==> iter(od)' + root = self.__root + curr = root[1] + while curr is not root: + yield curr[2] + curr = curr[1] + + def __reversed__(self): + 'od.__reversed__() <==> reversed(od)' + root = self.__root + curr = root[0] + while curr is not root: + yield curr[2] + curr = curr[0] + + def clear(self): + 'od.clear() -> None. Remove all items from od.' + try: + for node in self.__map.itervalues(): + del node[:] + root = self.__root + root[:] = [root, root, None] + self.__map.clear() + except AttributeError: + pass + dict.clear(self) + + def popitem(self, last=True): + '''od.popitem() -> (k, v), return and remove a (key, value) pair. + Pairs are returned in LIFO order if last is true or FIFO order if false. + + ''' + if not self: + raise KeyError('dictionary is empty') + root = self.__root + if last: + link = root[0] + link_prev = link[0] + link_prev[1] = root + root[0] = link_prev + else: + link = root[1] + link_next = link[1] + root[1] = link_next + link_next[0] = root + key = link[2] + del self.__map[key] + value = dict.pop(self, key) + return key, value + + # -- the following methods do not depend on the internal structure -- + + def keys(self): + 'od.keys() -> list of keys in od' + return list(self) + + def values(self): + 'od.values() -> list of values in od' + return [self[key] for key in self] + + def items(self): + 'od.items() -> list of (key, value) pairs in od' + return [(key, self[key]) for key in self] + + def iterkeys(self): + 'od.iterkeys() -> an iterator over the keys in od' + return iter(self) + + def itervalues(self): + 'od.itervalues -> an iterator over the values in od' + for k in self: + yield self[k] + + def iteritems(self): + 'od.iteritems -> an iterator over the (key, value) items in od' + for k in self: + yield (k, self[k]) + + def update(*args, **kwds): + '''od.update(E, **F) -> None. Update od from dict/iterable E and F. + + If E is a dict instance, does: for k in E: od[k] = E[k] + If E has a .keys() method, does: for k in E.keys(): od[k] = E[k] + Or if E is an iterable of items, does: for k, v in E: od[k] = v + In either case, this is followed by: for k, v in F.items(): od[k] = v + + ''' + if len(args) > 2: + raise TypeError('update() takes at most 2 positional ' + 'arguments (%d given)' % (len(args),)) + elif not args: + raise TypeError('update() takes at least 1 argument (0 given)') + self = args[0] + # Make progressively weaker assumptions about "other" + other = () + if len(args) == 2: + other = args[1] + if isinstance(other, dict): + for key in other: + self[key] = other[key] + elif hasattr(other, 'keys'): + for key in other.keys(): + self[key] = other[key] + else: + for key, value in other: + self[key] = value + for key, value in kwds.items(): + self[key] = value + + __update = update # let subclasses override update without breaking __init__ + + __marker = object() + + def pop(self, key, default=__marker): + '''od.pop(k[,d]) -> v, remove specified key and return the corresponding value. + If key is not found, d is returned if given, otherwise KeyError is raised. + + ''' + if key in self: + result = self[key] + del self[key] + return result + if default is self.__marker: + raise KeyError(key) + return default + + def setdefault(self, key, default=None): + 'od.setdefault(k[,d]) -> od.get(k,d), also set od[k]=d if k not in od' + if key in self: + return self[key] + self[key] = default + return default + + def __repr__(self, _repr_running={}): + 'od.__repr__() <==> repr(od)' + call_key = id(self), _get_ident() + if call_key in _repr_running: + return '...' + _repr_running[call_key] = 1 + try: + if not self: + return '%s()' % (self.__class__.__name__,) + return '%s(%r)' % (self.__class__.__name__, self.items()) + finally: + del _repr_running[call_key] + + def __reduce__(self): + 'Return state information for pickling' + items = [[k, self[k]] for k in self] + inst_dict = vars(self).copy() + for k in vars(OrderedDict()): + inst_dict.pop(k, None) + if inst_dict: + return (self.__class__, (items,), inst_dict) + return self.__class__, (items,) + + def copy(self): + 'od.copy() -> a shallow copy of od' + return self.__class__(self) + + @classmethod + def fromkeys(cls, iterable, value=None): + '''OD.fromkeys(S[, v]) -> New ordered dictionary with keys from S + and values equal to v (which defaults to None). + + ''' + d = cls() + for key in iterable: + d[key] = value + return d + + def __eq__(self, other): + '''od.__eq__(y) <==> od==y. Comparison to another OD is order-sensitive + while comparison to a regular mapping is order-insensitive. + + ''' + if isinstance(other, OrderedDict): + return len(self)==len(other) and self.items() == other.items() + return dict.__eq__(self, other) + + def __ne__(self, other): + return not self == other + + # -- the following methods are only used in Python 2.7 -- + + def viewkeys(self): + "od.viewkeys() -> a set-like object providing a view on od's keys" + return KeysView(self) + + def viewvalues(self): + "od.viewvalues() -> an object providing a view on od's values" + return ValuesView(self) + + def viewitems(self): + "od.viewitems() -> a set-like object providing a view on od's items" + return ItemsView(self) diff --git a/lib/spack/spack/cmd/activate.py b/lib/spack/spack/cmd/activate.py index 1004f1f8e6..1e44948d24 100644 --- a/lib/spack/spack/cmd/activate.py +++ b/lib/spack/spack/cmd/activate.py @@ -22,7 +22,7 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -from external import argparse +import argparse import llnl.util.tty as tty import spack import spack.cmd diff --git a/lib/spack/spack/cmd/checksum.py b/lib/spack/spack/cmd/checksum.py index 8a448450c2..c940d57781 100644 --- a/lib/spack/spack/cmd/checksum.py +++ b/lib/spack/spack/cmd/checksum.py @@ -24,7 +24,7 @@ ############################################################################## import os import re -from external import argparse +import argparse import hashlib from pprint import pprint from subprocess import CalledProcessError diff --git a/lib/spack/spack/cmd/clean.py b/lib/spack/spack/cmd/clean.py index c20136ebe5..e303b3d634 100644 --- a/lib/spack/spack/cmd/clean.py +++ b/lib/spack/spack/cmd/clean.py @@ -22,7 +22,7 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -from external import argparse +import argparse import llnl.util.tty as tty diff --git a/lib/spack/spack/cmd/compiler.py b/lib/spack/spack/cmd/compiler.py index 2a64dc914e..3173d11070 100644 --- a/lib/spack/spack/cmd/compiler.py +++ b/lib/spack/spack/cmd/compiler.py @@ -22,7 +22,7 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -from external import argparse +import argparse import llnl.util.tty as tty from llnl.util.tty.color import colorize diff --git a/lib/spack/spack/cmd/config.py b/lib/spack/spack/cmd/config.py index 8c18f88b64..78972a8be0 100644 --- a/lib/spack/spack/cmd/config.py +++ b/lib/spack/spack/cmd/config.py @@ -23,7 +23,7 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import sys -from external import argparse +import argparse import llnl.util.tty as tty diff --git a/lib/spack/spack/cmd/create.py b/lib/spack/spack/cmd/create.py index 46e6bcec14..05cf170e39 100644 --- a/lib/spack/spack/cmd/create.py +++ b/lib/spack/spack/cmd/create.py @@ -27,7 +27,7 @@ import os import hashlib import re -from external.ordereddict import OrderedDict +from ordereddict_backport import OrderedDict import llnl.util.tty as tty from llnl.util.filesystem import mkdirp diff --git a/lib/spack/spack/cmd/deactivate.py b/lib/spack/spack/cmd/deactivate.py index 1f0e303cdf..5a2b353fa2 100644 --- a/lib/spack/spack/cmd/deactivate.py +++ b/lib/spack/spack/cmd/deactivate.py @@ -22,7 +22,7 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -from external import argparse +import argparse import llnl.util.tty as tty import spack diff --git a/lib/spack/spack/cmd/dependents.py b/lib/spack/spack/cmd/dependents.py index 652f243b98..129a4eeb23 100644 --- a/lib/spack/spack/cmd/dependents.py +++ b/lib/spack/spack/cmd/dependents.py @@ -22,7 +22,7 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -from external import argparse +import argparse import llnl.util.tty as tty diff --git a/lib/spack/spack/cmd/diy.py b/lib/spack/spack/cmd/diy.py index 9f8a6d39db..d6bd1fbb79 100644 --- a/lib/spack/spack/cmd/diy.py +++ b/lib/spack/spack/cmd/diy.py @@ -24,7 +24,7 @@ ############################################################################## import sys import os -from external import argparse +import argparse import llnl.util.tty as tty diff --git a/lib/spack/spack/cmd/env.py b/lib/spack/spack/cmd/env.py index bde76b5daf..ae8e95491e 100644 --- a/lib/spack/spack/cmd/env.py +++ b/lib/spack/spack/cmd/env.py @@ -23,7 +23,7 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import os -from external import argparse +import argparse import llnl.util.tty as tty import spack.cmd import spack.build_environment as build_env diff --git a/lib/spack/spack/cmd/extensions.py b/lib/spack/spack/cmd/extensions.py index 7cadc424b0..86dec7b9e7 100644 --- a/lib/spack/spack/cmd/extensions.py +++ b/lib/spack/spack/cmd/extensions.py @@ -23,7 +23,7 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import sys -from external import argparse +import argparse import llnl.util.tty as tty from llnl.util.tty.colify import colify diff --git a/lib/spack/spack/cmd/fetch.py b/lib/spack/spack/cmd/fetch.py index 0ccebd9486..1dd8703daf 100644 --- a/lib/spack/spack/cmd/fetch.py +++ b/lib/spack/spack/cmd/fetch.py @@ -22,7 +22,7 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -from external import argparse +import argparse import spack import spack.cmd diff --git a/lib/spack/spack/cmd/find.py b/lib/spack/spack/cmd/find.py index 0b0dd6ef6f..05fa620c59 100644 --- a/lib/spack/spack/cmd/find.py +++ b/lib/spack/spack/cmd/find.py @@ -25,7 +25,7 @@ import sys import collections import itertools -from external import argparse +import argparse from StringIO import StringIO import llnl.util.tty as tty diff --git a/lib/spack/spack/cmd/graph.py b/lib/spack/spack/cmd/graph.py index cb93a1b543..586a02c53b 100644 --- a/lib/spack/spack/cmd/graph.py +++ b/lib/spack/spack/cmd/graph.py @@ -22,7 +22,7 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -from external import argparse +import argparse import spack import spack.cmd diff --git a/lib/spack/spack/cmd/install.py b/lib/spack/spack/cmd/install.py index 836a6260c8..e4338e222f 100644 --- a/lib/spack/spack/cmd/install.py +++ b/lib/spack/spack/cmd/install.py @@ -22,7 +22,7 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -from external import argparse +import argparse import llnl.util.tty as tty diff --git a/lib/spack/spack/cmd/list.py b/lib/spack/spack/cmd/list.py index 1f0978a18e..0b55d9fb7b 100644 --- a/lib/spack/spack/cmd/list.py +++ b/lib/spack/spack/cmd/list.py @@ -24,7 +24,7 @@ ############################################################################## import sys import llnl.util.tty as tty -from external import argparse +import argparse from llnl.util.tty.colify import colify import spack diff --git a/lib/spack/spack/cmd/load.py b/lib/spack/spack/cmd/load.py index 06574d9725..5bc6b15784 100644 --- a/lib/spack/spack/cmd/load.py +++ b/lib/spack/spack/cmd/load.py @@ -22,7 +22,7 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -from external import argparse +import argparse import spack.modules description ="Add package to environment using modules." diff --git a/lib/spack/spack/cmd/location.py b/lib/spack/spack/cmd/location.py index e8e9c3f277..6fbab9782f 100644 --- a/lib/spack/spack/cmd/location.py +++ b/lib/spack/spack/cmd/location.py @@ -24,7 +24,7 @@ ############################################################################## import os import sys -from external import argparse +import argparse import llnl.util.tty as tty from llnl.util.filesystem import join_path diff --git a/lib/spack/spack/cmd/md5.py b/lib/spack/spack/cmd/md5.py index dfa1be412b..8be0a7ad4c 100644 --- a/lib/spack/spack/cmd/md5.py +++ b/lib/spack/spack/cmd/md5.py @@ -24,7 +24,7 @@ ############################################################################## import os import hashlib -from external import argparse +import argparse import llnl.util.tty as tty from llnl.util.filesystem import * diff --git a/lib/spack/spack/cmd/mirror.py b/lib/spack/spack/cmd/mirror.py index 2356170a9a..4599944f1c 100644 --- a/lib/spack/spack/cmd/mirror.py +++ b/lib/spack/spack/cmd/mirror.py @@ -26,7 +26,7 @@ import os import sys from datetime import datetime -from external import argparse +import argparse import llnl.util.tty as tty from llnl.util.tty.colify import colify diff --git a/lib/spack/spack/cmd/module.py b/lib/spack/spack/cmd/module.py index 654b0cb2fa..c3daed6402 100644 --- a/lib/spack/spack/cmd/module.py +++ b/lib/spack/spack/cmd/module.py @@ -25,7 +25,7 @@ import sys import os import shutil -from external import argparse +import argparse import llnl.util.tty as tty from llnl.util.lang import partition_list diff --git a/lib/spack/spack/cmd/patch.py b/lib/spack/spack/cmd/patch.py index a6556c4828..2356583b07 100644 --- a/lib/spack/spack/cmd/patch.py +++ b/lib/spack/spack/cmd/patch.py @@ -22,7 +22,7 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -from external import argparse +import argparse import spack.cmd import spack diff --git a/lib/spack/spack/cmd/pkg.py b/lib/spack/spack/cmd/pkg.py index 055b7c2062..ae5efd9d9c 100644 --- a/lib/spack/spack/cmd/pkg.py +++ b/lib/spack/spack/cmd/pkg.py @@ -24,7 +24,7 @@ ############################################################################## import os -from external import argparse +import argparse import llnl.util.tty as tty from llnl.util.tty.colify import colify diff --git a/lib/spack/spack/cmd/providers.py b/lib/spack/spack/cmd/providers.py index 2bcdc9fba2..1a652c82d1 100644 --- a/lib/spack/spack/cmd/providers.py +++ b/lib/spack/spack/cmd/providers.py @@ -23,7 +23,7 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import os -from external import argparse +import argparse from llnl.util.tty.colify import colify diff --git a/lib/spack/spack/cmd/python.py b/lib/spack/spack/cmd/python.py index 7bd2e45ce0..15c45654c2 100644 --- a/lib/spack/spack/cmd/python.py +++ b/lib/spack/spack/cmd/python.py @@ -25,7 +25,7 @@ import os import sys import code -from external import argparse +import argparse import platform import spack diff --git a/lib/spack/spack/cmd/reindex.py b/lib/spack/spack/cmd/reindex.py index b584729ea4..c0008930c4 100644 --- a/lib/spack/spack/cmd/reindex.py +++ b/lib/spack/spack/cmd/reindex.py @@ -22,7 +22,7 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -from external import argparse +import argparse import spack description = "Rebuild Spack's package database." diff --git a/lib/spack/spack/cmd/restage.py b/lib/spack/spack/cmd/restage.py index e735a12c32..9230cf5a1a 100644 --- a/lib/spack/spack/cmd/restage.py +++ b/lib/spack/spack/cmd/restage.py @@ -22,7 +22,7 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -from external import argparse +import argparse import llnl.util.tty as tty diff --git a/lib/spack/spack/cmd/spec.py b/lib/spack/spack/cmd/spec.py index 407519313c..af7ec1b36c 100644 --- a/lib/spack/spack/cmd/spec.py +++ b/lib/spack/spack/cmd/spec.py @@ -22,7 +22,7 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -from external import argparse +import argparse import spack.cmd import llnl.util.tty as tty diff --git a/lib/spack/spack/cmd/stage.py b/lib/spack/spack/cmd/stage.py index f3dc97be17..09cf0e1a1c 100644 --- a/lib/spack/spack/cmd/stage.py +++ b/lib/spack/spack/cmd/stage.py @@ -23,7 +23,7 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import os -from external import argparse +import argparse import llnl.util.tty as tty import spack diff --git a/lib/spack/spack/cmd/test-install.py b/lib/spack/spack/cmd/test-install.py index 68b761d5dc..ba0cb39baf 100644 --- a/lib/spack/spack/cmd/test-install.py +++ b/lib/spack/spack/cmd/test-install.py @@ -22,7 +22,7 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -from external import argparse +import argparse import xml.etree.ElementTree as ET import itertools import re diff --git a/lib/spack/spack/cmd/uninstall.py b/lib/spack/spack/cmd/uninstall.py index e80f2d2636..93575e005d 100644 --- a/lib/spack/spack/cmd/uninstall.py +++ b/lib/spack/spack/cmd/uninstall.py @@ -23,7 +23,7 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import sys -from external import argparse +import argparse import llnl.util.tty as tty from llnl.util.tty.colify import colify diff --git a/lib/spack/spack/cmd/unload.py b/lib/spack/spack/cmd/unload.py index 6442c48cb1..24e49b3f24 100644 --- a/lib/spack/spack/cmd/unload.py +++ b/lib/spack/spack/cmd/unload.py @@ -22,7 +22,7 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -from external import argparse +import argparse import spack.modules description ="Remove package from environment using module." diff --git a/lib/spack/spack/cmd/unuse.py b/lib/spack/spack/cmd/unuse.py index 2a7229a3a0..7f0b384ea0 100644 --- a/lib/spack/spack/cmd/unuse.py +++ b/lib/spack/spack/cmd/unuse.py @@ -22,7 +22,7 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -from external import argparse +import argparse import spack.modules description ="Remove package from environment using dotkit." diff --git a/lib/spack/spack/cmd/use.py b/lib/spack/spack/cmd/use.py index e34c194739..4990fea2f8 100644 --- a/lib/spack/spack/cmd/use.py +++ b/lib/spack/spack/cmd/use.py @@ -22,7 +22,7 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -from external import argparse +import argparse import spack.modules description ="Add package to environment using dotkit." diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 3e91958c2c..41afe8b232 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -90,12 +90,12 @@ import os import exceptions import sys -from external.ordereddict import OrderedDict +from ordereddict_backport import OrderedDict from llnl.util.lang import memoized import spack.error -from external import yaml -from external.yaml.error import MarkedYAMLError +import yaml +from yaml.error import MarkedYAMLError import llnl.util.tty as tty from llnl.util.filesystem import mkdirp diff --git a/lib/spack/spack/database.py b/lib/spack/spack/database.py index e0c14a0455..908ffc7fa4 100644 --- a/lib/spack/spack/database.py +++ b/lib/spack/spack/database.py @@ -43,8 +43,8 @@ import os import time import socket -from external import yaml -from external.yaml.error import MarkedYAMLError, YAMLError +import yaml +from yaml.error import MarkedYAMLError, YAMLError import llnl.util.tty as tty from llnl.util.filesystem import * diff --git a/lib/spack/spack/directory_layout.py b/lib/spack/spack/directory_layout.py index da8f4187cc..918405cab6 100644 --- a/lib/spack/spack/directory_layout.py +++ b/lib/spack/spack/directory_layout.py @@ -29,7 +29,7 @@ import hashlib import shutil import glob import tempfile -from external import yaml +import yaml import llnl.util.tty as tty from llnl.util.filesystem import join_path, mkdirp diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 7b79feb311..92880e9cbf 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -96,8 +96,8 @@ import hashlib import base64 from StringIO import StringIO from operator import attrgetter -from external import yaml -from external.yaml.error import MarkedYAMLError +import yaml +from yaml.error import MarkedYAMLError import llnl.util.tty as tty from llnl.util.lang import * diff --git a/lib/spack/spack/test/python_version.py b/lib/spack/spack/test/python_version.py index 5779d31ed2..ba7bab6f4b 100644 --- a/lib/spack/spack/test/python_version.py +++ b/lib/spack/spack/test/python_version.py @@ -34,7 +34,7 @@ import re import llnl.util.tty as tty -from external import pyqver2 +import pyqver2 import spack spack_max_version = (2,6) diff --git a/lib/spack/spack/version.py b/lib/spack/spack/version.py index 35db05e018..ffce2d1ff8 100644 --- a/lib/spack/spack/version.py +++ b/lib/spack/spack/version.py @@ -48,7 +48,7 @@ import sys import re from bisect import bisect_left from functools import wraps -from external.functools import total_ordering +from functools_backport import total_ordering # Valid version characters VALID_VERSION = r'[A-Za-z0-9_.-]' -- cgit v1.2.3-70-g09d2 From 9c30e0210b55a6f5bca048359135ae5a83039de5 Mon Sep 17 00:00:00 2001 From: Stephen Herbein Date: Thu, 12 Nov 2015 08:15:38 -0800 Subject: Fixed -l flag for 'spack extensions' --- lib/spack/spack/cmd/extensions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/cmd/extensions.py b/lib/spack/spack/cmd/extensions.py index 7cadc424b0..99be448288 100644 --- a/lib/spack/spack/cmd/extensions.py +++ b/lib/spack/spack/cmd/extensions.py @@ -37,7 +37,7 @@ description = "List extensions for package." def setup_parser(subparser): format_group = subparser.add_mutually_exclusive_group() format_group.add_argument( - '-l', '--long', action='store_const', dest='mode', const='long', + '-l', '--long', action='store_true', dest='long', help='Show dependency hashes as well as versions.') format_group.add_argument( '-p', '--paths', action='store_const', dest='mode', const='paths', @@ -95,4 +95,4 @@ def extensions(parser, args): tty.msg("None activated.") return tty.msg("%d currently activated:" % len(activated)) - spack.cmd.find.display_specs(activated.values(), mode=args.mode) + spack.cmd.find.display_specs(activated.values(), mode=args.mode, long=args.long) -- cgit v1.2.3-70-g09d2 From 72c9604bcb9f0f28d0e7d63be9500b99960d118b Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Thu, 12 Nov 2015 15:17:09 -0800 Subject: Small fix to prevent this test from interfering with others. --- lib/spack/spack/test/unit_install.py | 39 ++++++++++++++++++++---------------- 1 file changed, 22 insertions(+), 17 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/test/unit_install.py b/lib/spack/spack/test/unit_install.py index c4b9092f05..7168272997 100644 --- a/lib/spack/spack/test/unit_install.py +++ b/lib/spack/spack/test/unit_install.py @@ -26,16 +26,16 @@ import unittest import itertools import spack -test_install = __import__("spack.cmd.test-install", +test_install = __import__("spack.cmd.test-install", fromlist=["BuildId", "create_test_output", "TestResult"]) class MockOutput(object): def __init__(self): self.results = {} - + def add_test(self, buildId, passed=True, buildInfo=None): self.results[buildId] = passed - + def write_to(self, stream): pass @@ -45,14 +45,14 @@ class MockSpec(object): self.name = name self.version = version self.hash = hashStr if hashStr else hash((name, version)) - + def traverse(self, order=None): - allDeps = itertools.chain.from_iterable(i.traverse() for i in + allDeps = itertools.chain.from_iterable(i.traverse() for i in self.dependencies.itervalues()) return set(itertools.chain([self], allDeps)) - + def dag_hash(self): - return self.hash + return self.hash def to_yaml(self): return "<<>>".format(test_install.BuildId(self).stringId()) @@ -75,47 +75,52 @@ class UnitInstallTest(unittest.TestCase): def setUp(self): super(UnitInstallTest, self).setUp() - + pkgX.installed = False pkgY.installed = False + self.saved_db = spack.db pkgDb = MockPackageDb({specX:pkgX, specY:pkgY}) spack.db = pkgDb + def tearDown(self): super(UnitInstallTest, self).tearDown() - + + spack.db = self.saved_db + def test_installing_both(self): mo = MockOutput() - + pkgX.installed = True pkgY.installed = True test_install.create_test_output(specX, [specX, specY], mo, getLogFunc=test_fetch_log) - - self.assertEqual(mo.results, - {bIdX:test_install.TestResult.PASSED, + + self.assertEqual(mo.results, + {bIdX:test_install.TestResult.PASSED, bIdY:test_install.TestResult.PASSED}) + def test_dependency_already_installed(self): mo = MockOutput() - + pkgX.installed = True pkgY.installed = True test_install.create_test_output(specX, [specX], mo, getLogFunc=test_fetch_log) - + self.assertEqual(mo.results, {bIdX:test_install.TestResult.PASSED}) #TODO: add test(s) where Y fails to install + class MockPackageDb(object): def __init__(self, init=None): self.specToPkg = {} if init: self.specToPkg.update(init) - + def get(self, spec): return self.specToPkg[spec] def test_fetch_log(path): return [] - -- cgit v1.2.3-70-g09d2 From 38fdd063d9bc90aff7d934ab2bd2f02df71d9138 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Thu, 12 Nov 2015 15:17:39 -0800 Subject: Fix and move NamespaceTrie to spack.util.naming - fix up routines in namespace trie. - trie can now hold intermediate elements. - trie now has a test case. --- lib/spack/spack/packages.py | 41 +---------------- lib/spack/spack/test/__init__.py | 3 +- lib/spack/spack/test/namespace_trie.py | 83 ++++++++++++++++++++++++++++++++++ lib/spack/spack/util/naming.py | 69 +++++++++++++++++++++++++++- 4 files changed, 154 insertions(+), 42 deletions(-) create mode 100644 lib/spack/spack/test/namespace_trie.py (limited to 'lib') diff --git a/lib/spack/spack/packages.py b/lib/spack/spack/packages.py index 6005523bc0..3a74ad6790 100644 --- a/lib/spack/spack/packages.py +++ b/lib/spack/spack/packages.py @@ -58,45 +58,6 @@ def _autospec(function): return converter -class NamespaceTrie(object): - def __init__(self): - self._elements = {} - - - def __setitem__(self, namespace, repo): - parts = namespace.split('.') - cur = self._elements - for p in parts[:-1]: - if p not in cur: - cur[p] = {} - cur = cur[p] - - cur[parts[-1]] = repo - - - def __getitem__(self, namespace): - parts = namespace.split('.') - cur = self._elements - for p in parts: - if p not in cur: - raise KeyError("Can't find namespace %s in trie" % namespace) - cur = cur[p] - return cur - - - def __contains__(self, namespace): - parts = namespace.split('.') - cur = self._elements - for p in parts: - if not isinstance(cur, dict): - return False - if p not in cur: - return False - cur = cur[p] - return True - - - class PackageFinder(object): """A PackageFinder is a wrapper around a list of PackageDBs. @@ -172,7 +133,7 @@ class PackageFinder(object): def providers_for(self, vpkg_name): - # TODO: THIS IS WRONG; shoudl use more than first repo + # TODO: THIS IS WRONG; should use more than first repo return self.repos[0].providers_for(vpkg_name) diff --git a/lib/spack/spack/test/__init__.py b/lib/spack/spack/test/__init__.py index 0f776bfea4..620d1fd362 100644 --- a/lib/spack/spack/test/__init__.py +++ b/lib/spack/spack/test/__init__.py @@ -59,7 +59,8 @@ test_names = ['versions', 'configure_guess', 'unit_install', 'lock', - 'database'] + 'database', + 'namespace_trie'] def list_tests(): diff --git a/lib/spack/spack/test/namespace_trie.py b/lib/spack/spack/test/namespace_trie.py new file mode 100644 index 0000000000..191abbe9e6 --- /dev/null +++ b/lib/spack/spack/test/namespace_trie.py @@ -0,0 +1,83 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +import unittest +from spack.util.naming import NamespaceTrie + + +class NamespaceTrieTest(unittest.TestCase): + + def setUp(self): + self.trie = NamespaceTrie() + + + def test_add_single(self): + self.trie['foo'] = 'bar' + self.assertEqual(self.trie['foo'], 'bar') + self.assertTrue('foo' in self.trie) + + + def test_add_multiple(self): + self.trie['foo.bar'] = 'baz' + self.assertEqual(self.trie['foo.bar'], 'baz') + + self.assertFalse('foo' in self.trie) + self.assertFalse('foo.bar.baz' in self.trie) + self.assertTrue('foo.bar' in self.trie) + + + def test_add_three(self): + # add a three-level namespace + self.trie['foo.bar.baz'] = 'quux' + self.assertEqual(self.trie['foo.bar.baz'], 'quux') + + self.assertFalse('foo' in self.trie) + self.assertFalse('foo.bar' in self.trie) + self.assertTrue('foo.bar.baz' in self.trie) + + # Try to add a second element in a higher space + self.trie['foo.bar'] = 'blah' + + self.assertFalse('foo' in self.trie) + + self.assertTrue('foo.bar' in self.trie) + self.assertEqual(self.trie['foo.bar'], 'blah') + + self.assertTrue('foo.bar.baz' in self.trie) + self.assertEqual(self.trie['foo.bar.baz'], 'quux') + + + def test_add_none_single(self): + self.trie['foo'] = None + self.assertEqual(self.trie['foo'], None) + self.assertTrue('foo' in self.trie) + + + def test_add_none_multiple(self): + self.trie['foo.bar'] = None + self.assertEqual(self.trie['foo.bar'], None) + + self.assertFalse('foo' in self.trie) + self.assertFalse('foo.bar.baz' in self.trie) + self.assertTrue('foo.bar' in self.trie) diff --git a/lib/spack/spack/util/naming.py b/lib/spack/spack/util/naming.py index a7b6e2b436..475062bb38 100644 --- a/lib/spack/spack/util/naming.py +++ b/lib/spack/spack/util/naming.py @@ -3,11 +3,12 @@ from __future__ import absolute_import import string import itertools import re +from StringIO import StringIO import spack __all__ = ['mod_to_class', 'spack_module_to_python_module', 'valid_module_name', - 'validate_module_name', 'possible_spack_module_names'] + 'validate_module_name', 'possible_spack_module_names', 'NamespaceTrie'] # Valid module names can contain '-' but can't start with it. _valid_module_re = r'^\w[\w-]*$' @@ -90,3 +91,69 @@ class InvalidModuleNameError(spack.error.SpackError): super(InvalidModuleNameError, self).__init__( "Invalid module name: " + name) self.name = name + + +class NamespaceTrie(object): + class Element(object): + def __init__(self, value): + self.value = value + + + def __init__(self, separator='.'): + self._subspaces = {} + self._value = None + self._sep = separator + + + def __setitem__(self, namespace, value): + first, sep, rest = namespace.partition(self._sep) + + if not first: + self._value = NamespaceTrie.Element(value) + return + + if first not in self._subspaces: + self._subspaces[first] = NamespaceTrie() + + self._subspaces[first][rest] = value + + + def _get_helper(self, namespace, full_name): + first, sep, rest = namespace.partition(self._sep) + if not first: + if not self._value: + raise KeyError("Can't find namespace '%s' in trie" % full_name) + return self._value.value + elif first not in self._subspaces: + raise KeyError("Can't find namespace '%s' in trie" % full_name) + else: + return self._subspaces[first]._get_helper(rest, full_name) + + + def __getitem__(self, namespace): + return self._get_helper(namespace, namespace) + + + def __contains__(self, namespace): + first, sep, rest = namespace.partition(self._sep) + if not first: + return self._value is not None + elif first not in self._subspaces: + return False + else: + return rest in self._subspaces[first] + + + def _str_helper(self, stream, level=0): + indent = (level * ' ') + for name in sorted(self._subspaces): + stream.write(indent + name + '\n') + if self._value: + stream.write(indent + ' ' + repr(self._value.value)) + stream.write(self._subspaces[name]._str_helper(stream, level+1)) + + + def __str__(self): + stream = StringIO() + self._str_helper(stream) + return stream.getvalue() -- cgit v1.2.3-70-g09d2 From 5e93dd65929e97559acdb1227e5418b7896bb244 Mon Sep 17 00:00:00 2001 From: Tom Scogland Date: Fri, 20 Nov 2015 11:49:46 -0800 Subject: modified warning to reference spack clean clean --dist has been removed, command fixed to something that works. --- lib/spack/spack/fetch_strategy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/fetch_strategy.py b/lib/spack/spack/fetch_strategy.py index b810023c5a..5e6850d14b 100644 --- a/lib/spack/spack/fetch_strategy.py +++ b/lib/spack/spack/fetch_strategy.py @@ -190,7 +190,7 @@ class URLFetchStrategy(FetchStrategy): if content_types and 'text/html' in content_types[-1]: tty.warn("The contents of " + self.archive_file + " look like HTML.", "The checksum will likely be bad. If it is, you can use", - "'spack clean --dist' to remove the bad archive, then fix", + "'spack clean ' to remove the bad archive, then fix", "your internet gateway issue and install again.") if not self.archive_file: -- cgit v1.2.3-70-g09d2 From 8c06b92225699fdd85185688b88d559a54c8a794 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 22 Nov 2015 20:28:59 -0800 Subject: Working PackageFinder / Repo --- .mailmap | 2 + lib/spack/spack/__init__.py | 6 - lib/spack/spack/packages.py | 375 ++++++++++++++++++++----------- lib/spack/spack/patch.py | 2 +- lib/spack/spack/test/config.py | 1 - lib/spack/spack/test/namespace_trie.py | 61 +++-- lib/spack/spack/test/package_sanity.py | 5 +- lib/spack/spack/test/packages.py | 6 +- lib/spack/spack/test/url_substitution.py | 1 - lib/spack/spack/util/naming.py | 32 ++- 10 files changed, 331 insertions(+), 160 deletions(-) (limited to 'lib') diff --git a/.mailmap b/.mailmap index 1cc13c1eb1..1b99da32b5 100644 --- a/.mailmap +++ b/.mailmap @@ -9,3 +9,5 @@ Saravan Pantham Saravan Pantham Tom Scogland Tom Scogland Tom Scogland Joachim Protze jprotze +Gregory L. Lee Gregory L. Lee +Gregory L. Lee Gregory Lee diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index eccec12d3b..549d0a9a0f 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -51,12 +51,6 @@ opt_path = join_path(prefix, "opt") install_path = join_path(opt_path, "spack") etc_path = join_path(prefix, "etc") -# -# Setup the spack.repos namespace -# -from spack.repo_loader import RepoNamespace -repos = RepoNamespace() - # # Set up the default packages database. # diff --git a/lib/spack/spack/packages.py b/lib/spack/spack/packages.py index 3a74ad6790..8114b7f1aa 100644 --- a/lib/spack/spack/packages.py +++ b/lib/spack/spack/packages.py @@ -31,6 +31,7 @@ import imp import re import itertools import traceback +from bisect import bisect_left from external import yaml import llnl.util.tty as tty @@ -58,11 +59,19 @@ def _autospec(function): return converter +def _make_namespace_module(ns): + module = imp.new_module(ns) + module.__file__ = "(spack namespace)" + module.__path__ = [] + module.__package__ = ns + return module + + class PackageFinder(object): - """A PackageFinder is a wrapper around a list of PackageDBs. + """A PackageFinder is a wrapper around a list of Repos. - It functions exactly like a PackageDB, but it operates on the - combined results of the PackageDBs in its list instead of on a + It functions exactly like a Repo, but it operates on the + combined results of the Repos in its list instead of on a single package repository. """ def __init__(self, *repo_dirs): @@ -70,61 +79,74 @@ class PackageFinder(object): self.by_namespace = NamespaceTrie() self.by_path = {} + self._all_package_names = [] + self._provider_index = None + for root in repo_dirs: - repo = PackageDB(root) + repo = Repo(root) self.put_last(repo) - def _check_repo(self, repo): + def swap(self, other): + """Convenience function to make swapping repostiories easier. + + This is currently used by mock tests. + TODO: Maybe there is a cleaner way. + + """ + attrs = ['repos', 'by_namespace', 'by_path', '_all_package_names', '_provider_index'] + for attr in attrs: + tmp = getattr(self, attr) + setattr(self, attr, getattr(other, attr)) + setattr(other, attr, tmp) + + + def _add(self, repo): + """Add a repository to the namespace and path indexes. + + Checks for duplicates -- two repos can't have the same root + directory, and they provide have the same namespace. + + """ if repo.root in self.by_path: raise DuplicateRepoError("Package repos are the same", repo, self.by_path[repo.root]) if repo.namespace in self.by_namespace: - tty.error("Package repos cannot have the same name", - repo, self.by_namespace[repo.namespace]) + raise DuplicateRepoError("Package repos cannot have the same name", + repo, self.by_namespace[repo.namespace]) - - def _add(self, repo): - self._check_repo(repo) + # Add repo to the pkg indexes self.by_namespace[repo.namespace] = repo self.by_path[repo.root] = repo + # add names to the cached name list + new_pkgs = set(repo.all_package_names()) + new_pkgs.update(set(self._all_package_names)) + self._all_package_names = sorted(new_pkgs, key=lambda n:n.lower()) + def put_first(self, repo): + """Add repo first in the search path.""" self._add(repo) self.repos.insert(0, repo) def put_last(self, repo): + """Add repo last in the search path.""" self._add(repo) self.repos.append(repo) def remove(self, repo): + """Remove a repo from the search path.""" if repo in self.repos: self.repos.remove(repo) - def swap(self, other): - repos = self.repos - by_namespace = self.by_namespace - by_path = self.by_path - - self.repos = other.repos - self.by_namespace = other.by_namespace - self.by_pah = other.by_path - - other.repos = repos - other.by_namespace = by_namespace - other.by_path = by_path - - def all_package_names(self): - all_pkgs = set() - for repo in self.repos: - all_pkgs.update(set(repo.all_package_names())) - return all_pkgs + """Return all unique package names in all repositories.""" + return self._all_package_names def all_packages(self): @@ -132,109 +154,82 @@ class PackageFinder(object): yield self.get(name) - def providers_for(self, vpkg_name): - # TODO: THIS IS WRONG; should use more than first repo - return self.repos[0].providers_for(vpkg_name) - + @_autospec + def providers_for(self, vpkg_spec): + if self._provider_index is None: + self._provider_index = ProviderIndex(self.all_package_names()) - def _get_spack_pkg_name(self, repo, py_module_name): - """Allow users to import Spack packages using legal Python identifiers. + providers = self._provider_index.providers_for(vpkg_spec) + if not providers: + raise UnknownPackageError(vpkg_spec.name) + return providers - A python identifier might map to many different Spack package - names due to hyphen/underscore ambiguity. - Easy example: - num3proxy -> 3proxy + def find_module(self, fullname, path=None): + """Implements precedence for overlaid namespaces. - Ambiguous: - foo_bar -> foo_bar, foo-bar + Loop checks each namespace in self.repos for packages, and + also handles loading empty containing namespaces. - More ambiguous: - foo_bar_baz -> foo_bar_baz, foo-bar-baz, foo_bar-baz, foo-bar_baz """ - if py_module_name in repo: - return py_module_name - - options = possible_spack_module_names(py_module_name) - options.remove(py_module_name) - for name in options: - if name in repo: - return name - - return None + # namespaces are added to repo, and package modules are leaves. + namespace, dot, module_name = fullname.rpartition('.') + # If it's a module in some repo, or if it is the repo's + # namespace, let the repo handle it. + for repo in self.repos: + if namespace == repo.namespace: + if repo.real_name(module_name): + return repo + elif fullname == repo.namespace: + return repo - def find_module(self, fullname, path=None): - if fullname in self.by_namespace: + # No repo provides the namespace, but it is a valid prefix of + # something in the PackageFinder. + if self.by_namespace.is_prefix(fullname): return self - namespace, dot, module_name = fullname.rpartition('.') - if namespace not in self.by_namespace: - return None - - repo = self.by_namespace[namespace] - name = self._get_spack_pkg_name(repo, module_name) - if not name: - return None - - return self + return None def load_module(self, fullname): + """Loads containing namespaces when necessary. + + See ``Repo`` for how actual package modules are loaded. + """ if fullname in sys.modules: return sys.modules[fullname] - if fullname in self.by_namespace: - ns = self.by_namespace[fullname] - module = imp.new_module(fullname) - module.__file__ = "" - module.__path__ = [] - module.__package__ = fullname + # partition fullname into prefix and module name. + namespace, dot, module_name = fullname.rpartition('.') - else: - namespace, dot, module_name = fullname.rpartition('.') - if namespace not in self.by_namespace: - raise ImportError( - "No Spack repository with namespace %s" % namespace) - - repo = self.by_namespace[namespace] - name = self._get_spack_pkg_name(repo, module_name) - if not name: - raise ImportError( - "No module %s in Spack repository %s" % (module_name, repo)) - - fullname = namespace + '.' + name - file_path = os.path.join(repo.root, name, package_file_name) - module = imp.load_source(fullname, file_path) - module.__package__ = namespace + if not self.by_namespace.is_prefix(fullname): + raise ImportError("No such Spack repo: %s" % fullname) + module = _make_namespace_module(namespace) module.__loader__ = self sys.modules[fullname] = module return module - def _find_repo_for_spec(self, spec): - """Find a repo that contains the supplied spec's package. - - Raises UnknownPackageErrorfor if not found. - """ + def repo_for_pkg(self, pkg_name): for repo in self.repos: - if spec.name in repo: + if pkg_name in repo: return repo - raise UnknownPackageError(spec.name) + raise UnknownPackageError(pkg_name) @_autospec def get(self, spec, new=False): - return self._find_repo_for_spec(spec).get(spec, new) + """Find a repo that contains the supplied spec's package. + Raises UnknownPackageError if not found. + """ + return self.repo_for_pkg(spec.name).get(spec) - def get_repo(self, namespace): - if namespace in self.by_namespace: - repo = self.by_namespace[namespace] - if isinstance(repo, PackageDB): - return repo - return None + + def dirname_for_package_name(self, pkg_name): + return self.repo_for_pkg(pkg_name).dirname_for_package_name(pkg_name) def exists(self, pkg_name): @@ -246,7 +241,7 @@ class PackageFinder(object): -class PackageDB(object): +class Repo(object): """Class representing a package repository in the filesystem. Each package repository must have a top-level configuration file @@ -278,10 +273,107 @@ class PackageDB(object): tty.die(("Invalid namespace '%s' in '%s'. Namespaces must be " "valid python identifiers separated by '.'") % (self.namespace, self.root)) + self._names = self.namespace.split('.') # These are internal cache variables. + self._modules = {} + self._classes = {} self._instances = {} + self._provider_index = None + self._all_package_names = None + + # make sure the namespace for packages in this repo exists. + self._create_namespace() + + + def _create_namespace(self): + """Create this repo's namespace module and insert it into sys.modules. + + Ensures that modules loaded via the repo have a home, and that + we don't get runtime warnings from Python's module system. + + """ + for l in range(1, len(self._names)+1): + ns = '.'.join(self._names[:l]) + if not ns in sys.modules: + sys.modules[ns] = _make_namespace_module(ns) + sys.modules[ns].__loader__ = self + + + def real_name(self, import_name): + """Allow users to import Spack packages using Python identifiers. + + A python identifier might map to many different Spack package + names due to hyphen/underscore ambiguity. + + Easy example: + num3proxy -> 3proxy + + Ambiguous: + foo_bar -> foo_bar, foo-bar + + More ambiguous: + foo_bar_baz -> foo_bar_baz, foo-bar-baz, foo_bar-baz, foo-bar_baz + """ + if import_name in self: + return import_name + + options = possible_spack_module_names(import_name) + options.remove(import_name) + for name in options: + if name in self: + return name + return None + + + def is_prefix(self, fullname): + """True if fullname is a prefix of this Repo's namespace.""" + parts = fullname.split('.') + return self._names[:len(parts)] == parts + + + def find_module(self, fullname, path=None): + """Python find_module import hook. + + Returns this Repo if it can load the module; None if not. + """ + if self.is_prefix(fullname): + return self + + namespace, dot, module_name = fullname.rpartition('.') + if namespace == self.namespace: + if self.real_name(module_name): + return self + + return None + + + def load_module(self, fullname): + """Python importer load hook. + + Tries to load the module; raises an ImportError if it can't. + """ + if fullname in sys.modules: + return sys.modules[fullname] + + namespace, dot, module_name = fullname.rpartition('.') + + if self.is_prefix(fullname): + module = _make_namespace_module(fullname) + + elif namespace == self.namespace: + real_name = self.real_name(module_name) + if not real_name: + raise ImportError("No module %s in repo %s" % (module_name, namespace)) + module = self._get_pkg_module(real_name) + + else: + raise ImportError("No module %s in repo %s" % (fullname, self.namespace)) + + module.__loader__ = self + sys.modules[fullname] = module + return module def _read_config(self): @@ -307,15 +399,14 @@ class PackageDB(object): if spec.virtual: raise UnknownPackageError(spec.name) - if new: - if spec in self._instances: - del self._instances[spec] + if new and spec in self._instances: + del self._instances[spec] if not spec in self._instances: - package_class = self.get_class_for_package_name(spec.name, spec.namespace) + PackageClass = self._get_pkg_class(spec.name) try: copy = spec.copy() - self._instances[copy] = package_class(copy) + self._instances[copy] = PackageClass(copy) except Exception, e: if spack.debug: sys.excepthook(*sys.exc_info()) @@ -353,7 +444,7 @@ class PackageDB(object): def filename_for_package_name(self, pkg_name): """Get the filename for the module we should load for a particular - package. Packages for a pacakge DB live in + package. Packages for a Repo live in ``$root//package.py`` This will return a proper package.py path even if the @@ -366,19 +457,20 @@ class PackageDB(object): return join_path(pkg_dir, package_file_name) - @memoized def all_package_names(self): - """Generator function for all packages. This looks for - ``/package.py`` files within the repo direcotories""" - all_package_names = [] + """Returns a sorted list of all package names in the Repo.""" + if self._all_package_names is None: + self._all_package_names = [] + + for pkg_name in os.listdir(self.root): + pkg_dir = join_path(self.root, pkg_name) + pkg_file = join_path(pkg_dir, package_file_name) + if os.path.isfile(pkg_file): + self._all_package_names.append(pkg_name) - for pkg_name in os.listdir(self.root): - pkg_dir = join_path(self.root, pkg_name) - pkg_file = join_path(pkg_dir, package_file_name) - if os.path.isfile(pkg_file): - all_package_names.append(pkg_name) + self._all_package_names.sort() - return sorted(all_package_names) + return self._all_package_names def all_packages(self): @@ -386,27 +478,54 @@ class PackageDB(object): yield self.get(name) - @memoized def exists(self, pkg_name): """Whether a package with the supplied name exists.""" - return os.path.exists(self.filename_for_package_name(pkg_name)) + # This does a binary search in the sorted list. + idx = bisect_left(self.all_package_names(), pkg_name) + return self._all_package_names[idx] == pkg_name - @memoized - def get_class_for_package_name(self, pkg_name, reponame = None): - """Get an instance of the class for a particular package.""" - file_path = self.filename_for_package_name(pkg_name) + def _get_pkg_module(self, pkg_name): + """Create a module for a particular package. + + This caches the module within this Repo *instance*. It does + *not* add it to ``sys.modules``. So, you can construct + multiple Repos for testing and ensure that the module will be + loaded once per repo. + + """ + if pkg_name not in self._modules: + file_path = self.filename_for_package_name(pkg_name) + + if not os.path.exists(file_path): + raise UnknownPackageError(pkg_name, self.namespace) - if os.path.exists(file_path): if not os.path.isfile(file_path): tty.die("Something's wrong. '%s' is not a file!" % file_path) + if not os.access(file_path, os.R_OK): tty.die("Cannot read '%s'!" % file_path) - else: - raise UnknownPackageError(pkg_name, self.namespace) + fullname = "%s.%s" % (self.namespace, pkg_name) + + module = imp.load_source(fullname, file_path) + module.__package__ = self.namespace + module.__loader__ = self + self._modules[pkg_name] = module + + return self._modules[pkg_name] + + + def _get_pkg_class(self, pkg_name): + """Get the class for the package out of its module. + + First loads (or fetches from cache) a module for the + package. Then extracts the package class from the module + according to Spack's naming convention. + """ class_name = mod_to_class(pkg_name) - module = __import__(self.namespace + '.' + pkg_name, fromlist=[class_name]) + module = self._get_pkg_module(pkg_name) + cls = getattr(module, class_name) if not inspect.isclass(cls): tty.die("%s.%s is not a class" % (pkg_name, class_name)) @@ -415,7 +534,7 @@ class PackageDB(object): def __str__(self): - return "" % (self.namespace, self.root) + return "" % (self.namespace, self.root) def __repr__(self): diff --git a/lib/spack/spack/patch.py b/lib/spack/spack/patch.py index da5fa1646b..fe6e0a65a3 100644 --- a/lib/spack/spack/patch.py +++ b/lib/spack/spack/patch.py @@ -54,7 +54,7 @@ class Patch(object): if '://' in path_or_url: self.url = path_or_url else: - pkg_dir = spack.db.dirname_for_package_name(pkg_name) + pkg_dir = spack.db.dirname_for_package_name(self.pkg_name) self.path = join_path(pkg_dir, path_or_url) if not os.path.isfile(self.path): raise NoSuchPatchFileError(pkg_name, self.path) diff --git a/lib/spack/spack/test/config.py b/lib/spack/spack/test/config.py index eed182a257..b1195dfe4e 100644 --- a/lib/spack/spack/test/config.py +++ b/lib/spack/spack/test/config.py @@ -27,7 +27,6 @@ import shutil import os from tempfile import mkdtemp import spack -from spack.packages import PackageDB from spack.test.mock_packages_test import * # Some sample compiler config data diff --git a/lib/spack/spack/test/namespace_trie.py b/lib/spack/spack/test/namespace_trie.py index 191abbe9e6..2fa13688e6 100644 --- a/lib/spack/spack/test/namespace_trie.py +++ b/lib/spack/spack/test/namespace_trie.py @@ -34,50 +34,81 @@ class NamespaceTrieTest(unittest.TestCase): def test_add_single(self): self.trie['foo'] = 'bar' + + self.assertTrue(self.trie.is_prefix('foo')) + self.assertTrue(self.trie.has_value('foo')) self.assertEqual(self.trie['foo'], 'bar') - self.assertTrue('foo' in self.trie) def test_add_multiple(self): self.trie['foo.bar'] = 'baz' + + self.assertFalse(self.trie.has_value('foo')) + self.assertTrue(self.trie.is_prefix('foo')) + + self.assertTrue(self.trie.is_prefix('foo.bar')) + self.assertTrue(self.trie.has_value('foo.bar')) self.assertEqual(self.trie['foo.bar'], 'baz') - self.assertFalse('foo' in self.trie) - self.assertFalse('foo.bar.baz' in self.trie) - self.assertTrue('foo.bar' in self.trie) + self.assertFalse(self.trie.is_prefix('foo.bar.baz')) + self.assertFalse(self.trie.has_value('foo.bar.baz')) def test_add_three(self): # add a three-level namespace self.trie['foo.bar.baz'] = 'quux' + + self.assertTrue(self.trie.is_prefix('foo')) + self.assertFalse(self.trie.has_value('foo')) + + self.assertTrue(self.trie.is_prefix('foo.bar')) + self.assertFalse(self.trie.has_value('foo.bar')) + + self.assertTrue(self.trie.is_prefix('foo.bar.baz')) + self.assertTrue(self.trie.has_value('foo.bar.baz')) self.assertEqual(self.trie['foo.bar.baz'], 'quux') - self.assertFalse('foo' in self.trie) - self.assertFalse('foo.bar' in self.trie) - self.assertTrue('foo.bar.baz' in self.trie) + self.assertFalse(self.trie.is_prefix('foo.bar.baz.quux')) + self.assertFalse(self.trie.has_value('foo.bar.baz.quux')) - # Try to add a second element in a higher space + # Try to add a second element in a prefix namespace self.trie['foo.bar'] = 'blah' - self.assertFalse('foo' in self.trie) + self.assertTrue(self.trie.is_prefix('foo')) + self.assertFalse(self.trie.has_value('foo')) - self.assertTrue('foo.bar' in self.trie) + self.assertTrue(self.trie.is_prefix('foo.bar')) + self.assertTrue(self.trie.has_value('foo.bar')) self.assertEqual(self.trie['foo.bar'], 'blah') - self.assertTrue('foo.bar.baz' in self.trie) + self.assertTrue(self.trie.is_prefix('foo.bar.baz')) + self.assertTrue(self.trie.has_value('foo.bar.baz')) self.assertEqual(self.trie['foo.bar.baz'], 'quux') + self.assertFalse(self.trie.is_prefix('foo.bar.baz.quux')) + self.assertFalse(self.trie.has_value('foo.bar.baz.quux')) + def test_add_none_single(self): self.trie['foo'] = None + self.assertTrue(self.trie.is_prefix('foo')) + self.assertTrue(self.trie.has_value('foo')) self.assertEqual(self.trie['foo'], None) - self.assertTrue('foo' in self.trie) + + self.assertFalse(self.trie.is_prefix('foo.bar')) + self.assertFalse(self.trie.has_value('foo.bar')) + def test_add_none_multiple(self): self.trie['foo.bar'] = None + + self.assertTrue(self.trie.is_prefix('foo')) + self.assertFalse(self.trie.has_value('foo')) + + self.assertTrue(self.trie.is_prefix('foo.bar')) + self.assertTrue(self.trie.has_value('foo.bar')) self.assertEqual(self.trie['foo.bar'], None) - self.assertFalse('foo' in self.trie) - self.assertFalse('foo.bar.baz' in self.trie) - self.assertTrue('foo.bar' in self.trie) + self.assertFalse(self.trie.is_prefix('foo.bar.baz')) + self.assertFalse(self.trie.has_value('foo.bar.baz')) diff --git a/lib/spack/spack/test/package_sanity.py b/lib/spack/spack/test/package_sanity.py index a5925ea066..a398568244 100644 --- a/lib/spack/spack/test/package_sanity.py +++ b/lib/spack/spack/test/package_sanity.py @@ -28,7 +28,6 @@ This test does sanity checks on Spack's builtin package database. import unittest import spack -import spack.url as url from spack.packages import PackageFinder @@ -45,7 +44,7 @@ class PackageSanityTest(unittest.TestCase): self.check_db() - def test_get_all_mock_packages(self): + def ztest_get_all_mock_packages(self): """Get the mock packages once each too.""" db = PackageFinder(spack.mock_packages_path) spack.db.swap(db) @@ -53,7 +52,7 @@ class PackageSanityTest(unittest.TestCase): spack.db.swap(db) - def test_url_versions(self): + def ztest_url_versions(self): """Check URLs for regular packages, if they are explicitly defined.""" for pkg in spack.db.all_packages(): for v, vattrs in pkg.versions.items(): diff --git a/lib/spack/spack/test/packages.py b/lib/spack/spack/test/packages.py index 42bd91ec5c..2d19d9ddc7 100644 --- a/lib/spack/spack/test/packages.py +++ b/lib/spack/spack/test/packages.py @@ -27,7 +27,7 @@ import unittest from llnl.util.filesystem import join_path import spack -import spack.packages as packages +from spack.packages import Repo from spack.util.naming import mod_to_class from spack.test.mock_packages_test import * @@ -44,7 +44,7 @@ class PackagesTest(MockPackagesTest): def test_package_filename(self): - repo = spack.db.get_repo('gov.llnl.spack.mock') + repo = Repo(spack.mock_packages_path) filename = repo.filename_for_package_name('mpich') self.assertEqual(filename, join_path(spack.mock_packages_path, 'mpich', 'package.py')) @@ -55,7 +55,7 @@ class PackagesTest(MockPackagesTest): def test_nonexisting_package_filename(self): - repo = spack.db.get_repo('gov.llnl.spack.mock') + repo = Repo(spack.mock_packages_path) filename = repo.filename_for_package_name('some-nonexisting-package') self.assertEqual(filename, join_path(spack.mock_packages_path, 'some-nonexisting-package', 'package.py')) diff --git a/lib/spack/spack/test/url_substitution.py b/lib/spack/spack/test/url_substitution.py index db7ddd251d..3ff76f63be 100644 --- a/lib/spack/spack/test/url_substitution.py +++ b/lib/spack/spack/test/url_substitution.py @@ -29,7 +29,6 @@ import unittest import spack import spack.url as url -from spack.packages import PackageDB class PackageSanityTest(unittest.TestCase): diff --git a/lib/spack/spack/util/naming.py b/lib/spack/spack/util/naming.py index 475062bb38..26ca86c77f 100644 --- a/lib/spack/spack/util/naming.py +++ b/lib/spack/spack/util/naming.py @@ -134,14 +134,42 @@ class NamespaceTrie(object): return self._get_helper(namespace, namespace) - def __contains__(self, namespace): + def is_prefix(self, namespace): + """True if the namespace has a value, or if it's the prefix of one that does.""" + first, sep, rest = namespace.partition(self._sep) + if not first: + return True + elif first not in self._subspaces: + return False + else: + return self._subspaces[first].is_prefix(rest) + + + def is_leaf(self, namespace): + """True if this namespace has no children in the trie.""" + first, sep, rest = namespace.partition(self._sep) + if not first: + return bool(self._subspaces) + elif first not in self._subspaces: + return False + else: + return self._subspaces[first].is_leaf(rest) + + + def has_value(self, namespace): + """True if there is a value set for the given namespace.""" first, sep, rest = namespace.partition(self._sep) if not first: return self._value is not None elif first not in self._subspaces: return False else: - return rest in self._subspaces[first] + return self._subspaces[first].has_value(rest) + + + def __contains__(self, namespace): + """Returns whether a value has been set for the namespace.""" + return self.has_value(namespace) def _str_helper(self, stream, level=0): -- cgit v1.2.3-70-g09d2 From 099fa1df34a157a323aa023b69c52399a441b81b Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 12 Nov 2015 13:23:19 -0800 Subject: Use nose to run unit tests. 1. Adding a plugin to keep track of the total number of tests run as well as the number of tests with failures/errors. 2. Some nose plugins (including xunit which will be added in a future commit) assign stdout to a stream object that does not have a .fileno attribute. spack.util.executable.Executable now avoids passing stdout to subprocess (and always uses subprocess.PIPE) TODO: 1. Still need to figure out how to activate the plugin (as of now it is being ignored by nose). Newer versions of nose appear to make this simpler (e.g. the "addplugins" argument to nose.run) 2. Need to include new version of nose in order to use xunit --- lib/spack/spack/test/__init__.py | 27 ++++++------- lib/spack/spack/test/tally_plugin.py | 73 ++++++++++++++++++++++++++++++++++++ lib/spack/spack/test/unit_install.py | 6 +-- lib/spack/spack/util/executable.py | 7 +++- 4 files changed, 93 insertions(+), 20 deletions(-) create mode 100644 lib/spack/spack/test/tally_plugin.py (limited to 'lib') diff --git a/lib/spack/spack/test/__init__.py b/lib/spack/spack/test/__init__.py index 0f776bfea4..9d577d6539 100644 --- a/lib/spack/spack/test/__init__.py +++ b/lib/spack/spack/test/__init__.py @@ -24,7 +24,9 @@ ############################################################################## import sys import unittest +import nose +from spack.test.tally_plugin import Tally import llnl.util.tty as tty from llnl.util.tty.colify import colify @@ -81,28 +83,23 @@ def run(names, verbose=False): "Valid names are:") colify(sorted(test_names), indent=4) sys.exit(1) - - runner = unittest.TextTestRunner(verbosity=verbosity) - - testsRun = errors = failures = 0 + + tally = Tally() + tally.enabled = True for test in names: module = 'spack.test.' + test print module - suite = unittest.defaultTestLoader.loadTestsFromName(module) - + tty.msg("Running test: %s" % test) - result = runner.run(suite) - testsRun += result.testsRun - errors += len(result.errors) - failures += len(result.failures) + result = nose.run(argv=["", module], plugins=[tally]) - succeeded = not errors and not failures + succeeded = not tally.failCount and not tally.errorCount tty.msg("Tests Complete.", - "%5d tests run" % testsRun, - "%5d failures" % failures, - "%5d errors" % errors) + "%5d tests run" % tally.numberOfTests, + "%5d failures" % tally.failCount, + "%5d errors" % tally.errorCount) - if not errors and not failures: + if succeeded: tty.info("OK", format='g') else: tty.info("FAIL", format='r') diff --git a/lib/spack/spack/test/tally_plugin.py b/lib/spack/spack/test/tally_plugin.py new file mode 100644 index 0000000000..d8638bf7e3 --- /dev/null +++ b/lib/spack/spack/test/tally_plugin.py @@ -0,0 +1,73 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from nose.plugins import Plugin + +import os + +class Tally(Plugin): + name = 'tally' + + def __init__(self): + super(Tally, self).__init__() + self.successes = set() + self.failures = set() + self.errors = set() + + @property + def successCount(self): + return len(self.successes) + + @property + def failCount(self): + return len(self.failures) + + @property + def errorCount(self): + return len(self.errors) + + @property + def numberOfTests(self): + return self.errorCount + self.failCount + self.successCount + + def options(self, parser, env=os.environ): + super(Tally, self).options(parser, env=env) + + def configure(self, options, conf): + super(Tally, self).configure(options, conf) + + def begin(self): + print ">>> TALLY PLUGIN BEGIN" + + def addSuccess(self, test): + self.successes.add(test) + + def addError(self, test, err): + self.errors.add(test) + + def addFailure(self, test, err): + test.failures.add(test) + + def finalize(self, result): + pass diff --git a/lib/spack/spack/test/unit_install.py b/lib/spack/spack/test/unit_install.py index c4b9092f05..0bcb3f3767 100644 --- a/lib/spack/spack/test/unit_install.py +++ b/lib/spack/spack/test/unit_install.py @@ -90,7 +90,7 @@ class UnitInstallTest(unittest.TestCase): pkgX.installed = True pkgY.installed = True - test_install.create_test_output(specX, [specX, specY], mo, getLogFunc=test_fetch_log) + test_install.create_test_output(specX, [specX, specY], mo, getLogFunc=mock_fetch_log) self.assertEqual(mo.results, {bIdX:test_install.TestResult.PASSED, @@ -101,7 +101,7 @@ class UnitInstallTest(unittest.TestCase): pkgX.installed = True pkgY.installed = True - test_install.create_test_output(specX, [specX], mo, getLogFunc=test_fetch_log) + test_install.create_test_output(specX, [specX], mo, getLogFunc=mock_fetch_log) self.assertEqual(mo.results, {bIdX:test_install.TestResult.PASSED}) @@ -116,6 +116,6 @@ class MockPackageDb(object): def get(self, spec): return self.specToPkg[spec] -def test_fetch_log(path): +def mock_fetch_log(path): return [] diff --git a/lib/spack/spack/util/executable.py b/lib/spack/spack/util/executable.py index d1dfb62ffb..6848a7e2a5 100644 --- a/lib/spack/spack/util/executable.py +++ b/lib/spack/spack/util/executable.py @@ -95,11 +95,14 @@ class Executable(object): proc = subprocess.Popen( cmd, stdin=input, - stderr=error, - stdout=subprocess.PIPE if return_output else output) + stderr=subprocess.PIPE, + stdout=subprocess.PIPE) out, err = proc.communicate() self.returncode = proc.returncode + output.write(out) + error.write(err) + rc = proc.returncode if fail_on_error and rc != 0 and (rc not in ignore_errors): raise ProcessError("Command exited with status %d:" -- cgit v1.2.3-70-g09d2 From 70049185a5b3d132374dcb36138eaa081503f96c Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 24 Nov 2015 10:39:42 -0800 Subject: Activation of Tally plugin to count tests. It doesn't appear to be actually counting the individual unit tests correctly so I need to chase that down. --- lib/spack/spack/test/__init__.py | 5 +++-- lib/spack/spack/test/tally_plugin.py | 3 --- 2 files changed, 3 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/test/__init__.py b/lib/spack/spack/test/__init__.py index 9d577d6539..327a4886b9 100644 --- a/lib/spack/spack/test/__init__.py +++ b/lib/spack/spack/test/__init__.py @@ -85,13 +85,14 @@ def run(names, verbose=False): sys.exit(1) tally = Tally() - tally.enabled = True for test in names: module = 'spack.test.' + test print module tty.msg("Running test: %s" % test) - result = nose.run(argv=["", module], plugins=[tally]) + + activateTally = "--with-%s" % spack.test.tally_plugin.Tally.name + result = nose.run(argv=["", activateTally, module], addplugins=[tally]) succeeded = not tally.failCount and not tally.errorCount tty.msg("Tests Complete.", diff --git a/lib/spack/spack/test/tally_plugin.py b/lib/spack/spack/test/tally_plugin.py index d8638bf7e3..b38f4f3134 100644 --- a/lib/spack/spack/test/tally_plugin.py +++ b/lib/spack/spack/test/tally_plugin.py @@ -57,9 +57,6 @@ class Tally(Plugin): def configure(self, options, conf): super(Tally, self).configure(options, conf) - def begin(self): - print ">>> TALLY PLUGIN BEGIN" - def addSuccess(self, test): self.successes.add(test) -- cgit v1.2.3-70-g09d2 From 5081ba6802818d5a5533985c0e3a0c1fa5b6321b Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 24 Nov 2015 10:45:07 -0800 Subject: It appears the same test object was returned multiple times for different unit tests, so tracking tests with sets wouldn't work unless I extracted the details relevant to the particular test. For now a simple count will work so using a set was unnecessary anyways. --- lib/spack/spack/test/tally_plugin.py | 24 ++++++------------------ 1 file changed, 6 insertions(+), 18 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/test/tally_plugin.py b/lib/spack/spack/test/tally_plugin.py index b38f4f3134..e314fae6ce 100644 --- a/lib/spack/spack/test/tally_plugin.py +++ b/lib/spack/spack/test/tally_plugin.py @@ -31,21 +31,9 @@ class Tally(Plugin): def __init__(self): super(Tally, self).__init__() - self.successes = set() - self.failures = set() - self.errors = set() - - @property - def successCount(self): - return len(self.successes) - - @property - def failCount(self): - return len(self.failures) - - @property - def errorCount(self): - return len(self.errors) + self.successCount = 0 + self.failCount = 0 + self.errorCount = 0 @property def numberOfTests(self): @@ -58,13 +46,13 @@ class Tally(Plugin): super(Tally, self).configure(options, conf) def addSuccess(self, test): - self.successes.add(test) + self.successCount += 1 def addError(self, test, err): - self.errors.add(test) + self.errorCount += 1 def addFailure(self, test, err): - test.failures.add(test) + self.failCount += 1 def finalize(self, result): pass -- cgit v1.2.3-70-g09d2 From 0adf93853d802266e939bffb99b6b9f60a88188e Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 24 Nov 2015 17:58:17 -0800 Subject: Update log_output to handle cases where sys.stdout/stderr streams do not have an associated file descriptor (e.g. holds for nose tests where sys.stdout is assigned to a StringIO object) --- lib/spack/llnl/util/tty/log.py | 34 +++++++++++++++++++++++++--------- 1 file changed, 25 insertions(+), 9 deletions(-) (limited to 'lib') diff --git a/lib/spack/llnl/util/tty/log.py b/lib/spack/llnl/util/tty/log.py index 5a52d45bc7..b26e0d250b 100644 --- a/lib/spack/llnl/util/tty/log.py +++ b/lib/spack/llnl/util/tty/log.py @@ -122,6 +122,10 @@ class log_output(object): self.force_color = force_color self.debug = debug + # Default is to try file-descriptor reassignment unless the system + # out/err streams do not have an associated file descriptor + self.directAssignment = False + def trace(self, frame, event, arg): """Jumps to __exit__ on the child process.""" raise _SkipWithBlock() @@ -185,13 +189,21 @@ class log_output(object): # Child: redirect output, execute the with block. os.close(read) - # Save old stdout and stderr - self._stdout = os.dup(sys.stdout.fileno()) - self._stderr = os.dup(sys.stderr.fileno()) - - # redirect to the pipe. - os.dup2(write, sys.stdout.fileno()) - os.dup2(write, sys.stderr.fileno()) + try: + # Save old stdout and stderr + self._stdout = os.dup(sys.stdout.fileno()) + self._stderr = os.dup(sys.stderr.fileno()) + + # redirect to the pipe. + os.dup2(write, sys.stdout.fileno()) + os.dup2(write, sys.stderr.fileno()) + except AttributeError: + self.directAssignment = True + self._stdout = sys.stdout + self._stderr = sys.stderr + output_redirect = os.fdopen(write, 'w') + sys.stdout = output_redirect + sys.stderr = output_redirect if self.force_color: color._force_color = True @@ -218,8 +230,12 @@ class log_output(object): # # TODO: think about how this works outside install. # TODO: ideally would propagate exception to parent... - os.dup2(self._stdout, sys.stdout.fileno()) - os.dup2(self._stderr, sys.stderr.fileno()) + if self.directAssignment: + sys.stdout = self._stdout + sys.stderr = self._stderr + else: + os.dup2(self._stdout, sys.stdout.fileno()) + os.dup2(self._stderr, sys.stderr.fileno()) return False -- cgit v1.2.3-70-g09d2 From 18dea24df8710bb65fd61278fb6bdf45d28d2aa6 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 24 Nov 2015 19:08:37 -0800 Subject: Use the xunit nose plugin to generate JUnit XML test results from Spack's unit tests. --- lib/spack/spack/cmd/test.py | 13 ++++++++++++- lib/spack/spack/test/__init__.py | 12 +++++++++--- 2 files changed, 21 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/cmd/test.py b/lib/spack/spack/cmd/test.py index b1418ac2f1..001ebfaef0 100644 --- a/lib/spack/spack/cmd/test.py +++ b/lib/spack/spack/cmd/test.py @@ -22,8 +22,10 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## +import os from pprint import pprint +from llnl.util.filesystem import join_path, mkdirp from llnl.util.tty.colify import colify from llnl.util.lang import list_modules @@ -37,6 +39,9 @@ def setup_parser(subparser): 'names', nargs='*', help="Names of tests to run.") subparser.add_argument( '-l', '--list', action='store_true', dest='list', help="Show available tests") + # TODO: make XML output optional + subparser.add_argument( + '-d', '--outputDir', dest='outputDir', help="Nose creates XML files in this directory") subparser.add_argument( '-v', '--verbose', action='store_true', dest='verbose', help="verbose output") @@ -48,4 +53,10 @@ def test(parser, args): colify(spack.test.list_tests(), indent=2) else: - spack.test.run(args.names, args.verbose) + if not args.outputDir: + outputDir = join_path(os.getcwd(), "test-output") + else: + outputDir = args.outputDir + if not os.path.exists(outputDir): + mkdirp(outputDir) + spack.test.run(args.names, outputDir, args.verbose) diff --git a/lib/spack/spack/test/__init__.py b/lib/spack/spack/test/__init__.py index 327a4886b9..158e8bdfcb 100644 --- a/lib/spack/spack/test/__init__.py +++ b/lib/spack/spack/test/__init__.py @@ -27,6 +27,7 @@ import unittest import nose from spack.test.tally_plugin import Tally +from llnl.util.filesystem import join_path import llnl.util.tty as tty from llnl.util.tty.colify import colify @@ -69,7 +70,7 @@ def list_tests(): return test_names -def run(names, verbose=False): +def run(names, outputDir, verbose=False): """Run tests with the supplied names. Names should be a list. If it's empty, run ALL of Spack's tests.""" verbosity = 1 if not verbose else 2 @@ -91,8 +92,13 @@ def run(names, verbose=False): tty.msg("Running test: %s" % test) - activateTally = "--with-%s" % spack.test.tally_plugin.Tally.name - result = nose.run(argv=["", activateTally, module], addplugins=[tally]) + xmlOutputFname = "unittests-{0}.xml".format(test) + xmlOutputPath = join_path(outputDir, xmlOutputFname) + runOpts = ["--with-%s" % spack.test.tally_plugin.Tally.name, + "--with-xunit", + "--xunit-file={0}".format(xmlOutputPath)] + argv = [""] + runOpts + [module] + result = nose.run(argv=argv, addplugins=[tally]) succeeded = not tally.failCount and not tally.errorCount tty.msg("Tests Complete.", -- cgit v1.2.3-70-g09d2 From e6d232bfefda069d876466faaa0c8e9e4a11b4f3 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Wed, 25 Nov 2015 09:58:10 -0800 Subject: Rename packages.py to repository.py, PackageFinder to RepoPath. --- lib/spack/spack/__init__.py | 4 +- lib/spack/spack/cmd/repo.py | 2 +- lib/spack/spack/cmd/uninstall.py | 4 +- lib/spack/spack/packages.py | 620 ---------------------------- lib/spack/spack/repo_loader.py | 110 ----- lib/spack/spack/repository.py | 627 +++++++++++++++++++++++++++++ lib/spack/spack/test/directory_layout.py | 4 +- lib/spack/spack/test/mock_packages_test.py | 4 +- lib/spack/spack/test/package_sanity.py | 4 +- lib/spack/spack/test/packages.py | 2 +- 10 files changed, 639 insertions(+), 742 deletions(-) delete mode 100644 lib/spack/spack/packages.py delete mode 100644 lib/spack/spack/repo_loader.py create mode 100644 lib/spack/spack/repository.py (limited to 'lib') diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index 549d0a9a0f..20ae8c9272 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -54,11 +54,11 @@ etc_path = join_path(prefix, "etc") # # Set up the default packages database. # -import spack.packages +import spack.repository _repo_paths = spack.config.get_repos_config() if not _repo_paths: tty.die("Spack configuration contains no package repositories.") -db = spack.packages.PackageFinder(*_repo_paths) +db = spack.repository.RepoPath(*_repo_paths) sys.meta_path.append(db) # diff --git a/lib/spack/spack/cmd/repo.py b/lib/spack/spack/cmd/repo.py index e290f60b7b..87f21b833d 100644 --- a/lib/spack/spack/cmd/repo.py +++ b/lib/spack/spack/cmd/repo.py @@ -32,7 +32,7 @@ from llnl.util.filesystem import join_path, mkdirp import spack.spec import spack.config from spack.util.environment import get_path -from spack.packages import repo_config_filename +from spack.repository import repo_config_filename import os import exceptions diff --git a/lib/spack/spack/cmd/uninstall.py b/lib/spack/spack/cmd/uninstall.py index e80f2d2636..eba76ef71d 100644 --- a/lib/spack/spack/cmd/uninstall.py +++ b/lib/spack/spack/cmd/uninstall.py @@ -30,7 +30,7 @@ from llnl.util.tty.colify import colify import spack import spack.cmd -import spack.packages +import spack.repository from spack.cmd.find import display_specs from spack.package import PackageStillNeededError @@ -80,7 +80,7 @@ def uninstall(parser, args): # should work if package is known to spack pkgs.append(s.package) - except spack.packages.UnknownPackageError, e: + except spack.repository.UnknownPackageError, e: # The package.py file has gone away -- but still want to uninstall. spack.Package(s).do_uninstall(force=True) diff --git a/lib/spack/spack/packages.py b/lib/spack/spack/packages.py deleted file mode 100644 index 8114b7f1aa..0000000000 --- a/lib/spack/spack/packages.py +++ /dev/null @@ -1,620 +0,0 @@ -############################################################################## -# Copyright (c) 2013, Lawrence Livermore National Security, LLC. -# Produced at the Lawrence Livermore National Laboratory. -# -# This file is part of Spack. -# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. -# LLNL-CODE-647188 -# -# For details, see https://scalability-llnl.github.io/spack -# Please also see the LICENSE file for our notice and the LGPL. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License (as published by -# the Free Software Foundation) version 2.1 dated February 1999. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and -# conditions of the GNU General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -############################################################################## -import os -import exceptions -import sys -import inspect -import glob -import imp -import re -import itertools -import traceback -from bisect import bisect_left -from external import yaml - -import llnl.util.tty as tty -from llnl.util.filesystem import join_path -from llnl.util.lang import * - -import spack.error -import spack.spec -from spack.virtual import ProviderIndex -from spack.util.naming import * - -# Filename for package repo names -repo_config_filename = '_repo.yaml' - -# Filename for packages in repos. -package_file_name = 'package.py' - -def _autospec(function): - """Decorator that automatically converts the argument of a single-arg - function to a Spec.""" - def converter(self, spec_like, *args, **kwargs): - if not isinstance(spec_like, spack.spec.Spec): - spec_like = spack.spec.Spec(spec_like) - return function(self, spec_like, *args, **kwargs) - return converter - - -def _make_namespace_module(ns): - module = imp.new_module(ns) - module.__file__ = "(spack namespace)" - module.__path__ = [] - module.__package__ = ns - return module - - -class PackageFinder(object): - """A PackageFinder is a wrapper around a list of Repos. - - It functions exactly like a Repo, but it operates on the - combined results of the Repos in its list instead of on a - single package repository. - """ - def __init__(self, *repo_dirs): - self.repos = [] - self.by_namespace = NamespaceTrie() - self.by_path = {} - - self._all_package_names = [] - self._provider_index = None - - for root in repo_dirs: - repo = Repo(root) - self.put_last(repo) - - - def swap(self, other): - """Convenience function to make swapping repostiories easier. - - This is currently used by mock tests. - TODO: Maybe there is a cleaner way. - - """ - attrs = ['repos', 'by_namespace', 'by_path', '_all_package_names', '_provider_index'] - for attr in attrs: - tmp = getattr(self, attr) - setattr(self, attr, getattr(other, attr)) - setattr(other, attr, tmp) - - - def _add(self, repo): - """Add a repository to the namespace and path indexes. - - Checks for duplicates -- two repos can't have the same root - directory, and they provide have the same namespace. - - """ - if repo.root in self.by_path: - raise DuplicateRepoError("Package repos are the same", - repo, self.by_path[repo.root]) - - if repo.namespace in self.by_namespace: - raise DuplicateRepoError("Package repos cannot have the same name", - repo, self.by_namespace[repo.namespace]) - - # Add repo to the pkg indexes - self.by_namespace[repo.namespace] = repo - self.by_path[repo.root] = repo - - # add names to the cached name list - new_pkgs = set(repo.all_package_names()) - new_pkgs.update(set(self._all_package_names)) - self._all_package_names = sorted(new_pkgs, key=lambda n:n.lower()) - - - def put_first(self, repo): - """Add repo first in the search path.""" - self._add(repo) - self.repos.insert(0, repo) - - - def put_last(self, repo): - """Add repo last in the search path.""" - self._add(repo) - self.repos.append(repo) - - - def remove(self, repo): - """Remove a repo from the search path.""" - if repo in self.repos: - self.repos.remove(repo) - - - def all_package_names(self): - """Return all unique package names in all repositories.""" - return self._all_package_names - - - def all_packages(self): - for name in self.all_package_names(): - yield self.get(name) - - - @_autospec - def providers_for(self, vpkg_spec): - if self._provider_index is None: - self._provider_index = ProviderIndex(self.all_package_names()) - - providers = self._provider_index.providers_for(vpkg_spec) - if not providers: - raise UnknownPackageError(vpkg_spec.name) - return providers - - - def find_module(self, fullname, path=None): - """Implements precedence for overlaid namespaces. - - Loop checks each namespace in self.repos for packages, and - also handles loading empty containing namespaces. - - """ - # namespaces are added to repo, and package modules are leaves. - namespace, dot, module_name = fullname.rpartition('.') - - # If it's a module in some repo, or if it is the repo's - # namespace, let the repo handle it. - for repo in self.repos: - if namespace == repo.namespace: - if repo.real_name(module_name): - return repo - elif fullname == repo.namespace: - return repo - - # No repo provides the namespace, but it is a valid prefix of - # something in the PackageFinder. - if self.by_namespace.is_prefix(fullname): - return self - - return None - - - def load_module(self, fullname): - """Loads containing namespaces when necessary. - - See ``Repo`` for how actual package modules are loaded. - """ - if fullname in sys.modules: - return sys.modules[fullname] - - # partition fullname into prefix and module name. - namespace, dot, module_name = fullname.rpartition('.') - - if not self.by_namespace.is_prefix(fullname): - raise ImportError("No such Spack repo: %s" % fullname) - - module = _make_namespace_module(namespace) - module.__loader__ = self - sys.modules[fullname] = module - return module - - - def repo_for_pkg(self, pkg_name): - for repo in self.repos: - if pkg_name in repo: - return repo - raise UnknownPackageError(pkg_name) - - - @_autospec - def get(self, spec, new=False): - """Find a repo that contains the supplied spec's package. - - Raises UnknownPackageError if not found. - """ - return self.repo_for_pkg(spec.name).get(spec) - - - def dirname_for_package_name(self, pkg_name): - return self.repo_for_pkg(pkg_name).dirname_for_package_name(pkg_name) - - - def exists(self, pkg_name): - return any(repo.exists(pkg_name) for repo in self.repos) - - - def __contains__(self, pkg_name): - return self.exists(pkg_name) - - - -class Repo(object): - """Class representing a package repository in the filesystem. - - Each package repository must have a top-level configuration file - called `_repo.yaml`. - - Currently, `_repo.yaml` this must define: - - `namespace`: - A Python namespace where the repository's packages should live. - - """ - def __init__(self, root): - """Instantiate a package repository from a filesystem path.""" - # Root directory, containing _repo.yaml and package dirs - self.root = root - - # Config file in /_repo.yaml - self.config_file = os.path.join(self.root, repo_config_filename) - - # Read configuration from _repo.yaml - config = self._read_config() - if not 'namespace' in config: - tty.die('Package repo in %s must define a namespace in %s.' - % (self.root, repo_config_filename)) - - # Check namespace in the repository configuration. - self.namespace = config['namespace'] - if not re.match(r'[a-zA-Z][a-zA-Z0-9_.]+', self.namespace): - tty.die(("Invalid namespace '%s' in '%s'. Namespaces must be " - "valid python identifiers separated by '.'") - % (self.namespace, self.root)) - self._names = self.namespace.split('.') - - # These are internal cache variables. - self._modules = {} - self._classes = {} - self._instances = {} - - self._provider_index = None - self._all_package_names = None - - # make sure the namespace for packages in this repo exists. - self._create_namespace() - - - def _create_namespace(self): - """Create this repo's namespace module and insert it into sys.modules. - - Ensures that modules loaded via the repo have a home, and that - we don't get runtime warnings from Python's module system. - - """ - for l in range(1, len(self._names)+1): - ns = '.'.join(self._names[:l]) - if not ns in sys.modules: - sys.modules[ns] = _make_namespace_module(ns) - sys.modules[ns].__loader__ = self - - - def real_name(self, import_name): - """Allow users to import Spack packages using Python identifiers. - - A python identifier might map to many different Spack package - names due to hyphen/underscore ambiguity. - - Easy example: - num3proxy -> 3proxy - - Ambiguous: - foo_bar -> foo_bar, foo-bar - - More ambiguous: - foo_bar_baz -> foo_bar_baz, foo-bar-baz, foo_bar-baz, foo-bar_baz - """ - if import_name in self: - return import_name - - options = possible_spack_module_names(import_name) - options.remove(import_name) - for name in options: - if name in self: - return name - return None - - - def is_prefix(self, fullname): - """True if fullname is a prefix of this Repo's namespace.""" - parts = fullname.split('.') - return self._names[:len(parts)] == parts - - - def find_module(self, fullname, path=None): - """Python find_module import hook. - - Returns this Repo if it can load the module; None if not. - """ - if self.is_prefix(fullname): - return self - - namespace, dot, module_name = fullname.rpartition('.') - if namespace == self.namespace: - if self.real_name(module_name): - return self - - return None - - - def load_module(self, fullname): - """Python importer load hook. - - Tries to load the module; raises an ImportError if it can't. - """ - if fullname in sys.modules: - return sys.modules[fullname] - - namespace, dot, module_name = fullname.rpartition('.') - - if self.is_prefix(fullname): - module = _make_namespace_module(fullname) - - elif namespace == self.namespace: - real_name = self.real_name(module_name) - if not real_name: - raise ImportError("No module %s in repo %s" % (module_name, namespace)) - module = self._get_pkg_module(real_name) - - else: - raise ImportError("No module %s in repo %s" % (fullname, self.namespace)) - - module.__loader__ = self - sys.modules[fullname] = module - return module - - - def _read_config(self): - """Check for a YAML config file in this db's root directory.""" - try: - with open(self.config_file) as reponame_file: - yaml_data = yaml.load(reponame_file) - - if (not yaml_data or 'repo' not in yaml_data or - not isinstance(yaml_data['repo'], dict)): - tty.die("Invalid %s in repository %s" - % (repo_config_filename, self.root)) - - return yaml_data['repo'] - - except exceptions.IOError, e: - tty.die("Error reading %s when opening %s" - % (self.config_file, self.root)) - - - @_autospec - def get(self, spec, new=False): - if spec.virtual: - raise UnknownPackageError(spec.name) - - if new and spec in self._instances: - del self._instances[spec] - - if not spec in self._instances: - PackageClass = self._get_pkg_class(spec.name) - try: - copy = spec.copy() - self._instances[copy] = PackageClass(copy) - except Exception, e: - if spack.debug: - sys.excepthook(*sys.exc_info()) - raise FailedConstructorError(spec.name, *sys.exc_info()) - - return self._instances[spec] - - - def purge(self): - """Clear entire package instance cache.""" - self._instances.clear() - - - @_autospec - def providers_for(self, vpkg_spec): - if self._provider_index is None: - self._provider_index = ProviderIndex(self.all_package_names()) - - providers = self._provider_index.providers_for(vpkg_spec) - if not providers: - raise UnknownPackageError(vpkg_spec.name) - return providers - - - @_autospec - def extensions_for(self, extendee_spec): - return [p for p in self.all_packages() if p.extends(extendee_spec)] - - - def dirname_for_package_name(self, pkg_name): - """Get the directory name for a particular package. This is the - directory that contains its package.py file.""" - return join_path(self.root, pkg_name) - - - def filename_for_package_name(self, pkg_name): - """Get the filename for the module we should load for a particular - package. Packages for a Repo live in - ``$root//package.py`` - - This will return a proper package.py path even if the - package doesn't exist yet, so callers will need to ensure - the package exists before importing. - """ - validate_module_name(pkg_name) - pkg_dir = self.dirname_for_package_name(pkg_name) - - return join_path(pkg_dir, package_file_name) - - - def all_package_names(self): - """Returns a sorted list of all package names in the Repo.""" - if self._all_package_names is None: - self._all_package_names = [] - - for pkg_name in os.listdir(self.root): - pkg_dir = join_path(self.root, pkg_name) - pkg_file = join_path(pkg_dir, package_file_name) - if os.path.isfile(pkg_file): - self._all_package_names.append(pkg_name) - - self._all_package_names.sort() - - return self._all_package_names - - - def all_packages(self): - for name in self.all_package_names(): - yield self.get(name) - - - def exists(self, pkg_name): - """Whether a package with the supplied name exists.""" - # This does a binary search in the sorted list. - idx = bisect_left(self.all_package_names(), pkg_name) - return self._all_package_names[idx] == pkg_name - - - def _get_pkg_module(self, pkg_name): - """Create a module for a particular package. - - This caches the module within this Repo *instance*. It does - *not* add it to ``sys.modules``. So, you can construct - multiple Repos for testing and ensure that the module will be - loaded once per repo. - - """ - if pkg_name not in self._modules: - file_path = self.filename_for_package_name(pkg_name) - - if not os.path.exists(file_path): - raise UnknownPackageError(pkg_name, self.namespace) - - if not os.path.isfile(file_path): - tty.die("Something's wrong. '%s' is not a file!" % file_path) - - if not os.access(file_path, os.R_OK): - tty.die("Cannot read '%s'!" % file_path) - - fullname = "%s.%s" % (self.namespace, pkg_name) - - module = imp.load_source(fullname, file_path) - module.__package__ = self.namespace - module.__loader__ = self - self._modules[pkg_name] = module - - return self._modules[pkg_name] - - - def _get_pkg_class(self, pkg_name): - """Get the class for the package out of its module. - - First loads (or fetches from cache) a module for the - package. Then extracts the package class from the module - according to Spack's naming convention. - """ - class_name = mod_to_class(pkg_name) - module = self._get_pkg_module(pkg_name) - - cls = getattr(module, class_name) - if not inspect.isclass(cls): - tty.die("%s.%s is not a class" % (pkg_name, class_name)) - - return cls - - - def __str__(self): - return "" % (self.namespace, self.root) - - - def __repr__(self): - return self.__str__() - - - def __contains__(self, pkg_name): - return self.exists(pkg_name) - - - # - # Below functions deal with installed packages, and should be - # moved to some other part of Spack (conbine with - # directory_layout?) - # - @_autospec - def get_installed(self, spec): - """Get all the installed specs that satisfy the provided spec constraint.""" - return [s for s in self.installed_package_specs() if s.satisfies(spec)] - - - @_autospec - def installed_extensions_for(self, extendee_spec): - for s in self.installed_package_specs(): - try: - if s.package.extends(extendee_spec): - yield s.package - except UnknownPackageError, e: - # Skip packages we know nothing about - continue - - - def installed_package_specs(self): - """Read installed package names straight from the install directory - layout. - """ - # Get specs from the directory layout but ensure that they're - # all normalized properly. - installed = [] - for spec in spack.install_layout.all_specs(): - spec.normalize() - installed.append(spec) - return installed - - - def installed_known_package_specs(self): - """Read installed package names straight from the install - directory layout, but return only specs for which the - package is known to this version of spack. - """ - for spec in spack.install_layout.all_specs(): - if self.exists(spec.name): - yield spec - - -class UnknownPackageError(spack.error.SpackError): - """Raised when we encounter a package spack doesn't have.""" - def __init__(self, name, repo=None): - msg = None - if repo: - msg = "Package %s not found in packagerepo %s." % (name, repo) - else: - msg = "Package %s not found." % name - super(UnknownPackageError, self).__init__(msg) - self.name = name - - -class DuplicateRepoError(spack.error.SpackError): - """Raised when duplicate repos are added to a PackageFinder.""" - def __init__(self, msg, repo1, repo2): - super(UnknownPackageError, self).__init__( - "%s: %s, %s" % (msg, repo1, repo2)) - - -class FailedConstructorError(spack.error.SpackError): - """Raised when a package's class constructor fails.""" - def __init__(self, name, exc_type, exc_obj, exc_tb): - super(FailedConstructorError, self).__init__( - "Class constructor failed for package '%s'." % name, - '\nCaused by:\n' + - ('%s: %s\n' % (exc_type.__name__, exc_obj)) + - ''.join(traceback.format_tb(exc_tb))) - self.name = name diff --git a/lib/spack/spack/repo_loader.py b/lib/spack/spack/repo_loader.py deleted file mode 100644 index 441011cf98..0000000000 --- a/lib/spack/spack/repo_loader.py +++ /dev/null @@ -1,110 +0,0 @@ -import re -import sys -import types -import traceback - -from llnl.util.lang import * -import spack - -# Name of module under which packages are imported -imported_packages_module = 'spack.repos' - -# Name of the package file inside a package directory -package_file_name = 'package.py' - -class LazyLoader: - """The LazyLoader handles cases when repo modules or classes - are imported. It watches for 'spack.repos.*' loads, then - redirects the load to the appropriate module.""" - def find_module(self, fullname, pathname): - if not fullname.startswith(imported_packages_module): - return None - - partial_name = fullname[len(imported_packages_module)+1:] - - print "partial: ", partial_name - print - - last_dot = partial_name.rfind('.') - repo = partial_name[:last_dot] - module = partial_name[last_dot+1:] - - repo_loader = spack.db.repo_loaders.get(repo) - if repo_loader: - try: - self.mod = repo_loader.get_module(module) - return self - except (ImportError, spack.packages.UnknownPackageError): - return None - - def load_module(self, fullname): - return self.mod - -#sys.meta_path.append(LazyLoader()) - -_reponames = {} -class RepoNamespace(types.ModuleType): - """The RepoNamespace holds the repository namespaces under - spack.repos. For example, when accessing spack.repos.original - this class will use __getattr__ to translate the 'original' - into one of spack's known repositories""" - def __init__(self): - sys.modules[imported_packages_module] = self - - def __getattr__(self, name): - if name in _reponames: - return _reponames[name] - raise AttributeError - - @property - def __file__(self): - return None - - @property - def __path__(self): - return [] - - -class RepoLoader(types.ModuleType): - """Each RepoLoader is associated with a repository, and the RepoLoader is - responsible for loading packages out of that repository. For example, - a RepoLoader may be responsible for spack.repos.original, and when someone - references spack.repos.original.libelf that RepoLoader will load the - libelf package.""" - def __init__(self, reponame, repopath): - self.path = repopath - self.reponame = reponame - self.module_name = imported_packages_module + '.' + reponame - if not reponame in _reponames: - _reponames[reponame] = self - - sys.modules[self.module_name] = self - - - @property - def __path__(self): - return [ self.path ] - - - def __getattr__(self, name): - if name[0] == '_': - raise AttributeError - return self.get_module(name) - - - @memoized - def get_module(self, pkg_name): - import os - import imp - import llnl.util.tty as tty - - - try: - module_name = imported_packages_module + '.' + self.reponame + '.' + pkg_name - module = imp.load_source(module_name, file_path) - - except ImportError, e: - tty.die("Error while importing %s from %s:\n%s" % ( - pkg_name, file_path, e.message)) - - return module diff --git a/lib/spack/spack/repository.py b/lib/spack/spack/repository.py new file mode 100644 index 0000000000..c1545b3654 --- /dev/null +++ b/lib/spack/spack/repository.py @@ -0,0 +1,627 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +import os +import exceptions +import sys +import inspect +import glob +import imp +import re +import itertools +import traceback +from bisect import bisect_left +from external import yaml + +import llnl.util.tty as tty +from llnl.util.filesystem import join_path +from llnl.util.lang import * + +import spack.error +import spack.spec +from spack.virtual import ProviderIndex +from spack.util.naming import * + +# Filename for package repo names +repo_config_filename = '_repo.yaml' + +# Filename for packages in repos. +package_file_name = 'package.py' + +def _autospec(function): + """Decorator that automatically converts the argument of a single-arg + function to a Spec.""" + def converter(self, spec_like, *args, **kwargs): + if not isinstance(spec_like, spack.spec.Spec): + spec_like = spack.spec.Spec(spec_like) + return function(self, spec_like, *args, **kwargs) + return converter + + +def _make_namespace_module(ns): + module = imp.new_module(ns) + module.__file__ = "(spack namespace)" + module.__path__ = [] + module.__package__ = ns + return module + + +class RepoPath(object): + """A RepoPath is a list of repos that function as one. + + It functions exactly like a Repo, but it operates on the + combined results of the Repos in its list instead of on a + single package repository. + """ + def __init__(self, *repo_dirs): + self.repos = [] + self.by_namespace = NamespaceTrie() + self.by_path = {} + + self._all_package_names = [] + self._provider_index = None + + for root in repo_dirs: + # Try to make it a repo if it's not one. + if not isinstance(root, Repo): + repo = Repo(root) + # Add the repo to the path. + self.put_last(repo) + + + def swap(self, other): + """Convenience function to make swapping repostiories easier. + + This is currently used by mock tests. + TODO: Maybe there is a cleaner way. + + """ + attrs = ['repos', + 'by_namespace', + 'by_path', + '_all_package_names', + '_provider_index'] + for attr in attrs: + tmp = getattr(self, attr) + setattr(self, attr, getattr(other, attr)) + setattr(other, attr, tmp) + + + def _add(self, repo): + """Add a repository to the namespace and path indexes. + + Checks for duplicates -- two repos can't have the same root + directory, and they provide have the same namespace. + + """ + if repo.root in self.by_path: + raise DuplicateRepoError("Package repos are the same", + repo, self.by_path[repo.root]) + + if repo.namespace in self.by_namespace: + raise DuplicateRepoError("Package repos cannot have the same name", + repo, self.by_namespace[repo.namespace]) + + # Add repo to the pkg indexes + self.by_namespace[repo.namespace] = repo + self.by_path[repo.root] = repo + + # add names to the cached name list + new_pkgs = set(repo.all_package_names()) + new_pkgs.update(set(self._all_package_names)) + self._all_package_names = sorted(new_pkgs, key=lambda n:n.lower()) + + + def put_first(self, repo): + """Add repo first in the search path.""" + self._add(repo) + self.repos.insert(0, repo) + + + def put_last(self, repo): + """Add repo last in the search path.""" + self._add(repo) + self.repos.append(repo) + + + def remove(self, repo): + """Remove a repo from the search path.""" + if repo in self.repos: + self.repos.remove(repo) + + + def all_package_names(self): + """Return all unique package names in all repositories.""" + return self._all_package_names + + + def all_packages(self): + for name in self.all_package_names(): + yield self.get(name) + + + @_autospec + def providers_for(self, vpkg_spec): + if self._provider_index is None: + self._provider_index = ProviderIndex(self.all_package_names()) + + providers = self._provider_index.providers_for(vpkg_spec) + if not providers: + raise UnknownPackageError(vpkg_spec.name) + return providers + + + def find_module(self, fullname, path=None): + """Implements precedence for overlaid namespaces. + + Loop checks each namespace in self.repos for packages, and + also handles loading empty containing namespaces. + + """ + # namespaces are added to repo, and package modules are leaves. + namespace, dot, module_name = fullname.rpartition('.') + + # If it's a module in some repo, or if it is the repo's + # namespace, let the repo handle it. + for repo in self.repos: + if namespace == repo.namespace: + if repo.real_name(module_name): + return repo + elif fullname == repo.namespace: + return repo + + # No repo provides the namespace, but it is a valid prefix of + # something in the RepoPath. + if self.by_namespace.is_prefix(fullname): + return self + + return None + + + def load_module(self, fullname): + """Loads containing namespaces when necessary. + + See ``Repo`` for how actual package modules are loaded. + """ + if fullname in sys.modules: + return sys.modules[fullname] + + # partition fullname into prefix and module name. + namespace, dot, module_name = fullname.rpartition('.') + + if not self.by_namespace.is_prefix(fullname): + raise ImportError("No such Spack repo: %s" % fullname) + + module = _make_namespace_module(namespace) + module.__loader__ = self + sys.modules[fullname] = module + return module + + + def repo_for_pkg(self, pkg_name): + for repo in self.repos: + if pkg_name in repo: + return repo + raise UnknownPackageError(pkg_name) + + + @_autospec + def get(self, spec, new=False): + """Find a repo that contains the supplied spec's package. + + Raises UnknownPackageError if not found. + """ + return self.repo_for_pkg(spec.name).get(spec) + + + def dirname_for_package_name(self, pkg_name): + return self.repo_for_pkg(pkg_name).dirname_for_package_name(pkg_name) + + + def exists(self, pkg_name): + return any(repo.exists(pkg_name) for repo in self.repos) + + + def __contains__(self, pkg_name): + return self.exists(pkg_name) + + + +class Repo(object): + """Class representing a package repository in the filesystem. + + Each package repository must have a top-level configuration file + called `_repo.yaml`. + + Currently, `_repo.yaml` this must define: + + `namespace`: + A Python namespace where the repository's packages should live. + + """ + def __init__(self, root): + """Instantiate a package repository from a filesystem path.""" + # Root directory, containing _repo.yaml and package dirs + self.root = root + + # Config file in /_repo.yaml + self.config_file = os.path.join(self.root, repo_config_filename) + + # Read configuration from _repo.yaml + config = self._read_config() + if not 'namespace' in config: + tty.die('Package repo in %s must define a namespace in %s.' + % (self.root, repo_config_filename)) + + # Check namespace in the repository configuration. + self.namespace = config['namespace'] + if not re.match(r'[a-zA-Z][a-zA-Z0-9_.]+', self.namespace): + tty.die(("Invalid namespace '%s' in '%s'. Namespaces must be " + "valid python identifiers separated by '.'") + % (self.namespace, self.root)) + self._names = self.namespace.split('.') + + # These are internal cache variables. + self._modules = {} + self._classes = {} + self._instances = {} + + self._provider_index = None + self._all_package_names = None + + # make sure the namespace for packages in this repo exists. + self._create_namespace() + + + def _create_namespace(self): + """Create this repo's namespace module and insert it into sys.modules. + + Ensures that modules loaded via the repo have a home, and that + we don't get runtime warnings from Python's module system. + + """ + for l in range(1, len(self._names)+1): + ns = '.'.join(self._names[:l]) + if not ns in sys.modules: + sys.modules[ns] = _make_namespace_module(ns) + sys.modules[ns].__loader__ = self + + + def real_name(self, import_name): + """Allow users to import Spack packages using Python identifiers. + + A python identifier might map to many different Spack package + names due to hyphen/underscore ambiguity. + + Easy example: + num3proxy -> 3proxy + + Ambiguous: + foo_bar -> foo_bar, foo-bar + + More ambiguous: + foo_bar_baz -> foo_bar_baz, foo-bar-baz, foo_bar-baz, foo-bar_baz + """ + if import_name in self: + return import_name + + options = possible_spack_module_names(import_name) + options.remove(import_name) + for name in options: + if name in self: + return name + return None + + + def is_prefix(self, fullname): + """True if fullname is a prefix of this Repo's namespace.""" + parts = fullname.split('.') + return self._names[:len(parts)] == parts + + + def find_module(self, fullname, path=None): + """Python find_module import hook. + + Returns this Repo if it can load the module; None if not. + """ + if self.is_prefix(fullname): + return self + + namespace, dot, module_name = fullname.rpartition('.') + if namespace == self.namespace: + if self.real_name(module_name): + return self + + return None + + + def load_module(self, fullname): + """Python importer load hook. + + Tries to load the module; raises an ImportError if it can't. + """ + if fullname in sys.modules: + return sys.modules[fullname] + + namespace, dot, module_name = fullname.rpartition('.') + + if self.is_prefix(fullname): + module = _make_namespace_module(fullname) + + elif namespace == self.namespace: + real_name = self.real_name(module_name) + if not real_name: + raise ImportError("No module %s in repo %s" % (module_name, namespace)) + module = self._get_pkg_module(real_name) + + else: + raise ImportError("No module %s in repo %s" % (fullname, self.namespace)) + + module.__loader__ = self + sys.modules[fullname] = module + return module + + + def _read_config(self): + """Check for a YAML config file in this db's root directory.""" + try: + with open(self.config_file) as reponame_file: + yaml_data = yaml.load(reponame_file) + + if (not yaml_data or 'repo' not in yaml_data or + not isinstance(yaml_data['repo'], dict)): + tty.die("Invalid %s in repository %s" + % (repo_config_filename, self.root)) + + return yaml_data['repo'] + + except exceptions.IOError, e: + tty.die("Error reading %s when opening %s" + % (self.config_file, self.root)) + + + @_autospec + def get(self, spec, new=False): + if spec.virtual: + raise UnknownPackageError(spec.name) + + if new and spec in self._instances: + del self._instances[spec] + + if not spec in self._instances: + PackageClass = self._get_pkg_class(spec.name) + try: + copy = spec.copy() + self._instances[copy] = PackageClass(copy) + except Exception, e: + if spack.debug: + sys.excepthook(*sys.exc_info()) + raise FailedConstructorError(spec.name, *sys.exc_info()) + + return self._instances[spec] + + + def purge(self): + """Clear entire package instance cache.""" + self._instances.clear() + + + @_autospec + def providers_for(self, vpkg_spec): + if self._provider_index is None: + self._provider_index = ProviderIndex(self.all_package_names()) + + providers = self._provider_index.providers_for(vpkg_spec) + if not providers: + raise UnknownPackageError(vpkg_spec.name) + return providers + + + @_autospec + def extensions_for(self, extendee_spec): + return [p for p in self.all_packages() if p.extends(extendee_spec)] + + + def dirname_for_package_name(self, pkg_name): + """Get the directory name for a particular package. This is the + directory that contains its package.py file.""" + return join_path(self.root, pkg_name) + + + def filename_for_package_name(self, pkg_name): + """Get the filename for the module we should load for a particular + package. Packages for a Repo live in + ``$root//package.py`` + + This will return a proper package.py path even if the + package doesn't exist yet, so callers will need to ensure + the package exists before importing. + """ + validate_module_name(pkg_name) + pkg_dir = self.dirname_for_package_name(pkg_name) + + return join_path(pkg_dir, package_file_name) + + + def all_package_names(self): + """Returns a sorted list of all package names in the Repo.""" + if self._all_package_names is None: + self._all_package_names = [] + + for pkg_name in os.listdir(self.root): + pkg_dir = join_path(self.root, pkg_name) + pkg_file = join_path(pkg_dir, package_file_name) + if os.path.isfile(pkg_file): + self._all_package_names.append(pkg_name) + + self._all_package_names.sort() + + return self._all_package_names + + + def all_packages(self): + for name in self.all_package_names(): + yield self.get(name) + + + def exists(self, pkg_name): + """Whether a package with the supplied name exists.""" + # This does a binary search in the sorted list. + idx = bisect_left(self.all_package_names(), pkg_name) + return self._all_package_names[idx] == pkg_name + + + def _get_pkg_module(self, pkg_name): + """Create a module for a particular package. + + This caches the module within this Repo *instance*. It does + *not* add it to ``sys.modules``. So, you can construct + multiple Repos for testing and ensure that the module will be + loaded once per repo. + + """ + if pkg_name not in self._modules: + file_path = self.filename_for_package_name(pkg_name) + + if not os.path.exists(file_path): + raise UnknownPackageError(pkg_name, self.namespace) + + if not os.path.isfile(file_path): + tty.die("Something's wrong. '%s' is not a file!" % file_path) + + if not os.access(file_path, os.R_OK): + tty.die("Cannot read '%s'!" % file_path) + + fullname = "%s.%s" % (self.namespace, pkg_name) + + module = imp.load_source(fullname, file_path) + module.__package__ = self.namespace + module.__loader__ = self + self._modules[pkg_name] = module + + return self._modules[pkg_name] + + + def _get_pkg_class(self, pkg_name): + """Get the class for the package out of its module. + + First loads (or fetches from cache) a module for the + package. Then extracts the package class from the module + according to Spack's naming convention. + """ + class_name = mod_to_class(pkg_name) + module = self._get_pkg_module(pkg_name) + + cls = getattr(module, class_name) + if not inspect.isclass(cls): + tty.die("%s.%s is not a class" % (pkg_name, class_name)) + + return cls + + + def __str__(self): + return "" % (self.namespace, self.root) + + + def __repr__(self): + return self.__str__() + + + def __contains__(self, pkg_name): + return self.exists(pkg_name) + + + # + # Below functions deal with installed packages, and should be + # moved to some other part of Spack (conbine with + # directory_layout?) + # + @_autospec + def get_installed(self, spec): + """Get all the installed specs that satisfy the provided spec constraint.""" + return [s for s in self.installed_package_specs() if s.satisfies(spec)] + + + @_autospec + def installed_extensions_for(self, extendee_spec): + for s in self.installed_package_specs(): + try: + if s.package.extends(extendee_spec): + yield s.package + except UnknownPackageError, e: + # Skip packages we know nothing about + continue + + + def installed_package_specs(self): + """Read installed package names straight from the install directory + layout. + """ + # Get specs from the directory layout but ensure that they're + # all normalized properly. + installed = [] + for spec in spack.install_layout.all_specs(): + spec.normalize() + installed.append(spec) + return installed + + + def installed_known_package_specs(self): + """Read installed package names straight from the install + directory layout, but return only specs for which the + package is known to this version of spack. + """ + for spec in spack.install_layout.all_specs(): + if self.exists(spec.name): + yield spec + + +class UnknownPackageError(spack.error.SpackError): + """Raised when we encounter a package spack doesn't have.""" + def __init__(self, name, repo=None): + msg = None + if repo: + msg = "Package %s not found in packagerepo %s." % (name, repo) + else: + msg = "Package %s not found." % name + super(UnknownPackageError, self).__init__(msg) + self.name = name + + +class DuplicateRepoError(spack.error.SpackError): + """Raised when duplicate repos are added to a RepoPath.""" + def __init__(self, msg, repo1, repo2): + super(UnknownPackageError, self).__init__( + "%s: %s, %s" % (msg, repo1, repo2)) + + +class FailedConstructorError(spack.error.SpackError): + """Raised when a package's class constructor fails.""" + def __init__(self, name, exc_type, exc_obj, exc_tb): + super(FailedConstructorError, self).__init__( + "Class constructor failed for package '%s'." % name, + '\nCaused by:\n' + + ('%s: %s\n' % (exc_type.__name__, exc_obj)) + + ''.join(traceback.format_tb(exc_tb))) + self.name = name diff --git a/lib/spack/spack/test/directory_layout.py b/lib/spack/spack/test/directory_layout.py index 55b3f0b18f..580620e0e8 100644 --- a/lib/spack/spack/test/directory_layout.py +++ b/lib/spack/spack/test/directory_layout.py @@ -34,7 +34,7 @@ from llnl.util.filesystem import * import spack from spack.spec import Spec -from spack.packages import PackageFinder +from spack.repository import RepoPath from spack.directory_layout import YamlDirectoryLayout # number of packages to test (to reduce test time) @@ -123,7 +123,7 @@ class DirectoryLayoutTest(unittest.TestCase): information about installed packages' specs to uninstall or query them again if the package goes away. """ - mock_db = PackageFinder(spack.mock_packages_path) + mock_db = RepoPath(spack.mock_packages_path) not_in_mock = set.difference( set(spack.db.all_package_names()), diff --git a/lib/spack/spack/test/mock_packages_test.py b/lib/spack/spack/test/mock_packages_test.py index 071c21b7e0..8c6273def0 100644 --- a/lib/spack/spack/test/mock_packages_test.py +++ b/lib/spack/spack/test/mock_packages_test.py @@ -27,7 +27,7 @@ import unittest import spack import spack.config -from spack.packages import PackageFinder +from spack.repository import RepoPath from spack.spec import Spec @@ -36,7 +36,7 @@ class MockPackagesTest(unittest.TestCase): # Use the mock packages database for these tests. This allows # us to set up contrived packages that don't interfere with # real ones. - self.db = PackageFinder(spack.mock_packages_path) + self.db = RepoPath(spack.mock_packages_path) spack.db.swap(self.db) spack.config.clear_config_caches() diff --git a/lib/spack/spack/test/package_sanity.py b/lib/spack/spack/test/package_sanity.py index a398568244..0a132fd701 100644 --- a/lib/spack/spack/test/package_sanity.py +++ b/lib/spack/spack/test/package_sanity.py @@ -28,7 +28,7 @@ This test does sanity checks on Spack's builtin package database. import unittest import spack -from spack.packages import PackageFinder +from spack.repository import RepoPath class PackageSanityTest(unittest.TestCase): @@ -46,7 +46,7 @@ class PackageSanityTest(unittest.TestCase): def ztest_get_all_mock_packages(self): """Get the mock packages once each too.""" - db = PackageFinder(spack.mock_packages_path) + db = RepoPath(spack.mock_packages_path) spack.db.swap(db) self.check_db() spack.db.swap(db) diff --git a/lib/spack/spack/test/packages.py b/lib/spack/spack/test/packages.py index 2d19d9ddc7..8a786e364e 100644 --- a/lib/spack/spack/test/packages.py +++ b/lib/spack/spack/test/packages.py @@ -27,7 +27,7 @@ import unittest from llnl.util.filesystem import join_path import spack -from spack.packages import Repo +from spack.repository import Repo from spack.util.naming import mod_to_class from spack.test.mock_packages_test import * -- cgit v1.2.3-70-g09d2 From 04f032d6e397ce219a673c93277683060def52fd Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Wed, 25 Nov 2015 10:01:44 -0800 Subject: Rename spack.db -> spack.repo Making distinction between install database and package repositories clearer. --- lib/spack/spack/__init__.py | 6 ++-- lib/spack/spack/cmd/checksum.py | 2 +- lib/spack/spack/cmd/clean.py | 2 +- lib/spack/spack/cmd/create.py | 4 +-- lib/spack/spack/cmd/diy.py | 4 +-- lib/spack/spack/cmd/edit.py | 2 +- lib/spack/spack/cmd/extensions.py | 2 +- lib/spack/spack/cmd/fetch.py | 2 +- lib/spack/spack/cmd/find.py | 2 +- lib/spack/spack/cmd/info.py | 2 +- lib/spack/spack/cmd/install.py | 2 +- lib/spack/spack/cmd/list.py | 2 +- lib/spack/spack/cmd/location.py | 6 ++-- lib/spack/spack/cmd/mirror.py | 2 +- lib/spack/spack/cmd/package-list.py | 2 +- lib/spack/spack/cmd/patch.py | 2 +- lib/spack/spack/cmd/pkg.py | 2 +- lib/spack/spack/cmd/providers.py | 2 +- lib/spack/spack/cmd/repo.py | 2 +- lib/spack/spack/cmd/restage.py | 2 +- lib/spack/spack/cmd/stage.py | 2 +- lib/spack/spack/cmd/test-install.py | 8 ++--- lib/spack/spack/cmd/urls.py | 2 +- lib/spack/spack/cmd/versions.py | 2 +- lib/spack/spack/database.py | 2 +- lib/spack/spack/graph.py | 2 +- lib/spack/spack/package.py | 4 +-- lib/spack/spack/patch.py | 2 +- lib/spack/spack/spec.py | 14 ++++----- lib/spack/spack/test/concretize.py | 12 ++++---- lib/spack/spack/test/database.py | 2 +- lib/spack/spack/test/directory_layout.py | 12 ++++---- lib/spack/spack/test/git_fetch.py | 2 +- lib/spack/spack/test/hg_fetch.py | 2 +- lib/spack/spack/test/install.py | 2 +- lib/spack/spack/test/mirror.py | 2 +- lib/spack/spack/test/mock_packages_test.py | 6 ++-- lib/spack/spack/test/multimethod.py | 48 +++++++++++++++--------------- lib/spack/spack/test/package_sanity.py | 10 +++---- lib/spack/spack/test/packages.py | 6 ++-- lib/spack/spack/test/python_version.py | 4 +-- lib/spack/spack/test/svn_fetch.py | 2 +- lib/spack/spack/test/unit_install.py | 6 ++-- 43 files changed, 103 insertions(+), 103 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index 20ae8c9272..4f481ce937 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -58,8 +58,8 @@ import spack.repository _repo_paths = spack.config.get_repos_config() if not _repo_paths: tty.die("Spack configuration contains no package repositories.") -db = spack.repository.RepoPath(*_repo_paths) -sys.meta_path.append(db) +repo = spack.repository.RepoPath(*_repo_paths) +sys.meta_path.append(repo) # # Set up the installed packages database @@ -146,7 +146,7 @@ sys_type = None # When packages call 'from spack import *', this extra stuff is brought in. # # Spack internal code should call 'import spack' and accesses other -# variables (spack.db, paths, etc.) directly. +# variables (spack.repo, paths, etc.) directly. # # TODO: maybe this should be separated out and should go in build_environment.py? # TODO: it's not clear where all the stuff that needs to be included in packages diff --git a/lib/spack/spack/cmd/checksum.py b/lib/spack/spack/cmd/checksum.py index 8a448450c2..a9a5d11cca 100644 --- a/lib/spack/spack/cmd/checksum.py +++ b/lib/spack/spack/cmd/checksum.py @@ -81,7 +81,7 @@ def get_checksums(versions, urls, **kwargs): def checksum(parser, args): # get the package we're going to generate checksums for - pkg = spack.db.get(args.package) + pkg = spack.repo.get(args.package) # If the user asked for specific versions, use those. if args.versions: diff --git a/lib/spack/spack/cmd/clean.py b/lib/spack/spack/cmd/clean.py index c20136ebe5..5546060e09 100644 --- a/lib/spack/spack/cmd/clean.py +++ b/lib/spack/spack/cmd/clean.py @@ -42,5 +42,5 @@ def clean(parser, args): specs = spack.cmd.parse_specs(args.packages, concretize=True) for spec in specs: - package = spack.db.get(spec) + package = spack.repo.get(spec) package.do_clean() diff --git a/lib/spack/spack/cmd/create.py b/lib/spack/spack/cmd/create.py index 1502942f2c..475c4993b5 100644 --- a/lib/spack/spack/cmd/create.py +++ b/lib/spack/spack/cmd/create.py @@ -172,7 +172,7 @@ def create(parser, args): tty.msg("Creating template for package %s" % name) # Create a directory for the new package. - pkg_path = spack.db.filename_for_package_name(name, package_repo) + pkg_path = spack.repo.filename_for_package_name(name, package_repo) if os.path.exists(pkg_path) and not args.force: tty.die("%s already exists." % pkg_path) else: @@ -214,7 +214,7 @@ def create(parser, args): name = 'py-%s' % name # Create a directory for the new package. - pkg_path = spack.db.filename_for_package_name(name) + pkg_path = spack.repo.filename_for_package_name(name) if os.path.exists(pkg_path) and not args.force: tty.die("%s already exists." % pkg_path) else: diff --git a/lib/spack/spack/cmd/diy.py b/lib/spack/spack/cmd/diy.py index 9f8a6d39db..9b8d2e1f6f 100644 --- a/lib/spack/spack/cmd/diy.py +++ b/lib/spack/spack/cmd/diy.py @@ -61,7 +61,7 @@ def diy(self, args): # Take a write lock before checking for existence. with spack.installed_db.write_transaction(): spec = specs[0] - if not spack.db.exists(spec.name): + if not spack.repo.exists(spec.name): tty.warn("No such package: %s" % spec.name) create = tty.get_yes_or_no("Create this package?", default=False) if not create: @@ -76,7 +76,7 @@ def diy(self, args): tty.die("spack diy spec must have a single, concrete version.") spec.concretize() - package = spack.db.get(spec) + package = spack.repo.get(spec) if package.installed: tty.error("Already installed in %s" % package.prefix) diff --git a/lib/spack/spack/cmd/edit.py b/lib/spack/spack/cmd/edit.py index 9081d12516..cb63c6fecd 100644 --- a/lib/spack/spack/cmd/edit.py +++ b/lib/spack/spack/cmd/edit.py @@ -54,7 +54,7 @@ class ${class_name}(Package): def edit_package(name, force=False): - path = spack.db.filename_for_package_name(name) + path = spack.repo.filename_for_package_name(name) if os.path.exists(path): if not os.path.isfile(path): diff --git a/lib/spack/spack/cmd/extensions.py b/lib/spack/spack/cmd/extensions.py index 7cadc424b0..39c8e1f8c0 100644 --- a/lib/spack/spack/cmd/extensions.py +++ b/lib/spack/spack/cmd/extensions.py @@ -71,7 +71,7 @@ def extensions(parser, args): args.mode = 'short' # List package names of extensions - extensions = spack.db.extensions_for(spec) + extensions = spack.repo.extensions_for(spec) if not extensions: tty.msg("%s has no extensions." % spec.cshort_spec) return diff --git a/lib/spack/spack/cmd/fetch.py b/lib/spack/spack/cmd/fetch.py index 0ccebd9486..efaa7acc13 100644 --- a/lib/spack/spack/cmd/fetch.py +++ b/lib/spack/spack/cmd/fetch.py @@ -46,5 +46,5 @@ def fetch(parser, args): specs = spack.cmd.parse_specs(args.packages, concretize=True) for spec in specs: - package = spack.db.get(spec) + package = spack.repo.get(spec) package.do_fetch() diff --git a/lib/spack/spack/cmd/find.py b/lib/spack/spack/cmd/find.py index 0b0dd6ef6f..5c0f2f521e 100644 --- a/lib/spack/spack/cmd/find.py +++ b/lib/spack/spack/cmd/find.py @@ -137,7 +137,7 @@ def find(parser, args): # Filter out specs that don't exist. query_specs = spack.cmd.parse_specs(args.query_specs) query_specs, nonexisting = partition_list( - query_specs, lambda s: spack.db.exists(s.name)) + query_specs, lambda s: spack.repo.exists(s.name)) if nonexisting: msg = "No such package%s: " % ('s' if len(nonexisting) > 1 else '') diff --git a/lib/spack/spack/cmd/info.py b/lib/spack/spack/cmd/info.py index 085e4db44d..2b61850c2d 100644 --- a/lib/spack/spack/cmd/info.py +++ b/lib/spack/spack/cmd/info.py @@ -105,5 +105,5 @@ def print_text_info(pkg): def info(parser, args): - pkg = spack.db.get(args.name) + pkg = spack.repo.get(args.name) print_text_info(pkg) diff --git a/lib/spack/spack/cmd/install.py b/lib/spack/spack/cmd/install.py index 836a6260c8..0942a8e383 100644 --- a/lib/spack/spack/cmd/install.py +++ b/lib/spack/spack/cmd/install.py @@ -70,7 +70,7 @@ def install(parser, args): specs = spack.cmd.parse_specs(args.packages, concretize=True) for spec in specs: - package = spack.db.get(spec) + package = spack.repo.get(spec) with spack.installed_db.write_transaction(): package.do_install( keep_prefix=args.keep_prefix, diff --git a/lib/spack/spack/cmd/list.py b/lib/spack/spack/cmd/list.py index 1f0978a18e..d20aded6ca 100644 --- a/lib/spack/spack/cmd/list.py +++ b/lib/spack/spack/cmd/list.py @@ -43,7 +43,7 @@ def setup_parser(subparser): def list(parser, args): # Start with all package names. - pkgs = spack.db.all_package_names() + pkgs = spack.repo.all_package_names() # filter if a filter arg was provided if args.filter: diff --git a/lib/spack/spack/cmd/location.py b/lib/spack/spack/cmd/location.py index e8e9c3f277..7475c65867 100644 --- a/lib/spack/spack/cmd/location.py +++ b/lib/spack/spack/cmd/location.py @@ -72,7 +72,7 @@ def location(parser, args): print spack.prefix elif args.packages: - print spack.db.root + print spack.repo.root elif args.stages: print spack.stage_path @@ -94,12 +94,12 @@ def location(parser, args): if args.package_dir: # This one just needs the spec name. - print join_path(spack.db.root, spec.name) + print join_path(spack.repo.root, spec.name) else: # These versions need concretized specs. spec.concretize() - pkg = spack.db.get(spec) + pkg = spack.repo.get(spec) if args.stage_dir: print pkg.stage.path diff --git a/lib/spack/spack/cmd/mirror.py b/lib/spack/spack/cmd/mirror.py index 2356170a9a..9a507e69db 100644 --- a/lib/spack/spack/cmd/mirror.py +++ b/lib/spack/spack/cmd/mirror.py @@ -128,7 +128,7 @@ def mirror_create(args): # If nothing is passed, use all packages. if not specs: - specs = [Spec(n) for n in spack.db.all_package_names()] + specs = [Spec(n) for n in spack.repo.all_package_names()] specs.sort(key=lambda s: s.format("$_$@").lower()) # Default name for directory is spack-mirror- diff --git a/lib/spack/spack/cmd/package-list.py b/lib/spack/spack/cmd/package-list.py index f048482845..a14b06bf7f 100644 --- a/lib/spack/spack/cmd/package-list.py +++ b/lib/spack/spack/cmd/package-list.py @@ -48,7 +48,7 @@ def rst_table(elts): def print_rst_package_list(): """Print out information on all packages in restructured text.""" - pkgs = sorted(spack.db.all_packages(), key=lambda s:s.name.lower()) + pkgs = sorted(spack.repo.all_packages(), key=lambda s:s.name.lower()) print ".. _package-list:" print diff --git a/lib/spack/spack/cmd/patch.py b/lib/spack/spack/cmd/patch.py index a6556c4828..600cad87fe 100644 --- a/lib/spack/spack/cmd/patch.py +++ b/lib/spack/spack/cmd/patch.py @@ -47,5 +47,5 @@ def patch(parser, args): specs = spack.cmd.parse_specs(args.packages, concretize=True) for spec in specs: - package = spack.db.get(spec) + package = spack.repo.get(spec) package.do_patch() diff --git a/lib/spack/spack/cmd/pkg.py b/lib/spack/spack/cmd/pkg.py index 055b7c2062..5b70188941 100644 --- a/lib/spack/spack/cmd/pkg.py +++ b/lib/spack/spack/cmd/pkg.py @@ -85,7 +85,7 @@ def list_packages(rev): def pkg_add(args): for pkg_name in args.packages: - filename = spack.db.filename_for_package_name(pkg_name) + filename = spack.repo.filename_for_package_name(pkg_name) if not os.path.isfile(filename): tty.die("No such package: %s. Path does not exist:" % pkg_name, filename) diff --git a/lib/spack/spack/cmd/providers.py b/lib/spack/spack/cmd/providers.py index 2bcdc9fba2..872afa98d5 100644 --- a/lib/spack/spack/cmd/providers.py +++ b/lib/spack/spack/cmd/providers.py @@ -39,4 +39,4 @@ def setup_parser(subparser): def providers(parser, args): for spec in spack.cmd.parse_specs(args.vpkg_spec): - colify(sorted(spack.db.providers_for(spec)), indent=4) + colify(sorted(spack.repo.providers_for(spec)), indent=4) diff --git a/lib/spack/spack/cmd/repo.py b/lib/spack/spack/cmd/repo.py index 87f21b833d..85cc83730c 100644 --- a/lib/spack/spack/cmd/repo.py +++ b/lib/spack/spack/cmd/repo.py @@ -108,7 +108,7 @@ def repo_remove(args): def repo_list(args): """List package sources and their mnemoics""" - root_names = spack.db.repos + root_names = spack.repo.repos max_len = max(len(s[0]) for s in root_names) fmt = "%%-%ds%%s" % (max_len + 4) for root in root_names: diff --git a/lib/spack/spack/cmd/restage.py b/lib/spack/spack/cmd/restage.py index e735a12c32..a9ee26a539 100644 --- a/lib/spack/spack/cmd/restage.py +++ b/lib/spack/spack/cmd/restage.py @@ -42,5 +42,5 @@ def restage(parser, args): specs = spack.cmd.parse_specs(args.packages, concretize=True) for spec in specs: - package = spack.db.get(spec) + package = spack.repo.get(spec) package.do_restage() diff --git a/lib/spack/spack/cmd/stage.py b/lib/spack/spack/cmd/stage.py index f3dc97be17..f734b3dd3b 100644 --- a/lib/spack/spack/cmd/stage.py +++ b/lib/spack/spack/cmd/stage.py @@ -49,5 +49,5 @@ def stage(parser, args): specs = spack.cmd.parse_specs(args.specs, concretize=True) for spec in specs: - package = spack.db.get(spec) + package = spack.repo.get(spec) package.do_stage() diff --git a/lib/spack/spack/cmd/test-install.py b/lib/spack/spack/cmd/test-install.py index 68b761d5dc..7a8921d058 100644 --- a/lib/spack/spack/cmd/test-install.py +++ b/lib/spack/spack/cmd/test-install.py @@ -115,7 +115,7 @@ def fetch_log(path): def failed_dependencies(spec): return set(childSpec for childSpec in spec.dependencies.itervalues() if not - spack.db.get(childSpec).installed) + spack.repo.get(childSpec).installed) def create_test_output(topSpec, newInstalls, output, getLogFunc=fetch_log): @@ -126,7 +126,7 @@ def create_test_output(topSpec, newInstalls, output, getLogFunc=fetch_log): continue failedDeps = failed_dependencies(spec) - package = spack.db.get(spec) + package = spack.repo.get(spec) if failedDeps: result = TestResult.SKIPPED dep = iter(failedDeps).next() @@ -171,7 +171,7 @@ def test_install(parser, args): newInstalls = set() for spec in topSpec.traverse(): - package = spack.db.get(spec) + package = spack.repo.get(spec) if not package.installed: newInstalls.add(spec) @@ -188,7 +188,7 @@ def test_install(parser, args): # Calling do_install for the top-level package would be sufficient but # this attempts to keep going if any package fails (other packages which # are not dependents may succeed) - package = spack.db.get(spec) + package = spack.repo.get(spec) if (not failed_dependencies(spec)) and (not package.installed): try: package.do_install( diff --git a/lib/spack/spack/cmd/urls.py b/lib/spack/spack/cmd/urls.py index 417ce3ab68..14ce4c369f 100644 --- a/lib/spack/spack/cmd/urls.py +++ b/lib/spack/spack/cmd/urls.py @@ -41,7 +41,7 @@ def setup_parser(subparser): def urls(parser, args): urls = set() - for pkg in spack.db.all_packages(): + for pkg in spack.repo.all_packages(): url = getattr(pkg.__class__, 'url', None) if url: urls.add(url) diff --git a/lib/spack/spack/cmd/versions.py b/lib/spack/spack/cmd/versions.py index ed16728261..a5ea5e8f83 100644 --- a/lib/spack/spack/cmd/versions.py +++ b/lib/spack/spack/cmd/versions.py @@ -34,7 +34,7 @@ def setup_parser(subparser): def versions(parser, args): - pkg = spack.db.get(args.package) + pkg = spack.repo.get(args.package) safe_versions = pkg.versions fetched_versions = pkg.fetch_remote_versions() diff --git a/lib/spack/spack/database.py b/lib/spack/spack/database.py index e0c14a0455..8e380083f3 100644 --- a/lib/spack/spack/database.py +++ b/lib/spack/spack/database.py @@ -549,7 +549,7 @@ class Database(object): for key, rec in self._data.items(): if installed is not any and rec.installed != installed: continue - if known is not any and spack.db.exists(rec.spec.name) != known: + if known is not any and spack.repo.exists(rec.spec.name) != known: continue if query_spec is any or rec.spec.satisfies(query_spec): results.append(rec.spec) diff --git a/lib/spack/spack/graph.py b/lib/spack/spack/graph.py index 5fb6a9cd23..0600016a51 100644 --- a/lib/spack/spack/graph.py +++ b/lib/spack/spack/graph.py @@ -523,7 +523,7 @@ def graph_dot(*specs, **kwargs): return '"%s"' % string if not specs: - specs = [p.name for p in spack.db.all_packages()] + specs = [p.name for p in spack.repo.all_packages()] else: roots = specs specs = set() diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 39d71bb4b9..95b74bc961 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -372,7 +372,7 @@ class Package(object): self._total_time = 0.0 if self.is_extension: - spack.db.get(self.extendee_spec)._check_extendable() + spack.repo.get(self.extendee_spec)._check_extendable() @property @@ -548,7 +548,7 @@ class Package(object): yield spec continue - for pkg in spack.db.get(name).preorder_traversal(visited, **kwargs): + for pkg in spack.repo.get(name).preorder_traversal(visited, **kwargs): yield pkg diff --git a/lib/spack/spack/patch.py b/lib/spack/spack/patch.py index fe6e0a65a3..fef42fb5ee 100644 --- a/lib/spack/spack/patch.py +++ b/lib/spack/spack/patch.py @@ -54,7 +54,7 @@ class Patch(object): if '://' in path_or_url: self.url = path_or_url else: - pkg_dir = spack.db.dirname_for_package_name(self.pkg_name) + pkg_dir = spack.repo.dirname_for_package_name(self.pkg_name) self.path = join_path(pkg_dir, path_or_url) if not os.path.isfile(self.path): raise NoSuchPatchFileError(pkg_name, self.path) diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 5e59f240a4..bb0f194c13 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -487,7 +487,7 @@ class Spec(object): @property def package(self): - return spack.db.get(self) + return spack.repo.get(self) @property @@ -505,7 +505,7 @@ class Spec(object): @staticmethod def is_virtual(name): """Test if a name is virtual without requiring a Spec.""" - return not spack.db.exists(name) + return not spack.repo.exists(name) @property @@ -798,7 +798,7 @@ class Spec(object): return changed for spec in virtuals: - providers = spack.db.providers_for(spec) + providers = spack.repo.providers_for(spec) concrete = spack.concretizer.choose_provider(spec, providers) concrete = concrete.copy() spec._replace_with(concrete) @@ -909,7 +909,7 @@ class Spec(object): the dependency. If no conditions are True (and we don't depend on it), return None. """ - pkg = spack.db.get(self.name) + pkg = spack.repo.get(self.name) conditions = pkg.dependencies[name] # evaluate when specs to figure out constraints on the dependency. @@ -1037,7 +1037,7 @@ class Spec(object): any_change = False changed = True - pkg = spack.db.get(self.name) + pkg = spack.repo.get(self.name) while changed: changed = False for dep_name in pkg.dependencies: @@ -1115,7 +1115,7 @@ class Spec(object): for spec in self.traverse(): # Don't get a package for a virtual name. if not spec.virtual: - spack.db.get(spec.name) + spack.repo.get(spec.name) # validate compiler in addition to the package name. if spec.compiler: @@ -1247,7 +1247,7 @@ class Spec(object): # A concrete provider can satisfy a virtual dependency. if not self.virtual and other.virtual: - pkg = spack.db.get(self.name) + pkg = spack.repo.get(self.name) if pkg.provides(other.name): for provided, when_spec in pkg.provided.items(): if self.satisfies(when_spec, deps=False, strict=strict): diff --git a/lib/spack/spack/test/concretize.py b/lib/spack/spack/test/concretize.py index b3a77d076a..8842efe5a0 100644 --- a/lib/spack/spack/test/concretize.py +++ b/lib/spack/spack/test/concretize.py @@ -125,22 +125,22 @@ class ConcretizeTest(MockPackagesTest): we ask for some advanced version. """ self.assertTrue(not any(spec.satisfies('mpich2@:1.0') - for spec in spack.db.providers_for('mpi@2.1'))) + for spec in spack.repo.providers_for('mpi@2.1'))) self.assertTrue(not any(spec.satisfies('mpich2@:1.1') - for spec in spack.db.providers_for('mpi@2.2'))) + for spec in spack.repo.providers_for('mpi@2.2'))) self.assertTrue(not any(spec.satisfies('mpich2@:1.1') - for spec in spack.db.providers_for('mpi@2.2'))) + for spec in spack.repo.providers_for('mpi@2.2'))) self.assertTrue(not any(spec.satisfies('mpich@:1') - for spec in spack.db.providers_for('mpi@2'))) + for spec in spack.repo.providers_for('mpi@2'))) self.assertTrue(not any(spec.satisfies('mpich@:1') - for spec in spack.db.providers_for('mpi@3'))) + for spec in spack.repo.providers_for('mpi@3'))) self.assertTrue(not any(spec.satisfies('mpich2') - for spec in spack.db.providers_for('mpi@3'))) + for spec in spack.repo.providers_for('mpi@3'))) def test_virtual_is_fully_expanded_for_callpath(self): diff --git a/lib/spack/spack/test/database.py b/lib/spack/spack/test/database.py index 8416143f2d..5ce010ae8f 100644 --- a/lib/spack/spack/test/database.py +++ b/lib/spack/spack/test/database.py @@ -79,7 +79,7 @@ class DatabaseTest(MockPackagesTest): def _mock_install(self, spec): s = Spec(spec) - pkg = spack.db.get(s.concretized()) + pkg = spack.repo.get(s.concretized()) pkg.do_install(fake=True) diff --git a/lib/spack/spack/test/directory_layout.py b/lib/spack/spack/test/directory_layout.py index 580620e0e8..ded978b818 100644 --- a/lib/spack/spack/test/directory_layout.py +++ b/lib/spack/spack/test/directory_layout.py @@ -62,7 +62,7 @@ class DirectoryLayoutTest(unittest.TestCase): finally that the directory can be removed by the directory layout. """ - packages = list(spack.db.all_packages())[:max_packages] + packages = list(spack.repo.all_packages())[:max_packages] for pkg in packages: spec = pkg.spec @@ -126,14 +126,14 @@ class DirectoryLayoutTest(unittest.TestCase): mock_db = RepoPath(spack.mock_packages_path) not_in_mock = set.difference( - set(spack.db.all_package_names()), + set(spack.repo.all_package_names()), set(mock_db.all_package_names())) packages = list(not_in_mock)[:max_packages] # Create all the packages that are not in mock. installed_specs = {} for pkg_name in packages: - spec = spack.db.get(pkg_name).spec + spec = spack.repo.get(pkg_name).spec # If a spec fails to concretize, just skip it. If it is a # real error, it will be caught by concretization tests. @@ -145,7 +145,7 @@ class DirectoryLayoutTest(unittest.TestCase): self.layout.create_install_directory(spec) installed_specs[spec] = self.layout.path_for_spec(spec) - spack.db.swap(mock_db) + spack.repo.swap(mock_db) # Now check that even without the package files, we know # enough to read a spec from the spec file. @@ -160,12 +160,12 @@ class DirectoryLayoutTest(unittest.TestCase): self.assertTrue(spec.eq_dag(spec_from_file)) self.assertEqual(spec.dag_hash(), spec_from_file.dag_hash()) - spack.db.swap(mock_db) + spack.repo.swap(mock_db) def test_find(self): """Test that finding specs within an install layout works.""" - packages = list(spack.db.all_packages())[:max_packages] + packages = list(spack.repo.all_packages())[:max_packages] # Create install prefixes for all packages in the list installed_specs = {} diff --git a/lib/spack/spack/test/git_fetch.py b/lib/spack/spack/test/git_fetch.py index 9700bd7533..bf31bd2a5b 100644 --- a/lib/spack/spack/test/git_fetch.py +++ b/lib/spack/spack/test/git_fetch.py @@ -50,7 +50,7 @@ class GitFetchTest(MockPackagesTest): spec = Spec('git-test') spec.concretize() - self.pkg = spack.db.get(spec, new=True) + self.pkg = spack.repo.get(spec, new=True) def tearDown(self): diff --git a/lib/spack/spack/test/hg_fetch.py b/lib/spack/spack/test/hg_fetch.py index 531dfabaa1..559fc2959a 100644 --- a/lib/spack/spack/test/hg_fetch.py +++ b/lib/spack/spack/test/hg_fetch.py @@ -47,7 +47,7 @@ class HgFetchTest(MockPackagesTest): spec = Spec('hg-test') spec.concretize() - self.pkg = spack.db.get(spec, new=True) + self.pkg = spack.repo.get(spec, new=True) def tearDown(self): diff --git a/lib/spack/spack/test/install.py b/lib/spack/spack/test/install.py index 5659e97a4d..b280576c77 100644 --- a/lib/spack/spack/test/install.py +++ b/lib/spack/spack/test/install.py @@ -78,7 +78,7 @@ class InstallTest(MockPackagesTest): self.assertTrue(spec.concrete) # Get the package - pkg = spack.db.get(spec) + pkg = spack.repo.get(spec) # Fake the URL for the package so it downloads from a file. pkg.fetcher = URLFetchStrategy(self.repo.url) diff --git a/lib/spack/spack/test/mirror.py b/lib/spack/spack/test/mirror.py index 89ab14359e..edd075a9a9 100644 --- a/lib/spack/spack/test/mirror.py +++ b/lib/spack/spack/test/mirror.py @@ -55,7 +55,7 @@ class MirrorTest(MockPackagesTest): spec.concretize() # Get the package and fix its fetch args to point to a mock repo - pkg = spack.db.get(spec) + pkg = spack.repo.get(spec) repo = MockRepoClass() self.repos[name] = repo diff --git a/lib/spack/spack/test/mock_packages_test.py b/lib/spack/spack/test/mock_packages_test.py index 8c6273def0..2150b40876 100644 --- a/lib/spack/spack/test/mock_packages_test.py +++ b/lib/spack/spack/test/mock_packages_test.py @@ -37,7 +37,7 @@ class MockPackagesTest(unittest.TestCase): # us to set up contrived packages that don't interfere with # real ones. self.db = RepoPath(spack.mock_packages_path) - spack.db.swap(self.db) + spack.repo.swap(self.db) spack.config.clear_config_caches() self.real_scopes = spack.config.config_scopes @@ -59,7 +59,7 @@ class MockPackagesTest(unittest.TestCase): spec = Spec(spec) # Save original dependencies before making any changes. - pkg = spack.db.get(pkg_name) + pkg = spack.repo.get(pkg_name) if pkg_name not in self.saved_deps: self.saved_deps[pkg_name] = (pkg, pkg.dependencies.copy()) @@ -69,7 +69,7 @@ class MockPackagesTest(unittest.TestCase): def cleanmock(self): """Restore the real packages path after any test.""" - spack.db.swap(self.db) + spack.repo.swap(self.db) spack.config.config_scopes = self.real_scopes spack.config.clear_config_caches() diff --git a/lib/spack/spack/test/multimethod.py b/lib/spack/spack/test/multimethod.py index cd5d9e625e..56e8b3f343 100644 --- a/lib/spack/spack/test/multimethod.py +++ b/lib/spack/spack/test/multimethod.py @@ -38,92 +38,92 @@ from spack.test.mock_packages_test import * class MultiMethodTest(MockPackagesTest): def test_no_version_match(self): - pkg = spack.db.get('multimethod@2.0') + pkg = spack.repo.get('multimethod@2.0') self.assertRaises(NoSuchMethodError, pkg.no_version_2) def test_one_version_match(self): - pkg = spack.db.get('multimethod@1.0') + pkg = spack.repo.get('multimethod@1.0') self.assertEqual(pkg.no_version_2(), 1) - pkg = spack.db.get('multimethod@3.0') + pkg = spack.repo.get('multimethod@3.0') self.assertEqual(pkg.no_version_2(), 3) - pkg = spack.db.get('multimethod@4.0') + pkg = spack.repo.get('multimethod@4.0') self.assertEqual(pkg.no_version_2(), 4) def test_version_overlap(self): - pkg = spack.db.get('multimethod@2.0') + pkg = spack.repo.get('multimethod@2.0') self.assertEqual(pkg.version_overlap(), 1) - pkg = spack.db.get('multimethod@5.0') + pkg = spack.repo.get('multimethod@5.0') self.assertEqual(pkg.version_overlap(), 2) def test_mpi_version(self): - pkg = spack.db.get('multimethod^mpich@3.0.4') + pkg = spack.repo.get('multimethod^mpich@3.0.4') self.assertEqual(pkg.mpi_version(), 3) - pkg = spack.db.get('multimethod^mpich2@1.2') + pkg = spack.repo.get('multimethod^mpich2@1.2') self.assertEqual(pkg.mpi_version(), 2) - pkg = spack.db.get('multimethod^mpich@1.0') + pkg = spack.repo.get('multimethod^mpich@1.0') self.assertEqual(pkg.mpi_version(), 1) def test_undefined_mpi_version(self): - pkg = spack.db.get('multimethod^mpich@0.4') + pkg = spack.repo.get('multimethod^mpich@0.4') self.assertEqual(pkg.mpi_version(), 1) - pkg = spack.db.get('multimethod^mpich@1.4') + pkg = spack.repo.get('multimethod^mpich@1.4') self.assertEqual(pkg.mpi_version(), 1) def test_default_works(self): - pkg = spack.db.get('multimethod%gcc') + pkg = spack.repo.get('multimethod%gcc') self.assertEqual(pkg.has_a_default(), 'gcc') - pkg = spack.db.get('multimethod%intel') + pkg = spack.repo.get('multimethod%intel') self.assertEqual(pkg.has_a_default(), 'intel') - pkg = spack.db.get('multimethod%pgi') + pkg = spack.repo.get('multimethod%pgi') self.assertEqual(pkg.has_a_default(), 'default') def test_architecture_match(self): - pkg = spack.db.get('multimethod=x86_64') + pkg = spack.repo.get('multimethod=x86_64') self.assertEqual(pkg.different_by_architecture(), 'x86_64') - pkg = spack.db.get('multimethod=ppc64') + pkg = spack.repo.get('multimethod=ppc64') self.assertEqual(pkg.different_by_architecture(), 'ppc64') - pkg = spack.db.get('multimethod=ppc32') + pkg = spack.repo.get('multimethod=ppc32') self.assertEqual(pkg.different_by_architecture(), 'ppc32') - pkg = spack.db.get('multimethod=arm64') + pkg = spack.repo.get('multimethod=arm64') self.assertEqual(pkg.different_by_architecture(), 'arm64') - pkg = spack.db.get('multimethod=macos') + pkg = spack.repo.get('multimethod=macos') self.assertRaises(NoSuchMethodError, pkg.different_by_architecture) def test_dependency_match(self): - pkg = spack.db.get('multimethod^zmpi') + pkg = spack.repo.get('multimethod^zmpi') self.assertEqual(pkg.different_by_dep(), 'zmpi') - pkg = spack.db.get('multimethod^mpich') + pkg = spack.repo.get('multimethod^mpich') self.assertEqual(pkg.different_by_dep(), 'mpich') # If we try to switch on some entirely different dep, it's ambiguous, # but should take the first option - pkg = spack.db.get('multimethod^foobar') + pkg = spack.repo.get('multimethod^foobar') self.assertEqual(pkg.different_by_dep(), 'mpich') def test_virtual_dep_match(self): - pkg = spack.db.get('multimethod^mpich2') + pkg = spack.repo.get('multimethod^mpich2') self.assertEqual(pkg.different_by_virtual_dep(), 2) - pkg = spack.db.get('multimethod^mpich@1.0') + pkg = spack.repo.get('multimethod^mpich@1.0') self.assertEqual(pkg.different_by_virtual_dep(), 1) diff --git a/lib/spack/spack/test/package_sanity.py b/lib/spack/spack/test/package_sanity.py index 0a132fd701..b2d3da91b1 100644 --- a/lib/spack/spack/test/package_sanity.py +++ b/lib/spack/spack/test/package_sanity.py @@ -35,8 +35,8 @@ class PackageSanityTest(unittest.TestCase): def check_db(self): """Get all packages in a DB to make sure they work.""" - for name in spack.db.all_package_names(): - spack.db.get(name) + for name in spack.repo.all_package_names(): + spack.repo.get(name) def test_get_all_packages(self): @@ -47,14 +47,14 @@ class PackageSanityTest(unittest.TestCase): def ztest_get_all_mock_packages(self): """Get the mock packages once each too.""" db = RepoPath(spack.mock_packages_path) - spack.db.swap(db) + spack.repo.swap(db) self.check_db() - spack.db.swap(db) + spack.repo.swap(db) def ztest_url_versions(self): """Check URLs for regular packages, if they are explicitly defined.""" - for pkg in spack.db.all_packages(): + for pkg in spack.repo.all_packages(): for v, vattrs in pkg.versions.items(): if 'url' in vattrs: # If there is a url for the version check it. diff --git a/lib/spack/spack/test/packages.py b/lib/spack/spack/test/packages.py index 8a786e364e..e39def2ff2 100644 --- a/lib/spack/spack/test/packages.py +++ b/lib/spack/spack/test/packages.py @@ -35,11 +35,11 @@ from spack.test.mock_packages_test import * class PackagesTest(MockPackagesTest): def test_load_package(self): - pkg = spack.db.get('mpich') + pkg = spack.repo.get('mpich') def test_package_name(self): - pkg = spack.db.get('mpich') + pkg = spack.repo.get('mpich') self.assertEqual(pkg.name, 'mpich') @@ -50,7 +50,7 @@ class PackagesTest(MockPackagesTest): def test_package_name(self): - pkg = spack.db.get('mpich') + pkg = spack.repo.get('mpich') self.assertEqual(pkg.name, 'mpich') diff --git a/lib/spack/spack/test/python_version.py b/lib/spack/spack/test/python_version.py index 5779d31ed2..d2f7ea45b2 100644 --- a/lib/spack/spack/test/python_version.py +++ b/lib/spack/spack/test/python_version.py @@ -54,8 +54,8 @@ class PythonVersionTest(unittest.TestCase): def package_py_files(self): - for name in spack.db.all_package_names(): - yield spack.db.filename_for_package_name(name) + for name in spack.repo.all_package_names(): + yield spack.repo.filename_for_package_name(name) def check_python_versions(self, *files): diff --git a/lib/spack/spack/test/svn_fetch.py b/lib/spack/spack/test/svn_fetch.py index 184fe8faa1..8abd089847 100644 --- a/lib/spack/spack/test/svn_fetch.py +++ b/lib/spack/spack/test/svn_fetch.py @@ -49,7 +49,7 @@ class SvnFetchTest(MockPackagesTest): spec = Spec('svn-test') spec.concretize() - self.pkg = spack.db.get(spec, new=True) + self.pkg = spack.repo.get(spec, new=True) def tearDown(self): diff --git a/lib/spack/spack/test/unit_install.py b/lib/spack/spack/test/unit_install.py index 7168272997..477ca0436d 100644 --- a/lib/spack/spack/test/unit_install.py +++ b/lib/spack/spack/test/unit_install.py @@ -79,15 +79,15 @@ class UnitInstallTest(unittest.TestCase): pkgX.installed = False pkgY.installed = False - self.saved_db = spack.db + self.saved_db = spack.repo pkgDb = MockPackageDb({specX:pkgX, specY:pkgY}) - spack.db = pkgDb + spack.repo = pkgDb def tearDown(self): super(UnitInstallTest, self).tearDown() - spack.db = self.saved_db + spack.repo = self.saved_db def test_installing_both(self): mo = MockOutput() -- cgit v1.2.3-70-g09d2 From 60f7756626271b46ee7d32a80ba1e78b337d6f91 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Wed, 25 Nov 2015 12:28:43 -0800 Subject: Executable.__call__ original semantics were to avoid redirecting process output to the output stream if return_output was set. This branch undid that and this commit restores those semantics. --- lib/spack/spack/util/executable.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/util/executable.py b/lib/spack/spack/util/executable.py index 6848a7e2a5..6088d4514e 100644 --- a/lib/spack/spack/util/executable.py +++ b/lib/spack/spack/util/executable.py @@ -100,7 +100,8 @@ class Executable(object): out, err = proc.communicate() self.returncode = proc.returncode - output.write(out) + if not return_output: + output.write(out) error.write(err) rc = proc.returncode -- cgit v1.2.3-70-g09d2 From d50a18d9ebe38137690dfd0a6972616c0e07271c Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Wed, 25 Nov 2015 17:10:45 -0800 Subject: XML output for unit tests is now enabled with an option (disabled by default) --- lib/spack/spack/cmd/test.py | 21 ++++++++++++++------- lib/spack/spack/test/__init__.py | 12 +++++++----- lib/spack/spack/test/tally_plugin.py | 1 + 3 files changed, 22 insertions(+), 12 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/cmd/test.py b/lib/spack/spack/cmd/test.py index 001ebfaef0..f93d126851 100644 --- a/lib/spack/spack/cmd/test.py +++ b/lib/spack/spack/cmd/test.py @@ -39,9 +39,12 @@ def setup_parser(subparser): 'names', nargs='*', help="Names of tests to run.") subparser.add_argument( '-l', '--list', action='store_true', dest='list', help="Show available tests") - # TODO: make XML output optional subparser.add_argument( - '-d', '--outputDir', dest='outputDir', help="Nose creates XML files in this directory") + '--createXmlOutput', action='store_true', dest='createXmlOutput', + help="Create JUnit XML from test results") + subparser.add_argument( + '--xmlOutputDir', dest='xmlOutputDir', + help="Nose creates XML files in this directory") subparser.add_argument( '-v', '--verbose', action='store_true', dest='verbose', help="verbose output") @@ -53,10 +56,14 @@ def test(parser, args): colify(spack.test.list_tests(), indent=2) else: - if not args.outputDir: - outputDir = join_path(os.getcwd(), "test-output") + if not args.createXmlOutput: + outputDir = None else: - outputDir = args.outputDir - if not os.path.exists(outputDir): - mkdirp(outputDir) + if not args.xmlOutputDir: + outputDir = join_path(os.getcwd(), "test-output") + else: + outputDir = os.path.abspath(args.xmlOutputDir) + + if not os.path.exists(outputDir): + mkdirp(outputDir) spack.test.run(args.names, outputDir, args.verbose) diff --git a/lib/spack/spack/test/__init__.py b/lib/spack/spack/test/__init__.py index 158e8bdfcb..cb75f15ca1 100644 --- a/lib/spack/spack/test/__init__.py +++ b/lib/spack/spack/test/__init__.py @@ -92,11 +92,13 @@ def run(names, outputDir, verbose=False): tty.msg("Running test: %s" % test) - xmlOutputFname = "unittests-{0}.xml".format(test) - xmlOutputPath = join_path(outputDir, xmlOutputFname) - runOpts = ["--with-%s" % spack.test.tally_plugin.Tally.name, - "--with-xunit", - "--xunit-file={0}".format(xmlOutputPath)] + runOpts = ["--with-%s" % spack.test.tally_plugin.Tally.name] + + if outputDir: + xmlOutputFname = "unittests-{0}.xml".format(test) + xmlOutputPath = join_path(outputDir, xmlOutputFname) + runOpts += ["--with-xunit", + "--xunit-file={0}".format(xmlOutputPath)] argv = [""] + runOpts + [module] result = nose.run(argv=argv, addplugins=[tally]) diff --git a/lib/spack/spack/test/tally_plugin.py b/lib/spack/spack/test/tally_plugin.py index e314fae6ce..c167d58529 100644 --- a/lib/spack/spack/test/tally_plugin.py +++ b/lib/spack/spack/test/tally_plugin.py @@ -35,6 +35,7 @@ class Tally(Plugin): self.failCount = 0 self.errorCount = 0 + # TODO: this doesn't account for the possibility of skipped tests @property def numberOfTests(self): return self.errorCount + self.failCount + self.successCount -- cgit v1.2.3-70-g09d2 From fc96f62794a5c718afb5904d93bb9370ed4a261d Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Wed, 25 Nov 2015 18:08:17 -0800 Subject: Updated documentation/naming in tally_plugin to reflect that the count of the total number of tests run excludes skipped tests --- lib/spack/spack/test/__init__.py | 2 +- lib/spack/spack/test/tally_plugin.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/test/__init__.py b/lib/spack/spack/test/__init__.py index cb75f15ca1..7da3895cec 100644 --- a/lib/spack/spack/test/__init__.py +++ b/lib/spack/spack/test/__init__.py @@ -104,7 +104,7 @@ def run(names, outputDir, verbose=False): succeeded = not tally.failCount and not tally.errorCount tty.msg("Tests Complete.", - "%5d tests run" % tally.numberOfTests, + "%5d tests run" % tally.numberOfTestsRun, "%5d failures" % tally.failCount, "%5d errors" % tally.errorCount) diff --git a/lib/spack/spack/test/tally_plugin.py b/lib/spack/spack/test/tally_plugin.py index c167d58529..9ca898c47c 100644 --- a/lib/spack/spack/test/tally_plugin.py +++ b/lib/spack/spack/test/tally_plugin.py @@ -35,9 +35,9 @@ class Tally(Plugin): self.failCount = 0 self.errorCount = 0 - # TODO: this doesn't account for the possibility of skipped tests @property - def numberOfTests(self): + def numberOfTestsRun(self): + """Excludes skipped tests""" return self.errorCount + self.failCount + self.successCount def options(self, parser, env=os.environ): -- cgit v1.2.3-70-g09d2 From 1fe626ec7cd8b8a2bceb9e73dd9597d9f99813cf Mon Sep 17 00:00:00 2001 From: Massimiliano Culpo Date: Thu, 26 Nov 2015 17:53:33 +0100 Subject: resource directive : sketch of implementation + clang / llvm use case --- lib/spack/llnl/util/lang.py | 2 +- lib/spack/spack/directives.py | 36 ++++++++++++++++++++++---- lib/spack/spack/package.py | 51 +++++++++++++++++++++++++++++++------ lib/spack/spack/resource.py | 37 +++++++++++++++++++++++++++ var/spack/packages/clang/package.py | 7 +++++ var/spack/packages/llvm/package.py | 15 +++++++++-- 6 files changed, 132 insertions(+), 16 deletions(-) create mode 100644 lib/spack/spack/resource.py (limited to 'lib') diff --git a/lib/spack/llnl/util/lang.py b/lib/spack/llnl/util/lang.py index 156ee34c9e..e726368794 100644 --- a/lib/spack/llnl/util/lang.py +++ b/lib/spack/llnl/util/lang.py @@ -112,7 +112,7 @@ def partition_list(elements, predicate): def caller_locals(): """This will return the locals of the *parent* of the caller. - This allows a fucntion to insert variables into its caller's + This allows a function to insert variables into its caller's scope. Yes, this is some black magic, and yes it's useful for implementing things like depends_on and provides. """ diff --git a/lib/spack/spack/directives.py b/lib/spack/spack/directives.py index 78039ac6f9..4d8333fd7d 100644 --- a/lib/spack/spack/directives.py +++ b/lib/spack/spack/directives.py @@ -44,11 +44,12 @@ The available directives are: * ``variant`` """ -__all__ = [ 'depends_on', 'extends', 'provides', 'patch', 'version', - 'variant' ] +__all__ = ['depends_on', 'extends', 'provides', 'patch', 'version', + 'variant', 'resource'] import re import inspect +import functools from llnl.util.lang import * @@ -60,7 +61,8 @@ from spack.version import Version from spack.patch import Patch from spack.variant import Variant from spack.spec import Spec, parse_anonymous_spec - +from spack.resource import Resource +from spack.fetch_strategy import URLFetchStrategy # # This is a list of all directives, built up as they are defined in @@ -79,8 +81,8 @@ class directive(object): """Decorator for Spack directives. Spack directives allow you to modify a package while it is being - defined, e.g. to add version or depenency information. Directives - are one of the key pieces of Spack's package "langauge", which is + defined, e.g. to add version or dependency information. Directives + are one of the key pieces of Spack's package "language", which is embedded in python. Here's an example directive: @@ -141,6 +143,7 @@ class directive(object): def __call__(self, directive_function): directives[directive_function.__name__] = self + @functools.wraps(directive_function) def wrapped(*args, **kwargs): pkg = DictWrapper(caller_locals()) self.ensure_dicts(pkg) @@ -259,6 +262,29 @@ def variant(pkg, name, default=False, description=""): pkg.variants[name] = Variant(default, description) +@directive('resources') +def resource(pkg, **kwargs): + """ + Define an external resource to be fetched and staged when building the package. Based on the keywords present in the + dictionary the appropriate FetchStrategy will be used for the resource. + + List of recognized keywords: + + * 'when' : represents the condition upon which the resource is needed (optional) + * 'destination' : path where to extract / checkout the resource (optional) + + """ + when = kwargs.get('when', pkg.name) + # FIXME : currently I assume destination to be a relative path (rooted at pkg.stage.source_path) + destination = kwargs.get('destination', "") + when_spec = parse_anonymous_spec(when, pkg.name) + resources = pkg.resources.setdefault(when_spec, []) + # FIXME : change URLFetchStrategy with a factory that selects based on kwargs + fetcher = URLFetchStrategy(**kwargs) + # FIXME : should we infer the name somehow if not passed ? + name = kwargs.get('name') + resources.append(Resource(name, fetcher, destination)) + class DirectiveError(spack.error.SpackError): """This is raised when something is wrong with a package directive.""" def __init__(self, directive, message): diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index c631a35bf3..9fee962df3 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -639,26 +639,53 @@ class Package(object): "Will not fetch %s." % self.spec.format('$_$@'), checksum_msg) self.stage.fetch() + + ########## + # Fetch resources + resources = self._get_resources() + # FIXME : choose the unique name appropriately. Is there a function somewhere for the base name ? + pieces = [self.name, str(self.version), self.spec.dag_hash()] + for resource in resources: + resource_stage_folder = '-'.join(pieces + [resource.name]) + stage = Stage(resource.fetcher, name=resource_stage_folder) + resource.fetcher.set_stage(stage) + resource.fetcher.fetch() + ########## + self._fetch_time = time.time() - start_time if spack.do_checksum and self.version in self.versions: self.stage.check() - def do_stage(self): """Unpacks the fetched tarball, then changes into the expanded tarball directory.""" if not self.spec.concrete: raise ValueError("Can only stage concrete packages.") - self.do_fetch() + def _expand_archive(stage, name=self.name): + archive_dir = stage.source_path + if not archive_dir: + stage.expand_archive() + tty.msg("Created stage in %s." % stage.path) + else: + tty.msg("Already staged %s in %s." % (name, stage.path)) - archive_dir = self.stage.source_path - if not archive_dir: - self.stage.expand_archive() - tty.msg("Created stage in %s." % self.stage.path) - else: - tty.msg("Already staged %s in %s." % (self.name, self.stage.path)) + + self.do_fetch() + _expand_archive(self.stage) + + ########## + # Stage resources in appropriate path + resources = self._get_resources() + for resource in resources: + stage = resource.fetcher.stage + _expand_archive(stage, resource.name) + link_path = join_path(self.stage.source_path, resource.destination, os.path.basename(stage.source_path)) + if not os.path.exists(link_path): + # Create a symlink + os.symlink(stage.source_path, link_path) + ########## self.stage.chdir_to_source() @@ -730,6 +757,14 @@ class Package(object): mkdirp(self.prefix.man1) + def _get_resources(self): + resources = [] + # Select the resources that are needed for this build + for when_spec, resource_list in self.resources.items(): + if when_spec in self.spec: + resources.extend(resource_list) + return resources + def _build_logger(self, log_path): """Create a context manager to log build output.""" diff --git a/lib/spack/spack/resource.py b/lib/spack/spack/resource.py new file mode 100644 index 0000000000..00ee2d49e4 --- /dev/null +++ b/lib/spack/spack/resource.py @@ -0,0 +1,37 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +""" +Describes an optional resource needed for a build. Typically a bunch of sources that can be built in-tree within another +package to enable optional features. +""" + +class Resource(object): + """ + Represents an optional resource. Aggregates a name, a fetcher and a destination. + """ + def __init__(self, name, fetcher, destination): + self.name = name + self.fetcher = fetcher + self.destination = destination diff --git a/var/spack/packages/clang/package.py b/var/spack/packages/clang/package.py index 4f977bf9a4..d0869362f5 100644 --- a/var/spack/packages/clang/package.py +++ b/var/spack/packages/clang/package.py @@ -39,6 +39,13 @@ class Clang(Package): version('3.6.2', 'ff862793682f714bb7862325b9c06e20', url='http://llvm.org/releases/3.6.2/cfe-3.6.2.src.tar.xz') version('3.5.1', '93f9532f8f7e6f1d8e5c1116907051cb', url='http://llvm.org/releases/3.5.1/cfe-3.5.1.src.tar.xz') + ########## + # @3.7.0 + resource(name='clang-tools-extra', + url='http://llvm.org/releases/3.7.0/clang-tools-extra-3.7.0.src.tar.xz', + md5='d5a87dacb65d981a427a536f6964642e', destination='tools', when='@3.7.0') + ########## + def install(self, spec, prefix): env['CXXFLAGS'] = self.compiler.cxx11_flag diff --git a/var/spack/packages/llvm/package.py b/var/spack/packages/llvm/package.py index a6759c3033..b68aa82aff 100644 --- a/var/spack/packages/llvm/package.py +++ b/var/spack/packages/llvm/package.py @@ -41,13 +41,24 @@ class Llvm(Package): depends_on('python@2.7:') + ########## + # @3.7.0 + # TODO : Add support for libc++ <- libc++ABI <- libunwind with variant? + resource(name='compiler-rt', + url='http://llvm.org/releases/3.7.0/compiler-rt-3.7.0.src.tar.xz', md5='383c10affd513026f08936b5525523f5', + destination='projects', when='@3.7.0') + resource(name='openmp', + url='http://llvm.org/releases/3.7.0/openmp-3.7.0.src.tar.xz', md5='f482c86fdead50ba246a1a2b0bbf206f', + destination='projects', when='@3.7.0') + ########## + def install(self, spec, prefix): env['CXXFLAGS'] = self.compiler.cxx11_flag with working_dir('spack-build', create=True): cmake('..', - '-DLLVM_REQUIRES_RTTI=1', - '-DPYTHON_EXECUTABLE=%s/bin/python' % spec['python'].prefix, + '-DLLVM_REQUIRES_RTTI:BOOL=ON', + '-DPYTHON_EXECUTABLE:PATH=%s/bin/python' % spec['python'].prefix, *std_cmake_args) make() make("install") -- cgit v1.2.3-70-g09d2 From 89d5127900dda96b2a583c4c1a9bdac8e51c1c15 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Thu, 26 Nov 2015 14:19:27 -0800 Subject: New, cleaner package repository structure. Package repositories now look like this: top-level-dir/ repo.yaml packages/ libelf/ package.py mpich/ package.py ... This leaves room at the top level for additional metadata, source, per-repo configs, indexes, etc., and it makes it easy to see that something is a spack repo (just look for repo.yaml and packages). --- lib/spack/spack/__init__.py | 15 +- lib/spack/spack/cmd/repo.py | 2 +- lib/spack/spack/repository.py | 178 ++- var/spack/mock_packages/_repo.yaml | 2 - var/spack/mock_packages/a/package.py | 12 - var/spack/mock_packages/b/package.py | 12 - var/spack/mock_packages/c/package.py | 12 - var/spack/mock_packages/callpath/package.py | 41 - var/spack/mock_packages/direct_mpich/package.py | 36 - var/spack/mock_packages/dyninst/package.py | 44 - var/spack/mock_packages/e/package.py | 12 - var/spack/mock_packages/fake/package.py | 34 - var/spack/mock_packages/git-test/package.py | 10 - var/spack/mock_packages/hg-test/package.py | 10 - var/spack/mock_packages/indirect_mpich/package.py | 41 - var/spack/mock_packages/libdwarf/package.py | 44 - var/spack/mock_packages/libelf/package.py | 43 - var/spack/mock_packages/mpich/package.py | 46 - var/spack/mock_packages/mpich2/package.py | 47 - var/spack/mock_packages/mpileaks/package.py | 43 - var/spack/mock_packages/multimethod/package.py | 143 -- .../mock_packages/optional-dep-test-2/package.py | 18 - .../mock_packages/optional-dep-test-3/package.py | 17 - .../mock_packages/optional-dep-test/package.py | 29 - var/spack/mock_packages/svn-test/package.py | 10 - .../trivial_install_test_package/package.py | 38 - var/spack/mock_packages/zmpi/package.py | 39 - var/spack/packages/ImageMagick/package.py | 37 - var/spack/packages/Mitos/package.py | 19 - var/spack/packages/R/package.py | 33 - var/spack/packages/SAMRAI/no-tool-build.patch | 20 - var/spack/packages/SAMRAI/package.py | 53 - var/spack/packages/_repo.yaml | 2 - var/spack/packages/activeharmony/package.py | 15 - var/spack/packages/adept-utils/package.py | 42 - var/spack/packages/apex/package.py | 34 - var/spack/packages/arpack/package.py | 41 - var/spack/packages/asciidoc/package.py | 18 - var/spack/packages/atk/package.py | 18 - var/spack/packages/atlas/package.py | 60 - var/spack/packages/autoconf/package.py | 14 - var/spack/packages/automaded/package.py | 51 - var/spack/packages/automake/package.py | 16 - var/spack/packages/bear/package.py | 17 - var/spack/packages/bib2xhtml/package.py | 27 - var/spack/packages/binutils/package.py | 30 - var/spack/packages/bison/package.py | 17 - var/spack/packages/boost/package.py | 66 - var/spack/packages/bowtie2/bowtie2-2.5.patch | 16 - var/spack/packages/bowtie2/package.py | 24 - var/spack/packages/boxlib/package.py | 25 - var/spack/packages/bzip2/package.py | 36 - var/spack/packages/cairo/package.py | 19 - var/spack/packages/callpath/package.py | 47 - var/spack/packages/cblas/package.py | 35 - var/spack/packages/cgm/package.py | 30 - var/spack/packages/clang/package.py | 51 - var/spack/packages/cloog/package.py | 26 - var/spack/packages/cmake/package.py | 45 - var/spack/packages/coreutils/package.py | 17 - var/spack/packages/cppcheck/package.py | 15 - var/spack/packages/cram/package.py | 15 - var/spack/packages/cscope/package.py | 17 - var/spack/packages/cube/package.py | 55 - var/spack/packages/czmq/package.py | 19 - var/spack/packages/dbus/package.py | 31 - var/spack/packages/docbook-xml/package.py | 19 - var/spack/packages/doxygen/package.py | 25 - var/spack/packages/dri2proto/package.py | 14 - var/spack/packages/dtcmp/package.py | 20 - var/spack/packages/dyninst/package.py | 68 - var/spack/packages/elfutils/package.py | 26 - var/spack/packages/extrae/package.py | 46 - var/spack/packages/exuberant-ctags/package.py | 14 - var/spack/packages/fish/package.py | 18 - var/spack/packages/flex/package.py | 15 - var/spack/packages/flux/package.py | 36 - var/spack/packages/fontconfig/package.py | 16 - var/spack/packages/freetype/package.py | 16 - var/spack/packages/gasnet/package.py | 35 - var/spack/packages/gcc/package.py | 122 -- var/spack/packages/gdk-pixbuf/package.py | 22 - var/spack/packages/geos/package.py | 31 - var/spack/packages/gflags/package.py | 21 - var/spack/packages/ghostscript/package.py | 17 - var/spack/packages/git/package.py | 27 - var/spack/packages/glib/package.py | 18 - var/spack/packages/glm/package.py | 19 - var/spack/packages/global/package.py | 24 - var/spack/packages/glog/package.py | 15 - var/spack/packages/gmp/package.py | 40 - var/spack/packages/gnutls/package.py | 22 - var/spack/packages/gperf/package.py | 19 - var/spack/packages/gperftools/package.py | 38 - var/spack/packages/graphlib/package.py | 14 - var/spack/packages/graphviz/package.py | 21 - var/spack/packages/gtkplus/package.py | 22 - var/spack/packages/harfbuzz/package.py | 20 - var/spack/packages/hdf5/package.py | 42 - var/spack/packages/hwloc/package.py | 25 - var/spack/packages/hypre/package.py | 32 - var/spack/packages/icu/package.py | 25 - var/spack/packages/icu4c/package.py | 17 - var/spack/packages/isl/package.py | 17 - var/spack/packages/jdk/package.py | 46 - var/spack/packages/jpeg/package.py | 14 - var/spack/packages/launchmon/package.py | 47 - .../packages/launchmon/patch.lmon_install_dir | 147 -- var/spack/packages/lcms/package.py | 19 - var/spack/packages/leveldb/package.py | 29 - var/spack/packages/libNBC/package.py | 43 - var/spack/packages/libarchive/package.py | 16 - var/spack/packages/libcircle/package.py | 18 - var/spack/packages/libdrm/package.py | 18 - var/spack/packages/libdwarf/package.py | 81 - var/spack/packages/libelf/package.py | 49 - var/spack/packages/libevent/package.py | 30 - var/spack/packages/libffi/package.py | 17 - var/spack/packages/libgcrypt/package.py | 19 - var/spack/packages/libgpg-error/package.py | 17 - var/spack/packages/libjpeg-turbo/package.py | 20 - var/spack/packages/libjson-c/package.py | 14 - var/spack/packages/libmng/package.py | 23 - var/spack/packages/libmonitor/package.py | 36 - var/spack/packages/libpciaccess/package.py | 21 - var/spack/packages/libpng/package.py | 15 - var/spack/packages/libsodium/package.py | 19 - var/spack/packages/libtiff/package.py | 18 - var/spack/packages/libtool/package.py | 14 - var/spack/packages/libunwind/package.py | 38 - var/spack/packages/libuuid/package.py | 16 - var/spack/packages/libxcb/package.py | 21 - var/spack/packages/libxml2/package.py | 20 - var/spack/packages/libxshmfence/package.py | 16 - var/spack/packages/libxslt/package.py | 24 - var/spack/packages/llvm-lld/package.py | 46 - var/spack/packages/llvm/package.py | 53 - var/spack/packages/lmdb/package.py | 39 - var/spack/packages/lua/package.py | 26 - var/spack/packages/lwgrp/package.py | 18 - var/spack/packages/lwm2/package.py | 18 - var/spack/packages/matio/package.py | 15 - var/spack/packages/memaxes/package.py | 19 - var/spack/packages/mesa/package.py | 34 - var/spack/packages/metis/package.py | 27 - var/spack/packages/mpc/package.py | 42 - var/spack/packages/mpe2/mpe2.patch | 12 - var/spack/packages/mpe2/package.py | 28 - var/spack/packages/mpfr/package.py | 41 - var/spack/packages/mpibash/mpibash-4.3.patch | 1565 -------------------- var/spack/packages/mpibash/package.py | 32 - var/spack/packages/mpich/package.py | 92 -- var/spack/packages/mpileaks/package.py | 44 - var/spack/packages/mrnet/package.py | 20 - var/spack/packages/munge/package.py | 20 - var/spack/packages/muster/package.py | 22 - .../mvapich2/ad_lustre_rwcontig_open_source.patch | 11 - var/spack/packages/mvapich2/package.py | 104 -- var/spack/packages/nasm/package.py | 14 - var/spack/packages/ncdu/package.py | 28 - var/spack/packages/ncurses/package.py | 33 - var/spack/packages/netcdf/netcdf-4.3.3-mpi.patch | 25 - var/spack/packages/netcdf/package.py | 27 - var/spack/packages/netgauge/package.py | 43 - var/spack/packages/netlib-blas/package.py | 46 - var/spack/packages/netlib-lapack/package.py | 59 - var/spack/packages/nettle/package.py | 17 - var/spack/packages/ompss/package.py | 50 - var/spack/packages/ompt-openmp/package.py | 23 - var/spack/packages/opari2/package.py | 65 - .../openmpi/ad_lustre_rwcontig_open_source.patch | 11 - var/spack/packages/openmpi/llnl-platforms.patch | 151 -- var/spack/packages/openmpi/package.py | 109 -- var/spack/packages/openssl/package.py | 26 - var/spack/packages/otf/package.py | 21 - var/spack/packages/otf2/package.py | 74 - var/spack/packages/pango/package.py | 19 - var/spack/packages/papi/package.py | 35 - var/spack/packages/paraver/package.py | 41 - var/spack/packages/paraview/package.py | 72 - var/spack/packages/parmetis/package.py | 26 - var/spack/packages/parpack/package.py | 43 - var/spack/packages/pcre/package.py | 15 - var/spack/packages/petsc/package.py | 40 - var/spack/packages/pidx/package.py | 21 - var/spack/packages/pixman/package.py | 18 - var/spack/packages/pkg-config/package.py | 17 - var/spack/packages/pmgr_collective/package.py | 37 - var/spack/packages/postgresql/package.py | 20 - var/spack/packages/ppl/package.py | 28 - var/spack/packages/protobuf/package.py | 16 - var/spack/packages/py-basemap/package.py | 20 - var/spack/packages/py-biopython/package.py | 15 - var/spack/packages/py-cffi/package.py | 17 - var/spack/packages/py-cython/package.py | 14 - var/spack/packages/py-dateutil/package.py | 16 - var/spack/packages/py-epydoc/package.py | 13 - var/spack/packages/py-genders/package.py | 15 - var/spack/packages/py-gnuplot/package.py | 14 - var/spack/packages/py-h5py/package.py | 19 - var/spack/packages/py-ipython/package.py | 16 - var/spack/packages/py-libxml2/package.py | 15 - var/spack/packages/py-lockfile/package.py | 23 - var/spack/packages/py-mako/package.py | 16 - var/spack/packages/py-matplotlib/package.py | 47 - var/spack/packages/py-mock/package.py | 17 - var/spack/packages/py-mpi4py/package.py | 14 - var/spack/packages/py-mx/package.py | 13 - var/spack/packages/py-nose/package.py | 17 - var/spack/packages/py-numpy/package.py | 28 - var/spack/packages/py-pandas/package.py | 25 - var/spack/packages/py-pexpect/package.py | 13 - var/spack/packages/py-pil/package.py | 14 - var/spack/packages/py-pmw/package.py | 13 - var/spack/packages/py-pychecker/package.py | 13 - var/spack/packages/py-pycparser/package.py | 15 - var/spack/packages/py-pyelftools/package.py | 13 - var/spack/packages/py-pygments/package.py | 15 - var/spack/packages/py-pylint/package.py | 17 - var/spack/packages/py-pypar/package.py | 14 - var/spack/packages/py-pyparsing/package.py | 13 - var/spack/packages/py-pyqt/package.py | 24 - var/spack/packages/py-pyside/package.py | 45 - var/spack/packages/py-python-daemon/package.py | 26 - var/spack/packages/py-pytz/package.py | 14 - var/spack/packages/py-rpy2/package.py | 17 - var/spack/packages/py-scientificpython/package.py | 16 - var/spack/packages/py-scikit-learn/package.py | 14 - var/spack/packages/py-scipy/package.py | 18 - var/spack/packages/py-setuptools/package.py | 15 - var/spack/packages/py-shiboken/package.py | 45 - var/spack/packages/py-sip/package.py | 21 - var/spack/packages/py-six/package.py | 14 - var/spack/packages/py-sphinx/package.py | 13 - var/spack/packages/py-sympy/package.py | 13 - var/spack/packages/py-virtualenv/package.py | 16 - var/spack/packages/py-yapf/package.py | 15 - var/spack/packages/python/package.py | 160 -- var/spack/packages/qhull/package.py | 27 - var/spack/packages/qt/package.py | 109 -- var/spack/packages/qthreads/package.py | 22 - var/spack/packages/ravel/package.py | 23 - var/spack/packages/readline/package.py | 21 - .../rose/add_spack_compiler_recognition.patch | 13 - var/spack/packages/rose/package.py | 39 - var/spack/packages/ruby/package.py | 41 - var/spack/packages/samtools/package.py | 18 - var/spack/packages/samtools/samtools1.2.patch | 20 - var/spack/packages/scalasca/package.py | 65 - var/spack/packages/scorep/package.py | 74 - var/spack/packages/scotch/package.py | 40 - var/spack/packages/scr/package.py | 44 - var/spack/packages/silo/package.py | 19 - var/spack/packages/snappy/package.py | 15 - var/spack/packages/spindle/package.py | 44 - var/spack/packages/sqlite/package.py | 40 - var/spack/packages/stat/configure_mpicxx.patch | 19 - var/spack/packages/stat/package.py | 40 - var/spack/packages/sundials/package.py | 39 - var/spack/packages/swig/package.py | 46 - var/spack/packages/task/package.py | 20 - var/spack/packages/taskd/package.py | 20 - var/spack/packages/tau/package.py | 36 - var/spack/packages/tcl/package.py | 22 - var/spack/packages/the_silver_searcher/package.py | 17 - var/spack/packages/thrift/package.py | 44 - var/spack/packages/tk/package.py | 22 - var/spack/packages/tmux/package.py | 24 - var/spack/packages/tmuxinator/package.py | 17 - var/spack/packages/trilinos/package.py | 50 - var/spack/packages/uncrustify/package.py | 14 - var/spack/packages/util-linux/package.py | 20 - var/spack/packages/vim/package.py | 83 -- var/spack/packages/vtk/package.py | 40 - var/spack/packages/wget/package.py | 21 - var/spack/packages/wx/package.py | 24 - var/spack/packages/wxpropgrid/package.py | 20 - var/spack/packages/xcb-proto/package.py | 15 - var/spack/packages/xz/package.py | 20 - var/spack/packages/yasm/package.py | 16 - var/spack/packages/zeromq/package.py | 20 - var/spack/packages/zlib/package.py | 18 - var/spack/packages/zsh/package.py | 16 - var/spack/repos/builtin.mock/packages/a/package.py | 12 + var/spack/repos/builtin.mock/packages/b/package.py | 12 + var/spack/repos/builtin.mock/packages/c/package.py | 12 + .../builtin.mock/packages/callpath/package.py | 41 + .../builtin.mock/packages/direct_mpich/package.py | 36 + .../repos/builtin.mock/packages/dyninst/package.py | 44 + var/spack/repos/builtin.mock/packages/e/package.py | 12 + .../repos/builtin.mock/packages/fake/package.py | 34 + .../builtin.mock/packages/git-test/package.py | 10 + .../repos/builtin.mock/packages/hg-test/package.py | 10 + .../packages/indirect_mpich/package.py | 41 + .../builtin.mock/packages/libdwarf/package.py | 44 + .../repos/builtin.mock/packages/libelf/package.py | 43 + .../repos/builtin.mock/packages/mpich/package.py | 46 + .../repos/builtin.mock/packages/mpich2/package.py | 47 + .../builtin.mock/packages/mpileaks/package.py | 43 + .../builtin.mock/packages/multimethod/package.py | 143 ++ .../packages/optional-dep-test-2/package.py | 18 + .../packages/optional-dep-test-3/package.py | 17 + .../packages/optional-dep-test/package.py | 29 + .../builtin.mock/packages/svn-test/package.py | 10 + .../trivial_install_test_package/package.py | 38 + .../repos/builtin.mock/packages/zmpi/package.py | 39 + var/spack/repos/builtin.mock/repo.yaml | 2 + .../repos/builtin/packages/ImageMagick/package.py | 37 + var/spack/repos/builtin/packages/Mitos/package.py | 19 + var/spack/repos/builtin/packages/R/package.py | 33 + .../builtin/packages/SAMRAI/no-tool-build.patch | 20 + var/spack/repos/builtin/packages/SAMRAI/package.py | 53 + .../builtin/packages/activeharmony/package.py | 15 + .../repos/builtin/packages/adept-utils/package.py | 42 + var/spack/repos/builtin/packages/apex/package.py | 34 + var/spack/repos/builtin/packages/arpack/package.py | 41 + .../repos/builtin/packages/asciidoc/package.py | 18 + var/spack/repos/builtin/packages/atk/package.py | 18 + var/spack/repos/builtin/packages/atlas/package.py | 60 + .../repos/builtin/packages/autoconf/package.py | 14 + .../repos/builtin/packages/automaded/package.py | 51 + .../repos/builtin/packages/automake/package.py | 16 + var/spack/repos/builtin/packages/bear/package.py | 17 + .../repos/builtin/packages/bib2xhtml/package.py | 27 + .../repos/builtin/packages/binutils/package.py | 30 + var/spack/repos/builtin/packages/bison/package.py | 17 + var/spack/repos/builtin/packages/boost/package.py | 66 + .../builtin/packages/bowtie2/bowtie2-2.5.patch | 16 + .../repos/builtin/packages/bowtie2/package.py | 24 + var/spack/repos/builtin/packages/boxlib/package.py | 25 + var/spack/repos/builtin/packages/bzip2/package.py | 36 + var/spack/repos/builtin/packages/cairo/package.py | 19 + .../repos/builtin/packages/callpath/package.py | 47 + var/spack/repos/builtin/packages/cblas/package.py | 35 + var/spack/repos/builtin/packages/cgm/package.py | 30 + var/spack/repos/builtin/packages/clang/package.py | 51 + var/spack/repos/builtin/packages/cloog/package.py | 26 + var/spack/repos/builtin/packages/cmake/package.py | 45 + .../repos/builtin/packages/coreutils/package.py | 17 + .../repos/builtin/packages/cppcheck/package.py | 15 + var/spack/repos/builtin/packages/cram/package.py | 15 + var/spack/repos/builtin/packages/cscope/package.py | 17 + var/spack/repos/builtin/packages/cube/package.py | 55 + var/spack/repos/builtin/packages/czmq/package.py | 19 + var/spack/repos/builtin/packages/dbus/package.py | 31 + .../repos/builtin/packages/docbook-xml/package.py | 19 + .../repos/builtin/packages/doxygen/package.py | 25 + .../repos/builtin/packages/dri2proto/package.py | 14 + var/spack/repos/builtin/packages/dtcmp/package.py | 20 + .../repos/builtin/packages/dyninst/package.py | 68 + .../repos/builtin/packages/elfutils/package.py | 26 + var/spack/repos/builtin/packages/extrae/package.py | 46 + .../builtin/packages/exuberant-ctags/package.py | 14 + var/spack/repos/builtin/packages/fish/package.py | 18 + var/spack/repos/builtin/packages/flex/package.py | 15 + var/spack/repos/builtin/packages/flux/package.py | 36 + .../repos/builtin/packages/fontconfig/package.py | 16 + .../repos/builtin/packages/freetype/package.py | 16 + var/spack/repos/builtin/packages/gasnet/package.py | 35 + var/spack/repos/builtin/packages/gcc/package.py | 122 ++ .../repos/builtin/packages/gdk-pixbuf/package.py | 22 + var/spack/repos/builtin/packages/geos/package.py | 31 + var/spack/repos/builtin/packages/gflags/package.py | 21 + .../repos/builtin/packages/ghostscript/package.py | 17 + var/spack/repos/builtin/packages/git/package.py | 27 + var/spack/repos/builtin/packages/glib/package.py | 18 + var/spack/repos/builtin/packages/glm/package.py | 19 + var/spack/repos/builtin/packages/global/package.py | 24 + var/spack/repos/builtin/packages/glog/package.py | 15 + var/spack/repos/builtin/packages/gmp/package.py | 40 + var/spack/repos/builtin/packages/gnutls/package.py | 22 + var/spack/repos/builtin/packages/gperf/package.py | 19 + .../repos/builtin/packages/gperftools/package.py | 38 + .../repos/builtin/packages/graphlib/package.py | 14 + .../repos/builtin/packages/graphviz/package.py | 21 + .../repos/builtin/packages/gtkplus/package.py | 22 + .../repos/builtin/packages/harfbuzz/package.py | 20 + var/spack/repos/builtin/packages/hdf5/package.py | 42 + var/spack/repos/builtin/packages/hwloc/package.py | 25 + var/spack/repos/builtin/packages/hypre/package.py | 32 + var/spack/repos/builtin/packages/icu/package.py | 25 + var/spack/repos/builtin/packages/icu4c/package.py | 17 + var/spack/repos/builtin/packages/isl/package.py | 17 + var/spack/repos/builtin/packages/jdk/package.py | 46 + var/spack/repos/builtin/packages/jpeg/package.py | 14 + .../repos/builtin/packages/launchmon/package.py | 47 + .../packages/launchmon/patch.lmon_install_dir | 147 ++ var/spack/repos/builtin/packages/lcms/package.py | 19 + .../repos/builtin/packages/leveldb/package.py | 29 + var/spack/repos/builtin/packages/libNBC/package.py | 43 + .../repos/builtin/packages/libarchive/package.py | 16 + .../repos/builtin/packages/libcircle/package.py | 18 + var/spack/repos/builtin/packages/libdrm/package.py | 18 + .../repos/builtin/packages/libdwarf/package.py | 81 + var/spack/repos/builtin/packages/libelf/package.py | 49 + .../repos/builtin/packages/libevent/package.py | 30 + var/spack/repos/builtin/packages/libffi/package.py | 17 + .../repos/builtin/packages/libgcrypt/package.py | 19 + .../repos/builtin/packages/libgpg-error/package.py | 17 + .../builtin/packages/libjpeg-turbo/package.py | 20 + .../repos/builtin/packages/libjson-c/package.py | 14 + var/spack/repos/builtin/packages/libmng/package.py | 23 + .../repos/builtin/packages/libmonitor/package.py | 36 + .../repos/builtin/packages/libpciaccess/package.py | 21 + var/spack/repos/builtin/packages/libpng/package.py | 15 + .../repos/builtin/packages/libsodium/package.py | 19 + .../repos/builtin/packages/libtiff/package.py | 18 + .../repos/builtin/packages/libtool/package.py | 14 + .../repos/builtin/packages/libunwind/package.py | 38 + .../repos/builtin/packages/libuuid/package.py | 16 + var/spack/repos/builtin/packages/libxcb/package.py | 21 + .../repos/builtin/packages/libxml2/package.py | 20 + .../repos/builtin/packages/libxshmfence/package.py | 16 + .../repos/builtin/packages/libxslt/package.py | 24 + .../repos/builtin/packages/llvm-lld/package.py | 46 + var/spack/repos/builtin/packages/llvm/package.py | 53 + var/spack/repos/builtin/packages/lmdb/package.py | 39 + var/spack/repos/builtin/packages/lua/package.py | 26 + var/spack/repos/builtin/packages/lwgrp/package.py | 18 + var/spack/repos/builtin/packages/lwm2/package.py | 18 + var/spack/repos/builtin/packages/matio/package.py | 15 + .../repos/builtin/packages/memaxes/package.py | 19 + var/spack/repos/builtin/packages/mesa/package.py | 34 + var/spack/repos/builtin/packages/metis/package.py | 27 + var/spack/repos/builtin/packages/mpc/package.py | 42 + var/spack/repos/builtin/packages/mpe2/mpe2.patch | 12 + var/spack/repos/builtin/packages/mpe2/package.py | 28 + var/spack/repos/builtin/packages/mpfr/package.py | 41 + .../builtin/packages/mpibash/mpibash-4.3.patch | 1565 ++++++++++++++++++++ .../repos/builtin/packages/mpibash/package.py | 32 + var/spack/repos/builtin/packages/mpich/package.py | 92 ++ .../repos/builtin/packages/mpileaks/package.py | 44 + var/spack/repos/builtin/packages/mrnet/package.py | 20 + var/spack/repos/builtin/packages/munge/package.py | 20 + var/spack/repos/builtin/packages/muster/package.py | 22 + .../mvapich2/ad_lustre_rwcontig_open_source.patch | 11 + .../repos/builtin/packages/mvapich2/package.py | 104 ++ var/spack/repos/builtin/packages/nasm/package.py | 14 + var/spack/repos/builtin/packages/ncdu/package.py | 28 + .../repos/builtin/packages/ncurses/package.py | 33 + .../builtin/packages/netcdf/netcdf-4.3.3-mpi.patch | 25 + var/spack/repos/builtin/packages/netcdf/package.py | 27 + .../repos/builtin/packages/netgauge/package.py | 43 + .../repos/builtin/packages/netlib-blas/package.py | 46 + .../builtin/packages/netlib-lapack/package.py | 59 + var/spack/repos/builtin/packages/nettle/package.py | 17 + var/spack/repos/builtin/packages/ompss/package.py | 50 + .../repos/builtin/packages/ompt-openmp/package.py | 23 + var/spack/repos/builtin/packages/opari2/package.py | 65 + .../openmpi/ad_lustre_rwcontig_open_source.patch | 11 + .../builtin/packages/openmpi/llnl-platforms.patch | 151 ++ .../repos/builtin/packages/openmpi/package.py | 109 ++ .../repos/builtin/packages/openssl/package.py | 26 + var/spack/repos/builtin/packages/otf/package.py | 21 + var/spack/repos/builtin/packages/otf2/package.py | 74 + var/spack/repos/builtin/packages/pango/package.py | 19 + var/spack/repos/builtin/packages/papi/package.py | 35 + .../repos/builtin/packages/paraver/package.py | 41 + .../repos/builtin/packages/paraview/package.py | 72 + .../repos/builtin/packages/parmetis/package.py | 26 + .../repos/builtin/packages/parpack/package.py | 43 + var/spack/repos/builtin/packages/pcre/package.py | 15 + var/spack/repos/builtin/packages/petsc/package.py | 40 + var/spack/repos/builtin/packages/pidx/package.py | 21 + var/spack/repos/builtin/packages/pixman/package.py | 18 + .../repos/builtin/packages/pkg-config/package.py | 17 + .../builtin/packages/pmgr_collective/package.py | 37 + .../repos/builtin/packages/postgresql/package.py | 20 + var/spack/repos/builtin/packages/ppl/package.py | 28 + .../repos/builtin/packages/protobuf/package.py | 16 + .../repos/builtin/packages/py-basemap/package.py | 20 + .../repos/builtin/packages/py-biopython/package.py | 15 + .../repos/builtin/packages/py-cffi/package.py | 17 + .../repos/builtin/packages/py-cython/package.py | 14 + .../repos/builtin/packages/py-dateutil/package.py | 16 + .../repos/builtin/packages/py-epydoc/package.py | 13 + .../repos/builtin/packages/py-genders/package.py | 15 + .../repos/builtin/packages/py-gnuplot/package.py | 14 + .../repos/builtin/packages/py-h5py/package.py | 19 + .../repos/builtin/packages/py-ipython/package.py | 16 + .../repos/builtin/packages/py-libxml2/package.py | 15 + .../repos/builtin/packages/py-lockfile/package.py | 23 + .../repos/builtin/packages/py-mako/package.py | 16 + .../builtin/packages/py-matplotlib/package.py | 47 + .../repos/builtin/packages/py-mock/package.py | 17 + .../repos/builtin/packages/py-mpi4py/package.py | 14 + var/spack/repos/builtin/packages/py-mx/package.py | 13 + .../repos/builtin/packages/py-nose/package.py | 17 + .../repos/builtin/packages/py-numpy/package.py | 28 + .../repos/builtin/packages/py-pandas/package.py | 25 + .../repos/builtin/packages/py-pexpect/package.py | 13 + var/spack/repos/builtin/packages/py-pil/package.py | 14 + var/spack/repos/builtin/packages/py-pmw/package.py | 13 + .../repos/builtin/packages/py-pychecker/package.py | 13 + .../repos/builtin/packages/py-pycparser/package.py | 15 + .../builtin/packages/py-pyelftools/package.py | 13 + .../repos/builtin/packages/py-pygments/package.py | 15 + .../repos/builtin/packages/py-pylint/package.py | 17 + .../repos/builtin/packages/py-pypar/package.py | 14 + .../repos/builtin/packages/py-pyparsing/package.py | 13 + .../repos/builtin/packages/py-pyqt/package.py | 24 + .../repos/builtin/packages/py-pyside/package.py | 45 + .../builtin/packages/py-python-daemon/package.py | 26 + .../repos/builtin/packages/py-pytz/package.py | 14 + .../repos/builtin/packages/py-rpy2/package.py | 17 + .../packages/py-scientificpython/package.py | 16 + .../builtin/packages/py-scikit-learn/package.py | 14 + .../repos/builtin/packages/py-scipy/package.py | 18 + .../builtin/packages/py-setuptools/package.py | 15 + .../repos/builtin/packages/py-shiboken/package.py | 45 + var/spack/repos/builtin/packages/py-sip/package.py | 21 + var/spack/repos/builtin/packages/py-six/package.py | 14 + .../repos/builtin/packages/py-sphinx/package.py | 13 + .../repos/builtin/packages/py-sympy/package.py | 13 + .../builtin/packages/py-virtualenv/package.py | 16 + .../repos/builtin/packages/py-yapf/package.py | 15 + var/spack/repos/builtin/packages/python/package.py | 160 ++ var/spack/repos/builtin/packages/qhull/package.py | 27 + var/spack/repos/builtin/packages/qt/package.py | 109 ++ .../repos/builtin/packages/qthreads/package.py | 22 + var/spack/repos/builtin/packages/ravel/package.py | 23 + .../repos/builtin/packages/readline/package.py | 21 + .../rose/add_spack_compiler_recognition.patch | 13 + var/spack/repos/builtin/packages/rose/package.py | 39 + var/spack/repos/builtin/packages/ruby/package.py | 41 + .../repos/builtin/packages/samtools/package.py | 18 + .../builtin/packages/samtools/samtools1.2.patch | 20 + .../repos/builtin/packages/scalasca/package.py | 65 + var/spack/repos/builtin/packages/scorep/package.py | 74 + var/spack/repos/builtin/packages/scotch/package.py | 40 + var/spack/repos/builtin/packages/scr/package.py | 44 + var/spack/repos/builtin/packages/silo/package.py | 19 + var/spack/repos/builtin/packages/snappy/package.py | 15 + .../repos/builtin/packages/spindle/package.py | 44 + var/spack/repos/builtin/packages/sqlite/package.py | 40 + .../builtin/packages/stat/configure_mpicxx.patch | 19 + var/spack/repos/builtin/packages/stat/package.py | 40 + .../repos/builtin/packages/sundials/package.py | 39 + var/spack/repos/builtin/packages/swig/package.py | 46 + var/spack/repos/builtin/packages/task/package.py | 20 + var/spack/repos/builtin/packages/taskd/package.py | 20 + var/spack/repos/builtin/packages/tau/package.py | 36 + var/spack/repos/builtin/packages/tcl/package.py | 22 + .../packages/the_silver_searcher/package.py | 17 + var/spack/repos/builtin/packages/thrift/package.py | 44 + var/spack/repos/builtin/packages/tk/package.py | 22 + var/spack/repos/builtin/packages/tmux/package.py | 24 + .../repos/builtin/packages/tmuxinator/package.py | 17 + .../repos/builtin/packages/trilinos/package.py | 50 + .../repos/builtin/packages/uncrustify/package.py | 14 + .../repos/builtin/packages/util-linux/package.py | 20 + var/spack/repos/builtin/packages/vim/package.py | 83 ++ var/spack/repos/builtin/packages/vtk/package.py | 40 + var/spack/repos/builtin/packages/wget/package.py | 21 + var/spack/repos/builtin/packages/wx/package.py | 24 + .../repos/builtin/packages/wxpropgrid/package.py | 20 + .../repos/builtin/packages/xcb-proto/package.py | 15 + var/spack/repos/builtin/packages/xz/package.py | 20 + var/spack/repos/builtin/packages/yasm/package.py | 16 + var/spack/repos/builtin/packages/zeromq/package.py | 20 + var/spack/repos/builtin/packages/zlib/package.py | 18 + var/spack/repos/builtin/packages/zsh/package.py | 16 + var/spack/repos/builtin/repo.yaml | 2 + 563 files changed, 10048 insertions(+), 9975 deletions(-) delete mode 100644 var/spack/mock_packages/_repo.yaml delete mode 100644 var/spack/mock_packages/a/package.py delete mode 100644 var/spack/mock_packages/b/package.py delete mode 100644 var/spack/mock_packages/c/package.py delete mode 100644 var/spack/mock_packages/callpath/package.py delete mode 100644 var/spack/mock_packages/direct_mpich/package.py delete mode 100644 var/spack/mock_packages/dyninst/package.py delete mode 100644 var/spack/mock_packages/e/package.py delete mode 100644 var/spack/mock_packages/fake/package.py delete mode 100644 var/spack/mock_packages/git-test/package.py delete mode 100644 var/spack/mock_packages/hg-test/package.py delete mode 100644 var/spack/mock_packages/indirect_mpich/package.py delete mode 100644 var/spack/mock_packages/libdwarf/package.py delete mode 100644 var/spack/mock_packages/libelf/package.py delete mode 100644 var/spack/mock_packages/mpich/package.py delete mode 100644 var/spack/mock_packages/mpich2/package.py delete mode 100644 var/spack/mock_packages/mpileaks/package.py delete mode 100644 var/spack/mock_packages/multimethod/package.py delete mode 100644 var/spack/mock_packages/optional-dep-test-2/package.py delete mode 100644 var/spack/mock_packages/optional-dep-test-3/package.py delete mode 100644 var/spack/mock_packages/optional-dep-test/package.py delete mode 100644 var/spack/mock_packages/svn-test/package.py delete mode 100644 var/spack/mock_packages/trivial_install_test_package/package.py delete mode 100644 var/spack/mock_packages/zmpi/package.py delete mode 100644 var/spack/packages/ImageMagick/package.py delete mode 100644 var/spack/packages/Mitos/package.py delete mode 100644 var/spack/packages/R/package.py delete mode 100644 var/spack/packages/SAMRAI/no-tool-build.patch delete mode 100644 var/spack/packages/SAMRAI/package.py delete mode 100644 var/spack/packages/_repo.yaml delete mode 100644 var/spack/packages/activeharmony/package.py delete mode 100644 var/spack/packages/adept-utils/package.py delete mode 100644 var/spack/packages/apex/package.py delete mode 100644 var/spack/packages/arpack/package.py delete mode 100644 var/spack/packages/asciidoc/package.py delete mode 100644 var/spack/packages/atk/package.py delete mode 100644 var/spack/packages/atlas/package.py delete mode 100644 var/spack/packages/autoconf/package.py delete mode 100644 var/spack/packages/automaded/package.py delete mode 100644 var/spack/packages/automake/package.py delete mode 100644 var/spack/packages/bear/package.py delete mode 100644 var/spack/packages/bib2xhtml/package.py delete mode 100644 var/spack/packages/binutils/package.py delete mode 100644 var/spack/packages/bison/package.py delete mode 100644 var/spack/packages/boost/package.py delete mode 100644 var/spack/packages/bowtie2/bowtie2-2.5.patch delete mode 100644 var/spack/packages/bowtie2/package.py delete mode 100644 var/spack/packages/boxlib/package.py delete mode 100644 var/spack/packages/bzip2/package.py delete mode 100644 var/spack/packages/cairo/package.py delete mode 100644 var/spack/packages/callpath/package.py delete mode 100644 var/spack/packages/cblas/package.py delete mode 100644 var/spack/packages/cgm/package.py delete mode 100644 var/spack/packages/clang/package.py delete mode 100644 var/spack/packages/cloog/package.py delete mode 100644 var/spack/packages/cmake/package.py delete mode 100644 var/spack/packages/coreutils/package.py delete mode 100644 var/spack/packages/cppcheck/package.py delete mode 100644 var/spack/packages/cram/package.py delete mode 100644 var/spack/packages/cscope/package.py delete mode 100644 var/spack/packages/cube/package.py delete mode 100644 var/spack/packages/czmq/package.py delete mode 100644 var/spack/packages/dbus/package.py delete mode 100644 var/spack/packages/docbook-xml/package.py delete mode 100644 var/spack/packages/doxygen/package.py delete mode 100644 var/spack/packages/dri2proto/package.py delete mode 100644 var/spack/packages/dtcmp/package.py delete mode 100644 var/spack/packages/dyninst/package.py delete mode 100644 var/spack/packages/elfutils/package.py delete mode 100644 var/spack/packages/extrae/package.py delete mode 100644 var/spack/packages/exuberant-ctags/package.py delete mode 100644 var/spack/packages/fish/package.py delete mode 100644 var/spack/packages/flex/package.py delete mode 100644 var/spack/packages/flux/package.py delete mode 100644 var/spack/packages/fontconfig/package.py delete mode 100644 var/spack/packages/freetype/package.py delete mode 100644 var/spack/packages/gasnet/package.py delete mode 100644 var/spack/packages/gcc/package.py delete mode 100644 var/spack/packages/gdk-pixbuf/package.py delete mode 100644 var/spack/packages/geos/package.py delete mode 100644 var/spack/packages/gflags/package.py delete mode 100644 var/spack/packages/ghostscript/package.py delete mode 100644 var/spack/packages/git/package.py delete mode 100644 var/spack/packages/glib/package.py delete mode 100644 var/spack/packages/glm/package.py delete mode 100644 var/spack/packages/global/package.py delete mode 100644 var/spack/packages/glog/package.py delete mode 100644 var/spack/packages/gmp/package.py delete mode 100644 var/spack/packages/gnutls/package.py delete mode 100644 var/spack/packages/gperf/package.py delete mode 100644 var/spack/packages/gperftools/package.py delete mode 100644 var/spack/packages/graphlib/package.py delete mode 100644 var/spack/packages/graphviz/package.py delete mode 100644 var/spack/packages/gtkplus/package.py delete mode 100644 var/spack/packages/harfbuzz/package.py delete mode 100644 var/spack/packages/hdf5/package.py delete mode 100644 var/spack/packages/hwloc/package.py delete mode 100644 var/spack/packages/hypre/package.py delete mode 100644 var/spack/packages/icu/package.py delete mode 100644 var/spack/packages/icu4c/package.py delete mode 100644 var/spack/packages/isl/package.py delete mode 100644 var/spack/packages/jdk/package.py delete mode 100644 var/spack/packages/jpeg/package.py delete mode 100644 var/spack/packages/launchmon/package.py delete mode 100644 var/spack/packages/launchmon/patch.lmon_install_dir delete mode 100644 var/spack/packages/lcms/package.py delete mode 100644 var/spack/packages/leveldb/package.py delete mode 100644 var/spack/packages/libNBC/package.py delete mode 100644 var/spack/packages/libarchive/package.py delete mode 100644 var/spack/packages/libcircle/package.py delete mode 100644 var/spack/packages/libdrm/package.py delete mode 100644 var/spack/packages/libdwarf/package.py delete mode 100644 var/spack/packages/libelf/package.py delete mode 100644 var/spack/packages/libevent/package.py delete mode 100644 var/spack/packages/libffi/package.py delete mode 100644 var/spack/packages/libgcrypt/package.py delete mode 100644 var/spack/packages/libgpg-error/package.py delete mode 100644 var/spack/packages/libjpeg-turbo/package.py delete mode 100644 var/spack/packages/libjson-c/package.py delete mode 100644 var/spack/packages/libmng/package.py delete mode 100644 var/spack/packages/libmonitor/package.py delete mode 100644 var/spack/packages/libpciaccess/package.py delete mode 100644 var/spack/packages/libpng/package.py delete mode 100644 var/spack/packages/libsodium/package.py delete mode 100644 var/spack/packages/libtiff/package.py delete mode 100644 var/spack/packages/libtool/package.py delete mode 100644 var/spack/packages/libunwind/package.py delete mode 100644 var/spack/packages/libuuid/package.py delete mode 100644 var/spack/packages/libxcb/package.py delete mode 100644 var/spack/packages/libxml2/package.py delete mode 100644 var/spack/packages/libxshmfence/package.py delete mode 100644 var/spack/packages/libxslt/package.py delete mode 100644 var/spack/packages/llvm-lld/package.py delete mode 100644 var/spack/packages/llvm/package.py delete mode 100644 var/spack/packages/lmdb/package.py delete mode 100644 var/spack/packages/lua/package.py delete mode 100644 var/spack/packages/lwgrp/package.py delete mode 100644 var/spack/packages/lwm2/package.py delete mode 100644 var/spack/packages/matio/package.py delete mode 100644 var/spack/packages/memaxes/package.py delete mode 100644 var/spack/packages/mesa/package.py delete mode 100644 var/spack/packages/metis/package.py delete mode 100644 var/spack/packages/mpc/package.py delete mode 100644 var/spack/packages/mpe2/mpe2.patch delete mode 100644 var/spack/packages/mpe2/package.py delete mode 100644 var/spack/packages/mpfr/package.py delete mode 100644 var/spack/packages/mpibash/mpibash-4.3.patch delete mode 100644 var/spack/packages/mpibash/package.py delete mode 100644 var/spack/packages/mpich/package.py delete mode 100644 var/spack/packages/mpileaks/package.py delete mode 100644 var/spack/packages/mrnet/package.py delete mode 100644 var/spack/packages/munge/package.py delete mode 100644 var/spack/packages/muster/package.py delete mode 100644 var/spack/packages/mvapich2/ad_lustre_rwcontig_open_source.patch delete mode 100644 var/spack/packages/mvapich2/package.py delete mode 100644 var/spack/packages/nasm/package.py delete mode 100644 var/spack/packages/ncdu/package.py delete mode 100644 var/spack/packages/ncurses/package.py delete mode 100644 var/spack/packages/netcdf/netcdf-4.3.3-mpi.patch delete mode 100644 var/spack/packages/netcdf/package.py delete mode 100644 var/spack/packages/netgauge/package.py delete mode 100644 var/spack/packages/netlib-blas/package.py delete mode 100644 var/spack/packages/netlib-lapack/package.py delete mode 100644 var/spack/packages/nettle/package.py delete mode 100644 var/spack/packages/ompss/package.py delete mode 100644 var/spack/packages/ompt-openmp/package.py delete mode 100644 var/spack/packages/opari2/package.py delete mode 100644 var/spack/packages/openmpi/ad_lustre_rwcontig_open_source.patch delete mode 100644 var/spack/packages/openmpi/llnl-platforms.patch delete mode 100644 var/spack/packages/openmpi/package.py delete mode 100644 var/spack/packages/openssl/package.py delete mode 100644 var/spack/packages/otf/package.py delete mode 100644 var/spack/packages/otf2/package.py delete mode 100644 var/spack/packages/pango/package.py delete mode 100644 var/spack/packages/papi/package.py delete mode 100644 var/spack/packages/paraver/package.py delete mode 100644 var/spack/packages/paraview/package.py delete mode 100644 var/spack/packages/parmetis/package.py delete mode 100644 var/spack/packages/parpack/package.py delete mode 100644 var/spack/packages/pcre/package.py delete mode 100644 var/spack/packages/petsc/package.py delete mode 100644 var/spack/packages/pidx/package.py delete mode 100644 var/spack/packages/pixman/package.py delete mode 100644 var/spack/packages/pkg-config/package.py delete mode 100644 var/spack/packages/pmgr_collective/package.py delete mode 100644 var/spack/packages/postgresql/package.py delete mode 100644 var/spack/packages/ppl/package.py delete mode 100644 var/spack/packages/protobuf/package.py delete mode 100644 var/spack/packages/py-basemap/package.py delete mode 100644 var/spack/packages/py-biopython/package.py delete mode 100644 var/spack/packages/py-cffi/package.py delete mode 100644 var/spack/packages/py-cython/package.py delete mode 100644 var/spack/packages/py-dateutil/package.py delete mode 100644 var/spack/packages/py-epydoc/package.py delete mode 100644 var/spack/packages/py-genders/package.py delete mode 100644 var/spack/packages/py-gnuplot/package.py delete mode 100644 var/spack/packages/py-h5py/package.py delete mode 100644 var/spack/packages/py-ipython/package.py delete mode 100644 var/spack/packages/py-libxml2/package.py delete mode 100644 var/spack/packages/py-lockfile/package.py delete mode 100644 var/spack/packages/py-mako/package.py delete mode 100644 var/spack/packages/py-matplotlib/package.py delete mode 100644 var/spack/packages/py-mock/package.py delete mode 100644 var/spack/packages/py-mpi4py/package.py delete mode 100644 var/spack/packages/py-mx/package.py delete mode 100644 var/spack/packages/py-nose/package.py delete mode 100644 var/spack/packages/py-numpy/package.py delete mode 100644 var/spack/packages/py-pandas/package.py delete mode 100644 var/spack/packages/py-pexpect/package.py delete mode 100644 var/spack/packages/py-pil/package.py delete mode 100644 var/spack/packages/py-pmw/package.py delete mode 100644 var/spack/packages/py-pychecker/package.py delete mode 100644 var/spack/packages/py-pycparser/package.py delete mode 100644 var/spack/packages/py-pyelftools/package.py delete mode 100644 var/spack/packages/py-pygments/package.py delete mode 100644 var/spack/packages/py-pylint/package.py delete mode 100644 var/spack/packages/py-pypar/package.py delete mode 100644 var/spack/packages/py-pyparsing/package.py delete mode 100644 var/spack/packages/py-pyqt/package.py delete mode 100644 var/spack/packages/py-pyside/package.py delete mode 100644 var/spack/packages/py-python-daemon/package.py delete mode 100644 var/spack/packages/py-pytz/package.py delete mode 100644 var/spack/packages/py-rpy2/package.py delete mode 100644 var/spack/packages/py-scientificpython/package.py delete mode 100644 var/spack/packages/py-scikit-learn/package.py delete mode 100644 var/spack/packages/py-scipy/package.py delete mode 100644 var/spack/packages/py-setuptools/package.py delete mode 100644 var/spack/packages/py-shiboken/package.py delete mode 100644 var/spack/packages/py-sip/package.py delete mode 100644 var/spack/packages/py-six/package.py delete mode 100644 var/spack/packages/py-sphinx/package.py delete mode 100644 var/spack/packages/py-sympy/package.py delete mode 100644 var/spack/packages/py-virtualenv/package.py delete mode 100644 var/spack/packages/py-yapf/package.py delete mode 100644 var/spack/packages/python/package.py delete mode 100644 var/spack/packages/qhull/package.py delete mode 100644 var/spack/packages/qt/package.py delete mode 100644 var/spack/packages/qthreads/package.py delete mode 100644 var/spack/packages/ravel/package.py delete mode 100644 var/spack/packages/readline/package.py delete mode 100644 var/spack/packages/rose/add_spack_compiler_recognition.patch delete mode 100644 var/spack/packages/rose/package.py delete mode 100644 var/spack/packages/ruby/package.py delete mode 100644 var/spack/packages/samtools/package.py delete mode 100644 var/spack/packages/samtools/samtools1.2.patch delete mode 100644 var/spack/packages/scalasca/package.py delete mode 100644 var/spack/packages/scorep/package.py delete mode 100644 var/spack/packages/scotch/package.py delete mode 100644 var/spack/packages/scr/package.py delete mode 100644 var/spack/packages/silo/package.py delete mode 100644 var/spack/packages/snappy/package.py delete mode 100644 var/spack/packages/spindle/package.py delete mode 100644 var/spack/packages/sqlite/package.py delete mode 100644 var/spack/packages/stat/configure_mpicxx.patch delete mode 100644 var/spack/packages/stat/package.py delete mode 100644 var/spack/packages/sundials/package.py delete mode 100644 var/spack/packages/swig/package.py delete mode 100644 var/spack/packages/task/package.py delete mode 100644 var/spack/packages/taskd/package.py delete mode 100644 var/spack/packages/tau/package.py delete mode 100644 var/spack/packages/tcl/package.py delete mode 100644 var/spack/packages/the_silver_searcher/package.py delete mode 100644 var/spack/packages/thrift/package.py delete mode 100644 var/spack/packages/tk/package.py delete mode 100644 var/spack/packages/tmux/package.py delete mode 100644 var/spack/packages/tmuxinator/package.py delete mode 100644 var/spack/packages/trilinos/package.py delete mode 100644 var/spack/packages/uncrustify/package.py delete mode 100644 var/spack/packages/util-linux/package.py delete mode 100644 var/spack/packages/vim/package.py delete mode 100644 var/spack/packages/vtk/package.py delete mode 100644 var/spack/packages/wget/package.py delete mode 100644 var/spack/packages/wx/package.py delete mode 100644 var/spack/packages/wxpropgrid/package.py delete mode 100644 var/spack/packages/xcb-proto/package.py delete mode 100644 var/spack/packages/xz/package.py delete mode 100644 var/spack/packages/yasm/package.py delete mode 100644 var/spack/packages/zeromq/package.py delete mode 100644 var/spack/packages/zlib/package.py delete mode 100644 var/spack/packages/zsh/package.py create mode 100644 var/spack/repos/builtin.mock/packages/a/package.py create mode 100644 var/spack/repos/builtin.mock/packages/b/package.py create mode 100644 var/spack/repos/builtin.mock/packages/c/package.py create mode 100644 var/spack/repos/builtin.mock/packages/callpath/package.py create mode 100644 var/spack/repos/builtin.mock/packages/direct_mpich/package.py create mode 100644 var/spack/repos/builtin.mock/packages/dyninst/package.py create mode 100644 var/spack/repos/builtin.mock/packages/e/package.py create mode 100644 var/spack/repos/builtin.mock/packages/fake/package.py create mode 100644 var/spack/repos/builtin.mock/packages/git-test/package.py create mode 100644 var/spack/repos/builtin.mock/packages/hg-test/package.py create mode 100644 var/spack/repos/builtin.mock/packages/indirect_mpich/package.py create mode 100644 var/spack/repos/builtin.mock/packages/libdwarf/package.py create mode 100644 var/spack/repos/builtin.mock/packages/libelf/package.py create mode 100644 var/spack/repos/builtin.mock/packages/mpich/package.py create mode 100644 var/spack/repos/builtin.mock/packages/mpich2/package.py create mode 100644 var/spack/repos/builtin.mock/packages/mpileaks/package.py create mode 100644 var/spack/repos/builtin.mock/packages/multimethod/package.py create mode 100644 var/spack/repos/builtin.mock/packages/optional-dep-test-2/package.py create mode 100644 var/spack/repos/builtin.mock/packages/optional-dep-test-3/package.py create mode 100644 var/spack/repos/builtin.mock/packages/optional-dep-test/package.py create mode 100644 var/spack/repos/builtin.mock/packages/svn-test/package.py create mode 100644 var/spack/repos/builtin.mock/packages/trivial_install_test_package/package.py create mode 100644 var/spack/repos/builtin.mock/packages/zmpi/package.py create mode 100644 var/spack/repos/builtin.mock/repo.yaml create mode 100644 var/spack/repos/builtin/packages/ImageMagick/package.py create mode 100644 var/spack/repos/builtin/packages/Mitos/package.py create mode 100644 var/spack/repos/builtin/packages/R/package.py create mode 100644 var/spack/repos/builtin/packages/SAMRAI/no-tool-build.patch create mode 100644 var/spack/repos/builtin/packages/SAMRAI/package.py create mode 100644 var/spack/repos/builtin/packages/activeharmony/package.py create mode 100644 var/spack/repos/builtin/packages/adept-utils/package.py create mode 100644 var/spack/repos/builtin/packages/apex/package.py create mode 100644 var/spack/repos/builtin/packages/arpack/package.py create mode 100644 var/spack/repos/builtin/packages/asciidoc/package.py create mode 100644 var/spack/repos/builtin/packages/atk/package.py create mode 100644 var/spack/repos/builtin/packages/atlas/package.py create mode 100644 var/spack/repos/builtin/packages/autoconf/package.py create mode 100644 var/spack/repos/builtin/packages/automaded/package.py create mode 100644 var/spack/repos/builtin/packages/automake/package.py create mode 100644 var/spack/repos/builtin/packages/bear/package.py create mode 100644 var/spack/repos/builtin/packages/bib2xhtml/package.py create mode 100644 var/spack/repos/builtin/packages/binutils/package.py create mode 100644 var/spack/repos/builtin/packages/bison/package.py create mode 100644 var/spack/repos/builtin/packages/boost/package.py create mode 100644 var/spack/repos/builtin/packages/bowtie2/bowtie2-2.5.patch create mode 100644 var/spack/repos/builtin/packages/bowtie2/package.py create mode 100644 var/spack/repos/builtin/packages/boxlib/package.py create mode 100644 var/spack/repos/builtin/packages/bzip2/package.py create mode 100644 var/spack/repos/builtin/packages/cairo/package.py create mode 100644 var/spack/repos/builtin/packages/callpath/package.py create mode 100644 var/spack/repos/builtin/packages/cblas/package.py create mode 100644 var/spack/repos/builtin/packages/cgm/package.py create mode 100644 var/spack/repos/builtin/packages/clang/package.py create mode 100644 var/spack/repos/builtin/packages/cloog/package.py create mode 100644 var/spack/repos/builtin/packages/cmake/package.py create mode 100644 var/spack/repos/builtin/packages/coreutils/package.py create mode 100644 var/spack/repos/builtin/packages/cppcheck/package.py create mode 100644 var/spack/repos/builtin/packages/cram/package.py create mode 100644 var/spack/repos/builtin/packages/cscope/package.py create mode 100644 var/spack/repos/builtin/packages/cube/package.py create mode 100644 var/spack/repos/builtin/packages/czmq/package.py create mode 100644 var/spack/repos/builtin/packages/dbus/package.py create mode 100644 var/spack/repos/builtin/packages/docbook-xml/package.py create mode 100644 var/spack/repos/builtin/packages/doxygen/package.py create mode 100644 var/spack/repos/builtin/packages/dri2proto/package.py create mode 100644 var/spack/repos/builtin/packages/dtcmp/package.py create mode 100644 var/spack/repos/builtin/packages/dyninst/package.py create mode 100644 var/spack/repos/builtin/packages/elfutils/package.py create mode 100644 var/spack/repos/builtin/packages/extrae/package.py create mode 100644 var/spack/repos/builtin/packages/exuberant-ctags/package.py create mode 100644 var/spack/repos/builtin/packages/fish/package.py create mode 100644 var/spack/repos/builtin/packages/flex/package.py create mode 100644 var/spack/repos/builtin/packages/flux/package.py create mode 100644 var/spack/repos/builtin/packages/fontconfig/package.py create mode 100644 var/spack/repos/builtin/packages/freetype/package.py create mode 100644 var/spack/repos/builtin/packages/gasnet/package.py create mode 100644 var/spack/repos/builtin/packages/gcc/package.py create mode 100644 var/spack/repos/builtin/packages/gdk-pixbuf/package.py create mode 100644 var/spack/repos/builtin/packages/geos/package.py create mode 100644 var/spack/repos/builtin/packages/gflags/package.py create mode 100644 var/spack/repos/builtin/packages/ghostscript/package.py create mode 100644 var/spack/repos/builtin/packages/git/package.py create mode 100644 var/spack/repos/builtin/packages/glib/package.py create mode 100644 var/spack/repos/builtin/packages/glm/package.py create mode 100644 var/spack/repos/builtin/packages/global/package.py create mode 100644 var/spack/repos/builtin/packages/glog/package.py create mode 100644 var/spack/repos/builtin/packages/gmp/package.py create mode 100644 var/spack/repos/builtin/packages/gnutls/package.py create mode 100644 var/spack/repos/builtin/packages/gperf/package.py create mode 100644 var/spack/repos/builtin/packages/gperftools/package.py create mode 100644 var/spack/repos/builtin/packages/graphlib/package.py create mode 100644 var/spack/repos/builtin/packages/graphviz/package.py create mode 100644 var/spack/repos/builtin/packages/gtkplus/package.py create mode 100644 var/spack/repos/builtin/packages/harfbuzz/package.py create mode 100644 var/spack/repos/builtin/packages/hdf5/package.py create mode 100644 var/spack/repos/builtin/packages/hwloc/package.py create mode 100644 var/spack/repos/builtin/packages/hypre/package.py create mode 100644 var/spack/repos/builtin/packages/icu/package.py create mode 100644 var/spack/repos/builtin/packages/icu4c/package.py create mode 100644 var/spack/repos/builtin/packages/isl/package.py create mode 100644 var/spack/repos/builtin/packages/jdk/package.py create mode 100644 var/spack/repos/builtin/packages/jpeg/package.py create mode 100644 var/spack/repos/builtin/packages/launchmon/package.py create mode 100644 var/spack/repos/builtin/packages/launchmon/patch.lmon_install_dir create mode 100644 var/spack/repos/builtin/packages/lcms/package.py create mode 100644 var/spack/repos/builtin/packages/leveldb/package.py create mode 100644 var/spack/repos/builtin/packages/libNBC/package.py create mode 100644 var/spack/repos/builtin/packages/libarchive/package.py create mode 100644 var/spack/repos/builtin/packages/libcircle/package.py create mode 100644 var/spack/repos/builtin/packages/libdrm/package.py create mode 100644 var/spack/repos/builtin/packages/libdwarf/package.py create mode 100644 var/spack/repos/builtin/packages/libelf/package.py create mode 100644 var/spack/repos/builtin/packages/libevent/package.py create mode 100644 var/spack/repos/builtin/packages/libffi/package.py create mode 100644 var/spack/repos/builtin/packages/libgcrypt/package.py create mode 100644 var/spack/repos/builtin/packages/libgpg-error/package.py create mode 100644 var/spack/repos/builtin/packages/libjpeg-turbo/package.py create mode 100644 var/spack/repos/builtin/packages/libjson-c/package.py create mode 100644 var/spack/repos/builtin/packages/libmng/package.py create mode 100644 var/spack/repos/builtin/packages/libmonitor/package.py create mode 100644 var/spack/repos/builtin/packages/libpciaccess/package.py create mode 100644 var/spack/repos/builtin/packages/libpng/package.py create mode 100644 var/spack/repos/builtin/packages/libsodium/package.py create mode 100644 var/spack/repos/builtin/packages/libtiff/package.py create mode 100644 var/spack/repos/builtin/packages/libtool/package.py create mode 100644 var/spack/repos/builtin/packages/libunwind/package.py create mode 100644 var/spack/repos/builtin/packages/libuuid/package.py create mode 100644 var/spack/repos/builtin/packages/libxcb/package.py create mode 100644 var/spack/repos/builtin/packages/libxml2/package.py create mode 100644 var/spack/repos/builtin/packages/libxshmfence/package.py create mode 100644 var/spack/repos/builtin/packages/libxslt/package.py create mode 100644 var/spack/repos/builtin/packages/llvm-lld/package.py create mode 100644 var/spack/repos/builtin/packages/llvm/package.py create mode 100644 var/spack/repos/builtin/packages/lmdb/package.py create mode 100644 var/spack/repos/builtin/packages/lua/package.py create mode 100644 var/spack/repos/builtin/packages/lwgrp/package.py create mode 100644 var/spack/repos/builtin/packages/lwm2/package.py create mode 100644 var/spack/repos/builtin/packages/matio/package.py create mode 100644 var/spack/repos/builtin/packages/memaxes/package.py create mode 100644 var/spack/repos/builtin/packages/mesa/package.py create mode 100644 var/spack/repos/builtin/packages/metis/package.py create mode 100644 var/spack/repos/builtin/packages/mpc/package.py create mode 100644 var/spack/repos/builtin/packages/mpe2/mpe2.patch create mode 100644 var/spack/repos/builtin/packages/mpe2/package.py create mode 100644 var/spack/repos/builtin/packages/mpfr/package.py create mode 100644 var/spack/repos/builtin/packages/mpibash/mpibash-4.3.patch create mode 100644 var/spack/repos/builtin/packages/mpibash/package.py create mode 100644 var/spack/repos/builtin/packages/mpich/package.py create mode 100644 var/spack/repos/builtin/packages/mpileaks/package.py create mode 100644 var/spack/repos/builtin/packages/mrnet/package.py create mode 100644 var/spack/repos/builtin/packages/munge/package.py create mode 100644 var/spack/repos/builtin/packages/muster/package.py create mode 100644 var/spack/repos/builtin/packages/mvapich2/ad_lustre_rwcontig_open_source.patch create mode 100644 var/spack/repos/builtin/packages/mvapich2/package.py create mode 100644 var/spack/repos/builtin/packages/nasm/package.py create mode 100644 var/spack/repos/builtin/packages/ncdu/package.py create mode 100644 var/spack/repos/builtin/packages/ncurses/package.py create mode 100644 var/spack/repos/builtin/packages/netcdf/netcdf-4.3.3-mpi.patch create mode 100644 var/spack/repos/builtin/packages/netcdf/package.py create mode 100644 var/spack/repos/builtin/packages/netgauge/package.py create mode 100644 var/spack/repos/builtin/packages/netlib-blas/package.py create mode 100644 var/spack/repos/builtin/packages/netlib-lapack/package.py create mode 100644 var/spack/repos/builtin/packages/nettle/package.py create mode 100644 var/spack/repos/builtin/packages/ompss/package.py create mode 100644 var/spack/repos/builtin/packages/ompt-openmp/package.py create mode 100644 var/spack/repos/builtin/packages/opari2/package.py create mode 100644 var/spack/repos/builtin/packages/openmpi/ad_lustre_rwcontig_open_source.patch create mode 100644 var/spack/repos/builtin/packages/openmpi/llnl-platforms.patch create mode 100644 var/spack/repos/builtin/packages/openmpi/package.py create mode 100644 var/spack/repos/builtin/packages/openssl/package.py create mode 100644 var/spack/repos/builtin/packages/otf/package.py create mode 100644 var/spack/repos/builtin/packages/otf2/package.py create mode 100644 var/spack/repos/builtin/packages/pango/package.py create mode 100644 var/spack/repos/builtin/packages/papi/package.py create mode 100644 var/spack/repos/builtin/packages/paraver/package.py create mode 100644 var/spack/repos/builtin/packages/paraview/package.py create mode 100644 var/spack/repos/builtin/packages/parmetis/package.py create mode 100644 var/spack/repos/builtin/packages/parpack/package.py create mode 100644 var/spack/repos/builtin/packages/pcre/package.py create mode 100644 var/spack/repos/builtin/packages/petsc/package.py create mode 100644 var/spack/repos/builtin/packages/pidx/package.py create mode 100644 var/spack/repos/builtin/packages/pixman/package.py create mode 100644 var/spack/repos/builtin/packages/pkg-config/package.py create mode 100644 var/spack/repos/builtin/packages/pmgr_collective/package.py create mode 100644 var/spack/repos/builtin/packages/postgresql/package.py create mode 100644 var/spack/repos/builtin/packages/ppl/package.py create mode 100644 var/spack/repos/builtin/packages/protobuf/package.py create mode 100644 var/spack/repos/builtin/packages/py-basemap/package.py create mode 100644 var/spack/repos/builtin/packages/py-biopython/package.py create mode 100644 var/spack/repos/builtin/packages/py-cffi/package.py create mode 100644 var/spack/repos/builtin/packages/py-cython/package.py create mode 100644 var/spack/repos/builtin/packages/py-dateutil/package.py create mode 100644 var/spack/repos/builtin/packages/py-epydoc/package.py create mode 100644 var/spack/repos/builtin/packages/py-genders/package.py create mode 100644 var/spack/repos/builtin/packages/py-gnuplot/package.py create mode 100644 var/spack/repos/builtin/packages/py-h5py/package.py create mode 100644 var/spack/repos/builtin/packages/py-ipython/package.py create mode 100644 var/spack/repos/builtin/packages/py-libxml2/package.py create mode 100644 var/spack/repos/builtin/packages/py-lockfile/package.py create mode 100644 var/spack/repos/builtin/packages/py-mako/package.py create mode 100644 var/spack/repos/builtin/packages/py-matplotlib/package.py create mode 100644 var/spack/repos/builtin/packages/py-mock/package.py create mode 100644 var/spack/repos/builtin/packages/py-mpi4py/package.py create mode 100644 var/spack/repos/builtin/packages/py-mx/package.py create mode 100644 var/spack/repos/builtin/packages/py-nose/package.py create mode 100644 var/spack/repos/builtin/packages/py-numpy/package.py create mode 100644 var/spack/repos/builtin/packages/py-pandas/package.py create mode 100644 var/spack/repos/builtin/packages/py-pexpect/package.py create mode 100644 var/spack/repos/builtin/packages/py-pil/package.py create mode 100644 var/spack/repos/builtin/packages/py-pmw/package.py create mode 100644 var/spack/repos/builtin/packages/py-pychecker/package.py create mode 100644 var/spack/repos/builtin/packages/py-pycparser/package.py create mode 100644 var/spack/repos/builtin/packages/py-pyelftools/package.py create mode 100644 var/spack/repos/builtin/packages/py-pygments/package.py create mode 100644 var/spack/repos/builtin/packages/py-pylint/package.py create mode 100644 var/spack/repos/builtin/packages/py-pypar/package.py create mode 100644 var/spack/repos/builtin/packages/py-pyparsing/package.py create mode 100644 var/spack/repos/builtin/packages/py-pyqt/package.py create mode 100644 var/spack/repos/builtin/packages/py-pyside/package.py create mode 100644 var/spack/repos/builtin/packages/py-python-daemon/package.py create mode 100644 var/spack/repos/builtin/packages/py-pytz/package.py create mode 100644 var/spack/repos/builtin/packages/py-rpy2/package.py create mode 100644 var/spack/repos/builtin/packages/py-scientificpython/package.py create mode 100644 var/spack/repos/builtin/packages/py-scikit-learn/package.py create mode 100644 var/spack/repos/builtin/packages/py-scipy/package.py create mode 100644 var/spack/repos/builtin/packages/py-setuptools/package.py create mode 100644 var/spack/repos/builtin/packages/py-shiboken/package.py create mode 100644 var/spack/repos/builtin/packages/py-sip/package.py create mode 100644 var/spack/repos/builtin/packages/py-six/package.py create mode 100644 var/spack/repos/builtin/packages/py-sphinx/package.py create mode 100644 var/spack/repos/builtin/packages/py-sympy/package.py create mode 100644 var/spack/repos/builtin/packages/py-virtualenv/package.py create mode 100644 var/spack/repos/builtin/packages/py-yapf/package.py create mode 100644 var/spack/repos/builtin/packages/python/package.py create mode 100644 var/spack/repos/builtin/packages/qhull/package.py create mode 100644 var/spack/repos/builtin/packages/qt/package.py create mode 100644 var/spack/repos/builtin/packages/qthreads/package.py create mode 100644 var/spack/repos/builtin/packages/ravel/package.py create mode 100644 var/spack/repos/builtin/packages/readline/package.py create mode 100644 var/spack/repos/builtin/packages/rose/add_spack_compiler_recognition.patch create mode 100644 var/spack/repos/builtin/packages/rose/package.py create mode 100644 var/spack/repos/builtin/packages/ruby/package.py create mode 100644 var/spack/repos/builtin/packages/samtools/package.py create mode 100644 var/spack/repos/builtin/packages/samtools/samtools1.2.patch create mode 100644 var/spack/repos/builtin/packages/scalasca/package.py create mode 100644 var/spack/repos/builtin/packages/scorep/package.py create mode 100644 var/spack/repos/builtin/packages/scotch/package.py create mode 100644 var/spack/repos/builtin/packages/scr/package.py create mode 100644 var/spack/repos/builtin/packages/silo/package.py create mode 100644 var/spack/repos/builtin/packages/snappy/package.py create mode 100644 var/spack/repos/builtin/packages/spindle/package.py create mode 100644 var/spack/repos/builtin/packages/sqlite/package.py create mode 100644 var/spack/repos/builtin/packages/stat/configure_mpicxx.patch create mode 100644 var/spack/repos/builtin/packages/stat/package.py create mode 100644 var/spack/repos/builtin/packages/sundials/package.py create mode 100644 var/spack/repos/builtin/packages/swig/package.py create mode 100644 var/spack/repos/builtin/packages/task/package.py create mode 100644 var/spack/repos/builtin/packages/taskd/package.py create mode 100644 var/spack/repos/builtin/packages/tau/package.py create mode 100644 var/spack/repos/builtin/packages/tcl/package.py create mode 100644 var/spack/repos/builtin/packages/the_silver_searcher/package.py create mode 100644 var/spack/repos/builtin/packages/thrift/package.py create mode 100644 var/spack/repos/builtin/packages/tk/package.py create mode 100644 var/spack/repos/builtin/packages/tmux/package.py create mode 100644 var/spack/repos/builtin/packages/tmuxinator/package.py create mode 100644 var/spack/repos/builtin/packages/trilinos/package.py create mode 100644 var/spack/repos/builtin/packages/uncrustify/package.py create mode 100644 var/spack/repos/builtin/packages/util-linux/package.py create mode 100644 var/spack/repos/builtin/packages/vim/package.py create mode 100644 var/spack/repos/builtin/packages/vtk/package.py create mode 100644 var/spack/repos/builtin/packages/wget/package.py create mode 100644 var/spack/repos/builtin/packages/wx/package.py create mode 100644 var/spack/repos/builtin/packages/wxpropgrid/package.py create mode 100644 var/spack/repos/builtin/packages/xcb-proto/package.py create mode 100644 var/spack/repos/builtin/packages/xz/package.py create mode 100644 var/spack/repos/builtin/packages/yasm/package.py create mode 100644 var/spack/repos/builtin/packages/zeromq/package.py create mode 100644 var/spack/repos/builtin/packages/zlib/package.py create mode 100644 var/spack/repos/builtin/packages/zsh/package.py create mode 100644 var/spack/repos/builtin/repo.yaml (limited to 'lib') diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index 4f481ce937..aab20cb260 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -43,7 +43,7 @@ test_path = join_path(module_path, "test") hooks_path = join_path(module_path, "hooks") var_path = join_path(spack_root, "var", "spack") stage_path = join_path(var_path, "stage") -packages_path = join_path(var_path, "packages") +repos_path = join_path(var_path, "repos") share_path = join_path(spack_root, "share", "spack") prefix = spack_root @@ -58,8 +58,12 @@ import spack.repository _repo_paths = spack.config.get_repos_config() if not _repo_paths: tty.die("Spack configuration contains no package repositories.") -repo = spack.repository.RepoPath(*_repo_paths) -sys.meta_path.append(repo) + +try: + repo = spack.repository.RepoPath(*_repo_paths) + sys.meta_path.append(repo) +except spack.repository.BadRepoError, e: + tty.die('Bad repository. %s' % e.message) # # Set up the installed packages database @@ -68,9 +72,10 @@ from spack.database import Database installed_db = Database(install_path) # -# Paths to mock files for testing. +# Paths to built-in Spack repositories. # -mock_packages_path = join_path(var_path, "mock_packages") +packages_path = join_path(repos_path, "builtin") +mock_packages_path = join_path(repos_path, "builtin.mock") mock_config_path = join_path(var_path, "mock_configs") mock_site_config = join_path(mock_config_path, "site_spackconfig") diff --git a/lib/spack/spack/cmd/repo.py b/lib/spack/spack/cmd/repo.py index 85cc83730c..395aa90bed 100644 --- a/lib/spack/spack/cmd/repo.py +++ b/lib/spack/spack/cmd/repo.py @@ -32,7 +32,7 @@ from llnl.util.filesystem import join_path, mkdirp import spack.spec import spack.config from spack.util.environment import get_path -from spack.repository import repo_config_filename +from spack.repository import repo_config_name import os import exceptions diff --git a/lib/spack/spack/repository.py b/lib/spack/spack/repository.py index c1545b3654..a2c0bbe147 100644 --- a/lib/spack/spack/repository.py +++ b/lib/spack/spack/repository.py @@ -26,28 +26,32 @@ import os import exceptions import sys import inspect -import glob import imp import re -import itertools import traceback from bisect import bisect_left from external import yaml import llnl.util.tty as tty from llnl.util.filesystem import join_path -from llnl.util.lang import * import spack.error import spack.spec from spack.virtual import ProviderIndex from spack.util.naming import * -# Filename for package repo names -repo_config_filename = '_repo.yaml' +# +# Super-namespace for all packages. +# Package modules are imported as spack.pkg... +# +repo_namespace = 'spack.pkg' -# Filename for packages in repos. -package_file_name = 'package.py' +# +# These names describe how repos should be laid out in the filesystem. +# +repo_config_name = 'repo.yaml' # Top-level filename for repo config. +packages_dir_name = 'packages' # Top-level repo directory containing pkgs. +package_file_name = 'package.py' # Filename for packages in a repository. def _autospec(function): """Decorator that automatically converts the argument of a single-arg @@ -74,7 +78,10 @@ class RepoPath(object): combined results of the Repos in its list instead of on a single package repository. """ - def __init__(self, *repo_dirs): + def __init__(self, *repo_dirs, **kwargs): + # super-namespace for all packages in the RepoPath + self.super_namespace = kwargs.get('namespace', repo_namespace) + self.repos = [] self.by_namespace = NamespaceTrie() self.by_path = {} @@ -82,11 +89,9 @@ class RepoPath(object): self._all_package_names = [] self._provider_index = None + # Add each repo to this path. for root in repo_dirs: - # Try to make it a repo if it's not one. - if not isinstance(root, Repo): - repo = Repo(root) - # Add the repo to the path. + repo = Repo(root, self.super_namespace) self.put_last(repo) @@ -120,11 +125,11 @@ class RepoPath(object): repo, self.by_path[repo.root]) if repo.namespace in self.by_namespace: - raise DuplicateRepoError("Package repos cannot have the same name", + raise DuplicateRepoError("Package repos cannot provide the same namespace", repo, self.by_namespace[repo.namespace]) # Add repo to the pkg indexes - self.by_namespace[repo.namespace] = repo + self.by_namespace[repo.full_namespace] = repo self.by_path[repo.root] = repo # add names to the cached name list @@ -185,10 +190,10 @@ class RepoPath(object): # If it's a module in some repo, or if it is the repo's # namespace, let the repo handle it. for repo in self.repos: - if namespace == repo.namespace: + if namespace == repo.full_namespace: if repo.real_name(module_name): return repo - elif fullname == repo.namespace: + elif fullname == repo.full_namespace: return repo # No repo provides the namespace, but it is a valid prefix of @@ -200,13 +205,14 @@ class RepoPath(object): def load_module(self, fullname): - """Loads containing namespaces when necessary. + """Handles loading container namespaces when necessary. See ``Repo`` for how actual package modules are loaded. """ if fullname in sys.modules: return sys.modules[fullname] + # partition fullname into prefix and module name. namespace, dot, module_name = fullname.rpartition('.') @@ -252,41 +258,67 @@ class Repo(object): """Class representing a package repository in the filesystem. Each package repository must have a top-level configuration file - called `_repo.yaml`. + called `repo.yaml`. - Currently, `_repo.yaml` this must define: + Currently, `repo.yaml` this must define: `namespace`: A Python namespace where the repository's packages should live. """ - def __init__(self, root): - """Instantiate a package repository from a filesystem path.""" + def __init__(self, root, namespace=repo_namespace): + """Instantiate a package repository from a filesystem path. + + Arguments: + root The root directory of the repository. + + namespace A super-namespace that will contain the repo-defined + namespace (this is generally jsut `spack.pkg`). The + super-namespace is Spack's way of separating repositories + from other python namespaces. + + """ # Root directory, containing _repo.yaml and package dirs self.root = root - # Config file in /_repo.yaml - self.config_file = os.path.join(self.root, repo_config_filename) + # super-namespace for all packages in the Repo + self.super_namespace = namespace - # Read configuration from _repo.yaml + # check and raise BadRepoError on fail. + def check(condition, msg): + if not condition: raise BadRepoError(msg) + + # Validate repository layout. + self.config_file = join_path(self.root, repo_config_name) + check(os.path.isfile(self.config_file), + "No %s found in '%s'" % (repo_config_name, root)) + self.packages_path = join_path(self.root, packages_dir_name) + check(os.path.isdir(self.packages_path), + "No directory '%s' found in '%s'" % (repo_config_name, root)) + + # Read configuration and validate namespace config = self._read_config() - if not 'namespace' in config: - tty.die('Package repo in %s must define a namespace in %s.' - % (self.root, repo_config_filename)) + check('namespace' in config, '%s must define a namespace.' + % join_path(self.root, repo_config_name)) - # Check namespace in the repository configuration. self.namespace = config['namespace'] - if not re.match(r'[a-zA-Z][a-zA-Z0-9_.]+', self.namespace): - tty.die(("Invalid namespace '%s' in '%s'. Namespaces must be " - "valid python identifiers separated by '.'") - % (self.namespace, self.root)) - self._names = self.namespace.split('.') + check(re.match(r'[a-zA-Z][a-zA-Z0-9_.]+', self.namespace), + ("Invalid namespace '%s' in repo '%s'. " % (self.namespace, self.root)) + + "Namespaces must be valid python identifiers separated by '.'") + + # Set up 'full_namespace' to include the super-namespace + if self.super_namespace: + self.full_namespace = "%s.%s" % (self.super_namespace, self.namespace) + else: + self.full_namespace = self.namespace + + # Keep name components around for checking prefixes. + self._names = self.full_namespace.split('.') # These are internal cache variables. self._modules = {} self._classes = {} self._instances = {} - self._provider_index = None self._all_package_names = None @@ -301,11 +333,27 @@ class Repo(object): we don't get runtime warnings from Python's module system. """ + parent = None for l in range(1, len(self._names)+1): ns = '.'.join(self._names[:l]) if not ns in sys.modules: - sys.modules[ns] = _make_namespace_module(ns) - sys.modules[ns].__loader__ = self + module = _make_namespace_module(ns) + module.__loader__ = self + sys.modules[ns] = module + + # Ensure the namespace is an atrribute of its parent, + # if it has not been set by something else already. + # + # This ensures that we can do things like: + # import spack.pkg.builtin.mpich as mpich + if parent: + modname = self._names[l-1] + if not hasattr(parent, modname): + setattr(parent, modname, module) + else: + # no need to set up a module, but keep track of the parent. + module = sys.modules[ns] + parent = module def real_name(self, import_name): @@ -349,7 +397,7 @@ class Repo(object): return self namespace, dot, module_name = fullname.rpartition('.') - if namespace == self.namespace: + if namespace == self.full_namespace: if self.real_name(module_name): return self @@ -369,14 +417,14 @@ class Repo(object): if self.is_prefix(fullname): module = _make_namespace_module(fullname) - elif namespace == self.namespace: + elif namespace == self.full_namespace: real_name = self.real_name(module_name) if not real_name: - raise ImportError("No module %s in repo %s" % (module_name, namespace)) + raise ImportError("No module %s in %s" % (module_name, self)) module = self._get_pkg_module(real_name) else: - raise ImportError("No module %s in repo %s" % (fullname, self.namespace)) + raise ImportError("No module %s in %s" % (fullname, self)) module.__loader__ = self sys.modules[fullname] = module @@ -392,7 +440,7 @@ class Repo(object): if (not yaml_data or 'repo' not in yaml_data or not isinstance(yaml_data['repo'], dict)): tty.die("Invalid %s in repository %s" - % (repo_config_filename, self.root)) + % (repo_config_name, self.root)) return yaml_data['repo'] @@ -446,7 +494,7 @@ class Repo(object): def dirname_for_package_name(self, pkg_name): """Get the directory name for a particular package. This is the directory that contains its package.py file.""" - return join_path(self.root, pkg_name) + return join_path(self.packages_path, pkg_name) def filename_for_package_name(self, pkg_name): @@ -460,7 +508,6 @@ class Repo(object): """ validate_module_name(pkg_name) pkg_dir = self.dirname_for_package_name(pkg_name) - return join_path(pkg_dir, package_file_name) @@ -469,12 +516,25 @@ class Repo(object): if self._all_package_names is None: self._all_package_names = [] - for pkg_name in os.listdir(self.root): - pkg_dir = join_path(self.root, pkg_name) - pkg_file = join_path(pkg_dir, package_file_name) - if os.path.isfile(pkg_file): - self._all_package_names.append(pkg_name) - + for pkg_name in os.listdir(self.packages_path): + # Skip non-directories in the package root. + pkg_dir = join_path(self.packages_path, pkg_name) + if not os.path.isdir(pkg_dir): + continue + + # Skip directories without a package.py in them. + pkg_file = join_path(self.packages_path, pkg_name, package_file_name) + if not os.path.isfile(pkg_file): + continue + + # Warn about invalid names that look like packages. + if not valid_module_name(pkg_name): + tty.warn("Skipping package at %s. '%s' is not a valid Spack module name." + % (pkg_dir, pkg_name)) + continue + + # All checks passed. Add it to the list. + self._all_package_names.append(pkg_name) self._all_package_names.sort() return self._all_package_names @@ -489,7 +549,8 @@ class Repo(object): """Whether a package with the supplied name exists.""" # This does a binary search in the sorted list. idx = bisect_left(self.all_package_names(), pkg_name) - return self._all_package_names[idx] == pkg_name + return (idx < len(self._all_package_names) and + self._all_package_names[idx] == pkg_name) def _get_pkg_module(self, pkg_name): @@ -505,7 +566,7 @@ class Repo(object): file_path = self.filename_for_package_name(pkg_name) if not os.path.exists(file_path): - raise UnknownPackageError(pkg_name, self.namespace) + raise UnknownPackageError(pkg_name, self) if not os.path.isfile(file_path): tty.die("Something's wrong. '%s' is not a file!" % file_path) @@ -513,10 +574,11 @@ class Repo(object): if not os.access(file_path, os.R_OK): tty.die("Cannot read '%s'!" % file_path) - fullname = "%s.%s" % (self.namespace, pkg_name) + # e.g., spack.pkg.builtin.mpich + fullname = "%s.%s" % (self.full_namespace, pkg_name) module = imp.load_source(fullname, file_path) - module.__package__ = self.namespace + module.__package__ = self.full_namespace module.__loader__ = self self._modules[pkg_name] = module @@ -541,7 +603,7 @@ class Repo(object): def __str__(self): - return "" % (self.namespace, self.root) + return "[Repo '%s' at '%s']" % (self.namespace, self.root) def __repr__(self): @@ -597,12 +659,18 @@ class Repo(object): yield spec +class BadRepoError(spack.error.SpackError): + """Raised when repo layout is invalid.""" + def __init__(self, msg): + super(BadRepoError, self).__init__(msg) + + class UnknownPackageError(spack.error.SpackError): """Raised when we encounter a package spack doesn't have.""" def __init__(self, name, repo=None): msg = None if repo: - msg = "Package %s not found in packagerepo %s." % (name, repo) + msg = "Package %s not found in repository %s." % (name, repo) else: msg = "Package %s not found." % name super(UnknownPackageError, self).__init__(msg) diff --git a/var/spack/mock_packages/_repo.yaml b/var/spack/mock_packages/_repo.yaml deleted file mode 100644 index b97b978de3..0000000000 --- a/var/spack/mock_packages/_repo.yaml +++ /dev/null @@ -1,2 +0,0 @@ -repo: - namespace: gov.llnl.spack.mock diff --git a/var/spack/mock_packages/a/package.py b/var/spack/mock_packages/a/package.py deleted file mode 100644 index fa63c08df0..0000000000 --- a/var/spack/mock_packages/a/package.py +++ /dev/null @@ -1,12 +0,0 @@ -from spack import * - -class A(Package): - """Simple package with no dependencies""" - - homepage = "http://www.example.com" - url = "http://www.example.com/a-1.0.tar.gz" - - version('1.0', '0123456789abcdef0123456789abcdef') - - def install(self, spec, prefix): - pass diff --git a/var/spack/mock_packages/b/package.py b/var/spack/mock_packages/b/package.py deleted file mode 100644 index cb88aa2157..0000000000 --- a/var/spack/mock_packages/b/package.py +++ /dev/null @@ -1,12 +0,0 @@ -from spack import * - -class B(Package): - """Simple package with no dependencies""" - - homepage = "http://www.example.com" - url = "http://www.example.com/b-1.0.tar.gz" - - version('1.0', '0123456789abcdef0123456789abcdef') - - def install(self, spec, prefix): - pass diff --git a/var/spack/mock_packages/c/package.py b/var/spack/mock_packages/c/package.py deleted file mode 100644 index f51b913fa9..0000000000 --- a/var/spack/mock_packages/c/package.py +++ /dev/null @@ -1,12 +0,0 @@ -from spack import * - -class C(Package): - """Simple package with no dependencies""" - - homepage = "http://www.example.com" - url = "http://www.example.com/c-1.0.tar.gz" - - version('1.0', '0123456789abcdef0123456789abcdef') - - def install(self, spec, prefix): - pass diff --git a/var/spack/mock_packages/callpath/package.py b/var/spack/mock_packages/callpath/package.py deleted file mode 100644 index 5b6b70ba2a..0000000000 --- a/var/spack/mock_packages/callpath/package.py +++ /dev/null @@ -1,41 +0,0 @@ -############################################################################## -# Copyright (c) 2013, Lawrence Livermore National Security, LLC. -# Produced at the Lawrence Livermore National Laboratory. -# -# This file is part of Spack. -# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. -# LLNL-CODE-647188 -# -# For details, see https://scalability-llnl.github.io/spack -# Please also see the LICENSE file for our notice and the LGPL. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License (as published by -# the Free Software Foundation) version 2.1 dated February 1999. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and -# conditions of the GNU General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -############################################################################## -from spack import * - -class Callpath(Package): - homepage = "https://github.com/tgamblin/callpath" - url = "http://github.com/tgamblin/callpath-1.0.tar.gz" - - version(0.8, 'foobarbaz') - version(0.9, 'foobarbaz') - version(1.0, 'foobarbaz') - - depends_on("dyninst") - depends_on("mpi") - - def install(self, spec, prefix): - configure("--prefix=%s" % prefix) - make() - make("install") diff --git a/var/spack/mock_packages/direct_mpich/package.py b/var/spack/mock_packages/direct_mpich/package.py deleted file mode 100644 index 2ced82521b..0000000000 --- a/var/spack/mock_packages/direct_mpich/package.py +++ /dev/null @@ -1,36 +0,0 @@ -############################################################################## -# Copyright (c) 2013, Lawrence Livermore National Security, LLC. -# Produced at the Lawrence Livermore National Laboratory. -# -# This file is part of Spack. -# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. -# LLNL-CODE-647188 -# -# For details, see https://scalability-llnl.github.io/spack -# Please also see the LICENSE file for our notice and the LGPL. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License (as published by -# the Free Software Foundation) version 2.1 dated February 1999. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and -# conditions of the GNU General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -############################################################################## -from spack import * - -class DirectMpich(Package): - homepage = "http://www.example.com" - url = "http://www.example.com/direct_mpich-1.0.tar.gz" - - version('1.0', 'foobarbaz') - - depends_on('mpich') - - def install(self, spec, prefix): - pass diff --git a/var/spack/mock_packages/dyninst/package.py b/var/spack/mock_packages/dyninst/package.py deleted file mode 100644 index 7998578da1..0000000000 --- a/var/spack/mock_packages/dyninst/package.py +++ /dev/null @@ -1,44 +0,0 @@ -############################################################################## -# Copyright (c) 2013, Lawrence Livermore National Security, LLC. -# Produced at the Lawrence Livermore National Laboratory. -# -# This file is part of Spack. -# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. -# LLNL-CODE-647188 -# -# For details, see https://scalability-llnl.github.io/spack -# Please also see the LICENSE file for our notice and the LGPL. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License (as published by -# the Free Software Foundation) version 2.1 dated February 1999. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and -# conditions of the GNU General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -############################################################################## -from spack import * - -class Dyninst(Package): - homepage = "https://paradyn.org" - url = "http://www.paradyn.org/release8.1/DyninstAPI-8.1.1.tgz" - - version('8.2', 'cxyzab', - url='http://www.paradyn.org/release8.2/DyninstAPI-8.2.tgz') - version('8.1.2', 'bcxyza', - url='http://www.paradyn.org/release8.1.2/DyninstAPI-8.1.2.tgz') - version('8.1.1', 'abcxyz', - url='http://www.paradyn.org/release8.1/DyninstAPI-8.1.1.tgz') - - depends_on("libelf") - depends_on("libdwarf") - - def install(self, spec, prefix): - configure("--prefix=%s" % prefix) - make() - make("install") diff --git a/var/spack/mock_packages/e/package.py b/var/spack/mock_packages/e/package.py deleted file mode 100644 index 76c6b64c7f..0000000000 --- a/var/spack/mock_packages/e/package.py +++ /dev/null @@ -1,12 +0,0 @@ -from spack import * - -class E(Package): - """Simple package with no dependencies""" - - homepage = "http://www.example.com" - url = "http://www.example.com/e-1.0.tar.gz" - - version('1.0', '0123456789abcdef0123456789abcdef') - - def install(self, spec, prefix): - pass diff --git a/var/spack/mock_packages/fake/package.py b/var/spack/mock_packages/fake/package.py deleted file mode 100644 index fb3c2bdd2e..0000000000 --- a/var/spack/mock_packages/fake/package.py +++ /dev/null @@ -1,34 +0,0 @@ -############################################################################## -# Copyright (c) 2013, Lawrence Livermore National Security, LLC. -# Produced at the Lawrence Livermore National Laboratory. -# -# This file is part of Spack. -# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. -# LLNL-CODE-647188 -# -# For details, see https://scalability-llnl.github.io/spack -# Please also see the LICENSE file for our notice and the LGPL. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License (as published by -# the Free Software Foundation) version 2.1 dated February 1999. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and -# conditions of the GNU General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -############################################################################## -from spack import * - -class Fake(Package): - homepage = "http://www.fake-spack-example.org" - url = "http://www.fake-spack-example.org/downloads/fake-1.0.tar.gz" - - version('1.0', 'foobarbaz') - - def install(self, spec, prefix): - pass diff --git a/var/spack/mock_packages/git-test/package.py b/var/spack/mock_packages/git-test/package.py deleted file mode 100644 index 689185463c..0000000000 --- a/var/spack/mock_packages/git-test/package.py +++ /dev/null @@ -1,10 +0,0 @@ -from spack import * - -class GitTest(Package): - """Mock package that uses git for fetching.""" - homepage = "http://www.git-fetch-example.com" - - version('git', git='to-be-filled-in-by-test') - - def install(self, spec, prefix): - pass diff --git a/var/spack/mock_packages/hg-test/package.py b/var/spack/mock_packages/hg-test/package.py deleted file mode 100644 index 462f1e4c3a..0000000000 --- a/var/spack/mock_packages/hg-test/package.py +++ /dev/null @@ -1,10 +0,0 @@ -from spack import * - -class HgTest(Package): - """Test package that does fetching with mercurial.""" - homepage = "http://www.hg-fetch-example.com" - - version('hg', hg='to-be-filled-in-by-test') - - def install(self, spec, prefix): - pass diff --git a/var/spack/mock_packages/indirect_mpich/package.py b/var/spack/mock_packages/indirect_mpich/package.py deleted file mode 100644 index daf8b4b166..0000000000 --- a/var/spack/mock_packages/indirect_mpich/package.py +++ /dev/null @@ -1,41 +0,0 @@ -############################################################################## -# Copyright (c) 2013, Lawrence Livermore National Security, LLC. -# Produced at the Lawrence Livermore National Laboratory. -# -# This file is part of Spack. -# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. -# LLNL-CODE-647188 -# -# For details, see https://scalability-llnl.github.io/spack -# Please also see the LICENSE file for our notice and the LGPL. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License (as published by -# the Free Software Foundation) version 2.1 dated February 1999. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and -# conditions of the GNU General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -############################################################################## -from spack import * - -class IndirectMpich(Package): - """Test case for a package that depends on MPI and one of its - dependencies requires a *particular version* of MPI. - """ - - homepage = "http://www.example.com" - url = "http://www.example.com/indirect_mpich-1.0.tar.gz" - - version(1.0, 'foobarbaz') - - depends_on('mpi') - depends_on('direct_mpich') - - def install(self, spec, prefix): - pass diff --git a/var/spack/mock_packages/libdwarf/package.py b/var/spack/mock_packages/libdwarf/package.py deleted file mode 100644 index 0b8df04cfb..0000000000 --- a/var/spack/mock_packages/libdwarf/package.py +++ /dev/null @@ -1,44 +0,0 @@ -############################################################################## -# Copyright (c) 2013, Lawrence Livermore National Security, LLC. -# Produced at the Lawrence Livermore National Laboratory. -# -# This file is part of Spack. -# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. -# LLNL-CODE-647188 -# -# For details, see https://scalability-llnl.github.io/spack -# Please also see the LICENSE file for our notice and the LGPL. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License (as published by -# the Free Software Foundation) version 2.1 dated February 1999. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and -# conditions of the GNU General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -############################################################################## -from spack import * -import os - -# Only build certain parts of dwarf because the other ones break. -dwarf_dirs = ['libdwarf', 'dwarfdump2'] - -class Libdwarf(Package): - homepage = "http://www.prevanders.net/dwarf.html" - url = "http://www.prevanders.net/libdwarf-20130729.tar.gz" - list_url = homepage - - version(20130729, "64b42692e947d5180e162e46c689dfbf") - version(20130207, 'foobarbaz') - version(20111030, 'foobarbaz') - version(20070703, 'foobarbaz') - - depends_on("libelf") - - def install(self, spec, prefix): - pass diff --git a/var/spack/mock_packages/libelf/package.py b/var/spack/mock_packages/libelf/package.py deleted file mode 100644 index 94c8f942cd..0000000000 --- a/var/spack/mock_packages/libelf/package.py +++ /dev/null @@ -1,43 +0,0 @@ -############################################################################## -# Copyright (c) 2013, Lawrence Livermore National Security, LLC. -# Produced at the Lawrence Livermore National Laboratory. -# -# This file is part of Spack. -# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. -# LLNL-CODE-647188 -# -# For details, see https://scalability-llnl.github.io/spack -# Please also see the LICENSE file for our notice and the LGPL. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License (as published by -# the Free Software Foundation) version 2.1 dated February 1999. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and -# conditions of the GNU General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -############################################################################## -from spack import * - -class Libelf(Package): - homepage = "http://www.mr511.de/software/english.html" - url = "http://www.mr511.de/software/libelf-0.8.13.tar.gz" - - version('0.8.13', '4136d7b4c04df68b686570afa26988ac') - version('0.8.12', 'e21f8273d9f5f6d43a59878dc274fec7') - version('0.8.10', '9db4d36c283d9790d8fa7df1f4d7b4d9') - - def install(self, spec, prefix): - configure("--prefix=%s" % prefix, - "--enable-shared", - "--disable-dependency-tracking", - "--disable-debug") - make() - - # The mkdir commands in libelf's intsall can fail in parallel - make("install", parallel=False) diff --git a/var/spack/mock_packages/mpich/package.py b/var/spack/mock_packages/mpich/package.py deleted file mode 100644 index f77d3efc5d..0000000000 --- a/var/spack/mock_packages/mpich/package.py +++ /dev/null @@ -1,46 +0,0 @@ -############################################################################## -# Copyright (c) 2013, Lawrence Livermore National Security, LLC. -# Produced at the Lawrence Livermore National Laboratory. -# -# This file is part of Spack. -# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. -# LLNL-CODE-647188 -# -# For details, see https://scalability-llnl.github.io/spack -# Please also see the LICENSE file for our notice and the LGPL. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License (as published by -# the Free Software Foundation) version 2.1 dated February 1999. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and -# conditions of the GNU General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -############################################################################## -from spack import * - -class Mpich(Package): - homepage = "http://www.mpich.org" - url = "http://www.mpich.org/static/downloads/3.0.4/mpich-3.0.4.tar.gz" - list_url = "http://www.mpich.org/static/downloads/" - list_depth = 2 - - variant('debug', default=False, - description="Compile MPICH with debug flags.") - - version('3.0.4', '9c5d5d4fe1e17dd12153f40bc5b6dbc0') - version('3.0.3', 'foobarbaz') - version('3.0.2', 'foobarbaz') - version('3.0.1', 'foobarbaz') - version('3.0', 'foobarbaz') - - provides('mpi@:3', when='@3:') - provides('mpi@:1', when='@:1') - - def install(self, spec, prefix): - pass diff --git a/var/spack/mock_packages/mpich2/package.py b/var/spack/mock_packages/mpich2/package.py deleted file mode 100644 index 827b94c8a4..0000000000 --- a/var/spack/mock_packages/mpich2/package.py +++ /dev/null @@ -1,47 +0,0 @@ -############################################################################## -# Copyright (c) 2013, Lawrence Livermore National Security, LLC. -# Produced at the Lawrence Livermore National Laboratory. -# -# This file is part of Spack. -# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. -# LLNL-CODE-647188 -# -# For details, see https://scalability-llnl.github.io/spack -# Please also see the LICENSE file for our notice and the LGPL. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License (as published by -# the Free Software Foundation) version 2.1 dated February 1999. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and -# conditions of the GNU General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -############################################################################## -from spack import * - -class Mpich2(Package): - homepage = "http://www.mpich.org" - url = "http://www.mpich.org/static/downloads/1.5/mpich2-1.5.tar.gz" - list_url = "http://www.mpich.org/static/downloads/" - list_depth = 2 - - version('1.5', '9c5d5d4fe1e17dd12153f40bc5b6dbc0') - version('1.4', 'foobarbaz') - version('1.3', 'foobarbaz') - version('1.2', 'foobarbaz') - version('1.1', 'foobarbaz') - version('1.0', 'foobarbaz') - - provides('mpi@:2.0') - provides('mpi@:2.1', when='@1.1:') - provides('mpi@:2.2', when='@1.2:') - - def install(self, spec, prefix): - configure("--prefix=%s" % prefix) - make() - make("install") diff --git a/var/spack/mock_packages/mpileaks/package.py b/var/spack/mock_packages/mpileaks/package.py deleted file mode 100644 index 3989f1b452..0000000000 --- a/var/spack/mock_packages/mpileaks/package.py +++ /dev/null @@ -1,43 +0,0 @@ -############################################################################## -# Copyright (c) 2013, Lawrence Livermore National Security, LLC. -# Produced at the Lawrence Livermore National Laboratory. -# -# This file is part of Spack. -# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. -# LLNL-CODE-647188 -# -# For details, see https://scalability-llnl.github.io/spack -# Please also see the LICENSE file for our notice and the LGPL. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License (as published by -# the Free Software Foundation) version 2.1 dated February 1999. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and -# conditions of the GNU General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -############################################################################## -from spack import * - -class Mpileaks(Package): - homepage = "http://www.llnl.gov" - url = "http://www.llnl.gov/mpileaks-1.0.tar.gz" - - version(1.0, 'foobarbaz') - version(2.1, 'foobarbaz') - version(2.2, 'foobarbaz') - version(2.3, 'foobarbaz') - - variant('debug', default=False, description='Debug variant') - variant('opt', default=False, description='Optimized variant') - - depends_on("mpi") - depends_on("callpath") - - def install(self, spec, prefix): - pass diff --git a/var/spack/mock_packages/multimethod/package.py b/var/spack/mock_packages/multimethod/package.py deleted file mode 100644 index 75b1606ffc..0000000000 --- a/var/spack/mock_packages/multimethod/package.py +++ /dev/null @@ -1,143 +0,0 @@ -############################################################################## -# Copyright (c) 2013, Lawrence Livermore National Security, LLC. -# Produced at the Lawrence Livermore National Laboratory. -# -# This file is part of Spack. -# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. -# LLNL-CODE-647188 -# -# For details, see https://scalability-llnl.github.io/spack -# Please also see the LICENSE file for our notice and the LGPL. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License (as published by -# the Free Software Foundation) version 2.1 dated February 1999. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and -# conditions of the GNU General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -############################################################################## -from spack import * - - -class Multimethod(Package): - """This package is designed for use with Spack's multimethod test. - It has a bunch of test cases for the @when decorator that the - test uses. - """ - - homepage = 'http://www.example.com/' - url = 'http://www.example.com/example-1.0.tar.gz' - - # - # These functions are only valid for versions 1, 2, and 3. - # - @when('@1.0') - def no_version_2(self): - return 1 - - @when('@3.0') - def no_version_2(self): - return 3 - - @when('@4.0') - def no_version_2(self): - return 4 - - - # - # These functions overlap, so there is ambiguity, but we'll take - # the first one. - # - @when('@:4') - def version_overlap(self): - return 1 - - @when('@2:') - def version_overlap(self): - return 2 - - - # - # More complicated case with cascading versions. - # - def mpi_version(self): - return 0 - - @when('^mpi@3:') - def mpi_version(self): - return 3 - - @when('^mpi@2:') - def mpi_version(self): - return 2 - - @when('^mpi@1:') - def mpi_version(self): - return 1 - - - # - # Use these to test whether the default method is called when no - # match is found. This also tests whether we can switch methods - # on compilers - # - def has_a_default(self): - return 'default' - - @when('%gcc') - def has_a_default(self): - return 'gcc' - - @when('%intel') - def has_a_default(self): - return 'intel' - - - - # - # Make sure we can switch methods on different architectures - # - @when('=x86_64') - def different_by_architecture(self): - return 'x86_64' - - @when('=ppc64') - def different_by_architecture(self): - return 'ppc64' - - @when('=ppc32') - def different_by_architecture(self): - return 'ppc32' - - @when('=arm64') - def different_by_architecture(self): - return 'arm64' - - - # - # Make sure we can switch methods on different dependencies - # - @when('^mpich') - def different_by_dep(self): - return 'mpich' - - @when('^zmpi') - def different_by_dep(self): - return 'zmpi' - - - # - # Make sure we can switch on virtual dependencies - # - def different_by_virtual_dep(self): - return 1 - - @when('^mpi@2:') - def different_by_virtual_dep(self): - return 2 diff --git a/var/spack/mock_packages/optional-dep-test-2/package.py b/var/spack/mock_packages/optional-dep-test-2/package.py deleted file mode 100644 index ef0587588e..0000000000 --- a/var/spack/mock_packages/optional-dep-test-2/package.py +++ /dev/null @@ -1,18 +0,0 @@ -from spack import * - -class OptionalDepTest2(Package): - """Depends on the optional-dep-test package""" - - homepage = "http://www.example.com" - url = "http://www.example.com/optional-dep-test-2-1.0.tar.gz" - - version('1.0', '0123456789abcdef0123456789abcdef') - - variant('odt', default=False) - variant('mpi', default=False) - - depends_on('optional-dep-test', when='+odt') - depends_on('optional-dep-test+mpi', when='+mpi') - - def install(self, spec, prefix): - pass diff --git a/var/spack/mock_packages/optional-dep-test-3/package.py b/var/spack/mock_packages/optional-dep-test-3/package.py deleted file mode 100644 index e6cb3bd6e7..0000000000 --- a/var/spack/mock_packages/optional-dep-test-3/package.py +++ /dev/null @@ -1,17 +0,0 @@ -from spack import * - -class OptionalDepTest3(Package): - """Depends on the optional-dep-test package""" - - homepage = "http://www.example.com" - url = "http://www.example.com/optional-dep-test-3-1.0.tar.gz" - - version('1.0', '0123456789abcdef0123456789abcdef') - - variant('var', default=False) - - depends_on('a', when='~var') - depends_on('b', when='+var') - - def install(self, spec, prefix): - pass diff --git a/var/spack/mock_packages/optional-dep-test/package.py b/var/spack/mock_packages/optional-dep-test/package.py deleted file mode 100644 index bb57576ca9..0000000000 --- a/var/spack/mock_packages/optional-dep-test/package.py +++ /dev/null @@ -1,29 +0,0 @@ -from spack import * - -class OptionalDepTest(Package): - """Description""" - - homepage = "http://www.example.com" - url = "http://www.example.com/optional_dep_test-1.0.tar.gz" - - version('1.0', '0123456789abcdef0123456789abcdef') - version('1.1', '0123456789abcdef0123456789abcdef') - - variant('a', default=False) - variant('f', default=False) - variant('mpi', default=False) - - depends_on('a', when='+a') - depends_on('b', when='@1.1') - depends_on('c', when='%intel') - depends_on('d', when='%intel@64.1') - depends_on('e', when='%clang@34:40') - - depends_on('f', when='+f') - depends_on('g', when='^f') - depends_on('mpi', when='^g') - - depends_on('mpi', when='+mpi') - - def install(self, spec, prefix): - pass diff --git a/var/spack/mock_packages/svn-test/package.py b/var/spack/mock_packages/svn-test/package.py deleted file mode 100644 index ba4d5522b4..0000000000 --- a/var/spack/mock_packages/svn-test/package.py +++ /dev/null @@ -1,10 +0,0 @@ -from spack import * - -class SvnTest(Package): - """Mock package that uses svn for fetching.""" - url = "http://www.example.com/svn-test-1.0.tar.gz" - - version('svn', 'to-be-filled-in-by-test') - - def install(self, spec, prefix): - pass diff --git a/var/spack/mock_packages/trivial_install_test_package/package.py b/var/spack/mock_packages/trivial_install_test_package/package.py deleted file mode 100644 index c4db9f5f07..0000000000 --- a/var/spack/mock_packages/trivial_install_test_package/package.py +++ /dev/null @@ -1,38 +0,0 @@ -############################################################################## -# Copyright (c) 2013, Lawrence Livermore National Security, LLC. -# Produced at the Lawrence Livermore National Laboratory. -# -# This file is part of Spack. -# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. -# LLNL-CODE-647188 -# -# For details, see https://scalability-llnl.github.io/spack -# Please also see the LICENSE file for our notice and the LGPL. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License (as published by -# the Free Software Foundation) version 2.1 dated February 1999. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and -# conditions of the GNU General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -############################################################################## -from spack import * - -class TrivialInstallTestPackage(Package): - """This package is a stub with a trivial install method. It allows us - to test the install and uninstall logic of spack.""" - homepage = "http://www.example.com/trivial_install" - url = "http://www.unit-test-should-replace-this-url/trivial_install-1.0.tar.gz" - - version('1.0', 'foobarbaz') - - def install(self, spec, prefix): - configure('--prefix=%s' % prefix) - make() - make('install') diff --git a/var/spack/mock_packages/zmpi/package.py b/var/spack/mock_packages/zmpi/package.py deleted file mode 100644 index 8c6ceda6d3..0000000000 --- a/var/spack/mock_packages/zmpi/package.py +++ /dev/null @@ -1,39 +0,0 @@ -############################################################################## -# Copyright (c) 2013, Lawrence Livermore National Security, LLC. -# Produced at the Lawrence Livermore National Laboratory. -# -# This file is part of Spack. -# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. -# LLNL-CODE-647188 -# -# For details, see https://scalability-llnl.github.io/spack -# Please also see the LICENSE file for our notice and the LGPL. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License (as published by -# the Free Software Foundation) version 2.1 dated February 1999. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and -# conditions of the GNU General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -############################################################################## -from spack import * - -class Zmpi(Package): - """This is a fake MPI package used to demonstrate virtual package providers - with dependencies.""" - homepage = "http://www.spack-fake-zmpi.org" - url = "http://www.spack-fake-zmpi.org/downloads/zmpi-1.0.tar.gz" - - version('1.0', 'foobarbaz') - - provides('mpi@:10.0') - depends_on('fake') - - def install(self, spec, prefix): - pass diff --git a/var/spack/packages/ImageMagick/package.py b/var/spack/packages/ImageMagick/package.py deleted file mode 100644 index 753ea80ca6..0000000000 --- a/var/spack/packages/ImageMagick/package.py +++ /dev/null @@ -1,37 +0,0 @@ -from spack import * - -class Imagemagick(Package): - """ImageMagick is a image processing library""" - homepage = "http://www.imagemagic.org" - - #------------------------------------------------------------------------- - # ImageMagick does not keep around anything but *-10 versions, so - # this URL may change. If you want the bleeding edge, you can - # uncomment it and see if it works but you may need to try to - # fetch a newer version (-6, -7, -8, -9, etc.) or you can stick - # wtih the older, stable, archived -10 versions below. - # - # TODO: would be nice if spack had a way to recommend avoiding a - # TODO: bleeding edge version, but not comment it out. - # ------------------------------------------------------------------------- - # version('6.9.0-6', 'c1bce7396c22995b8bdb56b7797b4a1b', - # url="http://www.imagemagick.org/download/ImageMagick-6.9.0-6.tar.bz2") - - #------------------------------------------------------------------------- - # *-10 versions are archived, so these versions should fetch reliably. - # ------------------------------------------------------------------------- - version('6.8.9-10', 'aa050bf9785e571c956c111377bbf57c', - url="http://sourceforge.net/projects/imagemagick/files/old-sources/6.x/6.8/ImageMagick-6.8.9-10.tar.gz/download") - - depends_on('libtool') - depends_on('jpeg') - depends_on('libpng') - depends_on('freetype') - depends_on('fontconfig') - depends_on('libtiff') - - def install(self, spec, prefix): - configure("--prefix=%s" % prefix) - - make() - make("install") diff --git a/var/spack/packages/Mitos/package.py b/var/spack/packages/Mitos/package.py deleted file mode 100644 index e312da3ffc..0000000000 --- a/var/spack/packages/Mitos/package.py +++ /dev/null @@ -1,19 +0,0 @@ -from spack import * - -class Mitos(Package): - """Mitos is a library and a tool for collecting sampled memory - performance data to view with MemAxes""" - - homepage = "https://github.com/scalability-llnl/Mitos" - url = "https://github.com/scalability-llnl/Mitos" - - version('0.9.1', 'c6cb57f3cae54f5157affd97ef7ef79e', git='https://github.com/scalability-llnl/Mitos.git', tag='v0.9.1') - - depends_on('dyninst@8.2.1:') - depends_on('hwloc') - - def install(self, spec, prefix): - with working_dir('spack-build', create=True): - cmake('..', *std_cmake_args) - make() - make("install") diff --git a/var/spack/packages/R/package.py b/var/spack/packages/R/package.py deleted file mode 100644 index 2e6f65a742..0000000000 --- a/var/spack/packages/R/package.py +++ /dev/null @@ -1,33 +0,0 @@ -from spack import * - -class R(Package): - """R is 'GNU S', a freely available language and environment for - statistical computing and graphics which provides a wide va - riety of statistical and graphical techniques: linear and - nonlinear modelling, statistical tests, time series analysis, - classification, clustering, etc. Please consult the R project - homepage for further information.""" - homepage = "http://www.example.com" - url = "http://cran.cnr.berkeley.edu/src/base/R-3/R-3.1.2.tar.gz" - - version('3.1.2', '3af29ec06704cbd08d4ba8d69250ae74') - - depends_on("readline") - depends_on("ncurses") - depends_on("icu") - depends_on("glib") - depends_on("zlib") - depends_on("libtiff") - depends_on("jpeg") - depends_on("cairo") - depends_on("pango") - depends_on("freetype") - depends_on("tcl") - depends_on("tk") - - def install(self, spec, prefix): - configure("--prefix=%s" % prefix, - "--enable-R-shlib", - "--enable-BLAS-shlib") - make() - make("install") diff --git a/var/spack/packages/SAMRAI/no-tool-build.patch b/var/spack/packages/SAMRAI/no-tool-build.patch deleted file mode 100644 index 1adf0cf721..0000000000 --- a/var/spack/packages/SAMRAI/no-tool-build.patch +++ /dev/null @@ -1,20 +0,0 @@ ---- SAMRAI/Makefile.in 2013-05-31 11:04:32.000000000 -0700 -+++ SAMRAI/Makefile.in.notools 2014-05-30 10:31:15.135979900 -0700 -@@ -8,7 +8,7 @@ - ## - ######################################################################### - --default: library tools -+default: library - - SAMRAI = @top_srcdir@ - SUBDIR = . -@@ -135,7 +135,7 @@ - done - $(MAKE) archive_remove_obj_names - --install: library tools -+install: library - $(INSTALL) -d -m 755 $(INSTDIR)/config - $(INSTALL) -d -m 755 $(INSTDIR)/lib - $(INSTALL) -d -m 755 $(INSTDIR)/bin diff --git a/var/spack/packages/SAMRAI/package.py b/var/spack/packages/SAMRAI/package.py deleted file mode 100644 index eef041f0d5..0000000000 --- a/var/spack/packages/SAMRAI/package.py +++ /dev/null @@ -1,53 +0,0 @@ -from spack import * - -class Samrai(Package): - """SAMRAI (Structured Adaptive Mesh Refinement Application Infrastructure) - is an object-oriented C++ software library enables exploration of numerical, - algorithmic, parallel computing, and software issues associated with applying - structured adaptive mesh refinement (SAMR) technology in large-scale parallel - application development. - """ - homepage = "https://computation.llnl.gov/project/SAMRAI/" - url = "https://computation.llnl.gov/project/SAMRAI/download/SAMRAI-v3.9.1.tar.gz" - list_url = homepage - - version('3.9.1', '232d04d0c995f5abf20d94350befd0b2') - version('3.7.3', '12d574eacadf8c9a70f1bb4cd1a69df6') - version('3.7.2', 'f6a716f171c9fdbf3cb12f71fa6e2737') - version('3.6.3-beta', 'ef0510bf2893042daedaca434e5ec6ce') - version('3.5.2-beta', 'd072d9d681eeb9ada15ce91bea784274') - version('3.5.0-beta', '1ad18a319fc573e12e2b1fbb6f6b0a19') - version('3.4.1-beta', '00814cbee2cb76bf8302aff56bbb385b') - version('3.3.3-beta', '1db3241d3e1cab913dc310d736c34388') - version('3.3.2-beta', 'e598a085dab979498fcb6c110c4dd26c') - version('2.4.4', '04fb048ed0efe7c531ac10c81cc5f6ac') - - depends_on("mpi") - depends_on("zlib") - depends_on("hdf5") - depends_on("boost") - - # don't build tools with gcc - patch('no-tool-build.patch', when='%gcc') - - # TODO: currently hard-coded to use openmpi - be careful! - def install(self, spec, prefix): - mpi = next(m for m in ('openmpi', 'mpich', 'mvapich') - if m in spec) - - configure( - "--prefix=%s" % prefix, - "--with-CXX=%s" % spec[mpi].prefix.bin + "/mpic++", - "--with-CC=%s" % spec[mpi].prefix.bin + "/mpicc", - "--with-hdf5=%s" % spec['hdf5'].prefix, - "--with-boost=%s" % spec['boost'].prefix, - "--with-zlib=%s" % spec['zlib'].prefix, - "--without-blas", - "--without-lapack", - "--with-hypre=no", - "--with-petsc=no", - "--enable-opt", - "--disable-debug") - - make() - make("install") diff --git a/var/spack/packages/_repo.yaml b/var/spack/packages/_repo.yaml deleted file mode 100644 index 4a371e1cad..0000000000 --- a/var/spack/packages/_repo.yaml +++ /dev/null @@ -1,2 +0,0 @@ -repo: - namespace: gov.llnl.spack diff --git a/var/spack/packages/activeharmony/package.py b/var/spack/packages/activeharmony/package.py deleted file mode 100644 index 45dcc7c0e8..0000000000 --- a/var/spack/packages/activeharmony/package.py +++ /dev/null @@ -1,15 +0,0 @@ -from spack import * - -class Activeharmony(Package): - """Active Harmony: a framework for auto-tuning (the automated search for values to improve the performance of a target application).""" - homepage = "http://www.dyninst.org/harmony" - url = "http://www.dyninst.org/sites/default/files/downloads/harmony/ah-4.5.tar.gz" - - version('4.5', 'caee5b864716d376e2c25d739251b2a9') - - def install(self, spec, prefix): - make("CFLAGS=-O3") - make("install", 'PREFIX=%s' % prefix) - -from spack import * - diff --git a/var/spack/packages/adept-utils/package.py b/var/spack/packages/adept-utils/package.py deleted file mode 100644 index e4a2e1523f..0000000000 --- a/var/spack/packages/adept-utils/package.py +++ /dev/null @@ -1,42 +0,0 @@ -############################################################################## -# Copyright (c) 2013, Lawrence Livermore National Security, LLC. -# Produced at the Lawrence Livermore National Laboratory. -# -# This file is part of Spack. -# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. -# LLNL-CODE-647188 -# -# For details, see https://scalability-llnl.github.io/spack -# Please also see the LICENSE file for our notice and the LGPL. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License (as published by -# the Free Software Foundation) version 2.1 dated February 1999. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and -# conditions of the GNU General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -############################################################################## -from spack import * - -class AdeptUtils(Package): - """Utility libraries for LLNL performance tools.""" - - homepage = "https://github.com/scalability-llnl/adept-utils" - url = "https://github.com/scalability-llnl/adept-utils/archive/v1.0.tar.gz" - - version('1.0.1', '731a310717adcb004d9d195130efee7d') - version('1.0', '5c6cd9badce56c945ac8551e34804397') - - depends_on("boost") - depends_on("mpi") - - def install(self, spec, prefix): - cmake(*std_cmake_args) - make() - make("install") diff --git a/var/spack/packages/apex/package.py b/var/spack/packages/apex/package.py deleted file mode 100644 index 6404d5208a..0000000000 --- a/var/spack/packages/apex/package.py +++ /dev/null @@ -1,34 +0,0 @@ -from spack import * -from spack.util.environment import * - -class Apex(Package): - homepage = "http://github.com/khuck/xpress-apex" - #url = "http://github.com/khuck/xpress-apex/archive/v0.1-release-candidate.tar.gz" - url = "http://github.com/khuck/xpress-apex" - - #version('0.1', '6e039c224387348296739f6bf360d081') - #version('master', branch='master', git='https://github.com/khuck/xpress-apex.git') - version('2015-10-21', git='https://github.com/khuck/xpress-apex.git', commit='d2e66ddde689120472fc57fc546d8cd80aab745c') - - depends_on("binutils+libiberty") - depends_on("boost@1.54:") - depends_on("cmake@2.8.12:") - depends_on("activeharmony@4.5:") - depends_on("ompt-openmp") - - def install(self, spec, prefix): - - path=get_path("PATH") - path.remove(spec["binutils"].prefix.bin) - path_set("PATH", path) - with working_dir("build", create=True): - cmake('-DBOOST_ROOT=%s' % spec['boost'].prefix, - '-DUSE_BFD=TRUE', - '-DBFD_ROOT=%s' % spec['binutils'].prefix, - '-DUSE_ACTIVEHARMONY=TRUE', - '-DACTIVEHARMONY_ROOT=%s' % spec['activeharmony'].prefix, - '-DUSE_OMPT=TRUE', - '-DOMPT_ROOT=%s' % spec['ompt-openmp'].prefix, - '..', *std_cmake_args) - make() - make("install") diff --git a/var/spack/packages/arpack/package.py b/var/spack/packages/arpack/package.py deleted file mode 100644 index 8c67c536f3..0000000000 --- a/var/spack/packages/arpack/package.py +++ /dev/null @@ -1,41 +0,0 @@ -from spack import * -import os -import shutil - -class Arpack(Package): - """A collection of Fortran77 subroutines designed to solve large scale - eigenvalue problems. - """ - homepage = "http://www.caam.rice.edu/software/ARPACK/" - url = "http://www.caam.rice.edu/software/ARPACK/SRC/arpack96.tar.gz" - - version('96', 'fffaa970198b285676f4156cebc8626e') - - depends_on('blas') - depends_on('lapack') - - def patch(self): - # Filter the cray makefile to make a spack one. - shutil.move('ARMAKES/ARmake.CRAY', 'ARmake.inc') - makefile = FileFilter('ARmake.inc') - - # Be sure to use Spack F77 wrapper - makefile.filter('^FC.*', 'FC = f77') - makefile.filter('^FFLAGS.*', 'FFLAGS = -O2 -g') - - # Set up some variables. - makefile.filter('^PLAT.*', 'PLAT = ') - makefile.filter('^home.*', 'home = %s' % os.getcwd()) - makefile.filter('^BLASdir.*', 'BLASdir = %s' % self.spec['blas'].prefix) - makefile.filter('^LAPACKdir.*', 'LAPACKdir = %s' % self.spec['lapack'].prefix) - - # build the library in our own prefix. - makefile.filter('^ARPACKLIB.*', 'ARPACKLIB = %s/libarpack.a' % os.getcwd()) - - - def install(self, spec, prefix): - with working_dir('SRC'): - make('all') - - mkdirp(prefix.lib) - install('libarpack.a', prefix.lib) diff --git a/var/spack/packages/asciidoc/package.py b/var/spack/packages/asciidoc/package.py deleted file mode 100644 index 828f3b3f4f..0000000000 --- a/var/spack/packages/asciidoc/package.py +++ /dev/null @@ -1,18 +0,0 @@ -from spack import * - -class Asciidoc(Package): - """ A presentable text document format for writing articles, UNIX man - pages and other small to medium sized documents.""" - homepage = "http://asciidoc.org" - url = "http://downloads.sourceforge.net/project/asciidoc/asciidoc/8.6.9/asciidoc-8.6.9.tar.gz" - - version('8.6.9', 'c59018f105be8d022714b826b0be130a') - - depends_on('libxml2') - depends_on('libxslt') - - def install(self, spec, prefix): - configure('--prefix=%s' % prefix) - - make() - make("install") diff --git a/var/spack/packages/atk/package.py b/var/spack/packages/atk/package.py deleted file mode 100644 index 769805b227..0000000000 --- a/var/spack/packages/atk/package.py +++ /dev/null @@ -1,18 +0,0 @@ -from spack import * - -class Atk(Package): - """ATK provides the set of accessibility interfaces that are - implemented by other toolkits and applications. Using the ATK - interfaces, accessibility tools have full access to view and - control running applications.""" - homepage = "https://developer.gnome.org/atk/" - url = "http://ftp.gnome.org/pub/gnome/sources/atk/2.14/atk-2.14.0.tar.xz" - - version('2.14.0', 'ecb7ca8469a5650581b1227d78051b8b') - - depends_on("glib") - - def install(self, spec, prefix): - configure("--prefix=%s" % prefix) - make() - make("install") diff --git a/var/spack/packages/atlas/package.py b/var/spack/packages/atlas/package.py deleted file mode 100644 index fc683363a7..0000000000 --- a/var/spack/packages/atlas/package.py +++ /dev/null @@ -1,60 +0,0 @@ -from spack import * -from spack.util.executable import Executable -import os - -class Atlas(Package): - """ - Automatically Tuned Linear Algebra Software, generic shared - ATLAS is an approach for the automatic generation and optimization of - numerical software. Currently ATLAS supplies optimized versions for the - complete set of linear algebra kernels known as the Basic Linear Algebra - Subroutines (BLAS), and a subset of the linear algebra routines in the - LAPACK library. - """ - homepage = "http://math-atlas.sourceforge.net/" - - version('3.11.34', '0b6c5389c095c4c8785fd0f724ec6825', - url='http://sourceforge.net/projects/math-atlas/files/Developer%20%28unstable%29/3.11.34/atlas3.11.34.tar.bz2/download') - version('3.10.2', 'a4e21f343dec8f22e7415e339f09f6da', - url='http://downloads.sourceforge.net/project/math-atlas/Stable/3.10.2/atlas3.10.2.tar.bz2') - - # TODO: make this provide BLAS once it works better. Create a way - # TODO: to mark "beta" packages and require explicit invocation. - - # provides('blas') - - - def patch(self): - # Disable thraed check. LLNL's environment does not allow - # disabling of CPU throttling in a way that ATLAS actually - # understands. - filter_file(r'^\s+if \(thrchk\) exit\(1\);', 'if (0) exit(1);', - 'CONFIG/src/config.c') - # TODO: investigate a better way to add the check back in - # TODO: using, say, MSRs. Or move this to a variant. - - @when('@:3.10') - def install(self, spec, prefix): - with working_dir('ATLAS-Build', create=True): - configure = Executable('../configure') - configure('--prefix=%s' % prefix, '-C', 'ic', 'cc', '-C', 'if', 'f77', "--dylibs") - make() - make('check') - make('ptcheck') - make('time') - make("install") - - - def install(self, spec, prefix): - with working_dir('ATLAS-Build', create=True): - configure = Executable('../configure') - configure('--incdir=%s' % prefix.include, - '--libdir=%s' % prefix.lib, - '--cc=cc', - "--shared") - - make() - make('check') - make('ptcheck') - make('time') - make("install") diff --git a/var/spack/packages/autoconf/package.py b/var/spack/packages/autoconf/package.py deleted file mode 100644 index 5189faf054..0000000000 --- a/var/spack/packages/autoconf/package.py +++ /dev/null @@ -1,14 +0,0 @@ -from spack import * - -class Autoconf(Package): - """Autoconf -- system configuration part of autotools""" - homepage = "https://www.gnu.org/software/autoconf/" - url = "http://ftp.gnu.org/gnu/autoconf/autoconf-2.69.tar.gz" - - version('2.69', '82d05e03b93e45f5a39b828dc9c6c29b') - - def install(self, spec, prefix): - configure("--prefix=%s" % prefix) - - make() - make("install") diff --git a/var/spack/packages/automaded/package.py b/var/spack/packages/automaded/package.py deleted file mode 100644 index 9fbd93e3b3..0000000000 --- a/var/spack/packages/automaded/package.py +++ /dev/null @@ -1,51 +0,0 @@ -############################################################################## -# Copyright (c) 2013, Lawrence Livermore National Security, LLC. -# Produced at the Lawrence Livermore National Laboratory. -# -# This file is part of Spack. -# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. -# LLNL-CODE-647188 -# -# For details, see https://scalability-llnl.github.io/spack -# Please also see the LICENSE file for our notice and the LGPL. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License (as published by -# the Free Software Foundation) version 2.1 dated February 1999. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and -# conditions of the GNU General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -############################################################################## -from spack import * - -class Automaded(Package): - """AutomaDeD (Automata-based Debugging for Dissimilar parallel - tasks) is a tool for automatic diagnosis of performance and - correctness problems in MPI applications. It creates - control-flow models of each MPI process and, when a failure - occurs, these models are leveraged to find the origin of - problems automatically. MPI calls are intercepted (using - wrappers) to create the models. When an MPI application hangs, - AutomaDeD creates a progress-dependence graph that helps - finding the process (or group of processes) that caused the hang. - """ - - homepage = "https://github.com/scalability-llnl/AutomaDeD" - url = "https://github.com/scalability-llnl/AutomaDeD/archive/v1.0.tar.gz" - - version('1.0', '16a3d4def2c4c77d0bc4b21de8b3ab03') - - depends_on('mpi') - depends_on('boost') - depends_on('callpath') - - def install(self, spec, prefix): - cmake("-DSTATE_TRACKER_WITH_CALLPATH=ON", *std_cmake_args) - make() - make("install") diff --git a/var/spack/packages/automake/package.py b/var/spack/packages/automake/package.py deleted file mode 100644 index 9115822730..0000000000 --- a/var/spack/packages/automake/package.py +++ /dev/null @@ -1,16 +0,0 @@ -from spack import * - -class Automake(Package): - """Automake -- make file builder part of autotools""" - homepage = "http://www.gnu.org/software/automake/" - url = "http://ftp.gnu.org/gnu/automake/automake-1.14.tar.gz" - - version('1.14.1', 'd052a3e884631b9c7892f2efce542d75') - - depends_on('autoconf') - - def install(self, spec, prefix): - configure("--prefix=%s" % prefix) - - make() - make("install") diff --git a/var/spack/packages/bear/package.py b/var/spack/packages/bear/package.py deleted file mode 100644 index 0d4436fccc..0000000000 --- a/var/spack/packages/bear/package.py +++ /dev/null @@ -1,17 +0,0 @@ -from spack import * - -class Bear(Package): - """Bear is a tool that generates a compilation database for clang tooling from non-cmake build systems.""" - homepage = "https://github.com/rizsotto/Bear" - url = "https://github.com/rizsotto/Bear/archive/2.0.4.tar.gz" - - version('2.0.4', 'fd8afb5e8e18f8737ba06f90bd77d011') - - depends_on("cmake") - depends_on("python") - - def install(self, spec, prefix): - cmake('.', *std_cmake_args) - - make("all") - make("install") diff --git a/var/spack/packages/bib2xhtml/package.py b/var/spack/packages/bib2xhtml/package.py deleted file mode 100644 index 7f8e0cfe5a..0000000000 --- a/var/spack/packages/bib2xhtml/package.py +++ /dev/null @@ -1,27 +0,0 @@ -from spack import * -from glob import glob - -class Bib2xhtml(Package): - """bib2xhtml is a program that converts BibTeX files into HTML.""" - homepage = "http://www.spinellis.gr/sw/textproc/bib2xhtml/" - url='http://www.spinellis.gr/sw/textproc/bib2xhtml/bib2xhtml-v3.0-15-gf506.tar.gz' - - version('3.0-15-gf506', 'a26ba02fe0053bbbf2277bdf0acf8645') - - def url_for_version(self, v): - return ('http://www.spinellis.gr/sw/textproc/bib2xhtml/bib2xhtml-v%s.tar.gz' % v) - - def install(self, spec, prefix): - # Add the bst include files to the install directory - bst_include = join_path(prefix.share, 'bib2xhtml') - mkdirp(bst_include) - for bstfile in glob('html-*bst'): - install(bstfile, bst_include) - - # Install the script and point it at the user's favorite perl - # and the bst include directory. - mkdirp(prefix.bin) - install('bib2xhtml', prefix.bin) - filter_file(r'#!/usr/bin/perl', - '#!/usr/bin/env BSTINPUTS=%s perl' % bst_include, - join_path(prefix.bin, 'bib2xhtml')) diff --git a/var/spack/packages/binutils/package.py b/var/spack/packages/binutils/package.py deleted file mode 100644 index cac0a0407f..0000000000 --- a/var/spack/packages/binutils/package.py +++ /dev/null @@ -1,30 +0,0 @@ -from spack import * - -class Binutils(Package): - """GNU binutils, which contain the linker, assembler, objdump and others""" - homepage = "http://www.gnu.org/software/binutils/" - url = "ftp://ftp.gnu.org/gnu/binutils/binutils-2.25.tar.bz2" - - version('2.25', 'd9f3303f802a5b6b0bb73a335ab89d66') - version('2.24', 'e0f71a7b2ddab0f8612336ac81d9636b') - version('2.23.2', '4f8fa651e35ef262edc01d60fb45702e') - version('2.20.1', '2b9dc8f2b7dbd5ec5992c6e29de0b764') - - variant('libiberty', default=False, description='Also install libiberty.') - - def install(self, spec, prefix): - configure_args = [ - '--prefix=%s' % prefix, - '--disable-dependency-tracking', - '--enable-interwork', - '--enable-multilib', - '--enable-shared', - '--enable-64-bit-bfd', - '--enable-targets=all'] - - if '+libiberty' in spec: - configure_args.append('--enable-install-libiberty') - - configure(*configure_args) - make() - make("install") diff --git a/var/spack/packages/bison/package.py b/var/spack/packages/bison/package.py deleted file mode 100644 index 7c526fb958..0000000000 --- a/var/spack/packages/bison/package.py +++ /dev/null @@ -1,17 +0,0 @@ -from spack import * - -class Bison(Package): - """Bison is a general-purpose parser generator that converts - an annotated context-free grammar into a deterministic LR or - generalized LR (GLR) parser employing LALR(1) parser tables.""" - - homepage = "http://www.gnu.org/software/bison/" - url = "http://ftp.gnu.org/gnu/bison/bison-3.0.tar.gz" - - version('3.0.4', 'a586e11cd4aff49c3ff6d3b6a4c9ccf8') - - def install(self, spec, prefix): - configure("--prefix=%s" % prefix) - - make() - make("install") diff --git a/var/spack/packages/boost/package.py b/var/spack/packages/boost/package.py deleted file mode 100644 index 35824d53a2..0000000000 --- a/var/spack/packages/boost/package.py +++ /dev/null @@ -1,66 +0,0 @@ -from spack import * - -class Boost(Package): - """Boost provides free peer-reviewed portable C++ source - libraries, emphasizing libraries that work well with the C++ - Standard Library. - - Boost libraries are intended to be widely useful, and usable - across a broad spectrum of applications. The Boost license - encourages both commercial and non-commercial use. - """ - homepage = "http://www.boost.org" - url = "http://downloads.sourceforge.net/project/boost/boost/1.55.0/boost_1_55_0.tar.bz2" - list_url = "http://sourceforge.net/projects/boost/files/boost/" - list_depth = 2 - - version('1.59.0', '6aa9a5c6a4ca1016edd0ed1178e3cb87') - version('1.58.0', 'b8839650e61e9c1c0a89f371dd475546') - version('1.57.0', '1be49befbdd9a5ce9def2983ba3e7b76') - version('1.56.0', 'a744cf167b05d72335f27c88115f211d') - version('1.55.0', 'd6eef4b4cacb2183f2bf265a5a03a354') - version('1.54.0', '15cb8c0803064faef0c4ddf5bc5ca279') - version('1.53.0', 'a00d22605d5dbcfb4c9936a9b35bc4c2') - version('1.52.0', '3a855e0f919107e0ca4de4d84ad3f750') - version('1.51.0', '4b6bd483b692fd138aef84ed2c8eb679') - version('1.50.0', '52dd00be775e689f55a987baebccc462') - version('1.49.0', '0d202cb811f934282dea64856a175698') - version('1.48.0', 'd1e9a7a7f532bb031a3c175d86688d95') - version('1.47.0', 'a2dc343f7bc7f83f8941e47ed4a18200') - version('1.46.1', '7375679575f4c8db605d426fc721d506') - version('1.46.0', '37b12f1702319b73876b0097982087e0') - version('1.45.0', 'd405c606354789d0426bc07bea617e58') - version('1.44.0', 'f02578f5218f217a9f20e9c30e119c6a') - version('1.43.0', 'dd49767bfb726b0c774f7db0cef91ed1') - version('1.42.0', '7bf3b4eb841b62ffb0ade2b82218ebe6') - version('1.41.0', '8bb65e133907db727a2a825c5400d0a6') - version('1.40.0', 'ec3875caeac8c52c7c129802a8483bd7') - version('1.39.0', 'a17281fd88c48e0d866e1a12deecbcc0') - version('1.38.0', '5eca2116d39d61382b8f8235915cb267') - version('1.37.0', '8d9f990bfb7e83769fa5f1d6f065bc92') - version('1.36.0', '328bfec66c312150e4c2a78dcecb504b') - version('1.35.0', 'dce952a7214e72d6597516bcac84048b') - version('1.34.1', '2d938467e8a448a2c9763e0a9f8ca7e5') - version('1.34.0', 'ed5b9291ffad776f8757a916e1726ad0') - - - def url_for_version(self, version): - """Handle Boost's weird URLs, which write the version two different ways.""" - parts = [str(p) for p in Version(version)] - dots = ".".join(parts) - underscores = "_".join(parts) - return "http://downloads.sourceforge.net/project/boost/boost/%s/boost_%s.tar.bz2" % ( - dots, underscores) - - - def install(self, spec, prefix): - bootstrap = Executable('./bootstrap.sh') - bootstrap() - - # b2 used to be called bjam, before 1.47 (sigh) - b2name = './b2' if spec.satisfies('@1.47:') else './bjam' - - b2 = Executable(b2name) - b2('install', - '-j %s' % make_jobs, - '--prefix=%s' % prefix) diff --git a/var/spack/packages/bowtie2/bowtie2-2.5.patch b/var/spack/packages/bowtie2/bowtie2-2.5.patch deleted file mode 100644 index 290be39c73..0000000000 --- a/var/spack/packages/bowtie2/bowtie2-2.5.patch +++ /dev/null @@ -1,16 +0,0 @@ ---- Makefile 2015-02-26 10:50:00.000000000 -0800 -+++ Makefile.new 2015-07-29 18:03:59.891357399 -0700 -@@ -22,10 +22,10 @@ - # - - INC = --GCC_PREFIX = $(shell dirname `which gcc`) -+GCC_PREFIX = - GCC_SUFFIX = --CC = $(GCC_PREFIX)/gcc$(GCC_SUFFIX) --CPP = $(GCC_PREFIX)/g++$(GCC_SUFFIX) -+CC = cc -+CPP = c++ - CXX = $(CPP) - HEADERS = $(wildcard *.h) - BOWTIE_MM = 1 diff --git a/var/spack/packages/bowtie2/package.py b/var/spack/packages/bowtie2/package.py deleted file mode 100644 index 339aab6598..0000000000 --- a/var/spack/packages/bowtie2/package.py +++ /dev/null @@ -1,24 +0,0 @@ -from spack import * -from glob import glob -class Bowtie2(Package): - """Description""" - homepage = "bowtie-bio.sourceforge.net/bowtie2/index.shtml" - version('2.2.5','51fa97a862d248d7ee660efc1147c75f', url = "http://downloads.sourceforge.net/project/bowtie-bio/bowtie2/2.2.5/bowtie2-2.2.5-source.zip") - - patch('bowtie2-2.5.patch',when='@2.2.5', level=0) - - def install(self, spec, prefix): - make() - mkdirp(prefix.bin) - for bow in glob("bowtie2*"): - install(bow, prefix.bin) - # install('bowtie2',prefix.bin) - # install('bowtie2-align-l',prefix.bin) - # install('bowtie2-align-s',prefix.bin) - # install('bowtie2-build',prefix.bin) - # install('bowtie2-build-l',prefix.bin) - # install('bowtie2-build-s',prefix.bin) - # install('bowtie2-inspect',prefix.bin) - # install('bowtie2-inspect-l',prefix.bin) - # install('bowtie2-inspect-s',prefix.bin) - diff --git a/var/spack/packages/boxlib/package.py b/var/spack/packages/boxlib/package.py deleted file mode 100644 index 4f1b71132f..0000000000 --- a/var/spack/packages/boxlib/package.py +++ /dev/null @@ -1,25 +0,0 @@ -from spack import * - -class Boxlib(Package): - """BoxLib, a software framework for massively parallel - block-structured adaptive mesh refinement (AMR) codes.""" - - homepage = "https://ccse.lbl.gov/BoxLib/" - url = "https://ccse.lbl.gov/pub/Downloads/BoxLib.git"; - - # TODO: figure out how best to version this. No tags in the repo! - version('master', git='https://ccse.lbl.gov/pub/Downloads/BoxLib.git') - - depends_on('mpi') - - def install(self, spec, prefix): - args = std_cmake_args - args += ['-DCCSE_ENABLE_MPI=1', - '-DCMAKE_C_COMPILER=%s' % which('mpicc'), - '-DCMAKE_CXX_COMPILER=%s' % which('mpicxx'), - '-DCMAKE_Fortran_COMPILER=%s' % which('mpif90')] - - cmake('.', *args) - make() - make("install") - diff --git a/var/spack/packages/bzip2/package.py b/var/spack/packages/bzip2/package.py deleted file mode 100644 index d88336664d..0000000000 --- a/var/spack/packages/bzip2/package.py +++ /dev/null @@ -1,36 +0,0 @@ -from spack import * -from glob import glob - -class Bzip2(Package): - """bzip2 is a freely available, patent free high-quality data - compressor. It typically compresses files to within 10% to 15% - of the best available techniques (the PPM family of statistical - compressors), whilst being around twice as fast at compression - and six times faster at decompression.""" - homepage = "http://www.bzip.org" - url = "http://www.bzip.org/1.0.6/bzip2-1.0.6.tar.gz" - - version('1.0.6', '00b516f4704d4a7cb50a1d97e6e8e15b') - - def install(self, spec, prefix): - # No configure system -- have to filter the makefile for this package. - filter_file(r'CC=gcc', 'CC=cc', 'Makefile', string=True) - - make('-f', 'Makefile-libbz2_so') - make('clean') - make("install", "PREFIX=%s" % prefix) - - bzip2_exe = join_path(prefix.bin, 'bzip2') - install('bzip2-shared', bzip2_exe) - for i, libfile in enumerate(glob('libbz2.so*')): - install(libfile, prefix.lib) - if i == 0: - symlink(join_path(prefix.lib, libfile), join_path(prefix.lib, 'libbz2.so')) - - bunzip2 = join_path(prefix.bin, 'bunzip2') - remove(bunzip2) - symlink(bzip2_exe, bunzip2) - - bzcat = join_path(prefix.bin, 'bzcat') - remove(bzcat) - symlink(bzip2_exe, bzcat) diff --git a/var/spack/packages/cairo/package.py b/var/spack/packages/cairo/package.py deleted file mode 100644 index e1ac8aaa7d..0000000000 --- a/var/spack/packages/cairo/package.py +++ /dev/null @@ -1,19 +0,0 @@ -from spack import * - -class Cairo(Package): - """Cairo is a 2D graphics library with support for multiple output devices.""" - homepage = "http://cairographics.org" - url = "http://cairographics.org/releases/cairo-1.14.0.tar.xz" - - version('1.14.0', 'fc3a5edeba703f906f2241b394f0cced') - - depends_on("libpng") - depends_on("glib") - depends_on("pixman") - depends_on("fontconfig@2.10.91:") # Require newer version of fontconfig. - - def install(self, spec, prefix): - configure("--prefix=%s" % prefix, - "--enable-tee") - make() - make("install") diff --git a/var/spack/packages/callpath/package.py b/var/spack/packages/callpath/package.py deleted file mode 100644 index f8a1eab9f7..0000000000 --- a/var/spack/packages/callpath/package.py +++ /dev/null @@ -1,47 +0,0 @@ -############################################################################## -# Copyright (c) 2013, Lawrence Livermore National Security, LLC. -# Produced at the Lawrence Livermore National Laboratory. -# -# This file is part of Spack. -# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. -# LLNL-CODE-647188 -# -# For details, see https://scalability-llnl.github.io/spack -# Please also see the LICENSE file for our notice and the LGPL. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License (as published by -# the Free Software Foundation) version 2.1 dated February 1999. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and -# conditions of the GNU General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -############################################################################## -from spack import * - -class Callpath(Package): - """Library for representing callpaths consistently in - distributed-memory performance tools.""" - - homepage = "https://github.com/scalability-llnl/callpath" - url = "https://github.com/scalability-llnl/callpath/archive/v1.0.1.tar.gz" - - version('1.0.2', 'b1994d5ee7c7db9d27586fc2dcf8f373') - version('1.0.1', '0047983d2a52c5c335f8ba7f5bab2325') - - depends_on("libelf") - depends_on("libdwarf") - depends_on("dyninst") - depends_on("adept-utils") - depends_on("mpi") - - def install(self, spec, prefix): - # TODO: offer options for the walker used. - cmake('.', "-DCALLPATH_WALKER=dyninst", *std_cmake_args) - make() - make("install") diff --git a/var/spack/packages/cblas/package.py b/var/spack/packages/cblas/package.py deleted file mode 100644 index 3cfe5ee588..0000000000 --- a/var/spack/packages/cblas/package.py +++ /dev/null @@ -1,35 +0,0 @@ -from spack import * -import os - -class Cblas(Package): - """The BLAS (Basic Linear Algebra Subprograms) are routines that - provide standard building blocks for performing basic vector and - matrix operations.""" - - homepage = "http://www.netlib.org/blas/_cblas/" - - # tarball has no version, but on the date below, this MD5 was correct. - version('2015-06-06', '1e8830f622d2112239a4a8a83b84209a', - url='http://www.netlib.org/blas/blast-forum/cblas.tgz') - - depends_on('blas') - parallel = False - - def patch(self): - mf = FileFilter('Makefile.in') - - mf.filter('^BLLIB =.*', 'BLLIB = %s/libblas.a' % self.spec['blas'].prefix.lib) - mf.filter('^CC =.*', 'CC = cc') - mf.filter('^FC =.*', 'FC = f90') - - - def install(self, spec, prefix): - make('all') - mkdirp(prefix.lib) - mkdirp(prefix.include) - - # Rename the generated lib file to libcblas.a - install('./lib/cblas_LINUX.a', '%s/libcblas.a' % prefix.lib) - install('./include/cblas.h','%s' % prefix.include) - install('./include/cblas_f77.h','%s' % prefix.include) - diff --git a/var/spack/packages/cgm/package.py b/var/spack/packages/cgm/package.py deleted file mode 100644 index 05d6395c5a..0000000000 --- a/var/spack/packages/cgm/package.py +++ /dev/null @@ -1,30 +0,0 @@ -from spack import * - -class Cgm(Package): - """The Common Geometry Module, Argonne (CGMA) is a code library - which provides geometry functionality used for mesh generation and - other applications.""" - homepage = "http://trac.mcs.anl.gov/projects/ITAPS/wiki/CGM" - url = "http://ftp.mcs.anl.gov/pub/fathom/cgm13.1.1.tar.gz" - - version('13.1.1', '4e8dbc4ba8f65767b29f985f7a23b01f') - version('13.1.0', 'a6c7b22660f164ce893fb974f9cb2028') - version('13.1' , '95f724bda04919fc76818a5b7bc0b4ed') - - depends_on("mpi") - - def patch(self): - filter_file('^(#include "CGMParallelConventions.h")', - '//\1', - 'geom/parallel/CGMReadParallel.cpp') - - - def install(self, spec, prefix): - configure("--with-mpi", - "--prefix=%s" % prefix, - "CFLAGS=-static", - "CXXFLAGS=-static", - "FCFLAGS=-static") - - make() - make("install") diff --git a/var/spack/packages/clang/package.py b/var/spack/packages/clang/package.py deleted file mode 100644 index 4f977bf9a4..0000000000 --- a/var/spack/packages/clang/package.py +++ /dev/null @@ -1,51 +0,0 @@ -############################################################################## -# Copyright (c) 2013, Lawrence Livermore National Security, LLC. -# Produced at the Lawrence Livermore National Laboratory. -# -# This file is part of Spack. -# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. -# LLNL-CODE-647188 -# -# For details, see https://scalability-llnl.github.io/spack -# Please also see the LICENSE file for our notice and the LGPL. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License (as published by -# the Free Software Foundation) version 2.1 dated February 1999. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and -# conditions of the GNU General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -############################################################################## -from spack import * - -class Clang(Package): - """The goal of the Clang project is to create a new C, C++, - Objective C and Objective C++ front-end for the LLVM compiler. - """ - homepage = 'http://clang.llvm.org' - url = 'http://llvm.org/releases/3.7.0/cfe-3.7.0.src.tar.xz' - - depends_on('llvm@3.7.0', when='@3.7.0') - depends_on('llvm@3.6.2', when='@3.6.2') - depends_on('llvm@3.5.1', when='@3.5.1') - - version('3.7.0', '8f9d27335e7331cf0a4711e952f21f01', url='http://llvm.org/releases/3.7.0/cfe-3.7.0.src.tar.xz') - version('3.6.2', 'ff862793682f714bb7862325b9c06e20', url='http://llvm.org/releases/3.6.2/cfe-3.6.2.src.tar.xz') - version('3.5.1', '93f9532f8f7e6f1d8e5c1116907051cb', url='http://llvm.org/releases/3.5.1/cfe-3.5.1.src.tar.xz') - - def install(self, spec, prefix): - env['CXXFLAGS'] = self.compiler.cxx11_flag - - with working_dir('spack-build', create=True): - cmake('..', - '-DCLANG_PATH_TO_LLVM_BUILD=%s' % spec['llvm'].prefix, - '-DLLVM_MAIN_SRC_DIR=%s' % spec['llvm'].prefix, - *std_cmake_args) - make() - make("install") diff --git a/var/spack/packages/cloog/package.py b/var/spack/packages/cloog/package.py deleted file mode 100644 index 814a33c76c..0000000000 --- a/var/spack/packages/cloog/package.py +++ /dev/null @@ -1,26 +0,0 @@ -from spack import * - -class Cloog(Package): - """CLooG is a free software and library to generate code for - scanning Z-polyhedra. That is, it finds a code (e.g. in C, - FORTRAN...) that reaches each integral point of one or more - parameterized polyhedra.""" - - homepage = "http://www.cloog.org" - url = "http://www.bastoul.net/cloog/pages/download/count.php3?url=./cloog-0.18.1.tar.gz" - list_url = "http://www.bastoul.net/cloog/pages/download" - - version('0.18.1', 'e34fca0540d840e5d0f6427e98c92252') - version('0.18.0', 'be78a47bd82523250eb3e91646db5b3d') - version('0.17.0', '0aa3302c81f65ca62c114e5264f8a802') - - depends_on("gmp") - depends_on("isl") - - def install(self, spec, prefix): - configure("--prefix=%s" % prefix, - "--with-osl=no", - "--with-isl=%s" % spec['isl'].prefix, - "--with-gmp=%s" % spec['gmp'].prefix) - make() - make("install") diff --git a/var/spack/packages/cmake/package.py b/var/spack/packages/cmake/package.py deleted file mode 100644 index 9efa370c8b..0000000000 --- a/var/spack/packages/cmake/package.py +++ /dev/null @@ -1,45 +0,0 @@ -############################################################################## -# Copyright (c) 2013, Lawrence Livermore National Security, LLC. -# Produced at the Lawrence Livermore National Laboratory. -# -# This file is part of Spack. -# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. -# LLNL-CODE-647188 -# -# For details, see https://scalability-llnl.github.io/spack -# Please also see the LICENSE file for our notice and the LGPL. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License (as published by -# the Free Software Foundation) version 2.1 dated February 1999. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and -# conditions of the GNU General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -############################################################################## -from spack import * - -class Cmake(Package): - """A cross-platform, open-source build system. CMake is a family of - tools designed to build, test and package software.""" - homepage = 'https://www.cmake.org' - - version('2.8.10.2', '097278785da7182ec0aea8769d06860c', - url = 'http://www.cmake.org/files/v2.8/cmake-2.8.10.2.tar.gz') - - version('3.0.2', 'db4c687a31444a929d2fdc36c4dfb95f', - url = 'http://www.cmake.org/files/v3.0/cmake-3.0.2.tar.gz') - -# version('3.0.1', 'e2e05d84cb44a42f1371d9995631dcf5') -# version('3.0.0', '21a1c85e1a3b803c4b48e7ff915a863e') - - def install(self, spec, prefix): - configure('--prefix=' + prefix, - '--parallel=' + str(make_jobs)) - make() - make('install') diff --git a/var/spack/packages/coreutils/package.py b/var/spack/packages/coreutils/package.py deleted file mode 100644 index 78c608d8eb..0000000000 --- a/var/spack/packages/coreutils/package.py +++ /dev/null @@ -1,17 +0,0 @@ -from spack import * - -class Coreutils(Package): - """The GNU Core Utilities are the basic file, shell and text - manipulation utilities of the GNU operating system. These are - the core utilities which are expected to exist on every - operating system. - """ - homepage = "http://www.gnu.org/software/coreutils/" - url = "http://ftp.gnu.org/gnu/coreutils/coreutils-8.23.tar.xz" - - version('8.23', 'abed135279f87ad6762ce57ff6d89c41') - - def install(self, spec, prefix): - configure("--prefix=%s" % prefix) - make() - make("install") diff --git a/var/spack/packages/cppcheck/package.py b/var/spack/packages/cppcheck/package.py deleted file mode 100644 index 8e98f457ee..0000000000 --- a/var/spack/packages/cppcheck/package.py +++ /dev/null @@ -1,15 +0,0 @@ -from spack import * - -class Cppcheck(Package): - """A tool for static C/C++ code analysis.""" - homepage = "http://cppcheck.sourceforge.net/" - url = "http://downloads.sourceforge.net/project/cppcheck/cppcheck/1.68/cppcheck-1.68.tar.bz2" - - version('1.68', 'c015195f5d61a542f350269030150708') - - def install(self, spec, prefix): - # cppcheck does not have a configure script - make() - # manually install the final cppcheck binary - mkdirp(prefix.bin) - install('cppcheck', prefix.bin) diff --git a/var/spack/packages/cram/package.py b/var/spack/packages/cram/package.py deleted file mode 100644 index 4b8ec56f25..0000000000 --- a/var/spack/packages/cram/package.py +++ /dev/null @@ -1,15 +0,0 @@ -from spack import * - -class Cram(Package): - """Cram runs many small MPI jobs inside one large MPI job.""" - homepage = "https://github.com/scalability-llnl/cram" - url = "http://github.com/scalability-llnl/cram/archive/v1.0.1.tar.gz" - - version('1.0.1', 'c73711e945cf5dc603e44395f6647f5e') - - depends_on("mpi") - - def install(self, spec, prefix): - cmake(".", *std_cmake_args) - make() - make("install") diff --git a/var/spack/packages/cscope/package.py b/var/spack/packages/cscope/package.py deleted file mode 100644 index 9aac0f7304..0000000000 --- a/var/spack/packages/cscope/package.py +++ /dev/null @@ -1,17 +0,0 @@ -from spack import * - -class Cscope(Package): - """Cscope is a developer's tool for browsing source code.""" - homepage = "http://http://cscope.sourceforge.net/" - url = "http://downloads.sourceforge.net/project/cscope/cscope/15.8b/cscope-15.8b.tar.gz" - - version('15.8b', '8f9409a238ee313a96f9f87fe0f3b176') - - # Can be configured to use flex (not necessary) - # ./configure --with-flex - - def install(self, spec, prefix): - configure('--prefix=%s' % prefix) - - make() - make("install") diff --git a/var/spack/packages/cube/package.py b/var/spack/packages/cube/package.py deleted file mode 100644 index d97cd25636..0000000000 --- a/var/spack/packages/cube/package.py +++ /dev/null @@ -1,55 +0,0 @@ -# FIXME: Add copyright statement -# -from spack import * -from contextlib import closing - -class Cube(Package): - """Cube the profile viewer for Score-P and Scalasca profiles. It - displays a multi-dimensional performance space consisting - of the dimensions (i) performance metric, (ii) call path, - and (iii) system resource.""" - - homepage = "http://www.scalasca.org/software/cube-4.x/download.html" - url = "http://apps.fz-juelich.de/scalasca/releases/cube/4.2/dist/cube-4.2.3.tar.gz" - - version('4.2.3', '8f95b9531f5a8f8134f279c2767c9b20') - - version('4.3TP1', 'a2090fbc7b2ba394bd5c09ba971e237f', - url = 'http://apps.fz-juelich.de/scalasca/releases/cube/4.3/dist/cube-4.3-TP1.tar.gz') - - # Using CC as C++ compiler provides quirky workaround for a Score-P build system attempt - # to guess a matching C compiler when configuring scorep-score - backend_user_provided = """\ -CC=cc -CXX=CC -F77=f77 -FC=f90 -#CFLAGS=-fPIC -#CXXFLAGS=-fPIC -""" - frontend_user_provided = """\ -CC_FOR_BUILD=cc -CXX_FOR_BUILD=CC -F77_FOR_BUILD=f70 -FC_FOR_BUILD=f90 -""" - - def install(self, spec, prefix): - # Use a custom compiler configuration, otherwise the score-p - # build system messes with spack's compiler settings. - # Create these three files in the build directory - - with closing(open("vendor/common/build-config/platforms/platform-backend-user-provided", "w")) as backend_file: - backend_file.write(self.backend_user_provided) - with closing(open("vendor/common/build-config/platforms/platform-frontend-user-provided", "w")) as frontend_file: - frontend_file.write(self.frontend_user_provided) - - configure_args = ["--prefix=%s" % prefix, - "--with-custom-compilers", - "--without-paraver", - "--without-gui"] - - configure(*configure_args) - - make(parallel=False) - make("install", parallel=False) diff --git a/var/spack/packages/czmq/package.py b/var/spack/packages/czmq/package.py deleted file mode 100644 index a2f1947554..0000000000 --- a/var/spack/packages/czmq/package.py +++ /dev/null @@ -1,19 +0,0 @@ -from spack import * - -class Czmq(Package): - """ A C interface to the ZMQ library """ - homepage = "http://czmq.zeromq.org" - url = "https://github.com/zeromq/czmq/archive/v3.0.2.tar.gz" - - version('3.0.2', '23e9885f7ee3ce88d99d0425f52e9be1', url='https://github.com/zeromq/czmq/archive/v3.0.2.tar.gz') - - depends_on('zeromq') - - def install(self, spec, prefix): - bash = which("bash") - bash("./autogen.sh") - configure("--prefix=%s" % prefix) - - make() - make("install") - diff --git a/var/spack/packages/dbus/package.py b/var/spack/packages/dbus/package.py deleted file mode 100644 index f7c302d611..0000000000 --- a/var/spack/packages/dbus/package.py +++ /dev/null @@ -1,31 +0,0 @@ -from spack import * - -class Dbus(Package): - """D-Bus is a message bus system, a simple way for applications to - talk to one another. D-Bus supplies both a system daemon (for - events such new hardware device printer queue ) and a - per-user-login-session daemon (for general IPC needs among user - applications). Also, the message bus is built on top of a - general one-to-one message passing framework, which can be used - by any two applications to communicate directly (without going - through the message bus daemon).""" - - homepage = "http://dbus.freedesktop.org/" - url = "http://dbus.freedesktop.org/releases/dbus/dbus-1.8.8.tar.gz" - - version('1.9.0', 'ec6895a4d5c0637b01f0d0e7689e2b36') - version('1.8.8', 'b9f4a18ee3faa1e07c04aa1d83239c43') - version('1.8.6', '6a08ba555d340e9dfe2d623b83c0eea8') - version('1.8.4', '4717cb8ab5b80978fcadf2b4f2f72e1b') - version('1.8.2', 'd6f709bbec0a022a1847c7caec9d6068') - - def install(self, spec, prefix): - configure( - "--prefix=%s" % prefix, - "--disable-systemd") - make() - make("install") - - # dbus needs a machine id generated after install - dbus_uuidgen = Executable(join_path(prefix.bin, 'dbus-uuidgen')) - dbus_uuidgen('--ensure') diff --git a/var/spack/packages/docbook-xml/package.py b/var/spack/packages/docbook-xml/package.py deleted file mode 100644 index fce1de7deb..0000000000 --- a/var/spack/packages/docbook-xml/package.py +++ /dev/null @@ -1,19 +0,0 @@ -import os -import glob -from spack import * - - -class DocbookXml(Package): - """Docbook DTD XML files.""" - homepage = "http://www.oasis-open.org/docbook" - url = "http://www.oasis-open.org/docbook/xml/4.5/docbook-xml-4.5.zip" - - version('4.5', '03083e288e87a7e829e437358da7ef9e') - - def install(self, spec, prefix): - cp = which('cp') - - install_args = ['-a', '-t', prefix] - install_args.extend(glob.glob('*')) - - cp(*install_args) diff --git a/var/spack/packages/doxygen/package.py b/var/spack/packages/doxygen/package.py deleted file mode 100644 index 3d4a4e47a7..0000000000 --- a/var/spack/packages/doxygen/package.py +++ /dev/null @@ -1,25 +0,0 @@ -#------------------------------------------------------------------------------ -# Author: Justin Too -# Date: September 11, 2015 -#------------------------------------------------------------------------------ - -from spack import * - -class Doxygen(Package): - """Doxygen is the de facto standard tool for generating documentation - from annotated C++ sources, but it also supports other popular programming - languages such as C, Objective-C, C#, PHP, Java, Python, IDL (Corba, - Microsoft, and UNO/OpenOffice flavors), Fortran, VHDL, Tcl, and to some extent D.. - """ - homepage = "http://www.stack.nl/~dimitri/doxygen/" - url = "http://ftp.stack.nl/pub/users/dimitri/doxygen-1.8.10.src.tar.gz" - - version('1.8.10', '79767ccd986f12a0f949015efb5f058f') - - depends_on("cmake@2.8.12:") - - def install(self, spec, prefix): - cmake('.', *std_cmake_args) - - make() - make("install") diff --git a/var/spack/packages/dri2proto/package.py b/var/spack/packages/dri2proto/package.py deleted file mode 100644 index 11dfa568e2..0000000000 --- a/var/spack/packages/dri2proto/package.py +++ /dev/null @@ -1,14 +0,0 @@ -from spack import * - -class Dri2proto(Package): - """DRI2 Protocol Headers.""" - homepage = "http://http://cgit.freedesktop.org/xorg/proto/dri2proto/" - url = "http://xorg.freedesktop.org/releases/individual/proto/dri2proto-2.8.tar.gz" - - version('2.8', '19ea18f63d8ae8053c9fa84b60365b77') - - def install(self, spec, prefix): - configure("--prefix=%s" % prefix) - - make() - make("install") diff --git a/var/spack/packages/dtcmp/package.py b/var/spack/packages/dtcmp/package.py deleted file mode 100644 index 9d940583c1..0000000000 --- a/var/spack/packages/dtcmp/package.py +++ /dev/null @@ -1,20 +0,0 @@ -import os -from spack import * - -class Dtcmp(Package): - """The Datatype Comparison Library provides comparison operations and - parallel sort algorithms for MPI applications.""" - - homepage = "https://github.com/hpc/dtcmp" - url = "https://github.com/hpc/dtcmp/releases/download/v1.0.3/dtcmp-1.0.3.tar.gz" - - version('1.0.3', 'cdd8ccf71e8ff67de2558594a7fcd317') - - depends_on('mpi') - depends_on('lwgrp') - - def install(self, spec, prefix): - configure("--prefix=" + prefix, - "--with-lwgrp=" + spec['lwgrp'].prefix) - make() - make("install") diff --git a/var/spack/packages/dyninst/package.py b/var/spack/packages/dyninst/package.py deleted file mode 100644 index 41ec57dd2f..0000000000 --- a/var/spack/packages/dyninst/package.py +++ /dev/null @@ -1,68 +0,0 @@ -############################################################################## -# Copyright (c) 2013, Lawrence Livermore National Security, LLC. -# Produced at the Lawrence Livermore National Laboratory. -# -# This file is part of Spack. -# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. -# LLNL-CODE-647188 -# -# For details, see https://scalability-llnl.github.io/spack -# Please also see the LICENSE file for our notice and the LGPL. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License (as published by -# the Free Software Foundation) version 2.1 dated February 1999. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and -# conditions of the GNU General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -############################################################################## -from spack import * - -class Dyninst(Package): - """API for dynamic binary instrumentation. Modify programs while they - are executing without recompiling, re-linking, or re-executing.""" - homepage = "https://paradyn.org" - url = "http://www.dyninst.org/sites/default/files/downloads/dyninst/8.1.2/DyninstAPI-8.1.2.tgz" - list_url = "http://www.dyninst.org/downloads/dyninst-8.x" - - version('8.2.1', 'abf60b7faabe7a2e4b54395757be39c7', - url="http://www.paradyn.org/release8.2/DyninstAPI-8.2.1.tgz") - version('8.1.2', 'bf03b33375afa66fe0efa46ce3f4b17a', - url="http://www.paradyn.org/release8.1.2/DyninstAPI-8.1.2.tgz") - version('8.1.1', 'd1a04e995b7aa70960cd1d1fac8bd6ac', - url="http://www.paradyn.org/release8.1/DyninstAPI-8.1.1.tgz") - - depends_on("libelf") - depends_on("libdwarf") - depends_on("boost@1.42:") - - # new version uses cmake - def install(self, spec, prefix): - libelf = spec['libelf'].prefix - libdwarf = spec['libdwarf'].prefix - - with working_dir('spack-build', create=True): - cmake('..', - '-DBoost_INCLUDE_DIR=%s' % spec['boost'].prefix.include, - '-DBoost_LIBRARY_DIR=%s' % spec['boost'].prefix.lib, - '-DBoost_NO_SYSTEM_PATHS=TRUE', - '-DLIBELF_INCLUDE_DIR=%s' % join_path(libelf.include, 'libelf'), - '-DLIBELF_LIBRARIES=%s' % join_path(libelf.lib, 'libelf.so'), - '-DLIBDWARF_INCLUDE_DIR=%s' % libdwarf.include, - '-DLIBDWARF_LIBRARIES=%s' % join_path(libdwarf.lib, 'libdwarf.so'), - *std_cmake_args) - make() - make("install") - - - @when('@:8.1') - def install(self, spec, prefix): - configure("--prefix=" + prefix) - make() - make("install") diff --git a/var/spack/packages/elfutils/package.py b/var/spack/packages/elfutils/package.py deleted file mode 100644 index 926d234584..0000000000 --- a/var/spack/packages/elfutils/package.py +++ /dev/null @@ -1,26 +0,0 @@ -from spack import * - -class Elfutils(Package): - """elfutils is a collection of various binary tools such as - eu-objdump, eu-readelf, and other utilities that allow you to - inspect and manipulate ELF files. Refer to Table 5.Tools Included - in elfutils for Red Hat Developer for a complete list of binary - tools that are distributed with the Red Hat Developer Toolset - version of elfutils.""" - - homepage = "https://fedorahosted.org/elfutils/" - - version('0.163', - git='git://git.fedorahosted.org/git/elfutils.git', - tag='elfutils-0.163') - - provides('elf') - - def install(self, spec, prefix): - autoreconf = which('autoreconf') - autoreconf('-if') - - configure('--prefix=%s' % prefix, '--enable-maintainer-mode') - make() - make("install") - diff --git a/var/spack/packages/extrae/package.py b/var/spack/packages/extrae/package.py deleted file mode 100644 index 3ad4cbaf86..0000000000 --- a/var/spack/packages/extrae/package.py +++ /dev/null @@ -1,46 +0,0 @@ -from spack import * - -# typical working line with extrae 3.0.1 -# ./configure --prefix=/usr/local --with-mpi=/usr/lib64/mpi/gcc/openmpi --with-unwind=/usr/local --with-papi=/usr --with-dwarf=/usr --with-elf=/usr --with-dyninst=/usr --with-binutils=/usr --with-xml-prefix=/usr --enable-openmp --enable-nanos --enable-pthread --disable-parallel-merge LDFLAGS=-pthread - -class Extrae(Package): - """Extrae is the package devoted to generate tracefiles which can - be analyzed later by Paraver. Extrae is a tool that uses - different interposition mechanisms to inject probes into the - target application so as to gather information regarding the - application performance. The Extrae instrumentation package can - instrument the MPI programin model, and the following parallel - programming models either alone or in conjunction with MPI : - OpenMP, CUDA, OpenCL, pthread, OmpSs""" - homepage = "http://www.bsc.es/computer-sciences/extrae" - url = "http://www.bsc.es/ssl/apps/performanceTools/files/extrae-3.0.1.tar.bz2" - version('3.0.1', 'a6a8ca96cd877723cd8cc5df6bdb922b') - - depends_on("mpi") - depends_on("dyninst") - depends_on("libunwind") - depends_on("boost") - depends_on("libdwarf") - depends_on("papi") - - def install(self, spec, prefix): - if 'openmpi' in spec: - mpi = spec['openmpi'] - elif 'mpich' in spec: - mpi = spec['mpich'] - elif 'mvapich2' in spec: - mpi = spec['mvapich2'] - - configure("--prefix=%s" % prefix, - "--with-mpi=%s" % mpi.prefix, - "--with-unwind=%s" % spec['libunwind'].prefix, - "--with-dyninst=%s" % spec['dyninst'].prefix, - "--with-boost=%s" % spec['boost'].prefix, - "--with-dwarf=%s" % spec['libdwarf'].prefix, - "--with-papi=%s" % spec['papi'].prefix, - "--with-dyninst-headers=%s" % spec['dyninst'].prefix.include, - "--with-dyninst-libs=%s" % spec['dyninst'].prefix.lib) - - make() - make("install", parallel=False) - diff --git a/var/spack/packages/exuberant-ctags/package.py b/var/spack/packages/exuberant-ctags/package.py deleted file mode 100644 index efd2b541b2..0000000000 --- a/var/spack/packages/exuberant-ctags/package.py +++ /dev/null @@ -1,14 +0,0 @@ -from spack import * - -class ExuberantCtags(Package): - """The canonical ctags generator""" - homepage = "ctags.sourceforge.net" - url = "http://downloads.sourceforge.net/project/ctags/ctags/5.8/ctags-5.8.tar.gz" - - version('5.8', 'c00f82ecdcc357434731913e5b48630d') - - def install(self, spec, prefix): - configure('--prefix=%s' % prefix) - - make() - make("install") diff --git a/var/spack/packages/fish/package.py b/var/spack/packages/fish/package.py deleted file mode 100644 index 1225558705..0000000000 --- a/var/spack/packages/fish/package.py +++ /dev/null @@ -1,18 +0,0 @@ -from spack import * - -class Fish(Package): - """fish is a smart and user-friendly command line shell for OS X, Linux, and - the rest of the family. - """ - - homepage = "http://fishshell.com/" - url = "http://fishshell.com/files/2.2.0/fish-2.2.0.tar.gz" - list_url = homepage - - version('2.2.0', 'a76339fd14ce2ec229283c53e805faac48c3e99d9e3ede9d82c0554acfc7b77a') - - def install(self, spec, prefix): - configure('--prefix=%s' % prefix) - - make() - make("install") diff --git a/var/spack/packages/flex/package.py b/var/spack/packages/flex/package.py deleted file mode 100644 index b065904912..0000000000 --- a/var/spack/packages/flex/package.py +++ /dev/null @@ -1,15 +0,0 @@ -from spack import * - -class Flex(Package): - """Flex is a tool for generating scanners.""" - - homepage = "http://flex.sourceforge.net/" - url = "http://download.sourceforge.net/flex/flex-2.5.39.tar.gz" - - version('2.5.39', 'e133e9ead8ec0a58d81166b461244fde') - - def install(self, spec, prefix): - configure("--prefix=%s" % prefix) - - make() - make("install") diff --git a/var/spack/packages/flux/package.py b/var/spack/packages/flux/package.py deleted file mode 100644 index c128f46be8..0000000000 --- a/var/spack/packages/flux/package.py +++ /dev/null @@ -1,36 +0,0 @@ -from spack import * -import os - -class Flux(Package): - """ A next-generation resource manager (pre-alpha) """ - - homepage = "https://github.com/flux-framework/flux-core" - url = "https://github.com/flux-framework/flux-core" - - version('master', branch='master', git='https://github.com/flux-framework/flux-core') - - # Also needs autotools, but should use the system version if available - depends_on("zeromq@4.0.4:") - depends_on("czmq@2.2:") - depends_on("lua@5.1:5.1.99") - depends_on("munge") - depends_on("libjson-c") - depends_on("libxslt") - # TODO: This provides a catalog, hacked with environment below for now - depends_on("docbook-xml") - depends_on("asciidoc") - depends_on("python") - depends_on("py-cffi") - - def install(self, spec, prefix): - # Bootstrap with autotools - bash = which('bash') - bash('./autogen.sh') - - # Fix asciidoc dependency on xml style sheets and whatnot - os.environ['XML_CATALOG_FILES'] = os.path.join(spec['docbook-xml'].prefix, - 'catalog.xml') - # Configure, compile & install - configure("--prefix=" + prefix) - make("install", "V=1") - diff --git a/var/spack/packages/fontconfig/package.py b/var/spack/packages/fontconfig/package.py deleted file mode 100644 index 89b13604e8..0000000000 --- a/var/spack/packages/fontconfig/package.py +++ /dev/null @@ -1,16 +0,0 @@ -from spack import * - -class Fontconfig(Package): - """Fontconfig customizing font access""" - homepage = "http://www.freedesktop.org/wiki/Software/fontconfig/" - url = "http://www.freedesktop.org/software/fontconfig/release/fontconfig-2.11.1.tar.gz" - - version('2.11.1' , 'e75e303b4f7756c2b16203a57ac87eba') - - depends_on('freetype') - - def install(self, spec, prefix): - configure("--prefix=%s" % prefix) - - make() - make("install") diff --git a/var/spack/packages/freetype/package.py b/var/spack/packages/freetype/package.py deleted file mode 100644 index 0309b858a1..0000000000 --- a/var/spack/packages/freetype/package.py +++ /dev/null @@ -1,16 +0,0 @@ -from spack import * - -class Freetype(Package): - """Font package""" - homepage = "http://http://www.freetype.org" - url = "http://download.savannah.gnu.org/releases/freetype/freetype-2.5.3.tar.gz" - - version('2.5.3' , 'cafe9f210e45360279c730d27bf071e9') - - depends_on('libpng') - - def install(self, spec, prefix): - configure("--prefix=%s" % prefix) - - make() - make("install") diff --git a/var/spack/packages/gasnet/package.py b/var/spack/packages/gasnet/package.py deleted file mode 100644 index 705961d1de..0000000000 --- a/var/spack/packages/gasnet/package.py +++ /dev/null @@ -1,35 +0,0 @@ -from spack import * - -class Gasnet(Package): - """GASNet is a language-independent, low-level networking layer - that provides network-independent, high-performance communication - primitives tailored for implementing parallel global address space - SPMD languages and libraries such as UPC, Co-Array Fortran, SHMEM, - Cray Chapel, and Titanium. - """ - homepage = "http://gasnet.lbl.gov" - url = "http://gasnet.lbl.gov/GASNet-1.24.0.tar.gz" - - version('1.24.0', 'c8afdf48381e8b5a7340bdb32ca0f41a') - - - def install(self, spec, prefix): - # TODO: don't use paths with @ in them. - change_sed_delimiter('@', ';', 'configure') - - configure("--prefix=%s" % prefix, - # TODO: factor IB suport out into architecture description. - "--enable-ibv", - "--enable-udp", - "--disable-mpi", - "--enable-par", - "--enable-mpi-compat", - "--enable-segment-fast", - "--disable-aligned-segments", - # TODO: make an option so that Legion can request builds with/without this. - # See the Legion webpage for details on when to/not to use. - "--disable-pshm", - "--with-segment-mmap-max=64MB") - - make() - make("install") diff --git a/var/spack/packages/gcc/package.py b/var/spack/packages/gcc/package.py deleted file mode 100644 index a49a1348aa..0000000000 --- a/var/spack/packages/gcc/package.py +++ /dev/null @@ -1,122 +0,0 @@ -############################################################################## -# Copyright (c) 2013, Lawrence Livermore National Security, LLC. -# Produced at the Lawrence Livermore National Laboratory. -# -# This file is part of Spack. -# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. -# LLNL-CODE-647188 -# -# For details, see https://scalability-llnl.github.io/spack -# Please also see the LICENSE file for our notice and the LGPL. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License (as published by -# the Free Software Foundation) version 2.1 dated February 1999. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and -# conditions of the GNU General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -############################################################################## -from spack import * - -from contextlib import closing -from glob import glob - -class Gcc(Package): - """The GNU Compiler Collection includes front ends for C, C++, - Objective-C, Fortran, and Java.""" - homepage = "https://gcc.gnu.org" - - url = "http://open-source-box.org/gcc/gcc-4.9.2/gcc-4.9.2.tar.bz2" - list_url = 'http://open-source-box.org/gcc/' - list_depth = 2 - - DEPENDS_ON_ISL_PREDICATE = '@5.0:' - - version('5.2.0', 'a51bcfeb3da7dd4c623e27207ed43467') - version('4.9.3', '6f831b4d251872736e8e9cc09746f327') - version('4.9.2', '4df8ee253b7f3863ad0b86359cd39c43') - version('4.9.1', 'fddf71348546af523353bd43d34919c1') - version('4.8.5', '80d2c2982a3392bb0b89673ff136e223') - version('4.8.4', '5a84a30839b2aca22a2d723de2a626ec') - version('4.7.4', '4c696da46297de6ae77a82797d2abe28') - version('4.6.4', 'b407a3d1480c11667f293bfb1f17d1a4') - version('4.5.4', '27e459c2566b8209ab064570e1b378f7') - - depends_on("mpfr") - depends_on("gmp") - depends_on("mpc") # when @4.5: - depends_on("binutils~libiberty") - - # Save these until we can do optional deps. - depends_on("isl", when=DEPENDS_ON_ISL_PREDICATE) - #depends_on("ppl") - #depends_on("cloog") - - def install(self, spec, prefix): - # libjava/configure needs a minor fix to install into spack paths. - filter_file(r"'@.*@'", "'@[[:alnum:]]*@'", 'libjava/configure', string=True) - - enabled_languages = set(('c', 'c++', 'fortran', 'java', 'objc')) - if spec.satisfies("@4.7.1:"): - enabled_languages.add('go') - - # Generic options to compile GCC - options = ["--prefix=%s" % prefix, - "--libdir=%s/lib64" % prefix, - "--disable-multilib", - "--enable-languages=" + ','.join(enabled_languages), - "--with-mpc=%s" % spec['mpc'].prefix, - "--with-mpfr=%s" % spec['mpfr'].prefix, - "--with-gmp=%s" % spec['gmp'].prefix, - "--enable-lto", - "--with-gnu-ld", - "--with-gnu-as", - "--with-quad"] - # Binutils - binutils_options = ["--with-stage1-ldflags=%s" % self.rpath_args, - "--with-boot-ldflags=%s" % self.rpath_args, - "--with-ld=%s/bin/ld" % spec['binutils'].prefix, - "--with-as=%s/bin/as" % spec['binutils'].prefix] - options.extend(binutils_options) - # Isl - if spec.satisfies(Gcc.DEPENDS_ON_ISL_PREDICATE): - isl_options = ["--with-isl=%s" % spec['isl'].prefix] - options.extend(isl_options) - - # Rest of install is straightforward. - configure(*options) - make() - make("install") - - self.write_rpath_specs() - - - @property - def spec_dir(self): - # e.g. lib64/gcc/x86_64-unknown-linux-gnu/4.9.2 - spec_dir = glob("%s/lib64/gcc/*/*" % self.prefix) - return spec_dir[0] if spec_dir else None - - - def write_rpath_specs(self): - """Generate a spec file so the linker adds a rpath to the libs - the compiler used to build the executable.""" - if not self.spec_dir: - tty.warn("Could not install specs for %s." % self.spec.format('$_$@')) - return - - gcc = Executable(join_path(self.prefix.bin, 'gcc')) - lines = gcc('-dumpspecs', return_output=True).strip().split("\n") - specs_file = join_path(self.spec_dir, 'specs') - with closing(open(specs_file, 'w')) as out: - for line in lines: - out.write(line + "\n") - if line.startswith("*link:"): - out.write("-rpath %s/lib:%s/lib64 \\\n"% (self.prefix, self.prefix)) - set_install_permissions(specs_file) diff --git a/var/spack/packages/gdk-pixbuf/package.py b/var/spack/packages/gdk-pixbuf/package.py deleted file mode 100644 index 14a5569984..0000000000 --- a/var/spack/packages/gdk-pixbuf/package.py +++ /dev/null @@ -1,22 +0,0 @@ -from spack import * - -class GdkPixbuf(Package): - """The Gdk Pixbuf is a toolkit for image loading and pixel buffer - manipulation. It is used by GTK+ 2 and GTK+ 3 to load and - manipulate images. In the past it was distributed as part of - GTK+ 2 but it was split off into a separate package in - preparation for the change to GTK+ 3.""" - homepage = "https://developer.gnome.org/gdk-pixbuf/" - url = "http://ftp.gnome.org/pub/gnome/sources/gdk-pixbuf/2.31/gdk-pixbuf-2.31.1.tar.xz" - - version('2.31.2', '6be6bbc4f356d4b79ab4226860ab8523') - - depends_on("glib") - depends_on("jpeg") - depends_on("libpng") - depends_on("libtiff") - - def install(self, spec, prefix): - configure("--prefix=%s" % prefix) - make() - make("install") diff --git a/var/spack/packages/geos/package.py b/var/spack/packages/geos/package.py deleted file mode 100644 index 4a2657e32f..0000000000 --- a/var/spack/packages/geos/package.py +++ /dev/null @@ -1,31 +0,0 @@ -from spack import * - -class Geos(Package): - """GEOS (Geometry Engine - Open Source) is a C++ port of the Java - Topology Suite (JTS). As such, it aims to contain the complete - functionality of JTS in C++. This includes all the OpenGIS - Simple Features for SQL spatial predicate functions and spatial - operators, as well as specific JTS enhanced topology functions.""" - - homepage = "http://trac.osgeo.org/geos/" - url = "http://download.osgeo.org/geos/geos-3.4.2.tar.bz2" - - version('3.4.2', 'fc5df2d926eb7e67f988a43a92683bae') - version('3.4.1', '4c930dec44c45c49cd71f3e0931ded7e') - version('3.4.0', 'e41318fc76b5dc764a69d43ac6b18488') - version('3.3.9', '4794c20f07721d5011c93efc6ccb8e4e') - version('3.3.8', '75be476d0831a2d14958fed76ca266de') - version('3.3.7', '95ab996d22672b067d92c7dee2170460') - version('3.3.6', '6fadfb941541875f4976f75fb0bbc800') - version('3.3.5', '2ba61afb7fe2c5ddf642d82d7b16e75b') - version('3.3.4', '1bb9f14d57ef06ffa41cb1d67acb55a1') - version('3.3.3', '8454e653d7ecca475153cc88fd1daa26') - - extends('python') - depends_on('swig') - - def install(self, spec, prefix): - configure("--prefix=%s" % prefix, - "--enable-python") - make() - make("install") diff --git a/var/spack/packages/gflags/package.py b/var/spack/packages/gflags/package.py deleted file mode 100644 index 62dd80a094..0000000000 --- a/var/spack/packages/gflags/package.py +++ /dev/null @@ -1,21 +0,0 @@ -import os -from spack import * - -class Gflags(Package): - """The gflags package contains a C++ library that implements - commandline flags processing. It includes built-in support for - standard types such as string and the ability to define flags - in the source file in which they are used. Online documentation - available at: https://gflags.github.io/gflags/""" - - homepage = "https://gflags.github.io/gflags" - url = "https://github.com/gflags/gflags/archive/v2.1.2.tar.gz" - - version('2.1.2', 'ac432de923f9de1e9780b5254884599f') - - def install(self, spec, prefix): - cmake("-DCMAKE_INSTALL_PREFIX=" + prefix, - "-DBUILD_SHARED_LIBS=ON") - make() - make("test") - make("install") diff --git a/var/spack/packages/ghostscript/package.py b/var/spack/packages/ghostscript/package.py deleted file mode 100644 index 0ab49d425f..0000000000 --- a/var/spack/packages/ghostscript/package.py +++ /dev/null @@ -1,17 +0,0 @@ -from spack import * - -class Ghostscript(Package): - """an interpreter for the PostScript language and for PDF. """ - homepage = "http://ghostscript.com/" - url = "http://downloads.ghostscript.com/public/ghostscript-9.16.tar.gz" - - version('9.16', '829319325bbdb83f5c81379a8f86f38f') - - parallel = False - - def install(self, spec, prefix): - configure("--prefix=%s" %prefix, "--enable-shared") - - make() - make("install") - diff --git a/var/spack/packages/git/package.py b/var/spack/packages/git/package.py deleted file mode 100644 index 0f1a3ba05b..0000000000 --- a/var/spack/packages/git/package.py +++ /dev/null @@ -1,27 +0,0 @@ -from spack import * - -class Git(Package): - """Git is a free and open source distributed version control - system designed to handle everything from small to very large - projects with speed and efficiency.""" - homepage = "http://git-scm.com" - url = "https://www.kernel.org/pub/software/scm/git/git-2.2.1.tar.xz" - - version('2.2.1', '43e01f9d96ba8c11611e0eef0d9f9f28') - - # Use system openssl. - # depends_on("openssl") - - # Use system perl for now. - # depends_on("perl") - # depends_on("pcre") - - depends_on("zlib") - - def install(self, spec, prefix): - configure("--prefix=%s" % prefix, - "--without-pcre", - "--without-python") - - make() - make("install") diff --git a/var/spack/packages/glib/package.py b/var/spack/packages/glib/package.py deleted file mode 100644 index 178f0b9df5..0000000000 --- a/var/spack/packages/glib/package.py +++ /dev/null @@ -1,18 +0,0 @@ -from spack import * - -class Glib(Package): - """The GLib package contains a low-level libraries useful for - providing data structure handling for C, portability wrappers - and interfaces for such runtime functionality as an event loop, - threads, dynamic loading and an object system.""" - homepage = "https://developer.gnome.org/glib/" - url = "http://ftp.gnome.org/pub/gnome/sources/glib/2.42/glib-2.42.1.tar.xz" - - version('2.42.1', '89c4119e50e767d3532158605ee9121a') - - depends_on("libffi") - - def install(self, spec, prefix): - configure("--prefix=%s" % prefix) - make() - make("install") diff --git a/var/spack/packages/glm/package.py b/var/spack/packages/glm/package.py deleted file mode 100644 index d00c301b4c..0000000000 --- a/var/spack/packages/glm/package.py +++ /dev/null @@ -1,19 +0,0 @@ -from spack import * - - -class Glm(Package): - """ - OpenGL Mathematics (GLM) is a header only C++ mathematics library for graphics software based on - the OpenGL Shading Language (GLSL) specification. - """ - - homepage = "https://github.com/g-truc/glm" - url = "https://github.com/g-truc/glm/archive/0.9.7.1.tar.gz" - - version('0.9.7.1', '61af6639cdf652d1cdd7117190afced8') - - def install(self, spec, prefix): - with working_dir('spack-build', create=True): - cmake('..', *std_cmake_args) - make() - make("install") diff --git a/var/spack/packages/global/package.py b/var/spack/packages/global/package.py deleted file mode 100644 index a77b1bdc09..0000000000 --- a/var/spack/packages/global/package.py +++ /dev/null @@ -1,24 +0,0 @@ -from spack import * -import os - - -class Global(Package): - """ The Gnu Global tagging system """ - # FIXME: add a proper url for your package's homepage here. - homepage = "http://www.gnu.org/software/global" - url = "http://tamacom.com/global/global-6.5.tar.gz" - - version('6.5', 'dfec818b4f53d91721e247cf7b218078') - - depends_on('exuberant-ctags') - - def install(self, spec, prefix): - config_args = ['--prefix={}'.format(prefix)] - - config_args.append('--with-exuberant-ctags={}'.format( - os.path.join(spec['exuberant-ctags'].prefix.bin, 'ctags'))) - - configure(*config_args) - - make() - make("install") diff --git a/var/spack/packages/glog/package.py b/var/spack/packages/glog/package.py deleted file mode 100644 index d73386b394..0000000000 --- a/var/spack/packages/glog/package.py +++ /dev/null @@ -1,15 +0,0 @@ -import os -from spack import * - -class Glog(Package): - """C++ implementation of the Google logging module.""" - - homepage = "https://github.com/google/glog" - url = "https://github.com/google/glog/archive/v0.3.3.tar.gz" - - version('0.3.3', 'c1f86af27bd9c73186730aa957607ed0') - - def install(self, spec, prefix): - configure("--prefix=" + prefix) - make() - make("install") diff --git a/var/spack/packages/gmp/package.py b/var/spack/packages/gmp/package.py deleted file mode 100644 index d6af821b34..0000000000 --- a/var/spack/packages/gmp/package.py +++ /dev/null @@ -1,40 +0,0 @@ -############################################################################## -# Copyright (c) 2013, Lawrence Livermore National Security, LLC. -# Produced at the Lawrence Livermore National Laboratory. -# -# This file is part of Spack. -# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. -# LLNL-CODE-647188 -# -# For details, see https://scalability-llnl.github.io/spack -# Please also see the LICENSE file for our notice and the LGPL. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License (as published by -# the Free Software Foundation) version 2.1 dated February 1999. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and -# conditions of the GNU General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -############################################################################## -from spack import * - -class Gmp(Package): - """GMP is a free library for arbitrary precision arithmetic, - operating on signed integers, rational numbers, and - floating-point numbers.""" - homepage = "https://gmplib.org" - url = "https://gmplib.org/download/gmp/gmp-6.0.0a.tar.bz2" - - version('6.0.0a', 'b7ff2d88cae7f8085bd5006096eed470') - version('6.0.0' , '6ef5869ae735db9995619135bd856b84') - - def install(self, spec, prefix): - configure("--prefix=%s" % prefix) - make() - make("install") diff --git a/var/spack/packages/gnutls/package.py b/var/spack/packages/gnutls/package.py deleted file mode 100644 index cf57a24a6d..0000000000 --- a/var/spack/packages/gnutls/package.py +++ /dev/null @@ -1,22 +0,0 @@ -from spack import * - -class Gnutls(Package): - """GnuTLS is a secure communications library implementing the SSL, - TLS and DTLS protocols and technologies around them. It - provides a simple C language application programming interface - (API) to access the secure communications protocols as well as - APIs to parse and write X.509, PKCS #12, OpenPGP and other - required structures. It is aimed to be portable and efficient - with focus on security and interoperability.""" - - homepage = "http://www.gnutls.org" - url = "ftp://ftp.gnutls.org/gcrypt/gnutls/v3.3/gnutls-3.3.9.tar.xz" - - version('3.3.9', 'ff61b77e39d09f1140ab5a9cf52c58b6') - - depends_on("nettle") - - def install(self, spec, prefix): - configure("--prefix=%s" % prefix) - make() - make("install") diff --git a/var/spack/packages/gperf/package.py b/var/spack/packages/gperf/package.py deleted file mode 100644 index 32551b67b4..0000000000 --- a/var/spack/packages/gperf/package.py +++ /dev/null @@ -1,19 +0,0 @@ -from spack import * - -class Gperf(Package): - """GNU gperf is a perfect hash function generator. For a given - list of strings, it produces a hash function and hash table, in - form of C or C++ code, for looking up a value depending on the - input string. The hash function is perfect, which means that the - hash table has no collisions, and the hash table lookup needs a - single string comparison only.""" - - homepage = "https://www.gnu.org/software/gperf/" - url = "http://ftp.gnu.org/pub/gnu/gperf/gperf-3.0.4.tar.gz" - - version('3.0.4', 'c1f1db32fb6598d6a93e6e88796a8632') - - def install(self, spec, prefix): - configure("--prefix=%s" % prefix) - make() - make("install") diff --git a/var/spack/packages/gperftools/package.py b/var/spack/packages/gperftools/package.py deleted file mode 100644 index 8900462324..0000000000 --- a/var/spack/packages/gperftools/package.py +++ /dev/null @@ -1,38 +0,0 @@ -############################################################################## -# Copyright (c) 2013, Lawrence Livermore National Security, LLC. -# Produced at the Lawrence Livermore National Laboratory. -# -# This file is part of Spack. -# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. -# LLNL-CODE-647188 -# -# For details, see https://scalability-llnl.github.io/spack -# Please also see the LICENSE file for our notice and the LGPL. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License (as published by -# the Free Software Foundation) version 2.1 dated February 1999. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and -# conditions of the GNU General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -############################################################################## -from spack import * - -class Gperftools(Package): - """Google's fast malloc/free implementation, especially for multi-threaded applications. - Contains tcmalloc, heap-checker, heap-profiler, and cpu-profiler.""" - homepage = "https://code.google.com/p/gperftools" - url = "https://googledrive.com/host/0B6NtGsLhIcf7MWxMMF9JdTN3UVk/gperftools-2.3.tar.gz" - - version('2.3', 'f54dd119f0e46ac1f13264f8d97adf90', url="https://googledrive.com/host/0B6NtGsLhIcf7MWxMMF9JdTN3UVk/gperftools-2.3.tar.gz") - - def install(self, spec, prefix): - configure("--prefix=" + prefix) - make() - make("install") diff --git a/var/spack/packages/graphlib/package.py b/var/spack/packages/graphlib/package.py deleted file mode 100644 index ddac0b2b66..0000000000 --- a/var/spack/packages/graphlib/package.py +++ /dev/null @@ -1,14 +0,0 @@ -from spack import * - -class Graphlib(Package): - """Library to create, manipulate, and export graphs Graphlib.""" - homepage = "http://https://github.com/lee218llnl/graphlib" - url = "https://github.com/lee218llnl/graphlib/archive/v2.0.0.tar.gz" - - version('2.0.0', '43c6df84f1d38ba5a5dce0ae19371a70') - - def install(self, spec, prefix): - cmake(".", *std_cmake_args) - - make() - make("install") diff --git a/var/spack/packages/graphviz/package.py b/var/spack/packages/graphviz/package.py deleted file mode 100644 index 7af7da1881..0000000000 --- a/var/spack/packages/graphviz/package.py +++ /dev/null @@ -1,21 +0,0 @@ -from spack import * - -class Graphviz(Package): - """Graph Visualization Software""" - homepage = "http://www.graphviz.org" - url = "http://www.graphviz.org/pub/graphviz/stable/SOURCES/graphviz-2.38.0.tar.gz" - - version('2.38.0', '5b6a829b2ac94efcd5fa3c223ed6d3ae') - - parallel = False - - depends_on("swig") - depends_on("python") - depends_on("ghostscript") - - def install(self, spec, prefix): - configure("--prefix=%s" %prefix) - - make() - make("install") - diff --git a/var/spack/packages/gtkplus/package.py b/var/spack/packages/gtkplus/package.py deleted file mode 100644 index 0ebc7100de..0000000000 --- a/var/spack/packages/gtkplus/package.py +++ /dev/null @@ -1,22 +0,0 @@ -from spack import * - -class Gtkplus(Package): - """The GTK+ 2 package contains libraries used for creating graphical user interfaces for applications.""" - homepage = "http://www.gtk.org" - - version('2.24.25', '612350704dd3aacb95355a4981930c6f', - url="http://ftp.gnome.org/pub/gnome/sources/gtk+/2.24/gtk+-2.24.25.tar.xz") - - depends_on("atk") - depends_on("gdk-pixbuf") - depends_on("pango") - - def patch(self): - # remove disable deprecated flag. - filter_file(r'CFLAGS="-DGDK_PIXBUF_DISABLE_DEPRECATED $CFLAGS"', - '', 'configure', string=True) - - def install(self, spec, prefix): - configure("--prefix=%s" % prefix) - make() - make("install") diff --git a/var/spack/packages/harfbuzz/package.py b/var/spack/packages/harfbuzz/package.py deleted file mode 100644 index ed7c42a909..0000000000 --- a/var/spack/packages/harfbuzz/package.py +++ /dev/null @@ -1,20 +0,0 @@ -from spack import * - -class Harfbuzz(Package): - """The Harfbuzz package contains an OpenType text shaping engine.""" - homepage = "http://www.freedesktop.org/wiki/Software/HarfBuzz/" - url = "http://www.freedesktop.org/software/harfbuzz/release/harfbuzz-0.9.37.tar.bz2" - - version('0.9.37', 'bfe733250e34629a188d82e3b971bc1e') - - depends_on("glib") - depends_on("icu") - depends_on("freetype") - - def patch(self): - change_sed_delimiter('@', ';', 'src/Makefile.in') - - def install(self, spec, prefix): - configure("--prefix=%s" % prefix) - make() - make("install") diff --git a/var/spack/packages/hdf5/package.py b/var/spack/packages/hdf5/package.py deleted file mode 100644 index 15e0ef9338..0000000000 --- a/var/spack/packages/hdf5/package.py +++ /dev/null @@ -1,42 +0,0 @@ -from spack import * - -class Hdf5(Package): - """HDF5 is a data model, library, and file format for storing and managing - data. It supports an unlimited variety of datatypes, and is designed for - flexible and efficient I/O and for high volume and complex data. - """ - - homepage = "http://www.hdfgroup.org/HDF5/" - url = "http://www.hdfgroup.org/ftp/HDF5/releases/hdf5-1.8.13/src/hdf5-1.8.13.tar.gz" - list_url = "http://www.hdfgroup.org/ftp/HDF5/releases" - list_depth = 3 - - version('1.8.15', '03cccb5b33dbe975fdcd8ae9dc021f24') - version('1.8.13', 'c03426e9e77d7766944654280b467289') - - depends_on("mpi") - depends_on("zlib") - - # TODO: currently hard-coded to use OpenMPI - def install(self, spec, prefix): - - configure( - "--prefix=%s" % prefix, - "--with-zlib=%s" % spec['zlib'].prefix, - "--enable-parallel", - "--enable-shared", - "CC=%s" % spec['mpich'].prefix.bin + "/mpicc", - "CXX=%s" % spec['mpich'].prefix.bin + "/mpic++") - - make() - make("install") - - def url_for_version(self, version): - v = str(version) - - if version == Version("1.2.2"): - return "http://www.hdfgroup.org/ftp/HDF5/releases/hdf5-" + v + ".tar.gz" - elif version < Version("1.7"): - return "http://www.hdfgroup.org/ftp/HDF5/releases/hdf5-" + version.up_to(2) + "/hdf5-" + v + ".tar.gz" - else: - return "http://www.hdfgroup.org/ftp/HDF5/releases/hdf5-" + v + "/src/hdf5-" + v + ".tar.gz" diff --git a/var/spack/packages/hwloc/package.py b/var/spack/packages/hwloc/package.py deleted file mode 100644 index 31a31f376a..0000000000 --- a/var/spack/packages/hwloc/package.py +++ /dev/null @@ -1,25 +0,0 @@ -from spack import * - -class Hwloc(Package): - """The Portable Hardware Locality (hwloc) software package - provides a portable abstraction (across OS, versions, - architectures, ...) of the hierarchical topology of modern - architectures, including NUMA memory nodes, sockets, shared - caches, cores and simultaneous multithreading. It also gathers - various system attributes such as cache and memory information - as well as the locality of I/O devices such as network - interfaces, InfiniBand HCAs or GPUs. It primarily aims at - helping applications with gathering information about modern - computing hardware so as to exploit it accordingly and - efficiently.""" - homepage = "http://www.open-mpi.org/projects/hwloc/" - url = "http://www.open-mpi.org/software/hwloc/v1.9/downloads/hwloc-1.9.tar.gz" - - version('1.9', '1f9f9155682fe8946a97c08896109508') - - def install(self, spec, prefix): - configure("--prefix=%s" % prefix) - - make() - make("install") - diff --git a/var/spack/packages/hypre/package.py b/var/spack/packages/hypre/package.py deleted file mode 100644 index 198b3f00dc..0000000000 --- a/var/spack/packages/hypre/package.py +++ /dev/null @@ -1,32 +0,0 @@ -from spack import * - -class Hypre(Package): - """Hypre is a library of high performance preconditioners that - features parallel multigrid methods for both structured and - unstructured grid problems.""" - - homepage = "https://computation.llnl.gov/project/linear_solvers/software.php" - url = "https://computation.llnl.gov/project/linear_solvers/download/hypre-2.10.0b.tar.gz" - - version('2.10.0b', '768be38793a35bb5d055905b271f5b8e') - - depends_on("mpi") - depends_on("blas") - depends_on("lapack") - - def install(self, spec, prefix): - blas_dir = spec['blas'].prefix - lapack_dir = spec['lapack'].prefix - - # Hypre's source is staged under ./src so we'll have to manually - # cd into it. - with working_dir("src"): - configure( - "--prefix=%s" % prefix, - "--with-blas-libs=blas", - "--with-blas-lib-dirs=%s/lib" % blas_dir, - "--with-lapack-libs=\"lapack blas\"", - "--with-lapack-lib-dirs=%s/lib" % lapack_dir, - "--with-MPI") - make() - make("install") diff --git a/var/spack/packages/icu/package.py b/var/spack/packages/icu/package.py deleted file mode 100644 index f256ec5712..0000000000 --- a/var/spack/packages/icu/package.py +++ /dev/null @@ -1,25 +0,0 @@ -from spack import * - -class Icu(Package): - """The International Components for Unicode (ICU) package is a - mature, widely used set of C/C++ libraries providing Unicode and - Globalization support for software applications. ICU is widely - portable and gives applications the same results on all - platforms.""" - # FIXME: add a proper url for your package's homepage here. - homepage = "http://www.example.com" - url = "http://download.icu-project.org/files/icu4c/54.1/icu4c-54_1-src.tgz" - - version('54.1', 'e844caed8f2ca24c088505b0d6271bc0') - - - def url_for_version(self, version): - return "http://download.icu-project.org/files/icu4c/%s/icu4c-%s-src.tgz" % ( - version, str(version).replace('.', '_')) - - - def install(self, spec, prefix): - with working_dir("source"): - configure("--prefix=%s" % prefix) - make() - make("install") diff --git a/var/spack/packages/icu4c/package.py b/var/spack/packages/icu4c/package.py deleted file mode 100644 index 55b44463b2..0000000000 --- a/var/spack/packages/icu4c/package.py +++ /dev/null @@ -1,17 +0,0 @@ -from spack import * - -class Icu4c(Package): - """ICU is a mature, widely used set of C/C++ and Java libraries - providing Unicode and Globalization support for software applications.""" - - homepage = "http://site.icu-project.org/" - url = "http://downloads.sourceforge.net/project/icu/ICU4C/54.1/icu4c-54_1-src.tgz" - - version('54_1', 'e844caed8f2ca24c088505b0d6271bc0') - - def install(self, spec, prefix): - cd("source") - configure("--prefix=%s" % prefix) - - make() - make("install") diff --git a/var/spack/packages/isl/package.py b/var/spack/packages/isl/package.py deleted file mode 100644 index 836ef3ea40..0000000000 --- a/var/spack/packages/isl/package.py +++ /dev/null @@ -1,17 +0,0 @@ -from spack import * - -class Isl(Package): - """isl is a thread-safe C library for manipulating sets and - relations of integer points bounded by affine constraints.""" - homepage = "http://isl.gforge.inria.fr" - url = "http://isl.gforge.inria.fr/isl-0.14.tar.bz2" - - version('0.14', 'acd347243fca5609e3df37dba47fd0bb') - - depends_on("gmp") - - def install(self, spec, prefix): - configure("--prefix=%s" % prefix, - "--with-gmp-prefix=%s" % spec['gmp'].prefix) - make() - make("install") diff --git a/var/spack/packages/jdk/package.py b/var/spack/packages/jdk/package.py deleted file mode 100644 index 8f8076dd14..0000000000 --- a/var/spack/packages/jdk/package.py +++ /dev/null @@ -1,46 +0,0 @@ -#------------------------------------------------------------------------------ -# Author: Justin Too -#------------------------------------------------------------------------------ -import distutils -from distutils import dir_util -from subprocess import call - -import spack -from spack import * -import llnl.util.tty as tty - -class Jdk(Package): - """The Java Development Kit (JDK) released by Oracle Corporation - in the form of a binary product aimed at Java developers.""" - homepage = "http://www.oracle.com/technetwork/java/javase/downloads/index.html" - - version('8u25-linux-x64', 'e145c03a7edc845215092786bcfba77e', - url="http://download.oracle.com/otn-pub/java/jdk/8u25-b17/jdk-8u25-linux-x64.tar.gz") - - # Oracle requires that you accept their License Agreement in order - # to access the Java packages in download.oracle.com. In order to - # automate this process, we need to utilize these additional curl - # commandline options. - # - # See http://stackoverflow.com/questions/10268583/how-to-automate-download-and-installation-of-java-jdk-on-linux - curl_options=[ - '-j', # junk cookies - '-H', # specify required License Agreement cookie - 'Cookie: oraclelicense=accept-securebackup-cookie'] - - def do_fetch(self): - # Add our custom curl commandline options - tty.msg( - "[Jdk] Adding required commandline options to curl " + - "before performing fetch: %s" % - (self.curl_options)) - - for option in self.curl_options: - spack.curl.add_default_arg(option) - - # Now perform the actual fetch - super(Jdk, self).do_fetch() - - - def install(self, spec, prefix): - distutils.dir_util.copy_tree(".", prefix) diff --git a/var/spack/packages/jpeg/package.py b/var/spack/packages/jpeg/package.py deleted file mode 100644 index 87820467db..0000000000 --- a/var/spack/packages/jpeg/package.py +++ /dev/null @@ -1,14 +0,0 @@ -from spack import * - -class Jpeg(Package): - """jpeg library""" - homepage = "http://www.ijg.org" - url = "http://www.ijg.org/files/jpegsrc.v9a.tar.gz" - - version('9a', '3353992aecaee1805ef4109aadd433e7') - - def install(self, spec, prefix): - configure("--prefix=%s" % prefix) - - make() - make("install") diff --git a/var/spack/packages/launchmon/package.py b/var/spack/packages/launchmon/package.py deleted file mode 100644 index 6fbe6a68d0..0000000000 --- a/var/spack/packages/launchmon/package.py +++ /dev/null @@ -1,47 +0,0 @@ -############################################################################## -# Copyright (c) 2014, Lawrence Livermore National Security, LLC. -# Produced at the Lawrence Livermore National Laboratory. -# -# This file is part of Spack. -# Written by Matthew LeGendre, legendre1@llnl.gov, All rights reserved. -# LLNL-CODE-647188 -# -# For details, see https://scalability-llnl.github.io/spack -# Please also see the LICENSE file for our notice and the LGPL. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License (as published by -# the Free Software Foundation) version 2.1 dated February 1999. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and -# conditions of the GNU General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -############################################################################## -from spack import * - -class Launchmon(Package): - """Software infrastructure that enables HPC run-time tools to - co-locate tool daemons with a parallel job.""" - homepage = "http://sourceforge.net/projects/launchmon" - url = "http://downloads.sourceforge.net/project/launchmon/launchmon/1.0.1%20release/launchmon-1.0.1.tar.gz" - - version('1.0.1', '2f12465803409fd07f91174a4389eb2b') - version('1.0.1-2', git='https://github.com/scalability-llnl/launchmon.git', commit='ff7e22424b8f375318951eb1c9282fcbbfa8aadf') - - depends_on('autoconf') - depends_on('automake') - depends_on('libtool') - - def install(self, spec, prefix): - configure( - "--prefix=" + prefix, - "--with-bootfabric=cobo", - "--with-rm=slurm") - - make() - make("install") diff --git a/var/spack/packages/launchmon/patch.lmon_install_dir b/var/spack/packages/launchmon/patch.lmon_install_dir deleted file mode 100644 index 8a1d93fdc9..0000000000 --- a/var/spack/packages/launchmon/patch.lmon_install_dir +++ /dev/null @@ -1,147 +0,0 @@ -Index: launchmon/src/linux/lmon_api/Makefile.am -=================================================================== ---- launchmon/src/linux/lmon_api/Makefile.am (revision 481) -+++ launchmon/src/linux/lmon_api/Makefile.am (working copy) -@@ -80,13 +80,10 @@ - libmonfeapi_la_CFLAGS = $(AM_CFLAGS) - libmonfeapi_la_CXXFLAGS = $(AM_CXXFLAGS) - --libmonfeapi_la_LDFLAGS = -L$(top_srcdir)/@COMMLOC@ \ -- -L$(top_srcdir)/@GCRYPTLOC@ \ -- -L$(top_srcdir)/@GPGERRLOC@ \ -- $(AM_LDFLAGS) \ -- -version-info @LMON_CURRENT@:@LMON_REVISION@:@LMON_AGE@ -+libmonfeapi_la_LDFLAGS = $(AM_LDFLAGS) \ -+ -version-info @LMON_CURRENT@:@LMON_REVISION@:@LMON_AGE@ - --libmonfeapi_la_LIBADD = @LIBPTHREAD@ @LIBCOMM@ @LIBGCRYPT@ @LIBGPGERR@ @LIBRT@ -+libmonfeapi_la_LIBADD = @LIBPTHREAD@ $(top_builddir)/@COMMLOC@/@LIBCOMM@ $(top_builddir)/@GCRYPTLOC@/@LIBGCRYPT@ $(top_builddir)/@GPGERRLOC@/@LIBGPGERR@ @LIBRT@ - - libmonbeapi_la_SOURCES = lmon_be.cxx \ - lmon_daemon_internal.cxx \ -@@ -113,13 +110,10 @@ - libmonbeapi_la_CFLAGS = $(AM_CFLAGS) - libmonbeapi_la_CXXFLAGS = $(AM_CXXFLAGS) - --libmonbeapi_la_LDFLAGS = -L$(top_srcdir)/@COMMLOC@ \ -- -L$(top_srcdir)/@GCRYPTLOC@ \ -- -L$(top_srcdir)/@GPGERRLOC@ \ -- $(AM_LDFLAGS) \ -+libmonbeapi_la_LDFLAGS = $(AM_LDFLAGS) \ - -version-info @LMON_CURRENT@:@LMON_REVISION@:@LMON_AGE@ - --libmonbeapi_la_LIBADD = @LIBCOMM@ @LIBGCRYPT@ @LIBGPGERR@ -+libmonbeapi_la_LIBADD = $(top_builddir)/@COMMLOC@/@LIBCOMM@ $(top_builddir)/@GCRYPTLOC@/@LIBGCRYPT@ $(top_builddir)/@GPGERRLOC@/@LIBGPGERR@ - - - # -@@ -146,10 +140,8 @@ - - libmonmwapi_la_CXXFLAGS = $(AM_CXXFLAGS) - --libmonmwapi_la_LDFLAGS = -L$(top_srcdir)/@COMMLOC@ \ -- -L$(top_srcdir)/@GCRYPTLOC@ \ -- -L$(top_srcdir)/@GPGERRLOC@ \ -- $(AM_LDFLAGS) \ -+libmonmwapi_la_LDFLAGS = $(AM_LDFLAGS) \ - -version-info @LMON_CURRENT@:@LMON_REVISION@:@LMON_AGE@ - --libmonmwapi_la_LIBADD = @LIBCOMM@ @LIBGCRYPT@ @LIBGPGERR@ -+ -+libmonmwapi_la_LIBADD = $(top_builddir)/@COMMLOC@/@LIBCOMM@ $(top_builddir)/@GCRYPTLOC@/@LIBGCRYPT@ $(top_builddir)/@GPGERRLOC@/@LIBGPGERR@ -Index: tools/cobo/test/Makefile.am -=================================================================== ---- tools/cobo/test/Makefile.am (revision 481) -+++ tools/cobo/test/Makefile.am (working copy) -@@ -37,12 +37,12 @@ - - client_SOURCES = client.c - --client_LDFLAGS = -L$(top_srcdir)/@COMMLOC@ -+client_LDFLAGS = - --client_LDADD = @LIBCOMM@ -+client_LDADD = $(top_srcdir)/@COMMLOC@/@LIBCOMM@ - - server_rsh_SOURCES = server_rsh.c - --server_rsh_LDFLAGS = -L$(top_srcdir)/@COMMLOC@ -+server_rsh_LDFLAGS = - --server_rsh_LDADD = @LIBCOMM@ -+server_rsh_LDADD = $(top_srcdir)/@COMMLOC@/@LIBCOMM@ -Index: tools/pmgr_collective/test/Makefile.am -=================================================================== ---- tools/pmgr_collective/test/Makefile.am (revision 481) -+++ tools/pmgr_collective/test/Makefile.am (working copy) -@@ -31,18 +31,18 @@ - ## Jun 10 2008 DHA: Copied from the old Makefile. - ## - --INCLUDES = -I$(top_srcdir)/@COMMLOC@ -+INCLUDES = - - noinst_PROGRAMS = client mpirun_rsh - - client_SOURCES = client.c - --client_LDFLAGS = -L$(top_srcdir)/@COMMLOC@ -+client_LDFLAGS = - --client_LDADD = @LIBCOMM@ -+client_LDADD = @COMMLOC@/@LIBCOMM@ - - mpirun_rsh_SOURCES = mpirun_rsh.c - --mpirun_rsh_LDFLAGS = -L$(top_srcdir)/@COMMLOC@ -+mpirun_rsh_LDFLAGS = - --mpirun_rsh_LDADD = @LIBCOMM@ -+mpirun_rsh_LDADD = @COMMLOC@/@LIBCOMM@ -Index: config/x_ac_bootfabric.m4 -=================================================================== ---- config/x_ac_bootfabric.m4 (revision 481) -+++ config/x_ac_bootfabric.m4 (working copy) -@@ -63,7 +63,7 @@ - #AC_DEFINE(TOOL_SS_ENV, "LMON_SHARED_SECRET", [Define TOOL_SS_ENV]) - #AC_DEFINE(TOOL_SCH_ENV, "LMON_SEC_CHK", [Define TOOL_SCH_ENV]) - #AC_SUBST(COMMLOC, tools/pmgr_collective/src) -- #AC_SUBST(LIBCOMM, -lpmgr_collective) -+ #AC_SUBST(LIBCOMM, libcobo.la) - #else - commfab_found="no" - AC_MSG_ERROR([--with-bootfabric=pmgr is given, but pmgr_collective has been deprecated]) -@@ -87,7 +87,7 @@ - AC_DEFINE(TOOL_SS_ENV, "LMON_SHARED_SECRET", [Define TOOL_SS_ENV]) - AC_DEFINE(TOOL_SCH_ENV, "LMON_SEC_CHK", [Define TOOL_SCH_ENV]) - AC_SUBST(COMMLOC, tools/cobo/src) -- AC_SUBST(LIBCOMM, -lcobo) -+ AC_SUBST(LIBCOMM, libcobo.la) - - if test "x$with_cobo_port" != "xcheck" -a "x$with_cobo_port" != "xyes"; then - AC_DEFINE(COBO_BEGIN_PORT, $with_cobo_port, [Define a beginning port for COBO_BASED]) -@@ -117,7 +117,7 @@ - AC_DEFINE(TOOL_SS_ENV, "LMON_SHARED_SECRET", [Define TOOL_SS_ENV]) - AC_DEFINE(TOOL_SCH_ENV, "LMON_SEC_CHK", [Define TOOL_SCH_ENV]) - AC_SUBST(COMMLOC, tools/cobo/src) -- AC_SUBST(LIBCOMM, -lcobo) -+ AC_SUBST(LIBCOMM, libcobo.la) - - if test "x$with_cobo_port" != "xcheck" -a "x$with_cobo_port" != "xyes"; then - AC_DEFINE(COBO_BEGIN_PORT, $with_cobo_port, [Define a beginning port for COBO_BASED]) -Index: config/x_ac_gcrpyt.m4 -=================================================================== ---- config/x_ac_gcrypt.m4 2011-10-22 00:50:38.000000000 -0700 -+++ config/x_ac_gcrypt.patched.m4 2014-03-14 11:33:59.189220000 -0700 -@@ -55,8 +55,8 @@ - AC_CONFIG_SUBDIRS([tools/libgpg-error]) - AC_SUBST(GPGERRLOC, [tools/libgpg-error/src]) - AC_SUBST(GCRYPTLOC, [tools/libgcrypt/src]) -- AC_SUBST(LIBGCRYPT, [-lgcrypt]) -- AC_SUBST(LIBGPGERR, [-lgpg-error]) -+ AC_SUBST(LIBGCRYPT, [libgcrypt.la]) -+ AC_SUBST(LIBGPGERR, [libgpg-error.la]) - gcrypt_configured="yes" - else - AC_MSG_ERROR([tools/libgpg-error or tools/libgcrypt not found]) - diff --git a/var/spack/packages/lcms/package.py b/var/spack/packages/lcms/package.py deleted file mode 100644 index a53c2f997a..0000000000 --- a/var/spack/packages/lcms/package.py +++ /dev/null @@ -1,19 +0,0 @@ -from spack import * - -class Lcms(Package): - """Little cms is a color management library. Implements fast - transforms between ICC profiles. It is focused on speed, and is - portable across several platforms (MIT license).""" - homepage = "http://www.littlecms.com" - url = "http://downloads.sourceforge.net/project/lcms/lcms/2.6/lcms2-2.6.tar.gz" - - version('2.6', 'f4c08d38ceade4a664ebff7228910a33') - - depends_on("jpeg") - depends_on("libtiff") - depends_on("zlib") - - def install(self, spec, prefix): - configure("--prefix=%s" % prefix) - make() - make("install") diff --git a/var/spack/packages/leveldb/package.py b/var/spack/packages/leveldb/package.py deleted file mode 100644 index da68a9cbcb..0000000000 --- a/var/spack/packages/leveldb/package.py +++ /dev/null @@ -1,29 +0,0 @@ -import os -import glob -from spack import * - -class Leveldb(Package): - """LevelDB is a fast key-value storage library written at Google - that provides an ordered mapping from string keys to string values.""" - - homepage = "https://github.com/google/leveldb" - url = "https://github.com/google/leveldb/archive/v1.18.tar.gz" - - version('1.18', '73770de34a2a5ab34498d2e05b2b7fa0') - - depends_on("snappy") - - def install(self, spec, prefix): - make() - - mkdirp(prefix.include) - mkdirp(prefix.lib) - - cp = which('cp') - - # cp --preserve=links libleveldb.* prefix/lib - args = glob.glob('libleveldb.*') - args.append(prefix + '/lib') - cp('--preserve=links', *args) - - cp('-r', 'include/leveldb', prefix + '/include') diff --git a/var/spack/packages/libNBC/package.py b/var/spack/packages/libNBC/package.py deleted file mode 100644 index 6d08f3219c..0000000000 --- a/var/spack/packages/libNBC/package.py +++ /dev/null @@ -1,43 +0,0 @@ -############################################################################## -# Copyright (c) 2013, Lawrence Livermore National Security, LLC. -# Produced at the Lawrence Livermore National Laboratory. -# -# This file is part of Spack. -# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. -# LLNL-CODE-647188 -# -# For details, see https://scalability-llnl.github.io/spack -# Please also see the LICENSE file for our notice and the LGPL. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License (as published by -# the Free Software Foundation) version 2.1 dated February 1999. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and -# conditions of the GNU General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -############################################################################## -from spack import * - -class Libnbc(Package): - """LibNBC is a prototypic implementation of a nonblocking - interface for MPI collective operations. Based on ANSI C and - MPI-1, it supports all MPI-1 collective operations in a - nonblocking manner. LibNBC is distributed under the BSD license. - """ - homepage = "http://unixer.de/research/nbcoll/libnbc/" - url = "http://unixer.de/research/nbcoll/libnbc/libNBC-1.1.1.tar.gz" - - version('1.1.1', 'ece5c94992591a9fa934a90e5dbe50ce') - - depends_on("mpi") - - def install(self, spec, prefix): - configure("--prefix=%s" % prefix) - make() - make("install") diff --git a/var/spack/packages/libarchive/package.py b/var/spack/packages/libarchive/package.py deleted file mode 100644 index cbd4b89cd0..0000000000 --- a/var/spack/packages/libarchive/package.py +++ /dev/null @@ -1,16 +0,0 @@ -from spack import * - -class Libarchive(Package): - """libarchive: C library and command-line tools for reading and - writing tar, cpio, zip, ISO, and other archive formats.""" - homepage = "http://www.libarchive.org" - url = "http://www.libarchive.org/downloads/libarchive-3.1.2.tar.gz" - - version('3.1.2', 'efad5a503f66329bb9d2f4308b5de98a') - version('3.1.1', '1f3d883daf7161a0065e42a15bbf168f') - version('3.1.0', '095a287bb1fd687ab50c85955692bf3a') - - def install(self, spec, prefix): - configure("--prefix=%s" % prefix) - make() - make("install") diff --git a/var/spack/packages/libcircle/package.py b/var/spack/packages/libcircle/package.py deleted file mode 100644 index 3f7c996fb0..0000000000 --- a/var/spack/packages/libcircle/package.py +++ /dev/null @@ -1,18 +0,0 @@ -import os -from spack import * - -class Libcircle(Package): - """libcircle provides an efficient distributed queue on a cluster, - using self-stabilizing work stealing.""" - - homepage = "https://github.com/hpc/libcircle" - - version('0.2.1-rc.1', '2b1369a5736457239f908abf88143ec2', - url='https://github.com/hpc/libcircle/releases/download/0.2.1-rc.1/libcircle-0.2.1-rc.1.tar.gz') - - depends_on('mpi') - - def install(self, spec, prefix): - configure("--prefix=" + prefix) - make() - make("install") diff --git a/var/spack/packages/libdrm/package.py b/var/spack/packages/libdrm/package.py deleted file mode 100644 index 00736b7811..0000000000 --- a/var/spack/packages/libdrm/package.py +++ /dev/null @@ -1,18 +0,0 @@ -from spack import * - -class Libdrm(Package): - """A userspace library for accessing the DRM, direct - rendering manager, on Linux, BSD and other operating - systems that support the ioctl interface.""" - - homepage = "http://dri.freedesktop.org/libdrm/" # no real website... - url = "http://dri.freedesktop.org/libdrm/libdrm-2.4.59.tar.gz" - - version('2.4.59', '105ac7af1afcd742d402ca7b4eb168b6') - version('2.4.33', '86e4e3debe7087d5404461e0032231c8') - - def install(self, spec, prefix): - configure("--prefix=%s" % prefix) - - make() - make("install") diff --git a/var/spack/packages/libdwarf/package.py b/var/spack/packages/libdwarf/package.py deleted file mode 100644 index 099a974e93..0000000000 --- a/var/spack/packages/libdwarf/package.py +++ /dev/null @@ -1,81 +0,0 @@ -############################################################################## -# Copyright (c) 2013, Lawrence Livermore National Security, LLC. -# Produced at the Lawrence Livermore National Laboratory. -# -# This file is part of Spack. -# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. -# LLNL-CODE-647188 -# -# For details, see https://scalability-llnl.github.io/spack -# Please also see the LICENSE file for our notice and the LGPL. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License (as published by -# the Free Software Foundation) version 2.1 dated February 1999. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and -# conditions of the GNU General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -############################################################################## -from spack import * -import os - -# Only build certain parts of dwarf because the other ones break. -dwarf_dirs = ['libdwarf', 'dwarfdump2'] - -class Libdwarf(Package): - """The DWARF Debugging Information Format is of interest to - programmers working on compilers and debuggers (and any one - interested in reading or writing DWARF information). It was - developed by a committee (known as the PLSIG at the time) - starting around 1991. Starting around 1991 SGI developed the - libdwarf and dwarfdump tools for internal use and as part of - SGI IRIX developer tools. Since that time dwarfdump and - libdwarf have been shipped (as an executable and archive - respectively, not source) with every release of the SGI - MIPS/IRIX C compiler.""" - - homepage = "http://www.prevanders.net/dwarf.html" - url = "http://www.prevanders.net/libdwarf-20130729.tar.gz" - list_url = homepage - - version('20130729', '4cc5e48693f7b93b7aa0261e63c0e21d') - version('20130207', '64b42692e947d5180e162e46c689dfbf') - version('20130126', 'ded74a5e90edb5a12aac3c29d260c5db') - - depends_on("libelf") - - parallel = False - - - def install(self, spec, prefix): - # dwarf build does not set arguments for ar properly - make.add_default_arg('ARFLAGS=rcs') - - # Dwarf doesn't provide an install, so we have to do it. - mkdirp(prefix.bin, prefix.include, prefix.lib, prefix.man1) - - with working_dir('libdwarf'): - configure("--prefix=" + prefix, "--enable-shared") - make() - - install('libdwarf.a', prefix.lib) - install('libdwarf.so', prefix.lib) - install('libdwarf.h', prefix.include) - install('dwarf.h', prefix.include) - - with working_dir('dwarfdump2'): - configure("--prefix=" + prefix) - - # This makefile has strings of copy commands that - # cause a race in parallel - make(parallel=False) - - install('dwarfdump', prefix.bin) - install('dwarfdump.conf', prefix.lib) - install('dwarfdump.1', prefix.man1) diff --git a/var/spack/packages/libelf/package.py b/var/spack/packages/libelf/package.py deleted file mode 100644 index 9338b8f393..0000000000 --- a/var/spack/packages/libelf/package.py +++ /dev/null @@ -1,49 +0,0 @@ -############################################################################## -# Copyright (c) 2013, Lawrence Livermore National Security, LLC. -# Produced at the Lawrence Livermore National Laboratory. -# -# This file is part of Spack. -# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. -# LLNL-CODE-647188 -# -# For details, see https://scalability-llnl.github.io/spack -# Please also see the LICENSE file for our notice and the LGPL. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License (as published by -# the Free Software Foundation) version 2.1 dated February 1999. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and -# conditions of the GNU General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -############################################################################## -from spack import * - -class Libelf(Package): - """libelf lets you read, modify or create ELF object files in an - architecture-independent way. The library takes care of size - and endian issues, e.g. you can process a file for SPARC - processors on an Intel-based system.""" - - homepage = "http://www.mr511.de/software/english.html" - url = "http://www.mr511.de/software/libelf-0.8.13.tar.gz" - - version('0.8.13', '4136d7b4c04df68b686570afa26988ac') - version('0.8.12', 'e21f8273d9f5f6d43a59878dc274fec7') - - provides('elf') - - def install(self, spec, prefix): - configure("--prefix=" + prefix, - "--enable-shared", - "--disable-dependency-tracking", - "--disable-debug") - make() - - # The mkdir commands in libelf's install can fail in parallel - make("install", parallel=False) diff --git a/var/spack/packages/libevent/package.py b/var/spack/packages/libevent/package.py deleted file mode 100644 index 11b1083d67..0000000000 --- a/var/spack/packages/libevent/package.py +++ /dev/null @@ -1,30 +0,0 @@ -from spack import * - -class Libevent(Package): - """The libevent API provides a mechanism to execute a callback function - when a specific event occurs on a file descriptor or after a timeout has been - reached. Furthermore, libevent also support callbacks due to signals or regular - timeouts. - """ - - homepage = "http://libevent.org" - url = "https://github.com/downloads/libevent/libevent/libevent-2.0.21-stable.tar.gz" - list_url = "http://libevent.org/old-releases.html" - - version('2.0.21', 'b2405cc9ebf264aa47ff615d9de527a2') - version('2.0.20', '94270cdee32c0cd0aa9f4ee6ede27e8e') - version('2.0.19', '91111579769f46055b0a438f5cc59572') - version('2.0.18', 'aa1ce9bc0dee7b8084f6855765f2c86a') - version('2.0.17', 'dad64aaaaff16b5fbec25160c06fee9a') - version('2.0.16', '899efcffccdb3d5111419df76e7dc8df') - version('2.0.15', '2643abe7ba242df15c08b2cc14ec8759') - version('2.0.14', 'cac0f379da35d3b98f83ac16fcfe1df4') - version('2.0.13', 'af786b4b3f790c9d3279792edf7867fc') - version('2.0.12', '42986228baf95e325778ed328a93e070') - - - def install(self, spec, prefix): - configure("--prefix=%s" % prefix) - - make() - make("install") diff --git a/var/spack/packages/libffi/package.py b/var/spack/packages/libffi/package.py deleted file mode 100644 index acec031717..0000000000 --- a/var/spack/packages/libffi/package.py +++ /dev/null @@ -1,17 +0,0 @@ -from spack import * - -class Libffi(Package): - """The libffi library provides a portable, high level programming - interface to various calling conventions. This allows a programmer - to call any function specified by a call interface description at - run time.""" - homepage = "https://sourceware.org/libffi/" - - version('3.2.1','83b89587607e3eb65c70d361f13bab43',url = "ftp://sourceware.org/pub/libffi/libffi-3.2.1.tar.gz") - #version('3.1', 'f5898b29bbfd70502831a212d9249d10',url = "ftp://sourceware.org/pub/libffi/libffi-3.1.tar.gz") # Has a bug $(lib64) instead of ${lib64} in libffi.pc - - def install(self, spec, prefix): - configure("--prefix=%s" % prefix) - make() - make("install") - diff --git a/var/spack/packages/libgcrypt/package.py b/var/spack/packages/libgcrypt/package.py deleted file mode 100644 index 1d0a57f317..0000000000 --- a/var/spack/packages/libgcrypt/package.py +++ /dev/null @@ -1,19 +0,0 @@ -from spack import * - -class Libgcrypt(Package): - """Libgcrypt is a general purpose cryptographic library based on - the code from GnuPG. It provides functions for all cryptographic - building blocks: symmetric ciphers, hash algorithms, MACs, public - key algorithms, large integer functions, random numbers and a lot - of supporting functions. """ - homepage = "http://www.gnu.org/software/libgcrypt/" - url = "ftp://ftp.gnupg.org/gcrypt/libgcrypt/libgcrypt-1.6.2.tar.bz2" - - version('1.6.2', 'b54395a93cb1e57619943c082da09d5f') - - depends_on("libgpg-error") - - def install(self, spec, prefix): - configure("--prefix=%s" % prefix) - make() - make("install") diff --git a/var/spack/packages/libgpg-error/package.py b/var/spack/packages/libgpg-error/package.py deleted file mode 100644 index 6c1d1a10a7..0000000000 --- a/var/spack/packages/libgpg-error/package.py +++ /dev/null @@ -1,17 +0,0 @@ -from spack import * - -class LibgpgError(Package): - """Libgpg-error is a small library that defines common error - values for all GnuPG components. Among these are GPG, GPGSM, - GPGME, GPG-Agent, libgcrypt, Libksba, DirMngr, Pinentry, - SmartCard Daemon and possibly more in the future. """ - - homepage = "https://www.gnupg.org/related_software/libgpg-error" - url = "ftp://ftp.gnupg.org/gcrypt/libgpg-error/libgpg-error-1.18.tar.bz2" - - version('1.18', '12312802d2065774b787cbfc22cc04e9') - - def install(self, spec, prefix): - configure("--prefix=%s" % prefix) - make() - make("install") diff --git a/var/spack/packages/libjpeg-turbo/package.py b/var/spack/packages/libjpeg-turbo/package.py deleted file mode 100644 index 07ee183947..0000000000 --- a/var/spack/packages/libjpeg-turbo/package.py +++ /dev/null @@ -1,20 +0,0 @@ -from spack import * - -class LibjpegTurbo(Package): - """libjpeg-turbo is a fork of the original IJG libjpeg which uses - SIMD to accelerate baseline JPEG compression and - decompression. libjpeg is a library that implements JPEG image - encoding, decoding and transcoding.""" - homepage = "http://libjpeg-turbo.virtualgl.org" - url = "http://downloads.sourceforge.net/libjpeg-turbo/libjpeg-turbo-1.3.1.tar.gz" - - version('1.3.1', '2c3a68129dac443a72815ff5bb374b05') - - # Can use either of these. - depends_on("yasm") - depends_on("nasm") - - def install(self, spec, prefix): - configure("--prefix=%s" % prefix) - make() - make("install") diff --git a/var/spack/packages/libjson-c/package.py b/var/spack/packages/libjson-c/package.py deleted file mode 100644 index c0801cce9c..0000000000 --- a/var/spack/packages/libjson-c/package.py +++ /dev/null @@ -1,14 +0,0 @@ -from spack import * - -class LibjsonC(Package): - """ A JSON implementation in C """ - homepage = "https://github.com/json-c/json-c/wiki" - url = "https://s3.amazonaws.com/json-c_releases/releases/json-c-0.11.tar.gz" - - version('0.11', 'aa02367d2f7a830bf1e3376f77881e98') - - def install(self, spec, prefix): - configure('--prefix=%s' % prefix) - - make() - make("install") diff --git a/var/spack/packages/libmng/package.py b/var/spack/packages/libmng/package.py deleted file mode 100644 index e5336ea2c2..0000000000 --- a/var/spack/packages/libmng/package.py +++ /dev/null @@ -1,23 +0,0 @@ -from spack import * - -class Libmng(Package): - """libmng -THE reference library for reading, displaying, writing - and examining Multiple-Image Network Graphics. MNG is the animation - extension to the popular PNG image-format.""" - homepage = "http://sourceforge.net/projects/libmng/" - url = "http://downloads.sourceforge.net/project/libmng/libmng-devel/2.0.2/libmng-2.0.2.tar.gz" - - version('2.0.2', '1ffefaed4aac98475ee6267422cbca55') - - depends_on("jpeg") - depends_on("zlib") - depends_on("lcms") - - def patch(self): - # jpeg requires stdio to beincluded before its headrs. - filter_file(r'^(\#include \)', '#include\n\\1', 'libmng_types.h') - - def install(self, spec, prefix): - configure("--prefix=%s" % prefix) - make() - make("install") diff --git a/var/spack/packages/libmonitor/package.py b/var/spack/packages/libmonitor/package.py deleted file mode 100644 index 3b95b86ddf..0000000000 --- a/var/spack/packages/libmonitor/package.py +++ /dev/null @@ -1,36 +0,0 @@ -############################################################################## -# Copyright (c) 2013, Lawrence Livermore National Security, LLC. -# Produced at the Lawrence Livermore National Laboratory. -# -# This file is part of Spack. -# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. -# LLNL-CODE-647188 -# -# For details, see https://scalability-llnl.github.io/spack -# Please also see the LICENSE file for our notice and the LGPL. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License (as published by -# the Free Software Foundation) version 2.1 dated February 1999. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and -# conditions of the GNU General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -############################################################################## -from spack import * - -class Libmonitor(Package): - """Libmonitor is a library for process and thread control.""" - homepage = "http://hpctoolkit.org" - - version('20130218', svn='http://libmonitor.googlecode.com/svn/trunk/', revision=146) - - def install(self, spec, prefix): - configure("--prefix=" + prefix) - make() - make("install") diff --git a/var/spack/packages/libpciaccess/package.py b/var/spack/packages/libpciaccess/package.py deleted file mode 100644 index 6022fc34a3..0000000000 --- a/var/spack/packages/libpciaccess/package.py +++ /dev/null @@ -1,21 +0,0 @@ -from spack import * - -class Libpciaccess(Package): - """Generic PCI access library.""" - - homepage = "http://cgit.freedesktop.org/xorg/lib/libpciaccess/" - url = "http://cgit.freedesktop.org/xorg/lib/libpciaccess/" - - version('0.13.4', git='http://anongit.freedesktop.org/git/xorg/lib/libpciaccess.git', - tag='libpciaccess-0.13.4') - - depends_on('autoconf') - depends_on('libtool') - - def install(self, spec, prefix): - from subprocess import call - call(["./autogen.sh"]) - configure("--prefix=%s" % prefix) - - make() - make("install") diff --git a/var/spack/packages/libpng/package.py b/var/spack/packages/libpng/package.py deleted file mode 100644 index e02b08663e..0000000000 --- a/var/spack/packages/libpng/package.py +++ /dev/null @@ -1,15 +0,0 @@ -from spack import * - -class Libpng(Package): - """libpng graphics file format""" - homepage = "http://www.libpng.org/pub/png/libpng.html" - url = "http://download.sourceforge.net/libpng/libpng-1.6.16.tar.gz" - - version('1.6.16', '1a4ad377919ab15b54f6cb6a3ae2622d') - version('1.6.15', '829a256f3de9307731d4f52dc071916d') - version('1.6.14', '2101b3de1d5f348925990f9aa8405660') - - def install(self, spec, prefix): - configure("--prefix=%s" % prefix) - make() - make("install") diff --git a/var/spack/packages/libsodium/package.py b/var/spack/packages/libsodium/package.py deleted file mode 100644 index 1c8a16d998..0000000000 --- a/var/spack/packages/libsodium/package.py +++ /dev/null @@ -1,19 +0,0 @@ -from spack import * - -class Libsodium(Package): - """Sodium is a modern, easy-to-use software library for encryption, - decryption, signatures, password hashing and more.""" - homepage = "https://download.libsodium.org/doc/" - url = "https://download.libsodium.org/libsodium/releases/libsodium-1.0.3.tar.gz" - - version('1.0.3', 'b3bcc98e34d3250f55ae196822307fab') - version('1.0.2', 'dc40eb23e293448c6fc908757738003f') - version('1.0.1', '9a221b49fba7281ceaaf5e278d0f4430') - version('1.0.0', '3093dabe4e038d09f0d150cef064b2f7') - version('0.7.1', 'c224fe3923d1dcfe418c65c8a7246316') - - def install(self, spec, prefix): - configure("--prefix=%s" % prefix) - - make() - make("install") diff --git a/var/spack/packages/libtiff/package.py b/var/spack/packages/libtiff/package.py deleted file mode 100644 index 63c6704cb8..0000000000 --- a/var/spack/packages/libtiff/package.py +++ /dev/null @@ -1,18 +0,0 @@ -from spack import * - -class Libtiff(Package): - """libtiff graphics format library""" - homepage = "http://www.remotesensing.org/libtiff/" - url = "http://download.osgeo.org/libtiff/tiff-4.0.3.tar.gz" - - version('4.0.3', '051c1068e6a0627f461948c365290410') - - depends_on('jpeg') - depends_on('zlib') - depends_on('xz') - - def install(self, spec, prefix): - configure("--prefix=%s" % prefix) - - make() - make("install") diff --git a/var/spack/packages/libtool/package.py b/var/spack/packages/libtool/package.py deleted file mode 100644 index a07daf9781..0000000000 --- a/var/spack/packages/libtool/package.py +++ /dev/null @@ -1,14 +0,0 @@ -from spack import * - -class Libtool(Package): - """libtool -- library building part of autotools""" - homepage = "https://www.gnu.org/software/libtool/" - url = "http://ftpmirror.gnu.org/libtool/libtool-2.4.2.tar.gz" - - version('2.4.2' , 'd2f3b7d4627e69e13514a40e72a24d50') - - def install(self, spec, prefix): - configure("--prefix=%s" % prefix) - - make() - make("install") diff --git a/var/spack/packages/libunwind/package.py b/var/spack/packages/libunwind/package.py deleted file mode 100644 index 239fcbcfd5..0000000000 --- a/var/spack/packages/libunwind/package.py +++ /dev/null @@ -1,38 +0,0 @@ -############################################################################## -# Copyright (c) 2013, Lawrence Livermore National Security, LLC. -# Produced at the Lawrence Livermore National Laboratory. -# -# This file is part of Spack. -# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. -# LLNL-CODE-647188 -# -# For details, see https://scalability-llnl.github.io/spack -# Please also see the LICENSE file for our notice and the LGPL. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License (as published by -# the Free Software Foundation) version 2.1 dated February 1999. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and -# conditions of the GNU General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -############################################################################## -from spack import * - -class Libunwind(Package): - """A portable and efficient C programming interface (API) to determine - the call-chain of a program.""" - homepage = "http://www.nongnu.org/libunwind/" - url = "http://download.savannah.gnu.org/releases/libunwind/libunwind-1.1.tar.gz" - - version('1.1', 'fb4ea2f6fbbe45bf032cd36e586883ce') - - def install(self, spec, prefix): - configure("--prefix=" + prefix) - make() - make("install") diff --git a/var/spack/packages/libuuid/package.py b/var/spack/packages/libuuid/package.py deleted file mode 100644 index 373c5bfcac..0000000000 --- a/var/spack/packages/libuuid/package.py +++ /dev/null @@ -1,16 +0,0 @@ -from spack import * - -class Libuuid(Package): - """Portable uuid C library""" - # FIXME: add a proper url for your package's homepage here. - homepage = "http://sourceforge.net/projects/libuuid/" - url = "http://downloads.sourceforge.net/project/libuuid/libuuid-1.0.3.tar.gz?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Flibuuid%2F&ts=1433881396&use_mirror=iweb" - - version('1.0.3', 'd44d866d06286c08ba0846aba1086d68') - - def install(self, spec, prefix): - configure("--prefix=%s" % prefix) - - # FIXME: Add logic to build and install here - make() - make("install") diff --git a/var/spack/packages/libxcb/package.py b/var/spack/packages/libxcb/package.py deleted file mode 100644 index 16a5525c0d..0000000000 --- a/var/spack/packages/libxcb/package.py +++ /dev/null @@ -1,21 +0,0 @@ -from spack import * - -class Libxcb(Package): - """The X protocol C-language Binding (XCB) is a replacement - for Xlib featuring a small footprint, latency hiding, direct - access to the protocol, improved threading support, and - extensibility.""" - - homepage = "http://xcb.freedesktop.org/" - url = "http://xcb.freedesktop.org/dist/libxcb-1.11.tar.gz" - - version('1.11', '1698dd837d7e6e94d029dbe8b3a82deb') - version('1.11.1', '118623c15a96b08622603a71d8789bf3') - depends_on("python") - depends_on("xcb-proto") - - def install(self, spec, prefix): - configure("--prefix=%s" % prefix) - - make() - make("install") diff --git a/var/spack/packages/libxml2/package.py b/var/spack/packages/libxml2/package.py deleted file mode 100644 index 3a0af6b368..0000000000 --- a/var/spack/packages/libxml2/package.py +++ /dev/null @@ -1,20 +0,0 @@ -from spack import * - -class Libxml2(Package): - """Libxml2 is the XML C parser and toolkit developed for the Gnome - project (but usable outside of the Gnome platform), it is free - software available under the MIT License.""" - homepage = "http://xmlsoft.org" - url = "http://xmlsoft.org/sources/libxml2-2.9.2.tar.gz" - - version('2.9.2', '9e6a9aca9d155737868b3dc5fd82f788') - - extends('python') - depends_on('zlib') - depends_on('xz') - - def install(self, spec, prefix): - configure("--prefix=%s" % prefix) - - make() - make("install") diff --git a/var/spack/packages/libxshmfence/package.py b/var/spack/packages/libxshmfence/package.py deleted file mode 100644 index 3aa2448b46..0000000000 --- a/var/spack/packages/libxshmfence/package.py +++ /dev/null @@ -1,16 +0,0 @@ -from spack import * - -class Libxshmfence(Package): - """This is a tiny library that exposes a event API on top of Linux - futexes.""" - - homepage = "http://keithp.com/blogs/dri3_extension/" # not really... - url = "http://xorg.freedesktop.org/archive/individual/lib/libxshmfence-1.2.tar.gz" - - version('1.2', 'f0b30c0fc568b22ec524859ee28556f1') - - def install(self, spec, prefix): - configure("--prefix=%s" % prefix) - - make() - make("install") diff --git a/var/spack/packages/libxslt/package.py b/var/spack/packages/libxslt/package.py deleted file mode 100644 index f97332d020..0000000000 --- a/var/spack/packages/libxslt/package.py +++ /dev/null @@ -1,24 +0,0 @@ -from spack import * - -class Libxslt(Package): - """Libxslt is the XSLT C library developed for the GNOME - project. XSLT itself is a an XML language to define - transformation for XML. Libxslt is based on libxml2 the XML C - library developed for the GNOME project. It also implements - most of the EXSLT set of processor-portable extensions - functions and some of Saxon's evaluate and expressions - extensions.""" - homepage = "http://www.xmlsoft.org/XSLT/index.html" - url = "http://xmlsoft.org/sources/libxslt-1.1.28.tar.gz" - - version('1.1.28', '9667bf6f9310b957254fdcf6596600b7') - - depends_on("libxml2") - depends_on("xz") - depends_on("zlib") - depends_on("libgcrypt") - - def install(self, spec, prefix): - configure("--prefix=%s" % prefix) - make() - make("install") diff --git a/var/spack/packages/llvm-lld/package.py b/var/spack/packages/llvm-lld/package.py deleted file mode 100644 index f229211396..0000000000 --- a/var/spack/packages/llvm-lld/package.py +++ /dev/null @@ -1,46 +0,0 @@ -############################################################################## -# Copyright (c) 2013, Lawrence Livermore National Security, LLC. -# Produced at the Lawrence Livermore National Laboratory. -# -# This file is part of Spack. -# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. -# LLNL-CODE-647188 -# -# For details, see https://scalability-llnl.github.io/spack -# Please also see the LICENSE file for our notice and the LGPL. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License (as published by -# the Free Software Foundation) version 2.1 dated February 1999. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and -# conditions of the GNU General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -############################################################################## -from spack import * - -class LlvmLld(Package): - """lld - The LLVM Linker - lld is a new set of modular code for creating linker tools.""" - homepage = "http://lld.llvm.org" - url = "http://llvm.org/releases/3.4/lld-3.4.src.tar.gz" - - depends_on('llvm') - - version('3.4', '3b6a17e58c8416c869c14dd37682f78e') - - def install(self, spec, prefix): - env['CXXFLAGS'] = self.compier.cxx11_flag - - with working_dir('spack-build', create=True): - cmake('..', - '-DLLD_PATH_TO_LLVM_BUILD=%s' % spec['llvm'].prefix, - '-DLLVM_MAIN_SRC_DIR=%s' % spec['llvm'].prefix, - *std_cmake_args) - make() - make("install") diff --git a/var/spack/packages/llvm/package.py b/var/spack/packages/llvm/package.py deleted file mode 100644 index a6759c3033..0000000000 --- a/var/spack/packages/llvm/package.py +++ /dev/null @@ -1,53 +0,0 @@ -############################################################################## -# Copyright (c) 2013, Lawrence Livermore National Security, LLC. -# Produced at the Lawrence Livermore National Laboratory. -# -# This file is part of Spack. -# Written by David Beckingsale, david@llnl.gov, All rights reserved. -# LLNL-CODE-647188 -# -# For details, see https://scalability-llnl.github.io/spack -# Please also see the LICENSE file for our notice and the LGPL. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License (as published by -# the Free Software Foundation) version 2.1 dated February 1999. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and -# conditions of the GNU General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -############################################################################## -from spack import * - - -class Llvm(Package): - """The LLVM Project is a collection of modular and reusable compiler and - toolchain technologies. Despite its name, LLVM has little to do with - traditional virtual machines, though it does provide helpful libraries - that can be used to build them. The name "LLVM" itself is not an acronym; - it is the full name of the project. - """ - homepage = 'http://llvm.org/' - url = 'http://llvm.org/releases/3.7.0/llvm-3.7.0.src.tar.xz' - - version('3.7.0', 'b98b9495e5655a672d6cb83e1a180f8e', url='http://llvm.org/releases/3.7.0/llvm-3.7.0.src.tar.xz') - version('3.6.2', '0c1ee3597d75280dee603bae9cbf5cc2', url='http://llvm.org/releases/3.6.2/llvm-3.6.2.src.tar.xz') - version('3.5.1', '2d3d8004f38852aa679e5945b8ce0b14', url='http://llvm.org/releases/3.5.1/llvm-3.5.1.src.tar.xz') - - depends_on('python@2.7:') - - def install(self, spec, prefix): - env['CXXFLAGS'] = self.compiler.cxx11_flag - - with working_dir('spack-build', create=True): - cmake('..', - '-DLLVM_REQUIRES_RTTI=1', - '-DPYTHON_EXECUTABLE=%s/bin/python' % spec['python'].prefix, - *std_cmake_args) - make() - make("install") diff --git a/var/spack/packages/lmdb/package.py b/var/spack/packages/lmdb/package.py deleted file mode 100644 index 875b8100c5..0000000000 --- a/var/spack/packages/lmdb/package.py +++ /dev/null @@ -1,39 +0,0 @@ -import os -from spack import * - -class Lmdb(Package): - """Read-only mirror of official repo on openldap.org. Issues and - pull requests here are ignored. Use OpenLDAP ITS for issues. - http://www.openldap.org/software/repo.html""" - - - homepage = "http://www.openldap.org/software/repo.html" - url = "https://github.com/LMDB/lmdb/archive/LMDB_0.9.16.tar.gz" - - version('0.9.16', '0de89730b8f3f5711c2b3a4ba517b648') - - def install(self, spec, prefix): - os.chdir('libraries/liblmdb') - - make() - - mkdirp(prefix.bin) - mkdirp(prefix + '/man/man1') - mkdirp(prefix.lib) - mkdirp(prefix.include) - - bins = ['mdb_stat', 'mdb_copy', 'mdb_dump', 'mdb_load'] - for f in bins: - install(f, prefix.bin) - - mans = ['mdb_stat.1', 'mdb_copy.1', 'mdb_dump.1', 'mdb_load.1'] - for f in mans: - install(f, prefix + '/man/man1') - - libs = ['liblmdb.a', 'liblmdb.so'] - for f in libs: - install(f, prefix.lib) - - includes = ['lmdb.h'] - for f in includes: - install(f, prefix.include) diff --git a/var/spack/packages/lua/package.py b/var/spack/packages/lua/package.py deleted file mode 100644 index 57c443cc2d..0000000000 --- a/var/spack/packages/lua/package.py +++ /dev/null @@ -1,26 +0,0 @@ -from spack import * -import os - -class Lua(Package): - """ The Lua programming language interpreter and library """ - homepage = "http://www.lua.org" - url = "http://www.lua.org/ftp/lua-5.1.5.tar.gz" - - version('5.3.1', '797adacada8d85761c079390ff1d9961') - version('5.3.0', 'a1b0a7e92d0c85bbff7a8d27bf29f8af') - version('5.2.4', '913fdb32207046b273fdb17aad70be13') - version('5.2.3', 'dc7f94ec6ff15c985d2d6ad0f1b35654') - version('5.2.2', 'efbb645e897eae37cad4344ce8b0a614') - version('5.2.1', 'ae08f641b45d737d12d30291a5e5f6e3') - version('5.2.0', 'f1ea831f397214bae8a265995ab1a93e') - version('5.1.5', '2e115fe26e435e33b0d5c022e4490567') - version('5.1.4', 'd0870f2de55d59c1c8419f36e8fac150') - version('5.1.3', 'a70a8dfaa150e047866dc01a46272599') - - depends_on('ncurses') - - def install(self, spec, prefix): - make('INSTALL_TOP=%s' % prefix, - 'MYLDFLAGS=-L%s/lib' % spec['ncurses'].prefix, - 'linux', - 'install') diff --git a/var/spack/packages/lwgrp/package.py b/var/spack/packages/lwgrp/package.py deleted file mode 100644 index 5963382b92..0000000000 --- a/var/spack/packages/lwgrp/package.py +++ /dev/null @@ -1,18 +0,0 @@ -import os -from spack import * - -class Lwgrp(Package): - """Thie light-weight group library provides process group - representations using O(log N) space and time.""" - - homepage = "https://github.com/hpc/lwgrp" - url = "https://github.com/hpc/lwgrp/releases/download/v1.0.2/lwgrp-1.0.2.tar.gz" - - version('1.0.2', 'ab7ba3bdd8534a651da5076f47f27d8a') - - depends_on('mpi') - - def install(self, spec, prefix): - configure("--prefix=" + prefix) - make() - make("install") diff --git a/var/spack/packages/lwm2/package.py b/var/spack/packages/lwm2/package.py deleted file mode 100644 index 31afff8816..0000000000 --- a/var/spack/packages/lwm2/package.py +++ /dev/null @@ -1,18 +0,0 @@ -from spack import * - -class Lwm2(Package): - """LWM2: Light Weight Measurement Module. This is a PMPI module - that can collect a number of time-sliced MPI and POSIX I/O - measurements from a program. - """ - homepage = "https://jay.grs.rwth-aachen.de/redmine/projects/lwm2" - - version('torus', hg='https://jay.grs.rwth-aachen.de/hg/lwm2', revision='torus') - - depends_on("papi") - depends_on("mpi") - - def install(self, spec, prefix): - configure("--prefix=%s" % prefix) - make() - make("install") diff --git a/var/spack/packages/matio/package.py b/var/spack/packages/matio/package.py deleted file mode 100644 index 12cfb80926..0000000000 --- a/var/spack/packages/matio/package.py +++ /dev/null @@ -1,15 +0,0 @@ -from spack import * - - -class Matio(Package): - """matio is an C library for reading and writing Matlab MAT files""" - homepage = "http://sourceforge.net/projects/matio/" - url = "http://downloads.sourceforge.net/project/matio/matio/1.5.2/matio-1.5.2.tar.gz" - - version('1.5.2', '85b007b99916c63791f28398f6a4c6f1') - - def install(self, spec, prefix): - configure('--prefix=%s' % prefix) - - make() - make("install") diff --git a/var/spack/packages/memaxes/package.py b/var/spack/packages/memaxes/package.py deleted file mode 100644 index 76d5d3f831..0000000000 --- a/var/spack/packages/memaxes/package.py +++ /dev/null @@ -1,19 +0,0 @@ -from spack import * - -class Memaxes(Package): - """MemAxes is a visualizer for sampled memory trace data.""" - - homepage = "https://github.com/scalability-llnl/MemAxes" - - version('0.5', '5874f3fda9fd2d313c0ff9684f915ab5', - url='https://github.com/scalability-llnl/MemAxes/archive/v0.5.tar.gz') - - depends_on("cmake@2.8.9:") - depends_on("qt@5:") - - def install(self, spec, prefix): - with working_dir('spack-build', create=True): - cmake('..', *std_cmake_args) - make() - make("install") - diff --git a/var/spack/packages/mesa/package.py b/var/spack/packages/mesa/package.py deleted file mode 100644 index 2a04a8fd51..0000000000 --- a/var/spack/packages/mesa/package.py +++ /dev/null @@ -1,34 +0,0 @@ -from spack import * - -class Mesa(Package): - """Mesa is an open-source implementation of the OpenGL - specification - a system for rendering interactive 3D graphics.""" - - homepage = "http://www.mesa3d.org" - url = "ftp://ftp.freedesktop.org/pub/mesa/older-versions/8.x/8.0.5/MesaLib-8.0.5.tar.gz" - # url = "ftp://ftp.freedesktop.org/pub/mesa/10.4.4/MesaLib-10.4.4.tar.gz" - - # version('10.4.4', '8d863a3c209bf5116b2babfccccc68ce') - version('8.0.5', 'cda5d101f43b8784fa60bdeaca4056f2') - - # mesa 7.x, 8.x, 9.x - depends_on("libdrm@2.4.33") - depends_on("llvm@3.0") - depends_on("libxml2") - - # patch("llvm-fixes.patch") # using newer llvm - - # mesa 10.x - # depends_on("py-mako") - # depends_on("flex") - # depends_on("bison") - # depends_on("dri2proto") - # depends_on("libxcb") - # depends_on("libxshmfence") - - - def install(self, spec, prefix): - configure("--prefix=%s" % prefix) - - make() - make("install") diff --git a/var/spack/packages/metis/package.py b/var/spack/packages/metis/package.py deleted file mode 100644 index 7ce5ae1925..0000000000 --- a/var/spack/packages/metis/package.py +++ /dev/null @@ -1,27 +0,0 @@ -from spack import * - -class Metis(Package): - """METIS is a set of serial programs for partitioning graphs, - partitioning finite element meshes, and producing fill reducing - orderings for sparse matrices. The algorithms implemented in - METIS are based on the multilevel recursive-bisection, - multilevel k-way, and multi-constraint partitioning schemes.""" - - homepage = "http://glaros.dtc.umn.edu/gkhome/metis/metis/overview" - url = "http://glaros.dtc.umn.edu/gkhome/fetch/sw/metis/metis-5.1.0.tar.gz" - - version('5.1.0', '5465e67079419a69e0116de24fce58fe') - - depends_on('mpi') - - def install(self, spec, prefix): - cmake(".", - '-DGKLIB_PATH=%s/GKlib' % pwd(), - '-DSHARED=1', - '-DCMAKE_C_COMPILER=mpicc', - '-DCMAKE_CXX_COMPILER=mpicxx', - '-DSHARED=1', - *std_cmake_args) - - make() - make("install") diff --git a/var/spack/packages/mpc/package.py b/var/spack/packages/mpc/package.py deleted file mode 100644 index 6fbfca3007..0000000000 --- a/var/spack/packages/mpc/package.py +++ /dev/null @@ -1,42 +0,0 @@ -############################################################################## -# Copyright (c) 2013, Lawrence Livermore National Security, LLC. -# Produced at the Lawrence Livermore National Laboratory. -# -# This file is part of Spack. -# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. -# LLNL-CODE-647188 -# -# For details, see https://scalability-llnl.github.io/spack -# Please also see the LICENSE file for our notice and the LGPL. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License (as published by -# the Free Software Foundation) version 2.1 dated February 1999. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and -# conditions of the GNU General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -############################################################################## -from spack import * - -class Mpc(Package): - """Gnu Mpc is a C library for the arithmetic of complex numbers - with arbitrarily high precision and correct rounding of the - result.""" - homepage = "http://www.multiprecision.org" - url = "ftp://ftp.gnu.org/gnu/mpc/mpc-1.0.2.tar.gz" - - version('1.0.2', '68fadff3358fb3e7976c7a398a0af4c3') - - depends_on("gmp") - depends_on("mpfr") - - def install(self, spec, prefix): - configure("--prefix=%s" % prefix) - make() - make("install") diff --git a/var/spack/packages/mpe2/mpe2.patch b/var/spack/packages/mpe2/mpe2.patch deleted file mode 100644 index 3ade1f04f4..0000000000 --- a/var/spack/packages/mpe2/mpe2.patch +++ /dev/null @@ -1,12 +0,0 @@ -diff -rupN mpe2-1.3.0/src/graphics/src/mpe_graphics.c mpe2-1.3.0.new/src/graphics/src/mpe_graphics.c ---- mpe2-1.3.0/src/graphics/src/mpe_graphics.c 2009-06-15 10:36:22.000000000 -0600 -+++ mpe2-1.3.0.new/src/graphics/src/mpe_graphics.c 2014-10-25 00:11:22.000000000 -0600 -@@ -982,7 +982,7 @@ char *string; - return MPE_ERR_BAD_ARGS; - } - -- printf("color = %d, string = %s\n",(int) color, string); -+//printf("color = %d, string = %s\n",(int) color, string); - - XBSetPixVal( graph->xwin, graph->xwin->cmapping[color] ); - returnVal = XDrawString( graph->xwin->disp, XBDrawable(graph->xwin), diff --git a/var/spack/packages/mpe2/package.py b/var/spack/packages/mpe2/package.py deleted file mode 100644 index 27295172cc..0000000000 --- a/var/spack/packages/mpe2/package.py +++ /dev/null @@ -1,28 +0,0 @@ -from spack import * - -class Mpe2(Package): - """Message Passing Extensions (MPE) -- Parallel, shared X window graphics""" - - homepage = "http://www.mcs.anl.gov/research/projects/perfvis/software/MPE/" - url = "ftp://ftp.mcs.anl.gov/pub/mpi/mpe/mpe2-1.3.0.tar.gz" - - version('1.3.0', '67bf0c7b2e573df3ba0d2059a96c2f7b') - - patch('mpe2.patch') - - depends_on("mpi") - - provides("mpe") - - def install(self, spec, prefix): - configure("--prefix=" + prefix, - "--x-includes=/usr/X11R6/include", - "--x-libraries=/usr/X11R6/lib", - "--enable-mpe_graphics=yes", - "--disable-f77", - "--enable-viewers=no", - "--enable-slog2=no", - "--with-mpicc=mpicc") - - make() - make("install") diff --git a/var/spack/packages/mpfr/package.py b/var/spack/packages/mpfr/package.py deleted file mode 100644 index 9c744a22df..0000000000 --- a/var/spack/packages/mpfr/package.py +++ /dev/null @@ -1,41 +0,0 @@ -############################################################################## -# Copyright (c) 2013, Lawrence Livermore National Security, LLC. -# Produced at the Lawrence Livermore National Laboratory. -# -# This file is part of Spack. -# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. -# LLNL-CODE-647188 -# -# For details, see https://scalability-llnl.github.io/spack -# Please also see the LICENSE file for our notice and the LGPL. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License (as published by -# the Free Software Foundation) version 2.1 dated February 1999. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and -# conditions of the GNU General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -############################################################################## -from spack import * - -class Mpfr(Package): - """The MPFR library is a C library for multiple-precision - floating-point computations with correct rounding.""" - homepage = "http://www.mpfr.org" - url = "http://www.mpfr.org/mpfr-current/mpfr-3.1.3.tar.bz2" - - version('3.1.3', '5fdfa3cfa5c86514ee4a241a1affa138') - # version('3.1.2', 'ee2c3ac63bf0c2359bf08fc3ee094c19') - - depends_on('gmp') - - def install(self, spec, prefix): - configure("--prefix=%s" % prefix) - make() - make("install") diff --git a/var/spack/packages/mpibash/mpibash-4.3.patch b/var/spack/packages/mpibash/mpibash-4.3.patch deleted file mode 100644 index 17e285b0bf..0000000000 --- a/var/spack/packages/mpibash/mpibash-4.3.patch +++ /dev/null @@ -1,1565 +0,0 @@ -diff -Naur bash-4.3/builtins/circle.def mpibash-4.3/builtins/circle.def ---- bash-4.3/builtins/circle.def 1969-12-31 17:00:00.000000000 -0700 -+++ mpibash-4.3/builtins/circle.def 2014-05-13 11:27:37.314100671 -0600 -@@ -0,0 +1,620 @@ -+This file is circle.def, from which is created circle.c. -+It implements all of the "circle_*" builtins in Bash. -+ -+$PRODUCES circle.c -+ -+#include -+ -+#include -+#if defined (HAVE_UNISTD_H) -+# ifdef _MINIX -+# include -+# endif -+# include -+#endif -+ -+#include "../bashintl.h" -+#include "../shell.h" -+#include "common.h" -+#include "bashgetopt.h" -+#include -+ -+extern int running_trap, trap_saved_exit_value; -+ -+static int circle_rank; /* Rank in the Libcircle job */ -+static SHELL_VAR *create_func = NULL; /* User-defined callback function for CIRCLE_cb_create. */ -+static SHELL_VAR *process_func = NULL; /* User-defined callback function for CIRCLE_cb_process. */ -+static SHELL_VAR *reduce_init_func = NULL; /* User-defined callback function for CIRCLE_cb_reduce_init. */ -+static SHELL_VAR *reduce_fini_func = NULL; /* User-defined callback function for CIRCLE_cb_reduce_fini. */ -+static SHELL_VAR *reduce_op_func = NULL; /* User-defined callback function for CIRCLE_cb_reduce_op. */ -+static CIRCLE_handle *current_handle = NULL; /* Active handle within a callback or NULL if not within a callback */ -+static int within_reduction = 0; /* 1=within a reduction callback; 0=not */ -+ -+/* Return with a usage message if no arguments remain. */ -+#define YES_ARGS(LIST) \ -+ if ((LIST) == 0) \ -+ { \ -+ builtin_usage (); \ -+ return (EX_USAGE); \ -+ } -+ -+/* Perform the same operation as bind_variable, but with VALUE being a -+ * number, not a string. */ -+static SHELL_VAR * -+bind_variable_number (name, value, flags) -+ const char *name; -+ long value; -+ int flags; -+{ -+ char numstr[25]; /* String version of VALUE */ -+ -+ sprintf (numstr, "%ld", value); -+ return bind_variable (name, numstr, flags); -+} -+ -+/* Invoke the user-defined creation-callback function (create_func). */ -+static void -+internal_create_func (handle) -+ CIRCLE_handle *handle; -+{ -+ WORD_LIST *funcargs; -+ -+ if (create_func == NULL) -+ return; -+ current_handle = handle; -+ funcargs = make_word_list (make_word ("cb_create"), NULL); -+ execute_shell_function (create_func, funcargs); -+ dispose_words (funcargs); -+ current_handle = NULL; -+} -+ -+/* Invoke the user-defined process-callback function (process_func). */ -+static void -+internal_process_func (handle) -+ CIRCLE_handle *handle; -+{ -+ WORD_LIST *funcargs; -+ -+ if (process_func == NULL) -+ return; -+ current_handle = handle; -+ funcargs = make_word_list (make_word ("cb_process"), NULL); -+ execute_shell_function (process_func, funcargs); -+ dispose_words (funcargs); -+ current_handle = NULL; -+} -+ -+/* Invoke the user-defined reduction-initiation callback function -+ * (reduce_init_func). */ -+static void -+internal_reduce_init_func (void) -+{ -+ WORD_LIST *funcargs; -+ -+ if (reduce_init_func == NULL) -+ return; -+ within_reduction = 1; -+ funcargs = make_word_list (make_word ("cb_reduce_init"), NULL); -+ execute_shell_function (reduce_init_func, funcargs); -+ dispose_words (funcargs); -+ within_reduction = 0; -+} -+ -+/* Invoke the user-defined reduction callback function -+ * (reduce_op_func). */ -+static void -+internal_reduce_op_func (buf1, size1, buf2, size2) -+ const void* buf1; -+ size_t size1; -+ const void* buf2; -+ size_t size2; -+{ -+ WORD_LIST *funcargs; -+ -+ if (reduce_op_func == NULL) -+ return; -+ within_reduction = 1; -+ funcargs = make_word_list (make_word (buf2), NULL); -+ funcargs = make_word_list (make_word (buf1), funcargs); -+ funcargs = make_word_list (make_word ("cb_reduce_op"), funcargs); -+ execute_shell_function (reduce_op_func, funcargs); -+ dispose_words (funcargs); -+ within_reduction = 0; -+} -+ -+/* Invoke the user-defined reduction-finalization callback function -+ * (reduce_fini_func). */ -+static void -+internal_reduce_fini_func (buf, size) -+ const void* buf; -+ size_t size; -+{ -+ WORD_LIST *funcargs; -+ -+ if (reduce_fini_func == NULL) -+ return; -+ funcargs = make_word_list (make_word (buf), NULL); -+ funcargs = make_word_list (make_word ("cb_reduce_fini"), funcargs); -+ execute_shell_function (reduce_fini_func, funcargs); -+ dispose_words (funcargs); -+} -+ -+/* Look up a user-provided callback function. */ -+static int -+find_callback_function (list, user_func) -+ WORD_LIST *list; -+ SHELL_VAR **user_func; -+{ -+ char *funcname; /* Name of the user-defined function. */ -+ -+ /* If no argument was provided, nullify the callback function. */ -+ if (list == NULL) -+ { -+ *user_func = NULL; -+ return EXECUTION_SUCCESS; -+ } -+ -+ /* Get the callback function. */ -+ funcname = list->word->word; -+ list = list->next; -+ no_args (list); -+ *user_func = find_function (funcname); -+ if (*user_func == NULL) -+ { -+ builtin_error (_("function %s not found"), funcname); -+ return EXECUTION_FAILURE; -+ } -+ return EXECUTION_SUCCESS; -+} -+ -+/* Initialize Libcircle. */ -+void -+initialize_libcircle (argc, argv) -+ int argc; -+ char **argv; -+{ -+ circle_rank = CIRCLE_init (argc, argv, CIRCLE_DEFAULT_FLAGS); -+ bind_variable_number ("circle_rank", circle_rank, 0); -+ CIRCLE_enable_logging (CIRCLE_LOG_WARN); -+ CIRCLE_cb_create (internal_create_func); -+ CIRCLE_cb_process (internal_process_func); -+ CIRCLE_cb_reduce_init (internal_reduce_init_func); -+ CIRCLE_cb_reduce_op (internal_reduce_op_func); -+ CIRCLE_cb_reduce_fini (internal_reduce_fini_func); -+} -+ -+/* Finalize Libcircle. */ -+void -+finalize_libcircle (void) -+{ -+ CIRCLE_finalize (); -+} -+ -+/* ---------------------------------------------------------------------- */ -+ -+$BUILTIN circle_set_options -+$FUNCTION circle_set_options_builtin -+$SHORT_DOC circle_set_options [flag]... -+Change Libcircle's run-time behavior. -+ -+Arguments: -+ FLAG "split_random", "split_equal", or "create_global" -+ -+Multiple flags can be provided. If no flags are provided, Libcircle -+reverts to its default options. -+ -+Exit Status: -+Returns 0 unless an invalid option is given. -+$END -+/*'*/ -+ -+/* Here is the circle_set_options builtin. */ -+int -+circle_set_options_builtin (list) -+ WORD_LIST *list; -+{ -+ char *word; /* One argument */ -+ int flags = 0; /* Flags to pass to CIRCLE_set_options */ -+ -+ if (list == NULL) -+ flags = CIRCLE_DEFAULT_FLAGS; -+ else -+ while (list != NULL) -+ { -+ word = list->word->word; -+ if (!strcmp (word, "split_random")) -+ flags |= CIRCLE_SPLIT_RANDOM; -+ else if (!strcmp (word, "split_equal")) -+ flags |= CIRCLE_SPLIT_EQUAL; -+ else if (!strcmp (word, "create_global")) -+ flags |= CIRCLE_CREATE_GLOBAL; -+ else -+ { -+ builtin_error (_("invalid flag \"%s\""), word); -+ return (EXECUTION_FAILURE); -+ } -+ list = list->next; -+ } -+ CIRCLE_set_options (flags); -+ return EXECUTION_SUCCESS; -+} -+ -+$BUILTIN circle_cb_create -+$FUNCTION circle_cb_create_builtin -+$SHORT_DOC circle_cb_create [func] -+Register a function that will create work when asked. -+ -+Arguments: -+ FUNC User-defined callback function that will invoke -+ circle_enqueue when called -+ -+If FUNC is omitted, no function will be associated with work creation. -+This can be used to nullify a previous circle_cb_create invocation. -+ -+Exit Status: -+Returns 0 unless an invalid function is given or an error occurs. -+$END -+ -+/* Here is the circle_cb_create builtin. */ -+int -+circle_cb_create_builtin (list) -+ WORD_LIST *list; -+{ -+ return find_callback_function (list, &create_func); -+} -+ -+$BUILTIN circle_cb_process -+$FUNCTION circle_cb_process_builtin -+$SHORT_DOC circle_cb_process [func] -+Register a function that will process work when asked. -+ -+Arguments: -+ FUNC User-defined callback function that will invoke -+ circle_enqueue when called -+ -+If FUNC is omitted, no function will be associated with work processing. -+This can be used to nullify a previous circle_cb_process invocation. -+ -+Exit Status: -+Returns 0 unless an invalid function is given or an error occurs. -+$END -+ -+/* Here is the circle_cb_process builtin. */ -+int -+circle_cb_process_builtin (list) -+ WORD_LIST *list; -+{ -+ return find_callback_function (list, &process_func); -+} -+ -+$BUILTIN circle_begin -+$FUNCTION circle_begin_builtin -+$SHORT_DOC circle_begin -+Begin creation and processing of the distributed work queue. -+ -+Exit Status: -+Returns 0 unless an error occurs. -+$END -+ -+/* Here is the circle_begin builtin. */ -+int -+circle_begin_builtin (list) -+ WORD_LIST *list; -+{ -+ no_args (list); -+ CIRCLE_begin (); -+ return EXECUTION_SUCCESS; -+} -+ -+$BUILTIN circle_enqueue -+$FUNCTION circle_enqueue_builtin -+$SHORT_DOC circle_enqueue work -+Enqueue work onto the distributed queue. -+ -+Arguments: -+ WORK "Work" as represented by an arbitrary string of limited -+ size (generally around 4KB) -+ -+Exit Status: -+Returns 0 unless an error occurs. -+$END -+ -+/* Here is the circle_enqueue builtin. */ -+int -+circle_enqueue_builtin (list) -+ WORD_LIST *list; -+{ -+ char *work; /* Work to perform */ -+ -+ /* Extract the work argument. */ -+ YES_ARGS (list); -+ work = list->word->word; -+ list = list->next; -+ no_args (list); -+ -+ /* Complain if we're not within a proper callback function. */ -+ if (current_handle == NULL) -+ { -+ builtin_error (_("not within a Libcircle \"create\" or \"process\" callback function")); -+ return EXECUTION_FAILURE; -+ } -+ -+ /* Enqueue the work. */ -+ if (current_handle->enqueue (work) == -1) -+ return EXECUTION_FAILURE; -+ return EXECUTION_SUCCESS; -+} -+ -+$BUILTIN circle_dequeue -+$FUNCTION circle_dequeue_builtin -+$SHORT_DOC circle_dequeue var -+Dequeue work from the distributed queue into a variable. -+ -+Arguments: -+ VAR Variable in which to receive previously enqueued "work" -+ -+Exit Status: -+Returns 0 unless an error occurs. -+$END -+ -+/* Here is the circle_dequeue builtin. */ -+int -+circle_dequeue_builtin (list) -+ WORD_LIST *list; -+{ -+ char *varname; /* Variable in which to store the work string */ -+ char work[CIRCLE_MAX_STRING_LEN+1]; /* Work to perform */ -+ -+ /* Extract the variable-name argument. */ -+ YES_ARGS (list); -+ varname = list->word->word; -+ list = list->next; -+ no_args (list); -+ -+ /* Complain if we're not within a callback function. */ -+ if (current_handle == NULL) -+ { -+ builtin_error (_("not within a Libcircle callback function")); -+ return EXECUTION_FAILURE; -+ } -+ -+ /* Dequeue the work and bind it to the given variable. */ -+ if (current_handle->dequeue (work) == -1) -+ return EXECUTION_FAILURE; -+ bind_variable (varname, work, 0); -+ return EXECUTION_SUCCESS; -+} -+ -+$BUILTIN circle_enable_logging -+$FUNCTION circle_enable_logging_builtin -+$SHORT_DOC circle_enable_logging log_level -+Change Libcircle's logging verbosity -+ -+Arguments: -+ LOG_LEVEL "fatal", "error", "warning", "info", or "debug" -+ -+Exit Status: -+Returns 0 unless an invalid option is given. -+$END -+/*'*/ -+ -+/* Here is the circle_enable_logging builtin. */ -+int -+circle_enable_logging_builtin (list) -+ WORD_LIST *list; -+{ -+ char *word; /* One argument */ -+ CIRCLE_loglevel loglevel; /* Level to set */ -+ -+ /* Parse the log level. */ -+ YES_ARGS (list); -+ word = list->word->word; -+ if (!strcmp (word, "fatal")) -+ loglevel = CIRCLE_LOG_FATAL; -+ else if (!strcmp (word, "error")) -+ loglevel = CIRCLE_LOG_ERR; -+ else if (!strcmp (word, "warning")) -+ loglevel = CIRCLE_LOG_WARN; -+ else if (!strcmp (word, "info")) -+ loglevel = CIRCLE_LOG_INFO; -+ else if (!strcmp (word, "debug")) -+ loglevel = CIRCLE_LOG_DBG; -+ else -+ { -+ builtin_error (_("invalid log level \"%s\""), word); -+ return (EXECUTION_FAILURE); -+ } -+ -+ /* Set the log level. */ -+ CIRCLE_enable_logging (loglevel); -+ return EXECUTION_SUCCESS; -+} -+ -+$BUILTIN circle_abort -+$FUNCTION circle_abort_builtin -+$SHORT_DOC circle_abort -+Terminate queue processing. -+ -+Exit Status: -+Returns 0 unless an error occurs. -+$END -+ -+/* Here is the circle_abort builtin. */ -+int -+circle_abort_builtin (list) -+ WORD_LIST *list; -+{ -+ no_args (list); -+ CIRCLE_abort (); -+ return EXECUTION_SUCCESS; -+} -+ -+$BUILTIN circle_checkpoint -+$FUNCTION circle_checkpoint_builtin -+$SHORT_DOC circle_checkpoint -+Checkpoint a work queue to disk. -+ -+Write a file called circle${circle_rank}.txt containing the current -+queue state of rank ${circle_rank}. On a later run, a worker can -+invoke circle_read_restarts to repopulate its queue from such a -+checkpoint file. -+ -+Exit Status: -+Returns 0 unless an error occurs. -+$END -+/*'*/ -+ -+/* Here is the circle_checkpoint builtin. */ -+int -+circle_checkpoint_builtin (list) -+ WORD_LIST *list; -+{ -+ no_args (list); -+ CIRCLE_checkpoint (); -+ return EXECUTION_SUCCESS; -+} -+ -+$BUILTIN circle_read_restarts -+$FUNCTION circle_read_restarts_builtin -+$SHORT_DOC circle_read_restarts -+Repopulate a work queue from a disk checkpoint. -+ -+Read queue contents from a file called circle${circle_rank}.txt, which -+was previously produced by circle_checkpoint. -+ -+Exit Status: -+Returns 0 unless an error occurs. -+$END -+/*'*/ -+ -+/* Here is the circle_read_restarts builtin. */ -+int -+circle_read_restarts_builtin (list) -+ WORD_LIST *list; -+{ -+ no_args (list); -+ CIRCLE_read_restarts (); -+ return EXECUTION_SUCCESS; -+} -+ -+$BUILTIN circle_cb_reduce_init -+$FUNCTION circle_cb_reduce_init_builtin -+$SHORT_DOC circle_cb_reduce_init [func] -+Register a function that will initiate a reduction operation. -+ -+Arguments: -+ FUNC User-defined callback function that will invoke -+ circle_reduce when called -+ -+FUNC will be invoked on all ranks. -+ -+If FUNC is omitted, no function will be associated with reduction -+initialization. This can be used to nullify a previous -+circle_cb_reduce_init invocation. -+ -+Exit Status: -+Returns 0 unless an invalid function is given or an error occurs. -+$END -+ -+/* Here is the circle_cb_reduce_init builtin. */ -+int -+circle_cb_reduce_init_builtin (list) -+ WORD_LIST *list; -+{ -+ return find_callback_function (list, &reduce_init_func); -+} -+ -+$BUILTIN circle_cb_reduce_op -+$FUNCTION circle_cb_reduce_op_builtin -+$SHORT_DOC circle_cb_reduce_op [func] -+Register a function that will complete a reduction operation. -+ -+Arguments: -+ FUNC User-defined callback function that will receive -+ two items to reduce and invoke circle_reduce on -+ the reduced value -+ -+If FUNC is omitted, no function will be associated with reduction -+execution. This can be used to nullify a previous circle_cb_reduce_op -+invocation. -+ -+Exit Status: -+Returns 0 unless an invalid function is given or an error occurs. -+$END -+ -+/* Here is the circle_cb_reduce_op builtin. */ -+int -+circle_cb_reduce_op_builtin (list) -+ WORD_LIST *list; -+{ -+ return find_callback_function (list, &reduce_op_func); -+} -+ -+$BUILTIN circle_cb_reduce_fini -+$FUNCTION circle_cb_reduce_fini_builtin -+$SHORT_DOC circle_cb_reduce_fini [func] -+Register a function that will complete a reduction operation. -+ -+Arguments: -+ FUNC User-defined callback function that will receive -+ the final reduced data -+ -+If FUNC is omitted, no function will be associated with reduction -+completion. This can be used to nullify a previous -+circle_cb_reduce_fini invocation. -+ -+Libcircle guarantees that FUNC will be invoked only on rank 0. -+ -+Exit Status: -+Returns 0 unless an invalid function is given or an error occurs. -+$END -+ -+/* Here is the circle_cb_reduce_fini builtin. */ -+int -+circle_cb_reduce_fini_builtin (list) -+ WORD_LIST *list; -+{ -+ return find_callback_function (list, &reduce_fini_func); -+} -+ -+$BUILTIN circle_reduce -+$FUNCTION circle_reduce_builtin -+$SHORT_DOC circle_reduce work -+Seed the next phase of a reduction operation -+ -+Arguments: -+ WORK "Work" as represented by an arbitrary string of limited -+ size (generally around 4KB) -+ -+This function should be called both by the callback function -+registered with circle_reduce_init and the callback function -+registered with circle_reduce_op. -+ -+Exit Status: -+Returns 0 unless an error occurs. -+$END -+ -+/* Here is the circle_reduce builtin. */ -+int -+circle_reduce_builtin (list) -+ WORD_LIST *list; -+{ -+ char *work; /* Work to perform */ -+ -+ /* Extract the work argument. */ -+ YES_ARGS (list); -+ work = list->word->word; -+ list = list->next; -+ no_args (list); -+ -+ /* Complain if we're not within a proper callback function. */ -+ if (!within_reduction) -+ { -+ builtin_error (_("not within a Libcircle \"reduce_init\" or \"reduce_op\" callback function")); -+ return EXECUTION_FAILURE; -+ } -+ -+ /* Reduce the work. */ -+ CIRCLE_reduce (work, strlen (work)); -+ return EXECUTION_SUCCESS; -+} -diff -Naur bash-4.3/builtins/Makefile.in mpibash-4.3/builtins/Makefile.in ---- bash-4.3/builtins/Makefile.in 2012-05-25 07:29:19.000000000 -0600 -+++ mpibash-4.3/builtins/Makefile.in 2014-05-13 11:27:37.314100671 -0600 -@@ -141,7 +141,9 @@ - $(srcdir)/times.def $(srcdir)/trap.def $(srcdir)/type.def \ - $(srcdir)/ulimit.def $(srcdir)/umask.def $(srcdir)/wait.def \ - $(srcdir)/reserved.def $(srcdir)/pushd.def $(srcdir)/shopt.def \ -- $(srcdir)/printf.def $(srcdir)/complete.def $(srcdir)/mapfile.def -+ $(srcdir)/printf.def $(srcdir)/complete.def $(srcdir)/mapfile.def \ -+ $(srcdir)/mpi.def \ -+@CIRCLE@ $(srcdir)/circle.def - - STATIC_SOURCE = common.c evalstring.c evalfile.c getopt.c bashgetopt.c \ - getopt.h -@@ -153,7 +155,9 @@ - jobs.o kill.o let.o mapfile.o \ - pushd.o read.o return.o set.o setattr.o shift.o source.o \ - suspend.o test.o times.o trap.o type.o ulimit.o umask.o \ -- wait.o getopts.o shopt.o printf.o getopt.o bashgetopt.o complete.o -+ wait.o getopts.o shopt.o printf.o getopt.o bashgetopt.o complete.o \ -+ mpi.o \ -+@CIRCLE@ circle.o - - CREATED_FILES = builtext.h builtins.c psize.aux pipesize.h tmpbuiltins.c \ - tmpbuiltins.h -@@ -317,6 +321,8 @@ - getopts.o: getopts.def - reserved.o: reserved.def - complete.o: complete.def -+@CIRCLE@ circle.o: circle.def -+mpi.o: mpi.def - - # C files - bashgetopt.o: ../config.h $(topdir)/bashansi.h $(BASHINCDIR)/ansi_stdlib.h -@@ -644,6 +650,19 @@ - mapfile.o: $(topdir)/subst.h $(topdir)/externs.h $(BASHINCDIR)/maxpath.h - mapfile.o: $(topdir)/shell.h $(topdir)/syntax.h $(topdir)/variables.h $(topdir)/conftypes.h - mapfile.o: $(topdir)/arrayfunc.h ../pathnames.h -+@CIRCLE@ circle.o: $(topdir)/command.h ../config.h $(BASHINCDIR)/memalloc.h $(topdir)/error.h -+@CIRCLE@ circle.o: $(topdir)/general.h $(topdir)/xmalloc.h $(topdir)/subst.h $(topdir)/externs.h -+@CIRCLE@ circle.o: $(topdir)/quit.h $(topdir)/dispose_cmd.h $(topdir)/make_cmd.h -+@CIRCLE@ circle.o: $(topdir)/shell.h $(topdir)/syntax.h $(topdir)/unwind_prot.h $(topdir)/variables.h $(topdir)/conftypes.h -+@CIRCLE@ circle.o: $(BASHINCDIR)/maxpath.h ../pathnames.h -+mpi.o: ../config.h ../config-top.h ../config-bot.h ../bashintl.h -+mpi.o: ../include/gettext.h ../shell.h ../config.h ../bashjmp.h -+mpi.o: ../include/posixjmp.h ../command.h ../syntax.h ../general.h -+mpi.o: ../bashtypes.h ../include/chartypes.h ../xmalloc.h ../bashansi.h -+mpi.o: ../error.h ../variables.h ../array.h ../assoc.h ../hashlib.h -+mpi.o: ../conftypes.h ../arrayfunc.h ../quit.h ../sig.h ../include/maxpath.h -+mpi.o: ../unwind_prot.h ../dispose_cmd.h ../make_cmd.h ../include/ocache.h -+mpi.o: ../subst.h ../pathnames.h ../externs.h common.h bashgetopt.h - - #bind.o: $(RL_LIBSRC)chardefs.h $(RL_LIBSRC)readline.h $(RL_LIBSRC)keymaps.h - -diff -Naur bash-4.3/builtins/mpi.def mpibash-4.3/builtins/mpi.def ---- bash-4.3/builtins/mpi.def 1969-12-31 17:00:00.000000000 -0700 -+++ mpibash-4.3/builtins/mpi.def 2014-05-13 11:27:37.314100671 -0600 -@@ -0,0 +1,744 @@ -+This file is mpi.def, from which is created mpi.c. -+It implements all of the "mpi_*" builtins in Bash. -+ -+$PRODUCES mpi.c -+ -+#include -+ -+#include -+#if defined (HAVE_UNISTD_H) -+# ifdef _MINIX -+# include -+# endif -+# include -+#endif -+ -+#include "../bashintl.h" -+#include "../shell.h" -+#include "common.h" -+#include "bashgetopt.h" -+#include -+ -+extern int running_trap, trap_saved_exit_value; -+ -+/* Keep track of who we are within MPI_COMM_WORLD. */ -+static int mpi_rank; -+static int mpi_num_ranks; -+ -+/* Try an MPI operation. Return with an error message on failure. */ -+#define MPI_TRY(STMT) \ -+ do \ -+ { \ -+ int mpierr; \ -+ mpierr = STMT; \ -+ if (mpierr != MPI_SUCCESS) \ -+ return report_mpi_error (mpierr); \ -+ } \ -+ while (0) -+ -+/* Return with a usage message if no arguments remain. */ -+#define YES_ARGS(LIST) \ -+ if ((LIST) == 0) \ -+ { \ -+ builtin_usage (); \ -+ return (EX_USAGE); \ -+ } -+ -+/* Return with an error message if a given variable is read-only or if -+ * we can't write to it for any other reason (e.g., it's defined as a -+ * function). */ -+#define REQUIRE_WRITABLE(NAME) \ -+ do \ -+ { \ -+ SHELL_VAR *bindvar = find_shell_variable (NAME); \ -+ if (bindvar) \ -+ { \ -+ if (readonly_p (bindvar)) \ -+ { \ -+ err_readonly (NAME); \ -+ return (EXECUTION_FAILURE); \ -+ } \ -+ if (unbind_variable (NAME) == -1) \ -+ { \ -+ builtin_error ("Failed to write to variable %s", NAME); \ -+ return (EXECUTION_FAILURE); \ -+ } \ -+ } \ -+ } \ -+ while (0) -+ -+/* Initialize MPI. */ -+void -+initialize_mpi (argc, argv) -+ int argc; -+ char **argv; -+{ -+ int init_done; -+ -+ MPI_Initialized (&init_done); -+ if (!init_done) -+ MPI_Init (&argc, &argv); -+ MPI_Errhandler_set (MPI_COMM_WORLD, MPI_ERRORS_RETURN); -+ MPI_Comm_rank (MPI_COMM_WORLD, &mpi_rank); -+ MPI_Comm_size (MPI_COMM_WORLD, &mpi_num_ranks); -+} -+ -+/* Finalize MPI. */ -+void -+finalize_mpi () -+{ -+ MPI_Finalize (); -+} -+ -+/* Parse an operation name into an MPI_Op. Return 1 on success, 0 on -+ * failure. */ -+static int -+parse_operation (char *name, MPI_Op *op) -+{ -+ /* Define a mapping from operator names to MPI_Op values. */ -+ typedef struct { -+ char *name; /* Operation name (e.g., "sum") */ -+ MPI_Op value; /* Operation value (e.g., MPI_SUM) */ -+ } opname2value_t; -+ static opname2value_t oplist[] = { -+ {"max", MPI_MAX}, -+ {"min", MPI_MIN}, -+ {"sum", MPI_SUM}, -+ {"prod", MPI_PROD}, -+ {"land", MPI_LAND}, -+ {"band", MPI_BAND}, -+ {"lor", MPI_LOR}, -+ {"bor", MPI_BOR}, -+ {"lxor", MPI_LXOR}, -+ {"bxor", MPI_BXOR}, -+ {"maxloc", MPI_MAXLOC}, -+ {"minloc", MPI_MINLOC} -+ }; -+ size_t i; -+ -+ for (i = 0; i < sizeof(oplist)/sizeof(opname2value_t); i++) -+ if (!strcmp(name, oplist[i].name)) -+ { -+ *op = oplist[i].value; -+ if (i > 0) -+ { -+ /* As a performance optimization, bubble up the value we -+ * just found. */ -+ opname2value_t prev = oplist[i - 1]; -+ oplist[i - 1] = oplist[i]; -+ oplist[i] = prev; -+ } -+ return 1; -+ } -+ return 0; -+} -+ -+/* Report an error to the user and return EXECUTION_FAILURE. */ -+static int -+report_mpi_error (mpierr) -+ int mpierr; -+{ -+ char errstr[MPI_MAX_ERROR_STRING]; -+ int errstrlen; -+ -+ MPI_Error_string (mpierr, errstr, &errstrlen); -+ builtin_error ("%s", errstr); -+ return EXECUTION_FAILURE; -+} -+ -+/* Perform the same operation as bind_variable, but with VALUE being a -+ * number, not a string. */ -+static SHELL_VAR * -+bind_variable_number (name, value, flags) -+ const char *name; -+ long value; -+ int flags; -+{ -+ char numstr[25]; /* String version of VALUE */ -+ -+ sprintf (numstr, "%ld", value); -+ return bind_variable (name, numstr, flags); -+} -+ -+/* Perform the same operation as bind_array_variable, but with VALUE -+ * being a number, not a string. */ -+static SHELL_VAR * -+bind_array_variable_number (name, ind, value, flags) -+ char *name; -+ arrayind_t ind; -+ long value; -+ int flags; -+{ -+ char numstr[25]; /* String version of VALUE */ -+ -+ sprintf (numstr, "%ld", value); -+ return bind_array_variable (name, ind, numstr, flags); -+} -+ -+/* Define a reduction-type function (allreduce, scan, exscan, etc.). */ -+typedef int (*reduction_func_t)(void *, void *, int, MPI_Datatype, MPI_Op, MPI_Comm); -+ -+/* Perform any reduction-type operation (allreduce, scan, exscan, etc.). */ -+static int -+reduction_like (list, funcname, func) -+ WORD_LIST *list; -+ char *funcname; -+ reduction_func_t func; -+{ -+ char *word; /* One argument */ -+ struct { -+ long int value; /* Reduced value */ -+ int rank; /* Rank associated with the above */ -+ } number, result; -+ MPI_Op operation = MPI_SUM; /* Operation to perform */ -+ char *varname; /* Name of the variable to bind the results to */ -+ intmax_t n; -+ int i; -+ -+ /* Parse "-O OPERATION" (optional), where OPERATION is a reduction -+ * operation. */ -+ YES_ARGS (list); -+ word = list->word->word; -+ if (ISOPTION (word, 'O')) -+ { -+ list = list->next; -+ if (list == 0) -+ { -+ sh_needarg (funcname); -+ return (EX_USAGE); -+ } -+ word = list->word->word; -+ if (!parse_operation (word, &operation)) -+ { -+ sh_invalidopt ("-O"); -+ return (EX_USAGE); -+ } -+ list = list->next; -+ } -+ -+ /* Parse the argument, which must be a number. */ -+ YES_ARGS (list); -+ word = list->word->word; -+ if (!legal_number (word, &n)) -+ { -+ sh_neednumarg (funcname); -+ return (EX_USAGE); -+ } -+ number.value = (long int) n; -+ number.rank = mpi_rank; -+ list = list->next; -+ -+ /* Parse the target variable, which must not be read-only. */ -+ YES_ARGS (list); -+ varname = list->word->word; -+ if (mpi_rank != 0 || func != MPI_Exscan) -+ REQUIRE_WRITABLE (varname); -+ list = list->next; -+ no_args (list); -+ -+ /* Perform the reduction operation. Bind the given array variable -+ * to the result and, for minloc/maxloc, the associated rank. */ -+ if (mpi_rank != 0 || func != MPI_Exscan) { -+ bind_array_variable (varname, 0, "", 0); -+ bind_array_variable (varname, 1, "", 0); -+ } -+ if (operation == MPI_MINLOC || operation == MPI_MAXLOC) -+ { -+ MPI_TRY (func (&number, &result, 1, MPI_LONG_INT, operation, MPI_COMM_WORLD)); -+ if (mpi_rank != 0 || func != MPI_Exscan) -+ bind_array_variable_number (varname, 1, result.rank, 0); -+ } -+ else -+ MPI_TRY (func (&number.value, &result.value, 1, MPI_LONG, operation, MPI_COMM_WORLD)); -+ if (mpi_rank != 0 || func != MPI_Exscan) -+ bind_array_variable_number (varname, 0, result.value, 0); -+ return EXECUTION_SUCCESS; -+} -+ -+$BUILTIN mpi_comm_rank -+$FUNCTION mpi_comm_rank_builtin -+$SHORT_DOC mpi_comm_rank name -+Return the process's rank in the MPI job. -+ -+Arguments: -+ NAME Scalar variable in which to receive the rank -+ -+Exit Status: -+Returns 0 unless an invalid option is given. -+$END -+/*'*/ -+ -+/* Here is the mpi_comm_rank builtin. */ -+int -+mpi_comm_rank_builtin (list) -+ WORD_LIST *list; -+{ -+ char *varname; /* Name of the variable to bind the results to */ -+ -+ YES_ARGS (list); -+ varname = list->word->word; -+ REQUIRE_WRITABLE (varname); -+ list = list->next; -+ no_args (list); -+ bind_variable_number (varname, mpi_rank, 0); -+ return EXECUTION_SUCCESS; -+} -+ -+$BUILTIN mpi_comm_size -+$FUNCTION mpi_comm_size_builtin -+$SHORT_DOC mpi_comm_size name -+Return the total number of ranks in the MPI job. -+ -+Arguments: -+ NAME Scalar variable in which to receive the number of ranks -+ -+Exit Status: -+Returns 0 unless an invalid option is given. -+$END -+ -+/* Here is the mpi_comm_size builtin. */ -+int -+mpi_comm_size_builtin (list) -+ WORD_LIST *list; -+{ -+ char *varname; /* Name of the variable to bind the results to */ -+ -+ YES_ARGS (list); -+ varname = list->word->word; -+ REQUIRE_WRITABLE (varname); -+ list = list->next; -+ no_args (list); -+ bind_variable_number (varname, mpi_num_ranks, 0); -+ return EXECUTION_SUCCESS; -+} -+ -+$BUILTIN mpi_abort -+$FUNCTION mpi_abort_builtin -+$SHORT_DOC mpi_abort [n] -+Abort all processes in the MPI job and exit the shell. -+ -+Exits not only the caller's shell (with a status of N) but also all -+remote shells that are part of the same MPI job. If N is omitted, the -+exit status is that of the last command executed. -+ -+This command should be used only in extreme circumstances. It is -+better for each process to exit normally on its own. -+$END -+/*'*/ -+ -+/* Here is the mpi_abort builtin. */ -+int -+mpi_abort_builtin (list) -+ WORD_LIST *list; -+{ -+ int exit_value; -+ -+ exit_value = (running_trap == 1 && list == 0) ? trap_saved_exit_value : get_exitstat (list); /* Copied from exit.def */ -+ MPI_TRY (MPI_Abort (MPI_COMM_WORLD, exit_value)); -+ return EXECUTION_FAILURE; -+} -+ -+$BUILTIN mpi_send -+$FUNCTION mpi_send_builtin -+$SHORT_DOC mpi_send [-t tag] rank message -+Send a message to a remote process in the same MPI job. -+ -+Options: -+ -t TAG Send the message using tag TAG (default: 0). TAG must -+ be a nonnegative integer. -+ -+Arguments: -+ RANK Whom to send the message to. RANK must be an integer in -+ the range [0, $(mpi_comm_size)-1]. -+ -+ MESSAGE String to send to rank RANK. -+ -+Exit Status: -+Returns 0 unless an invalid option is given or an error occurs. -+$END -+ -+/* Here is the mpi_send builtin. */ -+int -+mpi_send_builtin (list) -+ WORD_LIST *list; -+{ -+ char *word; /* One argument */ -+ intmax_t target_rank; /* MPI target rank */ -+ char *message; /* Message to send to rank target_rank */ -+ intmax_t tag = 0; /* Message tag to use */ -+ -+ /* Parse "-t TAG" (optional), where TAG is a number or "any". */ -+ YES_ARGS (list); -+ word = list->word->word; -+ if (ISOPTION (word, 't')) -+ { -+ list = list->next; -+ if (list == 0) -+ { -+ sh_needarg ("mpi_recv"); -+ return (EX_USAGE); -+ } -+ word = list->word->word; -+ if (!legal_number (word, &tag)) -+ { -+ sh_neednumarg ("-t"); -+ return (EX_USAGE); -+ } -+ list = list->next; -+ } -+ else if (*word == '-') -+ { -+ sh_invalidopt (word); -+ builtin_usage (); -+ return (EX_USAGE); -+ } -+ -+ /* Parse the target rank, which must be a number. */ -+ YES_ARGS (list); -+ word = list->word->word; -+ if (!legal_number (word, &target_rank)) -+ { -+ builtin_error (_("mpi_send: numeric rank required")); -+ return (EX_USAGE); -+ } -+ list = list->next; -+ -+ /* Parse the message to send. */ -+ YES_ARGS (list); -+ message = list->word->word; -+ list = list->next; -+ no_args (list); -+ -+ /* Send the message. */ -+ MPI_TRY (MPI_Send (message, strlen(message)+1, MPI_BYTE, (int)target_rank, (int)tag, MPI_COMM_WORLD)); -+ return EXECUTION_SUCCESS; -+} -+ -+ -+$BUILTIN mpi_recv -+$FUNCTION mpi_recv_builtin -+$SHORT_DOC mpi_recv [-t tag] rank name -+Receive a message from a remote process in the same MPI job. -+ -+Options: -+ -t TAG Receive only messages sent using tag TAG (default: 0). -+ TAG must be either a nonnegative integer or the string -+ "any" to receive messages sent using any tag. -+ -+Arguments: -+ RANK Receive only messages sent from sender RANK. RANK -+ must either be in the range [0, $(mpi_comm_size)-1] or -+ be the string "any" to receive messages from any sender. -+ -+ NAME Array variable in which to receive the message, sender -+ rank, and tag. -+ -+Exit Status: -+Returns 0 unless an invalid option is given or an error occurs. -+$END -+ -+/* Here is the mpi_recv builtin. */ -+int -+mpi_recv_builtin (list) -+ WORD_LIST *list; -+{ -+ char *word; /* One argument */ -+ intmax_t source_rank; /* MPI source rank */ -+ char *endptr; /* Used for parsing strings into numbers */ -+ MPI_Status status; /* Status of an MPI operation */ -+ int count; /* Message length in bytes */ -+ intmax_t tag = 0; /* Message tag to use */ -+ char *varname; /* Name of the variable to bind the results to */ -+ static char *message = NULL; /* Message received from MPI */ -+ static size_t alloced = 0; /* Number of bytes allocated for the above */ -+ int opt; /* Parsed option */ -+ -+ /* Parse any options provided. */ -+ reset_internal_getopt (); -+ while ((opt = internal_getopt (list, "t:")) != -1) -+ { -+ switch (opt) -+ { -+ case 't': -+ if (!strcmp (list_optarg, "any")) -+ tag = MPI_ANY_TAG; -+ else if (!legal_number (list_optarg, &tag)) -+ { -+ builtin_error (_("-t: numeric argument or \"any\" required")); -+ return (EX_USAGE); -+ } -+ break; -+ -+ default: -+ sh_invalidopt (word); -+ builtin_usage (); -+ return (EX_USAGE); -+ } -+ } -+ list = loptend; -+ -+ /* Parse the source rank, which must be a number or "any". */ -+ YES_ARGS (list); -+ word = list->word->word; -+ if (!legal_number (word, &source_rank)) -+ { -+ if (!strcmp (word, "any")) -+ source_rank = MPI_ANY_SOURCE; -+ else -+ { -+ builtin_error (_("mpi_recv: numeric rank or \"any\" required")); -+ return (EX_USAGE); -+ } -+ } -+ list = list->next; -+ -+ /* Parse the target variable, which must not be read-only. */ -+ YES_ARGS (list); -+ varname = list->word->word; -+ REQUIRE_WRITABLE (varname); -+ list = list->next; -+ no_args (list); -+ -+ /* Receive a message. Because we don't know long the message will -+ * be, we first probe to get the length. */ -+ MPI_TRY (MPI_Probe ((int)source_rank, (int)tag, MPI_COMM_WORLD, &status)); -+ MPI_TRY (MPI_Get_count (&status, MPI_BYTE, &count)); -+ if (alloced < count) -+ { -+ message = xrealloc (message, count); -+ alloced = count; -+ } -+ MPI_TRY (MPI_Recv (message, count, MPI_BYTE, status.MPI_SOURCE, status.MPI_TAG, MPI_COMM_WORLD, &status)); -+ bind_array_variable (varname, 0, message, 0); -+ bind_array_variable_number (varname, 1, status.MPI_SOURCE, 0); -+ bind_array_variable_number (varname, 2, status.MPI_TAG, 0); -+ return EXECUTION_SUCCESS; -+} -+ -+$BUILTIN mpi_barrier -+$FUNCTION mpi_barrier_builtin -+$SHORT_DOC mpi_barrier -+Synchronizes all of the processes in the MPI job. -+ -+No process will return from mpi_barrier until all processes have -+called mpi_barrier. -+ -+Exit Status: -+Returns 0 unless an invalid option is given or an error occurs. -+$END -+ -+/* Here is the mpi_barrier builtin. */ -+int -+mpi_barrier_builtin (list) -+ WORD_LIST *list; -+{ -+ no_args (list); -+ MPI_TRY (MPI_Barrier (MPI_COMM_WORLD)); -+ return EXECUTION_SUCCESS; -+} -+ -+$BUILTIN mpi_bcast -+$FUNCTION mpi_bcast_builtin -+$SHORT_DOC mpi_bcast [message] name -+Broadcast a message to all processes in the same MPI job. -+ -+Arguments: -+ MESSAGE String to broadcast from one process to all the others. -+ -+ NAME Scalar variable in which to receive the broadcast message. -+ -+Exactly one process in the MPI job must specify a message to -+broadcast. No process will return from mpi_bcast until all processes -+have called mpi_bcast. -+ -+Exit Status: -+Returns 0 unless an invalid option is given or an error occurs. -+$END -+ -+/* Here is the mpi_bcast builtin. */ -+int -+mpi_bcast_builtin (list) -+ WORD_LIST *list; -+{ -+ char *word; /* One argument */ -+ int root; /* MPI root rank */ -+ char *root_message; /* Message to broadcast */ -+ int msglen; /* Length in bytes of the above (including the NULL byte) */ -+ char *varname; /* Name of the variable to bind the results to */ -+ static int *all_lengths = NULL; /* List of every rank's msglen */ -+ static char *message = NULL; /* Message received from the root */ -+ static int alloced = 0; /* Bytes allocated for the above */ -+ int i; -+ -+ /* Parse the optional message and target variable, which must not be -+ * read-only. */ -+ YES_ARGS (list); -+ if (list->next == NULL) -+ { -+ /* Non-root */ -+ root_message = NULL; -+ msglen = -1; -+ } -+ else -+ { -+ /* Root */ -+ root_message = list->word->word; -+ msglen = (int) strlen(root_message) + 1; -+ list = list->next; -+ } -+ varname = list->word->word; -+ REQUIRE_WRITABLE (varname); -+ list = list->next; -+ no_args (list); -+ -+ /* Acquire global agreement on the root and the message size. */ -+ if (all_lengths == NULL) -+ all_lengths = xmalloc (mpi_num_ranks*sizeof(int)); -+ MPI_TRY (MPI_Allgather (&msglen, 1, MPI_INT, all_lengths, 1, MPI_INT, MPI_COMM_WORLD)); -+ root = -1; -+ for (i = 0; i < mpi_num_ranks; i++) -+ { -+ if (all_lengths[i] == -1) -+ continue; -+ if (root != -1) -+ { -+ builtin_error (_("mpi_bcast: more than one process specified a message")); -+ return (EXECUTION_FAILURE); -+ } -+ root = i; -+ msglen = all_lengths[i]; -+ } -+ if (root == -1) -+ { -+ builtin_error (_("mpi_bcast: no process specified a message")); -+ return (EXECUTION_FAILURE); -+ } -+ -+ /* Broadcast the message. */ -+ if (mpi_rank == root) -+ { -+ MPI_TRY (MPI_Bcast (root_message, msglen, MPI_BYTE, root, MPI_COMM_WORLD)); -+ bind_variable (varname, root_message, 0); -+ } -+ else -+ { -+ if (alloced < msglen) -+ { -+ message = xrealloc (message, msglen); -+ alloced = msglen; -+ } -+ MPI_TRY (MPI_Bcast (message, msglen, MPI_BYTE, root, MPI_COMM_WORLD)); -+ bind_variable (varname, message, 0); -+ } -+ return EXECUTION_SUCCESS; -+} -+ -+$BUILTIN mpi_scan -+$FUNCTION mpi_scan_builtin -+$SHORT_DOC mpi_scan number name -+Perform an inclusive scan across all processes in the same MPI job. -+ -+ -O OPERATION Operation to perform. Must be one of "max", "min", -+ "sum", "prod", "land", "band", "lor", "bor", "lxor", -+ "bxor", "maxloc", or "minloc" (default: "sum"). -+ -+Arguments: -+ NUMBER Integer to use in the scan operation. -+ -+ NAME Array variable in which to receive the result and, in -+ the case of maxloc and minloc, the associated rank. -+ -+In an inclusive-scan operation, each process i presents a number, -+a[i]. Once all processes in the MPI job have presented their number, -+the command returns a[0] to rank 0, a[0]+a[1] to rank 1, -+a[0]+a[1]+a[2] to rank 2, and so forth. The -O option enables "+" to -+be replaced with other operations. -+ -+Inclusive scans can be useful for assigning a unique index to each -+process in the MPI job. -+ -+Exit Status: -+Returns 0 unless an invalid option is given or an error occurs. -+$END -+ -+/* Here is the mpi_scan builtin. */ -+int -+mpi_scan_builtin (list) -+ WORD_LIST *list; -+{ -+ return reduction_like (list, "mpi_scan", MPI_Scan); -+} -+ -+$BUILTIN mpi_exscan -+$FUNCTION mpi_exscan_builtin -+$SHORT_DOC mpi_exscan number name -+Perform an exclusive scan across all processes in the same MPI job. -+ -+ -O OPERATION Operation to perform. Must be one of "max", "min", -+ "sum", "prod", "land", "band", "lor", "bor", "lxor", -+ "bxor", "maxloc", or "minloc" (default: "sum"). -+ -+Arguments: -+ NUMBER Integer to use in the scan operation. -+ -+ NAME Array variable in which to receive the result and, in -+ the case of maxloc and minloc, the associated rank. -+ -+In a exclusive-scan operation, each process i presents a number, a[i]. -+Once all processes in the MPI job have presented their number, the -+command assigns a[0] to NAME on rank 1, a[0]+a[1] to NAME on rank 2, -+a[0]+a[1]+a[2] to NAME on rank 3, and so forth. No assignment is -+performed on rank 0. The -O option enables "+" to be replaced with -+other operations. -+ -+Exclusive scans can be useful for assigning a unique index to each -+process in the MPI job. -+ -+Exit Status: -+Returns 0 unless an invalid option is given or an error occurs. -+$END -+ -+/* Here is the mpi_exscan builtin. */ -+int -+mpi_exscan_builtin (list) -+ WORD_LIST *list; -+{ -+ return reduction_like (list, "mpi_exscan", MPI_Exscan); -+} -+ -+$BUILTIN mpi_allreduce -+$FUNCTION mpi_allreduce_builtin -+$SHORT_DOC mpi_allreduce number name -+Reduce numbers from all processes in an MPI job to a single number. -+ -+Options: -+ -+ -O OPERATION Operation to perform. Must be one of "max", "min", -+ "sum", "prod", "land", "band", "lor", "bor", "lxor", -+ "bxor", "maxloc", or "minloc" (default: "sum"). -+ -+Arguments: -+ NUMBER Integer to use in the allreduce operation. -+ -+ NAME Array variable in which to receive the result and, in -+ the case of maxloc and minloc, the associated rank. -+ -+In an all-reduce operation, each process i presents a number, a[i]. -+Once all processes in the MPI job have presented their number, the -+command returns a[0]+a[1]+...+a[n-1] to all ranks. The -O option -+enables "+" to be replaced with other operations. -+ -+All-reduces can be useful for reaching global agreement (e.g., of a -+termination condition). -+ -+Exit Status: -+Returns 0 unless an invalid option is given or an error occurs. -+$END -+ -+/* Here is the mpi_allreduce builtin. */ -+int -+mpi_allreduce_builtin (list) -+ WORD_LIST *list; -+{ -+ return reduction_like (list, "mpi_allreduce", MPI_Allreduce); -+} -diff -Naur bash-4.3/config.h.in mpibash-4.3/config.h.in ---- bash-4.3/config.h.in 2013-06-29 15:35:33.000000000 -0600 -+++ mpibash-4.3/config.h.in 2014-05-13 11:27:37.314100671 -0600 -@@ -1147,6 +1147,12 @@ - /* Define if you have the `__argz_stringify' function. */ - #undef HAVE___ARGZ_STRINGIFY - -+/* Define if you have both the header file and the libcircle library. */ -+#undef HAVE_LIBCIRCLE -+ -+/* Define if you have the `CIRCLE_cb_reduce_op' function. */ -+#undef HAVE_CIRCLE_CB_REDUCE_OP -+ - /* End additions for lib/intl */ - - #include "config-bot.h" -diff -Naur bash-4.3/configure.ac mpibash-4.3/configure.ac ---- bash-4.3/configure.ac 2014-02-11 08:37:53.000000000 -0700 -+++ mpibash-4.3/configure.ac 2014-05-13 11:27:37.302100179 -0600 -@@ -24,7 +24,7 @@ - AC_REVISION([for Bash 4.3, version 4.063])dnl - - define(bashvers, 4.3) --define(relstatus, release) -+define(relstatus, MPI) - - AC_INIT([bash], bashvers-relstatus, [bug-bash@gnu.org]) - -@@ -813,6 +813,21 @@ - fi - ]) - -+dnl Ensure that we can find an MPI library. -+AC_CHECK_FUNCS([MPI_Init], [], [ -+ AC_MSG_ERROR([Cannot continue without MPI. Consider specifying CC=mpicc.])]) -+ -+dnl If we have Libcircle, use it, too. -+AC_SEARCH_LIBS([CIRCLE_cb_create], [circle], [AC_CHECK_HEADERS([libcircle.h])]) -+if test "x$ac_cv_header_libcircle_h" = xyes; then -+ libcircle_make_prefix="" -+ AC_DEFINE([HAVE_LIBCIRCLE], [1], [Define if you have the Libcircle header and library.]) -+ AC_CHECK_FUNCS([CIRCLE_cb_reduce_op]) -+else -+ libcircle_make_prefix="#" -+fi -+AC_SUBST([CIRCLE], [$libcircle_make_prefix]) -+ - BASH_CHECK_DECL(strtoimax) - BASH_CHECK_DECL(strtol) - BASH_CHECK_DECL(strtoll) -diff -Naur bash-4.3/Makefile.in mpibash-4.3/Makefile.in ---- bash-4.3/Makefile.in 2014-01-25 14:27:30.000000000 -0700 -+++ mpibash-4.3/Makefile.in 2014-05-13 11:27:37.314100671 -0600 -@@ -104,7 +104,7 @@ - VERSPROG = bashversion$(EXEEXT) - VERSOBJ = bashversion.$(OBJEXT) - --Program = bash$(EXEEXT) -+Program = mpibash$(EXEEXT) - Version = @BASHVERS@ - PatchLevel = `$(BUILD_DIR)/$(VERSPROG) -p` - RELSTATUS = @RELSTATUS@ -diff -Naur bash-4.3/shell.c mpibash-4.3/shell.c ---- bash-4.3/shell.c 2014-01-14 06:04:32.000000000 -0700 -+++ mpibash-4.3/shell.c 2014-05-13 11:27:37.314100671 -0600 -@@ -107,6 +107,13 @@ - extern char *primary_prompt, *secondary_prompt; - extern char *this_command_name; - -+extern void initialize_mpi __P((int, char **)); -+extern void finalize_mpi __P((void)); -+#ifdef HAVE_LIBCIRCLE -+extern void initialize_libcircle __P((int, char **)); -+extern void finalize_libcircle __P((void)); -+#endif -+ - /* Non-zero means that this shell has already been run; i.e. you should - call shell_reinitialize () if you need to start afresh. */ - int shell_initialized = 0; -@@ -324,7 +331,7 @@ - static void init_interactive_script __P((void)); - - static void set_shell_name __P((char *)); --static void shell_initialize __P((void)); -+static void shell_initialize __P((int, char **)); - static void shell_reinitialize __P((void)); - - static void show_shell_usage __P((FILE *, int)); -@@ -561,7 +568,7 @@ - - /* From here on in, the shell must be a normal functioning shell. - Variables from the environment are expected to be set, etc. */ -- shell_initialize (); -+ shell_initialize (argc, argv); - - set_default_lang (); - set_default_locale_vars (); -@@ -941,6 +948,12 @@ - end_job_control (); - #endif /* JOB_CONTROL */ - -+#ifdef HAVE_LIBCIRCLE -+ finalize_libcircle (); -+#else -+ finalize_mpi (); -+#endif -+ - /* Always return the exit status of the last command to our parent. */ - sh_exit (s); - } -@@ -1691,7 +1704,9 @@ - /* Do whatever is necessary to initialize the shell. - Put new initializations in here. */ - static void --shell_initialize () -+shell_initialize (argc, argv) -+ int argc; -+ char **argv; - { - char hostname[256]; - -@@ -1760,6 +1775,17 @@ - initialize_shell_options (privileged_mode||running_setuid); - initialize_bashopts (privileged_mode||running_setuid); - #endif -+ -+ /* Initialize Libcircle and MPI. */ -+#ifdef HAVE_LIBCIRCLE -+ initialize_libcircle (argc, argv); -+ initialize_mpi (argc, argv); -+ bind_variable ("libcircle", "yes", 0); -+#else -+ initialize_mpi (argc, argv); -+ bind_variable ("libcircle", "no", 0); -+#endif -+ bind_variable ("mpibash", "yes", 0); - } - - /* Function called by main () when it appears that the shell has already diff --git a/var/spack/packages/mpibash/package.py b/var/spack/packages/mpibash/package.py deleted file mode 100644 index d0f6dafed6..0000000000 --- a/var/spack/packages/mpibash/package.py +++ /dev/null @@ -1,32 +0,0 @@ -import os -from spack import * - -class Mpibash(Package): - """Parallel scripting right from the Bourne-Again Shell (Bash)""" - homepage = "http://www.ccs3.lanl.gov/~pakin/software/mpibash-4.3.html" - - version('4.3', '81348932d5da294953e15d4814c74dd1', - url="http://ftp.gnu.org/gnu/bash/bash-4.3.tar.gz") - - # patch -p1 < ../mpibash-4.3.patch - patch('mpibash-4.3.patch', level=1, when='@4.3') - - # above patch modifies configure.ac - depends_on('autoconf') - - # uses MPI_Exscan which is in MPI-1.2 and later - depends_on('mpi@1.2:') - - depends_on('libcircle') - - def install(self, spec, prefix): - # run autoconf to rebuild configure - autoconf = which('autoconf') - autoconf() - - configure("--prefix=" + prefix, - "CC=mpicc") - - make(parallel=False) - - make("install") diff --git a/var/spack/packages/mpich/package.py b/var/spack/packages/mpich/package.py deleted file mode 100644 index d48bf878f6..0000000000 --- a/var/spack/packages/mpich/package.py +++ /dev/null @@ -1,92 +0,0 @@ -############################################################################## -# Copyright (c) 2013, Lawrence Livermore National Security, LLC. -# Produced at the Lawrence Livermore National Laboratory. -# -# This file is part of Spack. -# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. -# LLNL-CODE-647188 -# -# For details, see https://scalability-llnl.github.io/spack -# Please also see the LICENSE file for our notice and the LGPL. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License (as published by -# the Free Software Foundation) version 2.1 dated February 1999. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and -# conditions of the GNU General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -############################################################################## -from spack import * -import os - -class Mpich(Package): - """MPICH is a high performance and widely portable implementation of - the Message Passing Interface (MPI) standard.""" - homepage = "http://www.mpich.org" - url = "http://www.mpich.org/static/downloads/3.0.4/mpich-3.0.4.tar.gz" - list_url = "http://www.mpich.org/static/downloads/" - list_depth = 2 - - version('3.1.4', '2ab544607986486562e076b83937bba2') - version('3.1.3', '93cb17f91ac758cbf9174ecb03563778') - version('3.1.2', '7fbf4b81dcb74b07ae85939d1ceee7f1') - version('3.1.1', '40dc408b1e03cc36d80209baaa2d32b7') - version('3.1', '5643dd176499bfb7d25079aaff25f2ec') - version('3.0.4', '9c5d5d4fe1e17dd12153f40bc5b6dbc0') - - provides('mpi@:3.0', when='@3:') - provides('mpi@:1.3', when='@1:') - - def setup_dependent_environment(self, module, spec, dep_spec): - """For dependencies, make mpicc's use spack wrapper.""" - os.environ['MPICH_CC'] = 'cc' - os.environ['MPICH_CXX'] = 'c++' - os.environ['MPICH_F77'] = 'f77' - os.environ['MPICH_F90'] = 'f90' - - - def install(self, spec, prefix): - config_args = ["--prefix=" + prefix, - "--enable-shared"] - - # TODO: Spack should make it so that you can't actually find - # these compilers if they're "disabled" for the current - # compiler configuration. - if not self.compiler.f77: - config_args.append("--disable-f77") - - if not self.compiler.fc: - config_args.append("--disable-fc") - - configure(*config_args) - make() - make("install") - - self.filter_compilers() - - - def filter_compilers(self): - """Run after install to make the MPI compilers use the - compilers that Spack built the package with. - - If this isn't done, they'll have CC, CXX, F77, and FC set - to Spack's generic cc, c++, f77, and f90. We want them to - be bound to whatever compiler they were built with. - """ - bin = self.prefix.bin - mpicc = os.path.join(bin, 'mpicc') - mpicxx = os.path.join(bin, 'mpicxx') - mpif77 = os.path.join(bin, 'mpif77') - mpif90 = os.path.join(bin, 'mpif90') - - kwargs = { 'ignore_absent' : True, 'backup' : False, 'string' : True } - filter_file('CC="cc"', 'CC="%s"' % self.compiler.cc, mpicc, **kwargs) - filter_file('CXX="c++"', 'CXX="%s"' % self.compiler.cxx, mpicxx, **kwargs) - filter_file('F77="f77"', 'F77="%s"' % self.compiler.f77, mpif77, **kwargs) - filter_file('FC="f90"', 'FC="%s"' % self.compiler.fc, mpif90, **kwargs) diff --git a/var/spack/packages/mpileaks/package.py b/var/spack/packages/mpileaks/package.py deleted file mode 100644 index 4ef866588c..0000000000 --- a/var/spack/packages/mpileaks/package.py +++ /dev/null @@ -1,44 +0,0 @@ -############################################################################## -# Copyright (c) 2013, Lawrence Livermore National Security, LLC. -# Produced at the Lawrence Livermore National Laboratory. -# -# This file is part of Spack. -# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. -# LLNL-CODE-647188 -# -# For details, see https://scalability-llnl.github.io/spack -# Please also see the LICENSE file for our notice and the LGPL. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License (as published by -# the Free Software Foundation) version 2.1 dated February 1999. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and -# conditions of the GNU General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -############################################################################## -from spack import * - -class Mpileaks(Package): - """Tool to detect and report leaked MPI objects like MPI_Requests and MPI_Datatypes.""" - - homepage = "https://github.com/hpc/mpileaks" - url = "https://github.com/hpc/mpileaks/releases/download/v1.0/mpileaks-1.0.tar.gz" - - version('1.0', '8838c574b39202a57d7c2d68692718aa') - - depends_on("mpi") - depends_on("adept-utils") - depends_on("callpath") - - def install(self, spec, prefix): - configure("--prefix=" + prefix, - "--with-adept-utils=" + spec['adept-utils'].prefix, - "--with-callpath=" + spec['callpath'].prefix) - make() - make("install") diff --git a/var/spack/packages/mrnet/package.py b/var/spack/packages/mrnet/package.py deleted file mode 100644 index 6e9766f275..0000000000 --- a/var/spack/packages/mrnet/package.py +++ /dev/null @@ -1,20 +0,0 @@ -from spack import * - -class Mrnet(Package): - """The MRNet Multi-Cast Reduction Network.""" - homepage = "http://paradyn.org/mrnet" - url = "ftp://ftp.cs.wisc.edu/paradyn/mrnet/mrnet_4.0.0.tar.gz" - - version('4.0.0', 'd00301c078cba57ef68613be32ceea2f') - version('4.1.0', '5a248298b395b329e2371bf25366115c') - - parallel = False - - depends_on("boost") - - def install(self, spec, prefix): - configure("--prefix=%s" %prefix, "--enable-shared") - - make() - make("install") - diff --git a/var/spack/packages/munge/package.py b/var/spack/packages/munge/package.py deleted file mode 100644 index c737ca0354..0000000000 --- a/var/spack/packages/munge/package.py +++ /dev/null @@ -1,20 +0,0 @@ -from spack import * -import os - -class Munge(Package): - """ MUNGE Uid 'N' Gid Emporium """ - homepage = "https://code.google.com/p/munge/" - url = "https://github.com/dun/munge/releases/download/munge-0.5.11/munge-0.5.11.tar.bz2" - - version('0.5.11', 'bd8fca8d5f4c1fcbef1816482d49ee01', url='https://github.com/dun/munge/releases/download/munge-0.5.11/munge-0.5.11.tar.bz2') - - depends_on('openssl') - depends_on('libgcrypt') - - def install(self, spec, prefix): - os.makedirs(os.path.join(prefix, "lib/systemd/system")) - configure("--prefix=%s" % prefix) - - make() - make("install") - diff --git a/var/spack/packages/muster/package.py b/var/spack/packages/muster/package.py deleted file mode 100644 index 722daf3d7f..0000000000 --- a/var/spack/packages/muster/package.py +++ /dev/null @@ -1,22 +0,0 @@ -from spack import * - -class Muster(Package): - """The Muster library provides implementations of sequential and - parallel K-Medoids clustering algorithms. It is intended as a - general framework for parallel cluster analysis, particularly - for performance data analysis on systems with very large - numbers of processes. - """ - homepage = "https://github.com/scalability-llnl/muster" - url = "https://github.com/scalability-llnl/muster/archive/v1.0.tar.gz" - - version('1.0.1', 'd709787db7e080447afb6571ac17723c') - version('1.0', '2eec6979a4a36d3a65a792d12969be16') - - depends_on("boost") - depends_on("mpi") - - def install(self, spec, prefix): - cmake(".", *std_cmake_args) - make() - make("install") diff --git a/var/spack/packages/mvapich2/ad_lustre_rwcontig_open_source.patch b/var/spack/packages/mvapich2/ad_lustre_rwcontig_open_source.patch deleted file mode 100644 index ff85845cf8..0000000000 --- a/var/spack/packages/mvapich2/ad_lustre_rwcontig_open_source.patch +++ /dev/null @@ -1,11 +0,0 @@ ---- a/src/mpi/romio/adio/ad_lustre/ad_lustre_rwcontig.c 2013-12-10 12:05:44.806417000 -0800 -+++ b/src/mpi/romio/adio/ad_lustre/ad_lustre_rwcontig.c 2013-12-10 11:53:03.295622000 -0800 -@@ -8,7 +8,7 @@ - * Copyright (C) 2008 Sun Microsystems, Lustre group - */ - --#define _XOPEN_SOURCE 600 -+//#define _XOPEN_SOURCE 600 - #include - #include - #include "ad_lustre.h" diff --git a/var/spack/packages/mvapich2/package.py b/var/spack/packages/mvapich2/package.py deleted file mode 100644 index ca0b1287c1..0000000000 --- a/var/spack/packages/mvapich2/package.py +++ /dev/null @@ -1,104 +0,0 @@ -import os -from spack import * - -class Mvapich2(Package): - """mvapich2 is an MPI implmenetation for infiniband networks.""" - homepage = "http://mvapich.cse.ohio-state.edu/" - - version('1.9', '5dc58ed08fd3142c260b70fe297e127c', - url="http://mvapich.cse.ohio-state.edu/download/mvapich2/mv2/mvapich2-1.9.tgz") - patch('ad_lustre_rwcontig_open_source.patch', when='@1.9') - - version('2.0', '9fbb68a4111a8b6338e476dc657388b4', - url='http://mvapich.cse.ohio-state.edu/download/mvapich/mv2/mvapich2-2.0.tar.gz') - - provides('mpi@:2.2', when='@1.9') # MVAPICH2-1.9 supports MPI 2.2 - provides('mpi@:3.0', when='@2.0') # MVAPICH2-2.0 supports MPI 3.0 - - - def install(self, spec, prefix): - # we'll set different configure flags depending on our environment - configure_args = [] - - # TODO: The MPICH*_FLAGS have a different name for 1.9 - - if '+debug' in spec: - # set configure flags for debug build - configure_args.append("--disable-fast") - configure_args.append("--enable-g=dbg") - configure_args.append("--enable-error-checking=runtime") - configure_args.append("--enable-error-messages=all") - configure_args.append("--enable-nmpi-as-mpi") - - if "%gnu" in spec: - # set variables for GNU compilers - os.environ['MPICHLIB_CFLAGS'] = "-g -O0" - os.environ['MPICHLIB_CXXFLAGS'] = "-g -O0" - os.environ['MPICHLIB_FFLAGS'] = "-g -O0 -fno-second-underscore" - os.environ['MPICHLIB_F90FLAGS'] = "-g -O0 -fno-second-underscore" - elif "%intel" in spec: - # set variables for Inel compilers - os.environ['MPICHLIB_CFLAGS'] = "-g -O0" - os.environ['MPICHLIB_CXXFLAGS'] = "-g -O0" - os.environ['MPICHLIB_FFLAGS'] = "-g -O0" - os.environ['MPICHLIB_F90FLAGS'] = "-g -O0" - elif "%pgi" in spec: - # set variables for PGI compilers - os.environ['MPICHLIB_CFLAGS'] = "-g -O0 -fPIC" - os.environ['MPICHLIB_CXXFLAGS'] = "-g -O0 -fPIC" - os.environ['MPICHLIB_FFLAGS'] = "-g -O0 -fPIC" - os.environ['MPICHLIB_F90FLAGS'] = "-g -O0 -fPIC" - - else: - # set configure flags for normal optimizations - configure_args.append("--enable-fast=all") - configure_args.append("--enable-g=dbg") - configure_args.append("--enable-nmpi-as-mpi") - - if "%gnu" in spec: - # set variables for what compilers - os.environ['MPICHLIB_CFLAGS'] = "-g -O2" - os.environ['MPICHLIB_CXXFLAGS'] = "-g -O2" - os.environ['MPICHLIB_FFLAGS'] = "-g -O2 -fno-second-underscore" - os.environ['MPICHLIB_F90FLAGS'] = "-g -O2 -fno-second-underscore" - elif "%intel" in spec: - # set variables for Inel compilers - os.environ['MPICHLIB_CFLAGS'] = "-g -O2" - os.environ['MPICHLIB_CXXFLAGS'] = "-g -O2" - os.environ['MPICHLIB_FFLAGS'] = "-g -O2" - os.environ['MPICHLIB_F90FLAGS'] = "-g -O2" - elif "%pgi" in spec: - # set variables for PGI compilers - os.environ['MPICHLIB_CFLAGS'] = "-g -O2 -fPIC" - os.environ['MPICHLIB_CXXFLAGS'] = "-g -O2 -fPIC" - os.environ['MPICHLIB_FFLAGS'] = "-g -O2 -fPIC" - os.environ['MPICHLIB_F90FLAGS'] = "-g -O2 -fPIC" - - # determine network type by variant - if "+psm" in spec: - # throw this flag on QLogic systems to use PSM - configure_args.append("--with-device=ch3:psm") - else: - # throw this flag on IB systems - configure_args.append("--with-device=ch3:mrail", "--with-rdma=gen2") - - # TODO: shared-memory build - - # TODO: CUDA - - # TODO: other file systems like panasis - - configure( - "--prefix=" + prefix, - "--enable-f77", "--enable-fc", "--enable-cxx", - "--enable-shared", "--enable-sharedlibs=gcc", - "--enable-debuginfo", - "--with-pm=no", "--with-pmi=slurm", - "--enable-romio", "--with-file-system=lustre+nfs+ufs", - "--disable-mpe", "--without-mpe", - "--disable-silent-rules", - *configure_args) - - make() - - make("install") diff --git a/var/spack/packages/nasm/package.py b/var/spack/packages/nasm/package.py deleted file mode 100644 index 933b6a62c5..0000000000 --- a/var/spack/packages/nasm/package.py +++ /dev/null @@ -1,14 +0,0 @@ -from spack import * - -class Nasm(Package): - """NASM (Netwide Assembler) is an 80x86 assembler designed for - portability and modularity. It includes a disassembler as well.""" - homepage = "http://www.nasm.us" - url = "http://www.nasm.us/pub/nasm/releasebuilds/2.11.06/nasm-2.11.06.tar.xz" - - version('2.11.06', '2b958e9f5d200641e6fc9564977aecc5') - - def install(self, spec, prefix): - configure("--prefix=%s" % prefix) - make() - make("install") diff --git a/var/spack/packages/ncdu/package.py b/var/spack/packages/ncdu/package.py deleted file mode 100644 index 234f9730d6..0000000000 --- a/var/spack/packages/ncdu/package.py +++ /dev/null @@ -1,28 +0,0 @@ -from spack import * - -class Ncdu(Package): - """ - Ncdu is a disk usage analyzer with an ncurses interface. It is designed - to find space hogs on a remote server where you don't have an entire - gaphical setup available, but it is a useful tool even on regular desktop - systems. Ncdu aims to be fast, simple and easy to use, and should be able - to run in any minimal POSIX-like environment with ncurses installed. - """ - - homepage = "http://dev.yorhel.nl/ncdu" - url = "http://dev.yorhel.nl/download/ncdu-1.11.tar.gz" - - version('1.11', '9e44240a5356b029f05f0e70a63c4d12') - version('1.10', '7535decc8d54eca811493e82d4bfab2d') - version('1.9' , '93258079db897d28bb8890e2db89b1fb') - version('1.8' , '94d7a821f8a0d7ba8ef3dd926226f7d5') - version('1.7' , '172047c29d232724cc62e773e82e592a') - - depends_on("ncurses") - - def install(self, spec, prefix): - configure('--prefix=%s' % prefix, - '--with-ncurses=%s' % spec['ncurses']) - - make() - make("install") diff --git a/var/spack/packages/ncurses/package.py b/var/spack/packages/ncurses/package.py deleted file mode 100644 index cc180bbae1..0000000000 --- a/var/spack/packages/ncurses/package.py +++ /dev/null @@ -1,33 +0,0 @@ -from spack import * - -class Ncurses(Package): - """The ncurses (new curses) library is a free software emulation of curses - in System V Release 4.0, and more. It uses terminfo format, supports pads and - color and multiple highlights and forms characters and function-key mapping, - and has all the other SYSV-curses enhancements over BSD curses. - """ - - homepage = "http://invisible-island.net/ncurses/ncurses.html" - - version('5.9', '8cb9c412e5f2d96bc6f459aa8c6282a1', - url='http://ftp.gnu.org/pub/gnu/ncurses/ncurses-5.9.tar.gz') - version('6.0', 'ee13d052e1ead260d7c28071f46eefb1', - url='http://ftp.gnu.org/pub/gnu/ncurses/ncurses-6.0.tar.gz') - - def install(self, spec, prefix): - configure("--prefix=%s" % prefix, - "--with-shared", - "--enable-widec", - "--disable-pc-files", - "--without-ada") - make() - make("install") - - configure("--prefix=%s" % prefix, - "--with-shared", - "--disable-widec", - "--disable-pc-files", - "--without-ada") - make() - make("install") - diff --git a/var/spack/packages/netcdf/netcdf-4.3.3-mpi.patch b/var/spack/packages/netcdf/netcdf-4.3.3-mpi.patch deleted file mode 100644 index 46dda5fc9d..0000000000 --- a/var/spack/packages/netcdf/netcdf-4.3.3-mpi.patch +++ /dev/null @@ -1,25 +0,0 @@ -diff -Nur netcdf-4.3.3/CMakeLists.txt netcdf-4.3.3.mpi/CMakeLists.txt ---- netcdf-4.3.3/CMakeLists.txt 2015-02-12 16:44:35.000000000 -0500 -+++ netcdf-4.3.3.mpi/CMakeLists.txt 2015-10-14 16:44:41.176300658 -0400 -@@ -753,6 +753,7 @@ - SET(USE_PARALLEL OFF CACHE BOOL "") - MESSAGE(STATUS "Cannot find HDF5 library built with parallel support. Disabling parallel build.") - ELSE() -+ FIND_PACKAGE(MPI REQUIRED) - SET(USE_PARALLEL ON CACHE BOOL "") - SET(STATUS_PARALLEL "ON") - ENDIF() -diff -Nur netcdf-4.3.3/liblib/CMakeLists.txt netcdf-4.3.3.mpi/liblib/CMakeLists.txt ---- netcdf-4.3.3/liblib/CMakeLists.txt 2015-02-12 16:44:35.000000000 -0500 -+++ netcdf-4.3.3.mpi/liblib/CMakeLists.txt 2015-10-14 16:44:57.757793634 -0400 -@@ -71,6 +71,10 @@ - SET(TLL_LIBS ${TLL_LIBS} ${CURL_LIBRARY}) - ENDIF() - -+IF(USE_PARALLEL) -+ SET(TLL_LIBS ${TLL_LIBS} ${MPI_C_LIBRARIES}) -+ENDIF() -+ - IF(USE_HDF4) - SET(TLL_LIBS ${TLL_LIBS} ${HDF4_LIBRARIES}) - ENDIF() diff --git a/var/spack/packages/netcdf/package.py b/var/spack/packages/netcdf/package.py deleted file mode 100644 index e1e0d836c6..0000000000 --- a/var/spack/packages/netcdf/package.py +++ /dev/null @@ -1,27 +0,0 @@ -from spack import * - -class Netcdf(Package): - """NetCDF is a set of software libraries and self-describing, machine-independent - data formats that support the creation, access, and sharing of array-oriented - scientific data.""" - - homepage = "http://www.unidata.ucar.edu/software/netcdf/" - url = "ftp://ftp.unidata.ucar.edu/pub/netcdf/netcdf-4.3.3.tar.gz" - - version('4.3.3', '5fbd0e108a54bd82cb5702a73f56d2ae') - - patch('netcdf-4.3.3-mpi.patch') - - # Dependencies: - # >HDF5 - depends_on("hdf5") - - def install(self, spec, prefix): - with working_dir('spack-build', create=True): - cmake('..', - "-DCMAKE_INSTALL_PREFIX:PATH=%s" % prefix, - "-DENABLE_DAP:BOOL=OFF", # Disable DAP. - "-DBUILD_SHARED_LIBS:BOOL=OFF") # Don't build shared libraries (use static libs). - - make() - make("install") diff --git a/var/spack/packages/netgauge/package.py b/var/spack/packages/netgauge/package.py deleted file mode 100644 index c2378b0718..0000000000 --- a/var/spack/packages/netgauge/package.py +++ /dev/null @@ -1,43 +0,0 @@ -############################################################################## -# Copyright (c) 2013, Lawrence Livermore National Security, LLC. -# Produced at the Lawrence Livermore National Laboratory. -# -# This file is part of Spack. -# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. -# LLNL-CODE-647188 -# -# For details, see https://scalability-llnl.github.io/spack -# Please also see the LICENSE file for our notice and the LGPL. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License (as published by -# the Free Software Foundation) version 2.1 dated February 1999. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and -# conditions of the GNU General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -############################################################################## -from spack import * - -class Netgauge(Package): - """Netgauge is a high-precision network parameter measurement - tool. It supports benchmarking of many different network protocols - and communication patterns. The main focus lies on accuracy, - statistical analysis and easy extensibility. - """ - homepage = "http://unixer.de/research/netgauge/" - url = "http://unixer.de/research/netgauge/netgauge-2.4.6.tar.gz" - - version('2.4.6', 'e0e040ec6452e93ca21ccc54deac1d7f') - - depends_on("mpi") - - def install(self, spec, prefix): - configure("--prefix=%s" % prefix) - make() - make("install") diff --git a/var/spack/packages/netlib-blas/package.py b/var/spack/packages/netlib-blas/package.py deleted file mode 100644 index 85e97323d3..0000000000 --- a/var/spack/packages/netlib-blas/package.py +++ /dev/null @@ -1,46 +0,0 @@ -from spack import * -import os - - -class NetlibBlas(Package): - """Netlib reference BLAS""" - homepage = "http://www.netlib.org/lapack/" - url = "http://www.netlib.org/lapack/lapack-3.5.0.tgz" - - version('3.5.0', 'b1d3e3e425b2e44a06760ff173104bdf') - - variant('fpic', default=False, description="Build with -fpic compiler option") - - # virtual dependency - provides('blas') - - # Doesn't always build correctly in parallel - parallel = False - - def patch(self): - os.symlink('make.inc.example', 'make.inc') - - mf = FileFilter('make.inc') - mf.filter('^FORTRAN.*', 'FORTRAN = f90') - mf.filter('^LOADER.*', 'LOADER = f90') - mf.filter('^CC =.*', 'CC = cc') - - if '+fpic' in self.spec: - mf.filter('^OPTS.*=.*', 'OPTS = -O2 -frecursive -fpic') - mf.filter('^CFLAGS =.*', 'CFLAGS = -O3 -fpic') - - - def install(self, spec, prefix): - make('blaslib') - - # Tests that blas builds correctly - make('blas_testing') - - # No install provided - mkdirp(prefix.lib) - install('librefblas.a', prefix.lib) - - # Blas virtual package should provide blas.a and libblas.a - with working_dir(prefix.lib): - symlink('librefblas.a', 'blas.a') - symlink('librefblas.a', 'libblas.a') diff --git a/var/spack/packages/netlib-lapack/package.py b/var/spack/packages/netlib-lapack/package.py deleted file mode 100644 index fb6b99e27c..0000000000 --- a/var/spack/packages/netlib-lapack/package.py +++ /dev/null @@ -1,59 +0,0 @@ -from spack import * - -class NetlibLapack(Package): - """ - LAPACK version 3.X is a comprehensive FORTRAN library that does - linear algebra operations including matrix inversions, least - squared solutions to linear sets of equations, eigenvector - analysis, singular value decomposition, etc. It is a very - comprehensive and reputable package that has found extensive - use in the scientific community. - """ - homepage = "http://www.netlib.org/lapack/" - url = "http://www.netlib.org/lapack/lapack-3.5.0.tgz" - - version('3.5.0', 'b1d3e3e425b2e44a06760ff173104bdf') - version('3.4.2', '61bf1a8a4469d4bdb7604f5897179478') - version('3.4.1', '44c3869c38c8335c2b9c2a8bb276eb55') - version('3.4.0', '02d5706ec03ba885fc246e5fa10d8c70') - version('3.3.1', 'd0d533ec9a5b74933c2a1e84eedc58b4') - - variant('shared', default=False, description="Build shared library version") - - # virtual dependency - provides('lapack') - - # blas is a virtual dependency. - depends_on('blas') - - depends_on('cmake') - - # Doesn't always build correctly in parallel - parallel = False - - @when('^netlib-blas') - def get_blas_libs(self): - blas = self.spec['netlib-blas'] - return [join_path(blas.prefix.lib, 'blas.a')] - - - @when('^atlas') - def get_blas_libs(self): - blas = self.spec['atlas'] - return [join_path(blas.prefix.lib, l) - for l in ('libf77blas.a', 'libatlas.a')] - - - def install(self, spec, prefix): - blas_libs = ";".join(self.get_blas_libs()) - cmake_args = [".", '-DBLAS_LIBRARIES=' + blas_libs] - - if '+shared' in spec: - cmake_args.append('-DBUILD_SHARED_LIBS=ON') - - cmake_args += std_cmake_args - - cmake(*cmake_args) - make() - make("install") - diff --git a/var/spack/packages/nettle/package.py b/var/spack/packages/nettle/package.py deleted file mode 100644 index cd600b0b87..0000000000 --- a/var/spack/packages/nettle/package.py +++ /dev/null @@ -1,17 +0,0 @@ -from spack import * - -class Nettle(Package): - """The Nettle package contains the low-level cryptographic library - that is designed to fit easily in many contexts.""" - - homepage = "http://www.example.com" - url = "http://ftp.gnu.org/gnu/nettle/nettle-2.7.1.tar.gz" - - version('2.7', '2caa1bd667c35db71becb93c5d89737f') - - depends_on('gmp') - - def install(self, spec, prefix): - configure("--prefix=%s" % prefix) - make() - make("install") diff --git a/var/spack/packages/ompss/package.py b/var/spack/packages/ompss/package.py deleted file mode 100644 index e09e0a624f..0000000000 --- a/var/spack/packages/ompss/package.py +++ /dev/null @@ -1,50 +0,0 @@ -from spack import * -import os -import glob - -# working config lines for ompss 14.06 : -#./nanox-0.7/config.log: $ ./configure --prefix=/usr/gapps/exmatex/ompss --with-mcc=/usr/gapps/exmatex/ompss/ --with-hwloc=/usr -#./mcxx-1.99.2/config.log: $ ./configure --prefix=/usr/gapps/exmatex/ompss --with-nanox=/usr/gapps/exmatex/ompss --enable-ompss --with-mpi=/opt/mvapich2-intel-shmem-1.7 --enable-tl-openmp-profile --enable-tl-openmp-intel - -class Ompss(Package): - """OmpSs is an effort to integrate features from the StarSs - programming model developed by BSC into a single programming - model. In particular, our objective is to extend OpenMP with - new directives to support asynchronous parallelism and - heterogeneity (devices like GPUs). However, it can also be - understood as new directives extending other accelerator based - APIs like CUDA or OpenCL. Our OmpSs environment is built on top - of our Mercurium compiler and Nanos++ runtime system.""" - homepage = "http://pm.bsc.es/" - url = "http://pm.bsc.es/sites/default/files/ftp/ompss/releases/ompss-14.10.tar.gz" - list_url = 'http://pm.bsc.es/ompss-downloads' - - version('14.10', '404d161265748f2f96bb35fd8c7e79ee') - - # all dependencies are optional, really - depends_on("mpi") - #depends_on("openmp") - depends_on("hwloc") - depends_on("extrae") - - def install(self, spec, prefix): - if 'openmpi' in spec: - mpi = spec['openmpi'] - elif 'mpich' in spec: - mpi = spec['mpich'] - elif 'mvapich' in spec: - mpi = spec['mvapich'] - - openmp_options = ["--enable-tl-openmp-profile"] - if spec.satisfies('%intel'): - openmp_options.append( "--enable-tl-openmp-intel" ) - - os.chdir(glob.glob('./nanox-*').pop()) - configure("--prefix=%s" % prefix, "--with-mcc=%s" % prefix, "--with-extrae=%s" % spec['extrae'].prefix, "--with-hwloc=%s" % spec['hwloc'].prefix) - make() - make("install") - - os.chdir(glob.glob('../mcxx-*').pop()) - configure("--prefix=%s" % prefix, "--with-nanox=%s" % prefix, "--enable-ompss", "--with-mpi=%s" % mpi.prefix, *openmp_options) - make() - make("install") diff --git a/var/spack/packages/ompt-openmp/package.py b/var/spack/packages/ompt-openmp/package.py deleted file mode 100644 index 5d380ebd77..0000000000 --- a/var/spack/packages/ompt-openmp/package.py +++ /dev/null @@ -1,23 +0,0 @@ -from spack import * - -class OmptOpenmp(Package): - """LLVM/Clang OpenMP runtime with OMPT support. This is a fork of the OpenMPToolsInterface/LLVM-openmp fork of the official LLVM OpenMP mirror. This library provides a drop-in replacement of the OpenMP runtimes for GCC, Intel and LLVM/Clang.""" - homepage = "https://github.com/OpenMPToolsInterface/LLVM-openmp" - url = "http://github.com/khuck/LLVM-openmp/archive/v0.1-spack.tar.gz" - - version('spack', '35227b2726e377faa433fc841226e036') - - # depends_on("foo") - - def install(self, spec, prefix): - with working_dir("runtime/build", create=True): - - # FIXME: Modify the configure line to suit your build system here. - cmake('-DCMAKE_C_COMPILER=%s' % self.compiler.cc, - '-DCMAKE_CXX_COMPILER=%s' % self.compiler.cxx, - '-DCMAKE_INSTALL_PREFIX=%s' % prefix, - '..', *std_cmake_args) - - # FIXME: Add logic to build and install here - make() - make("install") diff --git a/var/spack/packages/opari2/package.py b/var/spack/packages/opari2/package.py deleted file mode 100644 index daaee61e3a..0000000000 --- a/var/spack/packages/opari2/package.py +++ /dev/null @@ -1,65 +0,0 @@ -# FIXME: Add copyright statement here - -from spack import * -from contextlib import closing - -class Opari2(Package): - """OPARI2 is a source-to-source instrumentation tool for OpenMP and - hybrid codes. It surrounds OpenMP directives and runtime library - calls with calls to the POMP2 measurement interface. - OPARI2 will provide you with a new initialization method that allows - for multi-directory and parallel builds as well as the usage of - pre-instrumented libraries. Furthermore, an efficient way of - tracking parent-child relationships was added. Additionally, we - extended OPARI2 to support instrumentation of OpenMP 3.0 - tied tasks. """ - - homepage = "http://www.vi-hps.org/projects/score-p" - url = "http://www.vi-hps.org/upload/packages/opari2/opari2-1.1.2.tar.gz" - - version('1.1.2', '9a262c7ca05ff0ab5f7775ae96f3539e') - - backend_user_provided = """\ -CC=cc -CXX=c++ -F77=f77 -FC=f90 -CFLAGS=-fPIC -CXXFLAGS=-fPIC -""" - frontend_user_provided = """\ -CC_FOR_BUILD=cc -CXX_FOR_BUILD=c++ -F77_FOR_BUILD=f70 -FC_FOR_BUILD=f90 -CFLAGS_FOR_BUILD=-fPIC -CXXFLAGS_FOR_BUILD=-fPIC -""" - mpi_user_provided = """\ -MPICC=mpicc -MPICXX=mpicxx -MPIF77=mpif77 -MPIFC=mpif90 -MPI_CFLAGS=-fPIC -MPI_CXXFLAGS=-fPIC -""" - - def install(self, spec, prefix): - # Use a custom compiler configuration, otherwise the score-p - # build system messes with spack's compiler settings. - # Create these three files in the build directory - with closing(open("platform-backend-user-provided", "w")) as backend_file: - backend_file.write(self.backend_user_provided) - with closing(open("platform-frontend-user-provided", "w")) as frontend_file: - frontend_file.write(self.frontend_user_provided) - with closing(open("platform-mpi-user-provided", "w")) as mpi_file: - mpi_file.write(self.mpi_user_provided) - - # FIXME: Modify the configure line to suit your build system here. - configure("--prefix=%s" % prefix, - "--with-custom-compilers", - "--enable-shared") - - # FIXME: Add logic to build and install here - make() - make("install") diff --git a/var/spack/packages/openmpi/ad_lustre_rwcontig_open_source.patch b/var/spack/packages/openmpi/ad_lustre_rwcontig_open_source.patch deleted file mode 100644 index daa825ccbe..0000000000 --- a/var/spack/packages/openmpi/ad_lustre_rwcontig_open_source.patch +++ /dev/null @@ -1,11 +0,0 @@ ---- a/ompi/mca/io/romio/romio/adio/ad_lustre/ad_lustre_rwcontig.c 2013-12-10 12:05:44.806417000 -0800 -+++ b/ompi/mca/io/romio/romio/adio/ad_lustre/ad_lustre_rwcontig.c 2013-12-10 11:53:03.295622000 -0800 -@@ -8,7 +8,7 @@ - * Copyright (C) 2008 Sun Microsystems, Lustre group - */ - --#define _XOPEN_SOURCE 600 -+//#define _XOPEN_SOURCE 600 - #include - #include - #include "ad_lustre.h" diff --git a/var/spack/packages/openmpi/llnl-platforms.patch b/var/spack/packages/openmpi/llnl-platforms.patch deleted file mode 100644 index f515743c4d..0000000000 --- a/var/spack/packages/openmpi/llnl-platforms.patch +++ /dev/null @@ -1,151 +0,0 @@ -diff -Nuar openmpi-1.6.5.orig/contrib/platform/llnl/optimized openmpi-1.6.5.llnl/contrib/platform/llnl/optimized ---- openmpi-1.6.5.orig/contrib/platform/llnl/optimized 1969-12-31 16:00:00.000000000 -0800 -+++ openmpi-1.6.5.llnl/contrib/platform/llnl/optimized 2013-08-08 23:47:12.704029000 -0700 -@@ -0,0 +1,29 @@ -+enable_dlopen=no -+enable_mem_debug=no -+enable_mem_profile=no -+enable_debug_symbols=no -+enable_binaries=yes -+enable_heterogeneous=no -+enable_debug=no -+enable_shared=yes -+enable_static=yes -+enable_memchecker=no -+enable_ipv6=no -+enable_mpi_f77=yes -+enable_mpi_f90=yes -+enable_mpi_cxx=yes -+enable_mpi_cxx_seek=yes -+enable_cxx_exceptions=no -+enable_ft_thread=no -+enable_per_user_config_files=no -+enable_mca_no_build=carto,crs,filem,routed-linear,snapc,pml-dr,pml-crcp2,pml-crcpw,pml-v,pml-example,crcp,btl-tcp -+enable_contrib_no_build=libnbc,vt -+with_slurm=yes -+with_pmi=yes -+with_tm=no -+with_openib=yes -+with_psm=yes -+with_devel_headers=yes -+with_io_romio_flags=--with-file-system=ufs+nfs+lustre -+with_memory_manager=ptmalloc2 -+with_valgrind=no -diff -Nuar openmpi-1.6.5.orig/contrib/platform/llnl/optimized.conf openmpi-1.6.5.llnl/contrib/platform/llnl/optimized.conf ---- openmpi-1.6.5.orig/contrib/platform/llnl/optimized.conf 1969-12-31 16:00:00.000000000 -0800 -+++ openmpi-1.6.5.llnl/contrib/platform/llnl/optimized.conf 2013-08-08 23:43:52.907553000 -0700 -@@ -0,0 +1,114 @@ -+# -+# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana -+# University Research and Technology -+# Corporation. All rights reserved. -+# Copyright (c) 2004-2005 The University of Tennessee and The University -+# of Tennessee Research Foundation. All rights -+# reserved. -+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, -+# University of Stuttgart. All rights reserved. -+# Copyright (c) 2004-2005 The Regents of the University of California. -+# All rights reserved. -+# Copyright (c) 2006 Cisco Systems, Inc. All rights reserved. -+# Copyright (c) 2011 Los Alamos National Security, LLC. -+# All rights reserved. -+# $COPYRIGHT$ -+# -+# Additional copyrights may follow -+# -+# $HEADER$ -+# -+ -+# This is the default system-wide MCA parameters defaults file. -+# Specifically, the MCA parameter "mca_param_files" defaults to a -+# value of -+# "$HOME/.openmpi/mca-params.conf:$sysconf/openmpi-mca-params.conf" -+# (this file is the latter of the two). So if the default value of -+# mca_param_files is not changed, this file is used to set system-wide -+# MCA parameters. This file can therefore be used to set system-wide -+# default MCA parameters for all users. Of course, users can override -+# these values if they want, but this file is an excellent location -+# for setting system-specific MCA parameters for those users who don't -+# know / care enough to investigate the proper values for them. -+ -+# Note that this file is only applicable where it is visible (in a -+# filesystem sense). Specifically, MPI processes each read this file -+# during their startup to determine what default values for MCA -+# parameters should be used. mpirun does not bundle up the values in -+# this file from the node where it was run and send them to all nodes; -+# the default value decisions are effectively distributed. Hence, -+# these values are only applicable on nodes that "see" this file. If -+# $sysconf is a directory on a local disk, it is likely that changes -+# to this file will need to be propagated to other nodes. If $sysconf -+# is a directory that is shared via a networked filesystem, changes to -+# this file will be visible to all nodes that share this $sysconf. -+ -+# The format is straightforward: one per line, mca_param_name = -+# rvalue. Quoting is ignored (so if you use quotes or escape -+# characters, they'll be included as part of the value). For example: -+ -+# Disable run-time MPI parameter checking -+# mpi_param_check = 0 -+ -+# Note that the value "~/" will be expanded to the current user's home -+# directory. For example: -+ -+# Change component loading path -+# component_path = /usr/local/lib/openmpi:~/my_openmpi_components -+ -+# See "ompi_info --param all all" for a full listing of Open MPI MCA -+# parameters available and their default values. -+# -+ -+# Basic behavior to smooth startup -+mca_component_show_load_errors = 0 -+orte_abort_timeout = 10 -+opal_set_max_sys_limits = 1 -+orte_report_launch_progress = 1 -+ -+# Define timeout for daemons to report back during launch -+orte_startup_timeout = 10000 -+ -+## Protect the shared file systems -+orte_no_session_dirs = /p,/usr/local,/usr/global,/nfs/tmp1,/nfs/tmp2 -+orte_tmpdir_base = /tmp -+ -+## Require an allocation to run - protects the frontend -+## from inadvertent job executions -+orte_allocation_required = 1 -+ -+## MPI behavior -+## Do NOT specify mpi_leave_pinned so system -+## can figure out for itself whether or not -+## it is supported and usable -+orte_notifier = syslog -+ -+## Add the interface for out-of-band communication -+## and set it up -+oob_tcp_if_include=ib0 -+oob_tcp_peer_retries = 1000 -+oob_tcp_disable_family = IPv6 -+oob_tcp_listen_mode = listen_thread -+oob_tcp_sndbuf = 32768 -+oob_tcp_rcvbuf = 32768 -+ -+## Define the MPI interconnects -+btl = sm,openib,self -+ -+## We are using the PSM MTL by default -+## There can only be one! -+pml = cm -+ -+## Setup OpenIB - just in case -+btl_openib_want_fork_support = 0 -+btl_openib_cpc_include = oob -+btl_openib_receive_queues = S,4096,1024:S,12288,512:S,65536,512 -+ -+## Enable cpu affinity -+opal_paffinity_alone = 1 -+ -+## Setup MPI options -+mpi_show_handle_leaks = 0 -+mpi_warn_on_fork = 1 -+mpi_abort_print_stack = 0 -+ diff --git a/var/spack/packages/openmpi/package.py b/var/spack/packages/openmpi/package.py deleted file mode 100644 index 5e429dedf5..0000000000 --- a/var/spack/packages/openmpi/package.py +++ /dev/null @@ -1,109 +0,0 @@ -import os - -from spack import * - - -class Openmpi(Package): - """Open MPI is a project combining technologies and resources from - several other projects (FT-MPI, LA-MPI, LAM/MPI, and PACX-MPI) - in order to build the best MPI library available. A completely - new MPI-2 compliant implementation, Open MPI offers advantages - for system and software vendors, application developers and - computer science researchers. - """ - - homepage = "http://www.open-mpi.org" - - version('1.10.0', '280cf952de68369cebaca886c5ce0304', - url = "http://www.open-mpi.org/software/ompi/v1.10/downloads/openmpi-1.10.0.tar.bz2") - version('1.8.8', '0dab8e602372da1425e9242ae37faf8c', - url = 'http://www.open-mpi.org/software/ompi/v1.8/downloads/openmpi-1.8.8.tar.bz2') - version('1.6.5', '03aed2a4aa4d0b27196962a2a65fc475', - url = "http://www.open-mpi.org/software/ompi/v1.6/downloads/openmpi-1.6.5.tar.bz2") - - patch('ad_lustre_rwcontig_open_source.patch', when="@1.6.5") - patch('llnl-platforms.patch', when="@1.6.5") - - provides('mpi@:2.2', when='@1.6.5') # Open MPI 1.6.5 supports MPI-2.2 - provides('mpi@:3.0', when='@1.8.8') # Open MPI 1.8.8 supports MPI-3.0 - provides('mpi@:3.0', when='@1.10.0') # Open MPI 1.10.0 supports MPI-3.0 - - - def setup_dependent_environment(self, module, spec, dep_spec): - """For dependencies, make mpicc's use spack wrapper.""" - os.environ['OMPI_CC'] = 'cc' - os.environ['OMPI_CXX'] = 'c++' - os.environ['OMPI_FC'] = 'f90' - os.environ['OMPI_F77'] = 'f77' - - - def install(self, spec, prefix): - config_args = ["--prefix=%s" % prefix] - - # TODO: use variants for this, e.g. +lanl, +llnl, etc. - # use this for LANL builds, but for LLNL builds, we need: - # "--with-platform=contrib/platform/llnl/optimized" - if self.version == ver("1.6.5") and '+lanl' in spec: - config_args.append("--with-platform=contrib/platform/lanl/tlcc2/optimized-nopanasas") - - # TODO: Spack should make it so that you can't actually find - # these compilers if they're "disabled" for the current - # compiler configuration. - if not self.compiler.f77 and not self.compiler.fc: - config_args.append("--enable-mpi-fortran=no") - - configure(*config_args) - make() - make("install") - - self.filter_compilers() - - - def filter_compilers(self): - """Run after install to make the MPI compilers use the - compilers that Spack built the package with. - - If this isn't done, they'll have CC, CXX and FC set - to Spack's generic cc, c++ and f90. We want them to - be bound to whatever compiler they were built with. - """ - kwargs = { 'ignore_absent' : True, 'backup' : False, 'string' : False } - dir = os.path.join(self.prefix, 'share/openmpi/') - - cc_wrappers = ['mpicc-vt-wrapper-data.txt', 'mpicc-wrapper-data.txt', - 'ortecc-wrapper-data.txt', 'shmemcc-wrapper-data.txt'] - - cxx_wrappers = ['mpic++-vt-wrapper-data.txt', 'mpic++-wrapper-data.txt', - 'ortec++-wrapper-data.txt'] - - fc_wrappers = ['mpifort-vt-wrapper-data.txt', - 'mpifort-wrapper-data.txt', 'shmemfort-wrapper-data.txt'] - - for wrapper in cc_wrappers: - filter_file('compiler=.*', 'compiler=%s' % self.compiler.cc, - os.path.join(dir, wrapper), **kwargs) - - for wrapper in cxx_wrappers: - filter_file('compiler=.*', 'compiler=%s' % self.compiler.cxx, - os.path.join(dir, wrapper), **kwargs) - - for wrapper in fc_wrappers: - filter_file('compiler=.*', 'compiler=%s' % self.compiler.fc, - os.path.join(dir, wrapper), **kwargs) - - # These are symlinks in newer versions, so check that here - f77_wrappers = ['mpif77-vt-wrapper-data.txt', 'mpif77-wrapper-data.txt'] - f90_wrappers = ['mpif90-vt-wrapper-data.txt', 'mpif90-wrapper-data.txt'] - - for wrapper in f77_wrappers: - path = os.path.join(dir, wrapper) - if not os.path.islink(path): - filter_file('compiler=.*', 'compiler=%s' % self.compiler.f77, - path, **kwargs) - for wrapper in f90_wrappers: - path = os.path.join(dir, wrapper) - if not os.path.islink(path): - filter_file('compiler=.*', 'compiler=%s' % self.compiler.fc, - path, **kwargs) - - diff --git a/var/spack/packages/openssl/package.py b/var/spack/packages/openssl/package.py deleted file mode 100644 index c5a8aeb9dc..0000000000 --- a/var/spack/packages/openssl/package.py +++ /dev/null @@ -1,26 +0,0 @@ -from spack import * - -class Openssl(Package): - """The OpenSSL Project is a collaborative effort to develop a - robust, commercial-grade, full-featured, and Open Source - toolkit implementing the Secure Sockets Layer (SSL v2/v3) and - Transport Layer Security (TLS v1) protocols as well as a - full-strength general purpose cryptography library.""" - homepage = "http://www.openssl.org" - url = "http://www.openssl.org/source/openssl-1.0.1h.tar.gz" - - version('1.0.1h', '8d6d684a9430d5cc98a62a5d8fbda8cf') - - depends_on("zlib") - parallel = False - - def install(self, spec, prefix): - config = Executable("./config") - config("--prefix=%s" % prefix, - "--openssldir=%s/etc/openssl" % prefix, - "zlib", - "no-krb5", - "shared") - - make() - make("install") diff --git a/var/spack/packages/otf/package.py b/var/spack/packages/otf/package.py deleted file mode 100644 index 52893dd265..0000000000 --- a/var/spack/packages/otf/package.py +++ /dev/null @@ -1,21 +0,0 @@ -from spack import * - -class Otf(Package): - """To improve scalability for very large and massively parallel - traces the Open Trace Format (OTF) is developed at ZIH as a - successor format to the Vampir Trace Format (VTF3).""" - - homepage = "http://tu-dresden.de/die_tu_dresden/zentrale_einrichtungen/zih/forschung/projekte/otf/index_html/document_view?set_language=en" - url = "http://wwwpub.zih.tu-dresden.de/%7Emlieber/dcount/dcount.php?package=otf&get=OTF-1.12.5salmon.tar.gz" - - version('1.12.5salmon', 'bf260198633277031330e3356dcb4eec') - - depends_on('zlib') - - def install(self, spec, prefix): - configure('--prefix=%s' % prefix, - '--without-vtf3', - '--with-zlib', - '--with-zlibsymbols') - make() - make("install") diff --git a/var/spack/packages/otf2/package.py b/var/spack/packages/otf2/package.py deleted file mode 100644 index fa0a5898b6..0000000000 --- a/var/spack/packages/otf2/package.py +++ /dev/null @@ -1,74 +0,0 @@ -# FIXME: Add copyright - -from spack import * -from contextlib import closing -import os - -class Otf2(Package): - """The Open Trace Format 2 is a highly scalable, memory efficient event - trace data format plus support library.""" - - homepage = "http://www.vi-hps.org/score-p" - url = "http://www.vi-hps.org/upload/packages/otf2/otf2-1.4.tar.gz" - - version('1.4', 'a23c42e936eb9209c4e08b61c3cf5092', - url="http://www.vi-hps.org/upload/packages/otf2/otf2-1.4.tar.gz") - version('1.3.1', 'd0ffc4e858455ace4f596f910e68c9f2', - url="http://www.vi-hps.org/upload/packages/otf2/otf2-1.3.1.tar.gz") - version('1.2.1', '8fb3e11fb7489896596ae2c7c83d7fc8', - url="http://www.vi-hps.org/upload/packages/otf2/otf2-1.2.1.tar.gz") - - backend_user_provided = """\ -CC=cc -CXX=c++ -F77=f77 -FC=f90 -CFLAGS=-fPIC -CXXFLAGS=-fPIC -""" - frontend_user_provided = """\ -CC_FOR_BUILD=cc -CXX_FOR_BUILD=c++ -F77_FOR_BUILD=f70 -FC_FOR_BUILD=f90 -CFLAGS_FOR_BUILD=-fPIC -CXXFLAGS_FOR_BUILD=-fPIC -""" - mpi_user_provided = """\ -MPICC=cc -MPICXX=c++ -MPIF77=f77 -MPIFC=f90 -MPI_CFLAGS=-fPIC -MPI_CXXFLAGS=-fPIC -""" - - @when('@:1.2.1') - def version_specific_args(self): - return ["--with-platform=disabled", "CC=cc", "CXX=c++", "F77=f77", "F90=f90", "CFLAGS=-fPIC", "CXXFLAGS=-fPIC"] - - @when('@1.3:') - def version_specific_args(self): - # TODO: figure out what scorep's build does as of otf2 1.3 - return ["--with-custom-compilers"] - - def install(self, spec, prefix): - # Use a custom compiler configuration, otherwise the score-p - # build system messes with spack's compiler settings. - # Create these three files in the build directory - with closing(open("platform-backend-user-provided", "w")) as backend_file: - backend_file.write(self.backend_user_provided) - with closing(open("platform-frontend-user-provided", "w")) as frontend_file: - frontend_file.write(self.frontend_user_provided) - with closing(open("platform-mpi-user-provided", "w")) as mpi_file: - mpi_file.write(self.mpi_user_provided) - - configure_args=["--prefix=%s" % prefix, - "--enable-shared"] - - configure_args.extend(self.version_specific_args()) - - configure(*configure_args) - - make() - make("install") diff --git a/var/spack/packages/pango/package.py b/var/spack/packages/pango/package.py deleted file mode 100644 index df43625bf5..0000000000 --- a/var/spack/packages/pango/package.py +++ /dev/null @@ -1,19 +0,0 @@ -from spack import * - -class Pango(Package): - """Pango is a library for laying out and rendering of text, with - an emphasis on internationalization. It can be used anywhere - that text layout is needed, though most of the work on Pango so - far has been done in the context of the GTK+ widget toolkit.""" - homepage = "http://www.pango.org" - url = "http://ftp.gnome.org/pub/gnome/sources/pango/1.36/pango-1.36.8.tar.xz" - - version('1.36.8', '217a9a753006275215fa9fa127760ece') - - depends_on("harfbuzz") - depends_on("cairo") - - def install(self, spec, prefix): - configure("--prefix=%s" % prefix) - make() - make("install") diff --git a/var/spack/packages/papi/package.py b/var/spack/packages/papi/package.py deleted file mode 100644 index 596f7114d6..0000000000 --- a/var/spack/packages/papi/package.py +++ /dev/null @@ -1,35 +0,0 @@ -from spack import * -import os - -class Papi(Package): - """PAPI provides the tool designer and application engineer with a - consistent interface and methodology for use of the performance - counter hardware found in most major microprocessors. PAPI - enables software engineers to see, in near real time, the - relation between software performance and processor events. In - addition Component PAPI provides access to a collection of - components that expose performance measurement opportunites - across the hardware and software stack.""" - homepage = "http://icl.cs.utk.edu/papi/index.html" - url = "http://icl.cs.utk.edu/projects/papi/downloads/papi-5.3.0.tar.gz" - - version('5.3.0', '367961dd0ab426e5ae367c2713924ffb') - - def install(self, spec, prefix): - os.chdir("src/") - - configure_args=["--prefix=%s" % prefix] - - # need to force consistency in the use of compilers - if spec.satisfies('%gcc'): - configure_args.append('CC=gcc') - configure_args.append('MPICH_CC=gcc') - if spec.satisfies('%intel'): - configure_args.append('CC=icc') - configure_args.append('MPICH_CC=icc') - - configure(*configure_args) - - make() - make("install") - diff --git a/var/spack/packages/paraver/package.py b/var/spack/packages/paraver/package.py deleted file mode 100644 index 5f8a153d4c..0000000000 --- a/var/spack/packages/paraver/package.py +++ /dev/null @@ -1,41 +0,0 @@ -from spack import * -import os - -class Paraver(Package): - """"A very powerful performance visualization and analysis tool - based on traces that can be used to analyse any information that - is expressed on its input trace format. Traces for parallel MPI, - OpenMP and other programs can be genereated with Extrae.""" - homepage = "http://www.bsc.es/computer-sciences/performance-tools/paraver" - url = "http://www.bsc.es/ssl/apps/performanceTools/files/paraver-sources-4.5.3.tar.gz" - - version('4.5.3', '625de9ec0d639acd18d1aaa644b38f72') - - depends_on("boost") - #depends_on("extrae") - depends_on("wx") - depends_on("wxpropgrid") - - def install(self, spec, prefix): - os.chdir("ptools_common_files") - configure("--prefix=%s" % prefix) - make() - make("install") - - os.chdir("../paraver-kernel") - #"--with-extrae=%s" % spec['extrae'].prefix, - configure("--prefix=%s" % prefix, "--with-ptools-common-files=%s" % prefix, "--with-boost=%s" % spec['boost'].prefix, "--with-boost-serialization=boost_serialization") - make() - make("install") - - os.chdir("../paraver-toolset") - configure("--prefix=%s" % prefix) - make() - make("install") - - os.chdir("../wxparaver") - #"--with-extrae=%s" % spec['extrae'].prefix, - configure("--prefix=%s" % prefix, "--with-paraver=%s" % prefix, "--with-boost=%s" % spec['boost'].prefix, "--with-boost-serialization=boost_serialization", "--with-wxdir=%s" % spec['wx'].prefix.bin) - make() - make("install") - diff --git a/var/spack/packages/paraview/package.py b/var/spack/packages/paraview/package.py deleted file mode 100644 index a0ff812ca2..0000000000 --- a/var/spack/packages/paraview/package.py +++ /dev/null @@ -1,72 +0,0 @@ -from spack import * - -class Paraview(Package): - homepage = 'http://www.paraview.org' - url = 'http://www.paraview.org/files/v4.4/ParaView-v4.4.0-source.tar.gz' - - version('4.4.0', 'fa1569857dd680ebb4d7ff89c2227378', url='http://www.paraview.org/files/v4.4/ParaView-v4.4.0-source.tar.gz') - - variant('python', default=False, description='Enable Python support') - variant('matplotlib', default=False, description='Enable Matplotlib support') - variant('numpy', default=False, description='Enable NumPy support') - - variant('tcl', default=False, description='Enable TCL support') - - variant('mpi', default=False, description='Enable MPI support') - - variant('osmesa', default=False, description='Enable OSMesa support') - variant('qt', default=False, description='Enable Qt support') - - depends_on('python', when='+python') - depends_on('py-numpy', when='+python+numpy') - depends_on('py-matplotlib', when='+python+matplotlib') - depends_on('tcl', when='+tcl') - depends_on('mpi', when='+mpi') - depends_on('qt', when='+qt') - - depends_on('bzip2') - depends_on('freetype') - depends_on('hdf5') # drags in mpi - depends_on('jpeg') - depends_on('libpng') - depends_on('libtiff') - #depends_on('libxml2') # drags in python - depends_on('netcdf') - #depends_on('protobuf') # version mismatches? - #depends_on('sqlite') # external version not supported - depends_on('zlib') - - def install(self, spec, prefix): - with working_dir('spack-build', create=True): - def feature_to_bool(feature, on='ON', off='OFF'): - if feature in spec: - return on - return off - - def nfeature_to_bool(feature): - return feature_to_bool(feature, on='OFF', off='ON') - - feature_args = std_cmake_args[:] - feature_args.append('-DPARAVIEW_BUILD_QT_GUI:BOOL=%s' % feature_to_bool('+qt')) - feature_args.append('-DPARAVIEW_ENABLE_PYTHON:BOOL=%s' % feature_to_bool('+python')) - feature_args.append('-DPARAVIEW_USE_MPI:BOOL=%s' % feature_to_bool('+mpi')) - feature_args.append('-DVTK_ENABLE_TCL_WRAPPING:BOOL=%s' % feature_to_bool('+tcl')) - feature_args.append('-DVTK_OPENGL_HAS_OSMESA:BOOL=%s' % feature_to_bool('+osmesa')) - feature_args.append('-DVTK_USE_X:BOOL=%s' % nfeature_to_bool('+osmesa')) - feature_args.append('-DVTK_RENDERING_BACKEND:STRING=%s' % feature_to_bool('+opengl2', 'OpenGL2', 'OpenGL')) - - feature_args.extend(std_cmake_args) - - cmake('..', - '-DCMAKE_INSTALL_PREFIX:PATH=%s' % prefix, - '-DBUILD_TESTING:BOOL=OFF', - '-DVTK_USER_SYSTEM_FREETYPE:BOOL=ON', - '-DVTK_USER_SYSTEM_HDF5:BOOL=ON', - '-DVTK_USER_SYSTEM_JPEG:BOOL=ON', - #'-DVTK_USER_SYSTEM_LIBXML2:BOOL=ON', - '-DVTK_USER_SYSTEM_NETCDF:BOOL=ON', - '-DVTK_USER_SYSTEM_TIFF:BOOL=ON', - '-DVTK_USER_SYSTEM_ZLIB:BOOL=ON', - *feature_args) - make() - make('install') diff --git a/var/spack/packages/parmetis/package.py b/var/spack/packages/parmetis/package.py deleted file mode 100644 index d8cd337304..0000000000 --- a/var/spack/packages/parmetis/package.py +++ /dev/null @@ -1,26 +0,0 @@ -from spack import * - -class Parmetis(Package): - """ParMETIS is an MPI-based parallel library that implements a - variety of algorithms for partitioning unstructured graphs, - meshes, and for computing fill-reducing orderings of sparse - matrices.""" - homepage = "http://glaros.dtc.umn.edu/gkhome/metis/parmetis/overview" - url = "http://glaros.dtc.umn.edu/gkhome/fetch/sw/parmetis/parmetis-4.0.3.tar.gz" - - version('4.0.3', 'f69c479586bf6bb7aff6a9bc0c739628') - - depends_on('mpi') - - def install(self, spec, prefix): - cmake(".", - '-DGKLIB_PATH=%s/metis/GKlib' % pwd(), - '-DMETIS_PATH=%s/metis' % pwd(), - '-DSHARED=1', - '-DCMAKE_C_COMPILER=mpicc', - '-DCMAKE_CXX_COMPILER=mpicxx', - '-DSHARED=1', - *std_cmake_args) - - make() - make("install") diff --git a/var/spack/packages/parpack/package.py b/var/spack/packages/parpack/package.py deleted file mode 100644 index 622aceca04..0000000000 --- a/var/spack/packages/parpack/package.py +++ /dev/null @@ -1,43 +0,0 @@ -from spack import * -import os -import shutil - -class Parpack(Package): - """ARPACK is a collection of Fortran77 subroutines designed to solve large - scale eigenvalue problems.""" - - homepage = "http://www.caam.rice.edu/software/ARPACK/download.html" - url = "http://www.caam.rice.edu/software/ARPACK/SRC/parpack96.tar.Z" - - version('96', 'a175f70ff71837a33ff7e4b0b6054f43') - - depends_on('mpi') - depends_on('blas') - depends_on('lapack') - - def patch(self): - # Filter the CJ makefile to make a spack one. - shutil.move('ARMAKES/ARmake.CJ', 'ARmake.inc') - mf = FileFilter('ARmake.inc') - - # Be sure to use Spack F77 wrapper - mf.filter('^FC.*', 'FC = f77') - mf.filter('^FFLAGS.*', 'FFLAGS = -O2 -g') - - # Set up some variables. - mf.filter('^PLAT.*', 'PLAT = ') - mf.filter('^home.*', 'home = %s' % os.getcwd()) - mf.filter('^BLASdir.*', 'BLASdir = %s' % self.spec['blas'].prefix) - mf.filter('^LAPACKdir.*', 'LAPACKdir = %s' % self.spec['lapack'].prefix) - mf.filter('^MAKE.*', 'MAKE = make') - - # build the library in our own prefix. - mf.filter('^ARPACKLIB.*', 'PARPACKLIB = %s/libparpack.a' % os.getcwd()) - - - def install(self, spec, prefix): - with working_dir('PARPACK/SRC/MPI'): - make('all') - - mkdirp(prefix.lib) - install('libparpack.a', prefix.lib) diff --git a/var/spack/packages/pcre/package.py b/var/spack/packages/pcre/package.py deleted file mode 100644 index 3424048a6c..0000000000 --- a/var/spack/packages/pcre/package.py +++ /dev/null @@ -1,15 +0,0 @@ -from spack import * - -class Pcre(Package): - """The PCRE package contains Perl Compatible Regular Expression - libraries. These are useful for implementing regular expression - pattern matching using the same syntax and semantics as Perl 5.""" - homepage = "http://www.pcre.org""" - url = "ftp://ftp.csx.cam.ac.uk/pub/software/programming/pcre/pcre-8.36.tar.bz2" - - version('8.36', 'b767bc9af0c20bc9c1fe403b0d41ad97') - - def install(self, spec, prefix): - configure("--prefix=%s" % prefix) - make() - make("install") diff --git a/var/spack/packages/petsc/package.py b/var/spack/packages/petsc/package.py deleted file mode 100644 index 4864e39bf1..0000000000 --- a/var/spack/packages/petsc/package.py +++ /dev/null @@ -1,40 +0,0 @@ -from spack import * - -class Petsc(Package): - """PETSc is a suite of data structures and routines for the - scalable (parallel) solution of scientific applications modeled by - partial differential equations.""" - - homepage = "http://www.mcs.anl.gov/petsc/index.html" - url = "http://ftp.mcs.anl.gov/pub/petsc/release-snapshots/petsc-3.5.3.tar.gz" - - version('3.5.3', 'd4fd2734661e89f18ac6014b5dd1ef2f') - version('3.5.2', 'ad170802b3b058b5deb9cd1f968e7e13') - version('3.5.1', 'a557e029711ebf425544e117ffa44d8f') - - depends_on("boost") - depends_on("blas") - depends_on("lapack") - depends_on("hypre") - depends_on("parmetis") - depends_on("metis") - depends_on("hdf5") - depends_on("mpi") - - def install(self, spec, prefix): - configure("--prefix=%s" % prefix, - "CC=cc", - "CXX=c++", - "FC=f90", - "--with-blas-lib=%s/libblas.a" % spec['blas'].prefix.lib, - "--with-lapack-lib=%s/liblapack.a" % spec['lapack'].prefix.lib, - "--with-boost-dir=%s" % spec['boost'].prefix, - "--with-hypre-dir=%s" % spec['hypre'].prefix, - "--with-parmetis-dir=%s" % spec['parmetis'].prefix, - "--with-metis-dir=%s" % spec['metis'].prefix, - "--with-hdf5-dir=%s" % spec['hdf5'].prefix, - "--with-shared-libraries=0") - - # PETSc has its own way of doing parallel make. - make('MAKE_NP=%s' % make_jobs, parallel=False) - make("install") diff --git a/var/spack/packages/pidx/package.py b/var/spack/packages/pidx/package.py deleted file mode 100644 index 81aed62fb1..0000000000 --- a/var/spack/packages/pidx/package.py +++ /dev/null @@ -1,21 +0,0 @@ -from spack import * - -class Pidx(Package): - """PIDX Parallel I/O Library. - - PIDX is an efficient parallel I/O library that reads and writes - multiresolution IDX data files. - """ - - homepage = "http://www.cedmav.com/pidx" - - version('1.0', git='https://github.com/sci-visus/PIDX.git', - commit='6afa1cf71d1c41263296dc049c8fabaf73c296da') - - depends_on("mpi") - - def install(self, spec, prefix): - with working_dir('spack-build', create=True): - cmake('..', *std_cmake_args) - make() - make("install") diff --git a/var/spack/packages/pixman/package.py b/var/spack/packages/pixman/package.py deleted file mode 100644 index 895cbdbca5..0000000000 --- a/var/spack/packages/pixman/package.py +++ /dev/null @@ -1,18 +0,0 @@ -from spack import * - -class Pixman(Package): - """The Pixman package contains a library that provides low-level - pixel manipulation features such as image compositing and - trapezoid rasterization.""" - homepage = "http://www.pixman.org" - url = "http://cairographics.org/releases/pixman-0.32.6.tar.gz" - - version('0.32.6', '3a30859719a41bd0f5cccffbfefdd4c2') - - depends_on("libpng") - - def install(self, spec, prefix): - configure("--prefix=%s" % prefix, - "--disable-gtk") - make() - make("install") diff --git a/var/spack/packages/pkg-config/package.py b/var/spack/packages/pkg-config/package.py deleted file mode 100644 index 9964c6ce34..0000000000 --- a/var/spack/packages/pkg-config/package.py +++ /dev/null @@ -1,17 +0,0 @@ -from spack import * - -class PkgConfig(Package): - """pkg-config is a helper tool used when compiling applications and libraries""" - homepage = "http://www.freedesktop.org/wiki/Software/pkg-config/" - url = "http://pkgconfig.freedesktop.org/releases/pkg-config-0.28.tar.gz" - - version('0.28', 'aa3c86e67551adc3ac865160e34a2a0d') - - parallel = False - - def install(self, spec, prefix): - configure("--prefix=%s" %prefix, "--enable-shared") - - make() - make("install") - diff --git a/var/spack/packages/pmgr_collective/package.py b/var/spack/packages/pmgr_collective/package.py deleted file mode 100644 index 5d9b02acc3..0000000000 --- a/var/spack/packages/pmgr_collective/package.py +++ /dev/null @@ -1,37 +0,0 @@ -############################################################################## -# Copyright (c) 2013, Lawrence Livermore National Security, LLC. -# Produced at the Lawrence Livermore National Laboratory. -# -# This file is part of Spack. -# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. -# LLNL-CODE-647188 -# -# For details, see https://scalability-llnl.github.io/spack -# Please also see the LICENSE file for our notice and the LGPL. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License (as published by -# the Free Software Foundation) version 2.1 dated February 1999. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and -# conditions of the GNU General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -############################################################################## -from spack import * - -class PmgrCollective(Package): - """PMGR_COLLECTIVE provides a scalable network for bootstrapping - MPI jobs.""" - homepage = "http://www.sourceforge.net/projects/pmgrcollective" - url = "http://downloads.sourceforge.net/project/pmgrcollective/pmgrcollective/PMGR_COLLECTIVE-1.0/pmgr_collective-1.0.tgz" - - version('1.0', '0384d008774274cc3fc7b4d810dfd07e') - - def install(self, spec, prefix): - make('PREFIX="' + prefix + '"') - make('PREFIX="' + prefix + '"', "install") diff --git a/var/spack/packages/postgresql/package.py b/var/spack/packages/postgresql/package.py deleted file mode 100644 index 46922b7b71..0000000000 --- a/var/spack/packages/postgresql/package.py +++ /dev/null @@ -1,20 +0,0 @@ -from spack import * - -class Postgresql(Package): - """PostgreSQL is a powerful, open source object-relational - database system. It has more than 15 years of active - development and a proven architecture that has earned it a - strong reputation for reliability, data integrity, and - correctness.""" - homepage = "http://www.postgresql.org/" - url = "http://ftp.postgresql.org/pub/source/v9.3.4/postgresql-9.3.4.tar.bz2" - - version('9.3.4', 'd0a41f54c377b2d2fab4a003b0dac762') - - depends_on("openssl") - - def install(self, spec, prefix): - configure("--prefix=%s" % prefix, - "--with-openssl") - make() - make("install") diff --git a/var/spack/packages/ppl/package.py b/var/spack/packages/ppl/package.py deleted file mode 100644 index 018d5c523d..0000000000 --- a/var/spack/packages/ppl/package.py +++ /dev/null @@ -1,28 +0,0 @@ -from spack import * - -class Ppl(Package): - """The Parma Polyhedra Library (PPL) provides numerical - abstractions especially targeted at applications in the field of - analysis and verification of complex systems. These abstractions - include convex polyhedra, some special classes of polyhedra shapes - that offer interesting complexity/precision tradeoffs, and grids - which represent regularly spaced points that satisfy a set of - linear congruence relations. The library also supports finite - powersets and products of polyhedra and grids, a mixed integer - linear programming problem solver using an exact-arithmetic - version of the simplex algorithm, a parametric integer programming - solver, and primitives for termination analysis via the automatic - synthesis of linear ranking functions.""" - - homepage = "http://bugseng.com/products/ppl/" - url = "http://bugseng.com/products/ppl/download/ftp/releases/1.1/ppl-1.1.tar.gz" - - version('1.1', '4f2422c0ef3f409707af32108deb30a7') - - depends_on("gmp") - - def install(self, spec, prefix): - configure("--prefix=%s" % prefix, - "--with-gmp=%s" % spec['gmp'].prefix) - make() - make("install") diff --git a/var/spack/packages/protobuf/package.py b/var/spack/packages/protobuf/package.py deleted file mode 100644 index 34085c7ce9..0000000000 --- a/var/spack/packages/protobuf/package.py +++ /dev/null @@ -1,16 +0,0 @@ -import os -from spack import * - -class Protobuf(Package): - """Google's data interchange format.""" - - homepage = "https://developers.google.com/protocol-buffers" - url = "https://github.com/google/protobuf/releases/download/v2.5.0/protobuf-2.5.0.tar.bz2" - - version('2.5.0', 'a72001a9067a4c2c4e0e836d0f92ece4') - - def install(self, spec, prefix): - configure("--prefix=" + prefix) - make() - make("check") - make("install") diff --git a/var/spack/packages/py-basemap/package.py b/var/spack/packages/py-basemap/package.py deleted file mode 100644 index 45f1085ba1..0000000000 --- a/var/spack/packages/py-basemap/package.py +++ /dev/null @@ -1,20 +0,0 @@ -from spack import * -import os - -class PyBasemap(Package): - """The matplotlib basemap toolkit is a library for plotting 2D data on maps in Python.""" - homepage = "http://matplotlib.org/basemap/" - url = "https://downloads.sourceforge.net/project/matplotlib/matplotlib-toolkits/basemap-1.0.7/basemap-1.0.7.tar.gz" - - version('1.0.7', '48c0557ced9e2c6e440b28b3caff2de8') - - extends('python') - depends_on('py-setuptools') - depends_on('py-numpy') - depends_on('py-matplotlib') - depends_on('py-pil') - depends_on("geos") - - def install(self, spec, prefix): - env['GEOS_DIR'] = spec['geos'].prefix - python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-biopython/package.py b/var/spack/packages/py-biopython/package.py deleted file mode 100644 index 8ecaf48626..0000000000 --- a/var/spack/packages/py-biopython/package.py +++ /dev/null @@ -1,15 +0,0 @@ -from spack import * - -class PyBiopython(Package): - """It is a distributed collaborative effort to develop Python libraries and applications which address the needs of current and future work in bioinformatics.""" - homepage = "http://biopython.org/wiki/Main_Page" - url = "http://biopython.org/DIST/biopython-1.65.tar.gz" - - version('1.65', '143e7861ade85c0a8b5e2bbdd1da1f67') - - extends('python') - depends_on('py-mx') - depends_on('py-numpy') - - def install(self, spec, prefix): - python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-cffi/package.py b/var/spack/packages/py-cffi/package.py deleted file mode 100644 index a4d37483fe..0000000000 --- a/var/spack/packages/py-cffi/package.py +++ /dev/null @@ -1,17 +0,0 @@ -from spack import * - -class PyCffi(Package): - """Foreign Function Interface for Python calling C code""" - homepage = "http://cffi.readthedocs.org/en/latest/" - # base https://pypi.python.org/pypi/cffi - url = "https://pypi.python.org/packages/source/c/cffi/cffi-1.1.2.tar.gz#md5=" - - version('1.1.2', 'ca6e6c45b45caa87aee9adc7c796eaea') - - extends('python') - depends_on('py-setuptools') - depends_on('py-pycparser') - depends_on('libffi') - - def install(self, spec, prefix): - python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-cython/package.py b/var/spack/packages/py-cython/package.py deleted file mode 100644 index 68eb735ad9..0000000000 --- a/var/spack/packages/py-cython/package.py +++ /dev/null @@ -1,14 +0,0 @@ -from spack import * - -class PyCython(Package): - """The Cython compiler for writing C extensions for the Python language.""" - homepage = "https://pypi.python.org/pypi/cython" - url = "https://pypi.python.org/packages/source/C/Cython/cython-0.22.tar.gz" - - version('0.21.2', 'd21adb870c75680dc857cd05d41046a4') - version('0.22', '1ae25add4ef7b63ee9b4af697300d6b6') - - extends('python') - - def install(self, spec, prefix): - python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-dateutil/package.py b/var/spack/packages/py-dateutil/package.py deleted file mode 100644 index 0a17f2f2d2..0000000000 --- a/var/spack/packages/py-dateutil/package.py +++ /dev/null @@ -1,16 +0,0 @@ -from spack import * - -class PyDateutil(Package): - """Extensions to the standard Python datetime module.""" - homepage = "https://pypi.python.org/pypi/dateutil" - url = "https://pypi.python.org/packages/source/p/python-dateutil/python-dateutil-2.4.0.tar.gz" - - version('2.4.0', '75714163bb96bedd07685cdb2071b8bc') - version('2.4.2', '4ef68e1c485b09e9f034e10473e5add2') - - extends('python') - depends_on('py-setuptools') - depends_on('py-six') - - def install(self, spec, prefix): - python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-epydoc/package.py b/var/spack/packages/py-epydoc/package.py deleted file mode 100644 index af05510504..0000000000 --- a/var/spack/packages/py-epydoc/package.py +++ /dev/null @@ -1,13 +0,0 @@ -from spack import * - -class PyEpydoc(Package): - """Epydoc is a tool for generating API documentation documentation for Python modules, based on their docstrings.""" - homepage = "https://pypi.python.org/pypi/epydoc" - url = "https://pypi.python.org/packages/source/e/epydoc/epydoc-3.0.1.tar.gz" - - version('3.0.1', '36407974bd5da2af00bf90ca27feeb44') - - extends('python') - - def install(self, spec, prefix): - python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-genders/package.py b/var/spack/packages/py-genders/package.py deleted file mode 100644 index c49c8fd5b2..0000000000 --- a/var/spack/packages/py-genders/package.py +++ /dev/null @@ -1,15 +0,0 @@ -from spack import * - -class PyGenders(Package): - """Genders is a static cluster configuration database used for cluster configuration management. It is used by a variety of tools and scripts for management of large clusters.""" - homepage = "https://github.com/chaos/genders" - url = "https://github.com/chaos/genders/releases/download/genders-1-22-1/genders-1.22.tar.gz" - - version('1.22', '9ea59a024dcbddb85b0ed25ddca9bc8e', url='https://github.com/chaos/genders/releases/download/genders-1-22-1/genders-1.22.tar.gz') - extends('python') - - def install(self, spec, prefix): - configure("--prefix=%s" %prefix) - make(parallel=False) - make("install") - diff --git a/var/spack/packages/py-gnuplot/package.py b/var/spack/packages/py-gnuplot/package.py deleted file mode 100644 index ede4472c03..0000000000 --- a/var/spack/packages/py-gnuplot/package.py +++ /dev/null @@ -1,14 +0,0 @@ -from spack import * - -class PyGnuplot(Package): - """Gnuplot.py is a Python package that allows you to create graphs from within Python using the gnuplot plotting program.""" - homepage = "http://gnuplot-py.sourceforge.net/" - url = "http://downloads.sourceforge.net/project/gnuplot-py/Gnuplot-py/1.8/gnuplot-py-1.8.tar.gz" - - version('1.8', 'abd6f571e7aec68ae7db90a5217cd5b1') - - extends('python') - depends_on('py-numpy') - - def install(self, spec, prefix): - python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-h5py/package.py b/var/spack/packages/py-h5py/package.py deleted file mode 100644 index 6293da5407..0000000000 --- a/var/spack/packages/py-h5py/package.py +++ /dev/null @@ -1,19 +0,0 @@ -from spack import * -import re - -class PyH5py(Package): - """The h5py package provides both a high- and low-level interface to the HDF5 library from Python.""" - homepage = "https://pypi.python.org/pypi/h5py" - url = "https://pypi.python.org/packages/source/h/h5py/h5py-2.4.0.tar.gz" - - version('2.4.0', '80c9a94ae31f84885cc2ebe1323d6758') - version('2.5.0', '6e4301b5ad5da0d51b0a1e5ac19e3b74') - - extends('python', ignore=lambda f: re.match(r'bin/cy*', f)) - depends_on('hdf5') - depends_on('py-numpy') - depends_on('py-cython') - - def install(self, spec, prefix): - python('setup.py', 'configure', '--hdf5=%s' % spec['hdf5'].prefix) - python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-ipython/package.py b/var/spack/packages/py-ipython/package.py deleted file mode 100644 index 8d0e64a07f..0000000000 --- a/var/spack/packages/py-ipython/package.py +++ /dev/null @@ -1,16 +0,0 @@ -from spack import * - -class PyIpython(Package): - """IPython provides a rich toolkit to help you make the most out of using Python interactively.""" - homepage = "https://pypi.python.org/pypi/ipython" - url = "https://pypi.python.org/packages/source/i/ipython/ipython-2.3.1.tar.gz" - - version('2.3.1', '2b7085525dac11190bfb45bb8ec8dcbf') - version('3.1.0', 'a749d90c16068687b0ec45a27e72ef8f') - - extends('python') - depends_on('py-pygments') - depends_on('py-setuptools') - - def install(self, spec, prefix): - python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-libxml2/package.py b/var/spack/packages/py-libxml2/package.py deleted file mode 100644 index 59005428e4..0000000000 --- a/var/spack/packages/py-libxml2/package.py +++ /dev/null @@ -1,15 +0,0 @@ -from spack import * - -class PyLibxml2(Package): - """A Python wrapper around libxml2.""" - homepage = "https://xmlsoft.org/python.html" - url = "ftp://xmlsoft.org/libxml2/python/libxml2-python-2.6.21.tar.gz" - - version('2.6.21', '229dd2b3d110a77defeeaa73af83f7f3') - - extends('python') - depends_on('libxml2') - depends_on('libxslt') - - def install(self, spec, prefix): - python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-lockfile/package.py b/var/spack/packages/py-lockfile/package.py deleted file mode 100644 index 8722914d94..0000000000 --- a/var/spack/packages/py-lockfile/package.py +++ /dev/null @@ -1,23 +0,0 @@ -from spack import * - -class PyLockfile(Package): - """The lockfile package exports a LockFile class which provides a - simple API for locking files. Unlike the Windows msvcrt.locking - function, the fcntl.lockf and flock functions, and the - deprecated posixfile module, the API is identical across both - Unix (including Linux and Mac) and Windows platforms. The lock - mechanism relies on the atomic nature of the link (on Unix) and - mkdir (on Windows) system calls. An implementation based on - SQLite is also provided, more as a demonstration of the - possibilities it provides than as production-quality code. - """ - homepage = "https://pypi.python.org/pypi/lockfile" - url = "https://pypi.python.org/packages/source/l/lockfile/lockfile-0.10.2.tar.gz" - - version('0.10.2', '1aa6175a6d57f082cd12e7ac6102ab15') - - extends("python") - depends_on("py-setuptools") - - def install(self, spec, prefix): - python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-mako/package.py b/var/spack/packages/py-mako/package.py deleted file mode 100644 index 3e91ffd8e5..0000000000 --- a/var/spack/packages/py-mako/package.py +++ /dev/null @@ -1,16 +0,0 @@ -from spack import * - -class PyMako(Package): - """A super-fast templating language that borrows the best - ideas from the existing templating languages.""" - - homepage = "https://pypi.python.org/pypi/mako" - url = "https://pypi.python.org/packages/source/M/Mako/Mako-1.0.1.tar.gz" - - version('1.0.1', '9f0aafd177b039ef67b90ea350497a54') - - depends_on('py-setuptools') - extends('python') - - def install(self, spec, prefix): - python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-matplotlib/package.py b/var/spack/packages/py-matplotlib/package.py deleted file mode 100644 index e7ce3dfd24..0000000000 --- a/var/spack/packages/py-matplotlib/package.py +++ /dev/null @@ -1,47 +0,0 @@ -from spack import * -import os - -class PyMatplotlib(Package): - """Python plotting package.""" - homepage = "https://pypi.python.org/pypi/matplotlib" - url = "https://pypi.python.org/packages/source/m/matplotlib/matplotlib-1.4.2.tar.gz" - - version('1.4.2', '7d22efb6cce475025733c50487bd8898') - version('1.4.3', '86af2e3e3c61849ac7576a6f5ca44267') - - extends('python', ignore=r'bin/nosetests.*$') - - depends_on('py-pyside') - depends_on('py-ipython') - depends_on('py-pyparsing') - depends_on('py-six') - depends_on('py-dateutil') - depends_on('py-pytz') - depends_on('py-nose') - depends_on('py-numpy') - - depends_on('qt') - depends_on('bzip2') - depends_on('tcl') - depends_on('tk') - depends_on('qhull') - - def install(self, spec, prefix): - python('setup.py', 'install', '--prefix=%s' % prefix) - - if str(self.version) in ['1.4.2', '1.4.3']: - # hack to fix configuration file - config_file = None - for p,d,f in os.walk(prefix.lib): - for file in f: - if file.find('matplotlibrc') != -1: - config_file = join_path(p, 'matplotlibrc') - print config_file - if config_file == None: - raise InstallError('could not find config file') - filter_file(r'backend : pyside', - 'backend : Qt4Agg', - config_file) - filter_file(r'#backend.qt4 : PyQt4', - 'backend.qt4 : PySide', - config_file) diff --git a/var/spack/packages/py-mock/package.py b/var/spack/packages/py-mock/package.py deleted file mode 100644 index 3b08428ba0..0000000000 --- a/var/spack/packages/py-mock/package.py +++ /dev/null @@ -1,17 +0,0 @@ -from spack import * - -class PyMock(Package): - """mock is a library for testing in Python. It allows you to replace parts - of your system under test with mock objects and make assertions about how - they have been used.""" - - homepage = "https://github.com/testing-cabal/mock" - url = "https://pypi.python.org/packages/source/m/mock/mock-1.3.0.tar.gz" - - version('1.3.0', '73ee8a4afb3ff4da1b4afa287f39fdeb') - - extends('python') - depends_on('py-setuptools@17.1:') - - def install(self, spec, prefix): - python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-mpi4py/package.py b/var/spack/packages/py-mpi4py/package.py deleted file mode 100644 index 8001689a18..0000000000 --- a/var/spack/packages/py-mpi4py/package.py +++ /dev/null @@ -1,14 +0,0 @@ -from spack import * - -class PyMpi4py(Package): - """This package provides Python bindings for the Message Passing Interface (MPI) standard. It is implemented on top of the MPI-1/MPI-2 specification and exposes an API which grounds on the standard MPI-2 C++ bindings.""" - homepage = "https://pypi.python.org/pypi/mpi4py" - url = "https://pypi.python.org/packages/source/m/mpi4py/mpi4py-1.3.1.tar.gz" - - version('1.3.1', 'dbe9d22bdc8ed965c23a7ceb6f32fc3c') - extends('python') - depends_on('py-setuptools') - depends_on('mpi') - - def install(self, spec, prefix): - python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-mx/package.py b/var/spack/packages/py-mx/package.py deleted file mode 100644 index 717ee0562b..0000000000 --- a/var/spack/packages/py-mx/package.py +++ /dev/null @@ -1,13 +0,0 @@ -from spack import * - -class PyMx(Package): - """The eGenix.com mx Base Distribution for Python is a collection of professional quality software tools which enhance Python's usability in many important areas such as fast text searching, date/time processing and high speed data types.""" - homepage = "http://www.egenix.com/products/python/mxBase/" - url = "https://downloads.egenix.com/python/egenix-mx-base-3.2.8.tar.gz" - - version('3.2.8', '9d9d3a25f9dc051a15e97f452413423b') - - extends('python') - - def install(self, spec, prefix): - python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-nose/package.py b/var/spack/packages/py-nose/package.py deleted file mode 100644 index e7c6cf0264..0000000000 --- a/var/spack/packages/py-nose/package.py +++ /dev/null @@ -1,17 +0,0 @@ -from spack import * - -class PyNose(Package): - """nose extends the test loading and running features of unittest, - making it easier to write, find and run tests.""" - - homepage = "https://pypi.python.org/pypi/nose" - url = "https://pypi.python.org/packages/source/n/nose/nose-1.3.4.tar.gz" - - version('1.3.4', '6ed7169887580ddc9a8e16048d38274d') - version('1.3.6', '0ca546d81ca8309080fc80cb389e7a16') - - extends('python', ignore=r'bin/nosetests.*$') - depends_on('py-setuptools') - - def install(self, spec, prefix): - python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-numpy/package.py b/var/spack/packages/py-numpy/package.py deleted file mode 100644 index efa109a3e9..0000000000 --- a/var/spack/packages/py-numpy/package.py +++ /dev/null @@ -1,28 +0,0 @@ -from spack import * - -class PyNumpy(Package): - """array processing for numbers, strings, records, and objects.""" - homepage = "https://pypi.python.org/pypi/numpy" - url = "https://pypi.python.org/packages/source/n/numpy/numpy-1.9.1.tar.gz" - - version('1.9.1', '78842b73560ec378142665e712ae4ad9') - version('1.9.2', 'a1ed53432dbcd256398898d35bc8e645') - - extends('python') - depends_on('py-nose') - depends_on('netlib-blas+fpic') - depends_on('netlib-lapack+shared') - - def patch(self): - filter_file( - "possible_executables = \['(gfortran|g77|ifort|efl)", - "possible_executables = ['fc", - "numpy/distutils/fcompiler/gnu.py", - "numpy/distutils/fcompiler/intel.py") - - def install(self, spec, prefix): - with open('site.cfg', 'w') as f: - f.write('[DEFAULT]\n') - f.write('libraries=lapack,blas\n') - f.write('library_dirs=%s/lib:%s/lib\n' % (spec['blas'].prefix, spec['lapack'].prefix)) - python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-pandas/package.py b/var/spack/packages/py-pandas/package.py deleted file mode 100644 index 5b9997faa9..0000000000 --- a/var/spack/packages/py-pandas/package.py +++ /dev/null @@ -1,25 +0,0 @@ -from spack import * -import os - -class PyPandas(Package): - """pandas is a Python package providing fast, flexible, and expressive data structures designed to make working with relational or labeled data both easy and intuitive. It aims to be the fundamental high-level building block for doing practical, real world data analysis in Python. Additionally, it has the broader goal of becoming the most powerful and flexible open source data analysis / manipulation tool available in any language.""" - homepage = "http://pandas.pydata.org/" - url = "https://pypi.python.org/packages/source/p/pandas/pandas-0.16.0.tar.gz#md5=bfe311f05dc0c351f8955fbd1e296e73" - - version('0.16.0', 'bfe311f05dc0c351f8955fbd1e296e73') - version('0.16.1', 'fac4f25748f9610a3e00e765474bdea8') - - extends('python') - depends_on('py-dateutil') - depends_on('py-numpy') - depends_on('py-matplotlib') - depends_on('py-scipy') - depends_on('py-setuptools') - depends_on('py-pytz') - depends_on('libdrm') - depends_on('libpciaccess') - depends_on('llvm') - depends_on('mesa') - - def install(self, spec, prefix): - python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-pexpect/package.py b/var/spack/packages/py-pexpect/package.py deleted file mode 100644 index ff5fac84e0..0000000000 --- a/var/spack/packages/py-pexpect/package.py +++ /dev/null @@ -1,13 +0,0 @@ -from spack import * - -class PyPexpect(Package): - """Pexpect allows easy control of interactive console applications.""" - homepage = "https://pypi.python.org/pypi/pexpect" - url = "https://pypi.python.org/packages/source/p/pexpect/pexpect-3.3.tar.gz" - - version('3.3', '0de72541d3f1374b795472fed841dce8') - - extends('python') - - def install(self, spec, prefix): - python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-pil/package.py b/var/spack/packages/py-pil/package.py deleted file mode 100644 index 743b761981..0000000000 --- a/var/spack/packages/py-pil/package.py +++ /dev/null @@ -1,14 +0,0 @@ -from spack import * - -class PyPil(Package): - """The Python Imaging Library (PIL) adds image processing capabilities to your Python interpreter. This library supports many file formats, and provides powerful image processing and graphics capabilities.""" - - homepage = "http://www.pythonware.com/products/pil/" - url = "http://effbot.org/media/downloads/Imaging-1.1.7.tar.gz" - - version('1.1.7', 'fc14a54e1ce02a0225be8854bfba478e') - - extends('python') - - def install(self, spec, prefix): - python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-pmw/package.py b/var/spack/packages/py-pmw/package.py deleted file mode 100644 index 56131811e9..0000000000 --- a/var/spack/packages/py-pmw/package.py +++ /dev/null @@ -1,13 +0,0 @@ -from spack import * - -class PyPmw(Package): - """Pmw is a toolkit for building high-level compound widgets, or megawidgets, constructed using other widgets as component parts.""" - homepage = "https://pypi.python.org/pypi/Pmw" - url = "https://pypi.python.org/packages/source/P/Pmw/Pmw-2.0.0.tar.gz" - - version('2.0.0', 'c7c3f26c4f5abaa99807edefee578fc0') - - extends('python') - - def install(self, spec, prefix): - python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-pychecker/package.py b/var/spack/packages/py-pychecker/package.py deleted file mode 100644 index bda5a746aa..0000000000 --- a/var/spack/packages/py-pychecker/package.py +++ /dev/null @@ -1,13 +0,0 @@ -from spack import * - -class PyPychecker(Package): - """""" - homepage = "http://pychecker.sourceforge.net/" - url = "http://sourceforge.net/projects/pychecker/files/pychecker/0.8.19/pychecker-0.8.19.tar.gz" - - version('0.8.19', 'c37182863dfb09209d6ba4f38fce9d2b') - - extends('python') - - def install(self, spec, prefix): - python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-pycparser/package.py b/var/spack/packages/py-pycparser/package.py deleted file mode 100644 index f2bb679d25..0000000000 --- a/var/spack/packages/py-pycparser/package.py +++ /dev/null @@ -1,15 +0,0 @@ -from spack import * - -class PyPycparser(Package): - """pycparser is a complete parser of the C language, written in pure python""" - homepage = "https://github.com/eliben/pycparser" - url = "https://pypi.python.org/packages/source/p/pycparser/pycparser-2.13.tar.gz" - - version('2.13', 'e4fe1a2d341b22e25da0d22f034ef32f') - - - extends('python') - depends_on('py-setuptools') - - def install(self, spec, prefix): - python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-pyelftools/package.py b/var/spack/packages/py-pyelftools/package.py deleted file mode 100644 index d5ad32e624..0000000000 --- a/var/spack/packages/py-pyelftools/package.py +++ /dev/null @@ -1,13 +0,0 @@ -from spack import * - -class PyPyelftools(Package): - """A pure-Python library for parsing and analyzing ELF files and DWARF debugging information""" - homepage = "https://pypi.python.org/pypi/pyelftools" - url = "https://pypi.python.org/packages/source/p/pyelftools/pyelftools-0.23.tar.gz" - - version('0.23', 'aa7cefa8bd2f63d7b017440c9084f310') - - extends('python') - - def install(self, spec, prefix): - python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-pygments/package.py b/var/spack/packages/py-pygments/package.py deleted file mode 100644 index 7e07bf6869..0000000000 --- a/var/spack/packages/py-pygments/package.py +++ /dev/null @@ -1,15 +0,0 @@ -from spack import * - -class PyPygments(Package): - """Pygments is a syntax highlighting package written in Python.""" - homepage = "https://pypi.python.org/pypi/pygments" - url = "https://pypi.python.org/packages/source/P/Pygments/Pygments-2.0.1.tar.gz" - - version('2.0.1', 'e0daf4c14a4fe5b630da765904de4d6c') - version('2.0.2', '238587a1370d62405edabd0794b3ec4a') - - extends('python') - depends_on('py-setuptools') - - def install(self, spec, prefix): - python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-pylint/package.py b/var/spack/packages/py-pylint/package.py deleted file mode 100644 index 9579708c29..0000000000 --- a/var/spack/packages/py-pylint/package.py +++ /dev/null @@ -1,17 +0,0 @@ -from spack import * -import re - -class PyPylint(Package): - """array processing for numbers, strings, records, and objects.""" - homepage = "https://pypi.python.org/pypi/pylint" - url = "https://pypi.python.org/packages/source/p/pylint/pylint-1.4.1.tar.gz" - - version('1.4.1', 'df7c679bdcce5019389038847e4de622') - version('1.4.3', '5924c1c7ca5ca23647812f5971d0ea44') - - extends('python') - depends_on('py-nose') - depends_on('py-setuptools') - - def install(self, spec, prefix): - python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-pypar/package.py b/var/spack/packages/py-pypar/package.py deleted file mode 100644 index af9c76ccd8..0000000000 --- a/var/spack/packages/py-pypar/package.py +++ /dev/null @@ -1,14 +0,0 @@ -from spack import * - -class PyPypar(Package): - """Pypar is an efficient but easy-to-use module that allows programs written in Python to run in parallel on multiple processors and communicate using MPI.""" - homepage = "http://code.google.com/p/pypar/" - url = "https://pypar.googlecode.com/files/pypar-2.1.5_108.tgz" - - version('2.1.5_108', '7a1f28327d2a3b679f9455c843d850b8', url='https://pypar.googlecode.com/files/pypar-2.1.5_108.tgz') - extends('python') - depends_on('mpi') - - def install(self, spec, prefix): - with working_dir('source'): - python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-pyparsing/package.py b/var/spack/packages/py-pyparsing/package.py deleted file mode 100644 index a6e50ad139..0000000000 --- a/var/spack/packages/py-pyparsing/package.py +++ /dev/null @@ -1,13 +0,0 @@ -from spack import * - -class PyPyparsing(Package): - """A Python Parsing Module.""" - homepage = "https://pypi.python.org/pypi/pyparsing" - url = "https://pypi.python.org/packages/source/p/pyparsing/pyparsing-2.0.3.tar.gz" - - version('2.0.3', '0fe479be09fc2cf005f753d3acc35939') - - extends('python') - - def install(self, spec, prefix): - python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-pyqt/package.py b/var/spack/packages/py-pyqt/package.py deleted file mode 100644 index 8edca105bb..0000000000 --- a/var/spack/packages/py-pyqt/package.py +++ /dev/null @@ -1,24 +0,0 @@ -from spack import * - -class PyPyqt(Package): - """PyQt is a set of Python v2 and v3 bindings for Digia's Qt - application framework and runs on all platforms supported by Qt - including Windows, MacOS/X and Linux.""" - homepage = "http://www.riverbankcomputing.com/software/pyqt/intro" - url = "http://sourceforge.net/projects/pyqt/files/PyQt4/PyQt-4.11.3/PyQt-x11-gpl-4.11.3.tar.gz" - - version('4.11.3', '997c3e443165a89a559e0d96b061bf70') - - extends('python') - depends_on('py-sip') - - # TODO: allow qt5 when conditional deps are supported. - # TODO: Fix version matching so that @4 works like @:4 - depends_on('qt@:4') - - def install(self, spec, prefix): - python('configure.py', - '--confirm-license', - '--destdir=%s' % site_packages_dir) - make() - make('install') diff --git a/var/spack/packages/py-pyside/package.py b/var/spack/packages/py-pyside/package.py deleted file mode 100644 index bb5da44d02..0000000000 --- a/var/spack/packages/py-pyside/package.py +++ /dev/null @@ -1,45 +0,0 @@ -from spack import * -import os - -class PyPyside(Package): - """array processing for numbers, strings, records, and objects.""" - homepage = "https://pypi.python.org/pypi/pyside" - url = "https://pypi.python.org/packages/source/P/PySide/PySide-1.2.2.tar.gz" - - version('1.2.2', 'c45bc400c8a86d6b35f34c29e379e44d') - - # TODO: make build dependency - # depends_on("cmake") - - extends('python') - depends_on('py-setuptools') - depends_on('qt@:4') - - def patch(self): - """Undo PySide RPATH handling and add Spack RPATH.""" - # Figure out the special RPATH - pypkg = self.spec['python'].package - rpath = self.rpath - rpath.append(os.path.join(self.prefix, pypkg.site_packages_dir, 'PySide')) - - # Add Spack's standard CMake args to the sub-builds. - # They're called BY setup.py so we have to patch it. - filter_file( - r'OPTION_CMAKE,', - r'OPTION_CMAKE, ' + ( - '"-DCMAKE_INSTALL_RPATH_USE_LINK_PATH=FALSE", ' - '"-DCMAKE_INSTALL_RPATH=%s",' % ':'.join(rpath)), - 'setup.py') - - # PySide tries to patch ELF files to remove RPATHs - # Disable this and go with the one we set. - filter_file( - r'^\s*rpath_cmd\(pyside_path, srcpath\)', - r'#rpath_cmd(pyside_path, srcpath)', - 'pyside_postinstall.py') - - - def install(self, spec, prefix): - python('setup.py', 'install', - '--prefix=%s' % prefix, - '--jobs=%s' % make_jobs) diff --git a/var/spack/packages/py-python-daemon/package.py b/var/spack/packages/py-python-daemon/package.py deleted file mode 100644 index 12cbe9101c..0000000000 --- a/var/spack/packages/py-python-daemon/package.py +++ /dev/null @@ -1,26 +0,0 @@ -from spack import * - -class PyPythonDaemon(Package): - """Library to implement a well-behaved Unix daemon process. - - This library implements the well-behaved daemon specification of - PEP Standard daemon process. - - A well-behaved Unix daemon process is tricky to get right, but the - required steps are much the same for every daemon program. A - DaemonContext instance holds the behaviour and configured process - environment for the program; use the instance as a context manager - to enter a daemon state. - """ - homepage = "https://pypi.python.org/pypi/python-daemon/" - url = "https://pypi.python.org/packages/source/p/python-daemon/python-daemon-2.0.5.tar.gz" - - version('2.0.5', '73e7f49f525c51fa4a995aea4d80de41') - - extends("python") - depends_on("py-setuptools") - depends_on("py-lockfile") - - def install(self, spec, prefix): - python('setup.py', 'install', '--prefix=%s' % prefix) - diff --git a/var/spack/packages/py-pytz/package.py b/var/spack/packages/py-pytz/package.py deleted file mode 100644 index da6311a784..0000000000 --- a/var/spack/packages/py-pytz/package.py +++ /dev/null @@ -1,14 +0,0 @@ -from spack import * - -class PyPytz(Package): - """World timezone definitions, modern and historical.""" - homepage = "https://pypi.python.org/pypi/pytz" - url = "https://pypi.python.org/packages/source/p/pytz/pytz-2014.10.tar.gz" - - version('2014.10', 'eb1cb941a20c5b751352c52486aa1dd7') - version('2015.4', '417a47b1c432d90333e42084a605d3d8') - - extends('python') - - def install(self, spec, prefix): - python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-rpy2/package.py b/var/spack/packages/py-rpy2/package.py deleted file mode 100644 index a0b03d03e3..0000000000 --- a/var/spack/packages/py-rpy2/package.py +++ /dev/null @@ -1,17 +0,0 @@ -from spack import * - -class PyRpy2(Package): - """rpy2 is a redesign and rewrite of rpy. It is providing a low-level interface to R from Python, a proposed high-level interface, including wrappers to graphical libraries, as well as R-like structures and functions.""" - homepage = "https://pypi.python.org/pypi/rpy2" - url = "https://pypi.python.org/packages/source/r/rpy2/rpy2-2.5.4.tar.gz" - - version('2.5.4', '115a20ac30883f096da2bdfcab55196d') - version('2.5.6', 'a36e758b633ce6aec6a5f450bfee980f') - - extends('python') - depends_on('py-setuptools') - - depends_on('R') - - def install(self, spec, prefix): - python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-scientificpython/package.py b/var/spack/packages/py-scientificpython/package.py deleted file mode 100644 index df2c86caac..0000000000 --- a/var/spack/packages/py-scientificpython/package.py +++ /dev/null @@ -1,16 +0,0 @@ -from spack import * - -class PyScientificpython(Package): - """ScientificPython is a collection of Python modules for - scientific computing. It contains support for geometry, - mathematical functions, statistics, physical units, IO, - visualization, and parallelization.""" - - homepage = "https://sourcesup.renater.fr/projects/scientific-py/" - url = "https://sourcesup.renater.fr/frs/download.php/file/4411/ScientificPython-2.8.1.tar.gz" - version('2.8.1', '73ee0df19c7b58cdf2954261f0763c77') - - extends('python') - - def install(self, spec, prefix): - python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-scikit-learn/package.py b/var/spack/packages/py-scikit-learn/package.py deleted file mode 100644 index 5b078ce901..0000000000 --- a/var/spack/packages/py-scikit-learn/package.py +++ /dev/null @@ -1,14 +0,0 @@ -from spack import * - -class PyScikitLearn(Package): - """""" - homepage = "https://pypi.python.org/pypi/scikit-learn" - url = "https://pypi.python.org/packages/source/s/scikit-learn/scikit-learn-0.15.2.tar.gz" - - version('0.15.2', 'd9822ad0238e17b382a3c756ea94fe0d') - version('0.16.1', '363ddda501e3b6b61726aa40b8dbdb7e') - - extends('python') - - def install(self, spec, prefix): - python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-scipy/package.py b/var/spack/packages/py-scipy/package.py deleted file mode 100644 index 3a1124cc15..0000000000 --- a/var/spack/packages/py-scipy/package.py +++ /dev/null @@ -1,18 +0,0 @@ -from spack import * - -class PyScipy(Package): - """Scientific Library for Python.""" - homepage = "https://pypi.python.org/pypi/scipy" - url = "https://pypi.python.org/packages/source/s/scipy/scipy-0.15.0.tar.gz" - - version('0.15.0', '639112f077f0aeb6d80718dc5019dc7a') - version('0.15.1', 'be56cd8e60591d6332aac792a5880110') - - extends('python') - depends_on('py-nose') - depends_on('py-numpy') - depends_on('blas') - depends_on('lapack') - - def install(self, spec, prefix): - python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-setuptools/package.py b/var/spack/packages/py-setuptools/package.py deleted file mode 100644 index 760ad4d6db..0000000000 --- a/var/spack/packages/py-setuptools/package.py +++ /dev/null @@ -1,15 +0,0 @@ -from spack import * - -class PySetuptools(Package): - """Easily download, build, install, upgrade, and uninstall Python packages.""" - homepage = "https://pypi.python.org/pypi/setuptools" - url = "https://pypi.python.org/packages/source/s/setuptools/setuptools-11.3.tar.gz" - - version('11.3.1', '01f69212e019a2420c1693fb43593930') - version('16.0', '0ace0b96233516fc5f7c857d086aa3ad') - version('18.1', 'f72e87f34fbf07f299f6cb46256a0b06') - - extends('python') - - def install(self, spec, prefix): - python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-shiboken/package.py b/var/spack/packages/py-shiboken/package.py deleted file mode 100644 index e4bf4ce07e..0000000000 --- a/var/spack/packages/py-shiboken/package.py +++ /dev/null @@ -1,45 +0,0 @@ -from spack import * -import os - -class PyShiboken(Package): - """Shiboken generates bindings for C++ libraries using CPython source code.""" - homepage = "https://shiboken.readthedocs.org/" - url = "https://pypi.python.org/packages/source/S/Shiboken/Shiboken-1.2.2.tar.gz" - - version('1.2.2', '345cfebda221f525842e079a6141e555') - - # TODO: make build dependency - # depends_on("cmake") - - extends('python') - depends_on("py-setuptools") - depends_on("libxml2") - depends_on("qt@:4.8") - - def patch(self): - """Undo Shiboken RPATH handling and add Spack RPATH.""" - # Add Spack's standard CMake args to the sub-builds. - # They're called BY setup.py so we have to patch it. - pypkg = self.spec['python'].package - rpath = self.rpath - rpath.append(os.path.join(self.prefix, pypkg.site_packages_dir, 'Shiboken')) - - filter_file( - r'OPTION_CMAKE,', - r'OPTION_CMAKE, ' + ( - '"-DCMAKE_INSTALL_RPATH_USE_LINK_PATH=FALSE", ' - '"-DCMAKE_INSTALL_RPATH=%s",' % ':'.join(rpath)), - 'setup.py') - - # Shiboken tries to patch ELF files to remove RPATHs - # Disable this and go with the one we set. - filter_file( - r'^\s*rpath_cmd\(shiboken_path, srcpath\)', - r'#rpath_cmd(shiboken_path, srcpath)', - 'shiboken_postinstall.py') - - - def install(self, spec, prefix): - python('setup.py', 'install', - '--prefix=%s' % prefix, - '--jobs=%s' % make_jobs) diff --git a/var/spack/packages/py-sip/package.py b/var/spack/packages/py-sip/package.py deleted file mode 100644 index e4a6fb6961..0000000000 --- a/var/spack/packages/py-sip/package.py +++ /dev/null @@ -1,21 +0,0 @@ -from spack import * -import os - -class PySip(Package): - """SIP is a tool that makes it very easy to create Python bindings for C and C++ libraries.""" - homepage = "http://www.riverbankcomputing.com/software/sip/intro" - url = "http://sourceforge.net/projects/pyqt/files/sip/sip-4.16.5/sip-4.16.5.tar.gz" - - version('4.16.5', '6d01ea966a53e4c7ae5c5e48c40e49e5') - version('4.16.7', '32abc003980599d33ffd789734de4c36') - - extends('python') - - def install(self, spec, prefix): - python('configure.py', - '--destdir=%s' % site_packages_dir, - '--bindir=%s' % spec.prefix.bin, - '--incdir=%s' % python_include_dir, - '--sipdir=%s' % os.path.join(spec.prefix.share, 'sip')) - make() - make('install') diff --git a/var/spack/packages/py-six/package.py b/var/spack/packages/py-six/package.py deleted file mode 100644 index 05c5bd00a9..0000000000 --- a/var/spack/packages/py-six/package.py +++ /dev/null @@ -1,14 +0,0 @@ -from spack import * - -class PySix(Package): - """Python 2 and 3 compatibility utilities.""" - homepage = "https://pypi.python.org/pypi/six" - url = "https://pypi.python.org/packages/source/s/six/six-1.9.0.tar.gz" - - version('1.9.0', '476881ef4012262dfc8adc645ee786c4') - - extends('python') - depends_on('py-setuptools') - - def install(self, spec, prefix): - python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-sphinx/package.py b/var/spack/packages/py-sphinx/package.py deleted file mode 100644 index ec2e89a098..0000000000 --- a/var/spack/packages/py-sphinx/package.py +++ /dev/null @@ -1,13 +0,0 @@ -from spack import * - -class PySphinx(Package): - """Sphinx Documentation Generator.""" - homepage = "http://sphinx-doc.org" - url = "https://pypi.python.org/packages/source/S/Sphinx/Sphinx-1.3.1.tar.gz" - - version('1.3.1', '8786a194acf9673464c5455b11fd4332') - - extends('python') - - def install(self, spec, prefix): - python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-sympy/package.py b/var/spack/packages/py-sympy/package.py deleted file mode 100644 index c17e35b95f..0000000000 --- a/var/spack/packages/py-sympy/package.py +++ /dev/null @@ -1,13 +0,0 @@ -from spack import * - -class PySympy(Package): - """SymPy is a Python library for symbolic mathematics.""" - homepage = "https://pypi.python.org/pypi/sympy" - url = "https://pypi.python.org/packages/source/s/sympy/sympy-0.7.6.tar.gz" - - version('0.7.6', '3d04753974306d8a13830008e17babca') - - extends('python') - - def install(self, spec, prefix): - python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-virtualenv/package.py b/var/spack/packages/py-virtualenv/package.py deleted file mode 100644 index 037a6fc59f..0000000000 --- a/var/spack/packages/py-virtualenv/package.py +++ /dev/null @@ -1,16 +0,0 @@ -from spack import * -import shutil - -class PyVirtualenv(Package): - """virtualenv is a tool to create isolated Python environments.""" - homepage = "http://virtualenv.readthedocs.org/projects/virtualenv/" - url = "https://pypi.python.org/packages/source/v/virtualenv/virtualenv-1.11.6.tar.gz" - - version('1.11.6', 'f61cdd983d2c4e6aeabb70b1060d6f49') - version('13.0.1', '1ffc011bde6667f0e37ecd976f4934db') - - extends('python') - depends_on('py-setuptools') - - def install(self, spec, prefix): - python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-yapf/package.py b/var/spack/packages/py-yapf/package.py deleted file mode 100644 index 12ef191515..0000000000 --- a/var/spack/packages/py-yapf/package.py +++ /dev/null @@ -1,15 +0,0 @@ -from spack import * - -class PyYapf(Package): - """ Yet Another Python Formatter """ - homepage = "https://github.com/google/yapf" - # base https://pypi.python.org/pypi/cffi - url = "https://github.com/google/yapf/archive/v0.2.1.tar.gz" - - version('0.2.1', '348ccf86cf2057872e4451b204fb914c') - - extends('python') - depends_on('py-setuptools') - - def install(self, spec, prefix): - python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/python/package.py b/var/spack/packages/python/package.py deleted file mode 100644 index 000881a846..0000000000 --- a/var/spack/packages/python/package.py +++ /dev/null @@ -1,160 +0,0 @@ -import os -import re -from contextlib import closing -from llnl.util.lang import match_predicate - -from spack import * -import spack - - -class Python(Package): - """The Python programming language.""" - homepage = "http://www.python.org" - url = "http://www.python.org/ftp/python/2.7.8/Python-2.7.8.tar.xz" - - extendable = True - - version('2.7.8', 'd235bdfa75b8396942e360a70487ee00') - version('2.7.10', 'c685ef0b8e9f27b5e3db5db12b268ac6') - - depends_on("openssl") - depends_on("bzip2") - depends_on("readline") - depends_on("ncurses") - depends_on("sqlite") - - def install(self, spec, prefix): - # Need this to allow python build to find the Python installation. - env['PYTHONHOME'] = prefix - - # Rest of install is pretty standard. - configure("--prefix=%s" % prefix, - "--with-threads", - "--enable-shared") - make() - make("install") - - - # ======================================================================== - # Set up environment to make install easy for python extensions. - # ======================================================================== - - @property - def python_lib_dir(self): - return os.path.join('lib', 'python%d.%d' % self.version[:2]) - - - @property - def python_include_dir(self): - return os.path.join('include', 'python%d.%d' % self.version[:2]) - - - @property - def site_packages_dir(self): - return os.path.join(self.python_lib_dir, 'site-packages') - - - def setup_dependent_environment(self, module, spec, ext_spec): - """Called before python modules' install() methods. - - In most cases, extensions will only need to have one line:: - - python('setup.py', 'install', '--prefix=%s' % prefix) - """ - # Python extension builds can have a global python executable function - module.python = Executable(join_path(spec.prefix.bin, 'python')) - - # Add variables for lib/pythonX.Y and lib/pythonX.Y/site-packages dirs. - module.python_lib_dir = os.path.join(ext_spec.prefix, self.python_lib_dir) - module.python_include_dir = os.path.join(ext_spec.prefix, self.python_include_dir) - module.site_packages_dir = os.path.join(ext_spec.prefix, self.site_packages_dir) - - # Make the site packages directory if it does not exist already. - mkdirp(module.site_packages_dir) - - # Set PYTHONPATH to include site-packages dir for the - # extension and any other python extensions it depends on. - python_paths = [] - for d in ext_spec.traverse(): - if d.package.extends(self.spec): - python_paths.append(os.path.join(d.prefix, self.site_packages_dir)) - os.environ['PYTHONPATH'] = ':'.join(python_paths) - - - # ======================================================================== - # Handle specifics of activating and deactivating python modules. - # ======================================================================== - - def python_ignore(self, ext_pkg, args): - """Add some ignore files to activate/deactivate args.""" - ignore_arg = args.get('ignore', lambda f: False) - - # Always ignore easy-install.pth, as it needs to be merged. - patterns = [r'easy-install\.pth$'] - - # Ignore pieces of setuptools installed by other packages. - if ext_pkg.name != 'py-setuptools': - patterns.append(r'/site\.pyc?$') - patterns.append(r'setuptools\.pth') - patterns.append(r'bin/easy_install[^/]*$') - patterns.append(r'setuptools.*egg$') - - return match_predicate(ignore_arg, patterns) - - - def write_easy_install_pth(self, exts): - paths = [] - for ext in sorted(exts.values()): - ext_site_packages = os.path.join(ext.prefix, self.site_packages_dir) - easy_pth = "%s/easy-install.pth" % ext_site_packages - - if not os.path.isfile(easy_pth): - continue - - with closing(open(easy_pth)) as f: - for line in f: - line = line.rstrip() - - # Skip lines matching these criteria - if not line: continue - if re.search(r'^(import|#)', line): continue - if (ext.name != 'py-setuptools' and - re.search(r'setuptools.*egg$', line)): continue - - paths.append(line) - - site_packages = os.path.join(self.prefix, self.site_packages_dir) - main_pth = "%s/easy-install.pth" % site_packages - - if not paths: - if os.path.isfile(main_pth): - os.remove(main_pth) - - else: - with closing(open(main_pth, 'w')) as f: - f.write("import sys; sys.__plen = len(sys.path)\n") - for path in paths: - f.write("%s\n" % path) - f.write("import sys; new=sys.path[sys.__plen:]; del sys.path[sys.__plen:]; " - "p=getattr(sys,'__egginsert',0); sys.path[p:p]=new; sys.__egginsert = p+len(new)\n") - - - def activate(self, ext_pkg, **args): - ignore=self.python_ignore(ext_pkg, args) - args.update(ignore=ignore) - - super(Python, self).activate(ext_pkg, **args) - - exts = spack.install_layout.extension_map(self.spec) - exts[ext_pkg.name] = ext_pkg.spec - self.write_easy_install_pth(exts) - - - def deactivate(self, ext_pkg, **args): - args.update(ignore=self.python_ignore(ext_pkg, args)) - super(Python, self).deactivate(ext_pkg, **args) - - exts = spack.install_layout.extension_map(self.spec) - if ext_pkg.name in exts: # Make deactivate idempotent. - del exts[ext_pkg.name] - self.write_easy_install_pth(exts) diff --git a/var/spack/packages/qhull/package.py b/var/spack/packages/qhull/package.py deleted file mode 100644 index 9da4078a70..0000000000 --- a/var/spack/packages/qhull/package.py +++ /dev/null @@ -1,27 +0,0 @@ -from spack import * - -class Qhull(Package): - """Qhull computes the convex hull, Delaunay triangulation, Voronoi - diagram, halfspace intersection about a point, furt hest-site - Delaunay triangulation, and furthest-site Voronoi diagram. The - source code runs in 2-d, 3-d, 4-d, and higher dimensions. Qhull - implements the Quickhull algorithm for computing the convex - hull. It handles roundoff errors from floating point - arithmetic. It computes volumes, surface areas, and - approximations to the convex hull. - - Qhull does not support triangulation of non-convex surfaces, - mesh generation of non-convex objects, medium-sized inputs in - 9-D and higher, alpha shapes, weighted Voronoi diagrams, - Voronoi volumes, or constrained Delaunay triangulations.""" - - homepage = "http://www.qhull.org" - - version('1.0', 'd0f978c0d8dfb2e919caefa56ea2953c', - url="http://www.qhull.org/download/qhull-2012.1-src.tgz") - - def install(self, spec, prefix): - with working_dir('spack-build', create=True): - cmake('..', *std_cmake_args) - make() - make("install") diff --git a/var/spack/packages/qt/package.py b/var/spack/packages/qt/package.py deleted file mode 100644 index 0e4abe3b1d..0000000000 --- a/var/spack/packages/qt/package.py +++ /dev/null @@ -1,109 +0,0 @@ -import os -from spack import * -import os - -class Qt(Package): - """Qt is a comprehensive cross-platform C++ application framework.""" - homepage = "http://qt.io" - list_url = 'http://download.qt-project.org/official_releases/qt/' - list_depth = 2 - - version('5.4.0', 'e8654e4b37dd98039ba20da7a53877e6', - url='http://download.qt-project.org/official_releases/qt/5.4/5.4.0/single/qt-everywhere-opensource-src-5.4.0.tar.gz') - version('5.3.2', 'febb001129927a70174467ecb508a682', - url='http://download.qt.io/archive/qt/5.3/5.3.2/single/qt-everywhere-opensource-src-5.3.2.tar.gz') - - version('5.2.1', 'a78408c887c04c34ce615da690e0b4c8', - url='http://download.qt.io/archive/qt/5.2/5.2.1/single/qt-everywhere-opensource-src-5.2.1.tar.gz') - version('4.8.6', '2edbe4d6c2eff33ef91732602f3518eb', - url="http://download.qt-project.org/official_releases/qt/4.8/4.8.6/qt-everywhere-opensource-src-4.8.6.tar.gz") - - # Use system openssl for security. - #depends_on("openssl") - - depends_on("glib") - depends_on("gtkplus") - depends_on("libxml2") - depends_on("zlib") - depends_on("dbus") - depends_on("libtiff") - depends_on("libpng") - depends_on("libmng") - depends_on("jpeg") - - # Webkit - # depends_on("gperf") - # depends_on("flex") - # depends_on("bison") - # depends_on("ruby") - # depends_on("icu4c") - - # OpenGL hardware acceleration - depends_on("mesa") - depends_on("libxcb") - - - def setup_dependent_environment(self, module, spec, dep_spec): - """Dependencies of Qt find it using the QTDIR environment variable.""" - os.environ['QTDIR'] = self.prefix - - - def patch(self): - if self.spec.satisfies('@4'): - qmake_conf = 'mkspecs/common/g++-base.conf' - qmake_unix_conf = 'mkspecs/common/g++-unix.conf' - elif self.spec.satisfies('@5'): - qmake_conf = 'qtbase/mkspecs/common/g++-base.conf' - qmake_unix_conf = 'qtbase/mkspecs/common/g++-unix.conf' - else: - return - - # Fix qmake compilers in the default mkspec - filter_file(r'^QMAKE_COMPILER *=.*$', 'QMAKE_COMPILER = cc', qmake_conf) - filter_file(r'^QMAKE_CC *=.*$', 'QMAKE_CC = cc', qmake_conf) - filter_file(r'^QMAKE_CXX *=.*$', 'QMAKE_CXX = c++', qmake_conf) - filter_file(r'^QMAKE_LFLAGS_NOUNDEF *\+?=.*$', 'QMAKE_LFLAGS_NOUNDEF =', qmake_unix_conf) - - - @property - def common_config_args(self): - return [ - '-prefix', self.prefix, - '-v', - '-opensource', - '-opengl', - "-release", - '-shared', - '-confirm-license', - '-openssl-linked', - '-dbus-linked', - '-optimized-qmake', - '-no-openvg', - '-no-pch', - # NIS is deprecated in more recent glibc - "-no-nis"] - # Don't disable all the database drivers, but should - # really get them into spack at some point. - - - @when('@4') - def configure(self): - configure('-fast', - '-no-webkit', - *self.common_config_args) - - - @when('@5') - def configure(self): - configure('-no-eglfs', - '-no-directfb', - '-qt-xcb', - # If someone wants to get a webkit build working, be my guest! - '-skip', 'qtwebkit', - *self.common_config_args) - - - def install(self, spec, prefix): - self.configure() - make() - make("install") diff --git a/var/spack/packages/qthreads/package.py b/var/spack/packages/qthreads/package.py deleted file mode 100644 index dacdb71524..0000000000 --- a/var/spack/packages/qthreads/package.py +++ /dev/null @@ -1,22 +0,0 @@ -from spack import * - -class Qthreads(Package): - """The qthreads API is designed to make using large numbers of - threads convenient and easy, and to allow portable access to - threading constructs used in massively parallel shared memory - environments. The API maps well to both MTA-style threading and - PIM-style threading, and we provide an implementation of this - interface in both a standard SMP context as well as the SST - context. The qthreads API provides access to full/empty-bit - (FEB) semantics, where every word of memory can be marked - either full or empty, and a thread can wait for any word to - attain either state.""" - homepage = "http://www.cs.sandia.gov/qthreads/" - url = "https://qthreads.googlecode.com/files/qthread-1.10.tar.bz2" - - version('1.10', '5af8c8bbe88c2a6d45361643780d1671') - - def install(self, spec, prefix): - configure("--prefix=%s" % prefix) - make() - make("install") diff --git a/var/spack/packages/ravel/package.py b/var/spack/packages/ravel/package.py deleted file mode 100644 index 01fa941cfe..0000000000 --- a/var/spack/packages/ravel/package.py +++ /dev/null @@ -1,23 +0,0 @@ -from spack import * - -class Ravel(Package): - """Ravel is a parallel communication trace visualization tool that - orders events according to logical time.""" - - homepage = "https://github.com/scalability-llnl/ravel" - url = 'https://github.com/scalability-llnl/ravel/archive/v1.0.0.tar.gz' - - version('1.0.0', 'b25fece58331c2adfcce76c5036485c2') - - # TODO: make this a build dependency - depends_on('cmake@2.8.9:') - - depends_on('muster@1.0.1:') - depends_on('otf') - depends_on('otf2') - depends_on('qt@5:') - - def install(self, spec, prefix): - cmake('-Wno-dev', *std_cmake_args) - make() - make("install") diff --git a/var/spack/packages/readline/package.py b/var/spack/packages/readline/package.py deleted file mode 100644 index 1b870e0e7f..0000000000 --- a/var/spack/packages/readline/package.py +++ /dev/null @@ -1,21 +0,0 @@ -from spack import * - -class Readline(Package): - """The GNU Readline library provides a set of functions for use by - applications that allow users to edit command li nes as they - are typed in. Both Emacs and vi editing modes are - available. The Readline library includes additional functions - to maintain a list of previously-entered command lines, to - recall and perhaps reedit those lines, and perform csh-like - history expansion on previous commands. """ - homepage = "http://cnswww.cns.cwru.edu/php/chet/readline/rltop.html" - url = "ftp://ftp.cwru.edu/pub/bash/readline-6.3.tar.gz" - - version('6.3', '33c8fb279e981274f485fd91da77e94a') - - depends_on("ncurses") - - def install(self, spec, prefix): - configure("--prefix=%s" % prefix) - make("SHLIB_LIBS=-lncurses") - make("install") diff --git a/var/spack/packages/rose/add_spack_compiler_recognition.patch b/var/spack/packages/rose/add_spack_compiler_recognition.patch deleted file mode 100644 index ce61ae4e4c..0000000000 --- a/var/spack/packages/rose/add_spack_compiler_recognition.patch +++ /dev/null @@ -1,13 +0,0 @@ -diff --git a/config/compiler-defs.m4 b/config/compiler-defs.m4 -index d7d85d2..780c8de 100644 ---- a/config/compiler-defs.m4 -+++ b/config/compiler-defs.m4 -@@ -28,7 +28,7 @@ dnl predefined by a specific compiler - # g++|gcc|mpicc|mpic++|mpicxx|mpiCC) - # TOO (2/16/2011): added support for tensilica compilers, assuming they are - # like GCC (they use a GCC front-end) -- g++*|gcc*|mpicc|mpic++|mpicxx|mpiCC|xt-xc++|xt-xcc) -+ cc*|c++*|g++*|gcc*|mpicc|mpic++|mpicxx|mpiCC|xt-xc++|xt-xcc) - BACKEND_GCC_MAJOR=`echo|$BACKEND_CXX_COMPILER -dumpversion | cut -d\. -f1` - BACKEND_GCC_MINOR=`echo|$BACKEND_CXX_COMPILER -dumpversion | cut -d\. -f2` - BACKEND_GCC_PATCHLEVEL=`echo|$BACKEND_CXX_COMPILER -dumpversion | cut -d\. -f3` diff --git a/var/spack/packages/rose/package.py b/var/spack/packages/rose/package.py deleted file mode 100644 index 1d7294acab..0000000000 --- a/var/spack/packages/rose/package.py +++ /dev/null @@ -1,39 +0,0 @@ -#------------------------------------------------------------------------------ -# Author: Justin Too -#------------------------------------------------------------------------------ - -from spack import * - -class Rose(Package): - """A compiler infrastructure to build source-to-source program - transformation and analysis tools. - (Developed at Lawrence Livermore National Lab)""" - - homepage = "http://rosecompiler.org/" - url = "https://github.com/rose-compiler/edg4x-rose" - - version('master', branch='master', git='https://github.com/rose-compiler/edg4x-rose.git') - - patch('add_spack_compiler_recognition.patch') - - depends_on("autoconf@2.69") - depends_on("automake@1.14") - depends_on("libtool@2.4") - depends_on("boost@1.54.0") - depends_on("jdk@8u25-linux-x64") - - def install(self, spec, prefix): - # Bootstrap with autotools - bash = which('bash') - bash('build') - - # Configure, compile & install - with working_dir('rose-build', create=True): - boost = spec['boost'] - - configure = Executable('../configure') - configure("--prefix=" + prefix, - "--with-boost=" + boost.prefix, - "--disable-boost-version-check") - make("install-core") - diff --git a/var/spack/packages/ruby/package.py b/var/spack/packages/ruby/package.py deleted file mode 100644 index 6b6242362c..0000000000 --- a/var/spack/packages/ruby/package.py +++ /dev/null @@ -1,41 +0,0 @@ -from spack import * -import spack -import os - -class Ruby(Package): - """A dynamic, open source programming language with a focus on - simplicity and productivity.""" - - homepage = "https://www.ruby-lang.org/" - url = "http://cache.ruby-lang.org/pub/ruby/2.2/ruby-2.2.0.tar.gz" - - extendable = True - - version('2.2.0', 'cd03b28fd0b555970f5c4fd481700852') - - def install(self, spec, prefix): - configure("--prefix=%s" % prefix) - - make() - make("install") - - def setup_dependent_environment(self, module, spec, ext_spec): - """Called before ruby modules' install() methods. Sets GEM_HOME - and GEM_PATH to values appropriate for the package being built. - - In most cases, extensions will only need to have one line:: - - gem('install', '.gem') - """ - # Ruby extension builds have global ruby and gem functions - module.ruby = Executable(join_path(spec.prefix.bin, 'ruby')) - module.gem = Executable(join_path(spec.prefix.bin, 'gem')) - - # Set GEM_PATH to include dependent gem directories - ruby_paths = [] - for d in ext_spec.traverse(): - if d.package.extends(self.spec): - ruby_paths.append(d.prefix) - os.environ['GEM_PATH'] = ':'.join(ruby_paths) - # The actual installation path for this gem - os.environ['GEM_HOME'] = ext_spec.prefix diff --git a/var/spack/packages/samtools/package.py b/var/spack/packages/samtools/package.py deleted file mode 100644 index 72900398d8..0000000000 --- a/var/spack/packages/samtools/package.py +++ /dev/null @@ -1,18 +0,0 @@ -from spack import * - -class Samtools(Package): - """SAM Tools provide various utilities for manipulating alignments in the SAM format, - including sorting, merging, indexing and generating - alignments in a per-position format""" - - homepage = "www.htslib.org" - version('1.2','988ec4c3058a6ceda36503eebecd4122',url = "https://github.com/samtools/samtools/releases/download/1.2/samtools-1.2.tar.bz2") - - depends_on("zlib") - depends_on("mpc") - parallel=False - patch("samtools1.2.patch",level=0) - - def install(self, spec, prefix): - make("prefix=%s" % prefix, "install") - diff --git a/var/spack/packages/samtools/samtools1.2.patch b/var/spack/packages/samtools/samtools1.2.patch deleted file mode 100644 index ead3ab4e2c..0000000000 --- a/var/spack/packages/samtools/samtools1.2.patch +++ /dev/null @@ -1,20 +0,0 @@ ---- Makefile 2015-02-03 08:27:34.000000000 -0800 -+++ Makefile.new 2015-07-21 10:38:27.881406892 -0700 -@@ -26,7 +26,7 @@ - CFLAGS = -g -Wall -O2 - LDFLAGS = - LDLIBS = --DFLAGS= -D_FILE_OFFSET_BITS=64 -D_LARGEFILE64_SOURCE -D_CURSES_LIB=1 -+DFLAGS= -D_FILE_OFFSET_BITS=64 -D_LARGEFILE64_SOURCE -D_CURSES_LIB=0 - LOBJS= bam_aux.o bam.o bam_import.o sam.o \ - sam_header.o bam_plbuf.o - AOBJS= bam_index.o bam_plcmd.o sam_view.o \ -@@ -37,7 +37,7 @@ - faidx.o stats.o stats_isize.o bam_flags.o bam_split.o \ - bam_tview.o bam_tview_curses.o bam_tview_html.o bam_lpileup.o - INCLUDES= -I. -I$(HTSDIR) --LIBCURSES= -lcurses # -lXCurses -+#LIBCURSES= -lcurses # -lXCurses - - prefix = /usr/local - exec_prefix = $(prefix) diff --git a/var/spack/packages/scalasca/package.py b/var/spack/packages/scalasca/package.py deleted file mode 100644 index cf7a40c1f5..0000000000 --- a/var/spack/packages/scalasca/package.py +++ /dev/null @@ -1,65 +0,0 @@ -# FIXME: Add copyright - -from spack import * - -class Scalasca(Package): - """Scalasca is a software tool that supports the performance optimization - of parallel programs by measuring and analyzing their runtime behavior. - The analysis identifies potential performance bottlenecks - in - particular those concerning communication and synchronization - and - offers guidance in exploring their causes.""" - - # FIXME: add a proper url for your package's homepage here. - homepage = "http://www.scalasca.org" - url = "http://apps.fz-juelich.de/scalasca/releases/scalasca/2.1/dist/scalasca-2.1.tar.gz" - - version('2.1', 'bab9c2b021e51e2ba187feec442b96e6', - url = 'http://apps.fz-juelich.de/scalasca/releases/scalasca/2.1/dist/scalasca-2.1.tar.gz' ) - - depends_on("mpi") - depends_on("otf2@1.4") - depends_on("cube@4.2.3") - - backend_user_provided = """\ -CC=cc -CXX=c++ -F77=f77 -FC=f90 -CFLAGS=-fPIC -CXXFLAGS=-fPIC -""" - frontend_user_provided = """\ -CC_FOR_BUILD=cc -CXX_FOR_BUILD=c++ -F77_FOR_BUILD=f70 -FC_FOR_BUILD=f90 -CFLAGS_FOR_BUILD=-fPIC -CXXFLAGS_FOR_BUILD=-fPIC -""" - mpi_user_provided = """\ -MPICC=mpicc -MPICXX=mpicxx -MPIF77=mpif77 -MPIFC=mpif90 -MPI_CFLAGS=-fPIC -MPI_CXXFLAGS=-fPIC -""" - - def install(self, spec, prefix): - configure_args = ["--prefix=%s" % prefix, - "--with-custom-compilers", - "--with-otf2=%s" % spec['otf2'].prefix.bin, - "--with-cube=%s" % spec['cube'].prefix.bin, - "--enable-shared"] - - configure(*configure_args) - - make() - make("install") - - # FIXME: Modify the configure line to suit your build system here. - configure("--prefix=%s" % prefix) - - # FIXME: Add logic to build and install here - make() - make("install") diff --git a/var/spack/packages/scorep/package.py b/var/spack/packages/scorep/package.py deleted file mode 100644 index f013bd1cbb..0000000000 --- a/var/spack/packages/scorep/package.py +++ /dev/null @@ -1,74 +0,0 @@ -# FIXME: Add copyright statement - -from spack import * - -class Scorep(Package): - """The Score-P measurement infrastructure is a highly scalable and - easy-to-use tool suite for profiling, event tracing, and online - analysis of HPC applications.""" - - # FIXME: add a proper url for your package's homepage here. - homepage = "http://www.vi-hps.org/projects/score-p" - url = "http://www.vi-hps.org/upload/packages/scorep/scorep-1.2.3.tar.gz" - - version('1.3', '9db6f957b7f51fa01377a9537867a55c', - url = 'http://www.vi-hps.org/upload/packages/scorep/scorep-1.3.tar.gz') - - version('1.2.3', '4978084e7cbd05b94517aa8beaea0817') - - depends_on("mpi") - depends_on("papi") - # depends_on("otf2@1.2:1.2.1") # only Score-P 1.2.x - depends_on("otf2") - depends_on("opari2") - depends_on("cube@4.2:4.2.3") - - backend_user_provided = """\ -CC=cc -CXX=c++ -F77=f77 -FC=f90 -CFLAGS=-fPIC -CXXFLAGS=-fPIC -""" - frontend_user_provided = """\ -CC_FOR_BUILD=cc -CXX_FOR_BUILD=c++ -F77_FOR_BUILD=f70 -FC_FOR_BUILD=f90 -CFLAGS_FOR_BUILD=-fPIC -CXXFLAGS_FOR_BUILD=-fPIC -""" - mpi_user_provided = """\ -MPICC=mpicc -MPICXX=mpicxx -MPIF77=mpif77 -MPIFC=mpif90 -MPI_CFLAGS=-fPIC -MPI_CXXFLAGS=-fPIC -""" - - def install(self, spec, prefix): - # Use a custom compiler configuration, otherwise the score-p - # build system messes with spack's compiler settings. - # Create these three files in the build directory - with open("platform-backend-user-provided", "w") as backend_file: - backend_file.write(self.backend_user_provided) - with open("platform-frontend-user-provided", "w") as frontend_file: - frontend_file.write(self.frontend_user_provided) - with open("platform-mpi-user-provided", "w") as mpi_file: - mpi_file.write(self.mpi_user_provided) - - configure_args = ["--prefix=%s" % prefix, - "--with-custom-compilers", - "--with-otf2=%s" % spec['otf2'].prefix.bin, - "--with-opari2=%s" % spec['opari2'].prefix.bin, - "--with-cube=%s" % spec['cube'].prefix.bin, - "--with-papi-header=%s" % spec['papi'].prefix.include, - "--with-papi-lib=%s" % spec['papi'].prefix.lib, - "--enable-shared"] - - configure(*configure_args) - - make() - make("install") diff --git a/var/spack/packages/scotch/package.py b/var/spack/packages/scotch/package.py deleted file mode 100644 index 79289ff2ad..0000000000 --- a/var/spack/packages/scotch/package.py +++ /dev/null @@ -1,40 +0,0 @@ -from spack import * -import glob -import os - -class Scotch(Package): - """Scotch is a software package for graph and mesh/hypergraph - partitioning, graph clustering, and sparse matrix ordering.""" - homepage = "http://www.labri.fr/perso/pelegrin/scotch/" - url = "http://gforge.inria.fr/frs/download.php/file/34099/scotch_6.0.3.tar.gz" - list_url = "http://gforge.inria.fr/frs/?group_id=248" - - version('6.0.3', '10b0cc0f184de2de99859eafaca83cfc') - - depends_on('mpi') - - - def patch(self): - with working_dir('src/Make.inc'): - makefiles = glob.glob('Makefile.inc.x86-64_pc_linux2*') - filter_file(r'^CCS\s*=.*$', 'CCS = cc', *makefiles) - filter_file(r'^CCD\s*=.*$', 'CCD = cc', *makefiles) - - - def install(self, spec, prefix): - # Currently support gcc and icc on x86_64 (maybe others with - # vanilla makefile) - makefile = 'Make.inc/Makefile.inc.x86-64_pc_linux2' - if spec.satisfies('%icc'): - makefile += '.icc' - - with working_dir('src'): - force_symlink(makefile, 'Makefile.inc') - for app in ('scotch', 'ptscotch'): - make(app) - - install_tree('bin', prefix.bin) - install_tree('lib', prefix.lib) - install_tree('include', prefix.include) - install_tree('man/man1', prefix.share_man1) - diff --git a/var/spack/packages/scr/package.py b/var/spack/packages/scr/package.py deleted file mode 100644 index 9fb758f072..0000000000 --- a/var/spack/packages/scr/package.py +++ /dev/null @@ -1,44 +0,0 @@ -############################################################################## -# Copyright (c) 2013, Lawrence Livermore National Security, LLC. -# Produced at the Lawrence Livermore National Laboratory. -# -# This file is part of Spack. -# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. -# LLNL-CODE-647188 -# -# For details, see https://scalability-llnl.github.io/spack -# Please also see the LICENSE file for our notice and the LGPL. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License (as published by -# the Free Software Foundation) version 2.1 dated February 1999. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and -# conditions of the GNU General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -############################################################################## -from spack import * - -class Scr(Package): - """SCR caches checkpoint data in storage on the compute nodes of a - Linux cluster to provide a fast, scalable checkpoint/restart - capability for MPI codes""" - - homepage = "https://computation.llnl.gov/project/scr/" - - depends_on("mpi") -# depends_on("dtcmp") - - version('1.1-7', 'a5930e9ab27d1b7049447c2fd7734ebd', url='http://downloads.sourceforge.net/project/scalablecr/releases/scr-1.1-7.tar.gz') - version('1.1.8', '6a0f11ad18e27fcfc00a271ff587b06e', url='https://github.com/hpc/scr/releases/download/v1.1.8/scr-1.1.8.tar.gz') - - def install(self, spec, prefix): - configure("--prefix=" + prefix, - "--with-scr-config-file=" + prefix + "/etc/scr.conf") - make() - make("install") diff --git a/var/spack/packages/silo/package.py b/var/spack/packages/silo/package.py deleted file mode 100644 index 9eda11df15..0000000000 --- a/var/spack/packages/silo/package.py +++ /dev/null @@ -1,19 +0,0 @@ -from spack import * - -class Silo(Package): - """Silo is a library for reading and writing a wide variety of scientific data to binary, disk files.""" - - homepage = "http://wci.llnl.gov/simulation/computer-codes/silo" - url = "https://wci.llnl.gov/content/assets/docs/simulation/computer-codes/silo/silo-4.8/silo-4.8.tar.gz" - - #version('4.9', 'a83eda4f06761a86726e918fc55e782a') - version('4.8', 'b1cbc0e7ec435eb656dc4b53a23663c9') - - depends_on("hdf5@:1.8.12") - - def install(self, spec, prefix): - configure("--prefix=%s" % prefix, - "--with-hdf5=%s" %spec['hdf5'].prefix) - - make() - make("install") diff --git a/var/spack/packages/snappy/package.py b/var/spack/packages/snappy/package.py deleted file mode 100644 index c8f9ceef7d..0000000000 --- a/var/spack/packages/snappy/package.py +++ /dev/null @@ -1,15 +0,0 @@ -import os -from spack import * - -class Snappy(Package): - """A fast compressor/decompressor: https://code.google.com/p/snappy""" - - homepage = "https://code.google.com/p/snappy" - url = "https://github.com/google/snappy/releases/download/1.1.3/snappy-1.1.3.tar.gz" - - version('1.1.3', '7358c82f133dc77798e4c2062a749b73') - - def install(self, spec, prefix): - configure("--prefix=" + prefix) - make() - make("install") diff --git a/var/spack/packages/spindle/package.py b/var/spack/packages/spindle/package.py deleted file mode 100644 index 06a1e14284..0000000000 --- a/var/spack/packages/spindle/package.py +++ /dev/null @@ -1,44 +0,0 @@ -############################################################################## -# Copyright (c) 2014, Lawrence Livermore National Security, LLC. -# Produced at the Lawrence Livermore National Laboratory. -# -# This file is part of Spack. -# Written by Matthew LeGendre, legendre1@llnl.gov, All rights reserved. -# LLNL-CODE-647188 -# -# For details, see https://scalability-llnl.github.io/spack -# Please also see the LICENSE file for our notice and the LGPL. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License (as published by -# the Free Software Foundation) version 2.1 dated February 1999. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and -# conditions of the GNU General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -############################################################################## -from spack import * - -class Spindle(Package): - """Spindle improves the library-loading performance of dynamically - linked HPC applications. Without Spindle large MPI jobs can - overload on a shared file system when loading dynamically - linked libraries, causing site-wide performance problems. - """ - homepage = "https://computation.llnl.gov/project/spindle/" - url = "https://github.com/hpc/Spindle/archive/v0.8.1.tar.gz" - list_url = "https://github.com/hpc/Spindle/releases" - - version('0.8.1', 'f11793a6b9d8df2cd231fccb2857d912') - - depends_on("launchmon") - - def install(self, spec, prefix): - configure("--prefix=" + prefix) - make() - make("install") diff --git a/var/spack/packages/sqlite/package.py b/var/spack/packages/sqlite/package.py deleted file mode 100644 index 734b0b6cb6..0000000000 --- a/var/spack/packages/sqlite/package.py +++ /dev/null @@ -1,40 +0,0 @@ -############################################################################## -# Copyright (c) 2013, Lawrence Livermore National Security, LLC. -# Produced at the Lawrence Livermore National Laboratory. -# -# This file is part of Spack. -# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. -# LLNL-CODE-647188 -# -# For details, see https://scalability-llnl.github.io/spack -# Please also see the LICENSE file for our notice and the LGPL. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License (as published by -# the Free Software Foundation) version 2.1 dated February 1999. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and -# conditions of the GNU General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -############################################################################## -from spack import * - -class Sqlite(Package): - """SQLite3 is an SQL database engine in a C library. Programs that - link the SQLite3 library can have SQL database access without - running a separate RDBMS process. - """ - homepage = "www.sqlite.org" - - version('3.8.5', '0544ef6d7afd8ca797935ccc2685a9ed', - url='http://www.sqlite.org/2014/sqlite-autoconf-3080500.tar.gz') - - def install(self, spec, prefix): - configure("--prefix=" + prefix) - make() - make("install") diff --git a/var/spack/packages/stat/configure_mpicxx.patch b/var/spack/packages/stat/configure_mpicxx.patch deleted file mode 100644 index e09056d95c..0000000000 --- a/var/spack/packages/stat/configure_mpicxx.patch +++ /dev/null @@ -1,19 +0,0 @@ -commit 07ab6e565f939c54fff6580fc8463ea61662871a -Author: Gregory L. Lee -Date: Tue May 20 14:53:35 2014 -0700 - - re-boostrap to update configure - -diff --git a/configure b/configure -index 6c4af7d..30901ea 100755 ---- a/configure -+++ b/configure -@@ -15529,7 +15529,7 @@ fi - done - test -n "$MPICC" || MPICC="$CC" - -- for ac_prog in mpig++ mpiicpc mpxlC mpixlC -+ for ac_prog in mpig++ mpiCC mpicxx mpiicpc mpxlC mpixlC - do - # Extract the first word of "$ac_prog", so it can be a program name with args. - set dummy $ac_prog; ac_word=$2 diff --git a/var/spack/packages/stat/package.py b/var/spack/packages/stat/package.py deleted file mode 100644 index 5d81e62731..0000000000 --- a/var/spack/packages/stat/package.py +++ /dev/null @@ -1,40 +0,0 @@ -from spack import * - -class Stat(Package): - """Library to create, manipulate, and export graphs Graphlib.""" - homepage = "http://paradyn.org/STAT/STAT.html" - url = "https://github.com/lee218llnl/stat/archive/v2.0.0.tar.gz" - - version('2.2.0', '26bd69dd57a15afdd5d0ebdb0b7fb6fc') - version('2.1.0', 'ece26beaf057aa9134d62adcdda1ba91') - version('2.0.0', 'c7494210b0ba26b577171b92838e1a9b') - - variant('dysect', default=False, description="enable DySectAPI") - - depends_on('libelf') - depends_on('libdwarf') - depends_on('dyninst') - depends_on('graphlib') - depends_on('graphviz') - depends_on('launchmon') - depends_on('mrnet') - - patch('configure_mpicxx.patch', when='@2.1.0') - - def install(self, spec, prefix): - configure_args = [ - "--enable-gui", - "--prefix=%s" % prefix, - "--disable-examples", # Examples require MPI: avoid this dependency. - "--with-launchmon=%s" % spec['launchmon'].prefix, - "--with-mrnet=%s" % spec['mrnet'].prefix, - "--with-graphlib=%s" % spec['graphlib'].prefix, - "--with-stackwalker=%s" % spec['dyninst'].prefix, - "--with-libdwarf=%s" % spec['libdwarf'].prefix - ] - if '+dysect' in spec: - configure_args.append('--enable-dysectapi') - configure(*configure_args) - - make(parallel=False) - make("install") diff --git a/var/spack/packages/sundials/package.py b/var/spack/packages/sundials/package.py deleted file mode 100644 index 8b784c8c3c..0000000000 --- a/var/spack/packages/sundials/package.py +++ /dev/null @@ -1,39 +0,0 @@ -############################################################################## -# Copyright (c) 2013, Lawrence Livermore National Security, LLC. -# Produced at the Lawrence Livermore National Laboratory. -# -# This file is part of Spack. -# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. -# LLNL-CODE-647188 -# -# For details, see https://scalability-llnl.github.io/spack -# Please also see the LICENSE file for our notice and the LGPL. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License (as published by -# the Free Software Foundation) version 2.1 dated February 1999. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and -# conditions of the GNU General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -############################################################################## -from spack import * - -class Sundials(Package): - """SUNDIALS (SUite of Nonlinear and DIfferential/ALgebraic equation Solvers)""" - homepage = "http://computation.llnl.gov/casc/sundials/" - url = "http://computation.llnl.gov/casc/sundials/download/code/sundials-2.5.0.tar.gz" - - version('2.5.0', 'aba8b56eec600de3109cfb967aa3ba0f') - - depends_on("mpi") - - def install(self, spec, prefix): - configure("--prefix=%s" % prefix) - make() - make("install") diff --git a/var/spack/packages/swig/package.py b/var/spack/packages/swig/package.py deleted file mode 100644 index ee536d7063..0000000000 --- a/var/spack/packages/swig/package.py +++ /dev/null @@ -1,46 +0,0 @@ -############################################################################## -# Copyright (c) 2014, Lawrence Livermore National Security, LLC. -# Produced at the Lawrence Livermore National Laboratory. -# -# This file is part of Spack. -# Written by Matthew LeGendre, legendre1@llnl.gov, All rights reserved. -# LLNL-CODE-647188 -# -# For details, see https://scalability-llnl.github.io/spack -# Please also see the LICENSE file for our notice and the LGPL. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License (as published by -# the Free Software Foundation) version 2.1 dated February 1999. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and -# conditions of the GNU General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -############################################################################## -from spack import * - -class Swig(Package): - """SWIG is an interface compiler that connects programs written in - C and C++ with scripting languages such as Perl, Python, Ruby, - and Tcl. It works by taking the declarations found in C/C++ - header files and using them to generate the wrapper code that - scripting languages need to access the underlying C/C++ - code. In addition, SWIG provides a variety of customization - features that let you tailor the wrapping process to suit your - application.""" - homepage = "http://www.swig.org" - url = "http://prdownloads.sourceforge.net/swig/swig-3.0.2.tar.gz" - - version('3.0.2', '62f9b0d010cef36a13a010dc530d0d41') - - depends_on('pcre') - - def install(self, spec, prefix): - configure("--prefix=%s" % prefix) - make() - make("install") diff --git a/var/spack/packages/task/package.py b/var/spack/packages/task/package.py deleted file mode 100644 index 07f44cc45b..0000000000 --- a/var/spack/packages/task/package.py +++ /dev/null @@ -1,20 +0,0 @@ -from spack import * - -class Task(Package): - """Feature-rich console based todo list manager""" - homepage = "http://www.taskwarrior.org" - url = "http://taskwarrior.org/download/task-2.4.4.tar.gz" - - version('2.4.4', '517450c4a23a5842df3e9905b38801b3') - - depends_on("gnutls") - depends_on("libuuid") - # depends_on("gcc@4.8:") - - def install(self, spec, prefix): - with working_dir('spack-build', create=True): - cmake('-DCMAKE_BUILD_TYPE=release', - '..', - *std_cmake_args) - make() - make("install") diff --git a/var/spack/packages/taskd/package.py b/var/spack/packages/taskd/package.py deleted file mode 100644 index 66bc0cb484..0000000000 --- a/var/spack/packages/taskd/package.py +++ /dev/null @@ -1,20 +0,0 @@ -from spack import * - -class Taskd(Package): - """TaskWarrior task synchronization daemon""" - # FIXME: add a proper url for your package's homepage here. - homepage = "http://www.taskwarrior.org" - url = "http://taskwarrior.org/download/taskd-1.1.0.tar.gz" - - version('1.1.0', 'ac855828c16f199bdbc45fbc227388d0') - - depends_on("libuuid") - depends_on("gnutls") - - def install(self, spec, prefix): - with working_dir('spack-build', create=True): - cmake('-DCMAKE_BUILD_TYPE=release', - '..', - *std_cmake_args) - make() - make("install") diff --git a/var/spack/packages/tau/package.py b/var/spack/packages/tau/package.py deleted file mode 100644 index 048fac80aa..0000000000 --- a/var/spack/packages/tau/package.py +++ /dev/null @@ -1,36 +0,0 @@ -from spack import * - -import os -from llnl.util.filesystem import join_path - -class Tau(Package): - """A portable profiling and tracing toolkit for performance - analysis of parallel programs written in Fortran, C, C++, UPC, - Java, Python.""" - homepage = "http://www.cs.uoregon.edu/research/tau" - url = "http://www.cs.uoregon.edu/research/paracomp/tau/tauprofile/dist/tau-2.23.1.tar.gz" - - version('2.23.1', '6593b47ae1e7a838e632652f0426fe72') - - def install(self, spec, prefix): - # TAU isn't happy with directories that have '@' in the path. Sigh. - change_sed_delimiter('@', ';', 'configure') - change_sed_delimiter('@', ';', 'utils/FixMakefile') - change_sed_delimiter('@', ';', 'utils/FixMakefile.sed.default') - - # After that, it's relatively standard. - configure("-prefix=%s" % prefix) - make("install") - - # Link arch-specific directories into prefix since there is - # only one arch per prefix the way spack installs. - self.link_tau_arch_dirs() - - - def link_tau_arch_dirs(self): - for subdir in os.listdir(self.prefix): - for d in ('bin', 'lib'): - src = join_path(self.prefix, subdir, d) - dest = join_path(self.prefix, d) - if os.path.isdir(src) and not os.path.exists(dest): - os.symlink(join_path(subdir, d), dest) diff --git a/var/spack/packages/tcl/package.py b/var/spack/packages/tcl/package.py deleted file mode 100644 index 529adf7788..0000000000 --- a/var/spack/packages/tcl/package.py +++ /dev/null @@ -1,22 +0,0 @@ -from spack import * - -class Tcl(Package): - """Tcl (Tool Command Language) is a very powerful but easy to - learn dynamic programming language, suitable for a very wide - range of uses, including web and desktop applications, - networking, administration, testing and many more. Open source - and business-friendly, Tcl is a mature yet evolving language - that is truly cross platform, easily deployed and highly - extensible.""" - homepage = "http://www.tcl.tk" - - version('8.6.3', 'db382feca91754b7f93da16dc4cdad1f', - url="http://prdownloads.sourceforge.net/tcl/tcl8.6.3-src.tar.gz") - - depends_on('zlib') - - def install(self, spec, prefix): - with working_dir('unix'): - configure("--prefix=%s" % prefix) - make() - make("install") diff --git a/var/spack/packages/the_silver_searcher/package.py b/var/spack/packages/the_silver_searcher/package.py deleted file mode 100644 index e4020b6766..0000000000 --- a/var/spack/packages/the_silver_searcher/package.py +++ /dev/null @@ -1,17 +0,0 @@ -from spack import * - -class TheSilverSearcher(Package): - """Fast recursive grep alternative""" - homepage = "http://geoff.greer.fm/ag/" - url = "http://geoff.greer.fm/ag/releases/the_silver_searcher-0.30.0.tar.gz" - - version('0.30.0', '95e2e7859fab1156c835aff7413481db') - - depends_on('pcre') - depends_on('xz') - - def install(self, spec, prefix): - configure("--prefix=%s" % prefix) - - make() - make("install") diff --git a/var/spack/packages/thrift/package.py b/var/spack/packages/thrift/package.py deleted file mode 100644 index 0e15052f64..0000000000 --- a/var/spack/packages/thrift/package.py +++ /dev/null @@ -1,44 +0,0 @@ -from spack import * - -class Thrift(Package): - """The Apache Thrift software framework, for scalable cross-language services - development, combines a software stack with a code generation engine to build - services that work efficiently and seamlessly between C++, Java, Python, PHP, - Ruby, Erlang, Perl, Haskell, C#, Cocoa, JavaScript, Node.js, Smalltalk, OCaml - and Delphi and other languages.""" - - homepage = "http://thrift.apache.org" - url = "http://apache.mirrors.ionfish.org/thrift/0.9.2/thrift-0.9.2.tar.gz" - - version('0.9.2', '89f63cc4d0100912f4a1f8a9dee63678') - - extends("python") - - depends_on("autoconf") - depends_on("automake") - depends_on("bison") - depends_on("boost") - depends_on("flex") - depends_on("jdk") - depends_on("libtool") - depends_on("openssl") - depends_on("python") - - # Compilation fails for most languages, fortunately cpp installs fine - # All other languages (yes, including C) are omitted until someone needs them - def install(self, spec, prefix): - env["PY_PREFIX"] = prefix - env["JAVA_PREFIX"] = prefix - - configure("--prefix=%s" % prefix, - "--with-boost=%s" % spec['boost'].prefix, - "--with-c=no", - "--with-go=no", - "--with-python=yes", - "--with-lua=no", - "--with-php=no", - "--with-qt4=no", - "--enable-tests=no") - - make() - make("install") diff --git a/var/spack/packages/tk/package.py b/var/spack/packages/tk/package.py deleted file mode 100644 index 96736f6f95..0000000000 --- a/var/spack/packages/tk/package.py +++ /dev/null @@ -1,22 +0,0 @@ -from spack import * - -class Tk(Package): - """Tk is a graphical user interface toolkit that takes developing - desktop applications to a higher level than conventional - approaches. Tk is the standard GUI not only for Tcl, but for - many other dynamic languages, and can produce rich, native - applications that run unchanged across Windows, Mac OS X, Linux - and more.""" - homepage = "http://www.tcl.tk" - url = "http://prdownloads.sourceforge.net/tcl/tk8.6.3-src.tar.gz" - - version('src', '85ca4dbf4dcc19777fd456f6ee5d0221') - - depends_on("tcl") - - def install(self, spec, prefix): - with working_dir('unix'): - configure("--prefix=%s" % prefix, - "--with-tcl=%s" % spec['tcl'].prefix.lib) - make() - make("install") diff --git a/var/spack/packages/tmux/package.py b/var/spack/packages/tmux/package.py deleted file mode 100644 index 23d36db427..0000000000 --- a/var/spack/packages/tmux/package.py +++ /dev/null @@ -1,24 +0,0 @@ -from spack import * - -class Tmux(Package): - """tmux is a terminal multiplexer. What is a terminal multiplexer? It lets - you switch easily between several programs in one terminal, detach them (they - keep running in the background) and reattach them to a different terminal. And - do a lot more. - """ - - homepage = "http://tmux.sourceforge.net" - url = "http://downloads.sourceforge.net/project/tmux/tmux/tmux-1.9/tmux-1.9a.tar.gz" - - version('1.9a', 'b07601711f96f1d260b390513b509a2d') - - depends_on('libevent') - depends_on('ncurses') - - def install(self, spec, prefix): - configure( - "--prefix=%s" % prefix, - "PKG_CONFIG_PATH=%s:%s" % (spec['libevent'].prefix, spec['ncurses'].prefix)) - - make() - make("install") diff --git a/var/spack/packages/tmuxinator/package.py b/var/spack/packages/tmuxinator/package.py deleted file mode 100644 index 26c061cbd6..0000000000 --- a/var/spack/packages/tmuxinator/package.py +++ /dev/null @@ -1,17 +0,0 @@ -from spack import * - -class Tmuxinator(Package): - """A session configuration creator and manager for tmux""" - homepage = "https://github.com/tmuxinator/tmuxinator" - url = "https://github.com/tmuxinator/tmuxinator" - - version('0.6.11', - git='https://github.com/tmuxinator/tmuxinator', - tag='v0.6.11') - - extends('ruby') - - def install(self, spec, prefix): - gem('build', 'tmuxinator.gemspec') - gem('install', 'tmuxinator-{}.gem'.format(self.version)) - diff --git a/var/spack/packages/trilinos/package.py b/var/spack/packages/trilinos/package.py deleted file mode 100644 index 7c43f796a4..0000000000 --- a/var/spack/packages/trilinos/package.py +++ /dev/null @@ -1,50 +0,0 @@ -from spack import * - - -class Trilinos(Package): - """ - The Trilinos Project is an effort to develop algorithms and enabling technologies within an object-oriented - software framework for the solution of large-scale, complex multi-physics engineering and scientific problems. - A unique design feature of Trilinos is its focus on packages. - """ - homepage = "https://trilinos.org/" - url = "http://trilinos.csbsju.edu/download/files/trilinos-12.2.1-Source.tar.gz" - - version('12.2.1', '6161926ea247863c690e927687f83be9') - version('12.0.1', 'bd99741d047471e127b8296b2ec08017') - version('11.14.3', '2f4f83f8333e4233c57d0f01c4b57426') - version('11.14.2', 'a43590cf896c677890d75bfe75bc6254') - version('11.14.1', '40febc57f76668be8b6a77b7607bb67f') - - variant('mpi', default=True, description='Add a dependency on MPI and enables MPI dependent packages') - - # Everything should be compiled with -fpic - depends_on('blas') - depends_on('lapack') - depends_on('boost') - depends_on('netcdf') - depends_on('matio') - depends_on('glm') - depends_on('swig') - depends_on('mpi', when='+mpi') - - def install(self, spec, prefix): - - options = [ - '-DTrilinos_ENABLE_ALL_PACKAGES:BOOL=ON', - '-DTrilinos_ENABLE_TESTS:BOOL=OFF', - '-DTrilinos_ENABLE_EXAMPLES:BOOL=OFF', - '-DBUILD_SHARED_LIBS:BOOL=ON', - '-DBLAS_LIBRARY_DIRS:PATH=%s' % spec['blas'].prefix, - '-DLAPACK_LIBRARY_DIRS:PATH=%s' % spec['lapack'].prefix - ] - if '+mpi' in spec: - mpi_options = ['-DTPL_ENABLE_MPI:BOOL=ON'] - options.extend(mpi_options) - - # -DCMAKE_INSTALL_PREFIX and all the likes... - options.extend(std_cmake_args) - with working_dir('spack-build', create=True): - cmake('..', *options) - make() - make('install') diff --git a/var/spack/packages/uncrustify/package.py b/var/spack/packages/uncrustify/package.py deleted file mode 100644 index d3f2d1b473..0000000000 --- a/var/spack/packages/uncrustify/package.py +++ /dev/null @@ -1,14 +0,0 @@ -from spack import * - -class Uncrustify(Package): - """Source Code Beautifier for C, C++, C#, ObjectiveC, D, Java, Pawn and VALA""" - - homepage = "http://uncrustify.sourceforge.net/" - url = "http://downloads.sourceforge.net/project/uncrustify/uncrustify/uncrustify-0.61/uncrustify-0.61.tar.gz" - - version('0.61', 'b6140106e74c64e831d0b1c4b6cf7727') - - def install(self, spec, prefix): - configure("--prefix=%s" % prefix) - make() - make("install") diff --git a/var/spack/packages/util-linux/package.py b/var/spack/packages/util-linux/package.py deleted file mode 100644 index cb7ceabf57..0000000000 --- a/var/spack/packages/util-linux/package.py +++ /dev/null @@ -1,20 +0,0 @@ -from spack import * -import os - -class UtilLinux(Package): - """Util-linux is a suite of essential utilities for any Linux system.""" - - homepage = "http://freecode.com/projects/util-linux" - url = "https://www.kernel.org/pub/linux/utils/util-linux/v2.25/util-linux-2.25.tar.gz" - - version('2.25', 'f6d7fc6952ec69c4dc62c8d7c59c1d57') - - depends_on("python@2.7:") - - def install(self, spec, prefix): - configure("--prefix=%s" % prefix, - "PKG_CONFIG_PATH=%s/pkgconfig" % spec['python'].prefix.lib, - "--disable-use-tty-group") - - make() - make("install") diff --git a/var/spack/packages/vim/package.py b/var/spack/packages/vim/package.py deleted file mode 100644 index 4099b3257f..0000000000 --- a/var/spack/packages/vim/package.py +++ /dev/null @@ -1,83 +0,0 @@ -from spack import * - -class Vim(Package): - """Vim is a highly configurable text editor built to enable efficient text - editing. It is an improved version of the vi editor distributed with most - UNIX systems. Vim is often called a "programmer's editor," and so useful - for programming that many consider it an entire IDE. It's not just for - programmers, though. Vim is perfect for all kinds of text editing, from - composing email to editing configuration files. - """ - - homepage = "http://www.vim.org" - url = "ftp://ftp.vim.org/pub/vim/unix/vim-7.4.tar.bz2" - list_url = "http://ftp.vim.org/pub/vim/unix/" - - version('7.4', '607e135c559be642f210094ad023dc65') - version('7.3', '5b9510a17074e2b37d8bb38ae09edbf2') - version('7.2', 'f0901284b338e448bfd79ccca0041254') - version('7.1', '44c6b4914f38d6f9aa959640b89da329') - version('7.0', '4ca69757678272f718b1041c810d82d8') - version('6.4', '774c14d93ce58674b3b2c880edd12d77') - version('6.3', '821fda8f14d674346b87e3ef9cb96389') - version('6.2', 'c49d360bbd069d00e2a57804f2a123d9') - version('6.1.405', 'd220ff58f2c72ed606e6d0297c2f2a7c') - version('6.1', '7fd0f915adc7c0dab89772884268b030') - version('6.0', '9d9ca84d489af6b3f54639dd97af3774') - - feature_sets = ('huge', 'big', 'normal', 'small', 'tiny') - for fs in feature_sets: - variant(fs, default=False, description="Use '%s' feature set" % fs) - - variant('python', default=False, description="build with Python") - depends_on('python', when='+python') - - variant('ruby', default=False, description="build with Ruby") - depends_on('ruby', when='+ruby') - - variant('cscope', default=False, description="build with cscope support") - depends_on('cscope', when='+cscope') - - variant('gui', default=False, description="build with gui (gvim)") - # virtual dependency? - - def install(self, spec, prefix): - feature_set = None - for fs in self.feature_sets: - if "+" + fs in spec: - if feature_set is not None: - tty.error("Only one feature set allowed, both %s and %s specified" - % (feature_set, fs)) - feature_set = fs - if '+gui' in spec: - if feature_set is not None: - if feature_set is not 'huge': - tty.error("+gui variant requires 'huge' feature set, %s was specified" - % feature_set) - feature_set = 'huge' - if feature_set is None: - feature_set = 'normal' - - configure_args = [] - configure_args.append("--with-features=" + feature_set) - - if '+python' in spec: - configure_args.append("--enable-pythoninterp=yes") - else: - configure_args.append("--enable-pythoninterp=dynamic") - - if '+ruby' in spec: - configure_args.append("--enable-rubyinterp=yes") - else: - configure_args.append("--enable-rubyinterp=dynamic") - - if '+gui' in spec: - configure_args.append("--enable-gui=auto") - - if '+cscope' in spec: - configure_args.append("--enable-cscope") - - configure("--prefix=%s" % prefix, *configure_args) - - make() - make("install") diff --git a/var/spack/packages/vtk/package.py b/var/spack/packages/vtk/package.py deleted file mode 100644 index 4a27a8fedb..0000000000 --- a/var/spack/packages/vtk/package.py +++ /dev/null @@ -1,40 +0,0 @@ -from spack import * - -class Vtk(Package): - """The Visualization Toolkit (VTK) is an open-source, freely - available software system for 3D computer graphics, image - processing and visualization. """ - homepage = "http://www.vtk.org" - url = "http://www.vtk.org/files/release/6.1/VTK-6.1.0.tar.gz" - - version('6.1.0', '25e4dfb3bad778722dcaec80cd5dab7d') - - depends_on("qt") - - def install(self, spec, prefix): - with working_dir('spack-build', create=True): - cmake_args = [ - "..", - "-DBUILD_SHARED_LIBS=ON", - # Disable wrappers for other languages. - "-DVTK_WRAP_PYTHON=OFF", - "-DVTK_WRAP_JAVA=OFF", - "-DVTK_WRAP_TCL=OFF"] - cmake_args.extend(std_cmake_args) - - # Enable Qt support here. - cmake_args.extend([ - "-DQT_QMAKE_EXECUTABLE:PATH=%s/qmake" % spec['qt'].prefix.bin, - "-DVTK_Group_Qt:BOOL=ON", - # Ignore webkit because it's hard to build w/Qt - "-DVTK_Group_Qt=OFF", - "-DModule_vtkGUISupportQt:BOOL=ON", - "-DModule_vtkGUISupportQtOpenGL:BOOL=ON" - ]) - - if spec['qt'].satisfies('@5'): - cmake_args.append("-DVTK_QT_VERSION:STRING=5") - - cmake(*cmake_args) - make() - make("install") diff --git a/var/spack/packages/wget/package.py b/var/spack/packages/wget/package.py deleted file mode 100644 index c8fd025122..0000000000 --- a/var/spack/packages/wget/package.py +++ /dev/null @@ -1,21 +0,0 @@ -from spack import * - -class Wget(Package): - """GNU Wget is a free software package for retrieving files using - HTTP, HTTPS and FTP, the most widely-used Internet protocols. It - is a non-interactive commandline tool, so it may easily be called - from scripts, cron jobs, terminals without X-Windows support, - etc.""" - - homepage = "http://www.gnu.org/software/wget/" - url = "http://ftp.gnu.org/gnu/wget/wget-1.16.tar.xz" - - version('1.16', 'fe102975ab3a6c049777883f1bb9ad07') - - depends_on("openssl") - - def install(self, spec, prefix): - configure("--prefix=%s" % prefix, - "--with-ssl=openssl") - make() - make("install") diff --git a/var/spack/packages/wx/package.py b/var/spack/packages/wx/package.py deleted file mode 100644 index 1813a8c8a5..0000000000 --- a/var/spack/packages/wx/package.py +++ /dev/null @@ -1,24 +0,0 @@ -from spack import * - -class Wx(Package): - """wxWidgets is a C++ library that lets developers create - applications for Windows, Mac OS X, Linux and other platforms - with a single code base. It has popular language bindings for - Python, Perl, Ruby and many other languages, and unlike other - cross-platform toolkits, wxWidgets gives applications a truly - native look and feel because it uses the platform's native API - rather than emulating the GUI. It's also extensive, free, - open-source and mature.""" - homepage = "http://www.wxwidgets.org/" - - version('2.8.12', '2fa39da14bc06ea86fe902579fedc5b1', - url="https://sourceforge.net/projects/wxwindows/files/2.8.12/wxWidgets-2.8.12.tar.gz") - version('3.0.1', 'dad1f1cd9d4c370cbc22700dc492da31', - url="https://sourceforge.net/projects/wxwindows/files/3.0.1/wxWidgets-3.0.1.tar.bz2") - - def install(self, spec, prefix): - configure("--prefix=%s" % prefix, "--enable-unicode", "--disable-precomp-headers") - - make(parallel=False) - make("install") - diff --git a/var/spack/packages/wxpropgrid/package.py b/var/spack/packages/wxpropgrid/package.py deleted file mode 100644 index 790cead517..0000000000 --- a/var/spack/packages/wxpropgrid/package.py +++ /dev/null @@ -1,20 +0,0 @@ -from spack import * - -class Wxpropgrid(Package): - """wxPropertyGrid is a property sheet control for wxWidgets. In - other words, it is a specialized two-column grid for editing - properties such as strings, numbers, flagsets, string arrays, - and colours.""" - homepage = "http://wxpropgrid.sourceforge.net/" - url = "http://prdownloads.sourceforge.net/wxpropgrid/wxpropgrid-1.4.15-src.tar.gz" - - version('1.4.15', 'f44b5cd6fd60718bacfabbf7994f1e93') - - depends_on("wx") - - def install(self, spec, prefix): - configure("--prefix=%s" % prefix, "--with-wxdir=%s" % spec['wx'].prefix.bin, "--enable-unicode") - - make() - make("install") - diff --git a/var/spack/packages/xcb-proto/package.py b/var/spack/packages/xcb-proto/package.py deleted file mode 100644 index 17a94bd892..0000000000 --- a/var/spack/packages/xcb-proto/package.py +++ /dev/null @@ -1,15 +0,0 @@ -from spack import * - -class XcbProto(Package): - """Protocol for libxcb""" - - homepage = "http://xcb.freedesktop.org/" - url = "http://xcb.freedesktop.org/dist/xcb-proto-1.11.tar.gz" - - version('1.11', 'c8c6cb72c84f58270f4db1f39607f66a') - - def install(self, spec, prefix): - configure("--prefix=%s" % prefix) - - make() - make("install") diff --git a/var/spack/packages/xz/package.py b/var/spack/packages/xz/package.py deleted file mode 100644 index ba6c9733a7..0000000000 --- a/var/spack/packages/xz/package.py +++ /dev/null @@ -1,20 +0,0 @@ -from spack import * - -class Xz(Package): - """XZ Utils is free general-purpose data compression software with - high compression ratio. XZ Utils were written for POSIX-like - systems, but also work on some not-so-POSIX systems. XZ Utils are - the successor to LZMA Utils.""" - homepage = "http://tukaani.org/xz/" - url = "http://tukaani.org/xz/xz-5.2.0.tar.bz2" - - version('5.2.0', '867cc8611760240ebf3440bd6e170bb9', - url = 'http://tukaani.org/xz/xz-5.2.0.tar.bz2') - version('5.2.2', 'f90c9a0c8b259aee2234c4e0d7fd70af', - url = 'http://tukaani.org/xz/xz-5.2.2.tar.bz2') - - def install(self, spec, prefix): - configure("--prefix=%s" % prefix) - make() - make("install") - diff --git a/var/spack/packages/yasm/package.py b/var/spack/packages/yasm/package.py deleted file mode 100644 index d3a695b16d..0000000000 --- a/var/spack/packages/yasm/package.py +++ /dev/null @@ -1,16 +0,0 @@ -from spack import * - -class Yasm(Package): - """Yasm is a complete rewrite of the NASM-2.11.06 assembler. It - supports the x86 and AMD64 instruction sets, accepts NASM and - GAS assembler syntaxes and outputs binary, ELF32 and ELF64 - object formats.""" - homepage = "http://yasm.tortall.net" - url = "http://www.tortall.net/projects/yasm/releases/yasm-1.3.0.tar.gz" - - version('1.3.0', 'fc9e586751ff789b34b1f21d572d96af') - - def install(self, spec, prefix): - configure("--prefix=%s" % prefix) - make() - make("install") diff --git a/var/spack/packages/zeromq/package.py b/var/spack/packages/zeromq/package.py deleted file mode 100644 index b5a1e3d4cd..0000000000 --- a/var/spack/packages/zeromq/package.py +++ /dev/null @@ -1,20 +0,0 @@ -from spack import * - -class Zeromq(Package): - """ The ZMQ networking/concurrency library and core API """ - homepage = "http://zguide.zeromq.org/" - url = "http://download.zeromq.org/zeromq-4.1.2.tar.gz" - - version('4.1.2', '159c0c56a895472f02668e692d122685') - version('4.1.1', '0a4b44aa085644f25c177f79dc13f253') - version('4.0.7', '9b46f7e7b0704b83638ef0d461fd59ab') - version('4.0.6', 'd47dd09ed7ae6e7fd6f9a816d7f5fdf6') - version('4.0.5', '73c39f5eb01b9d7eaf74a5d899f1d03d') - - depends_on("libsodium") - - def install(self, spec, prefix): - configure("--with-libsodium","--prefix=%s" % prefix) - - make() - make("install") diff --git a/var/spack/packages/zlib/package.py b/var/spack/packages/zlib/package.py deleted file mode 100644 index 2770f781ac..0000000000 --- a/var/spack/packages/zlib/package.py +++ /dev/null @@ -1,18 +0,0 @@ -from spack import * - -class Zlib(Package): - """zlib is designed to be a free, general-purpose, legally unencumbered -- - that is, not covered by any patents -- lossless data-compression library for - use on virtually any computer hardware and operating system. - """ - - homepage = "http://zlib.net" - url = "http://zlib.net/zlib-1.2.8.tar.gz" - - version('1.2.8', '44d667c142d7cda120332623eab69f40') - - def install(self, spec, prefix): - configure("--prefix=%s" % prefix) - - make() - make("install") diff --git a/var/spack/packages/zsh/package.py b/var/spack/packages/zsh/package.py deleted file mode 100644 index 99ef9de2e5..0000000000 --- a/var/spack/packages/zsh/package.py +++ /dev/null @@ -1,16 +0,0 @@ -from spack import * - -class Zsh(Package): - """ The ZSH shell """ - homepage = "http://www.zsh.org" - url = "http://www.zsh.org/pub/zsh-5.0.8.tar.bz2" - - version('5.0.8', 'e6759e8dd7b714d624feffd0a73ba0fe') - - depends_on("pcre") - - def install(self, spec, prefix): - configure('--prefix=%s' % prefix) - - make() - make("install") diff --git a/var/spack/repos/builtin.mock/packages/a/package.py b/var/spack/repos/builtin.mock/packages/a/package.py new file mode 100644 index 0000000000..fa63c08df0 --- /dev/null +++ b/var/spack/repos/builtin.mock/packages/a/package.py @@ -0,0 +1,12 @@ +from spack import * + +class A(Package): + """Simple package with no dependencies""" + + homepage = "http://www.example.com" + url = "http://www.example.com/a-1.0.tar.gz" + + version('1.0', '0123456789abcdef0123456789abcdef') + + def install(self, spec, prefix): + pass diff --git a/var/spack/repos/builtin.mock/packages/b/package.py b/var/spack/repos/builtin.mock/packages/b/package.py new file mode 100644 index 0000000000..cb88aa2157 --- /dev/null +++ b/var/spack/repos/builtin.mock/packages/b/package.py @@ -0,0 +1,12 @@ +from spack import * + +class B(Package): + """Simple package with no dependencies""" + + homepage = "http://www.example.com" + url = "http://www.example.com/b-1.0.tar.gz" + + version('1.0', '0123456789abcdef0123456789abcdef') + + def install(self, spec, prefix): + pass diff --git a/var/spack/repos/builtin.mock/packages/c/package.py b/var/spack/repos/builtin.mock/packages/c/package.py new file mode 100644 index 0000000000..f51b913fa9 --- /dev/null +++ b/var/spack/repos/builtin.mock/packages/c/package.py @@ -0,0 +1,12 @@ +from spack import * + +class C(Package): + """Simple package with no dependencies""" + + homepage = "http://www.example.com" + url = "http://www.example.com/c-1.0.tar.gz" + + version('1.0', '0123456789abcdef0123456789abcdef') + + def install(self, spec, prefix): + pass diff --git a/var/spack/repos/builtin.mock/packages/callpath/package.py b/var/spack/repos/builtin.mock/packages/callpath/package.py new file mode 100644 index 0000000000..5b6b70ba2a --- /dev/null +++ b/var/spack/repos/builtin.mock/packages/callpath/package.py @@ -0,0 +1,41 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + +class Callpath(Package): + homepage = "https://github.com/tgamblin/callpath" + url = "http://github.com/tgamblin/callpath-1.0.tar.gz" + + version(0.8, 'foobarbaz') + version(0.9, 'foobarbaz') + version(1.0, 'foobarbaz') + + depends_on("dyninst") + depends_on("mpi") + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix) + make() + make("install") diff --git a/var/spack/repos/builtin.mock/packages/direct_mpich/package.py b/var/spack/repos/builtin.mock/packages/direct_mpich/package.py new file mode 100644 index 0000000000..2ced82521b --- /dev/null +++ b/var/spack/repos/builtin.mock/packages/direct_mpich/package.py @@ -0,0 +1,36 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + +class DirectMpich(Package): + homepage = "http://www.example.com" + url = "http://www.example.com/direct_mpich-1.0.tar.gz" + + version('1.0', 'foobarbaz') + + depends_on('mpich') + + def install(self, spec, prefix): + pass diff --git a/var/spack/repos/builtin.mock/packages/dyninst/package.py b/var/spack/repos/builtin.mock/packages/dyninst/package.py new file mode 100644 index 0000000000..7998578da1 --- /dev/null +++ b/var/spack/repos/builtin.mock/packages/dyninst/package.py @@ -0,0 +1,44 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + +class Dyninst(Package): + homepage = "https://paradyn.org" + url = "http://www.paradyn.org/release8.1/DyninstAPI-8.1.1.tgz" + + version('8.2', 'cxyzab', + url='http://www.paradyn.org/release8.2/DyninstAPI-8.2.tgz') + version('8.1.2', 'bcxyza', + url='http://www.paradyn.org/release8.1.2/DyninstAPI-8.1.2.tgz') + version('8.1.1', 'abcxyz', + url='http://www.paradyn.org/release8.1/DyninstAPI-8.1.1.tgz') + + depends_on("libelf") + depends_on("libdwarf") + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix) + make() + make("install") diff --git a/var/spack/repos/builtin.mock/packages/e/package.py b/var/spack/repos/builtin.mock/packages/e/package.py new file mode 100644 index 0000000000..76c6b64c7f --- /dev/null +++ b/var/spack/repos/builtin.mock/packages/e/package.py @@ -0,0 +1,12 @@ +from spack import * + +class E(Package): + """Simple package with no dependencies""" + + homepage = "http://www.example.com" + url = "http://www.example.com/e-1.0.tar.gz" + + version('1.0', '0123456789abcdef0123456789abcdef') + + def install(self, spec, prefix): + pass diff --git a/var/spack/repos/builtin.mock/packages/fake/package.py b/var/spack/repos/builtin.mock/packages/fake/package.py new file mode 100644 index 0000000000..fb3c2bdd2e --- /dev/null +++ b/var/spack/repos/builtin.mock/packages/fake/package.py @@ -0,0 +1,34 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + +class Fake(Package): + homepage = "http://www.fake-spack-example.org" + url = "http://www.fake-spack-example.org/downloads/fake-1.0.tar.gz" + + version('1.0', 'foobarbaz') + + def install(self, spec, prefix): + pass diff --git a/var/spack/repos/builtin.mock/packages/git-test/package.py b/var/spack/repos/builtin.mock/packages/git-test/package.py new file mode 100644 index 0000000000..689185463c --- /dev/null +++ b/var/spack/repos/builtin.mock/packages/git-test/package.py @@ -0,0 +1,10 @@ +from spack import * + +class GitTest(Package): + """Mock package that uses git for fetching.""" + homepage = "http://www.git-fetch-example.com" + + version('git', git='to-be-filled-in-by-test') + + def install(self, spec, prefix): + pass diff --git a/var/spack/repos/builtin.mock/packages/hg-test/package.py b/var/spack/repos/builtin.mock/packages/hg-test/package.py new file mode 100644 index 0000000000..462f1e4c3a --- /dev/null +++ b/var/spack/repos/builtin.mock/packages/hg-test/package.py @@ -0,0 +1,10 @@ +from spack import * + +class HgTest(Package): + """Test package that does fetching with mercurial.""" + homepage = "http://www.hg-fetch-example.com" + + version('hg', hg='to-be-filled-in-by-test') + + def install(self, spec, prefix): + pass diff --git a/var/spack/repos/builtin.mock/packages/indirect_mpich/package.py b/var/spack/repos/builtin.mock/packages/indirect_mpich/package.py new file mode 100644 index 0000000000..daf8b4b166 --- /dev/null +++ b/var/spack/repos/builtin.mock/packages/indirect_mpich/package.py @@ -0,0 +1,41 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + +class IndirectMpich(Package): + """Test case for a package that depends on MPI and one of its + dependencies requires a *particular version* of MPI. + """ + + homepage = "http://www.example.com" + url = "http://www.example.com/indirect_mpich-1.0.tar.gz" + + version(1.0, 'foobarbaz') + + depends_on('mpi') + depends_on('direct_mpich') + + def install(self, spec, prefix): + pass diff --git a/var/spack/repos/builtin.mock/packages/libdwarf/package.py b/var/spack/repos/builtin.mock/packages/libdwarf/package.py new file mode 100644 index 0000000000..0b8df04cfb --- /dev/null +++ b/var/spack/repos/builtin.mock/packages/libdwarf/package.py @@ -0,0 +1,44 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * +import os + +# Only build certain parts of dwarf because the other ones break. +dwarf_dirs = ['libdwarf', 'dwarfdump2'] + +class Libdwarf(Package): + homepage = "http://www.prevanders.net/dwarf.html" + url = "http://www.prevanders.net/libdwarf-20130729.tar.gz" + list_url = homepage + + version(20130729, "64b42692e947d5180e162e46c689dfbf") + version(20130207, 'foobarbaz') + version(20111030, 'foobarbaz') + version(20070703, 'foobarbaz') + + depends_on("libelf") + + def install(self, spec, prefix): + pass diff --git a/var/spack/repos/builtin.mock/packages/libelf/package.py b/var/spack/repos/builtin.mock/packages/libelf/package.py new file mode 100644 index 0000000000..94c8f942cd --- /dev/null +++ b/var/spack/repos/builtin.mock/packages/libelf/package.py @@ -0,0 +1,43 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + +class Libelf(Package): + homepage = "http://www.mr511.de/software/english.html" + url = "http://www.mr511.de/software/libelf-0.8.13.tar.gz" + + version('0.8.13', '4136d7b4c04df68b686570afa26988ac') + version('0.8.12', 'e21f8273d9f5f6d43a59878dc274fec7') + version('0.8.10', '9db4d36c283d9790d8fa7df1f4d7b4d9') + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix, + "--enable-shared", + "--disable-dependency-tracking", + "--disable-debug") + make() + + # The mkdir commands in libelf's intsall can fail in parallel + make("install", parallel=False) diff --git a/var/spack/repos/builtin.mock/packages/mpich/package.py b/var/spack/repos/builtin.mock/packages/mpich/package.py new file mode 100644 index 0000000000..f77d3efc5d --- /dev/null +++ b/var/spack/repos/builtin.mock/packages/mpich/package.py @@ -0,0 +1,46 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + +class Mpich(Package): + homepage = "http://www.mpich.org" + url = "http://www.mpich.org/static/downloads/3.0.4/mpich-3.0.4.tar.gz" + list_url = "http://www.mpich.org/static/downloads/" + list_depth = 2 + + variant('debug', default=False, + description="Compile MPICH with debug flags.") + + version('3.0.4', '9c5d5d4fe1e17dd12153f40bc5b6dbc0') + version('3.0.3', 'foobarbaz') + version('3.0.2', 'foobarbaz') + version('3.0.1', 'foobarbaz') + version('3.0', 'foobarbaz') + + provides('mpi@:3', when='@3:') + provides('mpi@:1', when='@:1') + + def install(self, spec, prefix): + pass diff --git a/var/spack/repos/builtin.mock/packages/mpich2/package.py b/var/spack/repos/builtin.mock/packages/mpich2/package.py new file mode 100644 index 0000000000..827b94c8a4 --- /dev/null +++ b/var/spack/repos/builtin.mock/packages/mpich2/package.py @@ -0,0 +1,47 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + +class Mpich2(Package): + homepage = "http://www.mpich.org" + url = "http://www.mpich.org/static/downloads/1.5/mpich2-1.5.tar.gz" + list_url = "http://www.mpich.org/static/downloads/" + list_depth = 2 + + version('1.5', '9c5d5d4fe1e17dd12153f40bc5b6dbc0') + version('1.4', 'foobarbaz') + version('1.3', 'foobarbaz') + version('1.2', 'foobarbaz') + version('1.1', 'foobarbaz') + version('1.0', 'foobarbaz') + + provides('mpi@:2.0') + provides('mpi@:2.1', when='@1.1:') + provides('mpi@:2.2', when='@1.2:') + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix) + make() + make("install") diff --git a/var/spack/repos/builtin.mock/packages/mpileaks/package.py b/var/spack/repos/builtin.mock/packages/mpileaks/package.py new file mode 100644 index 0000000000..3989f1b452 --- /dev/null +++ b/var/spack/repos/builtin.mock/packages/mpileaks/package.py @@ -0,0 +1,43 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + +class Mpileaks(Package): + homepage = "http://www.llnl.gov" + url = "http://www.llnl.gov/mpileaks-1.0.tar.gz" + + version(1.0, 'foobarbaz') + version(2.1, 'foobarbaz') + version(2.2, 'foobarbaz') + version(2.3, 'foobarbaz') + + variant('debug', default=False, description='Debug variant') + variant('opt', default=False, description='Optimized variant') + + depends_on("mpi") + depends_on("callpath") + + def install(self, spec, prefix): + pass diff --git a/var/spack/repos/builtin.mock/packages/multimethod/package.py b/var/spack/repos/builtin.mock/packages/multimethod/package.py new file mode 100644 index 0000000000..75b1606ffc --- /dev/null +++ b/var/spack/repos/builtin.mock/packages/multimethod/package.py @@ -0,0 +1,143 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class Multimethod(Package): + """This package is designed for use with Spack's multimethod test. + It has a bunch of test cases for the @when decorator that the + test uses. + """ + + homepage = 'http://www.example.com/' + url = 'http://www.example.com/example-1.0.tar.gz' + + # + # These functions are only valid for versions 1, 2, and 3. + # + @when('@1.0') + def no_version_2(self): + return 1 + + @when('@3.0') + def no_version_2(self): + return 3 + + @when('@4.0') + def no_version_2(self): + return 4 + + + # + # These functions overlap, so there is ambiguity, but we'll take + # the first one. + # + @when('@:4') + def version_overlap(self): + return 1 + + @when('@2:') + def version_overlap(self): + return 2 + + + # + # More complicated case with cascading versions. + # + def mpi_version(self): + return 0 + + @when('^mpi@3:') + def mpi_version(self): + return 3 + + @when('^mpi@2:') + def mpi_version(self): + return 2 + + @when('^mpi@1:') + def mpi_version(self): + return 1 + + + # + # Use these to test whether the default method is called when no + # match is found. This also tests whether we can switch methods + # on compilers + # + def has_a_default(self): + return 'default' + + @when('%gcc') + def has_a_default(self): + return 'gcc' + + @when('%intel') + def has_a_default(self): + return 'intel' + + + + # + # Make sure we can switch methods on different architectures + # + @when('=x86_64') + def different_by_architecture(self): + return 'x86_64' + + @when('=ppc64') + def different_by_architecture(self): + return 'ppc64' + + @when('=ppc32') + def different_by_architecture(self): + return 'ppc32' + + @when('=arm64') + def different_by_architecture(self): + return 'arm64' + + + # + # Make sure we can switch methods on different dependencies + # + @when('^mpich') + def different_by_dep(self): + return 'mpich' + + @when('^zmpi') + def different_by_dep(self): + return 'zmpi' + + + # + # Make sure we can switch on virtual dependencies + # + def different_by_virtual_dep(self): + return 1 + + @when('^mpi@2:') + def different_by_virtual_dep(self): + return 2 diff --git a/var/spack/repos/builtin.mock/packages/optional-dep-test-2/package.py b/var/spack/repos/builtin.mock/packages/optional-dep-test-2/package.py new file mode 100644 index 0000000000..ef0587588e --- /dev/null +++ b/var/spack/repos/builtin.mock/packages/optional-dep-test-2/package.py @@ -0,0 +1,18 @@ +from spack import * + +class OptionalDepTest2(Package): + """Depends on the optional-dep-test package""" + + homepage = "http://www.example.com" + url = "http://www.example.com/optional-dep-test-2-1.0.tar.gz" + + version('1.0', '0123456789abcdef0123456789abcdef') + + variant('odt', default=False) + variant('mpi', default=False) + + depends_on('optional-dep-test', when='+odt') + depends_on('optional-dep-test+mpi', when='+mpi') + + def install(self, spec, prefix): + pass diff --git a/var/spack/repos/builtin.mock/packages/optional-dep-test-3/package.py b/var/spack/repos/builtin.mock/packages/optional-dep-test-3/package.py new file mode 100644 index 0000000000..e6cb3bd6e7 --- /dev/null +++ b/var/spack/repos/builtin.mock/packages/optional-dep-test-3/package.py @@ -0,0 +1,17 @@ +from spack import * + +class OptionalDepTest3(Package): + """Depends on the optional-dep-test package""" + + homepage = "http://www.example.com" + url = "http://www.example.com/optional-dep-test-3-1.0.tar.gz" + + version('1.0', '0123456789abcdef0123456789abcdef') + + variant('var', default=False) + + depends_on('a', when='~var') + depends_on('b', when='+var') + + def install(self, spec, prefix): + pass diff --git a/var/spack/repos/builtin.mock/packages/optional-dep-test/package.py b/var/spack/repos/builtin.mock/packages/optional-dep-test/package.py new file mode 100644 index 0000000000..bb57576ca9 --- /dev/null +++ b/var/spack/repos/builtin.mock/packages/optional-dep-test/package.py @@ -0,0 +1,29 @@ +from spack import * + +class OptionalDepTest(Package): + """Description""" + + homepage = "http://www.example.com" + url = "http://www.example.com/optional_dep_test-1.0.tar.gz" + + version('1.0', '0123456789abcdef0123456789abcdef') + version('1.1', '0123456789abcdef0123456789abcdef') + + variant('a', default=False) + variant('f', default=False) + variant('mpi', default=False) + + depends_on('a', when='+a') + depends_on('b', when='@1.1') + depends_on('c', when='%intel') + depends_on('d', when='%intel@64.1') + depends_on('e', when='%clang@34:40') + + depends_on('f', when='+f') + depends_on('g', when='^f') + depends_on('mpi', when='^g') + + depends_on('mpi', when='+mpi') + + def install(self, spec, prefix): + pass diff --git a/var/spack/repos/builtin.mock/packages/svn-test/package.py b/var/spack/repos/builtin.mock/packages/svn-test/package.py new file mode 100644 index 0000000000..ba4d5522b4 --- /dev/null +++ b/var/spack/repos/builtin.mock/packages/svn-test/package.py @@ -0,0 +1,10 @@ +from spack import * + +class SvnTest(Package): + """Mock package that uses svn for fetching.""" + url = "http://www.example.com/svn-test-1.0.tar.gz" + + version('svn', 'to-be-filled-in-by-test') + + def install(self, spec, prefix): + pass diff --git a/var/spack/repos/builtin.mock/packages/trivial_install_test_package/package.py b/var/spack/repos/builtin.mock/packages/trivial_install_test_package/package.py new file mode 100644 index 0000000000..c4db9f5f07 --- /dev/null +++ b/var/spack/repos/builtin.mock/packages/trivial_install_test_package/package.py @@ -0,0 +1,38 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + +class TrivialInstallTestPackage(Package): + """This package is a stub with a trivial install method. It allows us + to test the install and uninstall logic of spack.""" + homepage = "http://www.example.com/trivial_install" + url = "http://www.unit-test-should-replace-this-url/trivial_install-1.0.tar.gz" + + version('1.0', 'foobarbaz') + + def install(self, spec, prefix): + configure('--prefix=%s' % prefix) + make() + make('install') diff --git a/var/spack/repos/builtin.mock/packages/zmpi/package.py b/var/spack/repos/builtin.mock/packages/zmpi/package.py new file mode 100644 index 0000000000..8c6ceda6d3 --- /dev/null +++ b/var/spack/repos/builtin.mock/packages/zmpi/package.py @@ -0,0 +1,39 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + +class Zmpi(Package): + """This is a fake MPI package used to demonstrate virtual package providers + with dependencies.""" + homepage = "http://www.spack-fake-zmpi.org" + url = "http://www.spack-fake-zmpi.org/downloads/zmpi-1.0.tar.gz" + + version('1.0', 'foobarbaz') + + provides('mpi@:10.0') + depends_on('fake') + + def install(self, spec, prefix): + pass diff --git a/var/spack/repos/builtin.mock/repo.yaml b/var/spack/repos/builtin.mock/repo.yaml new file mode 100644 index 0000000000..30b068da13 --- /dev/null +++ b/var/spack/repos/builtin.mock/repo.yaml @@ -0,0 +1,2 @@ +repo: + namespace: builtin.mock diff --git a/var/spack/repos/builtin/packages/ImageMagick/package.py b/var/spack/repos/builtin/packages/ImageMagick/package.py new file mode 100644 index 0000000000..753ea80ca6 --- /dev/null +++ b/var/spack/repos/builtin/packages/ImageMagick/package.py @@ -0,0 +1,37 @@ +from spack import * + +class Imagemagick(Package): + """ImageMagick is a image processing library""" + homepage = "http://www.imagemagic.org" + + #------------------------------------------------------------------------- + # ImageMagick does not keep around anything but *-10 versions, so + # this URL may change. If you want the bleeding edge, you can + # uncomment it and see if it works but you may need to try to + # fetch a newer version (-6, -7, -8, -9, etc.) or you can stick + # wtih the older, stable, archived -10 versions below. + # + # TODO: would be nice if spack had a way to recommend avoiding a + # TODO: bleeding edge version, but not comment it out. + # ------------------------------------------------------------------------- + # version('6.9.0-6', 'c1bce7396c22995b8bdb56b7797b4a1b', + # url="http://www.imagemagick.org/download/ImageMagick-6.9.0-6.tar.bz2") + + #------------------------------------------------------------------------- + # *-10 versions are archived, so these versions should fetch reliably. + # ------------------------------------------------------------------------- + version('6.8.9-10', 'aa050bf9785e571c956c111377bbf57c', + url="http://sourceforge.net/projects/imagemagick/files/old-sources/6.x/6.8/ImageMagick-6.8.9-10.tar.gz/download") + + depends_on('libtool') + depends_on('jpeg') + depends_on('libpng') + depends_on('freetype') + depends_on('fontconfig') + depends_on('libtiff') + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix) + + make() + make("install") diff --git a/var/spack/repos/builtin/packages/Mitos/package.py b/var/spack/repos/builtin/packages/Mitos/package.py new file mode 100644 index 0000000000..e312da3ffc --- /dev/null +++ b/var/spack/repos/builtin/packages/Mitos/package.py @@ -0,0 +1,19 @@ +from spack import * + +class Mitos(Package): + """Mitos is a library and a tool for collecting sampled memory + performance data to view with MemAxes""" + + homepage = "https://github.com/scalability-llnl/Mitos" + url = "https://github.com/scalability-llnl/Mitos" + + version('0.9.1', 'c6cb57f3cae54f5157affd97ef7ef79e', git='https://github.com/scalability-llnl/Mitos.git', tag='v0.9.1') + + depends_on('dyninst@8.2.1:') + depends_on('hwloc') + + def install(self, spec, prefix): + with working_dir('spack-build', create=True): + cmake('..', *std_cmake_args) + make() + make("install") diff --git a/var/spack/repos/builtin/packages/R/package.py b/var/spack/repos/builtin/packages/R/package.py new file mode 100644 index 0000000000..2e6f65a742 --- /dev/null +++ b/var/spack/repos/builtin/packages/R/package.py @@ -0,0 +1,33 @@ +from spack import * + +class R(Package): + """R is 'GNU S', a freely available language and environment for + statistical computing and graphics which provides a wide va + riety of statistical and graphical techniques: linear and + nonlinear modelling, statistical tests, time series analysis, + classification, clustering, etc. Please consult the R project + homepage for further information.""" + homepage = "http://www.example.com" + url = "http://cran.cnr.berkeley.edu/src/base/R-3/R-3.1.2.tar.gz" + + version('3.1.2', '3af29ec06704cbd08d4ba8d69250ae74') + + depends_on("readline") + depends_on("ncurses") + depends_on("icu") + depends_on("glib") + depends_on("zlib") + depends_on("libtiff") + depends_on("jpeg") + depends_on("cairo") + depends_on("pango") + depends_on("freetype") + depends_on("tcl") + depends_on("tk") + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix, + "--enable-R-shlib", + "--enable-BLAS-shlib") + make() + make("install") diff --git a/var/spack/repos/builtin/packages/SAMRAI/no-tool-build.patch b/var/spack/repos/builtin/packages/SAMRAI/no-tool-build.patch new file mode 100644 index 0000000000..1adf0cf721 --- /dev/null +++ b/var/spack/repos/builtin/packages/SAMRAI/no-tool-build.patch @@ -0,0 +1,20 @@ +--- SAMRAI/Makefile.in 2013-05-31 11:04:32.000000000 -0700 ++++ SAMRAI/Makefile.in.notools 2014-05-30 10:31:15.135979900 -0700 +@@ -8,7 +8,7 @@ + ## + ######################################################################### + +-default: library tools ++default: library + + SAMRAI = @top_srcdir@ + SUBDIR = . +@@ -135,7 +135,7 @@ + done + $(MAKE) archive_remove_obj_names + +-install: library tools ++install: library + $(INSTALL) -d -m 755 $(INSTDIR)/config + $(INSTALL) -d -m 755 $(INSTDIR)/lib + $(INSTALL) -d -m 755 $(INSTDIR)/bin diff --git a/var/spack/repos/builtin/packages/SAMRAI/package.py b/var/spack/repos/builtin/packages/SAMRAI/package.py new file mode 100644 index 0000000000..eef041f0d5 --- /dev/null +++ b/var/spack/repos/builtin/packages/SAMRAI/package.py @@ -0,0 +1,53 @@ +from spack import * + +class Samrai(Package): + """SAMRAI (Structured Adaptive Mesh Refinement Application Infrastructure) + is an object-oriented C++ software library enables exploration of numerical, + algorithmic, parallel computing, and software issues associated with applying + structured adaptive mesh refinement (SAMR) technology in large-scale parallel + application development. + """ + homepage = "https://computation.llnl.gov/project/SAMRAI/" + url = "https://computation.llnl.gov/project/SAMRAI/download/SAMRAI-v3.9.1.tar.gz" + list_url = homepage + + version('3.9.1', '232d04d0c995f5abf20d94350befd0b2') + version('3.7.3', '12d574eacadf8c9a70f1bb4cd1a69df6') + version('3.7.2', 'f6a716f171c9fdbf3cb12f71fa6e2737') + version('3.6.3-beta', 'ef0510bf2893042daedaca434e5ec6ce') + version('3.5.2-beta', 'd072d9d681eeb9ada15ce91bea784274') + version('3.5.0-beta', '1ad18a319fc573e12e2b1fbb6f6b0a19') + version('3.4.1-beta', '00814cbee2cb76bf8302aff56bbb385b') + version('3.3.3-beta', '1db3241d3e1cab913dc310d736c34388') + version('3.3.2-beta', 'e598a085dab979498fcb6c110c4dd26c') + version('2.4.4', '04fb048ed0efe7c531ac10c81cc5f6ac') + + depends_on("mpi") + depends_on("zlib") + depends_on("hdf5") + depends_on("boost") + + # don't build tools with gcc + patch('no-tool-build.patch', when='%gcc') + + # TODO: currently hard-coded to use openmpi - be careful! + def install(self, spec, prefix): + mpi = next(m for m in ('openmpi', 'mpich', 'mvapich') + if m in spec) + + configure( + "--prefix=%s" % prefix, + "--with-CXX=%s" % spec[mpi].prefix.bin + "/mpic++", + "--with-CC=%s" % spec[mpi].prefix.bin + "/mpicc", + "--with-hdf5=%s" % spec['hdf5'].prefix, + "--with-boost=%s" % spec['boost'].prefix, + "--with-zlib=%s" % spec['zlib'].prefix, + "--without-blas", + "--without-lapack", + "--with-hypre=no", + "--with-petsc=no", + "--enable-opt", + "--disable-debug") + + make() + make("install") diff --git a/var/spack/repos/builtin/packages/activeharmony/package.py b/var/spack/repos/builtin/packages/activeharmony/package.py new file mode 100644 index 0000000000..45dcc7c0e8 --- /dev/null +++ b/var/spack/repos/builtin/packages/activeharmony/package.py @@ -0,0 +1,15 @@ +from spack import * + +class Activeharmony(Package): + """Active Harmony: a framework for auto-tuning (the automated search for values to improve the performance of a target application).""" + homepage = "http://www.dyninst.org/harmony" + url = "http://www.dyninst.org/sites/default/files/downloads/harmony/ah-4.5.tar.gz" + + version('4.5', 'caee5b864716d376e2c25d739251b2a9') + + def install(self, spec, prefix): + make("CFLAGS=-O3") + make("install", 'PREFIX=%s' % prefix) + +from spack import * + diff --git a/var/spack/repos/builtin/packages/adept-utils/package.py b/var/spack/repos/builtin/packages/adept-utils/package.py new file mode 100644 index 0000000000..e4a2e1523f --- /dev/null +++ b/var/spack/repos/builtin/packages/adept-utils/package.py @@ -0,0 +1,42 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + +class AdeptUtils(Package): + """Utility libraries for LLNL performance tools.""" + + homepage = "https://github.com/scalability-llnl/adept-utils" + url = "https://github.com/scalability-llnl/adept-utils/archive/v1.0.tar.gz" + + version('1.0.1', '731a310717adcb004d9d195130efee7d') + version('1.0', '5c6cd9badce56c945ac8551e34804397') + + depends_on("boost") + depends_on("mpi") + + def install(self, spec, prefix): + cmake(*std_cmake_args) + make() + make("install") diff --git a/var/spack/repos/builtin/packages/apex/package.py b/var/spack/repos/builtin/packages/apex/package.py new file mode 100644 index 0000000000..6404d5208a --- /dev/null +++ b/var/spack/repos/builtin/packages/apex/package.py @@ -0,0 +1,34 @@ +from spack import * +from spack.util.environment import * + +class Apex(Package): + homepage = "http://github.com/khuck/xpress-apex" + #url = "http://github.com/khuck/xpress-apex/archive/v0.1-release-candidate.tar.gz" + url = "http://github.com/khuck/xpress-apex" + + #version('0.1', '6e039c224387348296739f6bf360d081') + #version('master', branch='master', git='https://github.com/khuck/xpress-apex.git') + version('2015-10-21', git='https://github.com/khuck/xpress-apex.git', commit='d2e66ddde689120472fc57fc546d8cd80aab745c') + + depends_on("binutils+libiberty") + depends_on("boost@1.54:") + depends_on("cmake@2.8.12:") + depends_on("activeharmony@4.5:") + depends_on("ompt-openmp") + + def install(self, spec, prefix): + + path=get_path("PATH") + path.remove(spec["binutils"].prefix.bin) + path_set("PATH", path) + with working_dir("build", create=True): + cmake('-DBOOST_ROOT=%s' % spec['boost'].prefix, + '-DUSE_BFD=TRUE', + '-DBFD_ROOT=%s' % spec['binutils'].prefix, + '-DUSE_ACTIVEHARMONY=TRUE', + '-DACTIVEHARMONY_ROOT=%s' % spec['activeharmony'].prefix, + '-DUSE_OMPT=TRUE', + '-DOMPT_ROOT=%s' % spec['ompt-openmp'].prefix, + '..', *std_cmake_args) + make() + make("install") diff --git a/var/spack/repos/builtin/packages/arpack/package.py b/var/spack/repos/builtin/packages/arpack/package.py new file mode 100644 index 0000000000..8c67c536f3 --- /dev/null +++ b/var/spack/repos/builtin/packages/arpack/package.py @@ -0,0 +1,41 @@ +from spack import * +import os +import shutil + +class Arpack(Package): + """A collection of Fortran77 subroutines designed to solve large scale + eigenvalue problems. + """ + homepage = "http://www.caam.rice.edu/software/ARPACK/" + url = "http://www.caam.rice.edu/software/ARPACK/SRC/arpack96.tar.gz" + + version('96', 'fffaa970198b285676f4156cebc8626e') + + depends_on('blas') + depends_on('lapack') + + def patch(self): + # Filter the cray makefile to make a spack one. + shutil.move('ARMAKES/ARmake.CRAY', 'ARmake.inc') + makefile = FileFilter('ARmake.inc') + + # Be sure to use Spack F77 wrapper + makefile.filter('^FC.*', 'FC = f77') + makefile.filter('^FFLAGS.*', 'FFLAGS = -O2 -g') + + # Set up some variables. + makefile.filter('^PLAT.*', 'PLAT = ') + makefile.filter('^home.*', 'home = %s' % os.getcwd()) + makefile.filter('^BLASdir.*', 'BLASdir = %s' % self.spec['blas'].prefix) + makefile.filter('^LAPACKdir.*', 'LAPACKdir = %s' % self.spec['lapack'].prefix) + + # build the library in our own prefix. + makefile.filter('^ARPACKLIB.*', 'ARPACKLIB = %s/libarpack.a' % os.getcwd()) + + + def install(self, spec, prefix): + with working_dir('SRC'): + make('all') + + mkdirp(prefix.lib) + install('libarpack.a', prefix.lib) diff --git a/var/spack/repos/builtin/packages/asciidoc/package.py b/var/spack/repos/builtin/packages/asciidoc/package.py new file mode 100644 index 0000000000..828f3b3f4f --- /dev/null +++ b/var/spack/repos/builtin/packages/asciidoc/package.py @@ -0,0 +1,18 @@ +from spack import * + +class Asciidoc(Package): + """ A presentable text document format for writing articles, UNIX man + pages and other small to medium sized documents.""" + homepage = "http://asciidoc.org" + url = "http://downloads.sourceforge.net/project/asciidoc/asciidoc/8.6.9/asciidoc-8.6.9.tar.gz" + + version('8.6.9', 'c59018f105be8d022714b826b0be130a') + + depends_on('libxml2') + depends_on('libxslt') + + def install(self, spec, prefix): + configure('--prefix=%s' % prefix) + + make() + make("install") diff --git a/var/spack/repos/builtin/packages/atk/package.py b/var/spack/repos/builtin/packages/atk/package.py new file mode 100644 index 0000000000..769805b227 --- /dev/null +++ b/var/spack/repos/builtin/packages/atk/package.py @@ -0,0 +1,18 @@ +from spack import * + +class Atk(Package): + """ATK provides the set of accessibility interfaces that are + implemented by other toolkits and applications. Using the ATK + interfaces, accessibility tools have full access to view and + control running applications.""" + homepage = "https://developer.gnome.org/atk/" + url = "http://ftp.gnome.org/pub/gnome/sources/atk/2.14/atk-2.14.0.tar.xz" + + version('2.14.0', 'ecb7ca8469a5650581b1227d78051b8b') + + depends_on("glib") + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix) + make() + make("install") diff --git a/var/spack/repos/builtin/packages/atlas/package.py b/var/spack/repos/builtin/packages/atlas/package.py new file mode 100644 index 0000000000..fc683363a7 --- /dev/null +++ b/var/spack/repos/builtin/packages/atlas/package.py @@ -0,0 +1,60 @@ +from spack import * +from spack.util.executable import Executable +import os + +class Atlas(Package): + """ + Automatically Tuned Linear Algebra Software, generic shared + ATLAS is an approach for the automatic generation and optimization of + numerical software. Currently ATLAS supplies optimized versions for the + complete set of linear algebra kernels known as the Basic Linear Algebra + Subroutines (BLAS), and a subset of the linear algebra routines in the + LAPACK library. + """ + homepage = "http://math-atlas.sourceforge.net/" + + version('3.11.34', '0b6c5389c095c4c8785fd0f724ec6825', + url='http://sourceforge.net/projects/math-atlas/files/Developer%20%28unstable%29/3.11.34/atlas3.11.34.tar.bz2/download') + version('3.10.2', 'a4e21f343dec8f22e7415e339f09f6da', + url='http://downloads.sourceforge.net/project/math-atlas/Stable/3.10.2/atlas3.10.2.tar.bz2') + + # TODO: make this provide BLAS once it works better. Create a way + # TODO: to mark "beta" packages and require explicit invocation. + + # provides('blas') + + + def patch(self): + # Disable thraed check. LLNL's environment does not allow + # disabling of CPU throttling in a way that ATLAS actually + # understands. + filter_file(r'^\s+if \(thrchk\) exit\(1\);', 'if (0) exit(1);', + 'CONFIG/src/config.c') + # TODO: investigate a better way to add the check back in + # TODO: using, say, MSRs. Or move this to a variant. + + @when('@:3.10') + def install(self, spec, prefix): + with working_dir('ATLAS-Build', create=True): + configure = Executable('../configure') + configure('--prefix=%s' % prefix, '-C', 'ic', 'cc', '-C', 'if', 'f77', "--dylibs") + make() + make('check') + make('ptcheck') + make('time') + make("install") + + + def install(self, spec, prefix): + with working_dir('ATLAS-Build', create=True): + configure = Executable('../configure') + configure('--incdir=%s' % prefix.include, + '--libdir=%s' % prefix.lib, + '--cc=cc', + "--shared") + + make() + make('check') + make('ptcheck') + make('time') + make("install") diff --git a/var/spack/repos/builtin/packages/autoconf/package.py b/var/spack/repos/builtin/packages/autoconf/package.py new file mode 100644 index 0000000000..5189faf054 --- /dev/null +++ b/var/spack/repos/builtin/packages/autoconf/package.py @@ -0,0 +1,14 @@ +from spack import * + +class Autoconf(Package): + """Autoconf -- system configuration part of autotools""" + homepage = "https://www.gnu.org/software/autoconf/" + url = "http://ftp.gnu.org/gnu/autoconf/autoconf-2.69.tar.gz" + + version('2.69', '82d05e03b93e45f5a39b828dc9c6c29b') + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix) + + make() + make("install") diff --git a/var/spack/repos/builtin/packages/automaded/package.py b/var/spack/repos/builtin/packages/automaded/package.py new file mode 100644 index 0000000000..9fbd93e3b3 --- /dev/null +++ b/var/spack/repos/builtin/packages/automaded/package.py @@ -0,0 +1,51 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + +class Automaded(Package): + """AutomaDeD (Automata-based Debugging for Dissimilar parallel + tasks) is a tool for automatic diagnosis of performance and + correctness problems in MPI applications. It creates + control-flow models of each MPI process and, when a failure + occurs, these models are leveraged to find the origin of + problems automatically. MPI calls are intercepted (using + wrappers) to create the models. When an MPI application hangs, + AutomaDeD creates a progress-dependence graph that helps + finding the process (or group of processes) that caused the hang. + """ + + homepage = "https://github.com/scalability-llnl/AutomaDeD" + url = "https://github.com/scalability-llnl/AutomaDeD/archive/v1.0.tar.gz" + + version('1.0', '16a3d4def2c4c77d0bc4b21de8b3ab03') + + depends_on('mpi') + depends_on('boost') + depends_on('callpath') + + def install(self, spec, prefix): + cmake("-DSTATE_TRACKER_WITH_CALLPATH=ON", *std_cmake_args) + make() + make("install") diff --git a/var/spack/repos/builtin/packages/automake/package.py b/var/spack/repos/builtin/packages/automake/package.py new file mode 100644 index 0000000000..9115822730 --- /dev/null +++ b/var/spack/repos/builtin/packages/automake/package.py @@ -0,0 +1,16 @@ +from spack import * + +class Automake(Package): + """Automake -- make file builder part of autotools""" + homepage = "http://www.gnu.org/software/automake/" + url = "http://ftp.gnu.org/gnu/automake/automake-1.14.tar.gz" + + version('1.14.1', 'd052a3e884631b9c7892f2efce542d75') + + depends_on('autoconf') + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix) + + make() + make("install") diff --git a/var/spack/repos/builtin/packages/bear/package.py b/var/spack/repos/builtin/packages/bear/package.py new file mode 100644 index 0000000000..0d4436fccc --- /dev/null +++ b/var/spack/repos/builtin/packages/bear/package.py @@ -0,0 +1,17 @@ +from spack import * + +class Bear(Package): + """Bear is a tool that generates a compilation database for clang tooling from non-cmake build systems.""" + homepage = "https://github.com/rizsotto/Bear" + url = "https://github.com/rizsotto/Bear/archive/2.0.4.tar.gz" + + version('2.0.4', 'fd8afb5e8e18f8737ba06f90bd77d011') + + depends_on("cmake") + depends_on("python") + + def install(self, spec, prefix): + cmake('.', *std_cmake_args) + + make("all") + make("install") diff --git a/var/spack/repos/builtin/packages/bib2xhtml/package.py b/var/spack/repos/builtin/packages/bib2xhtml/package.py new file mode 100644 index 0000000000..7f8e0cfe5a --- /dev/null +++ b/var/spack/repos/builtin/packages/bib2xhtml/package.py @@ -0,0 +1,27 @@ +from spack import * +from glob import glob + +class Bib2xhtml(Package): + """bib2xhtml is a program that converts BibTeX files into HTML.""" + homepage = "http://www.spinellis.gr/sw/textproc/bib2xhtml/" + url='http://www.spinellis.gr/sw/textproc/bib2xhtml/bib2xhtml-v3.0-15-gf506.tar.gz' + + version('3.0-15-gf506', 'a26ba02fe0053bbbf2277bdf0acf8645') + + def url_for_version(self, v): + return ('http://www.spinellis.gr/sw/textproc/bib2xhtml/bib2xhtml-v%s.tar.gz' % v) + + def install(self, spec, prefix): + # Add the bst include files to the install directory + bst_include = join_path(prefix.share, 'bib2xhtml') + mkdirp(bst_include) + for bstfile in glob('html-*bst'): + install(bstfile, bst_include) + + # Install the script and point it at the user's favorite perl + # and the bst include directory. + mkdirp(prefix.bin) + install('bib2xhtml', prefix.bin) + filter_file(r'#!/usr/bin/perl', + '#!/usr/bin/env BSTINPUTS=%s perl' % bst_include, + join_path(prefix.bin, 'bib2xhtml')) diff --git a/var/spack/repos/builtin/packages/binutils/package.py b/var/spack/repos/builtin/packages/binutils/package.py new file mode 100644 index 0000000000..cac0a0407f --- /dev/null +++ b/var/spack/repos/builtin/packages/binutils/package.py @@ -0,0 +1,30 @@ +from spack import * + +class Binutils(Package): + """GNU binutils, which contain the linker, assembler, objdump and others""" + homepage = "http://www.gnu.org/software/binutils/" + url = "ftp://ftp.gnu.org/gnu/binutils/binutils-2.25.tar.bz2" + + version('2.25', 'd9f3303f802a5b6b0bb73a335ab89d66') + version('2.24', 'e0f71a7b2ddab0f8612336ac81d9636b') + version('2.23.2', '4f8fa651e35ef262edc01d60fb45702e') + version('2.20.1', '2b9dc8f2b7dbd5ec5992c6e29de0b764') + + variant('libiberty', default=False, description='Also install libiberty.') + + def install(self, spec, prefix): + configure_args = [ + '--prefix=%s' % prefix, + '--disable-dependency-tracking', + '--enable-interwork', + '--enable-multilib', + '--enable-shared', + '--enable-64-bit-bfd', + '--enable-targets=all'] + + if '+libiberty' in spec: + configure_args.append('--enable-install-libiberty') + + configure(*configure_args) + make() + make("install") diff --git a/var/spack/repos/builtin/packages/bison/package.py b/var/spack/repos/builtin/packages/bison/package.py new file mode 100644 index 0000000000..7c526fb958 --- /dev/null +++ b/var/spack/repos/builtin/packages/bison/package.py @@ -0,0 +1,17 @@ +from spack import * + +class Bison(Package): + """Bison is a general-purpose parser generator that converts + an annotated context-free grammar into a deterministic LR or + generalized LR (GLR) parser employing LALR(1) parser tables.""" + + homepage = "http://www.gnu.org/software/bison/" + url = "http://ftp.gnu.org/gnu/bison/bison-3.0.tar.gz" + + version('3.0.4', 'a586e11cd4aff49c3ff6d3b6a4c9ccf8') + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix) + + make() + make("install") diff --git a/var/spack/repos/builtin/packages/boost/package.py b/var/spack/repos/builtin/packages/boost/package.py new file mode 100644 index 0000000000..35824d53a2 --- /dev/null +++ b/var/spack/repos/builtin/packages/boost/package.py @@ -0,0 +1,66 @@ +from spack import * + +class Boost(Package): + """Boost provides free peer-reviewed portable C++ source + libraries, emphasizing libraries that work well with the C++ + Standard Library. + + Boost libraries are intended to be widely useful, and usable + across a broad spectrum of applications. The Boost license + encourages both commercial and non-commercial use. + """ + homepage = "http://www.boost.org" + url = "http://downloads.sourceforge.net/project/boost/boost/1.55.0/boost_1_55_0.tar.bz2" + list_url = "http://sourceforge.net/projects/boost/files/boost/" + list_depth = 2 + + version('1.59.0', '6aa9a5c6a4ca1016edd0ed1178e3cb87') + version('1.58.0', 'b8839650e61e9c1c0a89f371dd475546') + version('1.57.0', '1be49befbdd9a5ce9def2983ba3e7b76') + version('1.56.0', 'a744cf167b05d72335f27c88115f211d') + version('1.55.0', 'd6eef4b4cacb2183f2bf265a5a03a354') + version('1.54.0', '15cb8c0803064faef0c4ddf5bc5ca279') + version('1.53.0', 'a00d22605d5dbcfb4c9936a9b35bc4c2') + version('1.52.0', '3a855e0f919107e0ca4de4d84ad3f750') + version('1.51.0', '4b6bd483b692fd138aef84ed2c8eb679') + version('1.50.0', '52dd00be775e689f55a987baebccc462') + version('1.49.0', '0d202cb811f934282dea64856a175698') + version('1.48.0', 'd1e9a7a7f532bb031a3c175d86688d95') + version('1.47.0', 'a2dc343f7bc7f83f8941e47ed4a18200') + version('1.46.1', '7375679575f4c8db605d426fc721d506') + version('1.46.0', '37b12f1702319b73876b0097982087e0') + version('1.45.0', 'd405c606354789d0426bc07bea617e58') + version('1.44.0', 'f02578f5218f217a9f20e9c30e119c6a') + version('1.43.0', 'dd49767bfb726b0c774f7db0cef91ed1') + version('1.42.0', '7bf3b4eb841b62ffb0ade2b82218ebe6') + version('1.41.0', '8bb65e133907db727a2a825c5400d0a6') + version('1.40.0', 'ec3875caeac8c52c7c129802a8483bd7') + version('1.39.0', 'a17281fd88c48e0d866e1a12deecbcc0') + version('1.38.0', '5eca2116d39d61382b8f8235915cb267') + version('1.37.0', '8d9f990bfb7e83769fa5f1d6f065bc92') + version('1.36.0', '328bfec66c312150e4c2a78dcecb504b') + version('1.35.0', 'dce952a7214e72d6597516bcac84048b') + version('1.34.1', '2d938467e8a448a2c9763e0a9f8ca7e5') + version('1.34.0', 'ed5b9291ffad776f8757a916e1726ad0') + + + def url_for_version(self, version): + """Handle Boost's weird URLs, which write the version two different ways.""" + parts = [str(p) for p in Version(version)] + dots = ".".join(parts) + underscores = "_".join(parts) + return "http://downloads.sourceforge.net/project/boost/boost/%s/boost_%s.tar.bz2" % ( + dots, underscores) + + + def install(self, spec, prefix): + bootstrap = Executable('./bootstrap.sh') + bootstrap() + + # b2 used to be called bjam, before 1.47 (sigh) + b2name = './b2' if spec.satisfies('@1.47:') else './bjam' + + b2 = Executable(b2name) + b2('install', + '-j %s' % make_jobs, + '--prefix=%s' % prefix) diff --git a/var/spack/repos/builtin/packages/bowtie2/bowtie2-2.5.patch b/var/spack/repos/builtin/packages/bowtie2/bowtie2-2.5.patch new file mode 100644 index 0000000000..290be39c73 --- /dev/null +++ b/var/spack/repos/builtin/packages/bowtie2/bowtie2-2.5.patch @@ -0,0 +1,16 @@ +--- Makefile 2015-02-26 10:50:00.000000000 -0800 ++++ Makefile.new 2015-07-29 18:03:59.891357399 -0700 +@@ -22,10 +22,10 @@ + # + + INC = +-GCC_PREFIX = $(shell dirname `which gcc`) ++GCC_PREFIX = + GCC_SUFFIX = +-CC = $(GCC_PREFIX)/gcc$(GCC_SUFFIX) +-CPP = $(GCC_PREFIX)/g++$(GCC_SUFFIX) ++CC = cc ++CPP = c++ + CXX = $(CPP) + HEADERS = $(wildcard *.h) + BOWTIE_MM = 1 diff --git a/var/spack/repos/builtin/packages/bowtie2/package.py b/var/spack/repos/builtin/packages/bowtie2/package.py new file mode 100644 index 0000000000..339aab6598 --- /dev/null +++ b/var/spack/repos/builtin/packages/bowtie2/package.py @@ -0,0 +1,24 @@ +from spack import * +from glob import glob +class Bowtie2(Package): + """Description""" + homepage = "bowtie-bio.sourceforge.net/bowtie2/index.shtml" + version('2.2.5','51fa97a862d248d7ee660efc1147c75f', url = "http://downloads.sourceforge.net/project/bowtie-bio/bowtie2/2.2.5/bowtie2-2.2.5-source.zip") + + patch('bowtie2-2.5.patch',when='@2.2.5', level=0) + + def install(self, spec, prefix): + make() + mkdirp(prefix.bin) + for bow in glob("bowtie2*"): + install(bow, prefix.bin) + # install('bowtie2',prefix.bin) + # install('bowtie2-align-l',prefix.bin) + # install('bowtie2-align-s',prefix.bin) + # install('bowtie2-build',prefix.bin) + # install('bowtie2-build-l',prefix.bin) + # install('bowtie2-build-s',prefix.bin) + # install('bowtie2-inspect',prefix.bin) + # install('bowtie2-inspect-l',prefix.bin) + # install('bowtie2-inspect-s',prefix.bin) + diff --git a/var/spack/repos/builtin/packages/boxlib/package.py b/var/spack/repos/builtin/packages/boxlib/package.py new file mode 100644 index 0000000000..4f1b71132f --- /dev/null +++ b/var/spack/repos/builtin/packages/boxlib/package.py @@ -0,0 +1,25 @@ +from spack import * + +class Boxlib(Package): + """BoxLib, a software framework for massively parallel + block-structured adaptive mesh refinement (AMR) codes.""" + + homepage = "https://ccse.lbl.gov/BoxLib/" + url = "https://ccse.lbl.gov/pub/Downloads/BoxLib.git"; + + # TODO: figure out how best to version this. No tags in the repo! + version('master', git='https://ccse.lbl.gov/pub/Downloads/BoxLib.git') + + depends_on('mpi') + + def install(self, spec, prefix): + args = std_cmake_args + args += ['-DCCSE_ENABLE_MPI=1', + '-DCMAKE_C_COMPILER=%s' % which('mpicc'), + '-DCMAKE_CXX_COMPILER=%s' % which('mpicxx'), + '-DCMAKE_Fortran_COMPILER=%s' % which('mpif90')] + + cmake('.', *args) + make() + make("install") + diff --git a/var/spack/repos/builtin/packages/bzip2/package.py b/var/spack/repos/builtin/packages/bzip2/package.py new file mode 100644 index 0000000000..d88336664d --- /dev/null +++ b/var/spack/repos/builtin/packages/bzip2/package.py @@ -0,0 +1,36 @@ +from spack import * +from glob import glob + +class Bzip2(Package): + """bzip2 is a freely available, patent free high-quality data + compressor. It typically compresses files to within 10% to 15% + of the best available techniques (the PPM family of statistical + compressors), whilst being around twice as fast at compression + and six times faster at decompression.""" + homepage = "http://www.bzip.org" + url = "http://www.bzip.org/1.0.6/bzip2-1.0.6.tar.gz" + + version('1.0.6', '00b516f4704d4a7cb50a1d97e6e8e15b') + + def install(self, spec, prefix): + # No configure system -- have to filter the makefile for this package. + filter_file(r'CC=gcc', 'CC=cc', 'Makefile', string=True) + + make('-f', 'Makefile-libbz2_so') + make('clean') + make("install", "PREFIX=%s" % prefix) + + bzip2_exe = join_path(prefix.bin, 'bzip2') + install('bzip2-shared', bzip2_exe) + for i, libfile in enumerate(glob('libbz2.so*')): + install(libfile, prefix.lib) + if i == 0: + symlink(join_path(prefix.lib, libfile), join_path(prefix.lib, 'libbz2.so')) + + bunzip2 = join_path(prefix.bin, 'bunzip2') + remove(bunzip2) + symlink(bzip2_exe, bunzip2) + + bzcat = join_path(prefix.bin, 'bzcat') + remove(bzcat) + symlink(bzip2_exe, bzcat) diff --git a/var/spack/repos/builtin/packages/cairo/package.py b/var/spack/repos/builtin/packages/cairo/package.py new file mode 100644 index 0000000000..e1ac8aaa7d --- /dev/null +++ b/var/spack/repos/builtin/packages/cairo/package.py @@ -0,0 +1,19 @@ +from spack import * + +class Cairo(Package): + """Cairo is a 2D graphics library with support for multiple output devices.""" + homepage = "http://cairographics.org" + url = "http://cairographics.org/releases/cairo-1.14.0.tar.xz" + + version('1.14.0', 'fc3a5edeba703f906f2241b394f0cced') + + depends_on("libpng") + depends_on("glib") + depends_on("pixman") + depends_on("fontconfig@2.10.91:") # Require newer version of fontconfig. + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix, + "--enable-tee") + make() + make("install") diff --git a/var/spack/repos/builtin/packages/callpath/package.py b/var/spack/repos/builtin/packages/callpath/package.py new file mode 100644 index 0000000000..f8a1eab9f7 --- /dev/null +++ b/var/spack/repos/builtin/packages/callpath/package.py @@ -0,0 +1,47 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + +class Callpath(Package): + """Library for representing callpaths consistently in + distributed-memory performance tools.""" + + homepage = "https://github.com/scalability-llnl/callpath" + url = "https://github.com/scalability-llnl/callpath/archive/v1.0.1.tar.gz" + + version('1.0.2', 'b1994d5ee7c7db9d27586fc2dcf8f373') + version('1.0.1', '0047983d2a52c5c335f8ba7f5bab2325') + + depends_on("libelf") + depends_on("libdwarf") + depends_on("dyninst") + depends_on("adept-utils") + depends_on("mpi") + + def install(self, spec, prefix): + # TODO: offer options for the walker used. + cmake('.', "-DCALLPATH_WALKER=dyninst", *std_cmake_args) + make() + make("install") diff --git a/var/spack/repos/builtin/packages/cblas/package.py b/var/spack/repos/builtin/packages/cblas/package.py new file mode 100644 index 0000000000..3cfe5ee588 --- /dev/null +++ b/var/spack/repos/builtin/packages/cblas/package.py @@ -0,0 +1,35 @@ +from spack import * +import os + +class Cblas(Package): + """The BLAS (Basic Linear Algebra Subprograms) are routines that + provide standard building blocks for performing basic vector and + matrix operations.""" + + homepage = "http://www.netlib.org/blas/_cblas/" + + # tarball has no version, but on the date below, this MD5 was correct. + version('2015-06-06', '1e8830f622d2112239a4a8a83b84209a', + url='http://www.netlib.org/blas/blast-forum/cblas.tgz') + + depends_on('blas') + parallel = False + + def patch(self): + mf = FileFilter('Makefile.in') + + mf.filter('^BLLIB =.*', 'BLLIB = %s/libblas.a' % self.spec['blas'].prefix.lib) + mf.filter('^CC =.*', 'CC = cc') + mf.filter('^FC =.*', 'FC = f90') + + + def install(self, spec, prefix): + make('all') + mkdirp(prefix.lib) + mkdirp(prefix.include) + + # Rename the generated lib file to libcblas.a + install('./lib/cblas_LINUX.a', '%s/libcblas.a' % prefix.lib) + install('./include/cblas.h','%s' % prefix.include) + install('./include/cblas_f77.h','%s' % prefix.include) + diff --git a/var/spack/repos/builtin/packages/cgm/package.py b/var/spack/repos/builtin/packages/cgm/package.py new file mode 100644 index 0000000000..05d6395c5a --- /dev/null +++ b/var/spack/repos/builtin/packages/cgm/package.py @@ -0,0 +1,30 @@ +from spack import * + +class Cgm(Package): + """The Common Geometry Module, Argonne (CGMA) is a code library + which provides geometry functionality used for mesh generation and + other applications.""" + homepage = "http://trac.mcs.anl.gov/projects/ITAPS/wiki/CGM" + url = "http://ftp.mcs.anl.gov/pub/fathom/cgm13.1.1.tar.gz" + + version('13.1.1', '4e8dbc4ba8f65767b29f985f7a23b01f') + version('13.1.0', 'a6c7b22660f164ce893fb974f9cb2028') + version('13.1' , '95f724bda04919fc76818a5b7bc0b4ed') + + depends_on("mpi") + + def patch(self): + filter_file('^(#include "CGMParallelConventions.h")', + '//\1', + 'geom/parallel/CGMReadParallel.cpp') + + + def install(self, spec, prefix): + configure("--with-mpi", + "--prefix=%s" % prefix, + "CFLAGS=-static", + "CXXFLAGS=-static", + "FCFLAGS=-static") + + make() + make("install") diff --git a/var/spack/repos/builtin/packages/clang/package.py b/var/spack/repos/builtin/packages/clang/package.py new file mode 100644 index 0000000000..4f977bf9a4 --- /dev/null +++ b/var/spack/repos/builtin/packages/clang/package.py @@ -0,0 +1,51 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + +class Clang(Package): + """The goal of the Clang project is to create a new C, C++, + Objective C and Objective C++ front-end for the LLVM compiler. + """ + homepage = 'http://clang.llvm.org' + url = 'http://llvm.org/releases/3.7.0/cfe-3.7.0.src.tar.xz' + + depends_on('llvm@3.7.0', when='@3.7.0') + depends_on('llvm@3.6.2', when='@3.6.2') + depends_on('llvm@3.5.1', when='@3.5.1') + + version('3.7.0', '8f9d27335e7331cf0a4711e952f21f01', url='http://llvm.org/releases/3.7.0/cfe-3.7.0.src.tar.xz') + version('3.6.2', 'ff862793682f714bb7862325b9c06e20', url='http://llvm.org/releases/3.6.2/cfe-3.6.2.src.tar.xz') + version('3.5.1', '93f9532f8f7e6f1d8e5c1116907051cb', url='http://llvm.org/releases/3.5.1/cfe-3.5.1.src.tar.xz') + + def install(self, spec, prefix): + env['CXXFLAGS'] = self.compiler.cxx11_flag + + with working_dir('spack-build', create=True): + cmake('..', + '-DCLANG_PATH_TO_LLVM_BUILD=%s' % spec['llvm'].prefix, + '-DLLVM_MAIN_SRC_DIR=%s' % spec['llvm'].prefix, + *std_cmake_args) + make() + make("install") diff --git a/var/spack/repos/builtin/packages/cloog/package.py b/var/spack/repos/builtin/packages/cloog/package.py new file mode 100644 index 0000000000..814a33c76c --- /dev/null +++ b/var/spack/repos/builtin/packages/cloog/package.py @@ -0,0 +1,26 @@ +from spack import * + +class Cloog(Package): + """CLooG is a free software and library to generate code for + scanning Z-polyhedra. That is, it finds a code (e.g. in C, + FORTRAN...) that reaches each integral point of one or more + parameterized polyhedra.""" + + homepage = "http://www.cloog.org" + url = "http://www.bastoul.net/cloog/pages/download/count.php3?url=./cloog-0.18.1.tar.gz" + list_url = "http://www.bastoul.net/cloog/pages/download" + + version('0.18.1', 'e34fca0540d840e5d0f6427e98c92252') + version('0.18.0', 'be78a47bd82523250eb3e91646db5b3d') + version('0.17.0', '0aa3302c81f65ca62c114e5264f8a802') + + depends_on("gmp") + depends_on("isl") + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix, + "--with-osl=no", + "--with-isl=%s" % spec['isl'].prefix, + "--with-gmp=%s" % spec['gmp'].prefix) + make() + make("install") diff --git a/var/spack/repos/builtin/packages/cmake/package.py b/var/spack/repos/builtin/packages/cmake/package.py new file mode 100644 index 0000000000..9efa370c8b --- /dev/null +++ b/var/spack/repos/builtin/packages/cmake/package.py @@ -0,0 +1,45 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + +class Cmake(Package): + """A cross-platform, open-source build system. CMake is a family of + tools designed to build, test and package software.""" + homepage = 'https://www.cmake.org' + + version('2.8.10.2', '097278785da7182ec0aea8769d06860c', + url = 'http://www.cmake.org/files/v2.8/cmake-2.8.10.2.tar.gz') + + version('3.0.2', 'db4c687a31444a929d2fdc36c4dfb95f', + url = 'http://www.cmake.org/files/v3.0/cmake-3.0.2.tar.gz') + +# version('3.0.1', 'e2e05d84cb44a42f1371d9995631dcf5') +# version('3.0.0', '21a1c85e1a3b803c4b48e7ff915a863e') + + def install(self, spec, prefix): + configure('--prefix=' + prefix, + '--parallel=' + str(make_jobs)) + make() + make('install') diff --git a/var/spack/repos/builtin/packages/coreutils/package.py b/var/spack/repos/builtin/packages/coreutils/package.py new file mode 100644 index 0000000000..78c608d8eb --- /dev/null +++ b/var/spack/repos/builtin/packages/coreutils/package.py @@ -0,0 +1,17 @@ +from spack import * + +class Coreutils(Package): + """The GNU Core Utilities are the basic file, shell and text + manipulation utilities of the GNU operating system. These are + the core utilities which are expected to exist on every + operating system. + """ + homepage = "http://www.gnu.org/software/coreutils/" + url = "http://ftp.gnu.org/gnu/coreutils/coreutils-8.23.tar.xz" + + version('8.23', 'abed135279f87ad6762ce57ff6d89c41') + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix) + make() + make("install") diff --git a/var/spack/repos/builtin/packages/cppcheck/package.py b/var/spack/repos/builtin/packages/cppcheck/package.py new file mode 100644 index 0000000000..8e98f457ee --- /dev/null +++ b/var/spack/repos/builtin/packages/cppcheck/package.py @@ -0,0 +1,15 @@ +from spack import * + +class Cppcheck(Package): + """A tool for static C/C++ code analysis.""" + homepage = "http://cppcheck.sourceforge.net/" + url = "http://downloads.sourceforge.net/project/cppcheck/cppcheck/1.68/cppcheck-1.68.tar.bz2" + + version('1.68', 'c015195f5d61a542f350269030150708') + + def install(self, spec, prefix): + # cppcheck does not have a configure script + make() + # manually install the final cppcheck binary + mkdirp(prefix.bin) + install('cppcheck', prefix.bin) diff --git a/var/spack/repos/builtin/packages/cram/package.py b/var/spack/repos/builtin/packages/cram/package.py new file mode 100644 index 0000000000..4b8ec56f25 --- /dev/null +++ b/var/spack/repos/builtin/packages/cram/package.py @@ -0,0 +1,15 @@ +from spack import * + +class Cram(Package): + """Cram runs many small MPI jobs inside one large MPI job.""" + homepage = "https://github.com/scalability-llnl/cram" + url = "http://github.com/scalability-llnl/cram/archive/v1.0.1.tar.gz" + + version('1.0.1', 'c73711e945cf5dc603e44395f6647f5e') + + depends_on("mpi") + + def install(self, spec, prefix): + cmake(".", *std_cmake_args) + make() + make("install") diff --git a/var/spack/repos/builtin/packages/cscope/package.py b/var/spack/repos/builtin/packages/cscope/package.py new file mode 100644 index 0000000000..9aac0f7304 --- /dev/null +++ b/var/spack/repos/builtin/packages/cscope/package.py @@ -0,0 +1,17 @@ +from spack import * + +class Cscope(Package): + """Cscope is a developer's tool for browsing source code.""" + homepage = "http://http://cscope.sourceforge.net/" + url = "http://downloads.sourceforge.net/project/cscope/cscope/15.8b/cscope-15.8b.tar.gz" + + version('15.8b', '8f9409a238ee313a96f9f87fe0f3b176') + + # Can be configured to use flex (not necessary) + # ./configure --with-flex + + def install(self, spec, prefix): + configure('--prefix=%s' % prefix) + + make() + make("install") diff --git a/var/spack/repos/builtin/packages/cube/package.py b/var/spack/repos/builtin/packages/cube/package.py new file mode 100644 index 0000000000..d97cd25636 --- /dev/null +++ b/var/spack/repos/builtin/packages/cube/package.py @@ -0,0 +1,55 @@ +# FIXME: Add copyright statement +# +from spack import * +from contextlib import closing + +class Cube(Package): + """Cube the profile viewer for Score-P and Scalasca profiles. It + displays a multi-dimensional performance space consisting + of the dimensions (i) performance metric, (ii) call path, + and (iii) system resource.""" + + homepage = "http://www.scalasca.org/software/cube-4.x/download.html" + url = "http://apps.fz-juelich.de/scalasca/releases/cube/4.2/dist/cube-4.2.3.tar.gz" + + version('4.2.3', '8f95b9531f5a8f8134f279c2767c9b20') + + version('4.3TP1', 'a2090fbc7b2ba394bd5c09ba971e237f', + url = 'http://apps.fz-juelich.de/scalasca/releases/cube/4.3/dist/cube-4.3-TP1.tar.gz') + + # Using CC as C++ compiler provides quirky workaround for a Score-P build system attempt + # to guess a matching C compiler when configuring scorep-score + backend_user_provided = """\ +CC=cc +CXX=CC +F77=f77 +FC=f90 +#CFLAGS=-fPIC +#CXXFLAGS=-fPIC +""" + frontend_user_provided = """\ +CC_FOR_BUILD=cc +CXX_FOR_BUILD=CC +F77_FOR_BUILD=f70 +FC_FOR_BUILD=f90 +""" + + def install(self, spec, prefix): + # Use a custom compiler configuration, otherwise the score-p + # build system messes with spack's compiler settings. + # Create these three files in the build directory + + with closing(open("vendor/common/build-config/platforms/platform-backend-user-provided", "w")) as backend_file: + backend_file.write(self.backend_user_provided) + with closing(open("vendor/common/build-config/platforms/platform-frontend-user-provided", "w")) as frontend_file: + frontend_file.write(self.frontend_user_provided) + + configure_args = ["--prefix=%s" % prefix, + "--with-custom-compilers", + "--without-paraver", + "--without-gui"] + + configure(*configure_args) + + make(parallel=False) + make("install", parallel=False) diff --git a/var/spack/repos/builtin/packages/czmq/package.py b/var/spack/repos/builtin/packages/czmq/package.py new file mode 100644 index 0000000000..a2f1947554 --- /dev/null +++ b/var/spack/repos/builtin/packages/czmq/package.py @@ -0,0 +1,19 @@ +from spack import * + +class Czmq(Package): + """ A C interface to the ZMQ library """ + homepage = "http://czmq.zeromq.org" + url = "https://github.com/zeromq/czmq/archive/v3.0.2.tar.gz" + + version('3.0.2', '23e9885f7ee3ce88d99d0425f52e9be1', url='https://github.com/zeromq/czmq/archive/v3.0.2.tar.gz') + + depends_on('zeromq') + + def install(self, spec, prefix): + bash = which("bash") + bash("./autogen.sh") + configure("--prefix=%s" % prefix) + + make() + make("install") + diff --git a/var/spack/repos/builtin/packages/dbus/package.py b/var/spack/repos/builtin/packages/dbus/package.py new file mode 100644 index 0000000000..f7c302d611 --- /dev/null +++ b/var/spack/repos/builtin/packages/dbus/package.py @@ -0,0 +1,31 @@ +from spack import * + +class Dbus(Package): + """D-Bus is a message bus system, a simple way for applications to + talk to one another. D-Bus supplies both a system daemon (for + events such new hardware device printer queue ) and a + per-user-login-session daemon (for general IPC needs among user + applications). Also, the message bus is built on top of a + general one-to-one message passing framework, which can be used + by any two applications to communicate directly (without going + through the message bus daemon).""" + + homepage = "http://dbus.freedesktop.org/" + url = "http://dbus.freedesktop.org/releases/dbus/dbus-1.8.8.tar.gz" + + version('1.9.0', 'ec6895a4d5c0637b01f0d0e7689e2b36') + version('1.8.8', 'b9f4a18ee3faa1e07c04aa1d83239c43') + version('1.8.6', '6a08ba555d340e9dfe2d623b83c0eea8') + version('1.8.4', '4717cb8ab5b80978fcadf2b4f2f72e1b') + version('1.8.2', 'd6f709bbec0a022a1847c7caec9d6068') + + def install(self, spec, prefix): + configure( + "--prefix=%s" % prefix, + "--disable-systemd") + make() + make("install") + + # dbus needs a machine id generated after install + dbus_uuidgen = Executable(join_path(prefix.bin, 'dbus-uuidgen')) + dbus_uuidgen('--ensure') diff --git a/var/spack/repos/builtin/packages/docbook-xml/package.py b/var/spack/repos/builtin/packages/docbook-xml/package.py new file mode 100644 index 0000000000..fce1de7deb --- /dev/null +++ b/var/spack/repos/builtin/packages/docbook-xml/package.py @@ -0,0 +1,19 @@ +import os +import glob +from spack import * + + +class DocbookXml(Package): + """Docbook DTD XML files.""" + homepage = "http://www.oasis-open.org/docbook" + url = "http://www.oasis-open.org/docbook/xml/4.5/docbook-xml-4.5.zip" + + version('4.5', '03083e288e87a7e829e437358da7ef9e') + + def install(self, spec, prefix): + cp = which('cp') + + install_args = ['-a', '-t', prefix] + install_args.extend(glob.glob('*')) + + cp(*install_args) diff --git a/var/spack/repos/builtin/packages/doxygen/package.py b/var/spack/repos/builtin/packages/doxygen/package.py new file mode 100644 index 0000000000..3d4a4e47a7 --- /dev/null +++ b/var/spack/repos/builtin/packages/doxygen/package.py @@ -0,0 +1,25 @@ +#------------------------------------------------------------------------------ +# Author: Justin Too +# Date: September 11, 2015 +#------------------------------------------------------------------------------ + +from spack import * + +class Doxygen(Package): + """Doxygen is the de facto standard tool for generating documentation + from annotated C++ sources, but it also supports other popular programming + languages such as C, Objective-C, C#, PHP, Java, Python, IDL (Corba, + Microsoft, and UNO/OpenOffice flavors), Fortran, VHDL, Tcl, and to some extent D.. + """ + homepage = "http://www.stack.nl/~dimitri/doxygen/" + url = "http://ftp.stack.nl/pub/users/dimitri/doxygen-1.8.10.src.tar.gz" + + version('1.8.10', '79767ccd986f12a0f949015efb5f058f') + + depends_on("cmake@2.8.12:") + + def install(self, spec, prefix): + cmake('.', *std_cmake_args) + + make() + make("install") diff --git a/var/spack/repos/builtin/packages/dri2proto/package.py b/var/spack/repos/builtin/packages/dri2proto/package.py new file mode 100644 index 0000000000..11dfa568e2 --- /dev/null +++ b/var/spack/repos/builtin/packages/dri2proto/package.py @@ -0,0 +1,14 @@ +from spack import * + +class Dri2proto(Package): + """DRI2 Protocol Headers.""" + homepage = "http://http://cgit.freedesktop.org/xorg/proto/dri2proto/" + url = "http://xorg.freedesktop.org/releases/individual/proto/dri2proto-2.8.tar.gz" + + version('2.8', '19ea18f63d8ae8053c9fa84b60365b77') + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix) + + make() + make("install") diff --git a/var/spack/repos/builtin/packages/dtcmp/package.py b/var/spack/repos/builtin/packages/dtcmp/package.py new file mode 100644 index 0000000000..9d940583c1 --- /dev/null +++ b/var/spack/repos/builtin/packages/dtcmp/package.py @@ -0,0 +1,20 @@ +import os +from spack import * + +class Dtcmp(Package): + """The Datatype Comparison Library provides comparison operations and + parallel sort algorithms for MPI applications.""" + + homepage = "https://github.com/hpc/dtcmp" + url = "https://github.com/hpc/dtcmp/releases/download/v1.0.3/dtcmp-1.0.3.tar.gz" + + version('1.0.3', 'cdd8ccf71e8ff67de2558594a7fcd317') + + depends_on('mpi') + depends_on('lwgrp') + + def install(self, spec, prefix): + configure("--prefix=" + prefix, + "--with-lwgrp=" + spec['lwgrp'].prefix) + make() + make("install") diff --git a/var/spack/repos/builtin/packages/dyninst/package.py b/var/spack/repos/builtin/packages/dyninst/package.py new file mode 100644 index 0000000000..41ec57dd2f --- /dev/null +++ b/var/spack/repos/builtin/packages/dyninst/package.py @@ -0,0 +1,68 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + +class Dyninst(Package): + """API for dynamic binary instrumentation. Modify programs while they + are executing without recompiling, re-linking, or re-executing.""" + homepage = "https://paradyn.org" + url = "http://www.dyninst.org/sites/default/files/downloads/dyninst/8.1.2/DyninstAPI-8.1.2.tgz" + list_url = "http://www.dyninst.org/downloads/dyninst-8.x" + + version('8.2.1', 'abf60b7faabe7a2e4b54395757be39c7', + url="http://www.paradyn.org/release8.2/DyninstAPI-8.2.1.tgz") + version('8.1.2', 'bf03b33375afa66fe0efa46ce3f4b17a', + url="http://www.paradyn.org/release8.1.2/DyninstAPI-8.1.2.tgz") + version('8.1.1', 'd1a04e995b7aa70960cd1d1fac8bd6ac', + url="http://www.paradyn.org/release8.1/DyninstAPI-8.1.1.tgz") + + depends_on("libelf") + depends_on("libdwarf") + depends_on("boost@1.42:") + + # new version uses cmake + def install(self, spec, prefix): + libelf = spec['libelf'].prefix + libdwarf = spec['libdwarf'].prefix + + with working_dir('spack-build', create=True): + cmake('..', + '-DBoost_INCLUDE_DIR=%s' % spec['boost'].prefix.include, + '-DBoost_LIBRARY_DIR=%s' % spec['boost'].prefix.lib, + '-DBoost_NO_SYSTEM_PATHS=TRUE', + '-DLIBELF_INCLUDE_DIR=%s' % join_path(libelf.include, 'libelf'), + '-DLIBELF_LIBRARIES=%s' % join_path(libelf.lib, 'libelf.so'), + '-DLIBDWARF_INCLUDE_DIR=%s' % libdwarf.include, + '-DLIBDWARF_LIBRARIES=%s' % join_path(libdwarf.lib, 'libdwarf.so'), + *std_cmake_args) + make() + make("install") + + + @when('@:8.1') + def install(self, spec, prefix): + configure("--prefix=" + prefix) + make() + make("install") diff --git a/var/spack/repos/builtin/packages/elfutils/package.py b/var/spack/repos/builtin/packages/elfutils/package.py new file mode 100644 index 0000000000..926d234584 --- /dev/null +++ b/var/spack/repos/builtin/packages/elfutils/package.py @@ -0,0 +1,26 @@ +from spack import * + +class Elfutils(Package): + """elfutils is a collection of various binary tools such as + eu-objdump, eu-readelf, and other utilities that allow you to + inspect and manipulate ELF files. Refer to Table 5.Tools Included + in elfutils for Red Hat Developer for a complete list of binary + tools that are distributed with the Red Hat Developer Toolset + version of elfutils.""" + + homepage = "https://fedorahosted.org/elfutils/" + + version('0.163', + git='git://git.fedorahosted.org/git/elfutils.git', + tag='elfutils-0.163') + + provides('elf') + + def install(self, spec, prefix): + autoreconf = which('autoreconf') + autoreconf('-if') + + configure('--prefix=%s' % prefix, '--enable-maintainer-mode') + make() + make("install") + diff --git a/var/spack/repos/builtin/packages/extrae/package.py b/var/spack/repos/builtin/packages/extrae/package.py new file mode 100644 index 0000000000..3ad4cbaf86 --- /dev/null +++ b/var/spack/repos/builtin/packages/extrae/package.py @@ -0,0 +1,46 @@ +from spack import * + +# typical working line with extrae 3.0.1 +# ./configure --prefix=/usr/local --with-mpi=/usr/lib64/mpi/gcc/openmpi --with-unwind=/usr/local --with-papi=/usr --with-dwarf=/usr --with-elf=/usr --with-dyninst=/usr --with-binutils=/usr --with-xml-prefix=/usr --enable-openmp --enable-nanos --enable-pthread --disable-parallel-merge LDFLAGS=-pthread + +class Extrae(Package): + """Extrae is the package devoted to generate tracefiles which can + be analyzed later by Paraver. Extrae is a tool that uses + different interposition mechanisms to inject probes into the + target application so as to gather information regarding the + application performance. The Extrae instrumentation package can + instrument the MPI programin model, and the following parallel + programming models either alone or in conjunction with MPI : + OpenMP, CUDA, OpenCL, pthread, OmpSs""" + homepage = "http://www.bsc.es/computer-sciences/extrae" + url = "http://www.bsc.es/ssl/apps/performanceTools/files/extrae-3.0.1.tar.bz2" + version('3.0.1', 'a6a8ca96cd877723cd8cc5df6bdb922b') + + depends_on("mpi") + depends_on("dyninst") + depends_on("libunwind") + depends_on("boost") + depends_on("libdwarf") + depends_on("papi") + + def install(self, spec, prefix): + if 'openmpi' in spec: + mpi = spec['openmpi'] + elif 'mpich' in spec: + mpi = spec['mpich'] + elif 'mvapich2' in spec: + mpi = spec['mvapich2'] + + configure("--prefix=%s" % prefix, + "--with-mpi=%s" % mpi.prefix, + "--with-unwind=%s" % spec['libunwind'].prefix, + "--with-dyninst=%s" % spec['dyninst'].prefix, + "--with-boost=%s" % spec['boost'].prefix, + "--with-dwarf=%s" % spec['libdwarf'].prefix, + "--with-papi=%s" % spec['papi'].prefix, + "--with-dyninst-headers=%s" % spec['dyninst'].prefix.include, + "--with-dyninst-libs=%s" % spec['dyninst'].prefix.lib) + + make() + make("install", parallel=False) + diff --git a/var/spack/repos/builtin/packages/exuberant-ctags/package.py b/var/spack/repos/builtin/packages/exuberant-ctags/package.py new file mode 100644 index 0000000000..efd2b541b2 --- /dev/null +++ b/var/spack/repos/builtin/packages/exuberant-ctags/package.py @@ -0,0 +1,14 @@ +from spack import * + +class ExuberantCtags(Package): + """The canonical ctags generator""" + homepage = "ctags.sourceforge.net" + url = "http://downloads.sourceforge.net/project/ctags/ctags/5.8/ctags-5.8.tar.gz" + + version('5.8', 'c00f82ecdcc357434731913e5b48630d') + + def install(self, spec, prefix): + configure('--prefix=%s' % prefix) + + make() + make("install") diff --git a/var/spack/repos/builtin/packages/fish/package.py b/var/spack/repos/builtin/packages/fish/package.py new file mode 100644 index 0000000000..1225558705 --- /dev/null +++ b/var/spack/repos/builtin/packages/fish/package.py @@ -0,0 +1,18 @@ +from spack import * + +class Fish(Package): + """fish is a smart and user-friendly command line shell for OS X, Linux, and + the rest of the family. + """ + + homepage = "http://fishshell.com/" + url = "http://fishshell.com/files/2.2.0/fish-2.2.0.tar.gz" + list_url = homepage + + version('2.2.0', 'a76339fd14ce2ec229283c53e805faac48c3e99d9e3ede9d82c0554acfc7b77a') + + def install(self, spec, prefix): + configure('--prefix=%s' % prefix) + + make() + make("install") diff --git a/var/spack/repos/builtin/packages/flex/package.py b/var/spack/repos/builtin/packages/flex/package.py new file mode 100644 index 0000000000..b065904912 --- /dev/null +++ b/var/spack/repos/builtin/packages/flex/package.py @@ -0,0 +1,15 @@ +from spack import * + +class Flex(Package): + """Flex is a tool for generating scanners.""" + + homepage = "http://flex.sourceforge.net/" + url = "http://download.sourceforge.net/flex/flex-2.5.39.tar.gz" + + version('2.5.39', 'e133e9ead8ec0a58d81166b461244fde') + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix) + + make() + make("install") diff --git a/var/spack/repos/builtin/packages/flux/package.py b/var/spack/repos/builtin/packages/flux/package.py new file mode 100644 index 0000000000..c128f46be8 --- /dev/null +++ b/var/spack/repos/builtin/packages/flux/package.py @@ -0,0 +1,36 @@ +from spack import * +import os + +class Flux(Package): + """ A next-generation resource manager (pre-alpha) """ + + homepage = "https://github.com/flux-framework/flux-core" + url = "https://github.com/flux-framework/flux-core" + + version('master', branch='master', git='https://github.com/flux-framework/flux-core') + + # Also needs autotools, but should use the system version if available + depends_on("zeromq@4.0.4:") + depends_on("czmq@2.2:") + depends_on("lua@5.1:5.1.99") + depends_on("munge") + depends_on("libjson-c") + depends_on("libxslt") + # TODO: This provides a catalog, hacked with environment below for now + depends_on("docbook-xml") + depends_on("asciidoc") + depends_on("python") + depends_on("py-cffi") + + def install(self, spec, prefix): + # Bootstrap with autotools + bash = which('bash') + bash('./autogen.sh') + + # Fix asciidoc dependency on xml style sheets and whatnot + os.environ['XML_CATALOG_FILES'] = os.path.join(spec['docbook-xml'].prefix, + 'catalog.xml') + # Configure, compile & install + configure("--prefix=" + prefix) + make("install", "V=1") + diff --git a/var/spack/repos/builtin/packages/fontconfig/package.py b/var/spack/repos/builtin/packages/fontconfig/package.py new file mode 100644 index 0000000000..89b13604e8 --- /dev/null +++ b/var/spack/repos/builtin/packages/fontconfig/package.py @@ -0,0 +1,16 @@ +from spack import * + +class Fontconfig(Package): + """Fontconfig customizing font access""" + homepage = "http://www.freedesktop.org/wiki/Software/fontconfig/" + url = "http://www.freedesktop.org/software/fontconfig/release/fontconfig-2.11.1.tar.gz" + + version('2.11.1' , 'e75e303b4f7756c2b16203a57ac87eba') + + depends_on('freetype') + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix) + + make() + make("install") diff --git a/var/spack/repos/builtin/packages/freetype/package.py b/var/spack/repos/builtin/packages/freetype/package.py new file mode 100644 index 0000000000..0309b858a1 --- /dev/null +++ b/var/spack/repos/builtin/packages/freetype/package.py @@ -0,0 +1,16 @@ +from spack import * + +class Freetype(Package): + """Font package""" + homepage = "http://http://www.freetype.org" + url = "http://download.savannah.gnu.org/releases/freetype/freetype-2.5.3.tar.gz" + + version('2.5.3' , 'cafe9f210e45360279c730d27bf071e9') + + depends_on('libpng') + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix) + + make() + make("install") diff --git a/var/spack/repos/builtin/packages/gasnet/package.py b/var/spack/repos/builtin/packages/gasnet/package.py new file mode 100644 index 0000000000..705961d1de --- /dev/null +++ b/var/spack/repos/builtin/packages/gasnet/package.py @@ -0,0 +1,35 @@ +from spack import * + +class Gasnet(Package): + """GASNet is a language-independent, low-level networking layer + that provides network-independent, high-performance communication + primitives tailored for implementing parallel global address space + SPMD languages and libraries such as UPC, Co-Array Fortran, SHMEM, + Cray Chapel, and Titanium. + """ + homepage = "http://gasnet.lbl.gov" + url = "http://gasnet.lbl.gov/GASNet-1.24.0.tar.gz" + + version('1.24.0', 'c8afdf48381e8b5a7340bdb32ca0f41a') + + + def install(self, spec, prefix): + # TODO: don't use paths with @ in them. + change_sed_delimiter('@', ';', 'configure') + + configure("--prefix=%s" % prefix, + # TODO: factor IB suport out into architecture description. + "--enable-ibv", + "--enable-udp", + "--disable-mpi", + "--enable-par", + "--enable-mpi-compat", + "--enable-segment-fast", + "--disable-aligned-segments", + # TODO: make an option so that Legion can request builds with/without this. + # See the Legion webpage for details on when to/not to use. + "--disable-pshm", + "--with-segment-mmap-max=64MB") + + make() + make("install") diff --git a/var/spack/repos/builtin/packages/gcc/package.py b/var/spack/repos/builtin/packages/gcc/package.py new file mode 100644 index 0000000000..a49a1348aa --- /dev/null +++ b/var/spack/repos/builtin/packages/gcc/package.py @@ -0,0 +1,122 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + +from contextlib import closing +from glob import glob + +class Gcc(Package): + """The GNU Compiler Collection includes front ends for C, C++, + Objective-C, Fortran, and Java.""" + homepage = "https://gcc.gnu.org" + + url = "http://open-source-box.org/gcc/gcc-4.9.2/gcc-4.9.2.tar.bz2" + list_url = 'http://open-source-box.org/gcc/' + list_depth = 2 + + DEPENDS_ON_ISL_PREDICATE = '@5.0:' + + version('5.2.0', 'a51bcfeb3da7dd4c623e27207ed43467') + version('4.9.3', '6f831b4d251872736e8e9cc09746f327') + version('4.9.2', '4df8ee253b7f3863ad0b86359cd39c43') + version('4.9.1', 'fddf71348546af523353bd43d34919c1') + version('4.8.5', '80d2c2982a3392bb0b89673ff136e223') + version('4.8.4', '5a84a30839b2aca22a2d723de2a626ec') + version('4.7.4', '4c696da46297de6ae77a82797d2abe28') + version('4.6.4', 'b407a3d1480c11667f293bfb1f17d1a4') + version('4.5.4', '27e459c2566b8209ab064570e1b378f7') + + depends_on("mpfr") + depends_on("gmp") + depends_on("mpc") # when @4.5: + depends_on("binutils~libiberty") + + # Save these until we can do optional deps. + depends_on("isl", when=DEPENDS_ON_ISL_PREDICATE) + #depends_on("ppl") + #depends_on("cloog") + + def install(self, spec, prefix): + # libjava/configure needs a minor fix to install into spack paths. + filter_file(r"'@.*@'", "'@[[:alnum:]]*@'", 'libjava/configure', string=True) + + enabled_languages = set(('c', 'c++', 'fortran', 'java', 'objc')) + if spec.satisfies("@4.7.1:"): + enabled_languages.add('go') + + # Generic options to compile GCC + options = ["--prefix=%s" % prefix, + "--libdir=%s/lib64" % prefix, + "--disable-multilib", + "--enable-languages=" + ','.join(enabled_languages), + "--with-mpc=%s" % spec['mpc'].prefix, + "--with-mpfr=%s" % spec['mpfr'].prefix, + "--with-gmp=%s" % spec['gmp'].prefix, + "--enable-lto", + "--with-gnu-ld", + "--with-gnu-as", + "--with-quad"] + # Binutils + binutils_options = ["--with-stage1-ldflags=%s" % self.rpath_args, + "--with-boot-ldflags=%s" % self.rpath_args, + "--with-ld=%s/bin/ld" % spec['binutils'].prefix, + "--with-as=%s/bin/as" % spec['binutils'].prefix] + options.extend(binutils_options) + # Isl + if spec.satisfies(Gcc.DEPENDS_ON_ISL_PREDICATE): + isl_options = ["--with-isl=%s" % spec['isl'].prefix] + options.extend(isl_options) + + # Rest of install is straightforward. + configure(*options) + make() + make("install") + + self.write_rpath_specs() + + + @property + def spec_dir(self): + # e.g. lib64/gcc/x86_64-unknown-linux-gnu/4.9.2 + spec_dir = glob("%s/lib64/gcc/*/*" % self.prefix) + return spec_dir[0] if spec_dir else None + + + def write_rpath_specs(self): + """Generate a spec file so the linker adds a rpath to the libs + the compiler used to build the executable.""" + if not self.spec_dir: + tty.warn("Could not install specs for %s." % self.spec.format('$_$@')) + return + + gcc = Executable(join_path(self.prefix.bin, 'gcc')) + lines = gcc('-dumpspecs', return_output=True).strip().split("\n") + specs_file = join_path(self.spec_dir, 'specs') + with closing(open(specs_file, 'w')) as out: + for line in lines: + out.write(line + "\n") + if line.startswith("*link:"): + out.write("-rpath %s/lib:%s/lib64 \\\n"% (self.prefix, self.prefix)) + set_install_permissions(specs_file) diff --git a/var/spack/repos/builtin/packages/gdk-pixbuf/package.py b/var/spack/repos/builtin/packages/gdk-pixbuf/package.py new file mode 100644 index 0000000000..14a5569984 --- /dev/null +++ b/var/spack/repos/builtin/packages/gdk-pixbuf/package.py @@ -0,0 +1,22 @@ +from spack import * + +class GdkPixbuf(Package): + """The Gdk Pixbuf is a toolkit for image loading and pixel buffer + manipulation. It is used by GTK+ 2 and GTK+ 3 to load and + manipulate images. In the past it was distributed as part of + GTK+ 2 but it was split off into a separate package in + preparation for the change to GTK+ 3.""" + homepage = "https://developer.gnome.org/gdk-pixbuf/" + url = "http://ftp.gnome.org/pub/gnome/sources/gdk-pixbuf/2.31/gdk-pixbuf-2.31.1.tar.xz" + + version('2.31.2', '6be6bbc4f356d4b79ab4226860ab8523') + + depends_on("glib") + depends_on("jpeg") + depends_on("libpng") + depends_on("libtiff") + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix) + make() + make("install") diff --git a/var/spack/repos/builtin/packages/geos/package.py b/var/spack/repos/builtin/packages/geos/package.py new file mode 100644 index 0000000000..4a2657e32f --- /dev/null +++ b/var/spack/repos/builtin/packages/geos/package.py @@ -0,0 +1,31 @@ +from spack import * + +class Geos(Package): + """GEOS (Geometry Engine - Open Source) is a C++ port of the Java + Topology Suite (JTS). As such, it aims to contain the complete + functionality of JTS in C++. This includes all the OpenGIS + Simple Features for SQL spatial predicate functions and spatial + operators, as well as specific JTS enhanced topology functions.""" + + homepage = "http://trac.osgeo.org/geos/" + url = "http://download.osgeo.org/geos/geos-3.4.2.tar.bz2" + + version('3.4.2', 'fc5df2d926eb7e67f988a43a92683bae') + version('3.4.1', '4c930dec44c45c49cd71f3e0931ded7e') + version('3.4.0', 'e41318fc76b5dc764a69d43ac6b18488') + version('3.3.9', '4794c20f07721d5011c93efc6ccb8e4e') + version('3.3.8', '75be476d0831a2d14958fed76ca266de') + version('3.3.7', '95ab996d22672b067d92c7dee2170460') + version('3.3.6', '6fadfb941541875f4976f75fb0bbc800') + version('3.3.5', '2ba61afb7fe2c5ddf642d82d7b16e75b') + version('3.3.4', '1bb9f14d57ef06ffa41cb1d67acb55a1') + version('3.3.3', '8454e653d7ecca475153cc88fd1daa26') + + extends('python') + depends_on('swig') + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix, + "--enable-python") + make() + make("install") diff --git a/var/spack/repos/builtin/packages/gflags/package.py b/var/spack/repos/builtin/packages/gflags/package.py new file mode 100644 index 0000000000..62dd80a094 --- /dev/null +++ b/var/spack/repos/builtin/packages/gflags/package.py @@ -0,0 +1,21 @@ +import os +from spack import * + +class Gflags(Package): + """The gflags package contains a C++ library that implements + commandline flags processing. It includes built-in support for + standard types such as string and the ability to define flags + in the source file in which they are used. Online documentation + available at: https://gflags.github.io/gflags/""" + + homepage = "https://gflags.github.io/gflags" + url = "https://github.com/gflags/gflags/archive/v2.1.2.tar.gz" + + version('2.1.2', 'ac432de923f9de1e9780b5254884599f') + + def install(self, spec, prefix): + cmake("-DCMAKE_INSTALL_PREFIX=" + prefix, + "-DBUILD_SHARED_LIBS=ON") + make() + make("test") + make("install") diff --git a/var/spack/repos/builtin/packages/ghostscript/package.py b/var/spack/repos/builtin/packages/ghostscript/package.py new file mode 100644 index 0000000000..0ab49d425f --- /dev/null +++ b/var/spack/repos/builtin/packages/ghostscript/package.py @@ -0,0 +1,17 @@ +from spack import * + +class Ghostscript(Package): + """an interpreter for the PostScript language and for PDF. """ + homepage = "http://ghostscript.com/" + url = "http://downloads.ghostscript.com/public/ghostscript-9.16.tar.gz" + + version('9.16', '829319325bbdb83f5c81379a8f86f38f') + + parallel = False + + def install(self, spec, prefix): + configure("--prefix=%s" %prefix, "--enable-shared") + + make() + make("install") + diff --git a/var/spack/repos/builtin/packages/git/package.py b/var/spack/repos/builtin/packages/git/package.py new file mode 100644 index 0000000000..0f1a3ba05b --- /dev/null +++ b/var/spack/repos/builtin/packages/git/package.py @@ -0,0 +1,27 @@ +from spack import * + +class Git(Package): + """Git is a free and open source distributed version control + system designed to handle everything from small to very large + projects with speed and efficiency.""" + homepage = "http://git-scm.com" + url = "https://www.kernel.org/pub/software/scm/git/git-2.2.1.tar.xz" + + version('2.2.1', '43e01f9d96ba8c11611e0eef0d9f9f28') + + # Use system openssl. + # depends_on("openssl") + + # Use system perl for now. + # depends_on("perl") + # depends_on("pcre") + + depends_on("zlib") + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix, + "--without-pcre", + "--without-python") + + make() + make("install") diff --git a/var/spack/repos/builtin/packages/glib/package.py b/var/spack/repos/builtin/packages/glib/package.py new file mode 100644 index 0000000000..178f0b9df5 --- /dev/null +++ b/var/spack/repos/builtin/packages/glib/package.py @@ -0,0 +1,18 @@ +from spack import * + +class Glib(Package): + """The GLib package contains a low-level libraries useful for + providing data structure handling for C, portability wrappers + and interfaces for such runtime functionality as an event loop, + threads, dynamic loading and an object system.""" + homepage = "https://developer.gnome.org/glib/" + url = "http://ftp.gnome.org/pub/gnome/sources/glib/2.42/glib-2.42.1.tar.xz" + + version('2.42.1', '89c4119e50e767d3532158605ee9121a') + + depends_on("libffi") + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix) + make() + make("install") diff --git a/var/spack/repos/builtin/packages/glm/package.py b/var/spack/repos/builtin/packages/glm/package.py new file mode 100644 index 0000000000..d00c301b4c --- /dev/null +++ b/var/spack/repos/builtin/packages/glm/package.py @@ -0,0 +1,19 @@ +from spack import * + + +class Glm(Package): + """ + OpenGL Mathematics (GLM) is a header only C++ mathematics library for graphics software based on + the OpenGL Shading Language (GLSL) specification. + """ + + homepage = "https://github.com/g-truc/glm" + url = "https://github.com/g-truc/glm/archive/0.9.7.1.tar.gz" + + version('0.9.7.1', '61af6639cdf652d1cdd7117190afced8') + + def install(self, spec, prefix): + with working_dir('spack-build', create=True): + cmake('..', *std_cmake_args) + make() + make("install") diff --git a/var/spack/repos/builtin/packages/global/package.py b/var/spack/repos/builtin/packages/global/package.py new file mode 100644 index 0000000000..a77b1bdc09 --- /dev/null +++ b/var/spack/repos/builtin/packages/global/package.py @@ -0,0 +1,24 @@ +from spack import * +import os + + +class Global(Package): + """ The Gnu Global tagging system """ + # FIXME: add a proper url for your package's homepage here. + homepage = "http://www.gnu.org/software/global" + url = "http://tamacom.com/global/global-6.5.tar.gz" + + version('6.5', 'dfec818b4f53d91721e247cf7b218078') + + depends_on('exuberant-ctags') + + def install(self, spec, prefix): + config_args = ['--prefix={}'.format(prefix)] + + config_args.append('--with-exuberant-ctags={}'.format( + os.path.join(spec['exuberant-ctags'].prefix.bin, 'ctags'))) + + configure(*config_args) + + make() + make("install") diff --git a/var/spack/repos/builtin/packages/glog/package.py b/var/spack/repos/builtin/packages/glog/package.py new file mode 100644 index 0000000000..d73386b394 --- /dev/null +++ b/var/spack/repos/builtin/packages/glog/package.py @@ -0,0 +1,15 @@ +import os +from spack import * + +class Glog(Package): + """C++ implementation of the Google logging module.""" + + homepage = "https://github.com/google/glog" + url = "https://github.com/google/glog/archive/v0.3.3.tar.gz" + + version('0.3.3', 'c1f86af27bd9c73186730aa957607ed0') + + def install(self, spec, prefix): + configure("--prefix=" + prefix) + make() + make("install") diff --git a/var/spack/repos/builtin/packages/gmp/package.py b/var/spack/repos/builtin/packages/gmp/package.py new file mode 100644 index 0000000000..d6af821b34 --- /dev/null +++ b/var/spack/repos/builtin/packages/gmp/package.py @@ -0,0 +1,40 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + +class Gmp(Package): + """GMP is a free library for arbitrary precision arithmetic, + operating on signed integers, rational numbers, and + floating-point numbers.""" + homepage = "https://gmplib.org" + url = "https://gmplib.org/download/gmp/gmp-6.0.0a.tar.bz2" + + version('6.0.0a', 'b7ff2d88cae7f8085bd5006096eed470') + version('6.0.0' , '6ef5869ae735db9995619135bd856b84') + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix) + make() + make("install") diff --git a/var/spack/repos/builtin/packages/gnutls/package.py b/var/spack/repos/builtin/packages/gnutls/package.py new file mode 100644 index 0000000000..cf57a24a6d --- /dev/null +++ b/var/spack/repos/builtin/packages/gnutls/package.py @@ -0,0 +1,22 @@ +from spack import * + +class Gnutls(Package): + """GnuTLS is a secure communications library implementing the SSL, + TLS and DTLS protocols and technologies around them. It + provides a simple C language application programming interface + (API) to access the secure communications protocols as well as + APIs to parse and write X.509, PKCS #12, OpenPGP and other + required structures. It is aimed to be portable and efficient + with focus on security and interoperability.""" + + homepage = "http://www.gnutls.org" + url = "ftp://ftp.gnutls.org/gcrypt/gnutls/v3.3/gnutls-3.3.9.tar.xz" + + version('3.3.9', 'ff61b77e39d09f1140ab5a9cf52c58b6') + + depends_on("nettle") + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix) + make() + make("install") diff --git a/var/spack/repos/builtin/packages/gperf/package.py b/var/spack/repos/builtin/packages/gperf/package.py new file mode 100644 index 0000000000..32551b67b4 --- /dev/null +++ b/var/spack/repos/builtin/packages/gperf/package.py @@ -0,0 +1,19 @@ +from spack import * + +class Gperf(Package): + """GNU gperf is a perfect hash function generator. For a given + list of strings, it produces a hash function and hash table, in + form of C or C++ code, for looking up a value depending on the + input string. The hash function is perfect, which means that the + hash table has no collisions, and the hash table lookup needs a + single string comparison only.""" + + homepage = "https://www.gnu.org/software/gperf/" + url = "http://ftp.gnu.org/pub/gnu/gperf/gperf-3.0.4.tar.gz" + + version('3.0.4', 'c1f1db32fb6598d6a93e6e88796a8632') + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix) + make() + make("install") diff --git a/var/spack/repos/builtin/packages/gperftools/package.py b/var/spack/repos/builtin/packages/gperftools/package.py new file mode 100644 index 0000000000..8900462324 --- /dev/null +++ b/var/spack/repos/builtin/packages/gperftools/package.py @@ -0,0 +1,38 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + +class Gperftools(Package): + """Google's fast malloc/free implementation, especially for multi-threaded applications. + Contains tcmalloc, heap-checker, heap-profiler, and cpu-profiler.""" + homepage = "https://code.google.com/p/gperftools" + url = "https://googledrive.com/host/0B6NtGsLhIcf7MWxMMF9JdTN3UVk/gperftools-2.3.tar.gz" + + version('2.3', 'f54dd119f0e46ac1f13264f8d97adf90', url="https://googledrive.com/host/0B6NtGsLhIcf7MWxMMF9JdTN3UVk/gperftools-2.3.tar.gz") + + def install(self, spec, prefix): + configure("--prefix=" + prefix) + make() + make("install") diff --git a/var/spack/repos/builtin/packages/graphlib/package.py b/var/spack/repos/builtin/packages/graphlib/package.py new file mode 100644 index 0000000000..ddac0b2b66 --- /dev/null +++ b/var/spack/repos/builtin/packages/graphlib/package.py @@ -0,0 +1,14 @@ +from spack import * + +class Graphlib(Package): + """Library to create, manipulate, and export graphs Graphlib.""" + homepage = "http://https://github.com/lee218llnl/graphlib" + url = "https://github.com/lee218llnl/graphlib/archive/v2.0.0.tar.gz" + + version('2.0.0', '43c6df84f1d38ba5a5dce0ae19371a70') + + def install(self, spec, prefix): + cmake(".", *std_cmake_args) + + make() + make("install") diff --git a/var/spack/repos/builtin/packages/graphviz/package.py b/var/spack/repos/builtin/packages/graphviz/package.py new file mode 100644 index 0000000000..7af7da1881 --- /dev/null +++ b/var/spack/repos/builtin/packages/graphviz/package.py @@ -0,0 +1,21 @@ +from spack import * + +class Graphviz(Package): + """Graph Visualization Software""" + homepage = "http://www.graphviz.org" + url = "http://www.graphviz.org/pub/graphviz/stable/SOURCES/graphviz-2.38.0.tar.gz" + + version('2.38.0', '5b6a829b2ac94efcd5fa3c223ed6d3ae') + + parallel = False + + depends_on("swig") + depends_on("python") + depends_on("ghostscript") + + def install(self, spec, prefix): + configure("--prefix=%s" %prefix) + + make() + make("install") + diff --git a/var/spack/repos/builtin/packages/gtkplus/package.py b/var/spack/repos/builtin/packages/gtkplus/package.py new file mode 100644 index 0000000000..0ebc7100de --- /dev/null +++ b/var/spack/repos/builtin/packages/gtkplus/package.py @@ -0,0 +1,22 @@ +from spack import * + +class Gtkplus(Package): + """The GTK+ 2 package contains libraries used for creating graphical user interfaces for applications.""" + homepage = "http://www.gtk.org" + + version('2.24.25', '612350704dd3aacb95355a4981930c6f', + url="http://ftp.gnome.org/pub/gnome/sources/gtk+/2.24/gtk+-2.24.25.tar.xz") + + depends_on("atk") + depends_on("gdk-pixbuf") + depends_on("pango") + + def patch(self): + # remove disable deprecated flag. + filter_file(r'CFLAGS="-DGDK_PIXBUF_DISABLE_DEPRECATED $CFLAGS"', + '', 'configure', string=True) + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix) + make() + make("install") diff --git a/var/spack/repos/builtin/packages/harfbuzz/package.py b/var/spack/repos/builtin/packages/harfbuzz/package.py new file mode 100644 index 0000000000..ed7c42a909 --- /dev/null +++ b/var/spack/repos/builtin/packages/harfbuzz/package.py @@ -0,0 +1,20 @@ +from spack import * + +class Harfbuzz(Package): + """The Harfbuzz package contains an OpenType text shaping engine.""" + homepage = "http://www.freedesktop.org/wiki/Software/HarfBuzz/" + url = "http://www.freedesktop.org/software/harfbuzz/release/harfbuzz-0.9.37.tar.bz2" + + version('0.9.37', 'bfe733250e34629a188d82e3b971bc1e') + + depends_on("glib") + depends_on("icu") + depends_on("freetype") + + def patch(self): + change_sed_delimiter('@', ';', 'src/Makefile.in') + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix) + make() + make("install") diff --git a/var/spack/repos/builtin/packages/hdf5/package.py b/var/spack/repos/builtin/packages/hdf5/package.py new file mode 100644 index 0000000000..15e0ef9338 --- /dev/null +++ b/var/spack/repos/builtin/packages/hdf5/package.py @@ -0,0 +1,42 @@ +from spack import * + +class Hdf5(Package): + """HDF5 is a data model, library, and file format for storing and managing + data. It supports an unlimited variety of datatypes, and is designed for + flexible and efficient I/O and for high volume and complex data. + """ + + homepage = "http://www.hdfgroup.org/HDF5/" + url = "http://www.hdfgroup.org/ftp/HDF5/releases/hdf5-1.8.13/src/hdf5-1.8.13.tar.gz" + list_url = "http://www.hdfgroup.org/ftp/HDF5/releases" + list_depth = 3 + + version('1.8.15', '03cccb5b33dbe975fdcd8ae9dc021f24') + version('1.8.13', 'c03426e9e77d7766944654280b467289') + + depends_on("mpi") + depends_on("zlib") + + # TODO: currently hard-coded to use OpenMPI + def install(self, spec, prefix): + + configure( + "--prefix=%s" % prefix, + "--with-zlib=%s" % spec['zlib'].prefix, + "--enable-parallel", + "--enable-shared", + "CC=%s" % spec['mpich'].prefix.bin + "/mpicc", + "CXX=%s" % spec['mpich'].prefix.bin + "/mpic++") + + make() + make("install") + + def url_for_version(self, version): + v = str(version) + + if version == Version("1.2.2"): + return "http://www.hdfgroup.org/ftp/HDF5/releases/hdf5-" + v + ".tar.gz" + elif version < Version("1.7"): + return "http://www.hdfgroup.org/ftp/HDF5/releases/hdf5-" + version.up_to(2) + "/hdf5-" + v + ".tar.gz" + else: + return "http://www.hdfgroup.org/ftp/HDF5/releases/hdf5-" + v + "/src/hdf5-" + v + ".tar.gz" diff --git a/var/spack/repos/builtin/packages/hwloc/package.py b/var/spack/repos/builtin/packages/hwloc/package.py new file mode 100644 index 0000000000..31a31f376a --- /dev/null +++ b/var/spack/repos/builtin/packages/hwloc/package.py @@ -0,0 +1,25 @@ +from spack import * + +class Hwloc(Package): + """The Portable Hardware Locality (hwloc) software package + provides a portable abstraction (across OS, versions, + architectures, ...) of the hierarchical topology of modern + architectures, including NUMA memory nodes, sockets, shared + caches, cores and simultaneous multithreading. It also gathers + various system attributes such as cache and memory information + as well as the locality of I/O devices such as network + interfaces, InfiniBand HCAs or GPUs. It primarily aims at + helping applications with gathering information about modern + computing hardware so as to exploit it accordingly and + efficiently.""" + homepage = "http://www.open-mpi.org/projects/hwloc/" + url = "http://www.open-mpi.org/software/hwloc/v1.9/downloads/hwloc-1.9.tar.gz" + + version('1.9', '1f9f9155682fe8946a97c08896109508') + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix) + + make() + make("install") + diff --git a/var/spack/repos/builtin/packages/hypre/package.py b/var/spack/repos/builtin/packages/hypre/package.py new file mode 100644 index 0000000000..198b3f00dc --- /dev/null +++ b/var/spack/repos/builtin/packages/hypre/package.py @@ -0,0 +1,32 @@ +from spack import * + +class Hypre(Package): + """Hypre is a library of high performance preconditioners that + features parallel multigrid methods for both structured and + unstructured grid problems.""" + + homepage = "https://computation.llnl.gov/project/linear_solvers/software.php" + url = "https://computation.llnl.gov/project/linear_solvers/download/hypre-2.10.0b.tar.gz" + + version('2.10.0b', '768be38793a35bb5d055905b271f5b8e') + + depends_on("mpi") + depends_on("blas") + depends_on("lapack") + + def install(self, spec, prefix): + blas_dir = spec['blas'].prefix + lapack_dir = spec['lapack'].prefix + + # Hypre's source is staged under ./src so we'll have to manually + # cd into it. + with working_dir("src"): + configure( + "--prefix=%s" % prefix, + "--with-blas-libs=blas", + "--with-blas-lib-dirs=%s/lib" % blas_dir, + "--with-lapack-libs=\"lapack blas\"", + "--with-lapack-lib-dirs=%s/lib" % lapack_dir, + "--with-MPI") + make() + make("install") diff --git a/var/spack/repos/builtin/packages/icu/package.py b/var/spack/repos/builtin/packages/icu/package.py new file mode 100644 index 0000000000..f256ec5712 --- /dev/null +++ b/var/spack/repos/builtin/packages/icu/package.py @@ -0,0 +1,25 @@ +from spack import * + +class Icu(Package): + """The International Components for Unicode (ICU) package is a + mature, widely used set of C/C++ libraries providing Unicode and + Globalization support for software applications. ICU is widely + portable and gives applications the same results on all + platforms.""" + # FIXME: add a proper url for your package's homepage here. + homepage = "http://www.example.com" + url = "http://download.icu-project.org/files/icu4c/54.1/icu4c-54_1-src.tgz" + + version('54.1', 'e844caed8f2ca24c088505b0d6271bc0') + + + def url_for_version(self, version): + return "http://download.icu-project.org/files/icu4c/%s/icu4c-%s-src.tgz" % ( + version, str(version).replace('.', '_')) + + + def install(self, spec, prefix): + with working_dir("source"): + configure("--prefix=%s" % prefix) + make() + make("install") diff --git a/var/spack/repos/builtin/packages/icu4c/package.py b/var/spack/repos/builtin/packages/icu4c/package.py new file mode 100644 index 0000000000..55b44463b2 --- /dev/null +++ b/var/spack/repos/builtin/packages/icu4c/package.py @@ -0,0 +1,17 @@ +from spack import * + +class Icu4c(Package): + """ICU is a mature, widely used set of C/C++ and Java libraries + providing Unicode and Globalization support for software applications.""" + + homepage = "http://site.icu-project.org/" + url = "http://downloads.sourceforge.net/project/icu/ICU4C/54.1/icu4c-54_1-src.tgz" + + version('54_1', 'e844caed8f2ca24c088505b0d6271bc0') + + def install(self, spec, prefix): + cd("source") + configure("--prefix=%s" % prefix) + + make() + make("install") diff --git a/var/spack/repos/builtin/packages/isl/package.py b/var/spack/repos/builtin/packages/isl/package.py new file mode 100644 index 0000000000..836ef3ea40 --- /dev/null +++ b/var/spack/repos/builtin/packages/isl/package.py @@ -0,0 +1,17 @@ +from spack import * + +class Isl(Package): + """isl is a thread-safe C library for manipulating sets and + relations of integer points bounded by affine constraints.""" + homepage = "http://isl.gforge.inria.fr" + url = "http://isl.gforge.inria.fr/isl-0.14.tar.bz2" + + version('0.14', 'acd347243fca5609e3df37dba47fd0bb') + + depends_on("gmp") + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix, + "--with-gmp-prefix=%s" % spec['gmp'].prefix) + make() + make("install") diff --git a/var/spack/repos/builtin/packages/jdk/package.py b/var/spack/repos/builtin/packages/jdk/package.py new file mode 100644 index 0000000000..8f8076dd14 --- /dev/null +++ b/var/spack/repos/builtin/packages/jdk/package.py @@ -0,0 +1,46 @@ +#------------------------------------------------------------------------------ +# Author: Justin Too +#------------------------------------------------------------------------------ +import distutils +from distutils import dir_util +from subprocess import call + +import spack +from spack import * +import llnl.util.tty as tty + +class Jdk(Package): + """The Java Development Kit (JDK) released by Oracle Corporation + in the form of a binary product aimed at Java developers.""" + homepage = "http://www.oracle.com/technetwork/java/javase/downloads/index.html" + + version('8u25-linux-x64', 'e145c03a7edc845215092786bcfba77e', + url="http://download.oracle.com/otn-pub/java/jdk/8u25-b17/jdk-8u25-linux-x64.tar.gz") + + # Oracle requires that you accept their License Agreement in order + # to access the Java packages in download.oracle.com. In order to + # automate this process, we need to utilize these additional curl + # commandline options. + # + # See http://stackoverflow.com/questions/10268583/how-to-automate-download-and-installation-of-java-jdk-on-linux + curl_options=[ + '-j', # junk cookies + '-H', # specify required License Agreement cookie + 'Cookie: oraclelicense=accept-securebackup-cookie'] + + def do_fetch(self): + # Add our custom curl commandline options + tty.msg( + "[Jdk] Adding required commandline options to curl " + + "before performing fetch: %s" % + (self.curl_options)) + + for option in self.curl_options: + spack.curl.add_default_arg(option) + + # Now perform the actual fetch + super(Jdk, self).do_fetch() + + + def install(self, spec, prefix): + distutils.dir_util.copy_tree(".", prefix) diff --git a/var/spack/repos/builtin/packages/jpeg/package.py b/var/spack/repos/builtin/packages/jpeg/package.py new file mode 100644 index 0000000000..87820467db --- /dev/null +++ b/var/spack/repos/builtin/packages/jpeg/package.py @@ -0,0 +1,14 @@ +from spack import * + +class Jpeg(Package): + """jpeg library""" + homepage = "http://www.ijg.org" + url = "http://www.ijg.org/files/jpegsrc.v9a.tar.gz" + + version('9a', '3353992aecaee1805ef4109aadd433e7') + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix) + + make() + make("install") diff --git a/var/spack/repos/builtin/packages/launchmon/package.py b/var/spack/repos/builtin/packages/launchmon/package.py new file mode 100644 index 0000000000..6fbe6a68d0 --- /dev/null +++ b/var/spack/repos/builtin/packages/launchmon/package.py @@ -0,0 +1,47 @@ +############################################################################## +# Copyright (c) 2014, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Matthew LeGendre, legendre1@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + +class Launchmon(Package): + """Software infrastructure that enables HPC run-time tools to + co-locate tool daemons with a parallel job.""" + homepage = "http://sourceforge.net/projects/launchmon" + url = "http://downloads.sourceforge.net/project/launchmon/launchmon/1.0.1%20release/launchmon-1.0.1.tar.gz" + + version('1.0.1', '2f12465803409fd07f91174a4389eb2b') + version('1.0.1-2', git='https://github.com/scalability-llnl/launchmon.git', commit='ff7e22424b8f375318951eb1c9282fcbbfa8aadf') + + depends_on('autoconf') + depends_on('automake') + depends_on('libtool') + + def install(self, spec, prefix): + configure( + "--prefix=" + prefix, + "--with-bootfabric=cobo", + "--with-rm=slurm") + + make() + make("install") diff --git a/var/spack/repos/builtin/packages/launchmon/patch.lmon_install_dir b/var/spack/repos/builtin/packages/launchmon/patch.lmon_install_dir new file mode 100644 index 0000000000..8a1d93fdc9 --- /dev/null +++ b/var/spack/repos/builtin/packages/launchmon/patch.lmon_install_dir @@ -0,0 +1,147 @@ +Index: launchmon/src/linux/lmon_api/Makefile.am +=================================================================== +--- launchmon/src/linux/lmon_api/Makefile.am (revision 481) ++++ launchmon/src/linux/lmon_api/Makefile.am (working copy) +@@ -80,13 +80,10 @@ + libmonfeapi_la_CFLAGS = $(AM_CFLAGS) + libmonfeapi_la_CXXFLAGS = $(AM_CXXFLAGS) + +-libmonfeapi_la_LDFLAGS = -L$(top_srcdir)/@COMMLOC@ \ +- -L$(top_srcdir)/@GCRYPTLOC@ \ +- -L$(top_srcdir)/@GPGERRLOC@ \ +- $(AM_LDFLAGS) \ +- -version-info @LMON_CURRENT@:@LMON_REVISION@:@LMON_AGE@ ++libmonfeapi_la_LDFLAGS = $(AM_LDFLAGS) \ ++ -version-info @LMON_CURRENT@:@LMON_REVISION@:@LMON_AGE@ + +-libmonfeapi_la_LIBADD = @LIBPTHREAD@ @LIBCOMM@ @LIBGCRYPT@ @LIBGPGERR@ @LIBRT@ ++libmonfeapi_la_LIBADD = @LIBPTHREAD@ $(top_builddir)/@COMMLOC@/@LIBCOMM@ $(top_builddir)/@GCRYPTLOC@/@LIBGCRYPT@ $(top_builddir)/@GPGERRLOC@/@LIBGPGERR@ @LIBRT@ + + libmonbeapi_la_SOURCES = lmon_be.cxx \ + lmon_daemon_internal.cxx \ +@@ -113,13 +110,10 @@ + libmonbeapi_la_CFLAGS = $(AM_CFLAGS) + libmonbeapi_la_CXXFLAGS = $(AM_CXXFLAGS) + +-libmonbeapi_la_LDFLAGS = -L$(top_srcdir)/@COMMLOC@ \ +- -L$(top_srcdir)/@GCRYPTLOC@ \ +- -L$(top_srcdir)/@GPGERRLOC@ \ +- $(AM_LDFLAGS) \ ++libmonbeapi_la_LDFLAGS = $(AM_LDFLAGS) \ + -version-info @LMON_CURRENT@:@LMON_REVISION@:@LMON_AGE@ + +-libmonbeapi_la_LIBADD = @LIBCOMM@ @LIBGCRYPT@ @LIBGPGERR@ ++libmonbeapi_la_LIBADD = $(top_builddir)/@COMMLOC@/@LIBCOMM@ $(top_builddir)/@GCRYPTLOC@/@LIBGCRYPT@ $(top_builddir)/@GPGERRLOC@/@LIBGPGERR@ + + + # +@@ -146,10 +140,8 @@ + + libmonmwapi_la_CXXFLAGS = $(AM_CXXFLAGS) + +-libmonmwapi_la_LDFLAGS = -L$(top_srcdir)/@COMMLOC@ \ +- -L$(top_srcdir)/@GCRYPTLOC@ \ +- -L$(top_srcdir)/@GPGERRLOC@ \ +- $(AM_LDFLAGS) \ ++libmonmwapi_la_LDFLAGS = $(AM_LDFLAGS) \ + -version-info @LMON_CURRENT@:@LMON_REVISION@:@LMON_AGE@ + +-libmonmwapi_la_LIBADD = @LIBCOMM@ @LIBGCRYPT@ @LIBGPGERR@ ++ ++libmonmwapi_la_LIBADD = $(top_builddir)/@COMMLOC@/@LIBCOMM@ $(top_builddir)/@GCRYPTLOC@/@LIBGCRYPT@ $(top_builddir)/@GPGERRLOC@/@LIBGPGERR@ +Index: tools/cobo/test/Makefile.am +=================================================================== +--- tools/cobo/test/Makefile.am (revision 481) ++++ tools/cobo/test/Makefile.am (working copy) +@@ -37,12 +37,12 @@ + + client_SOURCES = client.c + +-client_LDFLAGS = -L$(top_srcdir)/@COMMLOC@ ++client_LDFLAGS = + +-client_LDADD = @LIBCOMM@ ++client_LDADD = $(top_srcdir)/@COMMLOC@/@LIBCOMM@ + + server_rsh_SOURCES = server_rsh.c + +-server_rsh_LDFLAGS = -L$(top_srcdir)/@COMMLOC@ ++server_rsh_LDFLAGS = + +-server_rsh_LDADD = @LIBCOMM@ ++server_rsh_LDADD = $(top_srcdir)/@COMMLOC@/@LIBCOMM@ +Index: tools/pmgr_collective/test/Makefile.am +=================================================================== +--- tools/pmgr_collective/test/Makefile.am (revision 481) ++++ tools/pmgr_collective/test/Makefile.am (working copy) +@@ -31,18 +31,18 @@ + ## Jun 10 2008 DHA: Copied from the old Makefile. + ## + +-INCLUDES = -I$(top_srcdir)/@COMMLOC@ ++INCLUDES = + + noinst_PROGRAMS = client mpirun_rsh + + client_SOURCES = client.c + +-client_LDFLAGS = -L$(top_srcdir)/@COMMLOC@ ++client_LDFLAGS = + +-client_LDADD = @LIBCOMM@ ++client_LDADD = @COMMLOC@/@LIBCOMM@ + + mpirun_rsh_SOURCES = mpirun_rsh.c + +-mpirun_rsh_LDFLAGS = -L$(top_srcdir)/@COMMLOC@ ++mpirun_rsh_LDFLAGS = + +-mpirun_rsh_LDADD = @LIBCOMM@ ++mpirun_rsh_LDADD = @COMMLOC@/@LIBCOMM@ +Index: config/x_ac_bootfabric.m4 +=================================================================== +--- config/x_ac_bootfabric.m4 (revision 481) ++++ config/x_ac_bootfabric.m4 (working copy) +@@ -63,7 +63,7 @@ + #AC_DEFINE(TOOL_SS_ENV, "LMON_SHARED_SECRET", [Define TOOL_SS_ENV]) + #AC_DEFINE(TOOL_SCH_ENV, "LMON_SEC_CHK", [Define TOOL_SCH_ENV]) + #AC_SUBST(COMMLOC, tools/pmgr_collective/src) +- #AC_SUBST(LIBCOMM, -lpmgr_collective) ++ #AC_SUBST(LIBCOMM, libcobo.la) + #else + commfab_found="no" + AC_MSG_ERROR([--with-bootfabric=pmgr is given, but pmgr_collective has been deprecated]) +@@ -87,7 +87,7 @@ + AC_DEFINE(TOOL_SS_ENV, "LMON_SHARED_SECRET", [Define TOOL_SS_ENV]) + AC_DEFINE(TOOL_SCH_ENV, "LMON_SEC_CHK", [Define TOOL_SCH_ENV]) + AC_SUBST(COMMLOC, tools/cobo/src) +- AC_SUBST(LIBCOMM, -lcobo) ++ AC_SUBST(LIBCOMM, libcobo.la) + + if test "x$with_cobo_port" != "xcheck" -a "x$with_cobo_port" != "xyes"; then + AC_DEFINE(COBO_BEGIN_PORT, $with_cobo_port, [Define a beginning port for COBO_BASED]) +@@ -117,7 +117,7 @@ + AC_DEFINE(TOOL_SS_ENV, "LMON_SHARED_SECRET", [Define TOOL_SS_ENV]) + AC_DEFINE(TOOL_SCH_ENV, "LMON_SEC_CHK", [Define TOOL_SCH_ENV]) + AC_SUBST(COMMLOC, tools/cobo/src) +- AC_SUBST(LIBCOMM, -lcobo) ++ AC_SUBST(LIBCOMM, libcobo.la) + + if test "x$with_cobo_port" != "xcheck" -a "x$with_cobo_port" != "xyes"; then + AC_DEFINE(COBO_BEGIN_PORT, $with_cobo_port, [Define a beginning port for COBO_BASED]) +Index: config/x_ac_gcrpyt.m4 +=================================================================== +--- config/x_ac_gcrypt.m4 2011-10-22 00:50:38.000000000 -0700 ++++ config/x_ac_gcrypt.patched.m4 2014-03-14 11:33:59.189220000 -0700 +@@ -55,8 +55,8 @@ + AC_CONFIG_SUBDIRS([tools/libgpg-error]) + AC_SUBST(GPGERRLOC, [tools/libgpg-error/src]) + AC_SUBST(GCRYPTLOC, [tools/libgcrypt/src]) +- AC_SUBST(LIBGCRYPT, [-lgcrypt]) +- AC_SUBST(LIBGPGERR, [-lgpg-error]) ++ AC_SUBST(LIBGCRYPT, [libgcrypt.la]) ++ AC_SUBST(LIBGPGERR, [libgpg-error.la]) + gcrypt_configured="yes" + else + AC_MSG_ERROR([tools/libgpg-error or tools/libgcrypt not found]) + diff --git a/var/spack/repos/builtin/packages/lcms/package.py b/var/spack/repos/builtin/packages/lcms/package.py new file mode 100644 index 0000000000..a53c2f997a --- /dev/null +++ b/var/spack/repos/builtin/packages/lcms/package.py @@ -0,0 +1,19 @@ +from spack import * + +class Lcms(Package): + """Little cms is a color management library. Implements fast + transforms between ICC profiles. It is focused on speed, and is + portable across several platforms (MIT license).""" + homepage = "http://www.littlecms.com" + url = "http://downloads.sourceforge.net/project/lcms/lcms/2.6/lcms2-2.6.tar.gz" + + version('2.6', 'f4c08d38ceade4a664ebff7228910a33') + + depends_on("jpeg") + depends_on("libtiff") + depends_on("zlib") + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix) + make() + make("install") diff --git a/var/spack/repos/builtin/packages/leveldb/package.py b/var/spack/repos/builtin/packages/leveldb/package.py new file mode 100644 index 0000000000..da68a9cbcb --- /dev/null +++ b/var/spack/repos/builtin/packages/leveldb/package.py @@ -0,0 +1,29 @@ +import os +import glob +from spack import * + +class Leveldb(Package): + """LevelDB is a fast key-value storage library written at Google + that provides an ordered mapping from string keys to string values.""" + + homepage = "https://github.com/google/leveldb" + url = "https://github.com/google/leveldb/archive/v1.18.tar.gz" + + version('1.18', '73770de34a2a5ab34498d2e05b2b7fa0') + + depends_on("snappy") + + def install(self, spec, prefix): + make() + + mkdirp(prefix.include) + mkdirp(prefix.lib) + + cp = which('cp') + + # cp --preserve=links libleveldb.* prefix/lib + args = glob.glob('libleveldb.*') + args.append(prefix + '/lib') + cp('--preserve=links', *args) + + cp('-r', 'include/leveldb', prefix + '/include') diff --git a/var/spack/repos/builtin/packages/libNBC/package.py b/var/spack/repos/builtin/packages/libNBC/package.py new file mode 100644 index 0000000000..6d08f3219c --- /dev/null +++ b/var/spack/repos/builtin/packages/libNBC/package.py @@ -0,0 +1,43 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + +class Libnbc(Package): + """LibNBC is a prototypic implementation of a nonblocking + interface for MPI collective operations. Based on ANSI C and + MPI-1, it supports all MPI-1 collective operations in a + nonblocking manner. LibNBC is distributed under the BSD license. + """ + homepage = "http://unixer.de/research/nbcoll/libnbc/" + url = "http://unixer.de/research/nbcoll/libnbc/libNBC-1.1.1.tar.gz" + + version('1.1.1', 'ece5c94992591a9fa934a90e5dbe50ce') + + depends_on("mpi") + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix) + make() + make("install") diff --git a/var/spack/repos/builtin/packages/libarchive/package.py b/var/spack/repos/builtin/packages/libarchive/package.py new file mode 100644 index 0000000000..cbd4b89cd0 --- /dev/null +++ b/var/spack/repos/builtin/packages/libarchive/package.py @@ -0,0 +1,16 @@ +from spack import * + +class Libarchive(Package): + """libarchive: C library and command-line tools for reading and + writing tar, cpio, zip, ISO, and other archive formats.""" + homepage = "http://www.libarchive.org" + url = "http://www.libarchive.org/downloads/libarchive-3.1.2.tar.gz" + + version('3.1.2', 'efad5a503f66329bb9d2f4308b5de98a') + version('3.1.1', '1f3d883daf7161a0065e42a15bbf168f') + version('3.1.0', '095a287bb1fd687ab50c85955692bf3a') + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix) + make() + make("install") diff --git a/var/spack/repos/builtin/packages/libcircle/package.py b/var/spack/repos/builtin/packages/libcircle/package.py new file mode 100644 index 0000000000..3f7c996fb0 --- /dev/null +++ b/var/spack/repos/builtin/packages/libcircle/package.py @@ -0,0 +1,18 @@ +import os +from spack import * + +class Libcircle(Package): + """libcircle provides an efficient distributed queue on a cluster, + using self-stabilizing work stealing.""" + + homepage = "https://github.com/hpc/libcircle" + + version('0.2.1-rc.1', '2b1369a5736457239f908abf88143ec2', + url='https://github.com/hpc/libcircle/releases/download/0.2.1-rc.1/libcircle-0.2.1-rc.1.tar.gz') + + depends_on('mpi') + + def install(self, spec, prefix): + configure("--prefix=" + prefix) + make() + make("install") diff --git a/var/spack/repos/builtin/packages/libdrm/package.py b/var/spack/repos/builtin/packages/libdrm/package.py new file mode 100644 index 0000000000..00736b7811 --- /dev/null +++ b/var/spack/repos/builtin/packages/libdrm/package.py @@ -0,0 +1,18 @@ +from spack import * + +class Libdrm(Package): + """A userspace library for accessing the DRM, direct + rendering manager, on Linux, BSD and other operating + systems that support the ioctl interface.""" + + homepage = "http://dri.freedesktop.org/libdrm/" # no real website... + url = "http://dri.freedesktop.org/libdrm/libdrm-2.4.59.tar.gz" + + version('2.4.59', '105ac7af1afcd742d402ca7b4eb168b6') + version('2.4.33', '86e4e3debe7087d5404461e0032231c8') + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix) + + make() + make("install") diff --git a/var/spack/repos/builtin/packages/libdwarf/package.py b/var/spack/repos/builtin/packages/libdwarf/package.py new file mode 100644 index 0000000000..099a974e93 --- /dev/null +++ b/var/spack/repos/builtin/packages/libdwarf/package.py @@ -0,0 +1,81 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * +import os + +# Only build certain parts of dwarf because the other ones break. +dwarf_dirs = ['libdwarf', 'dwarfdump2'] + +class Libdwarf(Package): + """The DWARF Debugging Information Format is of interest to + programmers working on compilers and debuggers (and any one + interested in reading or writing DWARF information). It was + developed by a committee (known as the PLSIG at the time) + starting around 1991. Starting around 1991 SGI developed the + libdwarf and dwarfdump tools for internal use and as part of + SGI IRIX developer tools. Since that time dwarfdump and + libdwarf have been shipped (as an executable and archive + respectively, not source) with every release of the SGI + MIPS/IRIX C compiler.""" + + homepage = "http://www.prevanders.net/dwarf.html" + url = "http://www.prevanders.net/libdwarf-20130729.tar.gz" + list_url = homepage + + version('20130729', '4cc5e48693f7b93b7aa0261e63c0e21d') + version('20130207', '64b42692e947d5180e162e46c689dfbf') + version('20130126', 'ded74a5e90edb5a12aac3c29d260c5db') + + depends_on("libelf") + + parallel = False + + + def install(self, spec, prefix): + # dwarf build does not set arguments for ar properly + make.add_default_arg('ARFLAGS=rcs') + + # Dwarf doesn't provide an install, so we have to do it. + mkdirp(prefix.bin, prefix.include, prefix.lib, prefix.man1) + + with working_dir('libdwarf'): + configure("--prefix=" + prefix, "--enable-shared") + make() + + install('libdwarf.a', prefix.lib) + install('libdwarf.so', prefix.lib) + install('libdwarf.h', prefix.include) + install('dwarf.h', prefix.include) + + with working_dir('dwarfdump2'): + configure("--prefix=" + prefix) + + # This makefile has strings of copy commands that + # cause a race in parallel + make(parallel=False) + + install('dwarfdump', prefix.bin) + install('dwarfdump.conf', prefix.lib) + install('dwarfdump.1', prefix.man1) diff --git a/var/spack/repos/builtin/packages/libelf/package.py b/var/spack/repos/builtin/packages/libelf/package.py new file mode 100644 index 0000000000..9338b8f393 --- /dev/null +++ b/var/spack/repos/builtin/packages/libelf/package.py @@ -0,0 +1,49 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + +class Libelf(Package): + """libelf lets you read, modify or create ELF object files in an + architecture-independent way. The library takes care of size + and endian issues, e.g. you can process a file for SPARC + processors on an Intel-based system.""" + + homepage = "http://www.mr511.de/software/english.html" + url = "http://www.mr511.de/software/libelf-0.8.13.tar.gz" + + version('0.8.13', '4136d7b4c04df68b686570afa26988ac') + version('0.8.12', 'e21f8273d9f5f6d43a59878dc274fec7') + + provides('elf') + + def install(self, spec, prefix): + configure("--prefix=" + prefix, + "--enable-shared", + "--disable-dependency-tracking", + "--disable-debug") + make() + + # The mkdir commands in libelf's install can fail in parallel + make("install", parallel=False) diff --git a/var/spack/repos/builtin/packages/libevent/package.py b/var/spack/repos/builtin/packages/libevent/package.py new file mode 100644 index 0000000000..11b1083d67 --- /dev/null +++ b/var/spack/repos/builtin/packages/libevent/package.py @@ -0,0 +1,30 @@ +from spack import * + +class Libevent(Package): + """The libevent API provides a mechanism to execute a callback function + when a specific event occurs on a file descriptor or after a timeout has been + reached. Furthermore, libevent also support callbacks due to signals or regular + timeouts. + """ + + homepage = "http://libevent.org" + url = "https://github.com/downloads/libevent/libevent/libevent-2.0.21-stable.tar.gz" + list_url = "http://libevent.org/old-releases.html" + + version('2.0.21', 'b2405cc9ebf264aa47ff615d9de527a2') + version('2.0.20', '94270cdee32c0cd0aa9f4ee6ede27e8e') + version('2.0.19', '91111579769f46055b0a438f5cc59572') + version('2.0.18', 'aa1ce9bc0dee7b8084f6855765f2c86a') + version('2.0.17', 'dad64aaaaff16b5fbec25160c06fee9a') + version('2.0.16', '899efcffccdb3d5111419df76e7dc8df') + version('2.0.15', '2643abe7ba242df15c08b2cc14ec8759') + version('2.0.14', 'cac0f379da35d3b98f83ac16fcfe1df4') + version('2.0.13', 'af786b4b3f790c9d3279792edf7867fc') + version('2.0.12', '42986228baf95e325778ed328a93e070') + + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix) + + make() + make("install") diff --git a/var/spack/repos/builtin/packages/libffi/package.py b/var/spack/repos/builtin/packages/libffi/package.py new file mode 100644 index 0000000000..acec031717 --- /dev/null +++ b/var/spack/repos/builtin/packages/libffi/package.py @@ -0,0 +1,17 @@ +from spack import * + +class Libffi(Package): + """The libffi library provides a portable, high level programming + interface to various calling conventions. This allows a programmer + to call any function specified by a call interface description at + run time.""" + homepage = "https://sourceware.org/libffi/" + + version('3.2.1','83b89587607e3eb65c70d361f13bab43',url = "ftp://sourceware.org/pub/libffi/libffi-3.2.1.tar.gz") + #version('3.1', 'f5898b29bbfd70502831a212d9249d10',url = "ftp://sourceware.org/pub/libffi/libffi-3.1.tar.gz") # Has a bug $(lib64) instead of ${lib64} in libffi.pc + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix) + make() + make("install") + diff --git a/var/spack/repos/builtin/packages/libgcrypt/package.py b/var/spack/repos/builtin/packages/libgcrypt/package.py new file mode 100644 index 0000000000..1d0a57f317 --- /dev/null +++ b/var/spack/repos/builtin/packages/libgcrypt/package.py @@ -0,0 +1,19 @@ +from spack import * + +class Libgcrypt(Package): + """Libgcrypt is a general purpose cryptographic library based on + the code from GnuPG. It provides functions for all cryptographic + building blocks: symmetric ciphers, hash algorithms, MACs, public + key algorithms, large integer functions, random numbers and a lot + of supporting functions. """ + homepage = "http://www.gnu.org/software/libgcrypt/" + url = "ftp://ftp.gnupg.org/gcrypt/libgcrypt/libgcrypt-1.6.2.tar.bz2" + + version('1.6.2', 'b54395a93cb1e57619943c082da09d5f') + + depends_on("libgpg-error") + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix) + make() + make("install") diff --git a/var/spack/repos/builtin/packages/libgpg-error/package.py b/var/spack/repos/builtin/packages/libgpg-error/package.py new file mode 100644 index 0000000000..6c1d1a10a7 --- /dev/null +++ b/var/spack/repos/builtin/packages/libgpg-error/package.py @@ -0,0 +1,17 @@ +from spack import * + +class LibgpgError(Package): + """Libgpg-error is a small library that defines common error + values for all GnuPG components. Among these are GPG, GPGSM, + GPGME, GPG-Agent, libgcrypt, Libksba, DirMngr, Pinentry, + SmartCard Daemon and possibly more in the future. """ + + homepage = "https://www.gnupg.org/related_software/libgpg-error" + url = "ftp://ftp.gnupg.org/gcrypt/libgpg-error/libgpg-error-1.18.tar.bz2" + + version('1.18', '12312802d2065774b787cbfc22cc04e9') + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix) + make() + make("install") diff --git a/var/spack/repos/builtin/packages/libjpeg-turbo/package.py b/var/spack/repos/builtin/packages/libjpeg-turbo/package.py new file mode 100644 index 0000000000..07ee183947 --- /dev/null +++ b/var/spack/repos/builtin/packages/libjpeg-turbo/package.py @@ -0,0 +1,20 @@ +from spack import * + +class LibjpegTurbo(Package): + """libjpeg-turbo is a fork of the original IJG libjpeg which uses + SIMD to accelerate baseline JPEG compression and + decompression. libjpeg is a library that implements JPEG image + encoding, decoding and transcoding.""" + homepage = "http://libjpeg-turbo.virtualgl.org" + url = "http://downloads.sourceforge.net/libjpeg-turbo/libjpeg-turbo-1.3.1.tar.gz" + + version('1.3.1', '2c3a68129dac443a72815ff5bb374b05') + + # Can use either of these. + depends_on("yasm") + depends_on("nasm") + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix) + make() + make("install") diff --git a/var/spack/repos/builtin/packages/libjson-c/package.py b/var/spack/repos/builtin/packages/libjson-c/package.py new file mode 100644 index 0000000000..c0801cce9c --- /dev/null +++ b/var/spack/repos/builtin/packages/libjson-c/package.py @@ -0,0 +1,14 @@ +from spack import * + +class LibjsonC(Package): + """ A JSON implementation in C """ + homepage = "https://github.com/json-c/json-c/wiki" + url = "https://s3.amazonaws.com/json-c_releases/releases/json-c-0.11.tar.gz" + + version('0.11', 'aa02367d2f7a830bf1e3376f77881e98') + + def install(self, spec, prefix): + configure('--prefix=%s' % prefix) + + make() + make("install") diff --git a/var/spack/repos/builtin/packages/libmng/package.py b/var/spack/repos/builtin/packages/libmng/package.py new file mode 100644 index 0000000000..e5336ea2c2 --- /dev/null +++ b/var/spack/repos/builtin/packages/libmng/package.py @@ -0,0 +1,23 @@ +from spack import * + +class Libmng(Package): + """libmng -THE reference library for reading, displaying, writing + and examining Multiple-Image Network Graphics. MNG is the animation + extension to the popular PNG image-format.""" + homepage = "http://sourceforge.net/projects/libmng/" + url = "http://downloads.sourceforge.net/project/libmng/libmng-devel/2.0.2/libmng-2.0.2.tar.gz" + + version('2.0.2', '1ffefaed4aac98475ee6267422cbca55') + + depends_on("jpeg") + depends_on("zlib") + depends_on("lcms") + + def patch(self): + # jpeg requires stdio to beincluded before its headrs. + filter_file(r'^(\#include \)', '#include\n\\1', 'libmng_types.h') + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix) + make() + make("install") diff --git a/var/spack/repos/builtin/packages/libmonitor/package.py b/var/spack/repos/builtin/packages/libmonitor/package.py new file mode 100644 index 0000000000..3b95b86ddf --- /dev/null +++ b/var/spack/repos/builtin/packages/libmonitor/package.py @@ -0,0 +1,36 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + +class Libmonitor(Package): + """Libmonitor is a library for process and thread control.""" + homepage = "http://hpctoolkit.org" + + version('20130218', svn='http://libmonitor.googlecode.com/svn/trunk/', revision=146) + + def install(self, spec, prefix): + configure("--prefix=" + prefix) + make() + make("install") diff --git a/var/spack/repos/builtin/packages/libpciaccess/package.py b/var/spack/repos/builtin/packages/libpciaccess/package.py new file mode 100644 index 0000000000..6022fc34a3 --- /dev/null +++ b/var/spack/repos/builtin/packages/libpciaccess/package.py @@ -0,0 +1,21 @@ +from spack import * + +class Libpciaccess(Package): + """Generic PCI access library.""" + + homepage = "http://cgit.freedesktop.org/xorg/lib/libpciaccess/" + url = "http://cgit.freedesktop.org/xorg/lib/libpciaccess/" + + version('0.13.4', git='http://anongit.freedesktop.org/git/xorg/lib/libpciaccess.git', + tag='libpciaccess-0.13.4') + + depends_on('autoconf') + depends_on('libtool') + + def install(self, spec, prefix): + from subprocess import call + call(["./autogen.sh"]) + configure("--prefix=%s" % prefix) + + make() + make("install") diff --git a/var/spack/repos/builtin/packages/libpng/package.py b/var/spack/repos/builtin/packages/libpng/package.py new file mode 100644 index 0000000000..e02b08663e --- /dev/null +++ b/var/spack/repos/builtin/packages/libpng/package.py @@ -0,0 +1,15 @@ +from spack import * + +class Libpng(Package): + """libpng graphics file format""" + homepage = "http://www.libpng.org/pub/png/libpng.html" + url = "http://download.sourceforge.net/libpng/libpng-1.6.16.tar.gz" + + version('1.6.16', '1a4ad377919ab15b54f6cb6a3ae2622d') + version('1.6.15', '829a256f3de9307731d4f52dc071916d') + version('1.6.14', '2101b3de1d5f348925990f9aa8405660') + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix) + make() + make("install") diff --git a/var/spack/repos/builtin/packages/libsodium/package.py b/var/spack/repos/builtin/packages/libsodium/package.py new file mode 100644 index 0000000000..1c8a16d998 --- /dev/null +++ b/var/spack/repos/builtin/packages/libsodium/package.py @@ -0,0 +1,19 @@ +from spack import * + +class Libsodium(Package): + """Sodium is a modern, easy-to-use software library for encryption, + decryption, signatures, password hashing and more.""" + homepage = "https://download.libsodium.org/doc/" + url = "https://download.libsodium.org/libsodium/releases/libsodium-1.0.3.tar.gz" + + version('1.0.3', 'b3bcc98e34d3250f55ae196822307fab') + version('1.0.2', 'dc40eb23e293448c6fc908757738003f') + version('1.0.1', '9a221b49fba7281ceaaf5e278d0f4430') + version('1.0.0', '3093dabe4e038d09f0d150cef064b2f7') + version('0.7.1', 'c224fe3923d1dcfe418c65c8a7246316') + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix) + + make() + make("install") diff --git a/var/spack/repos/builtin/packages/libtiff/package.py b/var/spack/repos/builtin/packages/libtiff/package.py new file mode 100644 index 0000000000..63c6704cb8 --- /dev/null +++ b/var/spack/repos/builtin/packages/libtiff/package.py @@ -0,0 +1,18 @@ +from spack import * + +class Libtiff(Package): + """libtiff graphics format library""" + homepage = "http://www.remotesensing.org/libtiff/" + url = "http://download.osgeo.org/libtiff/tiff-4.0.3.tar.gz" + + version('4.0.3', '051c1068e6a0627f461948c365290410') + + depends_on('jpeg') + depends_on('zlib') + depends_on('xz') + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix) + + make() + make("install") diff --git a/var/spack/repos/builtin/packages/libtool/package.py b/var/spack/repos/builtin/packages/libtool/package.py new file mode 100644 index 0000000000..a07daf9781 --- /dev/null +++ b/var/spack/repos/builtin/packages/libtool/package.py @@ -0,0 +1,14 @@ +from spack import * + +class Libtool(Package): + """libtool -- library building part of autotools""" + homepage = "https://www.gnu.org/software/libtool/" + url = "http://ftpmirror.gnu.org/libtool/libtool-2.4.2.tar.gz" + + version('2.4.2' , 'd2f3b7d4627e69e13514a40e72a24d50') + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix) + + make() + make("install") diff --git a/var/spack/repos/builtin/packages/libunwind/package.py b/var/spack/repos/builtin/packages/libunwind/package.py new file mode 100644 index 0000000000..239fcbcfd5 --- /dev/null +++ b/var/spack/repos/builtin/packages/libunwind/package.py @@ -0,0 +1,38 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + +class Libunwind(Package): + """A portable and efficient C programming interface (API) to determine + the call-chain of a program.""" + homepage = "http://www.nongnu.org/libunwind/" + url = "http://download.savannah.gnu.org/releases/libunwind/libunwind-1.1.tar.gz" + + version('1.1', 'fb4ea2f6fbbe45bf032cd36e586883ce') + + def install(self, spec, prefix): + configure("--prefix=" + prefix) + make() + make("install") diff --git a/var/spack/repos/builtin/packages/libuuid/package.py b/var/spack/repos/builtin/packages/libuuid/package.py new file mode 100644 index 0000000000..373c5bfcac --- /dev/null +++ b/var/spack/repos/builtin/packages/libuuid/package.py @@ -0,0 +1,16 @@ +from spack import * + +class Libuuid(Package): + """Portable uuid C library""" + # FIXME: add a proper url for your package's homepage here. + homepage = "http://sourceforge.net/projects/libuuid/" + url = "http://downloads.sourceforge.net/project/libuuid/libuuid-1.0.3.tar.gz?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Flibuuid%2F&ts=1433881396&use_mirror=iweb" + + version('1.0.3', 'd44d866d06286c08ba0846aba1086d68') + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix) + + # FIXME: Add logic to build and install here + make() + make("install") diff --git a/var/spack/repos/builtin/packages/libxcb/package.py b/var/spack/repos/builtin/packages/libxcb/package.py new file mode 100644 index 0000000000..16a5525c0d --- /dev/null +++ b/var/spack/repos/builtin/packages/libxcb/package.py @@ -0,0 +1,21 @@ +from spack import * + +class Libxcb(Package): + """The X protocol C-language Binding (XCB) is a replacement + for Xlib featuring a small footprint, latency hiding, direct + access to the protocol, improved threading support, and + extensibility.""" + + homepage = "http://xcb.freedesktop.org/" + url = "http://xcb.freedesktop.org/dist/libxcb-1.11.tar.gz" + + version('1.11', '1698dd837d7e6e94d029dbe8b3a82deb') + version('1.11.1', '118623c15a96b08622603a71d8789bf3') + depends_on("python") + depends_on("xcb-proto") + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix) + + make() + make("install") diff --git a/var/spack/repos/builtin/packages/libxml2/package.py b/var/spack/repos/builtin/packages/libxml2/package.py new file mode 100644 index 0000000000..3a0af6b368 --- /dev/null +++ b/var/spack/repos/builtin/packages/libxml2/package.py @@ -0,0 +1,20 @@ +from spack import * + +class Libxml2(Package): + """Libxml2 is the XML C parser and toolkit developed for the Gnome + project (but usable outside of the Gnome platform), it is free + software available under the MIT License.""" + homepage = "http://xmlsoft.org" + url = "http://xmlsoft.org/sources/libxml2-2.9.2.tar.gz" + + version('2.9.2', '9e6a9aca9d155737868b3dc5fd82f788') + + extends('python') + depends_on('zlib') + depends_on('xz') + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix) + + make() + make("install") diff --git a/var/spack/repos/builtin/packages/libxshmfence/package.py b/var/spack/repos/builtin/packages/libxshmfence/package.py new file mode 100644 index 0000000000..3aa2448b46 --- /dev/null +++ b/var/spack/repos/builtin/packages/libxshmfence/package.py @@ -0,0 +1,16 @@ +from spack import * + +class Libxshmfence(Package): + """This is a tiny library that exposes a event API on top of Linux + futexes.""" + + homepage = "http://keithp.com/blogs/dri3_extension/" # not really... + url = "http://xorg.freedesktop.org/archive/individual/lib/libxshmfence-1.2.tar.gz" + + version('1.2', 'f0b30c0fc568b22ec524859ee28556f1') + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix) + + make() + make("install") diff --git a/var/spack/repos/builtin/packages/libxslt/package.py b/var/spack/repos/builtin/packages/libxslt/package.py new file mode 100644 index 0000000000..f97332d020 --- /dev/null +++ b/var/spack/repos/builtin/packages/libxslt/package.py @@ -0,0 +1,24 @@ +from spack import * + +class Libxslt(Package): + """Libxslt is the XSLT C library developed for the GNOME + project. XSLT itself is a an XML language to define + transformation for XML. Libxslt is based on libxml2 the XML C + library developed for the GNOME project. It also implements + most of the EXSLT set of processor-portable extensions + functions and some of Saxon's evaluate and expressions + extensions.""" + homepage = "http://www.xmlsoft.org/XSLT/index.html" + url = "http://xmlsoft.org/sources/libxslt-1.1.28.tar.gz" + + version('1.1.28', '9667bf6f9310b957254fdcf6596600b7') + + depends_on("libxml2") + depends_on("xz") + depends_on("zlib") + depends_on("libgcrypt") + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix) + make() + make("install") diff --git a/var/spack/repos/builtin/packages/llvm-lld/package.py b/var/spack/repos/builtin/packages/llvm-lld/package.py new file mode 100644 index 0000000000..f229211396 --- /dev/null +++ b/var/spack/repos/builtin/packages/llvm-lld/package.py @@ -0,0 +1,46 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + +class LlvmLld(Package): + """lld - The LLVM Linker + lld is a new set of modular code for creating linker tools.""" + homepage = "http://lld.llvm.org" + url = "http://llvm.org/releases/3.4/lld-3.4.src.tar.gz" + + depends_on('llvm') + + version('3.4', '3b6a17e58c8416c869c14dd37682f78e') + + def install(self, spec, prefix): + env['CXXFLAGS'] = self.compier.cxx11_flag + + with working_dir('spack-build', create=True): + cmake('..', + '-DLLD_PATH_TO_LLVM_BUILD=%s' % spec['llvm'].prefix, + '-DLLVM_MAIN_SRC_DIR=%s' % spec['llvm'].prefix, + *std_cmake_args) + make() + make("install") diff --git a/var/spack/repos/builtin/packages/llvm/package.py b/var/spack/repos/builtin/packages/llvm/package.py new file mode 100644 index 0000000000..a6759c3033 --- /dev/null +++ b/var/spack/repos/builtin/packages/llvm/package.py @@ -0,0 +1,53 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by David Beckingsale, david@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class Llvm(Package): + """The LLVM Project is a collection of modular and reusable compiler and + toolchain technologies. Despite its name, LLVM has little to do with + traditional virtual machines, though it does provide helpful libraries + that can be used to build them. The name "LLVM" itself is not an acronym; + it is the full name of the project. + """ + homepage = 'http://llvm.org/' + url = 'http://llvm.org/releases/3.7.0/llvm-3.7.0.src.tar.xz' + + version('3.7.0', 'b98b9495e5655a672d6cb83e1a180f8e', url='http://llvm.org/releases/3.7.0/llvm-3.7.0.src.tar.xz') + version('3.6.2', '0c1ee3597d75280dee603bae9cbf5cc2', url='http://llvm.org/releases/3.6.2/llvm-3.6.2.src.tar.xz') + version('3.5.1', '2d3d8004f38852aa679e5945b8ce0b14', url='http://llvm.org/releases/3.5.1/llvm-3.5.1.src.tar.xz') + + depends_on('python@2.7:') + + def install(self, spec, prefix): + env['CXXFLAGS'] = self.compiler.cxx11_flag + + with working_dir('spack-build', create=True): + cmake('..', + '-DLLVM_REQUIRES_RTTI=1', + '-DPYTHON_EXECUTABLE=%s/bin/python' % spec['python'].prefix, + *std_cmake_args) + make() + make("install") diff --git a/var/spack/repos/builtin/packages/lmdb/package.py b/var/spack/repos/builtin/packages/lmdb/package.py new file mode 100644 index 0000000000..875b8100c5 --- /dev/null +++ b/var/spack/repos/builtin/packages/lmdb/package.py @@ -0,0 +1,39 @@ +import os +from spack import * + +class Lmdb(Package): + """Read-only mirror of official repo on openldap.org. Issues and + pull requests here are ignored. Use OpenLDAP ITS for issues. + http://www.openldap.org/software/repo.html""" + + + homepage = "http://www.openldap.org/software/repo.html" + url = "https://github.com/LMDB/lmdb/archive/LMDB_0.9.16.tar.gz" + + version('0.9.16', '0de89730b8f3f5711c2b3a4ba517b648') + + def install(self, spec, prefix): + os.chdir('libraries/liblmdb') + + make() + + mkdirp(prefix.bin) + mkdirp(prefix + '/man/man1') + mkdirp(prefix.lib) + mkdirp(prefix.include) + + bins = ['mdb_stat', 'mdb_copy', 'mdb_dump', 'mdb_load'] + for f in bins: + install(f, prefix.bin) + + mans = ['mdb_stat.1', 'mdb_copy.1', 'mdb_dump.1', 'mdb_load.1'] + for f in mans: + install(f, prefix + '/man/man1') + + libs = ['liblmdb.a', 'liblmdb.so'] + for f in libs: + install(f, prefix.lib) + + includes = ['lmdb.h'] + for f in includes: + install(f, prefix.include) diff --git a/var/spack/repos/builtin/packages/lua/package.py b/var/spack/repos/builtin/packages/lua/package.py new file mode 100644 index 0000000000..57c443cc2d --- /dev/null +++ b/var/spack/repos/builtin/packages/lua/package.py @@ -0,0 +1,26 @@ +from spack import * +import os + +class Lua(Package): + """ The Lua programming language interpreter and library """ + homepage = "http://www.lua.org" + url = "http://www.lua.org/ftp/lua-5.1.5.tar.gz" + + version('5.3.1', '797adacada8d85761c079390ff1d9961') + version('5.3.0', 'a1b0a7e92d0c85bbff7a8d27bf29f8af') + version('5.2.4', '913fdb32207046b273fdb17aad70be13') + version('5.2.3', 'dc7f94ec6ff15c985d2d6ad0f1b35654') + version('5.2.2', 'efbb645e897eae37cad4344ce8b0a614') + version('5.2.1', 'ae08f641b45d737d12d30291a5e5f6e3') + version('5.2.0', 'f1ea831f397214bae8a265995ab1a93e') + version('5.1.5', '2e115fe26e435e33b0d5c022e4490567') + version('5.1.4', 'd0870f2de55d59c1c8419f36e8fac150') + version('5.1.3', 'a70a8dfaa150e047866dc01a46272599') + + depends_on('ncurses') + + def install(self, spec, prefix): + make('INSTALL_TOP=%s' % prefix, + 'MYLDFLAGS=-L%s/lib' % spec['ncurses'].prefix, + 'linux', + 'install') diff --git a/var/spack/repos/builtin/packages/lwgrp/package.py b/var/spack/repos/builtin/packages/lwgrp/package.py new file mode 100644 index 0000000000..5963382b92 --- /dev/null +++ b/var/spack/repos/builtin/packages/lwgrp/package.py @@ -0,0 +1,18 @@ +import os +from spack import * + +class Lwgrp(Package): + """Thie light-weight group library provides process group + representations using O(log N) space and time.""" + + homepage = "https://github.com/hpc/lwgrp" + url = "https://github.com/hpc/lwgrp/releases/download/v1.0.2/lwgrp-1.0.2.tar.gz" + + version('1.0.2', 'ab7ba3bdd8534a651da5076f47f27d8a') + + depends_on('mpi') + + def install(self, spec, prefix): + configure("--prefix=" + prefix) + make() + make("install") diff --git a/var/spack/repos/builtin/packages/lwm2/package.py b/var/spack/repos/builtin/packages/lwm2/package.py new file mode 100644 index 0000000000..31afff8816 --- /dev/null +++ b/var/spack/repos/builtin/packages/lwm2/package.py @@ -0,0 +1,18 @@ +from spack import * + +class Lwm2(Package): + """LWM2: Light Weight Measurement Module. This is a PMPI module + that can collect a number of time-sliced MPI and POSIX I/O + measurements from a program. + """ + homepage = "https://jay.grs.rwth-aachen.de/redmine/projects/lwm2" + + version('torus', hg='https://jay.grs.rwth-aachen.de/hg/lwm2', revision='torus') + + depends_on("papi") + depends_on("mpi") + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix) + make() + make("install") diff --git a/var/spack/repos/builtin/packages/matio/package.py b/var/spack/repos/builtin/packages/matio/package.py new file mode 100644 index 0000000000..12cfb80926 --- /dev/null +++ b/var/spack/repos/builtin/packages/matio/package.py @@ -0,0 +1,15 @@ +from spack import * + + +class Matio(Package): + """matio is an C library for reading and writing Matlab MAT files""" + homepage = "http://sourceforge.net/projects/matio/" + url = "http://downloads.sourceforge.net/project/matio/matio/1.5.2/matio-1.5.2.tar.gz" + + version('1.5.2', '85b007b99916c63791f28398f6a4c6f1') + + def install(self, spec, prefix): + configure('--prefix=%s' % prefix) + + make() + make("install") diff --git a/var/spack/repos/builtin/packages/memaxes/package.py b/var/spack/repos/builtin/packages/memaxes/package.py new file mode 100644 index 0000000000..76d5d3f831 --- /dev/null +++ b/var/spack/repos/builtin/packages/memaxes/package.py @@ -0,0 +1,19 @@ +from spack import * + +class Memaxes(Package): + """MemAxes is a visualizer for sampled memory trace data.""" + + homepage = "https://github.com/scalability-llnl/MemAxes" + + version('0.5', '5874f3fda9fd2d313c0ff9684f915ab5', + url='https://github.com/scalability-llnl/MemAxes/archive/v0.5.tar.gz') + + depends_on("cmake@2.8.9:") + depends_on("qt@5:") + + def install(self, spec, prefix): + with working_dir('spack-build', create=True): + cmake('..', *std_cmake_args) + make() + make("install") + diff --git a/var/spack/repos/builtin/packages/mesa/package.py b/var/spack/repos/builtin/packages/mesa/package.py new file mode 100644 index 0000000000..2a04a8fd51 --- /dev/null +++ b/var/spack/repos/builtin/packages/mesa/package.py @@ -0,0 +1,34 @@ +from spack import * + +class Mesa(Package): + """Mesa is an open-source implementation of the OpenGL + specification - a system for rendering interactive 3D graphics.""" + + homepage = "http://www.mesa3d.org" + url = "ftp://ftp.freedesktop.org/pub/mesa/older-versions/8.x/8.0.5/MesaLib-8.0.5.tar.gz" + # url = "ftp://ftp.freedesktop.org/pub/mesa/10.4.4/MesaLib-10.4.4.tar.gz" + + # version('10.4.4', '8d863a3c209bf5116b2babfccccc68ce') + version('8.0.5', 'cda5d101f43b8784fa60bdeaca4056f2') + + # mesa 7.x, 8.x, 9.x + depends_on("libdrm@2.4.33") + depends_on("llvm@3.0") + depends_on("libxml2") + + # patch("llvm-fixes.patch") # using newer llvm + + # mesa 10.x + # depends_on("py-mako") + # depends_on("flex") + # depends_on("bison") + # depends_on("dri2proto") + # depends_on("libxcb") + # depends_on("libxshmfence") + + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix) + + make() + make("install") diff --git a/var/spack/repos/builtin/packages/metis/package.py b/var/spack/repos/builtin/packages/metis/package.py new file mode 100644 index 0000000000..7ce5ae1925 --- /dev/null +++ b/var/spack/repos/builtin/packages/metis/package.py @@ -0,0 +1,27 @@ +from spack import * + +class Metis(Package): + """METIS is a set of serial programs for partitioning graphs, + partitioning finite element meshes, and producing fill reducing + orderings for sparse matrices. The algorithms implemented in + METIS are based on the multilevel recursive-bisection, + multilevel k-way, and multi-constraint partitioning schemes.""" + + homepage = "http://glaros.dtc.umn.edu/gkhome/metis/metis/overview" + url = "http://glaros.dtc.umn.edu/gkhome/fetch/sw/metis/metis-5.1.0.tar.gz" + + version('5.1.0', '5465e67079419a69e0116de24fce58fe') + + depends_on('mpi') + + def install(self, spec, prefix): + cmake(".", + '-DGKLIB_PATH=%s/GKlib' % pwd(), + '-DSHARED=1', + '-DCMAKE_C_COMPILER=mpicc', + '-DCMAKE_CXX_COMPILER=mpicxx', + '-DSHARED=1', + *std_cmake_args) + + make() + make("install") diff --git a/var/spack/repos/builtin/packages/mpc/package.py b/var/spack/repos/builtin/packages/mpc/package.py new file mode 100644 index 0000000000..6fbfca3007 --- /dev/null +++ b/var/spack/repos/builtin/packages/mpc/package.py @@ -0,0 +1,42 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + +class Mpc(Package): + """Gnu Mpc is a C library for the arithmetic of complex numbers + with arbitrarily high precision and correct rounding of the + result.""" + homepage = "http://www.multiprecision.org" + url = "ftp://ftp.gnu.org/gnu/mpc/mpc-1.0.2.tar.gz" + + version('1.0.2', '68fadff3358fb3e7976c7a398a0af4c3') + + depends_on("gmp") + depends_on("mpfr") + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix) + make() + make("install") diff --git a/var/spack/repos/builtin/packages/mpe2/mpe2.patch b/var/spack/repos/builtin/packages/mpe2/mpe2.patch new file mode 100644 index 0000000000..3ade1f04f4 --- /dev/null +++ b/var/spack/repos/builtin/packages/mpe2/mpe2.patch @@ -0,0 +1,12 @@ +diff -rupN mpe2-1.3.0/src/graphics/src/mpe_graphics.c mpe2-1.3.0.new/src/graphics/src/mpe_graphics.c +--- mpe2-1.3.0/src/graphics/src/mpe_graphics.c 2009-06-15 10:36:22.000000000 -0600 ++++ mpe2-1.3.0.new/src/graphics/src/mpe_graphics.c 2014-10-25 00:11:22.000000000 -0600 +@@ -982,7 +982,7 @@ char *string; + return MPE_ERR_BAD_ARGS; + } + +- printf("color = %d, string = %s\n",(int) color, string); ++//printf("color = %d, string = %s\n",(int) color, string); + + XBSetPixVal( graph->xwin, graph->xwin->cmapping[color] ); + returnVal = XDrawString( graph->xwin->disp, XBDrawable(graph->xwin), diff --git a/var/spack/repos/builtin/packages/mpe2/package.py b/var/spack/repos/builtin/packages/mpe2/package.py new file mode 100644 index 0000000000..27295172cc --- /dev/null +++ b/var/spack/repos/builtin/packages/mpe2/package.py @@ -0,0 +1,28 @@ +from spack import * + +class Mpe2(Package): + """Message Passing Extensions (MPE) -- Parallel, shared X window graphics""" + + homepage = "http://www.mcs.anl.gov/research/projects/perfvis/software/MPE/" + url = "ftp://ftp.mcs.anl.gov/pub/mpi/mpe/mpe2-1.3.0.tar.gz" + + version('1.3.0', '67bf0c7b2e573df3ba0d2059a96c2f7b') + + patch('mpe2.patch') + + depends_on("mpi") + + provides("mpe") + + def install(self, spec, prefix): + configure("--prefix=" + prefix, + "--x-includes=/usr/X11R6/include", + "--x-libraries=/usr/X11R6/lib", + "--enable-mpe_graphics=yes", + "--disable-f77", + "--enable-viewers=no", + "--enable-slog2=no", + "--with-mpicc=mpicc") + + make() + make("install") diff --git a/var/spack/repos/builtin/packages/mpfr/package.py b/var/spack/repos/builtin/packages/mpfr/package.py new file mode 100644 index 0000000000..9c744a22df --- /dev/null +++ b/var/spack/repos/builtin/packages/mpfr/package.py @@ -0,0 +1,41 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + +class Mpfr(Package): + """The MPFR library is a C library for multiple-precision + floating-point computations with correct rounding.""" + homepage = "http://www.mpfr.org" + url = "http://www.mpfr.org/mpfr-current/mpfr-3.1.3.tar.bz2" + + version('3.1.3', '5fdfa3cfa5c86514ee4a241a1affa138') + # version('3.1.2', 'ee2c3ac63bf0c2359bf08fc3ee094c19') + + depends_on('gmp') + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix) + make() + make("install") diff --git a/var/spack/repos/builtin/packages/mpibash/mpibash-4.3.patch b/var/spack/repos/builtin/packages/mpibash/mpibash-4.3.patch new file mode 100644 index 0000000000..17e285b0bf --- /dev/null +++ b/var/spack/repos/builtin/packages/mpibash/mpibash-4.3.patch @@ -0,0 +1,1565 @@ +diff -Naur bash-4.3/builtins/circle.def mpibash-4.3/builtins/circle.def +--- bash-4.3/builtins/circle.def 1969-12-31 17:00:00.000000000 -0700 ++++ mpibash-4.3/builtins/circle.def 2014-05-13 11:27:37.314100671 -0600 +@@ -0,0 +1,620 @@ ++This file is circle.def, from which is created circle.c. ++It implements all of the "circle_*" builtins in Bash. ++ ++$PRODUCES circle.c ++ ++#include ++ ++#include ++#if defined (HAVE_UNISTD_H) ++# ifdef _MINIX ++# include ++# endif ++# include ++#endif ++ ++#include "../bashintl.h" ++#include "../shell.h" ++#include "common.h" ++#include "bashgetopt.h" ++#include ++ ++extern int running_trap, trap_saved_exit_value; ++ ++static int circle_rank; /* Rank in the Libcircle job */ ++static SHELL_VAR *create_func = NULL; /* User-defined callback function for CIRCLE_cb_create. */ ++static SHELL_VAR *process_func = NULL; /* User-defined callback function for CIRCLE_cb_process. */ ++static SHELL_VAR *reduce_init_func = NULL; /* User-defined callback function for CIRCLE_cb_reduce_init. */ ++static SHELL_VAR *reduce_fini_func = NULL; /* User-defined callback function for CIRCLE_cb_reduce_fini. */ ++static SHELL_VAR *reduce_op_func = NULL; /* User-defined callback function for CIRCLE_cb_reduce_op. */ ++static CIRCLE_handle *current_handle = NULL; /* Active handle within a callback or NULL if not within a callback */ ++static int within_reduction = 0; /* 1=within a reduction callback; 0=not */ ++ ++/* Return with a usage message if no arguments remain. */ ++#define YES_ARGS(LIST) \ ++ if ((LIST) == 0) \ ++ { \ ++ builtin_usage (); \ ++ return (EX_USAGE); \ ++ } ++ ++/* Perform the same operation as bind_variable, but with VALUE being a ++ * number, not a string. */ ++static SHELL_VAR * ++bind_variable_number (name, value, flags) ++ const char *name; ++ long value; ++ int flags; ++{ ++ char numstr[25]; /* String version of VALUE */ ++ ++ sprintf (numstr, "%ld", value); ++ return bind_variable (name, numstr, flags); ++} ++ ++/* Invoke the user-defined creation-callback function (create_func). */ ++static void ++internal_create_func (handle) ++ CIRCLE_handle *handle; ++{ ++ WORD_LIST *funcargs; ++ ++ if (create_func == NULL) ++ return; ++ current_handle = handle; ++ funcargs = make_word_list (make_word ("cb_create"), NULL); ++ execute_shell_function (create_func, funcargs); ++ dispose_words (funcargs); ++ current_handle = NULL; ++} ++ ++/* Invoke the user-defined process-callback function (process_func). */ ++static void ++internal_process_func (handle) ++ CIRCLE_handle *handle; ++{ ++ WORD_LIST *funcargs; ++ ++ if (process_func == NULL) ++ return; ++ current_handle = handle; ++ funcargs = make_word_list (make_word ("cb_process"), NULL); ++ execute_shell_function (process_func, funcargs); ++ dispose_words (funcargs); ++ current_handle = NULL; ++} ++ ++/* Invoke the user-defined reduction-initiation callback function ++ * (reduce_init_func). */ ++static void ++internal_reduce_init_func (void) ++{ ++ WORD_LIST *funcargs; ++ ++ if (reduce_init_func == NULL) ++ return; ++ within_reduction = 1; ++ funcargs = make_word_list (make_word ("cb_reduce_init"), NULL); ++ execute_shell_function (reduce_init_func, funcargs); ++ dispose_words (funcargs); ++ within_reduction = 0; ++} ++ ++/* Invoke the user-defined reduction callback function ++ * (reduce_op_func). */ ++static void ++internal_reduce_op_func (buf1, size1, buf2, size2) ++ const void* buf1; ++ size_t size1; ++ const void* buf2; ++ size_t size2; ++{ ++ WORD_LIST *funcargs; ++ ++ if (reduce_op_func == NULL) ++ return; ++ within_reduction = 1; ++ funcargs = make_word_list (make_word (buf2), NULL); ++ funcargs = make_word_list (make_word (buf1), funcargs); ++ funcargs = make_word_list (make_word ("cb_reduce_op"), funcargs); ++ execute_shell_function (reduce_op_func, funcargs); ++ dispose_words (funcargs); ++ within_reduction = 0; ++} ++ ++/* Invoke the user-defined reduction-finalization callback function ++ * (reduce_fini_func). */ ++static void ++internal_reduce_fini_func (buf, size) ++ const void* buf; ++ size_t size; ++{ ++ WORD_LIST *funcargs; ++ ++ if (reduce_fini_func == NULL) ++ return; ++ funcargs = make_word_list (make_word (buf), NULL); ++ funcargs = make_word_list (make_word ("cb_reduce_fini"), funcargs); ++ execute_shell_function (reduce_fini_func, funcargs); ++ dispose_words (funcargs); ++} ++ ++/* Look up a user-provided callback function. */ ++static int ++find_callback_function (list, user_func) ++ WORD_LIST *list; ++ SHELL_VAR **user_func; ++{ ++ char *funcname; /* Name of the user-defined function. */ ++ ++ /* If no argument was provided, nullify the callback function. */ ++ if (list == NULL) ++ { ++ *user_func = NULL; ++ return EXECUTION_SUCCESS; ++ } ++ ++ /* Get the callback function. */ ++ funcname = list->word->word; ++ list = list->next; ++ no_args (list); ++ *user_func = find_function (funcname); ++ if (*user_func == NULL) ++ { ++ builtin_error (_("function %s not found"), funcname); ++ return EXECUTION_FAILURE; ++ } ++ return EXECUTION_SUCCESS; ++} ++ ++/* Initialize Libcircle. */ ++void ++initialize_libcircle (argc, argv) ++ int argc; ++ char **argv; ++{ ++ circle_rank = CIRCLE_init (argc, argv, CIRCLE_DEFAULT_FLAGS); ++ bind_variable_number ("circle_rank", circle_rank, 0); ++ CIRCLE_enable_logging (CIRCLE_LOG_WARN); ++ CIRCLE_cb_create (internal_create_func); ++ CIRCLE_cb_process (internal_process_func); ++ CIRCLE_cb_reduce_init (internal_reduce_init_func); ++ CIRCLE_cb_reduce_op (internal_reduce_op_func); ++ CIRCLE_cb_reduce_fini (internal_reduce_fini_func); ++} ++ ++/* Finalize Libcircle. */ ++void ++finalize_libcircle (void) ++{ ++ CIRCLE_finalize (); ++} ++ ++/* ---------------------------------------------------------------------- */ ++ ++$BUILTIN circle_set_options ++$FUNCTION circle_set_options_builtin ++$SHORT_DOC circle_set_options [flag]... ++Change Libcircle's run-time behavior. ++ ++Arguments: ++ FLAG "split_random", "split_equal", or "create_global" ++ ++Multiple flags can be provided. If no flags are provided, Libcircle ++reverts to its default options. ++ ++Exit Status: ++Returns 0 unless an invalid option is given. ++$END ++/*'*/ ++ ++/* Here is the circle_set_options builtin. */ ++int ++circle_set_options_builtin (list) ++ WORD_LIST *list; ++{ ++ char *word; /* One argument */ ++ int flags = 0; /* Flags to pass to CIRCLE_set_options */ ++ ++ if (list == NULL) ++ flags = CIRCLE_DEFAULT_FLAGS; ++ else ++ while (list != NULL) ++ { ++ word = list->word->word; ++ if (!strcmp (word, "split_random")) ++ flags |= CIRCLE_SPLIT_RANDOM; ++ else if (!strcmp (word, "split_equal")) ++ flags |= CIRCLE_SPLIT_EQUAL; ++ else if (!strcmp (word, "create_global")) ++ flags |= CIRCLE_CREATE_GLOBAL; ++ else ++ { ++ builtin_error (_("invalid flag \"%s\""), word); ++ return (EXECUTION_FAILURE); ++ } ++ list = list->next; ++ } ++ CIRCLE_set_options (flags); ++ return EXECUTION_SUCCESS; ++} ++ ++$BUILTIN circle_cb_create ++$FUNCTION circle_cb_create_builtin ++$SHORT_DOC circle_cb_create [func] ++Register a function that will create work when asked. ++ ++Arguments: ++ FUNC User-defined callback function that will invoke ++ circle_enqueue when called ++ ++If FUNC is omitted, no function will be associated with work creation. ++This can be used to nullify a previous circle_cb_create invocation. ++ ++Exit Status: ++Returns 0 unless an invalid function is given or an error occurs. ++$END ++ ++/* Here is the circle_cb_create builtin. */ ++int ++circle_cb_create_builtin (list) ++ WORD_LIST *list; ++{ ++ return find_callback_function (list, &create_func); ++} ++ ++$BUILTIN circle_cb_process ++$FUNCTION circle_cb_process_builtin ++$SHORT_DOC circle_cb_process [func] ++Register a function that will process work when asked. ++ ++Arguments: ++ FUNC User-defined callback function that will invoke ++ circle_enqueue when called ++ ++If FUNC is omitted, no function will be associated with work processing. ++This can be used to nullify a previous circle_cb_process invocation. ++ ++Exit Status: ++Returns 0 unless an invalid function is given or an error occurs. ++$END ++ ++/* Here is the circle_cb_process builtin. */ ++int ++circle_cb_process_builtin (list) ++ WORD_LIST *list; ++{ ++ return find_callback_function (list, &process_func); ++} ++ ++$BUILTIN circle_begin ++$FUNCTION circle_begin_builtin ++$SHORT_DOC circle_begin ++Begin creation and processing of the distributed work queue. ++ ++Exit Status: ++Returns 0 unless an error occurs. ++$END ++ ++/* Here is the circle_begin builtin. */ ++int ++circle_begin_builtin (list) ++ WORD_LIST *list; ++{ ++ no_args (list); ++ CIRCLE_begin (); ++ return EXECUTION_SUCCESS; ++} ++ ++$BUILTIN circle_enqueue ++$FUNCTION circle_enqueue_builtin ++$SHORT_DOC circle_enqueue work ++Enqueue work onto the distributed queue. ++ ++Arguments: ++ WORK "Work" as represented by an arbitrary string of limited ++ size (generally around 4KB) ++ ++Exit Status: ++Returns 0 unless an error occurs. ++$END ++ ++/* Here is the circle_enqueue builtin. */ ++int ++circle_enqueue_builtin (list) ++ WORD_LIST *list; ++{ ++ char *work; /* Work to perform */ ++ ++ /* Extract the work argument. */ ++ YES_ARGS (list); ++ work = list->word->word; ++ list = list->next; ++ no_args (list); ++ ++ /* Complain if we're not within a proper callback function. */ ++ if (current_handle == NULL) ++ { ++ builtin_error (_("not within a Libcircle \"create\" or \"process\" callback function")); ++ return EXECUTION_FAILURE; ++ } ++ ++ /* Enqueue the work. */ ++ if (current_handle->enqueue (work) == -1) ++ return EXECUTION_FAILURE; ++ return EXECUTION_SUCCESS; ++} ++ ++$BUILTIN circle_dequeue ++$FUNCTION circle_dequeue_builtin ++$SHORT_DOC circle_dequeue var ++Dequeue work from the distributed queue into a variable. ++ ++Arguments: ++ VAR Variable in which to receive previously enqueued "work" ++ ++Exit Status: ++Returns 0 unless an error occurs. ++$END ++ ++/* Here is the circle_dequeue builtin. */ ++int ++circle_dequeue_builtin (list) ++ WORD_LIST *list; ++{ ++ char *varname; /* Variable in which to store the work string */ ++ char work[CIRCLE_MAX_STRING_LEN+1]; /* Work to perform */ ++ ++ /* Extract the variable-name argument. */ ++ YES_ARGS (list); ++ varname = list->word->word; ++ list = list->next; ++ no_args (list); ++ ++ /* Complain if we're not within a callback function. */ ++ if (current_handle == NULL) ++ { ++ builtin_error (_("not within a Libcircle callback function")); ++ return EXECUTION_FAILURE; ++ } ++ ++ /* Dequeue the work and bind it to the given variable. */ ++ if (current_handle->dequeue (work) == -1) ++ return EXECUTION_FAILURE; ++ bind_variable (varname, work, 0); ++ return EXECUTION_SUCCESS; ++} ++ ++$BUILTIN circle_enable_logging ++$FUNCTION circle_enable_logging_builtin ++$SHORT_DOC circle_enable_logging log_level ++Change Libcircle's logging verbosity ++ ++Arguments: ++ LOG_LEVEL "fatal", "error", "warning", "info", or "debug" ++ ++Exit Status: ++Returns 0 unless an invalid option is given. ++$END ++/*'*/ ++ ++/* Here is the circle_enable_logging builtin. */ ++int ++circle_enable_logging_builtin (list) ++ WORD_LIST *list; ++{ ++ char *word; /* One argument */ ++ CIRCLE_loglevel loglevel; /* Level to set */ ++ ++ /* Parse the log level. */ ++ YES_ARGS (list); ++ word = list->word->word; ++ if (!strcmp (word, "fatal")) ++ loglevel = CIRCLE_LOG_FATAL; ++ else if (!strcmp (word, "error")) ++ loglevel = CIRCLE_LOG_ERR; ++ else if (!strcmp (word, "warning")) ++ loglevel = CIRCLE_LOG_WARN; ++ else if (!strcmp (word, "info")) ++ loglevel = CIRCLE_LOG_INFO; ++ else if (!strcmp (word, "debug")) ++ loglevel = CIRCLE_LOG_DBG; ++ else ++ { ++ builtin_error (_("invalid log level \"%s\""), word); ++ return (EXECUTION_FAILURE); ++ } ++ ++ /* Set the log level. */ ++ CIRCLE_enable_logging (loglevel); ++ return EXECUTION_SUCCESS; ++} ++ ++$BUILTIN circle_abort ++$FUNCTION circle_abort_builtin ++$SHORT_DOC circle_abort ++Terminate queue processing. ++ ++Exit Status: ++Returns 0 unless an error occurs. ++$END ++ ++/* Here is the circle_abort builtin. */ ++int ++circle_abort_builtin (list) ++ WORD_LIST *list; ++{ ++ no_args (list); ++ CIRCLE_abort (); ++ return EXECUTION_SUCCESS; ++} ++ ++$BUILTIN circle_checkpoint ++$FUNCTION circle_checkpoint_builtin ++$SHORT_DOC circle_checkpoint ++Checkpoint a work queue to disk. ++ ++Write a file called circle${circle_rank}.txt containing the current ++queue state of rank ${circle_rank}. On a later run, a worker can ++invoke circle_read_restarts to repopulate its queue from such a ++checkpoint file. ++ ++Exit Status: ++Returns 0 unless an error occurs. ++$END ++/*'*/ ++ ++/* Here is the circle_checkpoint builtin. */ ++int ++circle_checkpoint_builtin (list) ++ WORD_LIST *list; ++{ ++ no_args (list); ++ CIRCLE_checkpoint (); ++ return EXECUTION_SUCCESS; ++} ++ ++$BUILTIN circle_read_restarts ++$FUNCTION circle_read_restarts_builtin ++$SHORT_DOC circle_read_restarts ++Repopulate a work queue from a disk checkpoint. ++ ++Read queue contents from a file called circle${circle_rank}.txt, which ++was previously produced by circle_checkpoint. ++ ++Exit Status: ++Returns 0 unless an error occurs. ++$END ++/*'*/ ++ ++/* Here is the circle_read_restarts builtin. */ ++int ++circle_read_restarts_builtin (list) ++ WORD_LIST *list; ++{ ++ no_args (list); ++ CIRCLE_read_restarts (); ++ return EXECUTION_SUCCESS; ++} ++ ++$BUILTIN circle_cb_reduce_init ++$FUNCTION circle_cb_reduce_init_builtin ++$SHORT_DOC circle_cb_reduce_init [func] ++Register a function that will initiate a reduction operation. ++ ++Arguments: ++ FUNC User-defined callback function that will invoke ++ circle_reduce when called ++ ++FUNC will be invoked on all ranks. ++ ++If FUNC is omitted, no function will be associated with reduction ++initialization. This can be used to nullify a previous ++circle_cb_reduce_init invocation. ++ ++Exit Status: ++Returns 0 unless an invalid function is given or an error occurs. ++$END ++ ++/* Here is the circle_cb_reduce_init builtin. */ ++int ++circle_cb_reduce_init_builtin (list) ++ WORD_LIST *list; ++{ ++ return find_callback_function (list, &reduce_init_func); ++} ++ ++$BUILTIN circle_cb_reduce_op ++$FUNCTION circle_cb_reduce_op_builtin ++$SHORT_DOC circle_cb_reduce_op [func] ++Register a function that will complete a reduction operation. ++ ++Arguments: ++ FUNC User-defined callback function that will receive ++ two items to reduce and invoke circle_reduce on ++ the reduced value ++ ++If FUNC is omitted, no function will be associated with reduction ++execution. This can be used to nullify a previous circle_cb_reduce_op ++invocation. ++ ++Exit Status: ++Returns 0 unless an invalid function is given or an error occurs. ++$END ++ ++/* Here is the circle_cb_reduce_op builtin. */ ++int ++circle_cb_reduce_op_builtin (list) ++ WORD_LIST *list; ++{ ++ return find_callback_function (list, &reduce_op_func); ++} ++ ++$BUILTIN circle_cb_reduce_fini ++$FUNCTION circle_cb_reduce_fini_builtin ++$SHORT_DOC circle_cb_reduce_fini [func] ++Register a function that will complete a reduction operation. ++ ++Arguments: ++ FUNC User-defined callback function that will receive ++ the final reduced data ++ ++If FUNC is omitted, no function will be associated with reduction ++completion. This can be used to nullify a previous ++circle_cb_reduce_fini invocation. ++ ++Libcircle guarantees that FUNC will be invoked only on rank 0. ++ ++Exit Status: ++Returns 0 unless an invalid function is given or an error occurs. ++$END ++ ++/* Here is the circle_cb_reduce_fini builtin. */ ++int ++circle_cb_reduce_fini_builtin (list) ++ WORD_LIST *list; ++{ ++ return find_callback_function (list, &reduce_fini_func); ++} ++ ++$BUILTIN circle_reduce ++$FUNCTION circle_reduce_builtin ++$SHORT_DOC circle_reduce work ++Seed the next phase of a reduction operation ++ ++Arguments: ++ WORK "Work" as represented by an arbitrary string of limited ++ size (generally around 4KB) ++ ++This function should be called both by the callback function ++registered with circle_reduce_init and the callback function ++registered with circle_reduce_op. ++ ++Exit Status: ++Returns 0 unless an error occurs. ++$END ++ ++/* Here is the circle_reduce builtin. */ ++int ++circle_reduce_builtin (list) ++ WORD_LIST *list; ++{ ++ char *work; /* Work to perform */ ++ ++ /* Extract the work argument. */ ++ YES_ARGS (list); ++ work = list->word->word; ++ list = list->next; ++ no_args (list); ++ ++ /* Complain if we're not within a proper callback function. */ ++ if (!within_reduction) ++ { ++ builtin_error (_("not within a Libcircle \"reduce_init\" or \"reduce_op\" callback function")); ++ return EXECUTION_FAILURE; ++ } ++ ++ /* Reduce the work. */ ++ CIRCLE_reduce (work, strlen (work)); ++ return EXECUTION_SUCCESS; ++} +diff -Naur bash-4.3/builtins/Makefile.in mpibash-4.3/builtins/Makefile.in +--- bash-4.3/builtins/Makefile.in 2012-05-25 07:29:19.000000000 -0600 ++++ mpibash-4.3/builtins/Makefile.in 2014-05-13 11:27:37.314100671 -0600 +@@ -141,7 +141,9 @@ + $(srcdir)/times.def $(srcdir)/trap.def $(srcdir)/type.def \ + $(srcdir)/ulimit.def $(srcdir)/umask.def $(srcdir)/wait.def \ + $(srcdir)/reserved.def $(srcdir)/pushd.def $(srcdir)/shopt.def \ +- $(srcdir)/printf.def $(srcdir)/complete.def $(srcdir)/mapfile.def ++ $(srcdir)/printf.def $(srcdir)/complete.def $(srcdir)/mapfile.def \ ++ $(srcdir)/mpi.def \ ++@CIRCLE@ $(srcdir)/circle.def + + STATIC_SOURCE = common.c evalstring.c evalfile.c getopt.c bashgetopt.c \ + getopt.h +@@ -153,7 +155,9 @@ + jobs.o kill.o let.o mapfile.o \ + pushd.o read.o return.o set.o setattr.o shift.o source.o \ + suspend.o test.o times.o trap.o type.o ulimit.o umask.o \ +- wait.o getopts.o shopt.o printf.o getopt.o bashgetopt.o complete.o ++ wait.o getopts.o shopt.o printf.o getopt.o bashgetopt.o complete.o \ ++ mpi.o \ ++@CIRCLE@ circle.o + + CREATED_FILES = builtext.h builtins.c psize.aux pipesize.h tmpbuiltins.c \ + tmpbuiltins.h +@@ -317,6 +321,8 @@ + getopts.o: getopts.def + reserved.o: reserved.def + complete.o: complete.def ++@CIRCLE@ circle.o: circle.def ++mpi.o: mpi.def + + # C files + bashgetopt.o: ../config.h $(topdir)/bashansi.h $(BASHINCDIR)/ansi_stdlib.h +@@ -644,6 +650,19 @@ + mapfile.o: $(topdir)/subst.h $(topdir)/externs.h $(BASHINCDIR)/maxpath.h + mapfile.o: $(topdir)/shell.h $(topdir)/syntax.h $(topdir)/variables.h $(topdir)/conftypes.h + mapfile.o: $(topdir)/arrayfunc.h ../pathnames.h ++@CIRCLE@ circle.o: $(topdir)/command.h ../config.h $(BASHINCDIR)/memalloc.h $(topdir)/error.h ++@CIRCLE@ circle.o: $(topdir)/general.h $(topdir)/xmalloc.h $(topdir)/subst.h $(topdir)/externs.h ++@CIRCLE@ circle.o: $(topdir)/quit.h $(topdir)/dispose_cmd.h $(topdir)/make_cmd.h ++@CIRCLE@ circle.o: $(topdir)/shell.h $(topdir)/syntax.h $(topdir)/unwind_prot.h $(topdir)/variables.h $(topdir)/conftypes.h ++@CIRCLE@ circle.o: $(BASHINCDIR)/maxpath.h ../pathnames.h ++mpi.o: ../config.h ../config-top.h ../config-bot.h ../bashintl.h ++mpi.o: ../include/gettext.h ../shell.h ../config.h ../bashjmp.h ++mpi.o: ../include/posixjmp.h ../command.h ../syntax.h ../general.h ++mpi.o: ../bashtypes.h ../include/chartypes.h ../xmalloc.h ../bashansi.h ++mpi.o: ../error.h ../variables.h ../array.h ../assoc.h ../hashlib.h ++mpi.o: ../conftypes.h ../arrayfunc.h ../quit.h ../sig.h ../include/maxpath.h ++mpi.o: ../unwind_prot.h ../dispose_cmd.h ../make_cmd.h ../include/ocache.h ++mpi.o: ../subst.h ../pathnames.h ../externs.h common.h bashgetopt.h + + #bind.o: $(RL_LIBSRC)chardefs.h $(RL_LIBSRC)readline.h $(RL_LIBSRC)keymaps.h + +diff -Naur bash-4.3/builtins/mpi.def mpibash-4.3/builtins/mpi.def +--- bash-4.3/builtins/mpi.def 1969-12-31 17:00:00.000000000 -0700 ++++ mpibash-4.3/builtins/mpi.def 2014-05-13 11:27:37.314100671 -0600 +@@ -0,0 +1,744 @@ ++This file is mpi.def, from which is created mpi.c. ++It implements all of the "mpi_*" builtins in Bash. ++ ++$PRODUCES mpi.c ++ ++#include ++ ++#include ++#if defined (HAVE_UNISTD_H) ++# ifdef _MINIX ++# include ++# endif ++# include ++#endif ++ ++#include "../bashintl.h" ++#include "../shell.h" ++#include "common.h" ++#include "bashgetopt.h" ++#include ++ ++extern int running_trap, trap_saved_exit_value; ++ ++/* Keep track of who we are within MPI_COMM_WORLD. */ ++static int mpi_rank; ++static int mpi_num_ranks; ++ ++/* Try an MPI operation. Return with an error message on failure. */ ++#define MPI_TRY(STMT) \ ++ do \ ++ { \ ++ int mpierr; \ ++ mpierr = STMT; \ ++ if (mpierr != MPI_SUCCESS) \ ++ return report_mpi_error (mpierr); \ ++ } \ ++ while (0) ++ ++/* Return with a usage message if no arguments remain. */ ++#define YES_ARGS(LIST) \ ++ if ((LIST) == 0) \ ++ { \ ++ builtin_usage (); \ ++ return (EX_USAGE); \ ++ } ++ ++/* Return with an error message if a given variable is read-only or if ++ * we can't write to it for any other reason (e.g., it's defined as a ++ * function). */ ++#define REQUIRE_WRITABLE(NAME) \ ++ do \ ++ { \ ++ SHELL_VAR *bindvar = find_shell_variable (NAME); \ ++ if (bindvar) \ ++ { \ ++ if (readonly_p (bindvar)) \ ++ { \ ++ err_readonly (NAME); \ ++ return (EXECUTION_FAILURE); \ ++ } \ ++ if (unbind_variable (NAME) == -1) \ ++ { \ ++ builtin_error ("Failed to write to variable %s", NAME); \ ++ return (EXECUTION_FAILURE); \ ++ } \ ++ } \ ++ } \ ++ while (0) ++ ++/* Initialize MPI. */ ++void ++initialize_mpi (argc, argv) ++ int argc; ++ char **argv; ++{ ++ int init_done; ++ ++ MPI_Initialized (&init_done); ++ if (!init_done) ++ MPI_Init (&argc, &argv); ++ MPI_Errhandler_set (MPI_COMM_WORLD, MPI_ERRORS_RETURN); ++ MPI_Comm_rank (MPI_COMM_WORLD, &mpi_rank); ++ MPI_Comm_size (MPI_COMM_WORLD, &mpi_num_ranks); ++} ++ ++/* Finalize MPI. */ ++void ++finalize_mpi () ++{ ++ MPI_Finalize (); ++} ++ ++/* Parse an operation name into an MPI_Op. Return 1 on success, 0 on ++ * failure. */ ++static int ++parse_operation (char *name, MPI_Op *op) ++{ ++ /* Define a mapping from operator names to MPI_Op values. */ ++ typedef struct { ++ char *name; /* Operation name (e.g., "sum") */ ++ MPI_Op value; /* Operation value (e.g., MPI_SUM) */ ++ } opname2value_t; ++ static opname2value_t oplist[] = { ++ {"max", MPI_MAX}, ++ {"min", MPI_MIN}, ++ {"sum", MPI_SUM}, ++ {"prod", MPI_PROD}, ++ {"land", MPI_LAND}, ++ {"band", MPI_BAND}, ++ {"lor", MPI_LOR}, ++ {"bor", MPI_BOR}, ++ {"lxor", MPI_LXOR}, ++ {"bxor", MPI_BXOR}, ++ {"maxloc", MPI_MAXLOC}, ++ {"minloc", MPI_MINLOC} ++ }; ++ size_t i; ++ ++ for (i = 0; i < sizeof(oplist)/sizeof(opname2value_t); i++) ++ if (!strcmp(name, oplist[i].name)) ++ { ++ *op = oplist[i].value; ++ if (i > 0) ++ { ++ /* As a performance optimization, bubble up the value we ++ * just found. */ ++ opname2value_t prev = oplist[i - 1]; ++ oplist[i - 1] = oplist[i]; ++ oplist[i] = prev; ++ } ++ return 1; ++ } ++ return 0; ++} ++ ++/* Report an error to the user and return EXECUTION_FAILURE. */ ++static int ++report_mpi_error (mpierr) ++ int mpierr; ++{ ++ char errstr[MPI_MAX_ERROR_STRING]; ++ int errstrlen; ++ ++ MPI_Error_string (mpierr, errstr, &errstrlen); ++ builtin_error ("%s", errstr); ++ return EXECUTION_FAILURE; ++} ++ ++/* Perform the same operation as bind_variable, but with VALUE being a ++ * number, not a string. */ ++static SHELL_VAR * ++bind_variable_number (name, value, flags) ++ const char *name; ++ long value; ++ int flags; ++{ ++ char numstr[25]; /* String version of VALUE */ ++ ++ sprintf (numstr, "%ld", value); ++ return bind_variable (name, numstr, flags); ++} ++ ++/* Perform the same operation as bind_array_variable, but with VALUE ++ * being a number, not a string. */ ++static SHELL_VAR * ++bind_array_variable_number (name, ind, value, flags) ++ char *name; ++ arrayind_t ind; ++ long value; ++ int flags; ++{ ++ char numstr[25]; /* String version of VALUE */ ++ ++ sprintf (numstr, "%ld", value); ++ return bind_array_variable (name, ind, numstr, flags); ++} ++ ++/* Define a reduction-type function (allreduce, scan, exscan, etc.). */ ++typedef int (*reduction_func_t)(void *, void *, int, MPI_Datatype, MPI_Op, MPI_Comm); ++ ++/* Perform any reduction-type operation (allreduce, scan, exscan, etc.). */ ++static int ++reduction_like (list, funcname, func) ++ WORD_LIST *list; ++ char *funcname; ++ reduction_func_t func; ++{ ++ char *word; /* One argument */ ++ struct { ++ long int value; /* Reduced value */ ++ int rank; /* Rank associated with the above */ ++ } number, result; ++ MPI_Op operation = MPI_SUM; /* Operation to perform */ ++ char *varname; /* Name of the variable to bind the results to */ ++ intmax_t n; ++ int i; ++ ++ /* Parse "-O OPERATION" (optional), where OPERATION is a reduction ++ * operation. */ ++ YES_ARGS (list); ++ word = list->word->word; ++ if (ISOPTION (word, 'O')) ++ { ++ list = list->next; ++ if (list == 0) ++ { ++ sh_needarg (funcname); ++ return (EX_USAGE); ++ } ++ word = list->word->word; ++ if (!parse_operation (word, &operation)) ++ { ++ sh_invalidopt ("-O"); ++ return (EX_USAGE); ++ } ++ list = list->next; ++ } ++ ++ /* Parse the argument, which must be a number. */ ++ YES_ARGS (list); ++ word = list->word->word; ++ if (!legal_number (word, &n)) ++ { ++ sh_neednumarg (funcname); ++ return (EX_USAGE); ++ } ++ number.value = (long int) n; ++ number.rank = mpi_rank; ++ list = list->next; ++ ++ /* Parse the target variable, which must not be read-only. */ ++ YES_ARGS (list); ++ varname = list->word->word; ++ if (mpi_rank != 0 || func != MPI_Exscan) ++ REQUIRE_WRITABLE (varname); ++ list = list->next; ++ no_args (list); ++ ++ /* Perform the reduction operation. Bind the given array variable ++ * to the result and, for minloc/maxloc, the associated rank. */ ++ if (mpi_rank != 0 || func != MPI_Exscan) { ++ bind_array_variable (varname, 0, "", 0); ++ bind_array_variable (varname, 1, "", 0); ++ } ++ if (operation == MPI_MINLOC || operation == MPI_MAXLOC) ++ { ++ MPI_TRY (func (&number, &result, 1, MPI_LONG_INT, operation, MPI_COMM_WORLD)); ++ if (mpi_rank != 0 || func != MPI_Exscan) ++ bind_array_variable_number (varname, 1, result.rank, 0); ++ } ++ else ++ MPI_TRY (func (&number.value, &result.value, 1, MPI_LONG, operation, MPI_COMM_WORLD)); ++ if (mpi_rank != 0 || func != MPI_Exscan) ++ bind_array_variable_number (varname, 0, result.value, 0); ++ return EXECUTION_SUCCESS; ++} ++ ++$BUILTIN mpi_comm_rank ++$FUNCTION mpi_comm_rank_builtin ++$SHORT_DOC mpi_comm_rank name ++Return the process's rank in the MPI job. ++ ++Arguments: ++ NAME Scalar variable in which to receive the rank ++ ++Exit Status: ++Returns 0 unless an invalid option is given. ++$END ++/*'*/ ++ ++/* Here is the mpi_comm_rank builtin. */ ++int ++mpi_comm_rank_builtin (list) ++ WORD_LIST *list; ++{ ++ char *varname; /* Name of the variable to bind the results to */ ++ ++ YES_ARGS (list); ++ varname = list->word->word; ++ REQUIRE_WRITABLE (varname); ++ list = list->next; ++ no_args (list); ++ bind_variable_number (varname, mpi_rank, 0); ++ return EXECUTION_SUCCESS; ++} ++ ++$BUILTIN mpi_comm_size ++$FUNCTION mpi_comm_size_builtin ++$SHORT_DOC mpi_comm_size name ++Return the total number of ranks in the MPI job. ++ ++Arguments: ++ NAME Scalar variable in which to receive the number of ranks ++ ++Exit Status: ++Returns 0 unless an invalid option is given. ++$END ++ ++/* Here is the mpi_comm_size builtin. */ ++int ++mpi_comm_size_builtin (list) ++ WORD_LIST *list; ++{ ++ char *varname; /* Name of the variable to bind the results to */ ++ ++ YES_ARGS (list); ++ varname = list->word->word; ++ REQUIRE_WRITABLE (varname); ++ list = list->next; ++ no_args (list); ++ bind_variable_number (varname, mpi_num_ranks, 0); ++ return EXECUTION_SUCCESS; ++} ++ ++$BUILTIN mpi_abort ++$FUNCTION mpi_abort_builtin ++$SHORT_DOC mpi_abort [n] ++Abort all processes in the MPI job and exit the shell. ++ ++Exits not only the caller's shell (with a status of N) but also all ++remote shells that are part of the same MPI job. If N is omitted, the ++exit status is that of the last command executed. ++ ++This command should be used only in extreme circumstances. It is ++better for each process to exit normally on its own. ++$END ++/*'*/ ++ ++/* Here is the mpi_abort builtin. */ ++int ++mpi_abort_builtin (list) ++ WORD_LIST *list; ++{ ++ int exit_value; ++ ++ exit_value = (running_trap == 1 && list == 0) ? trap_saved_exit_value : get_exitstat (list); /* Copied from exit.def */ ++ MPI_TRY (MPI_Abort (MPI_COMM_WORLD, exit_value)); ++ return EXECUTION_FAILURE; ++} ++ ++$BUILTIN mpi_send ++$FUNCTION mpi_send_builtin ++$SHORT_DOC mpi_send [-t tag] rank message ++Send a message to a remote process in the same MPI job. ++ ++Options: ++ -t TAG Send the message using tag TAG (default: 0). TAG must ++ be a nonnegative integer. ++ ++Arguments: ++ RANK Whom to send the message to. RANK must be an integer in ++ the range [0, $(mpi_comm_size)-1]. ++ ++ MESSAGE String to send to rank RANK. ++ ++Exit Status: ++Returns 0 unless an invalid option is given or an error occurs. ++$END ++ ++/* Here is the mpi_send builtin. */ ++int ++mpi_send_builtin (list) ++ WORD_LIST *list; ++{ ++ char *word; /* One argument */ ++ intmax_t target_rank; /* MPI target rank */ ++ char *message; /* Message to send to rank target_rank */ ++ intmax_t tag = 0; /* Message tag to use */ ++ ++ /* Parse "-t TAG" (optional), where TAG is a number or "any". */ ++ YES_ARGS (list); ++ word = list->word->word; ++ if (ISOPTION (word, 't')) ++ { ++ list = list->next; ++ if (list == 0) ++ { ++ sh_needarg ("mpi_recv"); ++ return (EX_USAGE); ++ } ++ word = list->word->word; ++ if (!legal_number (word, &tag)) ++ { ++ sh_neednumarg ("-t"); ++ return (EX_USAGE); ++ } ++ list = list->next; ++ } ++ else if (*word == '-') ++ { ++ sh_invalidopt (word); ++ builtin_usage (); ++ return (EX_USAGE); ++ } ++ ++ /* Parse the target rank, which must be a number. */ ++ YES_ARGS (list); ++ word = list->word->word; ++ if (!legal_number (word, &target_rank)) ++ { ++ builtin_error (_("mpi_send: numeric rank required")); ++ return (EX_USAGE); ++ } ++ list = list->next; ++ ++ /* Parse the message to send. */ ++ YES_ARGS (list); ++ message = list->word->word; ++ list = list->next; ++ no_args (list); ++ ++ /* Send the message. */ ++ MPI_TRY (MPI_Send (message, strlen(message)+1, MPI_BYTE, (int)target_rank, (int)tag, MPI_COMM_WORLD)); ++ return EXECUTION_SUCCESS; ++} ++ ++ ++$BUILTIN mpi_recv ++$FUNCTION mpi_recv_builtin ++$SHORT_DOC mpi_recv [-t tag] rank name ++Receive a message from a remote process in the same MPI job. ++ ++Options: ++ -t TAG Receive only messages sent using tag TAG (default: 0). ++ TAG must be either a nonnegative integer or the string ++ "any" to receive messages sent using any tag. ++ ++Arguments: ++ RANK Receive only messages sent from sender RANK. RANK ++ must either be in the range [0, $(mpi_comm_size)-1] or ++ be the string "any" to receive messages from any sender. ++ ++ NAME Array variable in which to receive the message, sender ++ rank, and tag. ++ ++Exit Status: ++Returns 0 unless an invalid option is given or an error occurs. ++$END ++ ++/* Here is the mpi_recv builtin. */ ++int ++mpi_recv_builtin (list) ++ WORD_LIST *list; ++{ ++ char *word; /* One argument */ ++ intmax_t source_rank; /* MPI source rank */ ++ char *endptr; /* Used for parsing strings into numbers */ ++ MPI_Status status; /* Status of an MPI operation */ ++ int count; /* Message length in bytes */ ++ intmax_t tag = 0; /* Message tag to use */ ++ char *varname; /* Name of the variable to bind the results to */ ++ static char *message = NULL; /* Message received from MPI */ ++ static size_t alloced = 0; /* Number of bytes allocated for the above */ ++ int opt; /* Parsed option */ ++ ++ /* Parse any options provided. */ ++ reset_internal_getopt (); ++ while ((opt = internal_getopt (list, "t:")) != -1) ++ { ++ switch (opt) ++ { ++ case 't': ++ if (!strcmp (list_optarg, "any")) ++ tag = MPI_ANY_TAG; ++ else if (!legal_number (list_optarg, &tag)) ++ { ++ builtin_error (_("-t: numeric argument or \"any\" required")); ++ return (EX_USAGE); ++ } ++ break; ++ ++ default: ++ sh_invalidopt (word); ++ builtin_usage (); ++ return (EX_USAGE); ++ } ++ } ++ list = loptend; ++ ++ /* Parse the source rank, which must be a number or "any". */ ++ YES_ARGS (list); ++ word = list->word->word; ++ if (!legal_number (word, &source_rank)) ++ { ++ if (!strcmp (word, "any")) ++ source_rank = MPI_ANY_SOURCE; ++ else ++ { ++ builtin_error (_("mpi_recv: numeric rank or \"any\" required")); ++ return (EX_USAGE); ++ } ++ } ++ list = list->next; ++ ++ /* Parse the target variable, which must not be read-only. */ ++ YES_ARGS (list); ++ varname = list->word->word; ++ REQUIRE_WRITABLE (varname); ++ list = list->next; ++ no_args (list); ++ ++ /* Receive a message. Because we don't know long the message will ++ * be, we first probe to get the length. */ ++ MPI_TRY (MPI_Probe ((int)source_rank, (int)tag, MPI_COMM_WORLD, &status)); ++ MPI_TRY (MPI_Get_count (&status, MPI_BYTE, &count)); ++ if (alloced < count) ++ { ++ message = xrealloc (message, count); ++ alloced = count; ++ } ++ MPI_TRY (MPI_Recv (message, count, MPI_BYTE, status.MPI_SOURCE, status.MPI_TAG, MPI_COMM_WORLD, &status)); ++ bind_array_variable (varname, 0, message, 0); ++ bind_array_variable_number (varname, 1, status.MPI_SOURCE, 0); ++ bind_array_variable_number (varname, 2, status.MPI_TAG, 0); ++ return EXECUTION_SUCCESS; ++} ++ ++$BUILTIN mpi_barrier ++$FUNCTION mpi_barrier_builtin ++$SHORT_DOC mpi_barrier ++Synchronizes all of the processes in the MPI job. ++ ++No process will return from mpi_barrier until all processes have ++called mpi_barrier. ++ ++Exit Status: ++Returns 0 unless an invalid option is given or an error occurs. ++$END ++ ++/* Here is the mpi_barrier builtin. */ ++int ++mpi_barrier_builtin (list) ++ WORD_LIST *list; ++{ ++ no_args (list); ++ MPI_TRY (MPI_Barrier (MPI_COMM_WORLD)); ++ return EXECUTION_SUCCESS; ++} ++ ++$BUILTIN mpi_bcast ++$FUNCTION mpi_bcast_builtin ++$SHORT_DOC mpi_bcast [message] name ++Broadcast a message to all processes in the same MPI job. ++ ++Arguments: ++ MESSAGE String to broadcast from one process to all the others. ++ ++ NAME Scalar variable in which to receive the broadcast message. ++ ++Exactly one process in the MPI job must specify a message to ++broadcast. No process will return from mpi_bcast until all processes ++have called mpi_bcast. ++ ++Exit Status: ++Returns 0 unless an invalid option is given or an error occurs. ++$END ++ ++/* Here is the mpi_bcast builtin. */ ++int ++mpi_bcast_builtin (list) ++ WORD_LIST *list; ++{ ++ char *word; /* One argument */ ++ int root; /* MPI root rank */ ++ char *root_message; /* Message to broadcast */ ++ int msglen; /* Length in bytes of the above (including the NULL byte) */ ++ char *varname; /* Name of the variable to bind the results to */ ++ static int *all_lengths = NULL; /* List of every rank's msglen */ ++ static char *message = NULL; /* Message received from the root */ ++ static int alloced = 0; /* Bytes allocated for the above */ ++ int i; ++ ++ /* Parse the optional message and target variable, which must not be ++ * read-only. */ ++ YES_ARGS (list); ++ if (list->next == NULL) ++ { ++ /* Non-root */ ++ root_message = NULL; ++ msglen = -1; ++ } ++ else ++ { ++ /* Root */ ++ root_message = list->word->word; ++ msglen = (int) strlen(root_message) + 1; ++ list = list->next; ++ } ++ varname = list->word->word; ++ REQUIRE_WRITABLE (varname); ++ list = list->next; ++ no_args (list); ++ ++ /* Acquire global agreement on the root and the message size. */ ++ if (all_lengths == NULL) ++ all_lengths = xmalloc (mpi_num_ranks*sizeof(int)); ++ MPI_TRY (MPI_Allgather (&msglen, 1, MPI_INT, all_lengths, 1, MPI_INT, MPI_COMM_WORLD)); ++ root = -1; ++ for (i = 0; i < mpi_num_ranks; i++) ++ { ++ if (all_lengths[i] == -1) ++ continue; ++ if (root != -1) ++ { ++ builtin_error (_("mpi_bcast: more than one process specified a message")); ++ return (EXECUTION_FAILURE); ++ } ++ root = i; ++ msglen = all_lengths[i]; ++ } ++ if (root == -1) ++ { ++ builtin_error (_("mpi_bcast: no process specified a message")); ++ return (EXECUTION_FAILURE); ++ } ++ ++ /* Broadcast the message. */ ++ if (mpi_rank == root) ++ { ++ MPI_TRY (MPI_Bcast (root_message, msglen, MPI_BYTE, root, MPI_COMM_WORLD)); ++ bind_variable (varname, root_message, 0); ++ } ++ else ++ { ++ if (alloced < msglen) ++ { ++ message = xrealloc (message, msglen); ++ alloced = msglen; ++ } ++ MPI_TRY (MPI_Bcast (message, msglen, MPI_BYTE, root, MPI_COMM_WORLD)); ++ bind_variable (varname, message, 0); ++ } ++ return EXECUTION_SUCCESS; ++} ++ ++$BUILTIN mpi_scan ++$FUNCTION mpi_scan_builtin ++$SHORT_DOC mpi_scan number name ++Perform an inclusive scan across all processes in the same MPI job. ++ ++ -O OPERATION Operation to perform. Must be one of "max", "min", ++ "sum", "prod", "land", "band", "lor", "bor", "lxor", ++ "bxor", "maxloc", or "minloc" (default: "sum"). ++ ++Arguments: ++ NUMBER Integer to use in the scan operation. ++ ++ NAME Array variable in which to receive the result and, in ++ the case of maxloc and minloc, the associated rank. ++ ++In an inclusive-scan operation, each process i presents a number, ++a[i]. Once all processes in the MPI job have presented their number, ++the command returns a[0] to rank 0, a[0]+a[1] to rank 1, ++a[0]+a[1]+a[2] to rank 2, and so forth. The -O option enables "+" to ++be replaced with other operations. ++ ++Inclusive scans can be useful for assigning a unique index to each ++process in the MPI job. ++ ++Exit Status: ++Returns 0 unless an invalid option is given or an error occurs. ++$END ++ ++/* Here is the mpi_scan builtin. */ ++int ++mpi_scan_builtin (list) ++ WORD_LIST *list; ++{ ++ return reduction_like (list, "mpi_scan", MPI_Scan); ++} ++ ++$BUILTIN mpi_exscan ++$FUNCTION mpi_exscan_builtin ++$SHORT_DOC mpi_exscan number name ++Perform an exclusive scan across all processes in the same MPI job. ++ ++ -O OPERATION Operation to perform. Must be one of "max", "min", ++ "sum", "prod", "land", "band", "lor", "bor", "lxor", ++ "bxor", "maxloc", or "minloc" (default: "sum"). ++ ++Arguments: ++ NUMBER Integer to use in the scan operation. ++ ++ NAME Array variable in which to receive the result and, in ++ the case of maxloc and minloc, the associated rank. ++ ++In a exclusive-scan operation, each process i presents a number, a[i]. ++Once all processes in the MPI job have presented their number, the ++command assigns a[0] to NAME on rank 1, a[0]+a[1] to NAME on rank 2, ++a[0]+a[1]+a[2] to NAME on rank 3, and so forth. No assignment is ++performed on rank 0. The -O option enables "+" to be replaced with ++other operations. ++ ++Exclusive scans can be useful for assigning a unique index to each ++process in the MPI job. ++ ++Exit Status: ++Returns 0 unless an invalid option is given or an error occurs. ++$END ++ ++/* Here is the mpi_exscan builtin. */ ++int ++mpi_exscan_builtin (list) ++ WORD_LIST *list; ++{ ++ return reduction_like (list, "mpi_exscan", MPI_Exscan); ++} ++ ++$BUILTIN mpi_allreduce ++$FUNCTION mpi_allreduce_builtin ++$SHORT_DOC mpi_allreduce number name ++Reduce numbers from all processes in an MPI job to a single number. ++ ++Options: ++ ++ -O OPERATION Operation to perform. Must be one of "max", "min", ++ "sum", "prod", "land", "band", "lor", "bor", "lxor", ++ "bxor", "maxloc", or "minloc" (default: "sum"). ++ ++Arguments: ++ NUMBER Integer to use in the allreduce operation. ++ ++ NAME Array variable in which to receive the result and, in ++ the case of maxloc and minloc, the associated rank. ++ ++In an all-reduce operation, each process i presents a number, a[i]. ++Once all processes in the MPI job have presented their number, the ++command returns a[0]+a[1]+...+a[n-1] to all ranks. The -O option ++enables "+" to be replaced with other operations. ++ ++All-reduces can be useful for reaching global agreement (e.g., of a ++termination condition). ++ ++Exit Status: ++Returns 0 unless an invalid option is given or an error occurs. ++$END ++ ++/* Here is the mpi_allreduce builtin. */ ++int ++mpi_allreduce_builtin (list) ++ WORD_LIST *list; ++{ ++ return reduction_like (list, "mpi_allreduce", MPI_Allreduce); ++} +diff -Naur bash-4.3/config.h.in mpibash-4.3/config.h.in +--- bash-4.3/config.h.in 2013-06-29 15:35:33.000000000 -0600 ++++ mpibash-4.3/config.h.in 2014-05-13 11:27:37.314100671 -0600 +@@ -1147,6 +1147,12 @@ + /* Define if you have the `__argz_stringify' function. */ + #undef HAVE___ARGZ_STRINGIFY + ++/* Define if you have both the header file and the libcircle library. */ ++#undef HAVE_LIBCIRCLE ++ ++/* Define if you have the `CIRCLE_cb_reduce_op' function. */ ++#undef HAVE_CIRCLE_CB_REDUCE_OP ++ + /* End additions for lib/intl */ + + #include "config-bot.h" +diff -Naur bash-4.3/configure.ac mpibash-4.3/configure.ac +--- bash-4.3/configure.ac 2014-02-11 08:37:53.000000000 -0700 ++++ mpibash-4.3/configure.ac 2014-05-13 11:27:37.302100179 -0600 +@@ -24,7 +24,7 @@ + AC_REVISION([for Bash 4.3, version 4.063])dnl + + define(bashvers, 4.3) +-define(relstatus, release) ++define(relstatus, MPI) + + AC_INIT([bash], bashvers-relstatus, [bug-bash@gnu.org]) + +@@ -813,6 +813,21 @@ + fi + ]) + ++dnl Ensure that we can find an MPI library. ++AC_CHECK_FUNCS([MPI_Init], [], [ ++ AC_MSG_ERROR([Cannot continue without MPI. Consider specifying CC=mpicc.])]) ++ ++dnl If we have Libcircle, use it, too. ++AC_SEARCH_LIBS([CIRCLE_cb_create], [circle], [AC_CHECK_HEADERS([libcircle.h])]) ++if test "x$ac_cv_header_libcircle_h" = xyes; then ++ libcircle_make_prefix="" ++ AC_DEFINE([HAVE_LIBCIRCLE], [1], [Define if you have the Libcircle header and library.]) ++ AC_CHECK_FUNCS([CIRCLE_cb_reduce_op]) ++else ++ libcircle_make_prefix="#" ++fi ++AC_SUBST([CIRCLE], [$libcircle_make_prefix]) ++ + BASH_CHECK_DECL(strtoimax) + BASH_CHECK_DECL(strtol) + BASH_CHECK_DECL(strtoll) +diff -Naur bash-4.3/Makefile.in mpibash-4.3/Makefile.in +--- bash-4.3/Makefile.in 2014-01-25 14:27:30.000000000 -0700 ++++ mpibash-4.3/Makefile.in 2014-05-13 11:27:37.314100671 -0600 +@@ -104,7 +104,7 @@ + VERSPROG = bashversion$(EXEEXT) + VERSOBJ = bashversion.$(OBJEXT) + +-Program = bash$(EXEEXT) ++Program = mpibash$(EXEEXT) + Version = @BASHVERS@ + PatchLevel = `$(BUILD_DIR)/$(VERSPROG) -p` + RELSTATUS = @RELSTATUS@ +diff -Naur bash-4.3/shell.c mpibash-4.3/shell.c +--- bash-4.3/shell.c 2014-01-14 06:04:32.000000000 -0700 ++++ mpibash-4.3/shell.c 2014-05-13 11:27:37.314100671 -0600 +@@ -107,6 +107,13 @@ + extern char *primary_prompt, *secondary_prompt; + extern char *this_command_name; + ++extern void initialize_mpi __P((int, char **)); ++extern void finalize_mpi __P((void)); ++#ifdef HAVE_LIBCIRCLE ++extern void initialize_libcircle __P((int, char **)); ++extern void finalize_libcircle __P((void)); ++#endif ++ + /* Non-zero means that this shell has already been run; i.e. you should + call shell_reinitialize () if you need to start afresh. */ + int shell_initialized = 0; +@@ -324,7 +331,7 @@ + static void init_interactive_script __P((void)); + + static void set_shell_name __P((char *)); +-static void shell_initialize __P((void)); ++static void shell_initialize __P((int, char **)); + static void shell_reinitialize __P((void)); + + static void show_shell_usage __P((FILE *, int)); +@@ -561,7 +568,7 @@ + + /* From here on in, the shell must be a normal functioning shell. + Variables from the environment are expected to be set, etc. */ +- shell_initialize (); ++ shell_initialize (argc, argv); + + set_default_lang (); + set_default_locale_vars (); +@@ -941,6 +948,12 @@ + end_job_control (); + #endif /* JOB_CONTROL */ + ++#ifdef HAVE_LIBCIRCLE ++ finalize_libcircle (); ++#else ++ finalize_mpi (); ++#endif ++ + /* Always return the exit status of the last command to our parent. */ + sh_exit (s); + } +@@ -1691,7 +1704,9 @@ + /* Do whatever is necessary to initialize the shell. + Put new initializations in here. */ + static void +-shell_initialize () ++shell_initialize (argc, argv) ++ int argc; ++ char **argv; + { + char hostname[256]; + +@@ -1760,6 +1775,17 @@ + initialize_shell_options (privileged_mode||running_setuid); + initialize_bashopts (privileged_mode||running_setuid); + #endif ++ ++ /* Initialize Libcircle and MPI. */ ++#ifdef HAVE_LIBCIRCLE ++ initialize_libcircle (argc, argv); ++ initialize_mpi (argc, argv); ++ bind_variable ("libcircle", "yes", 0); ++#else ++ initialize_mpi (argc, argv); ++ bind_variable ("libcircle", "no", 0); ++#endif ++ bind_variable ("mpibash", "yes", 0); + } + + /* Function called by main () when it appears that the shell has already diff --git a/var/spack/repos/builtin/packages/mpibash/package.py b/var/spack/repos/builtin/packages/mpibash/package.py new file mode 100644 index 0000000000..d0f6dafed6 --- /dev/null +++ b/var/spack/repos/builtin/packages/mpibash/package.py @@ -0,0 +1,32 @@ +import os +from spack import * + +class Mpibash(Package): + """Parallel scripting right from the Bourne-Again Shell (Bash)""" + homepage = "http://www.ccs3.lanl.gov/~pakin/software/mpibash-4.3.html" + + version('4.3', '81348932d5da294953e15d4814c74dd1', + url="http://ftp.gnu.org/gnu/bash/bash-4.3.tar.gz") + + # patch -p1 < ../mpibash-4.3.patch + patch('mpibash-4.3.patch', level=1, when='@4.3') + + # above patch modifies configure.ac + depends_on('autoconf') + + # uses MPI_Exscan which is in MPI-1.2 and later + depends_on('mpi@1.2:') + + depends_on('libcircle') + + def install(self, spec, prefix): + # run autoconf to rebuild configure + autoconf = which('autoconf') + autoconf() + + configure("--prefix=" + prefix, + "CC=mpicc") + + make(parallel=False) + + make("install") diff --git a/var/spack/repos/builtin/packages/mpich/package.py b/var/spack/repos/builtin/packages/mpich/package.py new file mode 100644 index 0000000000..d48bf878f6 --- /dev/null +++ b/var/spack/repos/builtin/packages/mpich/package.py @@ -0,0 +1,92 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * +import os + +class Mpich(Package): + """MPICH is a high performance and widely portable implementation of + the Message Passing Interface (MPI) standard.""" + homepage = "http://www.mpich.org" + url = "http://www.mpich.org/static/downloads/3.0.4/mpich-3.0.4.tar.gz" + list_url = "http://www.mpich.org/static/downloads/" + list_depth = 2 + + version('3.1.4', '2ab544607986486562e076b83937bba2') + version('3.1.3', '93cb17f91ac758cbf9174ecb03563778') + version('3.1.2', '7fbf4b81dcb74b07ae85939d1ceee7f1') + version('3.1.1', '40dc408b1e03cc36d80209baaa2d32b7') + version('3.1', '5643dd176499bfb7d25079aaff25f2ec') + version('3.0.4', '9c5d5d4fe1e17dd12153f40bc5b6dbc0') + + provides('mpi@:3.0', when='@3:') + provides('mpi@:1.3', when='@1:') + + def setup_dependent_environment(self, module, spec, dep_spec): + """For dependencies, make mpicc's use spack wrapper.""" + os.environ['MPICH_CC'] = 'cc' + os.environ['MPICH_CXX'] = 'c++' + os.environ['MPICH_F77'] = 'f77' + os.environ['MPICH_F90'] = 'f90' + + + def install(self, spec, prefix): + config_args = ["--prefix=" + prefix, + "--enable-shared"] + + # TODO: Spack should make it so that you can't actually find + # these compilers if they're "disabled" for the current + # compiler configuration. + if not self.compiler.f77: + config_args.append("--disable-f77") + + if not self.compiler.fc: + config_args.append("--disable-fc") + + configure(*config_args) + make() + make("install") + + self.filter_compilers() + + + def filter_compilers(self): + """Run after install to make the MPI compilers use the + compilers that Spack built the package with. + + If this isn't done, they'll have CC, CXX, F77, and FC set + to Spack's generic cc, c++, f77, and f90. We want them to + be bound to whatever compiler they were built with. + """ + bin = self.prefix.bin + mpicc = os.path.join(bin, 'mpicc') + mpicxx = os.path.join(bin, 'mpicxx') + mpif77 = os.path.join(bin, 'mpif77') + mpif90 = os.path.join(bin, 'mpif90') + + kwargs = { 'ignore_absent' : True, 'backup' : False, 'string' : True } + filter_file('CC="cc"', 'CC="%s"' % self.compiler.cc, mpicc, **kwargs) + filter_file('CXX="c++"', 'CXX="%s"' % self.compiler.cxx, mpicxx, **kwargs) + filter_file('F77="f77"', 'F77="%s"' % self.compiler.f77, mpif77, **kwargs) + filter_file('FC="f90"', 'FC="%s"' % self.compiler.fc, mpif90, **kwargs) diff --git a/var/spack/repos/builtin/packages/mpileaks/package.py b/var/spack/repos/builtin/packages/mpileaks/package.py new file mode 100644 index 0000000000..4ef866588c --- /dev/null +++ b/var/spack/repos/builtin/packages/mpileaks/package.py @@ -0,0 +1,44 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + +class Mpileaks(Package): + """Tool to detect and report leaked MPI objects like MPI_Requests and MPI_Datatypes.""" + + homepage = "https://github.com/hpc/mpileaks" + url = "https://github.com/hpc/mpileaks/releases/download/v1.0/mpileaks-1.0.tar.gz" + + version('1.0', '8838c574b39202a57d7c2d68692718aa') + + depends_on("mpi") + depends_on("adept-utils") + depends_on("callpath") + + def install(self, spec, prefix): + configure("--prefix=" + prefix, + "--with-adept-utils=" + spec['adept-utils'].prefix, + "--with-callpath=" + spec['callpath'].prefix) + make() + make("install") diff --git a/var/spack/repos/builtin/packages/mrnet/package.py b/var/spack/repos/builtin/packages/mrnet/package.py new file mode 100644 index 0000000000..6e9766f275 --- /dev/null +++ b/var/spack/repos/builtin/packages/mrnet/package.py @@ -0,0 +1,20 @@ +from spack import * + +class Mrnet(Package): + """The MRNet Multi-Cast Reduction Network.""" + homepage = "http://paradyn.org/mrnet" + url = "ftp://ftp.cs.wisc.edu/paradyn/mrnet/mrnet_4.0.0.tar.gz" + + version('4.0.0', 'd00301c078cba57ef68613be32ceea2f') + version('4.1.0', '5a248298b395b329e2371bf25366115c') + + parallel = False + + depends_on("boost") + + def install(self, spec, prefix): + configure("--prefix=%s" %prefix, "--enable-shared") + + make() + make("install") + diff --git a/var/spack/repos/builtin/packages/munge/package.py b/var/spack/repos/builtin/packages/munge/package.py new file mode 100644 index 0000000000..c737ca0354 --- /dev/null +++ b/var/spack/repos/builtin/packages/munge/package.py @@ -0,0 +1,20 @@ +from spack import * +import os + +class Munge(Package): + """ MUNGE Uid 'N' Gid Emporium """ + homepage = "https://code.google.com/p/munge/" + url = "https://github.com/dun/munge/releases/download/munge-0.5.11/munge-0.5.11.tar.bz2" + + version('0.5.11', 'bd8fca8d5f4c1fcbef1816482d49ee01', url='https://github.com/dun/munge/releases/download/munge-0.5.11/munge-0.5.11.tar.bz2') + + depends_on('openssl') + depends_on('libgcrypt') + + def install(self, spec, prefix): + os.makedirs(os.path.join(prefix, "lib/systemd/system")) + configure("--prefix=%s" % prefix) + + make() + make("install") + diff --git a/var/spack/repos/builtin/packages/muster/package.py b/var/spack/repos/builtin/packages/muster/package.py new file mode 100644 index 0000000000..722daf3d7f --- /dev/null +++ b/var/spack/repos/builtin/packages/muster/package.py @@ -0,0 +1,22 @@ +from spack import * + +class Muster(Package): + """The Muster library provides implementations of sequential and + parallel K-Medoids clustering algorithms. It is intended as a + general framework for parallel cluster analysis, particularly + for performance data analysis on systems with very large + numbers of processes. + """ + homepage = "https://github.com/scalability-llnl/muster" + url = "https://github.com/scalability-llnl/muster/archive/v1.0.tar.gz" + + version('1.0.1', 'd709787db7e080447afb6571ac17723c') + version('1.0', '2eec6979a4a36d3a65a792d12969be16') + + depends_on("boost") + depends_on("mpi") + + def install(self, spec, prefix): + cmake(".", *std_cmake_args) + make() + make("install") diff --git a/var/spack/repos/builtin/packages/mvapich2/ad_lustre_rwcontig_open_source.patch b/var/spack/repos/builtin/packages/mvapich2/ad_lustre_rwcontig_open_source.patch new file mode 100644 index 0000000000..ff85845cf8 --- /dev/null +++ b/var/spack/repos/builtin/packages/mvapich2/ad_lustre_rwcontig_open_source.patch @@ -0,0 +1,11 @@ +--- a/src/mpi/romio/adio/ad_lustre/ad_lustre_rwcontig.c 2013-12-10 12:05:44.806417000 -0800 ++++ b/src/mpi/romio/adio/ad_lustre/ad_lustre_rwcontig.c 2013-12-10 11:53:03.295622000 -0800 +@@ -8,7 +8,7 @@ + * Copyright (C) 2008 Sun Microsystems, Lustre group + */ + +-#define _XOPEN_SOURCE 600 ++//#define _XOPEN_SOURCE 600 + #include + #include + #include "ad_lustre.h" diff --git a/var/spack/repos/builtin/packages/mvapich2/package.py b/var/spack/repos/builtin/packages/mvapich2/package.py new file mode 100644 index 0000000000..ca0b1287c1 --- /dev/null +++ b/var/spack/repos/builtin/packages/mvapich2/package.py @@ -0,0 +1,104 @@ +import os +from spack import * + +class Mvapich2(Package): + """mvapich2 is an MPI implmenetation for infiniband networks.""" + homepage = "http://mvapich.cse.ohio-state.edu/" + + version('1.9', '5dc58ed08fd3142c260b70fe297e127c', + url="http://mvapich.cse.ohio-state.edu/download/mvapich2/mv2/mvapich2-1.9.tgz") + patch('ad_lustre_rwcontig_open_source.patch', when='@1.9') + + version('2.0', '9fbb68a4111a8b6338e476dc657388b4', + url='http://mvapich.cse.ohio-state.edu/download/mvapich/mv2/mvapich2-2.0.tar.gz') + + provides('mpi@:2.2', when='@1.9') # MVAPICH2-1.9 supports MPI 2.2 + provides('mpi@:3.0', when='@2.0') # MVAPICH2-2.0 supports MPI 3.0 + + + def install(self, spec, prefix): + # we'll set different configure flags depending on our environment + configure_args = [] + + # TODO: The MPICH*_FLAGS have a different name for 1.9 + + if '+debug' in spec: + # set configure flags for debug build + configure_args.append("--disable-fast") + configure_args.append("--enable-g=dbg") + configure_args.append("--enable-error-checking=runtime") + configure_args.append("--enable-error-messages=all") + configure_args.append("--enable-nmpi-as-mpi") + + if "%gnu" in spec: + # set variables for GNU compilers + os.environ['MPICHLIB_CFLAGS'] = "-g -O0" + os.environ['MPICHLIB_CXXFLAGS'] = "-g -O0" + os.environ['MPICHLIB_FFLAGS'] = "-g -O0 -fno-second-underscore" + os.environ['MPICHLIB_F90FLAGS'] = "-g -O0 -fno-second-underscore" + elif "%intel" in spec: + # set variables for Inel compilers + os.environ['MPICHLIB_CFLAGS'] = "-g -O0" + os.environ['MPICHLIB_CXXFLAGS'] = "-g -O0" + os.environ['MPICHLIB_FFLAGS'] = "-g -O0" + os.environ['MPICHLIB_F90FLAGS'] = "-g -O0" + elif "%pgi" in spec: + # set variables for PGI compilers + os.environ['MPICHLIB_CFLAGS'] = "-g -O0 -fPIC" + os.environ['MPICHLIB_CXXFLAGS'] = "-g -O0 -fPIC" + os.environ['MPICHLIB_FFLAGS'] = "-g -O0 -fPIC" + os.environ['MPICHLIB_F90FLAGS'] = "-g -O0 -fPIC" + + else: + # set configure flags for normal optimizations + configure_args.append("--enable-fast=all") + configure_args.append("--enable-g=dbg") + configure_args.append("--enable-nmpi-as-mpi") + + if "%gnu" in spec: + # set variables for what compilers + os.environ['MPICHLIB_CFLAGS'] = "-g -O2" + os.environ['MPICHLIB_CXXFLAGS'] = "-g -O2" + os.environ['MPICHLIB_FFLAGS'] = "-g -O2 -fno-second-underscore" + os.environ['MPICHLIB_F90FLAGS'] = "-g -O2 -fno-second-underscore" + elif "%intel" in spec: + # set variables for Inel compilers + os.environ['MPICHLIB_CFLAGS'] = "-g -O2" + os.environ['MPICHLIB_CXXFLAGS'] = "-g -O2" + os.environ['MPICHLIB_FFLAGS'] = "-g -O2" + os.environ['MPICHLIB_F90FLAGS'] = "-g -O2" + elif "%pgi" in spec: + # set variables for PGI compilers + os.environ['MPICHLIB_CFLAGS'] = "-g -O2 -fPIC" + os.environ['MPICHLIB_CXXFLAGS'] = "-g -O2 -fPIC" + os.environ['MPICHLIB_FFLAGS'] = "-g -O2 -fPIC" + os.environ['MPICHLIB_F90FLAGS'] = "-g -O2 -fPIC" + + # determine network type by variant + if "+psm" in spec: + # throw this flag on QLogic systems to use PSM + configure_args.append("--with-device=ch3:psm") + else: + # throw this flag on IB systems + configure_args.append("--with-device=ch3:mrail", "--with-rdma=gen2") + + # TODO: shared-memory build + + # TODO: CUDA + + # TODO: other file systems like panasis + + configure( + "--prefix=" + prefix, + "--enable-f77", "--enable-fc", "--enable-cxx", + "--enable-shared", "--enable-sharedlibs=gcc", + "--enable-debuginfo", + "--with-pm=no", "--with-pmi=slurm", + "--enable-romio", "--with-file-system=lustre+nfs+ufs", + "--disable-mpe", "--without-mpe", + "--disable-silent-rules", + *configure_args) + + make() + + make("install") diff --git a/var/spack/repos/builtin/packages/nasm/package.py b/var/spack/repos/builtin/packages/nasm/package.py new file mode 100644 index 0000000000..933b6a62c5 --- /dev/null +++ b/var/spack/repos/builtin/packages/nasm/package.py @@ -0,0 +1,14 @@ +from spack import * + +class Nasm(Package): + """NASM (Netwide Assembler) is an 80x86 assembler designed for + portability and modularity. It includes a disassembler as well.""" + homepage = "http://www.nasm.us" + url = "http://www.nasm.us/pub/nasm/releasebuilds/2.11.06/nasm-2.11.06.tar.xz" + + version('2.11.06', '2b958e9f5d200641e6fc9564977aecc5') + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix) + make() + make("install") diff --git a/var/spack/repos/builtin/packages/ncdu/package.py b/var/spack/repos/builtin/packages/ncdu/package.py new file mode 100644 index 0000000000..234f9730d6 --- /dev/null +++ b/var/spack/repos/builtin/packages/ncdu/package.py @@ -0,0 +1,28 @@ +from spack import * + +class Ncdu(Package): + """ + Ncdu is a disk usage analyzer with an ncurses interface. It is designed + to find space hogs on a remote server where you don't have an entire + gaphical setup available, but it is a useful tool even on regular desktop + systems. Ncdu aims to be fast, simple and easy to use, and should be able + to run in any minimal POSIX-like environment with ncurses installed. + """ + + homepage = "http://dev.yorhel.nl/ncdu" + url = "http://dev.yorhel.nl/download/ncdu-1.11.tar.gz" + + version('1.11', '9e44240a5356b029f05f0e70a63c4d12') + version('1.10', '7535decc8d54eca811493e82d4bfab2d') + version('1.9' , '93258079db897d28bb8890e2db89b1fb') + version('1.8' , '94d7a821f8a0d7ba8ef3dd926226f7d5') + version('1.7' , '172047c29d232724cc62e773e82e592a') + + depends_on("ncurses") + + def install(self, spec, prefix): + configure('--prefix=%s' % prefix, + '--with-ncurses=%s' % spec['ncurses']) + + make() + make("install") diff --git a/var/spack/repos/builtin/packages/ncurses/package.py b/var/spack/repos/builtin/packages/ncurses/package.py new file mode 100644 index 0000000000..cc180bbae1 --- /dev/null +++ b/var/spack/repos/builtin/packages/ncurses/package.py @@ -0,0 +1,33 @@ +from spack import * + +class Ncurses(Package): + """The ncurses (new curses) library is a free software emulation of curses + in System V Release 4.0, and more. It uses terminfo format, supports pads and + color and multiple highlights and forms characters and function-key mapping, + and has all the other SYSV-curses enhancements over BSD curses. + """ + + homepage = "http://invisible-island.net/ncurses/ncurses.html" + + version('5.9', '8cb9c412e5f2d96bc6f459aa8c6282a1', + url='http://ftp.gnu.org/pub/gnu/ncurses/ncurses-5.9.tar.gz') + version('6.0', 'ee13d052e1ead260d7c28071f46eefb1', + url='http://ftp.gnu.org/pub/gnu/ncurses/ncurses-6.0.tar.gz') + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix, + "--with-shared", + "--enable-widec", + "--disable-pc-files", + "--without-ada") + make() + make("install") + + configure("--prefix=%s" % prefix, + "--with-shared", + "--disable-widec", + "--disable-pc-files", + "--without-ada") + make() + make("install") + diff --git a/var/spack/repos/builtin/packages/netcdf/netcdf-4.3.3-mpi.patch b/var/spack/repos/builtin/packages/netcdf/netcdf-4.3.3-mpi.patch new file mode 100644 index 0000000000..46dda5fc9d --- /dev/null +++ b/var/spack/repos/builtin/packages/netcdf/netcdf-4.3.3-mpi.patch @@ -0,0 +1,25 @@ +diff -Nur netcdf-4.3.3/CMakeLists.txt netcdf-4.3.3.mpi/CMakeLists.txt +--- netcdf-4.3.3/CMakeLists.txt 2015-02-12 16:44:35.000000000 -0500 ++++ netcdf-4.3.3.mpi/CMakeLists.txt 2015-10-14 16:44:41.176300658 -0400 +@@ -753,6 +753,7 @@ + SET(USE_PARALLEL OFF CACHE BOOL "") + MESSAGE(STATUS "Cannot find HDF5 library built with parallel support. Disabling parallel build.") + ELSE() ++ FIND_PACKAGE(MPI REQUIRED) + SET(USE_PARALLEL ON CACHE BOOL "") + SET(STATUS_PARALLEL "ON") + ENDIF() +diff -Nur netcdf-4.3.3/liblib/CMakeLists.txt netcdf-4.3.3.mpi/liblib/CMakeLists.txt +--- netcdf-4.3.3/liblib/CMakeLists.txt 2015-02-12 16:44:35.000000000 -0500 ++++ netcdf-4.3.3.mpi/liblib/CMakeLists.txt 2015-10-14 16:44:57.757793634 -0400 +@@ -71,6 +71,10 @@ + SET(TLL_LIBS ${TLL_LIBS} ${CURL_LIBRARY}) + ENDIF() + ++IF(USE_PARALLEL) ++ SET(TLL_LIBS ${TLL_LIBS} ${MPI_C_LIBRARIES}) ++ENDIF() ++ + IF(USE_HDF4) + SET(TLL_LIBS ${TLL_LIBS} ${HDF4_LIBRARIES}) + ENDIF() diff --git a/var/spack/repos/builtin/packages/netcdf/package.py b/var/spack/repos/builtin/packages/netcdf/package.py new file mode 100644 index 0000000000..e1e0d836c6 --- /dev/null +++ b/var/spack/repos/builtin/packages/netcdf/package.py @@ -0,0 +1,27 @@ +from spack import * + +class Netcdf(Package): + """NetCDF is a set of software libraries and self-describing, machine-independent + data formats that support the creation, access, and sharing of array-oriented + scientific data.""" + + homepage = "http://www.unidata.ucar.edu/software/netcdf/" + url = "ftp://ftp.unidata.ucar.edu/pub/netcdf/netcdf-4.3.3.tar.gz" + + version('4.3.3', '5fbd0e108a54bd82cb5702a73f56d2ae') + + patch('netcdf-4.3.3-mpi.patch') + + # Dependencies: + # >HDF5 + depends_on("hdf5") + + def install(self, spec, prefix): + with working_dir('spack-build', create=True): + cmake('..', + "-DCMAKE_INSTALL_PREFIX:PATH=%s" % prefix, + "-DENABLE_DAP:BOOL=OFF", # Disable DAP. + "-DBUILD_SHARED_LIBS:BOOL=OFF") # Don't build shared libraries (use static libs). + + make() + make("install") diff --git a/var/spack/repos/builtin/packages/netgauge/package.py b/var/spack/repos/builtin/packages/netgauge/package.py new file mode 100644 index 0000000000..c2378b0718 --- /dev/null +++ b/var/spack/repos/builtin/packages/netgauge/package.py @@ -0,0 +1,43 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + +class Netgauge(Package): + """Netgauge is a high-precision network parameter measurement + tool. It supports benchmarking of many different network protocols + and communication patterns. The main focus lies on accuracy, + statistical analysis and easy extensibility. + """ + homepage = "http://unixer.de/research/netgauge/" + url = "http://unixer.de/research/netgauge/netgauge-2.4.6.tar.gz" + + version('2.4.6', 'e0e040ec6452e93ca21ccc54deac1d7f') + + depends_on("mpi") + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix) + make() + make("install") diff --git a/var/spack/repos/builtin/packages/netlib-blas/package.py b/var/spack/repos/builtin/packages/netlib-blas/package.py new file mode 100644 index 0000000000..85e97323d3 --- /dev/null +++ b/var/spack/repos/builtin/packages/netlib-blas/package.py @@ -0,0 +1,46 @@ +from spack import * +import os + + +class NetlibBlas(Package): + """Netlib reference BLAS""" + homepage = "http://www.netlib.org/lapack/" + url = "http://www.netlib.org/lapack/lapack-3.5.0.tgz" + + version('3.5.0', 'b1d3e3e425b2e44a06760ff173104bdf') + + variant('fpic', default=False, description="Build with -fpic compiler option") + + # virtual dependency + provides('blas') + + # Doesn't always build correctly in parallel + parallel = False + + def patch(self): + os.symlink('make.inc.example', 'make.inc') + + mf = FileFilter('make.inc') + mf.filter('^FORTRAN.*', 'FORTRAN = f90') + mf.filter('^LOADER.*', 'LOADER = f90') + mf.filter('^CC =.*', 'CC = cc') + + if '+fpic' in self.spec: + mf.filter('^OPTS.*=.*', 'OPTS = -O2 -frecursive -fpic') + mf.filter('^CFLAGS =.*', 'CFLAGS = -O3 -fpic') + + + def install(self, spec, prefix): + make('blaslib') + + # Tests that blas builds correctly + make('blas_testing') + + # No install provided + mkdirp(prefix.lib) + install('librefblas.a', prefix.lib) + + # Blas virtual package should provide blas.a and libblas.a + with working_dir(prefix.lib): + symlink('librefblas.a', 'blas.a') + symlink('librefblas.a', 'libblas.a') diff --git a/var/spack/repos/builtin/packages/netlib-lapack/package.py b/var/spack/repos/builtin/packages/netlib-lapack/package.py new file mode 100644 index 0000000000..fb6b99e27c --- /dev/null +++ b/var/spack/repos/builtin/packages/netlib-lapack/package.py @@ -0,0 +1,59 @@ +from spack import * + +class NetlibLapack(Package): + """ + LAPACK version 3.X is a comprehensive FORTRAN library that does + linear algebra operations including matrix inversions, least + squared solutions to linear sets of equations, eigenvector + analysis, singular value decomposition, etc. It is a very + comprehensive and reputable package that has found extensive + use in the scientific community. + """ + homepage = "http://www.netlib.org/lapack/" + url = "http://www.netlib.org/lapack/lapack-3.5.0.tgz" + + version('3.5.0', 'b1d3e3e425b2e44a06760ff173104bdf') + version('3.4.2', '61bf1a8a4469d4bdb7604f5897179478') + version('3.4.1', '44c3869c38c8335c2b9c2a8bb276eb55') + version('3.4.0', '02d5706ec03ba885fc246e5fa10d8c70') + version('3.3.1', 'd0d533ec9a5b74933c2a1e84eedc58b4') + + variant('shared', default=False, description="Build shared library version") + + # virtual dependency + provides('lapack') + + # blas is a virtual dependency. + depends_on('blas') + + depends_on('cmake') + + # Doesn't always build correctly in parallel + parallel = False + + @when('^netlib-blas') + def get_blas_libs(self): + blas = self.spec['netlib-blas'] + return [join_path(blas.prefix.lib, 'blas.a')] + + + @when('^atlas') + def get_blas_libs(self): + blas = self.spec['atlas'] + return [join_path(blas.prefix.lib, l) + for l in ('libf77blas.a', 'libatlas.a')] + + + def install(self, spec, prefix): + blas_libs = ";".join(self.get_blas_libs()) + cmake_args = [".", '-DBLAS_LIBRARIES=' + blas_libs] + + if '+shared' in spec: + cmake_args.append('-DBUILD_SHARED_LIBS=ON') + + cmake_args += std_cmake_args + + cmake(*cmake_args) + make() + make("install") + diff --git a/var/spack/repos/builtin/packages/nettle/package.py b/var/spack/repos/builtin/packages/nettle/package.py new file mode 100644 index 0000000000..cd600b0b87 --- /dev/null +++ b/var/spack/repos/builtin/packages/nettle/package.py @@ -0,0 +1,17 @@ +from spack import * + +class Nettle(Package): + """The Nettle package contains the low-level cryptographic library + that is designed to fit easily in many contexts.""" + + homepage = "http://www.example.com" + url = "http://ftp.gnu.org/gnu/nettle/nettle-2.7.1.tar.gz" + + version('2.7', '2caa1bd667c35db71becb93c5d89737f') + + depends_on('gmp') + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix) + make() + make("install") diff --git a/var/spack/repos/builtin/packages/ompss/package.py b/var/spack/repos/builtin/packages/ompss/package.py new file mode 100644 index 0000000000..e09e0a624f --- /dev/null +++ b/var/spack/repos/builtin/packages/ompss/package.py @@ -0,0 +1,50 @@ +from spack import * +import os +import glob + +# working config lines for ompss 14.06 : +#./nanox-0.7/config.log: $ ./configure --prefix=/usr/gapps/exmatex/ompss --with-mcc=/usr/gapps/exmatex/ompss/ --with-hwloc=/usr +#./mcxx-1.99.2/config.log: $ ./configure --prefix=/usr/gapps/exmatex/ompss --with-nanox=/usr/gapps/exmatex/ompss --enable-ompss --with-mpi=/opt/mvapich2-intel-shmem-1.7 --enable-tl-openmp-profile --enable-tl-openmp-intel + +class Ompss(Package): + """OmpSs is an effort to integrate features from the StarSs + programming model developed by BSC into a single programming + model. In particular, our objective is to extend OpenMP with + new directives to support asynchronous parallelism and + heterogeneity (devices like GPUs). However, it can also be + understood as new directives extending other accelerator based + APIs like CUDA or OpenCL. Our OmpSs environment is built on top + of our Mercurium compiler and Nanos++ runtime system.""" + homepage = "http://pm.bsc.es/" + url = "http://pm.bsc.es/sites/default/files/ftp/ompss/releases/ompss-14.10.tar.gz" + list_url = 'http://pm.bsc.es/ompss-downloads' + + version('14.10', '404d161265748f2f96bb35fd8c7e79ee') + + # all dependencies are optional, really + depends_on("mpi") + #depends_on("openmp") + depends_on("hwloc") + depends_on("extrae") + + def install(self, spec, prefix): + if 'openmpi' in spec: + mpi = spec['openmpi'] + elif 'mpich' in spec: + mpi = spec['mpich'] + elif 'mvapich' in spec: + mpi = spec['mvapich'] + + openmp_options = ["--enable-tl-openmp-profile"] + if spec.satisfies('%intel'): + openmp_options.append( "--enable-tl-openmp-intel" ) + + os.chdir(glob.glob('./nanox-*').pop()) + configure("--prefix=%s" % prefix, "--with-mcc=%s" % prefix, "--with-extrae=%s" % spec['extrae'].prefix, "--with-hwloc=%s" % spec['hwloc'].prefix) + make() + make("install") + + os.chdir(glob.glob('../mcxx-*').pop()) + configure("--prefix=%s" % prefix, "--with-nanox=%s" % prefix, "--enable-ompss", "--with-mpi=%s" % mpi.prefix, *openmp_options) + make() + make("install") diff --git a/var/spack/repos/builtin/packages/ompt-openmp/package.py b/var/spack/repos/builtin/packages/ompt-openmp/package.py new file mode 100644 index 0000000000..5d380ebd77 --- /dev/null +++ b/var/spack/repos/builtin/packages/ompt-openmp/package.py @@ -0,0 +1,23 @@ +from spack import * + +class OmptOpenmp(Package): + """LLVM/Clang OpenMP runtime with OMPT support. This is a fork of the OpenMPToolsInterface/LLVM-openmp fork of the official LLVM OpenMP mirror. This library provides a drop-in replacement of the OpenMP runtimes for GCC, Intel and LLVM/Clang.""" + homepage = "https://github.com/OpenMPToolsInterface/LLVM-openmp" + url = "http://github.com/khuck/LLVM-openmp/archive/v0.1-spack.tar.gz" + + version('spack', '35227b2726e377faa433fc841226e036') + + # depends_on("foo") + + def install(self, spec, prefix): + with working_dir("runtime/build", create=True): + + # FIXME: Modify the configure line to suit your build system here. + cmake('-DCMAKE_C_COMPILER=%s' % self.compiler.cc, + '-DCMAKE_CXX_COMPILER=%s' % self.compiler.cxx, + '-DCMAKE_INSTALL_PREFIX=%s' % prefix, + '..', *std_cmake_args) + + # FIXME: Add logic to build and install here + make() + make("install") diff --git a/var/spack/repos/builtin/packages/opari2/package.py b/var/spack/repos/builtin/packages/opari2/package.py new file mode 100644 index 0000000000..daaee61e3a --- /dev/null +++ b/var/spack/repos/builtin/packages/opari2/package.py @@ -0,0 +1,65 @@ +# FIXME: Add copyright statement here + +from spack import * +from contextlib import closing + +class Opari2(Package): + """OPARI2 is a source-to-source instrumentation tool for OpenMP and + hybrid codes. It surrounds OpenMP directives and runtime library + calls with calls to the POMP2 measurement interface. + OPARI2 will provide you with a new initialization method that allows + for multi-directory and parallel builds as well as the usage of + pre-instrumented libraries. Furthermore, an efficient way of + tracking parent-child relationships was added. Additionally, we + extended OPARI2 to support instrumentation of OpenMP 3.0 + tied tasks. """ + + homepage = "http://www.vi-hps.org/projects/score-p" + url = "http://www.vi-hps.org/upload/packages/opari2/opari2-1.1.2.tar.gz" + + version('1.1.2', '9a262c7ca05ff0ab5f7775ae96f3539e') + + backend_user_provided = """\ +CC=cc +CXX=c++ +F77=f77 +FC=f90 +CFLAGS=-fPIC +CXXFLAGS=-fPIC +""" + frontend_user_provided = """\ +CC_FOR_BUILD=cc +CXX_FOR_BUILD=c++ +F77_FOR_BUILD=f70 +FC_FOR_BUILD=f90 +CFLAGS_FOR_BUILD=-fPIC +CXXFLAGS_FOR_BUILD=-fPIC +""" + mpi_user_provided = """\ +MPICC=mpicc +MPICXX=mpicxx +MPIF77=mpif77 +MPIFC=mpif90 +MPI_CFLAGS=-fPIC +MPI_CXXFLAGS=-fPIC +""" + + def install(self, spec, prefix): + # Use a custom compiler configuration, otherwise the score-p + # build system messes with spack's compiler settings. + # Create these three files in the build directory + with closing(open("platform-backend-user-provided", "w")) as backend_file: + backend_file.write(self.backend_user_provided) + with closing(open("platform-frontend-user-provided", "w")) as frontend_file: + frontend_file.write(self.frontend_user_provided) + with closing(open("platform-mpi-user-provided", "w")) as mpi_file: + mpi_file.write(self.mpi_user_provided) + + # FIXME: Modify the configure line to suit your build system here. + configure("--prefix=%s" % prefix, + "--with-custom-compilers", + "--enable-shared") + + # FIXME: Add logic to build and install here + make() + make("install") diff --git a/var/spack/repos/builtin/packages/openmpi/ad_lustre_rwcontig_open_source.patch b/var/spack/repos/builtin/packages/openmpi/ad_lustre_rwcontig_open_source.patch new file mode 100644 index 0000000000..daa825ccbe --- /dev/null +++ b/var/spack/repos/builtin/packages/openmpi/ad_lustre_rwcontig_open_source.patch @@ -0,0 +1,11 @@ +--- a/ompi/mca/io/romio/romio/adio/ad_lustre/ad_lustre_rwcontig.c 2013-12-10 12:05:44.806417000 -0800 ++++ b/ompi/mca/io/romio/romio/adio/ad_lustre/ad_lustre_rwcontig.c 2013-12-10 11:53:03.295622000 -0800 +@@ -8,7 +8,7 @@ + * Copyright (C) 2008 Sun Microsystems, Lustre group + */ + +-#define _XOPEN_SOURCE 600 ++//#define _XOPEN_SOURCE 600 + #include + #include + #include "ad_lustre.h" diff --git a/var/spack/repos/builtin/packages/openmpi/llnl-platforms.patch b/var/spack/repos/builtin/packages/openmpi/llnl-platforms.patch new file mode 100644 index 0000000000..f515743c4d --- /dev/null +++ b/var/spack/repos/builtin/packages/openmpi/llnl-platforms.patch @@ -0,0 +1,151 @@ +diff -Nuar openmpi-1.6.5.orig/contrib/platform/llnl/optimized openmpi-1.6.5.llnl/contrib/platform/llnl/optimized +--- openmpi-1.6.5.orig/contrib/platform/llnl/optimized 1969-12-31 16:00:00.000000000 -0800 ++++ openmpi-1.6.5.llnl/contrib/platform/llnl/optimized 2013-08-08 23:47:12.704029000 -0700 +@@ -0,0 +1,29 @@ ++enable_dlopen=no ++enable_mem_debug=no ++enable_mem_profile=no ++enable_debug_symbols=no ++enable_binaries=yes ++enable_heterogeneous=no ++enable_debug=no ++enable_shared=yes ++enable_static=yes ++enable_memchecker=no ++enable_ipv6=no ++enable_mpi_f77=yes ++enable_mpi_f90=yes ++enable_mpi_cxx=yes ++enable_mpi_cxx_seek=yes ++enable_cxx_exceptions=no ++enable_ft_thread=no ++enable_per_user_config_files=no ++enable_mca_no_build=carto,crs,filem,routed-linear,snapc,pml-dr,pml-crcp2,pml-crcpw,pml-v,pml-example,crcp,btl-tcp ++enable_contrib_no_build=libnbc,vt ++with_slurm=yes ++with_pmi=yes ++with_tm=no ++with_openib=yes ++with_psm=yes ++with_devel_headers=yes ++with_io_romio_flags=--with-file-system=ufs+nfs+lustre ++with_memory_manager=ptmalloc2 ++with_valgrind=no +diff -Nuar openmpi-1.6.5.orig/contrib/platform/llnl/optimized.conf openmpi-1.6.5.llnl/contrib/platform/llnl/optimized.conf +--- openmpi-1.6.5.orig/contrib/platform/llnl/optimized.conf 1969-12-31 16:00:00.000000000 -0800 ++++ openmpi-1.6.5.llnl/contrib/platform/llnl/optimized.conf 2013-08-08 23:43:52.907553000 -0700 +@@ -0,0 +1,114 @@ ++# ++# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana ++# University Research and Technology ++# Corporation. All rights reserved. ++# Copyright (c) 2004-2005 The University of Tennessee and The University ++# of Tennessee Research Foundation. All rights ++# reserved. ++# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, ++# University of Stuttgart. All rights reserved. ++# Copyright (c) 2004-2005 The Regents of the University of California. ++# All rights reserved. ++# Copyright (c) 2006 Cisco Systems, Inc. All rights reserved. ++# Copyright (c) 2011 Los Alamos National Security, LLC. ++# All rights reserved. ++# $COPYRIGHT$ ++# ++# Additional copyrights may follow ++# ++# $HEADER$ ++# ++ ++# This is the default system-wide MCA parameters defaults file. ++# Specifically, the MCA parameter "mca_param_files" defaults to a ++# value of ++# "$HOME/.openmpi/mca-params.conf:$sysconf/openmpi-mca-params.conf" ++# (this file is the latter of the two). So if the default value of ++# mca_param_files is not changed, this file is used to set system-wide ++# MCA parameters. This file can therefore be used to set system-wide ++# default MCA parameters for all users. Of course, users can override ++# these values if they want, but this file is an excellent location ++# for setting system-specific MCA parameters for those users who don't ++# know / care enough to investigate the proper values for them. ++ ++# Note that this file is only applicable where it is visible (in a ++# filesystem sense). Specifically, MPI processes each read this file ++# during their startup to determine what default values for MCA ++# parameters should be used. mpirun does not bundle up the values in ++# this file from the node where it was run and send them to all nodes; ++# the default value decisions are effectively distributed. Hence, ++# these values are only applicable on nodes that "see" this file. If ++# $sysconf is a directory on a local disk, it is likely that changes ++# to this file will need to be propagated to other nodes. If $sysconf ++# is a directory that is shared via a networked filesystem, changes to ++# this file will be visible to all nodes that share this $sysconf. ++ ++# The format is straightforward: one per line, mca_param_name = ++# rvalue. Quoting is ignored (so if you use quotes or escape ++# characters, they'll be included as part of the value). For example: ++ ++# Disable run-time MPI parameter checking ++# mpi_param_check = 0 ++ ++# Note that the value "~/" will be expanded to the current user's home ++# directory. For example: ++ ++# Change component loading path ++# component_path = /usr/local/lib/openmpi:~/my_openmpi_components ++ ++# See "ompi_info --param all all" for a full listing of Open MPI MCA ++# parameters available and their default values. ++# ++ ++# Basic behavior to smooth startup ++mca_component_show_load_errors = 0 ++orte_abort_timeout = 10 ++opal_set_max_sys_limits = 1 ++orte_report_launch_progress = 1 ++ ++# Define timeout for daemons to report back during launch ++orte_startup_timeout = 10000 ++ ++## Protect the shared file systems ++orte_no_session_dirs = /p,/usr/local,/usr/global,/nfs/tmp1,/nfs/tmp2 ++orte_tmpdir_base = /tmp ++ ++## Require an allocation to run - protects the frontend ++## from inadvertent job executions ++orte_allocation_required = 1 ++ ++## MPI behavior ++## Do NOT specify mpi_leave_pinned so system ++## can figure out for itself whether or not ++## it is supported and usable ++orte_notifier = syslog ++ ++## Add the interface for out-of-band communication ++## and set it up ++oob_tcp_if_include=ib0 ++oob_tcp_peer_retries = 1000 ++oob_tcp_disable_family = IPv6 ++oob_tcp_listen_mode = listen_thread ++oob_tcp_sndbuf = 32768 ++oob_tcp_rcvbuf = 32768 ++ ++## Define the MPI interconnects ++btl = sm,openib,self ++ ++## We are using the PSM MTL by default ++## There can only be one! ++pml = cm ++ ++## Setup OpenIB - just in case ++btl_openib_want_fork_support = 0 ++btl_openib_cpc_include = oob ++btl_openib_receive_queues = S,4096,1024:S,12288,512:S,65536,512 ++ ++## Enable cpu affinity ++opal_paffinity_alone = 1 ++ ++## Setup MPI options ++mpi_show_handle_leaks = 0 ++mpi_warn_on_fork = 1 ++mpi_abort_print_stack = 0 ++ diff --git a/var/spack/repos/builtin/packages/openmpi/package.py b/var/spack/repos/builtin/packages/openmpi/package.py new file mode 100644 index 0000000000..5e429dedf5 --- /dev/null +++ b/var/spack/repos/builtin/packages/openmpi/package.py @@ -0,0 +1,109 @@ +import os + +from spack import * + + +class Openmpi(Package): + """Open MPI is a project combining technologies and resources from + several other projects (FT-MPI, LA-MPI, LAM/MPI, and PACX-MPI) + in order to build the best MPI library available. A completely + new MPI-2 compliant implementation, Open MPI offers advantages + for system and software vendors, application developers and + computer science researchers. + """ + + homepage = "http://www.open-mpi.org" + + version('1.10.0', '280cf952de68369cebaca886c5ce0304', + url = "http://www.open-mpi.org/software/ompi/v1.10/downloads/openmpi-1.10.0.tar.bz2") + version('1.8.8', '0dab8e602372da1425e9242ae37faf8c', + url = 'http://www.open-mpi.org/software/ompi/v1.8/downloads/openmpi-1.8.8.tar.bz2') + version('1.6.5', '03aed2a4aa4d0b27196962a2a65fc475', + url = "http://www.open-mpi.org/software/ompi/v1.6/downloads/openmpi-1.6.5.tar.bz2") + + patch('ad_lustre_rwcontig_open_source.patch', when="@1.6.5") + patch('llnl-platforms.patch', when="@1.6.5") + + provides('mpi@:2.2', when='@1.6.5') # Open MPI 1.6.5 supports MPI-2.2 + provides('mpi@:3.0', when='@1.8.8') # Open MPI 1.8.8 supports MPI-3.0 + provides('mpi@:3.0', when='@1.10.0') # Open MPI 1.10.0 supports MPI-3.0 + + + def setup_dependent_environment(self, module, spec, dep_spec): + """For dependencies, make mpicc's use spack wrapper.""" + os.environ['OMPI_CC'] = 'cc' + os.environ['OMPI_CXX'] = 'c++' + os.environ['OMPI_FC'] = 'f90' + os.environ['OMPI_F77'] = 'f77' + + + def install(self, spec, prefix): + config_args = ["--prefix=%s" % prefix] + + # TODO: use variants for this, e.g. +lanl, +llnl, etc. + # use this for LANL builds, but for LLNL builds, we need: + # "--with-platform=contrib/platform/llnl/optimized" + if self.version == ver("1.6.5") and '+lanl' in spec: + config_args.append("--with-platform=contrib/platform/lanl/tlcc2/optimized-nopanasas") + + # TODO: Spack should make it so that you can't actually find + # these compilers if they're "disabled" for the current + # compiler configuration. + if not self.compiler.f77 and not self.compiler.fc: + config_args.append("--enable-mpi-fortran=no") + + configure(*config_args) + make() + make("install") + + self.filter_compilers() + + + def filter_compilers(self): + """Run after install to make the MPI compilers use the + compilers that Spack built the package with. + + If this isn't done, they'll have CC, CXX and FC set + to Spack's generic cc, c++ and f90. We want them to + be bound to whatever compiler they were built with. + """ + kwargs = { 'ignore_absent' : True, 'backup' : False, 'string' : False } + dir = os.path.join(self.prefix, 'share/openmpi/') + + cc_wrappers = ['mpicc-vt-wrapper-data.txt', 'mpicc-wrapper-data.txt', + 'ortecc-wrapper-data.txt', 'shmemcc-wrapper-data.txt'] + + cxx_wrappers = ['mpic++-vt-wrapper-data.txt', 'mpic++-wrapper-data.txt', + 'ortec++-wrapper-data.txt'] + + fc_wrappers = ['mpifort-vt-wrapper-data.txt', + 'mpifort-wrapper-data.txt', 'shmemfort-wrapper-data.txt'] + + for wrapper in cc_wrappers: + filter_file('compiler=.*', 'compiler=%s' % self.compiler.cc, + os.path.join(dir, wrapper), **kwargs) + + for wrapper in cxx_wrappers: + filter_file('compiler=.*', 'compiler=%s' % self.compiler.cxx, + os.path.join(dir, wrapper), **kwargs) + + for wrapper in fc_wrappers: + filter_file('compiler=.*', 'compiler=%s' % self.compiler.fc, + os.path.join(dir, wrapper), **kwargs) + + # These are symlinks in newer versions, so check that here + f77_wrappers = ['mpif77-vt-wrapper-data.txt', 'mpif77-wrapper-data.txt'] + f90_wrappers = ['mpif90-vt-wrapper-data.txt', 'mpif90-wrapper-data.txt'] + + for wrapper in f77_wrappers: + path = os.path.join(dir, wrapper) + if not os.path.islink(path): + filter_file('compiler=.*', 'compiler=%s' % self.compiler.f77, + path, **kwargs) + for wrapper in f90_wrappers: + path = os.path.join(dir, wrapper) + if not os.path.islink(path): + filter_file('compiler=.*', 'compiler=%s' % self.compiler.fc, + path, **kwargs) + + diff --git a/var/spack/repos/builtin/packages/openssl/package.py b/var/spack/repos/builtin/packages/openssl/package.py new file mode 100644 index 0000000000..c5a8aeb9dc --- /dev/null +++ b/var/spack/repos/builtin/packages/openssl/package.py @@ -0,0 +1,26 @@ +from spack import * + +class Openssl(Package): + """The OpenSSL Project is a collaborative effort to develop a + robust, commercial-grade, full-featured, and Open Source + toolkit implementing the Secure Sockets Layer (SSL v2/v3) and + Transport Layer Security (TLS v1) protocols as well as a + full-strength general purpose cryptography library.""" + homepage = "http://www.openssl.org" + url = "http://www.openssl.org/source/openssl-1.0.1h.tar.gz" + + version('1.0.1h', '8d6d684a9430d5cc98a62a5d8fbda8cf') + + depends_on("zlib") + parallel = False + + def install(self, spec, prefix): + config = Executable("./config") + config("--prefix=%s" % prefix, + "--openssldir=%s/etc/openssl" % prefix, + "zlib", + "no-krb5", + "shared") + + make() + make("install") diff --git a/var/spack/repos/builtin/packages/otf/package.py b/var/spack/repos/builtin/packages/otf/package.py new file mode 100644 index 0000000000..52893dd265 --- /dev/null +++ b/var/spack/repos/builtin/packages/otf/package.py @@ -0,0 +1,21 @@ +from spack import * + +class Otf(Package): + """To improve scalability for very large and massively parallel + traces the Open Trace Format (OTF) is developed at ZIH as a + successor format to the Vampir Trace Format (VTF3).""" + + homepage = "http://tu-dresden.de/die_tu_dresden/zentrale_einrichtungen/zih/forschung/projekte/otf/index_html/document_view?set_language=en" + url = "http://wwwpub.zih.tu-dresden.de/%7Emlieber/dcount/dcount.php?package=otf&get=OTF-1.12.5salmon.tar.gz" + + version('1.12.5salmon', 'bf260198633277031330e3356dcb4eec') + + depends_on('zlib') + + def install(self, spec, prefix): + configure('--prefix=%s' % prefix, + '--without-vtf3', + '--with-zlib', + '--with-zlibsymbols') + make() + make("install") diff --git a/var/spack/repos/builtin/packages/otf2/package.py b/var/spack/repos/builtin/packages/otf2/package.py new file mode 100644 index 0000000000..fa0a5898b6 --- /dev/null +++ b/var/spack/repos/builtin/packages/otf2/package.py @@ -0,0 +1,74 @@ +# FIXME: Add copyright + +from spack import * +from contextlib import closing +import os + +class Otf2(Package): + """The Open Trace Format 2 is a highly scalable, memory efficient event + trace data format plus support library.""" + + homepage = "http://www.vi-hps.org/score-p" + url = "http://www.vi-hps.org/upload/packages/otf2/otf2-1.4.tar.gz" + + version('1.4', 'a23c42e936eb9209c4e08b61c3cf5092', + url="http://www.vi-hps.org/upload/packages/otf2/otf2-1.4.tar.gz") + version('1.3.1', 'd0ffc4e858455ace4f596f910e68c9f2', + url="http://www.vi-hps.org/upload/packages/otf2/otf2-1.3.1.tar.gz") + version('1.2.1', '8fb3e11fb7489896596ae2c7c83d7fc8', + url="http://www.vi-hps.org/upload/packages/otf2/otf2-1.2.1.tar.gz") + + backend_user_provided = """\ +CC=cc +CXX=c++ +F77=f77 +FC=f90 +CFLAGS=-fPIC +CXXFLAGS=-fPIC +""" + frontend_user_provided = """\ +CC_FOR_BUILD=cc +CXX_FOR_BUILD=c++ +F77_FOR_BUILD=f70 +FC_FOR_BUILD=f90 +CFLAGS_FOR_BUILD=-fPIC +CXXFLAGS_FOR_BUILD=-fPIC +""" + mpi_user_provided = """\ +MPICC=cc +MPICXX=c++ +MPIF77=f77 +MPIFC=f90 +MPI_CFLAGS=-fPIC +MPI_CXXFLAGS=-fPIC +""" + + @when('@:1.2.1') + def version_specific_args(self): + return ["--with-platform=disabled", "CC=cc", "CXX=c++", "F77=f77", "F90=f90", "CFLAGS=-fPIC", "CXXFLAGS=-fPIC"] + + @when('@1.3:') + def version_specific_args(self): + # TODO: figure out what scorep's build does as of otf2 1.3 + return ["--with-custom-compilers"] + + def install(self, spec, prefix): + # Use a custom compiler configuration, otherwise the score-p + # build system messes with spack's compiler settings. + # Create these three files in the build directory + with closing(open("platform-backend-user-provided", "w")) as backend_file: + backend_file.write(self.backend_user_provided) + with closing(open("platform-frontend-user-provided", "w")) as frontend_file: + frontend_file.write(self.frontend_user_provided) + with closing(open("platform-mpi-user-provided", "w")) as mpi_file: + mpi_file.write(self.mpi_user_provided) + + configure_args=["--prefix=%s" % prefix, + "--enable-shared"] + + configure_args.extend(self.version_specific_args()) + + configure(*configure_args) + + make() + make("install") diff --git a/var/spack/repos/builtin/packages/pango/package.py b/var/spack/repos/builtin/packages/pango/package.py new file mode 100644 index 0000000000..df43625bf5 --- /dev/null +++ b/var/spack/repos/builtin/packages/pango/package.py @@ -0,0 +1,19 @@ +from spack import * + +class Pango(Package): + """Pango is a library for laying out and rendering of text, with + an emphasis on internationalization. It can be used anywhere + that text layout is needed, though most of the work on Pango so + far has been done in the context of the GTK+ widget toolkit.""" + homepage = "http://www.pango.org" + url = "http://ftp.gnome.org/pub/gnome/sources/pango/1.36/pango-1.36.8.tar.xz" + + version('1.36.8', '217a9a753006275215fa9fa127760ece') + + depends_on("harfbuzz") + depends_on("cairo") + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix) + make() + make("install") diff --git a/var/spack/repos/builtin/packages/papi/package.py b/var/spack/repos/builtin/packages/papi/package.py new file mode 100644 index 0000000000..596f7114d6 --- /dev/null +++ b/var/spack/repos/builtin/packages/papi/package.py @@ -0,0 +1,35 @@ +from spack import * +import os + +class Papi(Package): + """PAPI provides the tool designer and application engineer with a + consistent interface and methodology for use of the performance + counter hardware found in most major microprocessors. PAPI + enables software engineers to see, in near real time, the + relation between software performance and processor events. In + addition Component PAPI provides access to a collection of + components that expose performance measurement opportunites + across the hardware and software stack.""" + homepage = "http://icl.cs.utk.edu/papi/index.html" + url = "http://icl.cs.utk.edu/projects/papi/downloads/papi-5.3.0.tar.gz" + + version('5.3.0', '367961dd0ab426e5ae367c2713924ffb') + + def install(self, spec, prefix): + os.chdir("src/") + + configure_args=["--prefix=%s" % prefix] + + # need to force consistency in the use of compilers + if spec.satisfies('%gcc'): + configure_args.append('CC=gcc') + configure_args.append('MPICH_CC=gcc') + if spec.satisfies('%intel'): + configure_args.append('CC=icc') + configure_args.append('MPICH_CC=icc') + + configure(*configure_args) + + make() + make("install") + diff --git a/var/spack/repos/builtin/packages/paraver/package.py b/var/spack/repos/builtin/packages/paraver/package.py new file mode 100644 index 0000000000..5f8a153d4c --- /dev/null +++ b/var/spack/repos/builtin/packages/paraver/package.py @@ -0,0 +1,41 @@ +from spack import * +import os + +class Paraver(Package): + """"A very powerful performance visualization and analysis tool + based on traces that can be used to analyse any information that + is expressed on its input trace format. Traces for parallel MPI, + OpenMP and other programs can be genereated with Extrae.""" + homepage = "http://www.bsc.es/computer-sciences/performance-tools/paraver" + url = "http://www.bsc.es/ssl/apps/performanceTools/files/paraver-sources-4.5.3.tar.gz" + + version('4.5.3', '625de9ec0d639acd18d1aaa644b38f72') + + depends_on("boost") + #depends_on("extrae") + depends_on("wx") + depends_on("wxpropgrid") + + def install(self, spec, prefix): + os.chdir("ptools_common_files") + configure("--prefix=%s" % prefix) + make() + make("install") + + os.chdir("../paraver-kernel") + #"--with-extrae=%s" % spec['extrae'].prefix, + configure("--prefix=%s" % prefix, "--with-ptools-common-files=%s" % prefix, "--with-boost=%s" % spec['boost'].prefix, "--with-boost-serialization=boost_serialization") + make() + make("install") + + os.chdir("../paraver-toolset") + configure("--prefix=%s" % prefix) + make() + make("install") + + os.chdir("../wxparaver") + #"--with-extrae=%s" % spec['extrae'].prefix, + configure("--prefix=%s" % prefix, "--with-paraver=%s" % prefix, "--with-boost=%s" % spec['boost'].prefix, "--with-boost-serialization=boost_serialization", "--with-wxdir=%s" % spec['wx'].prefix.bin) + make() + make("install") + diff --git a/var/spack/repos/builtin/packages/paraview/package.py b/var/spack/repos/builtin/packages/paraview/package.py new file mode 100644 index 0000000000..a0ff812ca2 --- /dev/null +++ b/var/spack/repos/builtin/packages/paraview/package.py @@ -0,0 +1,72 @@ +from spack import * + +class Paraview(Package): + homepage = 'http://www.paraview.org' + url = 'http://www.paraview.org/files/v4.4/ParaView-v4.4.0-source.tar.gz' + + version('4.4.0', 'fa1569857dd680ebb4d7ff89c2227378', url='http://www.paraview.org/files/v4.4/ParaView-v4.4.0-source.tar.gz') + + variant('python', default=False, description='Enable Python support') + variant('matplotlib', default=False, description='Enable Matplotlib support') + variant('numpy', default=False, description='Enable NumPy support') + + variant('tcl', default=False, description='Enable TCL support') + + variant('mpi', default=False, description='Enable MPI support') + + variant('osmesa', default=False, description='Enable OSMesa support') + variant('qt', default=False, description='Enable Qt support') + + depends_on('python', when='+python') + depends_on('py-numpy', when='+python+numpy') + depends_on('py-matplotlib', when='+python+matplotlib') + depends_on('tcl', when='+tcl') + depends_on('mpi', when='+mpi') + depends_on('qt', when='+qt') + + depends_on('bzip2') + depends_on('freetype') + depends_on('hdf5') # drags in mpi + depends_on('jpeg') + depends_on('libpng') + depends_on('libtiff') + #depends_on('libxml2') # drags in python + depends_on('netcdf') + #depends_on('protobuf') # version mismatches? + #depends_on('sqlite') # external version not supported + depends_on('zlib') + + def install(self, spec, prefix): + with working_dir('spack-build', create=True): + def feature_to_bool(feature, on='ON', off='OFF'): + if feature in spec: + return on + return off + + def nfeature_to_bool(feature): + return feature_to_bool(feature, on='OFF', off='ON') + + feature_args = std_cmake_args[:] + feature_args.append('-DPARAVIEW_BUILD_QT_GUI:BOOL=%s' % feature_to_bool('+qt')) + feature_args.append('-DPARAVIEW_ENABLE_PYTHON:BOOL=%s' % feature_to_bool('+python')) + feature_args.append('-DPARAVIEW_USE_MPI:BOOL=%s' % feature_to_bool('+mpi')) + feature_args.append('-DVTK_ENABLE_TCL_WRAPPING:BOOL=%s' % feature_to_bool('+tcl')) + feature_args.append('-DVTK_OPENGL_HAS_OSMESA:BOOL=%s' % feature_to_bool('+osmesa')) + feature_args.append('-DVTK_USE_X:BOOL=%s' % nfeature_to_bool('+osmesa')) + feature_args.append('-DVTK_RENDERING_BACKEND:STRING=%s' % feature_to_bool('+opengl2', 'OpenGL2', 'OpenGL')) + + feature_args.extend(std_cmake_args) + + cmake('..', + '-DCMAKE_INSTALL_PREFIX:PATH=%s' % prefix, + '-DBUILD_TESTING:BOOL=OFF', + '-DVTK_USER_SYSTEM_FREETYPE:BOOL=ON', + '-DVTK_USER_SYSTEM_HDF5:BOOL=ON', + '-DVTK_USER_SYSTEM_JPEG:BOOL=ON', + #'-DVTK_USER_SYSTEM_LIBXML2:BOOL=ON', + '-DVTK_USER_SYSTEM_NETCDF:BOOL=ON', + '-DVTK_USER_SYSTEM_TIFF:BOOL=ON', + '-DVTK_USER_SYSTEM_ZLIB:BOOL=ON', + *feature_args) + make() + make('install') diff --git a/var/spack/repos/builtin/packages/parmetis/package.py b/var/spack/repos/builtin/packages/parmetis/package.py new file mode 100644 index 0000000000..d8cd337304 --- /dev/null +++ b/var/spack/repos/builtin/packages/parmetis/package.py @@ -0,0 +1,26 @@ +from spack import * + +class Parmetis(Package): + """ParMETIS is an MPI-based parallel library that implements a + variety of algorithms for partitioning unstructured graphs, + meshes, and for computing fill-reducing orderings of sparse + matrices.""" + homepage = "http://glaros.dtc.umn.edu/gkhome/metis/parmetis/overview" + url = "http://glaros.dtc.umn.edu/gkhome/fetch/sw/parmetis/parmetis-4.0.3.tar.gz" + + version('4.0.3', 'f69c479586bf6bb7aff6a9bc0c739628') + + depends_on('mpi') + + def install(self, spec, prefix): + cmake(".", + '-DGKLIB_PATH=%s/metis/GKlib' % pwd(), + '-DMETIS_PATH=%s/metis' % pwd(), + '-DSHARED=1', + '-DCMAKE_C_COMPILER=mpicc', + '-DCMAKE_CXX_COMPILER=mpicxx', + '-DSHARED=1', + *std_cmake_args) + + make() + make("install") diff --git a/var/spack/repos/builtin/packages/parpack/package.py b/var/spack/repos/builtin/packages/parpack/package.py new file mode 100644 index 0000000000..622aceca04 --- /dev/null +++ b/var/spack/repos/builtin/packages/parpack/package.py @@ -0,0 +1,43 @@ +from spack import * +import os +import shutil + +class Parpack(Package): + """ARPACK is a collection of Fortran77 subroutines designed to solve large + scale eigenvalue problems.""" + + homepage = "http://www.caam.rice.edu/software/ARPACK/download.html" + url = "http://www.caam.rice.edu/software/ARPACK/SRC/parpack96.tar.Z" + + version('96', 'a175f70ff71837a33ff7e4b0b6054f43') + + depends_on('mpi') + depends_on('blas') + depends_on('lapack') + + def patch(self): + # Filter the CJ makefile to make a spack one. + shutil.move('ARMAKES/ARmake.CJ', 'ARmake.inc') + mf = FileFilter('ARmake.inc') + + # Be sure to use Spack F77 wrapper + mf.filter('^FC.*', 'FC = f77') + mf.filter('^FFLAGS.*', 'FFLAGS = -O2 -g') + + # Set up some variables. + mf.filter('^PLAT.*', 'PLAT = ') + mf.filter('^home.*', 'home = %s' % os.getcwd()) + mf.filter('^BLASdir.*', 'BLASdir = %s' % self.spec['blas'].prefix) + mf.filter('^LAPACKdir.*', 'LAPACKdir = %s' % self.spec['lapack'].prefix) + mf.filter('^MAKE.*', 'MAKE = make') + + # build the library in our own prefix. + mf.filter('^ARPACKLIB.*', 'PARPACKLIB = %s/libparpack.a' % os.getcwd()) + + + def install(self, spec, prefix): + with working_dir('PARPACK/SRC/MPI'): + make('all') + + mkdirp(prefix.lib) + install('libparpack.a', prefix.lib) diff --git a/var/spack/repos/builtin/packages/pcre/package.py b/var/spack/repos/builtin/packages/pcre/package.py new file mode 100644 index 0000000000..3424048a6c --- /dev/null +++ b/var/spack/repos/builtin/packages/pcre/package.py @@ -0,0 +1,15 @@ +from spack import * + +class Pcre(Package): + """The PCRE package contains Perl Compatible Regular Expression + libraries. These are useful for implementing regular expression + pattern matching using the same syntax and semantics as Perl 5.""" + homepage = "http://www.pcre.org""" + url = "ftp://ftp.csx.cam.ac.uk/pub/software/programming/pcre/pcre-8.36.tar.bz2" + + version('8.36', 'b767bc9af0c20bc9c1fe403b0d41ad97') + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix) + make() + make("install") diff --git a/var/spack/repos/builtin/packages/petsc/package.py b/var/spack/repos/builtin/packages/petsc/package.py new file mode 100644 index 0000000000..4864e39bf1 --- /dev/null +++ b/var/spack/repos/builtin/packages/petsc/package.py @@ -0,0 +1,40 @@ +from spack import * + +class Petsc(Package): + """PETSc is a suite of data structures and routines for the + scalable (parallel) solution of scientific applications modeled by + partial differential equations.""" + + homepage = "http://www.mcs.anl.gov/petsc/index.html" + url = "http://ftp.mcs.anl.gov/pub/petsc/release-snapshots/petsc-3.5.3.tar.gz" + + version('3.5.3', 'd4fd2734661e89f18ac6014b5dd1ef2f') + version('3.5.2', 'ad170802b3b058b5deb9cd1f968e7e13') + version('3.5.1', 'a557e029711ebf425544e117ffa44d8f') + + depends_on("boost") + depends_on("blas") + depends_on("lapack") + depends_on("hypre") + depends_on("parmetis") + depends_on("metis") + depends_on("hdf5") + depends_on("mpi") + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix, + "CC=cc", + "CXX=c++", + "FC=f90", + "--with-blas-lib=%s/libblas.a" % spec['blas'].prefix.lib, + "--with-lapack-lib=%s/liblapack.a" % spec['lapack'].prefix.lib, + "--with-boost-dir=%s" % spec['boost'].prefix, + "--with-hypre-dir=%s" % spec['hypre'].prefix, + "--with-parmetis-dir=%s" % spec['parmetis'].prefix, + "--with-metis-dir=%s" % spec['metis'].prefix, + "--with-hdf5-dir=%s" % spec['hdf5'].prefix, + "--with-shared-libraries=0") + + # PETSc has its own way of doing parallel make. + make('MAKE_NP=%s' % make_jobs, parallel=False) + make("install") diff --git a/var/spack/repos/builtin/packages/pidx/package.py b/var/spack/repos/builtin/packages/pidx/package.py new file mode 100644 index 0000000000..81aed62fb1 --- /dev/null +++ b/var/spack/repos/builtin/packages/pidx/package.py @@ -0,0 +1,21 @@ +from spack import * + +class Pidx(Package): + """PIDX Parallel I/O Library. + + PIDX is an efficient parallel I/O library that reads and writes + multiresolution IDX data files. + """ + + homepage = "http://www.cedmav.com/pidx" + + version('1.0', git='https://github.com/sci-visus/PIDX.git', + commit='6afa1cf71d1c41263296dc049c8fabaf73c296da') + + depends_on("mpi") + + def install(self, spec, prefix): + with working_dir('spack-build', create=True): + cmake('..', *std_cmake_args) + make() + make("install") diff --git a/var/spack/repos/builtin/packages/pixman/package.py b/var/spack/repos/builtin/packages/pixman/package.py new file mode 100644 index 0000000000..895cbdbca5 --- /dev/null +++ b/var/spack/repos/builtin/packages/pixman/package.py @@ -0,0 +1,18 @@ +from spack import * + +class Pixman(Package): + """The Pixman package contains a library that provides low-level + pixel manipulation features such as image compositing and + trapezoid rasterization.""" + homepage = "http://www.pixman.org" + url = "http://cairographics.org/releases/pixman-0.32.6.tar.gz" + + version('0.32.6', '3a30859719a41bd0f5cccffbfefdd4c2') + + depends_on("libpng") + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix, + "--disable-gtk") + make() + make("install") diff --git a/var/spack/repos/builtin/packages/pkg-config/package.py b/var/spack/repos/builtin/packages/pkg-config/package.py new file mode 100644 index 0000000000..9964c6ce34 --- /dev/null +++ b/var/spack/repos/builtin/packages/pkg-config/package.py @@ -0,0 +1,17 @@ +from spack import * + +class PkgConfig(Package): + """pkg-config is a helper tool used when compiling applications and libraries""" + homepage = "http://www.freedesktop.org/wiki/Software/pkg-config/" + url = "http://pkgconfig.freedesktop.org/releases/pkg-config-0.28.tar.gz" + + version('0.28', 'aa3c86e67551adc3ac865160e34a2a0d') + + parallel = False + + def install(self, spec, prefix): + configure("--prefix=%s" %prefix, "--enable-shared") + + make() + make("install") + diff --git a/var/spack/repos/builtin/packages/pmgr_collective/package.py b/var/spack/repos/builtin/packages/pmgr_collective/package.py new file mode 100644 index 0000000000..5d9b02acc3 --- /dev/null +++ b/var/spack/repos/builtin/packages/pmgr_collective/package.py @@ -0,0 +1,37 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + +class PmgrCollective(Package): + """PMGR_COLLECTIVE provides a scalable network for bootstrapping + MPI jobs.""" + homepage = "http://www.sourceforge.net/projects/pmgrcollective" + url = "http://downloads.sourceforge.net/project/pmgrcollective/pmgrcollective/PMGR_COLLECTIVE-1.0/pmgr_collective-1.0.tgz" + + version('1.0', '0384d008774274cc3fc7b4d810dfd07e') + + def install(self, spec, prefix): + make('PREFIX="' + prefix + '"') + make('PREFIX="' + prefix + '"', "install") diff --git a/var/spack/repos/builtin/packages/postgresql/package.py b/var/spack/repos/builtin/packages/postgresql/package.py new file mode 100644 index 0000000000..46922b7b71 --- /dev/null +++ b/var/spack/repos/builtin/packages/postgresql/package.py @@ -0,0 +1,20 @@ +from spack import * + +class Postgresql(Package): + """PostgreSQL is a powerful, open source object-relational + database system. It has more than 15 years of active + development and a proven architecture that has earned it a + strong reputation for reliability, data integrity, and + correctness.""" + homepage = "http://www.postgresql.org/" + url = "http://ftp.postgresql.org/pub/source/v9.3.4/postgresql-9.3.4.tar.bz2" + + version('9.3.4', 'd0a41f54c377b2d2fab4a003b0dac762') + + depends_on("openssl") + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix, + "--with-openssl") + make() + make("install") diff --git a/var/spack/repos/builtin/packages/ppl/package.py b/var/spack/repos/builtin/packages/ppl/package.py new file mode 100644 index 0000000000..018d5c523d --- /dev/null +++ b/var/spack/repos/builtin/packages/ppl/package.py @@ -0,0 +1,28 @@ +from spack import * + +class Ppl(Package): + """The Parma Polyhedra Library (PPL) provides numerical + abstractions especially targeted at applications in the field of + analysis and verification of complex systems. These abstractions + include convex polyhedra, some special classes of polyhedra shapes + that offer interesting complexity/precision tradeoffs, and grids + which represent regularly spaced points that satisfy a set of + linear congruence relations. The library also supports finite + powersets and products of polyhedra and grids, a mixed integer + linear programming problem solver using an exact-arithmetic + version of the simplex algorithm, a parametric integer programming + solver, and primitives for termination analysis via the automatic + synthesis of linear ranking functions.""" + + homepage = "http://bugseng.com/products/ppl/" + url = "http://bugseng.com/products/ppl/download/ftp/releases/1.1/ppl-1.1.tar.gz" + + version('1.1', '4f2422c0ef3f409707af32108deb30a7') + + depends_on("gmp") + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix, + "--with-gmp=%s" % spec['gmp'].prefix) + make() + make("install") diff --git a/var/spack/repos/builtin/packages/protobuf/package.py b/var/spack/repos/builtin/packages/protobuf/package.py new file mode 100644 index 0000000000..34085c7ce9 --- /dev/null +++ b/var/spack/repos/builtin/packages/protobuf/package.py @@ -0,0 +1,16 @@ +import os +from spack import * + +class Protobuf(Package): + """Google's data interchange format.""" + + homepage = "https://developers.google.com/protocol-buffers" + url = "https://github.com/google/protobuf/releases/download/v2.5.0/protobuf-2.5.0.tar.bz2" + + version('2.5.0', 'a72001a9067a4c2c4e0e836d0f92ece4') + + def install(self, spec, prefix): + configure("--prefix=" + prefix) + make() + make("check") + make("install") diff --git a/var/spack/repos/builtin/packages/py-basemap/package.py b/var/spack/repos/builtin/packages/py-basemap/package.py new file mode 100644 index 0000000000..45f1085ba1 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-basemap/package.py @@ -0,0 +1,20 @@ +from spack import * +import os + +class PyBasemap(Package): + """The matplotlib basemap toolkit is a library for plotting 2D data on maps in Python.""" + homepage = "http://matplotlib.org/basemap/" + url = "https://downloads.sourceforge.net/project/matplotlib/matplotlib-toolkits/basemap-1.0.7/basemap-1.0.7.tar.gz" + + version('1.0.7', '48c0557ced9e2c6e440b28b3caff2de8') + + extends('python') + depends_on('py-setuptools') + depends_on('py-numpy') + depends_on('py-matplotlib') + depends_on('py-pil') + depends_on("geos") + + def install(self, spec, prefix): + env['GEOS_DIR'] = spec['geos'].prefix + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/repos/builtin/packages/py-biopython/package.py b/var/spack/repos/builtin/packages/py-biopython/package.py new file mode 100644 index 0000000000..8ecaf48626 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-biopython/package.py @@ -0,0 +1,15 @@ +from spack import * + +class PyBiopython(Package): + """It is a distributed collaborative effort to develop Python libraries and applications which address the needs of current and future work in bioinformatics.""" + homepage = "http://biopython.org/wiki/Main_Page" + url = "http://biopython.org/DIST/biopython-1.65.tar.gz" + + version('1.65', '143e7861ade85c0a8b5e2bbdd1da1f67') + + extends('python') + depends_on('py-mx') + depends_on('py-numpy') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/repos/builtin/packages/py-cffi/package.py b/var/spack/repos/builtin/packages/py-cffi/package.py new file mode 100644 index 0000000000..a4d37483fe --- /dev/null +++ b/var/spack/repos/builtin/packages/py-cffi/package.py @@ -0,0 +1,17 @@ +from spack import * + +class PyCffi(Package): + """Foreign Function Interface for Python calling C code""" + homepage = "http://cffi.readthedocs.org/en/latest/" + # base https://pypi.python.org/pypi/cffi + url = "https://pypi.python.org/packages/source/c/cffi/cffi-1.1.2.tar.gz#md5=" + + version('1.1.2', 'ca6e6c45b45caa87aee9adc7c796eaea') + + extends('python') + depends_on('py-setuptools') + depends_on('py-pycparser') + depends_on('libffi') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/repos/builtin/packages/py-cython/package.py b/var/spack/repos/builtin/packages/py-cython/package.py new file mode 100644 index 0000000000..68eb735ad9 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-cython/package.py @@ -0,0 +1,14 @@ +from spack import * + +class PyCython(Package): + """The Cython compiler for writing C extensions for the Python language.""" + homepage = "https://pypi.python.org/pypi/cython" + url = "https://pypi.python.org/packages/source/C/Cython/cython-0.22.tar.gz" + + version('0.21.2', 'd21adb870c75680dc857cd05d41046a4') + version('0.22', '1ae25add4ef7b63ee9b4af697300d6b6') + + extends('python') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/repos/builtin/packages/py-dateutil/package.py b/var/spack/repos/builtin/packages/py-dateutil/package.py new file mode 100644 index 0000000000..0a17f2f2d2 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-dateutil/package.py @@ -0,0 +1,16 @@ +from spack import * + +class PyDateutil(Package): + """Extensions to the standard Python datetime module.""" + homepage = "https://pypi.python.org/pypi/dateutil" + url = "https://pypi.python.org/packages/source/p/python-dateutil/python-dateutil-2.4.0.tar.gz" + + version('2.4.0', '75714163bb96bedd07685cdb2071b8bc') + version('2.4.2', '4ef68e1c485b09e9f034e10473e5add2') + + extends('python') + depends_on('py-setuptools') + depends_on('py-six') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/repos/builtin/packages/py-epydoc/package.py b/var/spack/repos/builtin/packages/py-epydoc/package.py new file mode 100644 index 0000000000..af05510504 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-epydoc/package.py @@ -0,0 +1,13 @@ +from spack import * + +class PyEpydoc(Package): + """Epydoc is a tool for generating API documentation documentation for Python modules, based on their docstrings.""" + homepage = "https://pypi.python.org/pypi/epydoc" + url = "https://pypi.python.org/packages/source/e/epydoc/epydoc-3.0.1.tar.gz" + + version('3.0.1', '36407974bd5da2af00bf90ca27feeb44') + + extends('python') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/repos/builtin/packages/py-genders/package.py b/var/spack/repos/builtin/packages/py-genders/package.py new file mode 100644 index 0000000000..c49c8fd5b2 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-genders/package.py @@ -0,0 +1,15 @@ +from spack import * + +class PyGenders(Package): + """Genders is a static cluster configuration database used for cluster configuration management. It is used by a variety of tools and scripts for management of large clusters.""" + homepage = "https://github.com/chaos/genders" + url = "https://github.com/chaos/genders/releases/download/genders-1-22-1/genders-1.22.tar.gz" + + version('1.22', '9ea59a024dcbddb85b0ed25ddca9bc8e', url='https://github.com/chaos/genders/releases/download/genders-1-22-1/genders-1.22.tar.gz') + extends('python') + + def install(self, spec, prefix): + configure("--prefix=%s" %prefix) + make(parallel=False) + make("install") + diff --git a/var/spack/repos/builtin/packages/py-gnuplot/package.py b/var/spack/repos/builtin/packages/py-gnuplot/package.py new file mode 100644 index 0000000000..ede4472c03 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-gnuplot/package.py @@ -0,0 +1,14 @@ +from spack import * + +class PyGnuplot(Package): + """Gnuplot.py is a Python package that allows you to create graphs from within Python using the gnuplot plotting program.""" + homepage = "http://gnuplot-py.sourceforge.net/" + url = "http://downloads.sourceforge.net/project/gnuplot-py/Gnuplot-py/1.8/gnuplot-py-1.8.tar.gz" + + version('1.8', 'abd6f571e7aec68ae7db90a5217cd5b1') + + extends('python') + depends_on('py-numpy') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/repos/builtin/packages/py-h5py/package.py b/var/spack/repos/builtin/packages/py-h5py/package.py new file mode 100644 index 0000000000..6293da5407 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-h5py/package.py @@ -0,0 +1,19 @@ +from spack import * +import re + +class PyH5py(Package): + """The h5py package provides both a high- and low-level interface to the HDF5 library from Python.""" + homepage = "https://pypi.python.org/pypi/h5py" + url = "https://pypi.python.org/packages/source/h/h5py/h5py-2.4.0.tar.gz" + + version('2.4.0', '80c9a94ae31f84885cc2ebe1323d6758') + version('2.5.0', '6e4301b5ad5da0d51b0a1e5ac19e3b74') + + extends('python', ignore=lambda f: re.match(r'bin/cy*', f)) + depends_on('hdf5') + depends_on('py-numpy') + depends_on('py-cython') + + def install(self, spec, prefix): + python('setup.py', 'configure', '--hdf5=%s' % spec['hdf5'].prefix) + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/repos/builtin/packages/py-ipython/package.py b/var/spack/repos/builtin/packages/py-ipython/package.py new file mode 100644 index 0000000000..8d0e64a07f --- /dev/null +++ b/var/spack/repos/builtin/packages/py-ipython/package.py @@ -0,0 +1,16 @@ +from spack import * + +class PyIpython(Package): + """IPython provides a rich toolkit to help you make the most out of using Python interactively.""" + homepage = "https://pypi.python.org/pypi/ipython" + url = "https://pypi.python.org/packages/source/i/ipython/ipython-2.3.1.tar.gz" + + version('2.3.1', '2b7085525dac11190bfb45bb8ec8dcbf') + version('3.1.0', 'a749d90c16068687b0ec45a27e72ef8f') + + extends('python') + depends_on('py-pygments') + depends_on('py-setuptools') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/repos/builtin/packages/py-libxml2/package.py b/var/spack/repos/builtin/packages/py-libxml2/package.py new file mode 100644 index 0000000000..59005428e4 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-libxml2/package.py @@ -0,0 +1,15 @@ +from spack import * + +class PyLibxml2(Package): + """A Python wrapper around libxml2.""" + homepage = "https://xmlsoft.org/python.html" + url = "ftp://xmlsoft.org/libxml2/python/libxml2-python-2.6.21.tar.gz" + + version('2.6.21', '229dd2b3d110a77defeeaa73af83f7f3') + + extends('python') + depends_on('libxml2') + depends_on('libxslt') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/repos/builtin/packages/py-lockfile/package.py b/var/spack/repos/builtin/packages/py-lockfile/package.py new file mode 100644 index 0000000000..8722914d94 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-lockfile/package.py @@ -0,0 +1,23 @@ +from spack import * + +class PyLockfile(Package): + """The lockfile package exports a LockFile class which provides a + simple API for locking files. Unlike the Windows msvcrt.locking + function, the fcntl.lockf and flock functions, and the + deprecated posixfile module, the API is identical across both + Unix (including Linux and Mac) and Windows platforms. The lock + mechanism relies on the atomic nature of the link (on Unix) and + mkdir (on Windows) system calls. An implementation based on + SQLite is also provided, more as a demonstration of the + possibilities it provides than as production-quality code. + """ + homepage = "https://pypi.python.org/pypi/lockfile" + url = "https://pypi.python.org/packages/source/l/lockfile/lockfile-0.10.2.tar.gz" + + version('0.10.2', '1aa6175a6d57f082cd12e7ac6102ab15') + + extends("python") + depends_on("py-setuptools") + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/repos/builtin/packages/py-mako/package.py b/var/spack/repos/builtin/packages/py-mako/package.py new file mode 100644 index 0000000000..3e91ffd8e5 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-mako/package.py @@ -0,0 +1,16 @@ +from spack import * + +class PyMako(Package): + """A super-fast templating language that borrows the best + ideas from the existing templating languages.""" + + homepage = "https://pypi.python.org/pypi/mako" + url = "https://pypi.python.org/packages/source/M/Mako/Mako-1.0.1.tar.gz" + + version('1.0.1', '9f0aafd177b039ef67b90ea350497a54') + + depends_on('py-setuptools') + extends('python') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/repos/builtin/packages/py-matplotlib/package.py b/var/spack/repos/builtin/packages/py-matplotlib/package.py new file mode 100644 index 0000000000..e7ce3dfd24 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-matplotlib/package.py @@ -0,0 +1,47 @@ +from spack import * +import os + +class PyMatplotlib(Package): + """Python plotting package.""" + homepage = "https://pypi.python.org/pypi/matplotlib" + url = "https://pypi.python.org/packages/source/m/matplotlib/matplotlib-1.4.2.tar.gz" + + version('1.4.2', '7d22efb6cce475025733c50487bd8898') + version('1.4.3', '86af2e3e3c61849ac7576a6f5ca44267') + + extends('python', ignore=r'bin/nosetests.*$') + + depends_on('py-pyside') + depends_on('py-ipython') + depends_on('py-pyparsing') + depends_on('py-six') + depends_on('py-dateutil') + depends_on('py-pytz') + depends_on('py-nose') + depends_on('py-numpy') + + depends_on('qt') + depends_on('bzip2') + depends_on('tcl') + depends_on('tk') + depends_on('qhull') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) + + if str(self.version) in ['1.4.2', '1.4.3']: + # hack to fix configuration file + config_file = None + for p,d,f in os.walk(prefix.lib): + for file in f: + if file.find('matplotlibrc') != -1: + config_file = join_path(p, 'matplotlibrc') + print config_file + if config_file == None: + raise InstallError('could not find config file') + filter_file(r'backend : pyside', + 'backend : Qt4Agg', + config_file) + filter_file(r'#backend.qt4 : PyQt4', + 'backend.qt4 : PySide', + config_file) diff --git a/var/spack/repos/builtin/packages/py-mock/package.py b/var/spack/repos/builtin/packages/py-mock/package.py new file mode 100644 index 0000000000..3b08428ba0 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-mock/package.py @@ -0,0 +1,17 @@ +from spack import * + +class PyMock(Package): + """mock is a library for testing in Python. It allows you to replace parts + of your system under test with mock objects and make assertions about how + they have been used.""" + + homepage = "https://github.com/testing-cabal/mock" + url = "https://pypi.python.org/packages/source/m/mock/mock-1.3.0.tar.gz" + + version('1.3.0', '73ee8a4afb3ff4da1b4afa287f39fdeb') + + extends('python') + depends_on('py-setuptools@17.1:') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/repos/builtin/packages/py-mpi4py/package.py b/var/spack/repos/builtin/packages/py-mpi4py/package.py new file mode 100644 index 0000000000..8001689a18 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-mpi4py/package.py @@ -0,0 +1,14 @@ +from spack import * + +class PyMpi4py(Package): + """This package provides Python bindings for the Message Passing Interface (MPI) standard. It is implemented on top of the MPI-1/MPI-2 specification and exposes an API which grounds on the standard MPI-2 C++ bindings.""" + homepage = "https://pypi.python.org/pypi/mpi4py" + url = "https://pypi.python.org/packages/source/m/mpi4py/mpi4py-1.3.1.tar.gz" + + version('1.3.1', 'dbe9d22bdc8ed965c23a7ceb6f32fc3c') + extends('python') + depends_on('py-setuptools') + depends_on('mpi') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/repos/builtin/packages/py-mx/package.py b/var/spack/repos/builtin/packages/py-mx/package.py new file mode 100644 index 0000000000..717ee0562b --- /dev/null +++ b/var/spack/repos/builtin/packages/py-mx/package.py @@ -0,0 +1,13 @@ +from spack import * + +class PyMx(Package): + """The eGenix.com mx Base Distribution for Python is a collection of professional quality software tools which enhance Python's usability in many important areas such as fast text searching, date/time processing and high speed data types.""" + homepage = "http://www.egenix.com/products/python/mxBase/" + url = "https://downloads.egenix.com/python/egenix-mx-base-3.2.8.tar.gz" + + version('3.2.8', '9d9d3a25f9dc051a15e97f452413423b') + + extends('python') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/repos/builtin/packages/py-nose/package.py b/var/spack/repos/builtin/packages/py-nose/package.py new file mode 100644 index 0000000000..e7c6cf0264 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-nose/package.py @@ -0,0 +1,17 @@ +from spack import * + +class PyNose(Package): + """nose extends the test loading and running features of unittest, + making it easier to write, find and run tests.""" + + homepage = "https://pypi.python.org/pypi/nose" + url = "https://pypi.python.org/packages/source/n/nose/nose-1.3.4.tar.gz" + + version('1.3.4', '6ed7169887580ddc9a8e16048d38274d') + version('1.3.6', '0ca546d81ca8309080fc80cb389e7a16') + + extends('python', ignore=r'bin/nosetests.*$') + depends_on('py-setuptools') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/repos/builtin/packages/py-numpy/package.py b/var/spack/repos/builtin/packages/py-numpy/package.py new file mode 100644 index 0000000000..efa109a3e9 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-numpy/package.py @@ -0,0 +1,28 @@ +from spack import * + +class PyNumpy(Package): + """array processing for numbers, strings, records, and objects.""" + homepage = "https://pypi.python.org/pypi/numpy" + url = "https://pypi.python.org/packages/source/n/numpy/numpy-1.9.1.tar.gz" + + version('1.9.1', '78842b73560ec378142665e712ae4ad9') + version('1.9.2', 'a1ed53432dbcd256398898d35bc8e645') + + extends('python') + depends_on('py-nose') + depends_on('netlib-blas+fpic') + depends_on('netlib-lapack+shared') + + def patch(self): + filter_file( + "possible_executables = \['(gfortran|g77|ifort|efl)", + "possible_executables = ['fc", + "numpy/distutils/fcompiler/gnu.py", + "numpy/distutils/fcompiler/intel.py") + + def install(self, spec, prefix): + with open('site.cfg', 'w') as f: + f.write('[DEFAULT]\n') + f.write('libraries=lapack,blas\n') + f.write('library_dirs=%s/lib:%s/lib\n' % (spec['blas'].prefix, spec['lapack'].prefix)) + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/repos/builtin/packages/py-pandas/package.py b/var/spack/repos/builtin/packages/py-pandas/package.py new file mode 100644 index 0000000000..5b9997faa9 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-pandas/package.py @@ -0,0 +1,25 @@ +from spack import * +import os + +class PyPandas(Package): + """pandas is a Python package providing fast, flexible, and expressive data structures designed to make working with relational or labeled data both easy and intuitive. It aims to be the fundamental high-level building block for doing practical, real world data analysis in Python. Additionally, it has the broader goal of becoming the most powerful and flexible open source data analysis / manipulation tool available in any language.""" + homepage = "http://pandas.pydata.org/" + url = "https://pypi.python.org/packages/source/p/pandas/pandas-0.16.0.tar.gz#md5=bfe311f05dc0c351f8955fbd1e296e73" + + version('0.16.0', 'bfe311f05dc0c351f8955fbd1e296e73') + version('0.16.1', 'fac4f25748f9610a3e00e765474bdea8') + + extends('python') + depends_on('py-dateutil') + depends_on('py-numpy') + depends_on('py-matplotlib') + depends_on('py-scipy') + depends_on('py-setuptools') + depends_on('py-pytz') + depends_on('libdrm') + depends_on('libpciaccess') + depends_on('llvm') + depends_on('mesa') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/repos/builtin/packages/py-pexpect/package.py b/var/spack/repos/builtin/packages/py-pexpect/package.py new file mode 100644 index 0000000000..ff5fac84e0 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-pexpect/package.py @@ -0,0 +1,13 @@ +from spack import * + +class PyPexpect(Package): + """Pexpect allows easy control of interactive console applications.""" + homepage = "https://pypi.python.org/pypi/pexpect" + url = "https://pypi.python.org/packages/source/p/pexpect/pexpect-3.3.tar.gz" + + version('3.3', '0de72541d3f1374b795472fed841dce8') + + extends('python') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/repos/builtin/packages/py-pil/package.py b/var/spack/repos/builtin/packages/py-pil/package.py new file mode 100644 index 0000000000..743b761981 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-pil/package.py @@ -0,0 +1,14 @@ +from spack import * + +class PyPil(Package): + """The Python Imaging Library (PIL) adds image processing capabilities to your Python interpreter. This library supports many file formats, and provides powerful image processing and graphics capabilities.""" + + homepage = "http://www.pythonware.com/products/pil/" + url = "http://effbot.org/media/downloads/Imaging-1.1.7.tar.gz" + + version('1.1.7', 'fc14a54e1ce02a0225be8854bfba478e') + + extends('python') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/repos/builtin/packages/py-pmw/package.py b/var/spack/repos/builtin/packages/py-pmw/package.py new file mode 100644 index 0000000000..56131811e9 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-pmw/package.py @@ -0,0 +1,13 @@ +from spack import * + +class PyPmw(Package): + """Pmw is a toolkit for building high-level compound widgets, or megawidgets, constructed using other widgets as component parts.""" + homepage = "https://pypi.python.org/pypi/Pmw" + url = "https://pypi.python.org/packages/source/P/Pmw/Pmw-2.0.0.tar.gz" + + version('2.0.0', 'c7c3f26c4f5abaa99807edefee578fc0') + + extends('python') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/repos/builtin/packages/py-pychecker/package.py b/var/spack/repos/builtin/packages/py-pychecker/package.py new file mode 100644 index 0000000000..bda5a746aa --- /dev/null +++ b/var/spack/repos/builtin/packages/py-pychecker/package.py @@ -0,0 +1,13 @@ +from spack import * + +class PyPychecker(Package): + """""" + homepage = "http://pychecker.sourceforge.net/" + url = "http://sourceforge.net/projects/pychecker/files/pychecker/0.8.19/pychecker-0.8.19.tar.gz" + + version('0.8.19', 'c37182863dfb09209d6ba4f38fce9d2b') + + extends('python') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/repos/builtin/packages/py-pycparser/package.py b/var/spack/repos/builtin/packages/py-pycparser/package.py new file mode 100644 index 0000000000..f2bb679d25 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-pycparser/package.py @@ -0,0 +1,15 @@ +from spack import * + +class PyPycparser(Package): + """pycparser is a complete parser of the C language, written in pure python""" + homepage = "https://github.com/eliben/pycparser" + url = "https://pypi.python.org/packages/source/p/pycparser/pycparser-2.13.tar.gz" + + version('2.13', 'e4fe1a2d341b22e25da0d22f034ef32f') + + + extends('python') + depends_on('py-setuptools') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/repos/builtin/packages/py-pyelftools/package.py b/var/spack/repos/builtin/packages/py-pyelftools/package.py new file mode 100644 index 0000000000..d5ad32e624 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-pyelftools/package.py @@ -0,0 +1,13 @@ +from spack import * + +class PyPyelftools(Package): + """A pure-Python library for parsing and analyzing ELF files and DWARF debugging information""" + homepage = "https://pypi.python.org/pypi/pyelftools" + url = "https://pypi.python.org/packages/source/p/pyelftools/pyelftools-0.23.tar.gz" + + version('0.23', 'aa7cefa8bd2f63d7b017440c9084f310') + + extends('python') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/repos/builtin/packages/py-pygments/package.py b/var/spack/repos/builtin/packages/py-pygments/package.py new file mode 100644 index 0000000000..7e07bf6869 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-pygments/package.py @@ -0,0 +1,15 @@ +from spack import * + +class PyPygments(Package): + """Pygments is a syntax highlighting package written in Python.""" + homepage = "https://pypi.python.org/pypi/pygments" + url = "https://pypi.python.org/packages/source/P/Pygments/Pygments-2.0.1.tar.gz" + + version('2.0.1', 'e0daf4c14a4fe5b630da765904de4d6c') + version('2.0.2', '238587a1370d62405edabd0794b3ec4a') + + extends('python') + depends_on('py-setuptools') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/repos/builtin/packages/py-pylint/package.py b/var/spack/repos/builtin/packages/py-pylint/package.py new file mode 100644 index 0000000000..9579708c29 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-pylint/package.py @@ -0,0 +1,17 @@ +from spack import * +import re + +class PyPylint(Package): + """array processing for numbers, strings, records, and objects.""" + homepage = "https://pypi.python.org/pypi/pylint" + url = "https://pypi.python.org/packages/source/p/pylint/pylint-1.4.1.tar.gz" + + version('1.4.1', 'df7c679bdcce5019389038847e4de622') + version('1.4.3', '5924c1c7ca5ca23647812f5971d0ea44') + + extends('python') + depends_on('py-nose') + depends_on('py-setuptools') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/repos/builtin/packages/py-pypar/package.py b/var/spack/repos/builtin/packages/py-pypar/package.py new file mode 100644 index 0000000000..af9c76ccd8 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-pypar/package.py @@ -0,0 +1,14 @@ +from spack import * + +class PyPypar(Package): + """Pypar is an efficient but easy-to-use module that allows programs written in Python to run in parallel on multiple processors and communicate using MPI.""" + homepage = "http://code.google.com/p/pypar/" + url = "https://pypar.googlecode.com/files/pypar-2.1.5_108.tgz" + + version('2.1.5_108', '7a1f28327d2a3b679f9455c843d850b8', url='https://pypar.googlecode.com/files/pypar-2.1.5_108.tgz') + extends('python') + depends_on('mpi') + + def install(self, spec, prefix): + with working_dir('source'): + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/repos/builtin/packages/py-pyparsing/package.py b/var/spack/repos/builtin/packages/py-pyparsing/package.py new file mode 100644 index 0000000000..a6e50ad139 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-pyparsing/package.py @@ -0,0 +1,13 @@ +from spack import * + +class PyPyparsing(Package): + """A Python Parsing Module.""" + homepage = "https://pypi.python.org/pypi/pyparsing" + url = "https://pypi.python.org/packages/source/p/pyparsing/pyparsing-2.0.3.tar.gz" + + version('2.0.3', '0fe479be09fc2cf005f753d3acc35939') + + extends('python') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/repos/builtin/packages/py-pyqt/package.py b/var/spack/repos/builtin/packages/py-pyqt/package.py new file mode 100644 index 0000000000..8edca105bb --- /dev/null +++ b/var/spack/repos/builtin/packages/py-pyqt/package.py @@ -0,0 +1,24 @@ +from spack import * + +class PyPyqt(Package): + """PyQt is a set of Python v2 and v3 bindings for Digia's Qt + application framework and runs on all platforms supported by Qt + including Windows, MacOS/X and Linux.""" + homepage = "http://www.riverbankcomputing.com/software/pyqt/intro" + url = "http://sourceforge.net/projects/pyqt/files/PyQt4/PyQt-4.11.3/PyQt-x11-gpl-4.11.3.tar.gz" + + version('4.11.3', '997c3e443165a89a559e0d96b061bf70') + + extends('python') + depends_on('py-sip') + + # TODO: allow qt5 when conditional deps are supported. + # TODO: Fix version matching so that @4 works like @:4 + depends_on('qt@:4') + + def install(self, spec, prefix): + python('configure.py', + '--confirm-license', + '--destdir=%s' % site_packages_dir) + make() + make('install') diff --git a/var/spack/repos/builtin/packages/py-pyside/package.py b/var/spack/repos/builtin/packages/py-pyside/package.py new file mode 100644 index 0000000000..bb5da44d02 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-pyside/package.py @@ -0,0 +1,45 @@ +from spack import * +import os + +class PyPyside(Package): + """array processing for numbers, strings, records, and objects.""" + homepage = "https://pypi.python.org/pypi/pyside" + url = "https://pypi.python.org/packages/source/P/PySide/PySide-1.2.2.tar.gz" + + version('1.2.2', 'c45bc400c8a86d6b35f34c29e379e44d') + + # TODO: make build dependency + # depends_on("cmake") + + extends('python') + depends_on('py-setuptools') + depends_on('qt@:4') + + def patch(self): + """Undo PySide RPATH handling and add Spack RPATH.""" + # Figure out the special RPATH + pypkg = self.spec['python'].package + rpath = self.rpath + rpath.append(os.path.join(self.prefix, pypkg.site_packages_dir, 'PySide')) + + # Add Spack's standard CMake args to the sub-builds. + # They're called BY setup.py so we have to patch it. + filter_file( + r'OPTION_CMAKE,', + r'OPTION_CMAKE, ' + ( + '"-DCMAKE_INSTALL_RPATH_USE_LINK_PATH=FALSE", ' + '"-DCMAKE_INSTALL_RPATH=%s",' % ':'.join(rpath)), + 'setup.py') + + # PySide tries to patch ELF files to remove RPATHs + # Disable this and go with the one we set. + filter_file( + r'^\s*rpath_cmd\(pyside_path, srcpath\)', + r'#rpath_cmd(pyside_path, srcpath)', + 'pyside_postinstall.py') + + + def install(self, spec, prefix): + python('setup.py', 'install', + '--prefix=%s' % prefix, + '--jobs=%s' % make_jobs) diff --git a/var/spack/repos/builtin/packages/py-python-daemon/package.py b/var/spack/repos/builtin/packages/py-python-daemon/package.py new file mode 100644 index 0000000000..12cbe9101c --- /dev/null +++ b/var/spack/repos/builtin/packages/py-python-daemon/package.py @@ -0,0 +1,26 @@ +from spack import * + +class PyPythonDaemon(Package): + """Library to implement a well-behaved Unix daemon process. + + This library implements the well-behaved daemon specification of + PEP Standard daemon process. + + A well-behaved Unix daemon process is tricky to get right, but the + required steps are much the same for every daemon program. A + DaemonContext instance holds the behaviour and configured process + environment for the program; use the instance as a context manager + to enter a daemon state. + """ + homepage = "https://pypi.python.org/pypi/python-daemon/" + url = "https://pypi.python.org/packages/source/p/python-daemon/python-daemon-2.0.5.tar.gz" + + version('2.0.5', '73e7f49f525c51fa4a995aea4d80de41') + + extends("python") + depends_on("py-setuptools") + depends_on("py-lockfile") + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) + diff --git a/var/spack/repos/builtin/packages/py-pytz/package.py b/var/spack/repos/builtin/packages/py-pytz/package.py new file mode 100644 index 0000000000..da6311a784 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-pytz/package.py @@ -0,0 +1,14 @@ +from spack import * + +class PyPytz(Package): + """World timezone definitions, modern and historical.""" + homepage = "https://pypi.python.org/pypi/pytz" + url = "https://pypi.python.org/packages/source/p/pytz/pytz-2014.10.tar.gz" + + version('2014.10', 'eb1cb941a20c5b751352c52486aa1dd7') + version('2015.4', '417a47b1c432d90333e42084a605d3d8') + + extends('python') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/repos/builtin/packages/py-rpy2/package.py b/var/spack/repos/builtin/packages/py-rpy2/package.py new file mode 100644 index 0000000000..a0b03d03e3 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-rpy2/package.py @@ -0,0 +1,17 @@ +from spack import * + +class PyRpy2(Package): + """rpy2 is a redesign and rewrite of rpy. It is providing a low-level interface to R from Python, a proposed high-level interface, including wrappers to graphical libraries, as well as R-like structures and functions.""" + homepage = "https://pypi.python.org/pypi/rpy2" + url = "https://pypi.python.org/packages/source/r/rpy2/rpy2-2.5.4.tar.gz" + + version('2.5.4', '115a20ac30883f096da2bdfcab55196d') + version('2.5.6', 'a36e758b633ce6aec6a5f450bfee980f') + + extends('python') + depends_on('py-setuptools') + + depends_on('R') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/repos/builtin/packages/py-scientificpython/package.py b/var/spack/repos/builtin/packages/py-scientificpython/package.py new file mode 100644 index 0000000000..df2c86caac --- /dev/null +++ b/var/spack/repos/builtin/packages/py-scientificpython/package.py @@ -0,0 +1,16 @@ +from spack import * + +class PyScientificpython(Package): + """ScientificPython is a collection of Python modules for + scientific computing. It contains support for geometry, + mathematical functions, statistics, physical units, IO, + visualization, and parallelization.""" + + homepage = "https://sourcesup.renater.fr/projects/scientific-py/" + url = "https://sourcesup.renater.fr/frs/download.php/file/4411/ScientificPython-2.8.1.tar.gz" + version('2.8.1', '73ee0df19c7b58cdf2954261f0763c77') + + extends('python') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/repos/builtin/packages/py-scikit-learn/package.py b/var/spack/repos/builtin/packages/py-scikit-learn/package.py new file mode 100644 index 0000000000..5b078ce901 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-scikit-learn/package.py @@ -0,0 +1,14 @@ +from spack import * + +class PyScikitLearn(Package): + """""" + homepage = "https://pypi.python.org/pypi/scikit-learn" + url = "https://pypi.python.org/packages/source/s/scikit-learn/scikit-learn-0.15.2.tar.gz" + + version('0.15.2', 'd9822ad0238e17b382a3c756ea94fe0d') + version('0.16.1', '363ddda501e3b6b61726aa40b8dbdb7e') + + extends('python') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/repos/builtin/packages/py-scipy/package.py b/var/spack/repos/builtin/packages/py-scipy/package.py new file mode 100644 index 0000000000..3a1124cc15 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-scipy/package.py @@ -0,0 +1,18 @@ +from spack import * + +class PyScipy(Package): + """Scientific Library for Python.""" + homepage = "https://pypi.python.org/pypi/scipy" + url = "https://pypi.python.org/packages/source/s/scipy/scipy-0.15.0.tar.gz" + + version('0.15.0', '639112f077f0aeb6d80718dc5019dc7a') + version('0.15.1', 'be56cd8e60591d6332aac792a5880110') + + extends('python') + depends_on('py-nose') + depends_on('py-numpy') + depends_on('blas') + depends_on('lapack') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/repos/builtin/packages/py-setuptools/package.py b/var/spack/repos/builtin/packages/py-setuptools/package.py new file mode 100644 index 0000000000..760ad4d6db --- /dev/null +++ b/var/spack/repos/builtin/packages/py-setuptools/package.py @@ -0,0 +1,15 @@ +from spack import * + +class PySetuptools(Package): + """Easily download, build, install, upgrade, and uninstall Python packages.""" + homepage = "https://pypi.python.org/pypi/setuptools" + url = "https://pypi.python.org/packages/source/s/setuptools/setuptools-11.3.tar.gz" + + version('11.3.1', '01f69212e019a2420c1693fb43593930') + version('16.0', '0ace0b96233516fc5f7c857d086aa3ad') + version('18.1', 'f72e87f34fbf07f299f6cb46256a0b06') + + extends('python') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/repos/builtin/packages/py-shiboken/package.py b/var/spack/repos/builtin/packages/py-shiboken/package.py new file mode 100644 index 0000000000..e4bf4ce07e --- /dev/null +++ b/var/spack/repos/builtin/packages/py-shiboken/package.py @@ -0,0 +1,45 @@ +from spack import * +import os + +class PyShiboken(Package): + """Shiboken generates bindings for C++ libraries using CPython source code.""" + homepage = "https://shiboken.readthedocs.org/" + url = "https://pypi.python.org/packages/source/S/Shiboken/Shiboken-1.2.2.tar.gz" + + version('1.2.2', '345cfebda221f525842e079a6141e555') + + # TODO: make build dependency + # depends_on("cmake") + + extends('python') + depends_on("py-setuptools") + depends_on("libxml2") + depends_on("qt@:4.8") + + def patch(self): + """Undo Shiboken RPATH handling and add Spack RPATH.""" + # Add Spack's standard CMake args to the sub-builds. + # They're called BY setup.py so we have to patch it. + pypkg = self.spec['python'].package + rpath = self.rpath + rpath.append(os.path.join(self.prefix, pypkg.site_packages_dir, 'Shiboken')) + + filter_file( + r'OPTION_CMAKE,', + r'OPTION_CMAKE, ' + ( + '"-DCMAKE_INSTALL_RPATH_USE_LINK_PATH=FALSE", ' + '"-DCMAKE_INSTALL_RPATH=%s",' % ':'.join(rpath)), + 'setup.py') + + # Shiboken tries to patch ELF files to remove RPATHs + # Disable this and go with the one we set. + filter_file( + r'^\s*rpath_cmd\(shiboken_path, srcpath\)', + r'#rpath_cmd(shiboken_path, srcpath)', + 'shiboken_postinstall.py') + + + def install(self, spec, prefix): + python('setup.py', 'install', + '--prefix=%s' % prefix, + '--jobs=%s' % make_jobs) diff --git a/var/spack/repos/builtin/packages/py-sip/package.py b/var/spack/repos/builtin/packages/py-sip/package.py new file mode 100644 index 0000000000..e4a6fb6961 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-sip/package.py @@ -0,0 +1,21 @@ +from spack import * +import os + +class PySip(Package): + """SIP is a tool that makes it very easy to create Python bindings for C and C++ libraries.""" + homepage = "http://www.riverbankcomputing.com/software/sip/intro" + url = "http://sourceforge.net/projects/pyqt/files/sip/sip-4.16.5/sip-4.16.5.tar.gz" + + version('4.16.5', '6d01ea966a53e4c7ae5c5e48c40e49e5') + version('4.16.7', '32abc003980599d33ffd789734de4c36') + + extends('python') + + def install(self, spec, prefix): + python('configure.py', + '--destdir=%s' % site_packages_dir, + '--bindir=%s' % spec.prefix.bin, + '--incdir=%s' % python_include_dir, + '--sipdir=%s' % os.path.join(spec.prefix.share, 'sip')) + make() + make('install') diff --git a/var/spack/repos/builtin/packages/py-six/package.py b/var/spack/repos/builtin/packages/py-six/package.py new file mode 100644 index 0000000000..05c5bd00a9 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-six/package.py @@ -0,0 +1,14 @@ +from spack import * + +class PySix(Package): + """Python 2 and 3 compatibility utilities.""" + homepage = "https://pypi.python.org/pypi/six" + url = "https://pypi.python.org/packages/source/s/six/six-1.9.0.tar.gz" + + version('1.9.0', '476881ef4012262dfc8adc645ee786c4') + + extends('python') + depends_on('py-setuptools') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/repos/builtin/packages/py-sphinx/package.py b/var/spack/repos/builtin/packages/py-sphinx/package.py new file mode 100644 index 0000000000..ec2e89a098 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-sphinx/package.py @@ -0,0 +1,13 @@ +from spack import * + +class PySphinx(Package): + """Sphinx Documentation Generator.""" + homepage = "http://sphinx-doc.org" + url = "https://pypi.python.org/packages/source/S/Sphinx/Sphinx-1.3.1.tar.gz" + + version('1.3.1', '8786a194acf9673464c5455b11fd4332') + + extends('python') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/repos/builtin/packages/py-sympy/package.py b/var/spack/repos/builtin/packages/py-sympy/package.py new file mode 100644 index 0000000000..c17e35b95f --- /dev/null +++ b/var/spack/repos/builtin/packages/py-sympy/package.py @@ -0,0 +1,13 @@ +from spack import * + +class PySympy(Package): + """SymPy is a Python library for symbolic mathematics.""" + homepage = "https://pypi.python.org/pypi/sympy" + url = "https://pypi.python.org/packages/source/s/sympy/sympy-0.7.6.tar.gz" + + version('0.7.6', '3d04753974306d8a13830008e17babca') + + extends('python') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/repos/builtin/packages/py-virtualenv/package.py b/var/spack/repos/builtin/packages/py-virtualenv/package.py new file mode 100644 index 0000000000..037a6fc59f --- /dev/null +++ b/var/spack/repos/builtin/packages/py-virtualenv/package.py @@ -0,0 +1,16 @@ +from spack import * +import shutil + +class PyVirtualenv(Package): + """virtualenv is a tool to create isolated Python environments.""" + homepage = "http://virtualenv.readthedocs.org/projects/virtualenv/" + url = "https://pypi.python.org/packages/source/v/virtualenv/virtualenv-1.11.6.tar.gz" + + version('1.11.6', 'f61cdd983d2c4e6aeabb70b1060d6f49') + version('13.0.1', '1ffc011bde6667f0e37ecd976f4934db') + + extends('python') + depends_on('py-setuptools') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/repos/builtin/packages/py-yapf/package.py b/var/spack/repos/builtin/packages/py-yapf/package.py new file mode 100644 index 0000000000..12ef191515 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-yapf/package.py @@ -0,0 +1,15 @@ +from spack import * + +class PyYapf(Package): + """ Yet Another Python Formatter """ + homepage = "https://github.com/google/yapf" + # base https://pypi.python.org/pypi/cffi + url = "https://github.com/google/yapf/archive/v0.2.1.tar.gz" + + version('0.2.1', '348ccf86cf2057872e4451b204fb914c') + + extends('python') + depends_on('py-setuptools') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/repos/builtin/packages/python/package.py b/var/spack/repos/builtin/packages/python/package.py new file mode 100644 index 0000000000..000881a846 --- /dev/null +++ b/var/spack/repos/builtin/packages/python/package.py @@ -0,0 +1,160 @@ +import os +import re +from contextlib import closing +from llnl.util.lang import match_predicate + +from spack import * +import spack + + +class Python(Package): + """The Python programming language.""" + homepage = "http://www.python.org" + url = "http://www.python.org/ftp/python/2.7.8/Python-2.7.8.tar.xz" + + extendable = True + + version('2.7.8', 'd235bdfa75b8396942e360a70487ee00') + version('2.7.10', 'c685ef0b8e9f27b5e3db5db12b268ac6') + + depends_on("openssl") + depends_on("bzip2") + depends_on("readline") + depends_on("ncurses") + depends_on("sqlite") + + def install(self, spec, prefix): + # Need this to allow python build to find the Python installation. + env['PYTHONHOME'] = prefix + + # Rest of install is pretty standard. + configure("--prefix=%s" % prefix, + "--with-threads", + "--enable-shared") + make() + make("install") + + + # ======================================================================== + # Set up environment to make install easy for python extensions. + # ======================================================================== + + @property + def python_lib_dir(self): + return os.path.join('lib', 'python%d.%d' % self.version[:2]) + + + @property + def python_include_dir(self): + return os.path.join('include', 'python%d.%d' % self.version[:2]) + + + @property + def site_packages_dir(self): + return os.path.join(self.python_lib_dir, 'site-packages') + + + def setup_dependent_environment(self, module, spec, ext_spec): + """Called before python modules' install() methods. + + In most cases, extensions will only need to have one line:: + + python('setup.py', 'install', '--prefix=%s' % prefix) + """ + # Python extension builds can have a global python executable function + module.python = Executable(join_path(spec.prefix.bin, 'python')) + + # Add variables for lib/pythonX.Y and lib/pythonX.Y/site-packages dirs. + module.python_lib_dir = os.path.join(ext_spec.prefix, self.python_lib_dir) + module.python_include_dir = os.path.join(ext_spec.prefix, self.python_include_dir) + module.site_packages_dir = os.path.join(ext_spec.prefix, self.site_packages_dir) + + # Make the site packages directory if it does not exist already. + mkdirp(module.site_packages_dir) + + # Set PYTHONPATH to include site-packages dir for the + # extension and any other python extensions it depends on. + python_paths = [] + for d in ext_spec.traverse(): + if d.package.extends(self.spec): + python_paths.append(os.path.join(d.prefix, self.site_packages_dir)) + os.environ['PYTHONPATH'] = ':'.join(python_paths) + + + # ======================================================================== + # Handle specifics of activating and deactivating python modules. + # ======================================================================== + + def python_ignore(self, ext_pkg, args): + """Add some ignore files to activate/deactivate args.""" + ignore_arg = args.get('ignore', lambda f: False) + + # Always ignore easy-install.pth, as it needs to be merged. + patterns = [r'easy-install\.pth$'] + + # Ignore pieces of setuptools installed by other packages. + if ext_pkg.name != 'py-setuptools': + patterns.append(r'/site\.pyc?$') + patterns.append(r'setuptools\.pth') + patterns.append(r'bin/easy_install[^/]*$') + patterns.append(r'setuptools.*egg$') + + return match_predicate(ignore_arg, patterns) + + + def write_easy_install_pth(self, exts): + paths = [] + for ext in sorted(exts.values()): + ext_site_packages = os.path.join(ext.prefix, self.site_packages_dir) + easy_pth = "%s/easy-install.pth" % ext_site_packages + + if not os.path.isfile(easy_pth): + continue + + with closing(open(easy_pth)) as f: + for line in f: + line = line.rstrip() + + # Skip lines matching these criteria + if not line: continue + if re.search(r'^(import|#)', line): continue + if (ext.name != 'py-setuptools' and + re.search(r'setuptools.*egg$', line)): continue + + paths.append(line) + + site_packages = os.path.join(self.prefix, self.site_packages_dir) + main_pth = "%s/easy-install.pth" % site_packages + + if not paths: + if os.path.isfile(main_pth): + os.remove(main_pth) + + else: + with closing(open(main_pth, 'w')) as f: + f.write("import sys; sys.__plen = len(sys.path)\n") + for path in paths: + f.write("%s\n" % path) + f.write("import sys; new=sys.path[sys.__plen:]; del sys.path[sys.__plen:]; " + "p=getattr(sys,'__egginsert',0); sys.path[p:p]=new; sys.__egginsert = p+len(new)\n") + + + def activate(self, ext_pkg, **args): + ignore=self.python_ignore(ext_pkg, args) + args.update(ignore=ignore) + + super(Python, self).activate(ext_pkg, **args) + + exts = spack.install_layout.extension_map(self.spec) + exts[ext_pkg.name] = ext_pkg.spec + self.write_easy_install_pth(exts) + + + def deactivate(self, ext_pkg, **args): + args.update(ignore=self.python_ignore(ext_pkg, args)) + super(Python, self).deactivate(ext_pkg, **args) + + exts = spack.install_layout.extension_map(self.spec) + if ext_pkg.name in exts: # Make deactivate idempotent. + del exts[ext_pkg.name] + self.write_easy_install_pth(exts) diff --git a/var/spack/repos/builtin/packages/qhull/package.py b/var/spack/repos/builtin/packages/qhull/package.py new file mode 100644 index 0000000000..9da4078a70 --- /dev/null +++ b/var/spack/repos/builtin/packages/qhull/package.py @@ -0,0 +1,27 @@ +from spack import * + +class Qhull(Package): + """Qhull computes the convex hull, Delaunay triangulation, Voronoi + diagram, halfspace intersection about a point, furt hest-site + Delaunay triangulation, and furthest-site Voronoi diagram. The + source code runs in 2-d, 3-d, 4-d, and higher dimensions. Qhull + implements the Quickhull algorithm for computing the convex + hull. It handles roundoff errors from floating point + arithmetic. It computes volumes, surface areas, and + approximations to the convex hull. + + Qhull does not support triangulation of non-convex surfaces, + mesh generation of non-convex objects, medium-sized inputs in + 9-D and higher, alpha shapes, weighted Voronoi diagrams, + Voronoi volumes, or constrained Delaunay triangulations.""" + + homepage = "http://www.qhull.org" + + version('1.0', 'd0f978c0d8dfb2e919caefa56ea2953c', + url="http://www.qhull.org/download/qhull-2012.1-src.tgz") + + def install(self, spec, prefix): + with working_dir('spack-build', create=True): + cmake('..', *std_cmake_args) + make() + make("install") diff --git a/var/spack/repos/builtin/packages/qt/package.py b/var/spack/repos/builtin/packages/qt/package.py new file mode 100644 index 0000000000..0e4abe3b1d --- /dev/null +++ b/var/spack/repos/builtin/packages/qt/package.py @@ -0,0 +1,109 @@ +import os +from spack import * +import os + +class Qt(Package): + """Qt is a comprehensive cross-platform C++ application framework.""" + homepage = "http://qt.io" + list_url = 'http://download.qt-project.org/official_releases/qt/' + list_depth = 2 + + version('5.4.0', 'e8654e4b37dd98039ba20da7a53877e6', + url='http://download.qt-project.org/official_releases/qt/5.4/5.4.0/single/qt-everywhere-opensource-src-5.4.0.tar.gz') + version('5.3.2', 'febb001129927a70174467ecb508a682', + url='http://download.qt.io/archive/qt/5.3/5.3.2/single/qt-everywhere-opensource-src-5.3.2.tar.gz') + + version('5.2.1', 'a78408c887c04c34ce615da690e0b4c8', + url='http://download.qt.io/archive/qt/5.2/5.2.1/single/qt-everywhere-opensource-src-5.2.1.tar.gz') + version('4.8.6', '2edbe4d6c2eff33ef91732602f3518eb', + url="http://download.qt-project.org/official_releases/qt/4.8/4.8.6/qt-everywhere-opensource-src-4.8.6.tar.gz") + + # Use system openssl for security. + #depends_on("openssl") + + depends_on("glib") + depends_on("gtkplus") + depends_on("libxml2") + depends_on("zlib") + depends_on("dbus") + depends_on("libtiff") + depends_on("libpng") + depends_on("libmng") + depends_on("jpeg") + + # Webkit + # depends_on("gperf") + # depends_on("flex") + # depends_on("bison") + # depends_on("ruby") + # depends_on("icu4c") + + # OpenGL hardware acceleration + depends_on("mesa") + depends_on("libxcb") + + + def setup_dependent_environment(self, module, spec, dep_spec): + """Dependencies of Qt find it using the QTDIR environment variable.""" + os.environ['QTDIR'] = self.prefix + + + def patch(self): + if self.spec.satisfies('@4'): + qmake_conf = 'mkspecs/common/g++-base.conf' + qmake_unix_conf = 'mkspecs/common/g++-unix.conf' + elif self.spec.satisfies('@5'): + qmake_conf = 'qtbase/mkspecs/common/g++-base.conf' + qmake_unix_conf = 'qtbase/mkspecs/common/g++-unix.conf' + else: + return + + # Fix qmake compilers in the default mkspec + filter_file(r'^QMAKE_COMPILER *=.*$', 'QMAKE_COMPILER = cc', qmake_conf) + filter_file(r'^QMAKE_CC *=.*$', 'QMAKE_CC = cc', qmake_conf) + filter_file(r'^QMAKE_CXX *=.*$', 'QMAKE_CXX = c++', qmake_conf) + filter_file(r'^QMAKE_LFLAGS_NOUNDEF *\+?=.*$', 'QMAKE_LFLAGS_NOUNDEF =', qmake_unix_conf) + + + @property + def common_config_args(self): + return [ + '-prefix', self.prefix, + '-v', + '-opensource', + '-opengl', + "-release", + '-shared', + '-confirm-license', + '-openssl-linked', + '-dbus-linked', + '-optimized-qmake', + '-no-openvg', + '-no-pch', + # NIS is deprecated in more recent glibc + "-no-nis"] + # Don't disable all the database drivers, but should + # really get them into spack at some point. + + + @when('@4') + def configure(self): + configure('-fast', + '-no-webkit', + *self.common_config_args) + + + @when('@5') + def configure(self): + configure('-no-eglfs', + '-no-directfb', + '-qt-xcb', + # If someone wants to get a webkit build working, be my guest! + '-skip', 'qtwebkit', + *self.common_config_args) + + + def install(self, spec, prefix): + self.configure() + make() + make("install") diff --git a/var/spack/repos/builtin/packages/qthreads/package.py b/var/spack/repos/builtin/packages/qthreads/package.py new file mode 100644 index 0000000000..dacdb71524 --- /dev/null +++ b/var/spack/repos/builtin/packages/qthreads/package.py @@ -0,0 +1,22 @@ +from spack import * + +class Qthreads(Package): + """The qthreads API is designed to make using large numbers of + threads convenient and easy, and to allow portable access to + threading constructs used in massively parallel shared memory + environments. The API maps well to both MTA-style threading and + PIM-style threading, and we provide an implementation of this + interface in both a standard SMP context as well as the SST + context. The qthreads API provides access to full/empty-bit + (FEB) semantics, where every word of memory can be marked + either full or empty, and a thread can wait for any word to + attain either state.""" + homepage = "http://www.cs.sandia.gov/qthreads/" + url = "https://qthreads.googlecode.com/files/qthread-1.10.tar.bz2" + + version('1.10', '5af8c8bbe88c2a6d45361643780d1671') + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix) + make() + make("install") diff --git a/var/spack/repos/builtin/packages/ravel/package.py b/var/spack/repos/builtin/packages/ravel/package.py new file mode 100644 index 0000000000..01fa941cfe --- /dev/null +++ b/var/spack/repos/builtin/packages/ravel/package.py @@ -0,0 +1,23 @@ +from spack import * + +class Ravel(Package): + """Ravel is a parallel communication trace visualization tool that + orders events according to logical time.""" + + homepage = "https://github.com/scalability-llnl/ravel" + url = 'https://github.com/scalability-llnl/ravel/archive/v1.0.0.tar.gz' + + version('1.0.0', 'b25fece58331c2adfcce76c5036485c2') + + # TODO: make this a build dependency + depends_on('cmake@2.8.9:') + + depends_on('muster@1.0.1:') + depends_on('otf') + depends_on('otf2') + depends_on('qt@5:') + + def install(self, spec, prefix): + cmake('-Wno-dev', *std_cmake_args) + make() + make("install") diff --git a/var/spack/repos/builtin/packages/readline/package.py b/var/spack/repos/builtin/packages/readline/package.py new file mode 100644 index 0000000000..1b870e0e7f --- /dev/null +++ b/var/spack/repos/builtin/packages/readline/package.py @@ -0,0 +1,21 @@ +from spack import * + +class Readline(Package): + """The GNU Readline library provides a set of functions for use by + applications that allow users to edit command li nes as they + are typed in. Both Emacs and vi editing modes are + available. The Readline library includes additional functions + to maintain a list of previously-entered command lines, to + recall and perhaps reedit those lines, and perform csh-like + history expansion on previous commands. """ + homepage = "http://cnswww.cns.cwru.edu/php/chet/readline/rltop.html" + url = "ftp://ftp.cwru.edu/pub/bash/readline-6.3.tar.gz" + + version('6.3', '33c8fb279e981274f485fd91da77e94a') + + depends_on("ncurses") + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix) + make("SHLIB_LIBS=-lncurses") + make("install") diff --git a/var/spack/repos/builtin/packages/rose/add_spack_compiler_recognition.patch b/var/spack/repos/builtin/packages/rose/add_spack_compiler_recognition.patch new file mode 100644 index 0000000000..ce61ae4e4c --- /dev/null +++ b/var/spack/repos/builtin/packages/rose/add_spack_compiler_recognition.patch @@ -0,0 +1,13 @@ +diff --git a/config/compiler-defs.m4 b/config/compiler-defs.m4 +index d7d85d2..780c8de 100644 +--- a/config/compiler-defs.m4 ++++ b/config/compiler-defs.m4 +@@ -28,7 +28,7 @@ dnl predefined by a specific compiler + # g++|gcc|mpicc|mpic++|mpicxx|mpiCC) + # TOO (2/16/2011): added support for tensilica compilers, assuming they are + # like GCC (they use a GCC front-end) +- g++*|gcc*|mpicc|mpic++|mpicxx|mpiCC|xt-xc++|xt-xcc) ++ cc*|c++*|g++*|gcc*|mpicc|mpic++|mpicxx|mpiCC|xt-xc++|xt-xcc) + BACKEND_GCC_MAJOR=`echo|$BACKEND_CXX_COMPILER -dumpversion | cut -d\. -f1` + BACKEND_GCC_MINOR=`echo|$BACKEND_CXX_COMPILER -dumpversion | cut -d\. -f2` + BACKEND_GCC_PATCHLEVEL=`echo|$BACKEND_CXX_COMPILER -dumpversion | cut -d\. -f3` diff --git a/var/spack/repos/builtin/packages/rose/package.py b/var/spack/repos/builtin/packages/rose/package.py new file mode 100644 index 0000000000..1d7294acab --- /dev/null +++ b/var/spack/repos/builtin/packages/rose/package.py @@ -0,0 +1,39 @@ +#------------------------------------------------------------------------------ +# Author: Justin Too +#------------------------------------------------------------------------------ + +from spack import * + +class Rose(Package): + """A compiler infrastructure to build source-to-source program + transformation and analysis tools. + (Developed at Lawrence Livermore National Lab)""" + + homepage = "http://rosecompiler.org/" + url = "https://github.com/rose-compiler/edg4x-rose" + + version('master', branch='master', git='https://github.com/rose-compiler/edg4x-rose.git') + + patch('add_spack_compiler_recognition.patch') + + depends_on("autoconf@2.69") + depends_on("automake@1.14") + depends_on("libtool@2.4") + depends_on("boost@1.54.0") + depends_on("jdk@8u25-linux-x64") + + def install(self, spec, prefix): + # Bootstrap with autotools + bash = which('bash') + bash('build') + + # Configure, compile & install + with working_dir('rose-build', create=True): + boost = spec['boost'] + + configure = Executable('../configure') + configure("--prefix=" + prefix, + "--with-boost=" + boost.prefix, + "--disable-boost-version-check") + make("install-core") + diff --git a/var/spack/repos/builtin/packages/ruby/package.py b/var/spack/repos/builtin/packages/ruby/package.py new file mode 100644 index 0000000000..6b6242362c --- /dev/null +++ b/var/spack/repos/builtin/packages/ruby/package.py @@ -0,0 +1,41 @@ +from spack import * +import spack +import os + +class Ruby(Package): + """A dynamic, open source programming language with a focus on + simplicity and productivity.""" + + homepage = "https://www.ruby-lang.org/" + url = "http://cache.ruby-lang.org/pub/ruby/2.2/ruby-2.2.0.tar.gz" + + extendable = True + + version('2.2.0', 'cd03b28fd0b555970f5c4fd481700852') + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix) + + make() + make("install") + + def setup_dependent_environment(self, module, spec, ext_spec): + """Called before ruby modules' install() methods. Sets GEM_HOME + and GEM_PATH to values appropriate for the package being built. + + In most cases, extensions will only need to have one line:: + + gem('install', '.gem') + """ + # Ruby extension builds have global ruby and gem functions + module.ruby = Executable(join_path(spec.prefix.bin, 'ruby')) + module.gem = Executable(join_path(spec.prefix.bin, 'gem')) + + # Set GEM_PATH to include dependent gem directories + ruby_paths = [] + for d in ext_spec.traverse(): + if d.package.extends(self.spec): + ruby_paths.append(d.prefix) + os.environ['GEM_PATH'] = ':'.join(ruby_paths) + # The actual installation path for this gem + os.environ['GEM_HOME'] = ext_spec.prefix diff --git a/var/spack/repos/builtin/packages/samtools/package.py b/var/spack/repos/builtin/packages/samtools/package.py new file mode 100644 index 0000000000..72900398d8 --- /dev/null +++ b/var/spack/repos/builtin/packages/samtools/package.py @@ -0,0 +1,18 @@ +from spack import * + +class Samtools(Package): + """SAM Tools provide various utilities for manipulating alignments in the SAM format, + including sorting, merging, indexing and generating + alignments in a per-position format""" + + homepage = "www.htslib.org" + version('1.2','988ec4c3058a6ceda36503eebecd4122',url = "https://github.com/samtools/samtools/releases/download/1.2/samtools-1.2.tar.bz2") + + depends_on("zlib") + depends_on("mpc") + parallel=False + patch("samtools1.2.patch",level=0) + + def install(self, spec, prefix): + make("prefix=%s" % prefix, "install") + diff --git a/var/spack/repos/builtin/packages/samtools/samtools1.2.patch b/var/spack/repos/builtin/packages/samtools/samtools1.2.patch new file mode 100644 index 0000000000..ead3ab4e2c --- /dev/null +++ b/var/spack/repos/builtin/packages/samtools/samtools1.2.patch @@ -0,0 +1,20 @@ +--- Makefile 2015-02-03 08:27:34.000000000 -0800 ++++ Makefile.new 2015-07-21 10:38:27.881406892 -0700 +@@ -26,7 +26,7 @@ + CFLAGS = -g -Wall -O2 + LDFLAGS = + LDLIBS = +-DFLAGS= -D_FILE_OFFSET_BITS=64 -D_LARGEFILE64_SOURCE -D_CURSES_LIB=1 ++DFLAGS= -D_FILE_OFFSET_BITS=64 -D_LARGEFILE64_SOURCE -D_CURSES_LIB=0 + LOBJS= bam_aux.o bam.o bam_import.o sam.o \ + sam_header.o bam_plbuf.o + AOBJS= bam_index.o bam_plcmd.o sam_view.o \ +@@ -37,7 +37,7 @@ + faidx.o stats.o stats_isize.o bam_flags.o bam_split.o \ + bam_tview.o bam_tview_curses.o bam_tview_html.o bam_lpileup.o + INCLUDES= -I. -I$(HTSDIR) +-LIBCURSES= -lcurses # -lXCurses ++#LIBCURSES= -lcurses # -lXCurses + + prefix = /usr/local + exec_prefix = $(prefix) diff --git a/var/spack/repos/builtin/packages/scalasca/package.py b/var/spack/repos/builtin/packages/scalasca/package.py new file mode 100644 index 0000000000..cf7a40c1f5 --- /dev/null +++ b/var/spack/repos/builtin/packages/scalasca/package.py @@ -0,0 +1,65 @@ +# FIXME: Add copyright + +from spack import * + +class Scalasca(Package): + """Scalasca is a software tool that supports the performance optimization + of parallel programs by measuring and analyzing their runtime behavior. + The analysis identifies potential performance bottlenecks - in + particular those concerning communication and synchronization - and + offers guidance in exploring their causes.""" + + # FIXME: add a proper url for your package's homepage here. + homepage = "http://www.scalasca.org" + url = "http://apps.fz-juelich.de/scalasca/releases/scalasca/2.1/dist/scalasca-2.1.tar.gz" + + version('2.1', 'bab9c2b021e51e2ba187feec442b96e6', + url = 'http://apps.fz-juelich.de/scalasca/releases/scalasca/2.1/dist/scalasca-2.1.tar.gz' ) + + depends_on("mpi") + depends_on("otf2@1.4") + depends_on("cube@4.2.3") + + backend_user_provided = """\ +CC=cc +CXX=c++ +F77=f77 +FC=f90 +CFLAGS=-fPIC +CXXFLAGS=-fPIC +""" + frontend_user_provided = """\ +CC_FOR_BUILD=cc +CXX_FOR_BUILD=c++ +F77_FOR_BUILD=f70 +FC_FOR_BUILD=f90 +CFLAGS_FOR_BUILD=-fPIC +CXXFLAGS_FOR_BUILD=-fPIC +""" + mpi_user_provided = """\ +MPICC=mpicc +MPICXX=mpicxx +MPIF77=mpif77 +MPIFC=mpif90 +MPI_CFLAGS=-fPIC +MPI_CXXFLAGS=-fPIC +""" + + def install(self, spec, prefix): + configure_args = ["--prefix=%s" % prefix, + "--with-custom-compilers", + "--with-otf2=%s" % spec['otf2'].prefix.bin, + "--with-cube=%s" % spec['cube'].prefix.bin, + "--enable-shared"] + + configure(*configure_args) + + make() + make("install") + + # FIXME: Modify the configure line to suit your build system here. + configure("--prefix=%s" % prefix) + + # FIXME: Add logic to build and install here + make() + make("install") diff --git a/var/spack/repos/builtin/packages/scorep/package.py b/var/spack/repos/builtin/packages/scorep/package.py new file mode 100644 index 0000000000..f013bd1cbb --- /dev/null +++ b/var/spack/repos/builtin/packages/scorep/package.py @@ -0,0 +1,74 @@ +# FIXME: Add copyright statement + +from spack import * + +class Scorep(Package): + """The Score-P measurement infrastructure is a highly scalable and + easy-to-use tool suite for profiling, event tracing, and online + analysis of HPC applications.""" + + # FIXME: add a proper url for your package's homepage here. + homepage = "http://www.vi-hps.org/projects/score-p" + url = "http://www.vi-hps.org/upload/packages/scorep/scorep-1.2.3.tar.gz" + + version('1.3', '9db6f957b7f51fa01377a9537867a55c', + url = 'http://www.vi-hps.org/upload/packages/scorep/scorep-1.3.tar.gz') + + version('1.2.3', '4978084e7cbd05b94517aa8beaea0817') + + depends_on("mpi") + depends_on("papi") + # depends_on("otf2@1.2:1.2.1") # only Score-P 1.2.x + depends_on("otf2") + depends_on("opari2") + depends_on("cube@4.2:4.2.3") + + backend_user_provided = """\ +CC=cc +CXX=c++ +F77=f77 +FC=f90 +CFLAGS=-fPIC +CXXFLAGS=-fPIC +""" + frontend_user_provided = """\ +CC_FOR_BUILD=cc +CXX_FOR_BUILD=c++ +F77_FOR_BUILD=f70 +FC_FOR_BUILD=f90 +CFLAGS_FOR_BUILD=-fPIC +CXXFLAGS_FOR_BUILD=-fPIC +""" + mpi_user_provided = """\ +MPICC=mpicc +MPICXX=mpicxx +MPIF77=mpif77 +MPIFC=mpif90 +MPI_CFLAGS=-fPIC +MPI_CXXFLAGS=-fPIC +""" + + def install(self, spec, prefix): + # Use a custom compiler configuration, otherwise the score-p + # build system messes with spack's compiler settings. + # Create these three files in the build directory + with open("platform-backend-user-provided", "w") as backend_file: + backend_file.write(self.backend_user_provided) + with open("platform-frontend-user-provided", "w") as frontend_file: + frontend_file.write(self.frontend_user_provided) + with open("platform-mpi-user-provided", "w") as mpi_file: + mpi_file.write(self.mpi_user_provided) + + configure_args = ["--prefix=%s" % prefix, + "--with-custom-compilers", + "--with-otf2=%s" % spec['otf2'].prefix.bin, + "--with-opari2=%s" % spec['opari2'].prefix.bin, + "--with-cube=%s" % spec['cube'].prefix.bin, + "--with-papi-header=%s" % spec['papi'].prefix.include, + "--with-papi-lib=%s" % spec['papi'].prefix.lib, + "--enable-shared"] + + configure(*configure_args) + + make() + make("install") diff --git a/var/spack/repos/builtin/packages/scotch/package.py b/var/spack/repos/builtin/packages/scotch/package.py new file mode 100644 index 0000000000..79289ff2ad --- /dev/null +++ b/var/spack/repos/builtin/packages/scotch/package.py @@ -0,0 +1,40 @@ +from spack import * +import glob +import os + +class Scotch(Package): + """Scotch is a software package for graph and mesh/hypergraph + partitioning, graph clustering, and sparse matrix ordering.""" + homepage = "http://www.labri.fr/perso/pelegrin/scotch/" + url = "http://gforge.inria.fr/frs/download.php/file/34099/scotch_6.0.3.tar.gz" + list_url = "http://gforge.inria.fr/frs/?group_id=248" + + version('6.0.3', '10b0cc0f184de2de99859eafaca83cfc') + + depends_on('mpi') + + + def patch(self): + with working_dir('src/Make.inc'): + makefiles = glob.glob('Makefile.inc.x86-64_pc_linux2*') + filter_file(r'^CCS\s*=.*$', 'CCS = cc', *makefiles) + filter_file(r'^CCD\s*=.*$', 'CCD = cc', *makefiles) + + + def install(self, spec, prefix): + # Currently support gcc and icc on x86_64 (maybe others with + # vanilla makefile) + makefile = 'Make.inc/Makefile.inc.x86-64_pc_linux2' + if spec.satisfies('%icc'): + makefile += '.icc' + + with working_dir('src'): + force_symlink(makefile, 'Makefile.inc') + for app in ('scotch', 'ptscotch'): + make(app) + + install_tree('bin', prefix.bin) + install_tree('lib', prefix.lib) + install_tree('include', prefix.include) + install_tree('man/man1', prefix.share_man1) + diff --git a/var/spack/repos/builtin/packages/scr/package.py b/var/spack/repos/builtin/packages/scr/package.py new file mode 100644 index 0000000000..9fb758f072 --- /dev/null +++ b/var/spack/repos/builtin/packages/scr/package.py @@ -0,0 +1,44 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + +class Scr(Package): + """SCR caches checkpoint data in storage on the compute nodes of a + Linux cluster to provide a fast, scalable checkpoint/restart + capability for MPI codes""" + + homepage = "https://computation.llnl.gov/project/scr/" + + depends_on("mpi") +# depends_on("dtcmp") + + version('1.1-7', 'a5930e9ab27d1b7049447c2fd7734ebd', url='http://downloads.sourceforge.net/project/scalablecr/releases/scr-1.1-7.tar.gz') + version('1.1.8', '6a0f11ad18e27fcfc00a271ff587b06e', url='https://github.com/hpc/scr/releases/download/v1.1.8/scr-1.1.8.tar.gz') + + def install(self, spec, prefix): + configure("--prefix=" + prefix, + "--with-scr-config-file=" + prefix + "/etc/scr.conf") + make() + make("install") diff --git a/var/spack/repos/builtin/packages/silo/package.py b/var/spack/repos/builtin/packages/silo/package.py new file mode 100644 index 0000000000..9eda11df15 --- /dev/null +++ b/var/spack/repos/builtin/packages/silo/package.py @@ -0,0 +1,19 @@ +from spack import * + +class Silo(Package): + """Silo is a library for reading and writing a wide variety of scientific data to binary, disk files.""" + + homepage = "http://wci.llnl.gov/simulation/computer-codes/silo" + url = "https://wci.llnl.gov/content/assets/docs/simulation/computer-codes/silo/silo-4.8/silo-4.8.tar.gz" + + #version('4.9', 'a83eda4f06761a86726e918fc55e782a') + version('4.8', 'b1cbc0e7ec435eb656dc4b53a23663c9') + + depends_on("hdf5@:1.8.12") + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix, + "--with-hdf5=%s" %spec['hdf5'].prefix) + + make() + make("install") diff --git a/var/spack/repos/builtin/packages/snappy/package.py b/var/spack/repos/builtin/packages/snappy/package.py new file mode 100644 index 0000000000..c8f9ceef7d --- /dev/null +++ b/var/spack/repos/builtin/packages/snappy/package.py @@ -0,0 +1,15 @@ +import os +from spack import * + +class Snappy(Package): + """A fast compressor/decompressor: https://code.google.com/p/snappy""" + + homepage = "https://code.google.com/p/snappy" + url = "https://github.com/google/snappy/releases/download/1.1.3/snappy-1.1.3.tar.gz" + + version('1.1.3', '7358c82f133dc77798e4c2062a749b73') + + def install(self, spec, prefix): + configure("--prefix=" + prefix) + make() + make("install") diff --git a/var/spack/repos/builtin/packages/spindle/package.py b/var/spack/repos/builtin/packages/spindle/package.py new file mode 100644 index 0000000000..06a1e14284 --- /dev/null +++ b/var/spack/repos/builtin/packages/spindle/package.py @@ -0,0 +1,44 @@ +############################################################################## +# Copyright (c) 2014, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Matthew LeGendre, legendre1@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + +class Spindle(Package): + """Spindle improves the library-loading performance of dynamically + linked HPC applications. Without Spindle large MPI jobs can + overload on a shared file system when loading dynamically + linked libraries, causing site-wide performance problems. + """ + homepage = "https://computation.llnl.gov/project/spindle/" + url = "https://github.com/hpc/Spindle/archive/v0.8.1.tar.gz" + list_url = "https://github.com/hpc/Spindle/releases" + + version('0.8.1', 'f11793a6b9d8df2cd231fccb2857d912') + + depends_on("launchmon") + + def install(self, spec, prefix): + configure("--prefix=" + prefix) + make() + make("install") diff --git a/var/spack/repos/builtin/packages/sqlite/package.py b/var/spack/repos/builtin/packages/sqlite/package.py new file mode 100644 index 0000000000..734b0b6cb6 --- /dev/null +++ b/var/spack/repos/builtin/packages/sqlite/package.py @@ -0,0 +1,40 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + +class Sqlite(Package): + """SQLite3 is an SQL database engine in a C library. Programs that + link the SQLite3 library can have SQL database access without + running a separate RDBMS process. + """ + homepage = "www.sqlite.org" + + version('3.8.5', '0544ef6d7afd8ca797935ccc2685a9ed', + url='http://www.sqlite.org/2014/sqlite-autoconf-3080500.tar.gz') + + def install(self, spec, prefix): + configure("--prefix=" + prefix) + make() + make("install") diff --git a/var/spack/repos/builtin/packages/stat/configure_mpicxx.patch b/var/spack/repos/builtin/packages/stat/configure_mpicxx.patch new file mode 100644 index 0000000000..e09056d95c --- /dev/null +++ b/var/spack/repos/builtin/packages/stat/configure_mpicxx.patch @@ -0,0 +1,19 @@ +commit 07ab6e565f939c54fff6580fc8463ea61662871a +Author: Gregory L. Lee +Date: Tue May 20 14:53:35 2014 -0700 + + re-boostrap to update configure + +diff --git a/configure b/configure +index 6c4af7d..30901ea 100755 +--- a/configure ++++ b/configure +@@ -15529,7 +15529,7 @@ fi + done + test -n "$MPICC" || MPICC="$CC" + +- for ac_prog in mpig++ mpiicpc mpxlC mpixlC ++ for ac_prog in mpig++ mpiCC mpicxx mpiicpc mpxlC mpixlC + do + # Extract the first word of "$ac_prog", so it can be a program name with args. + set dummy $ac_prog; ac_word=$2 diff --git a/var/spack/repos/builtin/packages/stat/package.py b/var/spack/repos/builtin/packages/stat/package.py new file mode 100644 index 0000000000..5d81e62731 --- /dev/null +++ b/var/spack/repos/builtin/packages/stat/package.py @@ -0,0 +1,40 @@ +from spack import * + +class Stat(Package): + """Library to create, manipulate, and export graphs Graphlib.""" + homepage = "http://paradyn.org/STAT/STAT.html" + url = "https://github.com/lee218llnl/stat/archive/v2.0.0.tar.gz" + + version('2.2.0', '26bd69dd57a15afdd5d0ebdb0b7fb6fc') + version('2.1.0', 'ece26beaf057aa9134d62adcdda1ba91') + version('2.0.0', 'c7494210b0ba26b577171b92838e1a9b') + + variant('dysect', default=False, description="enable DySectAPI") + + depends_on('libelf') + depends_on('libdwarf') + depends_on('dyninst') + depends_on('graphlib') + depends_on('graphviz') + depends_on('launchmon') + depends_on('mrnet') + + patch('configure_mpicxx.patch', when='@2.1.0') + + def install(self, spec, prefix): + configure_args = [ + "--enable-gui", + "--prefix=%s" % prefix, + "--disable-examples", # Examples require MPI: avoid this dependency. + "--with-launchmon=%s" % spec['launchmon'].prefix, + "--with-mrnet=%s" % spec['mrnet'].prefix, + "--with-graphlib=%s" % spec['graphlib'].prefix, + "--with-stackwalker=%s" % spec['dyninst'].prefix, + "--with-libdwarf=%s" % spec['libdwarf'].prefix + ] + if '+dysect' in spec: + configure_args.append('--enable-dysectapi') + configure(*configure_args) + + make(parallel=False) + make("install") diff --git a/var/spack/repos/builtin/packages/sundials/package.py b/var/spack/repos/builtin/packages/sundials/package.py new file mode 100644 index 0000000000..8b784c8c3c --- /dev/null +++ b/var/spack/repos/builtin/packages/sundials/package.py @@ -0,0 +1,39 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + +class Sundials(Package): + """SUNDIALS (SUite of Nonlinear and DIfferential/ALgebraic equation Solvers)""" + homepage = "http://computation.llnl.gov/casc/sundials/" + url = "http://computation.llnl.gov/casc/sundials/download/code/sundials-2.5.0.tar.gz" + + version('2.5.0', 'aba8b56eec600de3109cfb967aa3ba0f') + + depends_on("mpi") + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix) + make() + make("install") diff --git a/var/spack/repos/builtin/packages/swig/package.py b/var/spack/repos/builtin/packages/swig/package.py new file mode 100644 index 0000000000..ee536d7063 --- /dev/null +++ b/var/spack/repos/builtin/packages/swig/package.py @@ -0,0 +1,46 @@ +############################################################################## +# Copyright (c) 2014, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Matthew LeGendre, legendre1@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + +class Swig(Package): + """SWIG is an interface compiler that connects programs written in + C and C++ with scripting languages such as Perl, Python, Ruby, + and Tcl. It works by taking the declarations found in C/C++ + header files and using them to generate the wrapper code that + scripting languages need to access the underlying C/C++ + code. In addition, SWIG provides a variety of customization + features that let you tailor the wrapping process to suit your + application.""" + homepage = "http://www.swig.org" + url = "http://prdownloads.sourceforge.net/swig/swig-3.0.2.tar.gz" + + version('3.0.2', '62f9b0d010cef36a13a010dc530d0d41') + + depends_on('pcre') + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix) + make() + make("install") diff --git a/var/spack/repos/builtin/packages/task/package.py b/var/spack/repos/builtin/packages/task/package.py new file mode 100644 index 0000000000..07f44cc45b --- /dev/null +++ b/var/spack/repos/builtin/packages/task/package.py @@ -0,0 +1,20 @@ +from spack import * + +class Task(Package): + """Feature-rich console based todo list manager""" + homepage = "http://www.taskwarrior.org" + url = "http://taskwarrior.org/download/task-2.4.4.tar.gz" + + version('2.4.4', '517450c4a23a5842df3e9905b38801b3') + + depends_on("gnutls") + depends_on("libuuid") + # depends_on("gcc@4.8:") + + def install(self, spec, prefix): + with working_dir('spack-build', create=True): + cmake('-DCMAKE_BUILD_TYPE=release', + '..', + *std_cmake_args) + make() + make("install") diff --git a/var/spack/repos/builtin/packages/taskd/package.py b/var/spack/repos/builtin/packages/taskd/package.py new file mode 100644 index 0000000000..66bc0cb484 --- /dev/null +++ b/var/spack/repos/builtin/packages/taskd/package.py @@ -0,0 +1,20 @@ +from spack import * + +class Taskd(Package): + """TaskWarrior task synchronization daemon""" + # FIXME: add a proper url for your package's homepage here. + homepage = "http://www.taskwarrior.org" + url = "http://taskwarrior.org/download/taskd-1.1.0.tar.gz" + + version('1.1.0', 'ac855828c16f199bdbc45fbc227388d0') + + depends_on("libuuid") + depends_on("gnutls") + + def install(self, spec, prefix): + with working_dir('spack-build', create=True): + cmake('-DCMAKE_BUILD_TYPE=release', + '..', + *std_cmake_args) + make() + make("install") diff --git a/var/spack/repos/builtin/packages/tau/package.py b/var/spack/repos/builtin/packages/tau/package.py new file mode 100644 index 0000000000..048fac80aa --- /dev/null +++ b/var/spack/repos/builtin/packages/tau/package.py @@ -0,0 +1,36 @@ +from spack import * + +import os +from llnl.util.filesystem import join_path + +class Tau(Package): + """A portable profiling and tracing toolkit for performance + analysis of parallel programs written in Fortran, C, C++, UPC, + Java, Python.""" + homepage = "http://www.cs.uoregon.edu/research/tau" + url = "http://www.cs.uoregon.edu/research/paracomp/tau/tauprofile/dist/tau-2.23.1.tar.gz" + + version('2.23.1', '6593b47ae1e7a838e632652f0426fe72') + + def install(self, spec, prefix): + # TAU isn't happy with directories that have '@' in the path. Sigh. + change_sed_delimiter('@', ';', 'configure') + change_sed_delimiter('@', ';', 'utils/FixMakefile') + change_sed_delimiter('@', ';', 'utils/FixMakefile.sed.default') + + # After that, it's relatively standard. + configure("-prefix=%s" % prefix) + make("install") + + # Link arch-specific directories into prefix since there is + # only one arch per prefix the way spack installs. + self.link_tau_arch_dirs() + + + def link_tau_arch_dirs(self): + for subdir in os.listdir(self.prefix): + for d in ('bin', 'lib'): + src = join_path(self.prefix, subdir, d) + dest = join_path(self.prefix, d) + if os.path.isdir(src) and not os.path.exists(dest): + os.symlink(join_path(subdir, d), dest) diff --git a/var/spack/repos/builtin/packages/tcl/package.py b/var/spack/repos/builtin/packages/tcl/package.py new file mode 100644 index 0000000000..529adf7788 --- /dev/null +++ b/var/spack/repos/builtin/packages/tcl/package.py @@ -0,0 +1,22 @@ +from spack import * + +class Tcl(Package): + """Tcl (Tool Command Language) is a very powerful but easy to + learn dynamic programming language, suitable for a very wide + range of uses, including web and desktop applications, + networking, administration, testing and many more. Open source + and business-friendly, Tcl is a mature yet evolving language + that is truly cross platform, easily deployed and highly + extensible.""" + homepage = "http://www.tcl.tk" + + version('8.6.3', 'db382feca91754b7f93da16dc4cdad1f', + url="http://prdownloads.sourceforge.net/tcl/tcl8.6.3-src.tar.gz") + + depends_on('zlib') + + def install(self, spec, prefix): + with working_dir('unix'): + configure("--prefix=%s" % prefix) + make() + make("install") diff --git a/var/spack/repos/builtin/packages/the_silver_searcher/package.py b/var/spack/repos/builtin/packages/the_silver_searcher/package.py new file mode 100644 index 0000000000..e4020b6766 --- /dev/null +++ b/var/spack/repos/builtin/packages/the_silver_searcher/package.py @@ -0,0 +1,17 @@ +from spack import * + +class TheSilverSearcher(Package): + """Fast recursive grep alternative""" + homepage = "http://geoff.greer.fm/ag/" + url = "http://geoff.greer.fm/ag/releases/the_silver_searcher-0.30.0.tar.gz" + + version('0.30.0', '95e2e7859fab1156c835aff7413481db') + + depends_on('pcre') + depends_on('xz') + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix) + + make() + make("install") diff --git a/var/spack/repos/builtin/packages/thrift/package.py b/var/spack/repos/builtin/packages/thrift/package.py new file mode 100644 index 0000000000..0e15052f64 --- /dev/null +++ b/var/spack/repos/builtin/packages/thrift/package.py @@ -0,0 +1,44 @@ +from spack import * + +class Thrift(Package): + """The Apache Thrift software framework, for scalable cross-language services + development, combines a software stack with a code generation engine to build + services that work efficiently and seamlessly between C++, Java, Python, PHP, + Ruby, Erlang, Perl, Haskell, C#, Cocoa, JavaScript, Node.js, Smalltalk, OCaml + and Delphi and other languages.""" + + homepage = "http://thrift.apache.org" + url = "http://apache.mirrors.ionfish.org/thrift/0.9.2/thrift-0.9.2.tar.gz" + + version('0.9.2', '89f63cc4d0100912f4a1f8a9dee63678') + + extends("python") + + depends_on("autoconf") + depends_on("automake") + depends_on("bison") + depends_on("boost") + depends_on("flex") + depends_on("jdk") + depends_on("libtool") + depends_on("openssl") + depends_on("python") + + # Compilation fails for most languages, fortunately cpp installs fine + # All other languages (yes, including C) are omitted until someone needs them + def install(self, spec, prefix): + env["PY_PREFIX"] = prefix + env["JAVA_PREFIX"] = prefix + + configure("--prefix=%s" % prefix, + "--with-boost=%s" % spec['boost'].prefix, + "--with-c=no", + "--with-go=no", + "--with-python=yes", + "--with-lua=no", + "--with-php=no", + "--with-qt4=no", + "--enable-tests=no") + + make() + make("install") diff --git a/var/spack/repos/builtin/packages/tk/package.py b/var/spack/repos/builtin/packages/tk/package.py new file mode 100644 index 0000000000..96736f6f95 --- /dev/null +++ b/var/spack/repos/builtin/packages/tk/package.py @@ -0,0 +1,22 @@ +from spack import * + +class Tk(Package): + """Tk is a graphical user interface toolkit that takes developing + desktop applications to a higher level than conventional + approaches. Tk is the standard GUI not only for Tcl, but for + many other dynamic languages, and can produce rich, native + applications that run unchanged across Windows, Mac OS X, Linux + and more.""" + homepage = "http://www.tcl.tk" + url = "http://prdownloads.sourceforge.net/tcl/tk8.6.3-src.tar.gz" + + version('src', '85ca4dbf4dcc19777fd456f6ee5d0221') + + depends_on("tcl") + + def install(self, spec, prefix): + with working_dir('unix'): + configure("--prefix=%s" % prefix, + "--with-tcl=%s" % spec['tcl'].prefix.lib) + make() + make("install") diff --git a/var/spack/repos/builtin/packages/tmux/package.py b/var/spack/repos/builtin/packages/tmux/package.py new file mode 100644 index 0000000000..23d36db427 --- /dev/null +++ b/var/spack/repos/builtin/packages/tmux/package.py @@ -0,0 +1,24 @@ +from spack import * + +class Tmux(Package): + """tmux is a terminal multiplexer. What is a terminal multiplexer? It lets + you switch easily between several programs in one terminal, detach them (they + keep running in the background) and reattach them to a different terminal. And + do a lot more. + """ + + homepage = "http://tmux.sourceforge.net" + url = "http://downloads.sourceforge.net/project/tmux/tmux/tmux-1.9/tmux-1.9a.tar.gz" + + version('1.9a', 'b07601711f96f1d260b390513b509a2d') + + depends_on('libevent') + depends_on('ncurses') + + def install(self, spec, prefix): + configure( + "--prefix=%s" % prefix, + "PKG_CONFIG_PATH=%s:%s" % (spec['libevent'].prefix, spec['ncurses'].prefix)) + + make() + make("install") diff --git a/var/spack/repos/builtin/packages/tmuxinator/package.py b/var/spack/repos/builtin/packages/tmuxinator/package.py new file mode 100644 index 0000000000..26c061cbd6 --- /dev/null +++ b/var/spack/repos/builtin/packages/tmuxinator/package.py @@ -0,0 +1,17 @@ +from spack import * + +class Tmuxinator(Package): + """A session configuration creator and manager for tmux""" + homepage = "https://github.com/tmuxinator/tmuxinator" + url = "https://github.com/tmuxinator/tmuxinator" + + version('0.6.11', + git='https://github.com/tmuxinator/tmuxinator', + tag='v0.6.11') + + extends('ruby') + + def install(self, spec, prefix): + gem('build', 'tmuxinator.gemspec') + gem('install', 'tmuxinator-{}.gem'.format(self.version)) + diff --git a/var/spack/repos/builtin/packages/trilinos/package.py b/var/spack/repos/builtin/packages/trilinos/package.py new file mode 100644 index 0000000000..7c43f796a4 --- /dev/null +++ b/var/spack/repos/builtin/packages/trilinos/package.py @@ -0,0 +1,50 @@ +from spack import * + + +class Trilinos(Package): + """ + The Trilinos Project is an effort to develop algorithms and enabling technologies within an object-oriented + software framework for the solution of large-scale, complex multi-physics engineering and scientific problems. + A unique design feature of Trilinos is its focus on packages. + """ + homepage = "https://trilinos.org/" + url = "http://trilinos.csbsju.edu/download/files/trilinos-12.2.1-Source.tar.gz" + + version('12.2.1', '6161926ea247863c690e927687f83be9') + version('12.0.1', 'bd99741d047471e127b8296b2ec08017') + version('11.14.3', '2f4f83f8333e4233c57d0f01c4b57426') + version('11.14.2', 'a43590cf896c677890d75bfe75bc6254') + version('11.14.1', '40febc57f76668be8b6a77b7607bb67f') + + variant('mpi', default=True, description='Add a dependency on MPI and enables MPI dependent packages') + + # Everything should be compiled with -fpic + depends_on('blas') + depends_on('lapack') + depends_on('boost') + depends_on('netcdf') + depends_on('matio') + depends_on('glm') + depends_on('swig') + depends_on('mpi', when='+mpi') + + def install(self, spec, prefix): + + options = [ + '-DTrilinos_ENABLE_ALL_PACKAGES:BOOL=ON', + '-DTrilinos_ENABLE_TESTS:BOOL=OFF', + '-DTrilinos_ENABLE_EXAMPLES:BOOL=OFF', + '-DBUILD_SHARED_LIBS:BOOL=ON', + '-DBLAS_LIBRARY_DIRS:PATH=%s' % spec['blas'].prefix, + '-DLAPACK_LIBRARY_DIRS:PATH=%s' % spec['lapack'].prefix + ] + if '+mpi' in spec: + mpi_options = ['-DTPL_ENABLE_MPI:BOOL=ON'] + options.extend(mpi_options) + + # -DCMAKE_INSTALL_PREFIX and all the likes... + options.extend(std_cmake_args) + with working_dir('spack-build', create=True): + cmake('..', *options) + make() + make('install') diff --git a/var/spack/repos/builtin/packages/uncrustify/package.py b/var/spack/repos/builtin/packages/uncrustify/package.py new file mode 100644 index 0000000000..d3f2d1b473 --- /dev/null +++ b/var/spack/repos/builtin/packages/uncrustify/package.py @@ -0,0 +1,14 @@ +from spack import * + +class Uncrustify(Package): + """Source Code Beautifier for C, C++, C#, ObjectiveC, D, Java, Pawn and VALA""" + + homepage = "http://uncrustify.sourceforge.net/" + url = "http://downloads.sourceforge.net/project/uncrustify/uncrustify/uncrustify-0.61/uncrustify-0.61.tar.gz" + + version('0.61', 'b6140106e74c64e831d0b1c4b6cf7727') + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix) + make() + make("install") diff --git a/var/spack/repos/builtin/packages/util-linux/package.py b/var/spack/repos/builtin/packages/util-linux/package.py new file mode 100644 index 0000000000..cb7ceabf57 --- /dev/null +++ b/var/spack/repos/builtin/packages/util-linux/package.py @@ -0,0 +1,20 @@ +from spack import * +import os + +class UtilLinux(Package): + """Util-linux is a suite of essential utilities for any Linux system.""" + + homepage = "http://freecode.com/projects/util-linux" + url = "https://www.kernel.org/pub/linux/utils/util-linux/v2.25/util-linux-2.25.tar.gz" + + version('2.25', 'f6d7fc6952ec69c4dc62c8d7c59c1d57') + + depends_on("python@2.7:") + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix, + "PKG_CONFIG_PATH=%s/pkgconfig" % spec['python'].prefix.lib, + "--disable-use-tty-group") + + make() + make("install") diff --git a/var/spack/repos/builtin/packages/vim/package.py b/var/spack/repos/builtin/packages/vim/package.py new file mode 100644 index 0000000000..4099b3257f --- /dev/null +++ b/var/spack/repos/builtin/packages/vim/package.py @@ -0,0 +1,83 @@ +from spack import * + +class Vim(Package): + """Vim is a highly configurable text editor built to enable efficient text + editing. It is an improved version of the vi editor distributed with most + UNIX systems. Vim is often called a "programmer's editor," and so useful + for programming that many consider it an entire IDE. It's not just for + programmers, though. Vim is perfect for all kinds of text editing, from + composing email to editing configuration files. + """ + + homepage = "http://www.vim.org" + url = "ftp://ftp.vim.org/pub/vim/unix/vim-7.4.tar.bz2" + list_url = "http://ftp.vim.org/pub/vim/unix/" + + version('7.4', '607e135c559be642f210094ad023dc65') + version('7.3', '5b9510a17074e2b37d8bb38ae09edbf2') + version('7.2', 'f0901284b338e448bfd79ccca0041254') + version('7.1', '44c6b4914f38d6f9aa959640b89da329') + version('7.0', '4ca69757678272f718b1041c810d82d8') + version('6.4', '774c14d93ce58674b3b2c880edd12d77') + version('6.3', '821fda8f14d674346b87e3ef9cb96389') + version('6.2', 'c49d360bbd069d00e2a57804f2a123d9') + version('6.1.405', 'd220ff58f2c72ed606e6d0297c2f2a7c') + version('6.1', '7fd0f915adc7c0dab89772884268b030') + version('6.0', '9d9ca84d489af6b3f54639dd97af3774') + + feature_sets = ('huge', 'big', 'normal', 'small', 'tiny') + for fs in feature_sets: + variant(fs, default=False, description="Use '%s' feature set" % fs) + + variant('python', default=False, description="build with Python") + depends_on('python', when='+python') + + variant('ruby', default=False, description="build with Ruby") + depends_on('ruby', when='+ruby') + + variant('cscope', default=False, description="build with cscope support") + depends_on('cscope', when='+cscope') + + variant('gui', default=False, description="build with gui (gvim)") + # virtual dependency? + + def install(self, spec, prefix): + feature_set = None + for fs in self.feature_sets: + if "+" + fs in spec: + if feature_set is not None: + tty.error("Only one feature set allowed, both %s and %s specified" + % (feature_set, fs)) + feature_set = fs + if '+gui' in spec: + if feature_set is not None: + if feature_set is not 'huge': + tty.error("+gui variant requires 'huge' feature set, %s was specified" + % feature_set) + feature_set = 'huge' + if feature_set is None: + feature_set = 'normal' + + configure_args = [] + configure_args.append("--with-features=" + feature_set) + + if '+python' in spec: + configure_args.append("--enable-pythoninterp=yes") + else: + configure_args.append("--enable-pythoninterp=dynamic") + + if '+ruby' in spec: + configure_args.append("--enable-rubyinterp=yes") + else: + configure_args.append("--enable-rubyinterp=dynamic") + + if '+gui' in spec: + configure_args.append("--enable-gui=auto") + + if '+cscope' in spec: + configure_args.append("--enable-cscope") + + configure("--prefix=%s" % prefix, *configure_args) + + make() + make("install") diff --git a/var/spack/repos/builtin/packages/vtk/package.py b/var/spack/repos/builtin/packages/vtk/package.py new file mode 100644 index 0000000000..4a27a8fedb --- /dev/null +++ b/var/spack/repos/builtin/packages/vtk/package.py @@ -0,0 +1,40 @@ +from spack import * + +class Vtk(Package): + """The Visualization Toolkit (VTK) is an open-source, freely + available software system for 3D computer graphics, image + processing and visualization. """ + homepage = "http://www.vtk.org" + url = "http://www.vtk.org/files/release/6.1/VTK-6.1.0.tar.gz" + + version('6.1.0', '25e4dfb3bad778722dcaec80cd5dab7d') + + depends_on("qt") + + def install(self, spec, prefix): + with working_dir('spack-build', create=True): + cmake_args = [ + "..", + "-DBUILD_SHARED_LIBS=ON", + # Disable wrappers for other languages. + "-DVTK_WRAP_PYTHON=OFF", + "-DVTK_WRAP_JAVA=OFF", + "-DVTK_WRAP_TCL=OFF"] + cmake_args.extend(std_cmake_args) + + # Enable Qt support here. + cmake_args.extend([ + "-DQT_QMAKE_EXECUTABLE:PATH=%s/qmake" % spec['qt'].prefix.bin, + "-DVTK_Group_Qt:BOOL=ON", + # Ignore webkit because it's hard to build w/Qt + "-DVTK_Group_Qt=OFF", + "-DModule_vtkGUISupportQt:BOOL=ON", + "-DModule_vtkGUISupportQtOpenGL:BOOL=ON" + ]) + + if spec['qt'].satisfies('@5'): + cmake_args.append("-DVTK_QT_VERSION:STRING=5") + + cmake(*cmake_args) + make() + make("install") diff --git a/var/spack/repos/builtin/packages/wget/package.py b/var/spack/repos/builtin/packages/wget/package.py new file mode 100644 index 0000000000..c8fd025122 --- /dev/null +++ b/var/spack/repos/builtin/packages/wget/package.py @@ -0,0 +1,21 @@ +from spack import * + +class Wget(Package): + """GNU Wget is a free software package for retrieving files using + HTTP, HTTPS and FTP, the most widely-used Internet protocols. It + is a non-interactive commandline tool, so it may easily be called + from scripts, cron jobs, terminals without X-Windows support, + etc.""" + + homepage = "http://www.gnu.org/software/wget/" + url = "http://ftp.gnu.org/gnu/wget/wget-1.16.tar.xz" + + version('1.16', 'fe102975ab3a6c049777883f1bb9ad07') + + depends_on("openssl") + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix, + "--with-ssl=openssl") + make() + make("install") diff --git a/var/spack/repos/builtin/packages/wx/package.py b/var/spack/repos/builtin/packages/wx/package.py new file mode 100644 index 0000000000..1813a8c8a5 --- /dev/null +++ b/var/spack/repos/builtin/packages/wx/package.py @@ -0,0 +1,24 @@ +from spack import * + +class Wx(Package): + """wxWidgets is a C++ library that lets developers create + applications for Windows, Mac OS X, Linux and other platforms + with a single code base. It has popular language bindings for + Python, Perl, Ruby and many other languages, and unlike other + cross-platform toolkits, wxWidgets gives applications a truly + native look and feel because it uses the platform's native API + rather than emulating the GUI. It's also extensive, free, + open-source and mature.""" + homepage = "http://www.wxwidgets.org/" + + version('2.8.12', '2fa39da14bc06ea86fe902579fedc5b1', + url="https://sourceforge.net/projects/wxwindows/files/2.8.12/wxWidgets-2.8.12.tar.gz") + version('3.0.1', 'dad1f1cd9d4c370cbc22700dc492da31', + url="https://sourceforge.net/projects/wxwindows/files/3.0.1/wxWidgets-3.0.1.tar.bz2") + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix, "--enable-unicode", "--disable-precomp-headers") + + make(parallel=False) + make("install") + diff --git a/var/spack/repos/builtin/packages/wxpropgrid/package.py b/var/spack/repos/builtin/packages/wxpropgrid/package.py new file mode 100644 index 0000000000..790cead517 --- /dev/null +++ b/var/spack/repos/builtin/packages/wxpropgrid/package.py @@ -0,0 +1,20 @@ +from spack import * + +class Wxpropgrid(Package): + """wxPropertyGrid is a property sheet control for wxWidgets. In + other words, it is a specialized two-column grid for editing + properties such as strings, numbers, flagsets, string arrays, + and colours.""" + homepage = "http://wxpropgrid.sourceforge.net/" + url = "http://prdownloads.sourceforge.net/wxpropgrid/wxpropgrid-1.4.15-src.tar.gz" + + version('1.4.15', 'f44b5cd6fd60718bacfabbf7994f1e93') + + depends_on("wx") + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix, "--with-wxdir=%s" % spec['wx'].prefix.bin, "--enable-unicode") + + make() + make("install") + diff --git a/var/spack/repos/builtin/packages/xcb-proto/package.py b/var/spack/repos/builtin/packages/xcb-proto/package.py new file mode 100644 index 0000000000..17a94bd892 --- /dev/null +++ b/var/spack/repos/builtin/packages/xcb-proto/package.py @@ -0,0 +1,15 @@ +from spack import * + +class XcbProto(Package): + """Protocol for libxcb""" + + homepage = "http://xcb.freedesktop.org/" + url = "http://xcb.freedesktop.org/dist/xcb-proto-1.11.tar.gz" + + version('1.11', 'c8c6cb72c84f58270f4db1f39607f66a') + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix) + + make() + make("install") diff --git a/var/spack/repos/builtin/packages/xz/package.py b/var/spack/repos/builtin/packages/xz/package.py new file mode 100644 index 0000000000..ba6c9733a7 --- /dev/null +++ b/var/spack/repos/builtin/packages/xz/package.py @@ -0,0 +1,20 @@ +from spack import * + +class Xz(Package): + """XZ Utils is free general-purpose data compression software with + high compression ratio. XZ Utils were written for POSIX-like + systems, but also work on some not-so-POSIX systems. XZ Utils are + the successor to LZMA Utils.""" + homepage = "http://tukaani.org/xz/" + url = "http://tukaani.org/xz/xz-5.2.0.tar.bz2" + + version('5.2.0', '867cc8611760240ebf3440bd6e170bb9', + url = 'http://tukaani.org/xz/xz-5.2.0.tar.bz2') + version('5.2.2', 'f90c9a0c8b259aee2234c4e0d7fd70af', + url = 'http://tukaani.org/xz/xz-5.2.2.tar.bz2') + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix) + make() + make("install") + diff --git a/var/spack/repos/builtin/packages/yasm/package.py b/var/spack/repos/builtin/packages/yasm/package.py new file mode 100644 index 0000000000..d3a695b16d --- /dev/null +++ b/var/spack/repos/builtin/packages/yasm/package.py @@ -0,0 +1,16 @@ +from spack import * + +class Yasm(Package): + """Yasm is a complete rewrite of the NASM-2.11.06 assembler. It + supports the x86 and AMD64 instruction sets, accepts NASM and + GAS assembler syntaxes and outputs binary, ELF32 and ELF64 + object formats.""" + homepage = "http://yasm.tortall.net" + url = "http://www.tortall.net/projects/yasm/releases/yasm-1.3.0.tar.gz" + + version('1.3.0', 'fc9e586751ff789b34b1f21d572d96af') + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix) + make() + make("install") diff --git a/var/spack/repos/builtin/packages/zeromq/package.py b/var/spack/repos/builtin/packages/zeromq/package.py new file mode 100644 index 0000000000..b5a1e3d4cd --- /dev/null +++ b/var/spack/repos/builtin/packages/zeromq/package.py @@ -0,0 +1,20 @@ +from spack import * + +class Zeromq(Package): + """ The ZMQ networking/concurrency library and core API """ + homepage = "http://zguide.zeromq.org/" + url = "http://download.zeromq.org/zeromq-4.1.2.tar.gz" + + version('4.1.2', '159c0c56a895472f02668e692d122685') + version('4.1.1', '0a4b44aa085644f25c177f79dc13f253') + version('4.0.7', '9b46f7e7b0704b83638ef0d461fd59ab') + version('4.0.6', 'd47dd09ed7ae6e7fd6f9a816d7f5fdf6') + version('4.0.5', '73c39f5eb01b9d7eaf74a5d899f1d03d') + + depends_on("libsodium") + + def install(self, spec, prefix): + configure("--with-libsodium","--prefix=%s" % prefix) + + make() + make("install") diff --git a/var/spack/repos/builtin/packages/zlib/package.py b/var/spack/repos/builtin/packages/zlib/package.py new file mode 100644 index 0000000000..2770f781ac --- /dev/null +++ b/var/spack/repos/builtin/packages/zlib/package.py @@ -0,0 +1,18 @@ +from spack import * + +class Zlib(Package): + """zlib is designed to be a free, general-purpose, legally unencumbered -- + that is, not covered by any patents -- lossless data-compression library for + use on virtually any computer hardware and operating system. + """ + + homepage = "http://zlib.net" + url = "http://zlib.net/zlib-1.2.8.tar.gz" + + version('1.2.8', '44d667c142d7cda120332623eab69f40') + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix) + + make() + make("install") diff --git a/var/spack/repos/builtin/packages/zsh/package.py b/var/spack/repos/builtin/packages/zsh/package.py new file mode 100644 index 0000000000..99ef9de2e5 --- /dev/null +++ b/var/spack/repos/builtin/packages/zsh/package.py @@ -0,0 +1,16 @@ +from spack import * + +class Zsh(Package): + """ The ZSH shell """ + homepage = "http://www.zsh.org" + url = "http://www.zsh.org/pub/zsh-5.0.8.tar.bz2" + + version('5.0.8', 'e6759e8dd7b714d624feffd0a73ba0fe') + + depends_on("pcre") + + def install(self, spec, prefix): + configure('--prefix=%s' % prefix) + + make() + make("install") diff --git a/var/spack/repos/builtin/repo.yaml b/var/spack/repos/builtin/repo.yaml new file mode 100644 index 0000000000..54b282db6b --- /dev/null +++ b/var/spack/repos/builtin/repo.yaml @@ -0,0 +1,2 @@ +repo: + namespace: builtin -- cgit v1.2.3-70-g09d2 From 763d850a0e1766a0b915686504bc07f8f2a92058 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Thu, 26 Nov 2015 14:23:12 -0800 Subject: Add tests for importing packages directly. Added test for, e.g.: import spack.pkg.builtin.mock.mpich import spack.pkg.builtin.mock.mpich as mpich from spack.pkg.builtin.mock.mpich import Mpich Among others. These ensure that direct package imports work so that packages can be extended. --- lib/spack/spack/test/packages.py | 42 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 40 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/test/packages.py b/lib/spack/spack/test/packages.py index e39def2ff2..f023516eba 100644 --- a/lib/spack/spack/test/packages.py +++ b/lib/spack/spack/test/packages.py @@ -46,7 +46,8 @@ class PackagesTest(MockPackagesTest): def test_package_filename(self): repo = Repo(spack.mock_packages_path) filename = repo.filename_for_package_name('mpich') - self.assertEqual(filename, join_path(spack.mock_packages_path, 'mpich', 'package.py')) + self.assertEqual(filename, + join_path(spack.mock_packages_path, 'packages', 'mpich', 'package.py')) def test_package_name(self): @@ -57,7 +58,9 @@ class PackagesTest(MockPackagesTest): def test_nonexisting_package_filename(self): repo = Repo(spack.mock_packages_path) filename = repo.filename_for_package_name('some-nonexisting-package') - self.assertEqual(filename, join_path(spack.mock_packages_path, 'some-nonexisting-package', 'package.py')) + self.assertEqual( + filename, + join_path(spack.mock_packages_path, 'packages', 'some-nonexisting-package', 'package.py')) def test_package_class_names(self): @@ -66,3 +69,38 @@ class PackagesTest(MockPackagesTest): self.assertEqual('PmgrCollective', mod_to_class('pmgr-collective')) self.assertEqual('Pmgrcollective', mod_to_class('PmgrCollective')) self.assertEqual('_3db', mod_to_class('3db')) + + + # + # Below tests target direct imports of spack packages from the + # spack.pkg namespace + # + + def test_import_package(self): + import spack.pkg.builtin.mock.mpich + + + def test_import_package_as(self): + import spack.pkg.builtin.mock.mpich as mp + + + def test_import_class_from_package(self): + from spack.pkg.builtin.mock.mpich import Mpich + + + def test_import_module_from_package(self): + from spack.pkg.builtin.mock import mpich + + + def test_import_namespace_container_modules(self): + import spack.pkg + import spack.pkg as p + from spack import pkg + + import spack.pkg.builtin + import spack.pkg.builtin as b + from spack.pkg import builtin + + import spack.pkg.builtin.mock + import spack.pkg.builtin.mock as m + from spack.pkg.builtin import mock -- cgit v1.2.3-70-g09d2 From 7383bd393eb36b67f308b643d1ce02b53ebb255d Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Fri, 27 Nov 2015 23:06:18 -0800 Subject: Fixed bug #42: problem with satisfies() for virtual dependencies. - _cross_provider_maps() had suffered some bit rot (map returned was ill-formed but still worked for cases with one vdep) - ProviderIndex.satisfies() was only checking whether the result map was non-empty. It should check whether all common vdeps are *in* the result map, as that indicates there is *some* way to satisfy *all* of them. We were checking whether there was some way to satisfy *any one* of them, which is wrong. - Above would cause a problem when there is more than one vdep provider. - Added test that covers this case. - Added `constrained()` method to Spec. Analogous to `normalized()`: `constrain():constrained() :: normalize():normalized()` --- lib/spack/spack/spec.py | 7 +++++ lib/spack/spack/test/spec_semantics.py | 13 ++++++++- lib/spack/spack/virtual.py | 9 +++--- var/spack/mock_packages/netlib-blas/package.py | 36 +++++++++++++++++++++++ var/spack/mock_packages/netlib-lapack/package.py | 37 ++++++++++++++++++++++++ var/spack/mock_packages/openblas/package.py | 37 ++++++++++++++++++++++++ 6 files changed, 134 insertions(+), 5 deletions(-) create mode 100644 var/spack/mock_packages/netlib-blas/package.py create mode 100644 var/spack/mock_packages/netlib-lapack/package.py create mode 100644 var/spack/mock_packages/openblas/package.py (limited to 'lib') diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 92880e9cbf..a10edcc6fc 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -1207,6 +1207,13 @@ class Spec(object): return common + def constrained(self, other, deps=True): + """Return a constrained copy without modifying this spec.""" + clone = self.copy(deps=deps) + clone.constrain(other, deps) + return clone + + def dep_difference(self, other): """Returns dependencies in self that are not in other.""" mine = set(s.name for s in self.traverse(root=False)) diff --git a/lib/spack/spack/test/spec_semantics.py b/lib/spack/spack/test/spec_semantics.py index 6666dbbb52..64220e5893 100644 --- a/lib/spack/spack/test/spec_semantics.py +++ b/lib/spack/spack/test/spec_semantics.py @@ -190,11 +190,23 @@ class SpecSematicsTest(MockPackagesTest): def test_satisfies_virtual(self): + # Don't use check_satisfies: it checks constrain() too, and + # you can't constrain a non-virtual by a virtual. self.assertTrue(Spec('mpich').satisfies(Spec('mpi'))) self.assertTrue(Spec('mpich2').satisfies(Spec('mpi'))) self.assertTrue(Spec('zmpi').satisfies(Spec('mpi'))) + def test_satisfies_virtual_dep_with_virtual_constraint(self): + """Ensure we can satisfy virtual constraints when there are multiple + vdep providers in the specs.""" + self.assertTrue(Spec('netlib-lapack ^openblas').satisfies('netlib-lapack ^openblas')) + self.assertFalse(Spec('netlib-lapack ^netlib-blas').satisfies('netlib-lapack ^openblas')) + + self.assertFalse(Spec('netlib-lapack ^openblas').satisfies('netlib-lapack ^netlib-blas')) + self.assertTrue(Spec('netlib-lapack ^netlib-blas').satisfies('netlib-lapack ^netlib-blas')) + + # ================================================================================ # Indexing specs # ================================================================================ @@ -327,4 +339,3 @@ class SpecSematicsTest(MockPackagesTest): self.check_constrain_not_changed('libelf^foo+debug', 'libelf^foo+debug') self.check_constrain_not_changed('libelf^foo~debug', 'libelf^foo~debug') self.check_constrain_not_changed('libelf^foo=bgqos_0', 'libelf^foo=bgqos_0') - diff --git a/lib/spack/spack/virtual.py b/lib/spack/spack/virtual.py index c77b259d61..f92cc4509c 100644 --- a/lib/spack/spack/virtual.py +++ b/lib/spack/spack/virtual.py @@ -117,12 +117,13 @@ class ProviderIndex(object): return sorted(providers) - # TODO: this is pretty darned nasty, and inefficient. + # TODO: this is pretty darned nasty, and inefficient, but there + # are not that many vdeps in most specs. def _cross_provider_maps(self, lmap, rmap): result = {} for lspec, rspec in itertools.product(lmap, rmap): try: - constrained = lspec.copy().constrain(rspec) + constrained = lspec.constrained(rspec) except spack.spec.UnsatisfiableSpecError: continue @@ -130,7 +131,7 @@ class ProviderIndex(object): for lp_spec, rp_spec in itertools.product(lmap[lspec], rmap[rspec]): if lp_spec.name == rp_spec.name: try: - const = lp_spec.copy().constrain(rp_spec,deps=False) + const = lp_spec.constrained(rp_spec, deps=False) result.setdefault(constrained, set()).add(const) except spack.spec.UnsatisfiableSpecError: continue @@ -157,4 +158,4 @@ class ProviderIndex(object): if crossed: result[name] = crossed - return bool(result) + return all(c in result for c in common) diff --git a/var/spack/mock_packages/netlib-blas/package.py b/var/spack/mock_packages/netlib-blas/package.py new file mode 100644 index 0000000000..199327812e --- /dev/null +++ b/var/spack/mock_packages/netlib-blas/package.py @@ -0,0 +1,36 @@ +############################################################################## +# Copyright (c) 2013-2015, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + +class NetlibBlas(Package): + homepage = "http://www.netlib.org/lapack/" + url = "http://www.netlib.org/lapack/lapack-3.5.0.tgz" + + version('3.5.0', 'b1d3e3e425b2e44a06760ff173104bdf') + + provides('blas') + + def install(self, spec, prefix): + pass diff --git a/var/spack/mock_packages/netlib-lapack/package.py b/var/spack/mock_packages/netlib-lapack/package.py new file mode 100644 index 0000000000..8f7f236f1b --- /dev/null +++ b/var/spack/mock_packages/netlib-lapack/package.py @@ -0,0 +1,37 @@ +############################################################################## +# Copyright (c) 2013-2015, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + +class NetlibLapack(Package): + homepage = "http://www.netlib.org/lapack/" + url = "http://www.netlib.org/lapack/lapack-3.5.0.tgz" + + version('3.5.0', 'b1d3e3e425b2e44a06760ff173104bdf') + + provides('lapack') + depends_on('blas') + + def install(self, spec, prefix): + pass diff --git a/var/spack/mock_packages/openblas/package.py b/var/spack/mock_packages/openblas/package.py new file mode 100644 index 0000000000..9c2fb30573 --- /dev/null +++ b/var/spack/mock_packages/openblas/package.py @@ -0,0 +1,37 @@ +############################################################################## +# Copyright (c) 2013-2015, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + +class Openblas(Package): + """OpenBLAS: An optimized BLAS library""" + homepage = "http://www.openblas.net" + url = "http://github.com/xianyi/OpenBLAS/archive/v0.2.15.tar.gz" + + version('0.2.15', 'b1190f3d3471685f17cfd1ec1d252ac9') + + provides('blas') + + def install(self, spec, prefix): + pass -- cgit v1.2.3-70-g09d2 From 73ef06018e2a6542d90f0b65a42e4a0287525b7a Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sat, 28 Nov 2015 16:26:23 -0800 Subject: Integrate namespace attribute into spec, spec DAG, spec YAML. --- lib/spack/spack/config.py | 22 ++++++++---- lib/spack/spack/database.py | 2 ++ lib/spack/spack/directory_layout.py | 9 +++-- lib/spack/spack/repository.py | 29 ++++++++++----- lib/spack/spack/spec.py | 65 ++++++++++++++++++++++++++-------- lib/spack/spack/test/spec_semantics.py | 33 +++++++++++++++-- 6 files changed, 125 insertions(+), 35 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 66da91f629..36bf8a7fc3 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -115,7 +115,7 @@ class _ConfigCategory: self.result_dict = {} _config_sections[name] = self -_ConfigCategory('config', 'config.yaml', True, False) +_ConfigCategory('repos', 'repos.yaml', True, True) _ConfigCategory('compilers', 'compilers.yaml', True, True) _ConfigCategory('mirrors', 'mirrors.yaml', True, True) _ConfigCategory('view', 'views.yaml', True, True) @@ -212,7 +212,7 @@ def substitute_spack_prefix(path): return path.replace('$spack', spack.prefix) -def get_config(category='config'): +def get_config(category): """Get the confguration tree for a category. Strips off the top-level category entry from the dict @@ -233,6 +233,10 @@ def get_config(category='config'): continue result = result[category.name] + # ignore empty sections for easy commenting of single-line configs. + if result is None: + continue + category.files_read_from.insert(0, path) if category.merge: category.result_dict = _merge_yaml(category.result_dict, result) @@ -266,12 +270,18 @@ def get_compilers_config(arch=None): def get_repos_config(): - config = get_config() - if 'repos' not in config: + repo_list = get_config('repos') + if repo_list is None: return [] - repo_list = config['repos'] - return [substitute_spack_prefix(repo) for repo in repo_list] + if not isinstance(repo_list, list): + tty.die("Bad repository configuration. 'repos' element does not contain a list.") + + def expand_repo_path(path): + path = substitute_spack_prefix(path) + path = os.path.expanduser(path) + return path + return [expand_repo_path(repo) for repo in repo_list] def get_mirror_config(): diff --git a/lib/spack/spack/database.py b/lib/spack/spack/database.py index 8e380083f3..5b3bd7502f 100644 --- a/lib/spack/spack/database.py +++ b/lib/spack/spack/database.py @@ -211,6 +211,8 @@ class Database(object): child = self._read_spec_from_yaml(dep_hash, installs, hash_key) spec._add_dependency(child) + spec._normal = True + spec._concrete = True return spec diff --git a/lib/spack/spack/directory_layout.py b/lib/spack/spack/directory_layout.py index da8f4187cc..30e2c93950 100644 --- a/lib/spack/spack/directory_layout.py +++ b/lib/spack/spack/directory_layout.py @@ -211,9 +211,12 @@ class YamlDirectoryLayout(DirectoryLayout): with open(path) as f: spec = Spec.from_yaml(f) - # Specs read from actual installations are always concrete - spec._normal = True - spec._concrete = True + # Specs read from actual installs are always concrete, so mark + # all parts of the spec. + for s in spec.traverse(): + s._normal = True + s._concrete = True + return spec diff --git a/lib/spack/spack/repository.py b/lib/spack/spack/repository.py index a2c0bbe147..1b09d63cdd 100644 --- a/lib/spack/spack/repository.py +++ b/lib/spack/spack/repository.py @@ -238,7 +238,16 @@ class RepoPath(object): Raises UnknownPackageError if not found. """ - return self.repo_for_pkg(spec.name).get(spec) + # if the spec has a fully qualified namespace, we grab it + # directly and ignore overlay precedence. + if spec.namespace: + fullspace = '%s.%s' % (self.super_namespace, spec.namespace) + if not fullspace in self.by_namespace: + raise UnknownPackageError( + "No configured repository contains package %s." % spec.fullname) + return self.by_namespace[fullspace].get(spec) + else: + return self.repo_for_pkg(spec.name).get(spec) def dirname_for_package_name(self, pkg_name): @@ -454,20 +463,24 @@ class Repo(object): if spec.virtual: raise UnknownPackageError(spec.name) - if new and spec in self._instances: - del self._instances[spec] + if spec.namespace and spec.namespace != self.namespace: + raise UnknownPackageError("Repository %s does not contain package %s." + % (self.namespace, spec.fullname)) - if not spec in self._instances: + if new or spec not in self._instances: PackageClass = self._get_pkg_class(spec.name) try: - copy = spec.copy() - self._instances[copy] = PackageClass(copy) + package = PackageClass(spec.copy()) + self._instances[spec] = package + return package + except Exception, e: if spack.debug: sys.excepthook(*sys.exc_info()) - raise FailedConstructorError(spec.name, *sys.exc_info()) + raise FailedConstructorError(spec.fullname, *sys.exc_info()) - return self._instances[spec] + else: + return self._instances[spec] def purge(self): diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index bb0f194c13..303df6df38 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -465,6 +465,13 @@ class Spec(object): self.dependencies[spec.name] = spec spec.dependents[self.name] = self + # + # Public interface + # + @property + def fullname(self): + return '%s.%s' % (self.namespace, self.name) if self.namespace else self.name + @property def root(self): @@ -518,6 +525,7 @@ class Spec(object): return True self._concrete = bool(not self.virtual + and self.namespace is not None and self.versions.concrete and self.variants.concrete and self.architecture @@ -658,6 +666,12 @@ class Spec(object): 'dependencies' : dict((d, self.dependencies[d].dag_hash()) for d in sorted(self.dependencies)) } + + # Older concrete specs do not have a namespace. Omit for + # consistent hashing. + if not self.concrete or self.namespace: + d['namespace'] = self.namespace + if self.compiler: d.update(self.compiler.to_dict()) else: @@ -682,6 +696,7 @@ class Spec(object): node = node[name] spec = Spec(name) + spec.namespace = node.get('namespace', None) spec.versions = VersionList.from_dict(node) spec.architecture = node['arch'] @@ -834,7 +849,20 @@ class Spec(object): changed = any(changes) force=True - self._concrete = True + for s in self.traverse(): + # After concretizing, assign namespaces to anything left. + # Note that this doesn't count as a "change". The repository + # configuration is constant throughout a spack run, and + # normalize and concretize evaluate Packages using Repo.get(), + # which respects precedence. So, a namespace assignment isn't + # changing how a package name would have been interpreted and + # we can do it as late as possible to allow as much + # compatibility across repositories as possible. + if s.namespace is None: + s.namespace = spack.repo.repo_for_pkg(s.name).namespace + + # Mark everything in the spec as concrete, as well. + s._concrete = True def concretized(self): @@ -909,7 +937,7 @@ class Spec(object): the dependency. If no conditions are True (and we don't depend on it), return None. """ - pkg = spack.repo.get(self.name) + pkg = spack.repo.get(self.fullname) conditions = pkg.dependencies[name] # evaluate when specs to figure out constraints on the dependency. @@ -1037,7 +1065,7 @@ class Spec(object): any_change = False changed = True - pkg = spack.repo.get(self.name) + pkg = spack.repo.get(self.fullname) while changed: changed = False for dep_name in pkg.dependencies: @@ -1058,18 +1086,17 @@ class Spec(object): the root, and ONLY the ones that were explicitly provided are there. Normalization turns a partial flat spec into a DAG, where: - 1. ALL dependencies of the root package are in the DAG. - 2. Each node's dependencies dict only contains its direct deps. + 1. Known dependencies of the root package are in the DAG. + 2. Each node's dependencies dict only contains its known direct deps. 3. There is only ONE unique spec for each package in the DAG. * This includes virtual packages. If there a non-virtual package that provides a virtual package that is in the spec, then we replace the virtual package with the non-virtual one. - 4. The spec DAG matches package DAG, including default variant values. - TODO: normalize should probably implement some form of cycle detection, to ensure that the spec is actually a DAG. + """ if self._normal and not force: return False @@ -1115,7 +1142,7 @@ class Spec(object): for spec in self.traverse(): # Don't get a package for a virtual name. if not spec.virtual: - spack.repo.get(spec.name) + spack.repo.get(spec.fullname) # validate compiler in addition to the package name. if spec.compiler: @@ -1138,6 +1165,10 @@ class Spec(object): if not self.name == other.name: raise UnsatisfiableSpecNameError(self.name, other.name) + if other.namespace is not None: + if self.namespace is not None and other.namespace != self.namespace: + raise UnsatisfiableSpecNameError(self.fullname, other.fullname) + if not self.versions.overlaps(other.versions): raise UnsatisfiableVersionSpecError(self.versions, other.versions) @@ -1181,7 +1212,7 @@ class Spec(object): # TODO: might want more detail than this, e.g. specific deps # in violation. if this becomes a priority get rid of this - # check and be more specici about what's wrong. + # check and be more specific about what's wrong. if not other.satisfies_dependencies(self): raise UnsatisfiableDependencySpecError(other, self) @@ -1247,7 +1278,7 @@ class Spec(object): # A concrete provider can satisfy a virtual dependency. if not self.virtual and other.virtual: - pkg = spack.repo.get(self.name) + pkg = spack.repo.get(self.fullname) if pkg.provides(other.name): for provided, when_spec in pkg.provided.items(): if self.satisfies(when_spec, deps=False, strict=strict): @@ -1259,6 +1290,11 @@ class Spec(object): if self.name != other.name: return False + # namespaces either match, or other doesn't require one. + if other.namespace is not None: + if self.namespace is not None and self.namespace != other.namespace: + return False + if self.versions and other.versions: if not self.versions.satisfies(other.versions, strict=strict): return False @@ -1476,8 +1512,8 @@ class Spec(object): def _cmp_node(self): """Comparison key for just *this node* and not its deps.""" - return (self.name, self.versions, self.variants, - self.architecture, self.compiler) + return (self.name, self.namespace, self.versions, + self.variants, self.architecture, self.compiler) def eq_node(self, other): @@ -1507,7 +1543,7 @@ class Spec(object): in the format string. The format strings you can provide are:: $_ Package name - $. Long package name + $. Full package name (with namespace) $@ Version $% Compiler $%@ Compiler & compiler version @@ -1556,8 +1592,7 @@ class Spec(object): if c == '_': out.write(fmt % self.name) elif c == '.': - longname = '%s.%s.%s' % (self.namespace, self.name) if self.namespace else self.name - out.write(fmt % longname) + out.write(fmt % self.fullname) elif c == '@': if self.versions and self.versions != _any_version: write(fmt % (c + str(self.versions)), c) diff --git a/lib/spack/spack/test/spec_semantics.py b/lib/spack/spack/test/spec_semantics.py index 6666dbbb52..87b7b628ed 100644 --- a/lib/spack/spack/test/spec_semantics.py +++ b/lib/spack/spack/test/spec_semantics.py @@ -35,7 +35,10 @@ class SpecSematicsTest(MockPackagesTest): # ================================================================================ def check_satisfies(self, spec, anon_spec, concrete=False): left = Spec(spec, concrete=concrete) - right = parse_anonymous_spec(anon_spec, left.name) + try: + right = Spec(anon_spec) # if it's not anonymous, allow it. + except: + right = parse_anonymous_spec(anon_spec, left.name) # Satisfies is one-directional. self.assertTrue(left.satisfies(right)) @@ -48,7 +51,10 @@ class SpecSematicsTest(MockPackagesTest): def check_unsatisfiable(self, spec, anon_spec, concrete=False): left = Spec(spec, concrete=concrete) - right = parse_anonymous_spec(anon_spec, left.name) + try: + right = Spec(anon_spec) # if it's not anonymous, allow it. + except: + right = parse_anonymous_spec(anon_spec, left.name) self.assertFalse(left.satisfies(right)) self.assertFalse(left.satisfies(anon_spec)) @@ -88,6 +94,28 @@ class SpecSematicsTest(MockPackagesTest): self.check_satisfies('libdwarf^libelf@0.8.13', '^libelf@0:1') + def test_satisfies_namespace(self): + self.check_satisfies('builtin.mpich', 'mpich') + self.check_satisfies('builtin.mock.mpich', 'mpich') + + # TODO: only works for deps now, but shouldn't we allow this for root spec? + # self.check_satisfies('builtin.mock.mpich', 'mpi') + + self.check_satisfies('builtin.mock.mpich', 'builtin.mock.mpich') + + self.check_unsatisfiable('builtin.mock.mpich', 'builtin.mpich') + + + def test_satisfies_namespaced_dep(self): + """Ensure spec from same or unspecified namespace satisfies namespace constraint.""" + self.check_satisfies('mpileaks ^builtin.mock.mpich', '^mpich') + + self.check_satisfies('mpileaks ^builtin.mock.mpich', '^mpi') + self.check_satisfies('mpileaks ^builtin.mock.mpich', '^builtin.mock.mpich') + + self.check_unsatisfiable('mpileaks ^builtin.mock.mpich', '^builtin.mpich') + + def test_satisfies_compiler(self): self.check_satisfies('foo%gcc', '%gcc') self.check_satisfies('foo%intel', '%intel') @@ -327,4 +355,3 @@ class SpecSematicsTest(MockPackagesTest): self.check_constrain_not_changed('libelf^foo+debug', 'libelf^foo+debug') self.check_constrain_not_changed('libelf^foo~debug', 'libelf^foo~debug') self.check_constrain_not_changed('libelf^foo=bgqos_0', 'libelf^foo=bgqos_0') - -- cgit v1.2.3-70-g09d2 From dfcf567de0cc944f7cdd0bcdee3b841bd0fbb900 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sat, 28 Nov 2015 19:00:24 -0800 Subject: Add a cleaned up repo command. --- lib/spack/spack/cmd/repo.py | 139 ++++++++++++++++++++++++-------------------- 1 file changed, 75 insertions(+), 64 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/cmd/repo.py b/lib/spack/spack/cmd/repo.py index 395aa90bed..8a3ea5989e 100644 --- a/lib/spack/spack/cmd/repo.py +++ b/lib/spack/spack/cmd/repo.py @@ -22,103 +22,114 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## +import os +import re +import shutil + from external import argparse import llnl.util.tty as tty -from llnl.util.tty.color import colorize -from llnl.util.tty.colify import colify -from llnl.util.lang import index_by from llnl.util.filesystem import join_path, mkdirp import spack.spec import spack.config from spack.util.environment import get_path -from spack.repository import repo_config_name - -import os -import exceptions -from contextlib import closing +from spack.repository import packages_dir_name, repo_config_name, Repo -description = "Manage package sources" +description = "Manage package source repositories." def setup_parser(subparser): - sp = subparser.add_subparsers( - metavar='SUBCOMMAND', dest='repo_command') - - add_parser = sp.add_parser('add', help=repo_add.__doc__) - add_parser.add_argument('directory', help="Directory containing the packages.") + sp = subparser.add_subparsers(metavar='SUBCOMMAND', dest='repo_command') + # Create create_parser = sp.add_parser('create', help=repo_create.__doc__) - create_parser.add_argument('directory', help="Directory containing the packages.") - create_parser.add_argument('name', help="Name of new package repository.") - - remove_parser = sp.add_parser('remove', help=repo_remove.__doc__) - remove_parser.add_argument('name') + create_parser.add_argument( + 'namespace', help="Namespace to identify packages in the repository.") + create_parser.add_argument( + 'directory', help="Directory to create the repo in. Defaults to same as namespace.", nargs='?') + # List list_parser = sp.add_parser('list', help=repo_list.__doc__) -def add_to_config(dir): - config = spack.config.get_config() - user_config = spack.config.get_config('user') - orig = None - if config.has_value('repo', '', 'directories'): - orig = config.get_value('repo', '', 'directories') - if orig and dir in orig.split(':'): - return False +def repo_create(args): + """Create a new package repo for a particular namespace.""" + namespace = args.namespace + if not re.match(r'\w[\.\w-]*', namespace): + tty.die("Invalid namespace: '%s'" % namespace) + + root = args.directory + if not root: + root = namespace + + existed = False + if os.path.exists(root): + if os.path.isfile(root): + tty.die('File %s already exists and is not a directory' % root) + elif os.path.isdir(root): + if not os.access(root, os.R_OK | os.W_OK): + tty.die('Cannot create new repo in %s: cannot access directory.' % root) + if os.listdir(root): + tty.die('Cannot create new repo in %s: directory is not empty.' % root) + existed = True + + full_path = os.path.realpath(root) + parent = os.path.dirname(full_path) + if not os.access(parent, os.R_OK | os.W_OK): + tty.die("Cannot create repository in %s: can't access parent!" % root) - newsetting = orig + ':' + dir if orig else dir - user_config.set_value('repo', '', 'directories', newsetting) - user_config.write() - return True + try: + config_path = os.path.join(root, repo_config_name) + packages_path = os.path.join(root, packages_dir_name) + mkdirp(packages_path) + with open(config_path, 'w') as config: + config.write("repo:\n") + config.write(" namespace: '%s'\n" % namespace) -def repo_add(args): - """Add package sources to the Spack configuration.""" - if not add_to_config(args.directory): - tty.die('Repo directory %s already exists in the repo list' % dir) + except (IOError, OSError) as e: + tty.die('Failed to create new repository in %s.' % root, + "Caused by %s: %s" % (type(e), e)) + # try to clean up. + if existed: + shutil.rmtree(config_path, ignore_errors=True) + shutil.rmtree(packages_path, ignore_errors=True) + else: + shutil.rmtree(root, ignore_errors=True) -def repo_create(args): - """Create a new package repo at a directory and name""" - dir = args.directory - name = args.name - - if os.path.exists(dir) and not os.path.isdir(dir): - tty.die('File %s already exists and is not a directory' % dir) - if not os.path.exists(dir): - try: - mkdirp(dir) - except exceptions.OSError, e: - tty.die('Failed to create new directory %s' % dir) - path = os.path.join(dir, repo_config_filename) - try: - with closing(open(path, 'w')) as repofile: - repofile.write(name + '\n') - except exceptions.IOError, e: - tty.die('Could not create new file %s' % path) + tty.msg("Created repo with namespace '%s'." % namespace) + tty.msg("To register it with Spack, add a line like this to ~/.spack/repos.yaml:", + 'repos:', + ' - ' + full_path) - if not add_to_config(args.directory): - tty.warn('Repo directory %s already exists in the repo list' % dir) + +def repo_add(args): + """Remove a package source from the Spack configuration""" + # FIXME: how to deal with this with the current config architecture? + # FIXME: Repos do not have mnemonics, which I assumed would be simpler... should they have them after all? def repo_remove(args): """Remove a package source from the Spack configuration""" - pass + # FIXME: see above. def repo_list(args): """List package sources and their mnemoics""" - root_names = spack.repo.repos - max_len = max(len(s[0]) for s in root_names) - fmt = "%%-%ds%%s" % (max_len + 4) - for root in root_names: - print fmt % (root[0], root[1]) + roots = spack.config.get_repos_config() + repos = [Repo(r) for r in roots] + + msg = "%d package repositor" % len(repos) + msg += "y." if len(repos) == 1 else "ies." + tty.msg(msg) + max_ns_len = max(len(r.namespace) for r in repos) + for repo in repos: + fmt = "%%-%ds%%s" % (max_ns_len + 4) + print fmt % (repo.namespace, repo.root) def repo(parser, args): - action = { 'add' : repo_add, - 'create' : repo_create, - 'remove' : repo_remove, + action = { 'create' : repo_create, 'list' : repo_list } action[args.repo_command](args) -- cgit v1.2.3-70-g09d2 From a075d581eff5e07cd4b703cbcd1d1e5f490b1fb4 Mon Sep 17 00:00:00 2001 From: Massimiliano Culpo Date: Tue, 1 Dec 2015 12:56:46 +0100 Subject: resource : fetch strategy constructed from kwargs instead or hardcoded URLFetchStrategy --- lib/spack/spack/directives.py | 5 ++--- lib/spack/spack/fetch_strategy.py | 16 ++++++++++++++++ 2 files changed, 18 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/directives.py b/lib/spack/spack/directives.py index 4d8333fd7d..741df3a31b 100644 --- a/lib/spack/spack/directives.py +++ b/lib/spack/spack/directives.py @@ -62,7 +62,7 @@ from spack.patch import Patch from spack.variant import Variant from spack.spec import Spec, parse_anonymous_spec from spack.resource import Resource -from spack.fetch_strategy import URLFetchStrategy +from spack.fetch_strategy import from_kwargs # # This is a list of all directives, built up as they are defined in @@ -279,8 +279,7 @@ def resource(pkg, **kwargs): destination = kwargs.get('destination', "") when_spec = parse_anonymous_spec(when, pkg.name) resources = pkg.resources.setdefault(when_spec, []) - # FIXME : change URLFetchStrategy with a factory that selects based on kwargs - fetcher = URLFetchStrategy(**kwargs) + fetcher = from_kwargs(**kwargs) # FIXME : should we infer the name somehow if not passed ? name = kwargs.get('name') resources.append(Resource(name, fetcher, destination)) diff --git a/lib/spack/spack/fetch_strategy.py b/lib/spack/spack/fetch_strategy.py index 5e6850d14b..1f5ef16caa 100644 --- a/lib/spack/spack/fetch_strategy.py +++ b/lib/spack/spack/fetch_strategy.py @@ -634,6 +634,22 @@ def from_url(url): return URLFetchStrategy(url) +def from_kwargs(**kwargs): + """ + Construct the appropriate FetchStrategy from the given keyword arguments. + + :param kwargs: dictionary of keyword arguments + :return: fetcher or raise a FetchError exception + """ + for fetcher in all_strategies: + if fetcher.matches(kwargs): + return fetcher(**kwargs) + # Raise an error in case we can't instantiate any known strategy + message = "Cannot instantiate any FetchStrategy" + long_message = message + " from the given arguments : {arguments}".format(srguments=kwargs) + raise FetchError(message, long_message) + + def args_are_for(args, fetcher): fetcher.matches(args) -- cgit v1.2.3-70-g09d2 From d3d9b5401b4d265f4de927cd34828a8c8b99e923 Mon Sep 17 00:00:00 2001 From: Massimiliano Culpo Date: Tue, 1 Dec 2015 13:26:44 +0100 Subject: resources : changed name of stage folder --- lib/spack/spack/package.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 9fee962df3..c1dbbafc79 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -643,10 +643,9 @@ class Package(object): ########## # Fetch resources resources = self._get_resources() - # FIXME : choose the unique name appropriately. Is there a function somewhere for the base name ? - pieces = [self.name, str(self.version), self.spec.dag_hash()] for resource in resources: - resource_stage_folder = '-'.join(pieces + [resource.name]) + pieces = ['resource', resource.name, self.spec.dag_hash()] + resource_stage_folder = '-'.join(pieces) stage = Stage(resource.fetcher, name=resource_stage_folder) resource.fetcher.set_stage(stage) resource.fetcher.fetch() -- cgit v1.2.3-70-g09d2 From b85dccca927bdd2e9c5d30841544ef4328ce9d56 Mon Sep 17 00:00:00 2001 From: Massimiliano Culpo Date: Tue, 1 Dec 2015 15:12:26 +0100 Subject: resources : added error handling for destination keyword on resource directive --- lib/spack/spack/directives.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/directives.py b/lib/spack/spack/directives.py index 741df3a31b..48a7df7462 100644 --- a/lib/spack/spack/directives.py +++ b/lib/spack/spack/directives.py @@ -42,6 +42,7 @@ The available directives are: * ``extends`` * ``patch`` * ``variant`` + * ``resource`` """ __all__ = ['depends_on', 'extends', 'provides', 'patch', 'version', @@ -49,9 +50,11 @@ __all__ = ['depends_on', 'extends', 'provides', 'patch', 'version', import re import inspect +import os.path import functools from llnl.util.lang import * +from llnl.util.filesystem import join_path import spack import spack.spec @@ -271,16 +274,27 @@ def resource(pkg, **kwargs): List of recognized keywords: * 'when' : represents the condition upon which the resource is needed (optional) - * 'destination' : path where to extract / checkout the resource (optional) + * 'destination' : path where to extract / checkout the resource (optional). This path must be a relative path, + and it must fall inside the stage area of the main package. """ when = kwargs.get('when', pkg.name) - # FIXME : currently I assume destination to be a relative path (rooted at pkg.stage.source_path) destination = kwargs.get('destination', "") + # Check if the path is relative + if os.path.isabs(destination): + message = "The destination keyword of a resource directive can't be an absolute path.\n" + message += "\tdestination : '{dest}\n'".format(dest=destination) + raise RuntimeError(message) + # Check if the path falls within the main package stage area + test_path = 'stage_folder_root/' + normalized_destination = os.path.normpath(join_path(test_path, destination)) # Normalized absolute path + if test_path not in normalized_destination: + message = "The destination folder of a resource must fall within the main package stage directory.\n" + message += "\tdestination : '{dest}'\n".format(dest=destination) + raise RuntimeError(message) when_spec = parse_anonymous_spec(when, pkg.name) resources = pkg.resources.setdefault(when_spec, []) fetcher = from_kwargs(**kwargs) - # FIXME : should we infer the name somehow if not passed ? name = kwargs.get('name') resources.append(Resource(name, fetcher, destination)) -- cgit v1.2.3-70-g09d2 From 4b2168ab8ef5b5e69545a9bd82ef8804a4108d75 Mon Sep 17 00:00:00 2001 From: Massimiliano Culpo Date: Wed, 2 Dec 2015 10:13:39 +0100 Subject: resources : added missing check after fetch --- lib/spack/spack/package.py | 1 + 1 file changed, 1 insertion(+) (limited to 'lib') diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index c1dbbafc79..c10283a6ee 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -649,6 +649,7 @@ class Package(object): stage = Stage(resource.fetcher, name=resource_stage_folder) resource.fetcher.set_stage(stage) resource.fetcher.fetch() + resource.fetcher.check() ########## self._fetch_time = time.time() - start_time -- cgit v1.2.3-70-g09d2 From 39a3cfd4d9fc8d0981ba118de9a1e049fba4b144 Mon Sep 17 00:00:00 2001 From: Massimiliano Culpo Date: Wed, 2 Dec 2015 12:24:37 +0100 Subject: reource directive accepts 'basename' keyword llvm : libc++ variant --- lib/spack/spack/directives.py | 5 ++++- lib/spack/spack/package.py | 3 ++- lib/spack/spack/resource.py | 4 +++- var/spack/packages/llvm/package.py | 9 ++++++++- 4 files changed, 17 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/directives.py b/lib/spack/spack/directives.py index 48a7df7462..07ef32294b 100644 --- a/lib/spack/spack/directives.py +++ b/lib/spack/spack/directives.py @@ -276,10 +276,12 @@ def resource(pkg, **kwargs): * 'when' : represents the condition upon which the resource is needed (optional) * 'destination' : path where to extract / checkout the resource (optional). This path must be a relative path, and it must fall inside the stage area of the main package. + * 'basename' : basename of the resource source folder within destination (optional). """ when = kwargs.get('when', pkg.name) destination = kwargs.get('destination', "") + basename = kwargs.get('basename', None) # Check if the path is relative if os.path.isabs(destination): message = "The destination keyword of a resource directive can't be an absolute path.\n" @@ -296,7 +298,8 @@ def resource(pkg, **kwargs): resources = pkg.resources.setdefault(when_spec, []) fetcher = from_kwargs(**kwargs) name = kwargs.get('name') - resources.append(Resource(name, fetcher, destination)) + resources.append(Resource(name, fetcher, destination, basename)) + class DirectiveError(spack.error.SpackError): """This is raised when something is wrong with a package directive.""" diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index c10283a6ee..3f48231c75 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -681,7 +681,8 @@ class Package(object): for resource in resources: stage = resource.fetcher.stage _expand_archive(stage, resource.name) - link_path = join_path(self.stage.source_path, resource.destination, os.path.basename(stage.source_path)) + basename = os.path.basename(stage.source_path) if resource.basename is None else resource.basename + link_path = join_path(self.stage.source_path, resource.destination, basename) if not os.path.exists(link_path): # Create a symlink os.symlink(stage.source_path, link_path) diff --git a/lib/spack/spack/resource.py b/lib/spack/spack/resource.py index 00ee2d49e4..b20612686d 100644 --- a/lib/spack/spack/resource.py +++ b/lib/spack/spack/resource.py @@ -27,11 +27,13 @@ Describes an optional resource needed for a build. Typically a bunch of sources package to enable optional features. """ + class Resource(object): """ Represents an optional resource. Aggregates a name, a fetcher and a destination. """ - def __init__(self, name, fetcher, destination): + def __init__(self, name, fetcher, destination, basename): self.name = name self.fetcher = fetcher self.destination = destination + self.basename = basename diff --git a/var/spack/packages/llvm/package.py b/var/spack/packages/llvm/package.py index b68aa82aff..872a6c082b 100644 --- a/var/spack/packages/llvm/package.py +++ b/var/spack/packages/llvm/package.py @@ -41,15 +41,22 @@ class Llvm(Package): depends_on('python@2.7:') + variant('libcxx', default=False, description="Builds the LLVM Standard C++ library targeting C++11") + ########## # @3.7.0 - # TODO : Add support for libc++ <- libc++ABI <- libunwind with variant? resource(name='compiler-rt', url='http://llvm.org/releases/3.7.0/compiler-rt-3.7.0.src.tar.xz', md5='383c10affd513026f08936b5525523f5', destination='projects', when='@3.7.0') resource(name='openmp', url='http://llvm.org/releases/3.7.0/openmp-3.7.0.src.tar.xz', md5='f482c86fdead50ba246a1a2b0bbf206f', destination='projects', when='@3.7.0') + resource(name='libcxx', + url='http://llvm.org/releases/3.7.0/libcxx-3.7.0.src.tar.xz', md5='46aa5175cbe1ad42d6e9c995968e56dd', + destination='projects', basename='libcxx', when='+libcxx@3.7.0') + resource(name='libcxxabi', + url='http://llvm.org/releases/3.7.0/libcxxabi-3.7.0.src.tar.xz', md5='5aa769e2fca79fa5335cfae8f6258772', + destination='projects', basename='libcxxabi', when='+libcxx@3.7.0') ########## def install(self, spec, prefix): -- cgit v1.2.3-70-g09d2 From 50bd4d2e4ed3fed83afa30e7796eaa99989bc3ce Mon Sep 17 00:00:00 2001 From: Massimiliano Culpo Date: Wed, 2 Dec 2015 16:18:25 +0100 Subject: mirror : deals correctly with variants that optionally enable resources (if they are archive URLs) --- lib/spack/spack/mirror.py | 81 +++++++++++++++++++++++++++++++++------------- lib/spack/spack/package.py | 19 +++++++---- 2 files changed, 72 insertions(+), 28 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/mirror.py b/lib/spack/spack/mirror.py index 306c8085aa..1c38d79164 100644 --- a/lib/spack/spack/mirror.py +++ b/lib/spack/spack/mirror.py @@ -26,7 +26,7 @@ This file contains code for creating spack mirror directories. A mirror is an organized hierarchy containing specially named archive files. This enabled spack to know where to find files in a mirror if -the main server for a particualr package is down. Or, if the computer +the main server for a particular package is down. Or, if the computer where spack is run is not connected to the internet, it allows spack to download packages directly from a mirror (e.g., on an intranet). """ @@ -42,7 +42,7 @@ import spack.fetch_strategy as fs from spack.spec import Spec from spack.stage import Stage from spack.version import * -from spack.util.compression import extension +from spack.util.compression import extension, allowed_archive def mirror_archive_filename(spec): @@ -87,11 +87,26 @@ def get_matching_versions(specs, **kwargs): if v.satisfies(spec.versions): s = Spec(pkg.name) s.versions = VersionList([v]) + s.variants = spec.variants.copy() matching.append(s) return matching +def suggest_archive_basename(resource): + """ + Return a tentative basename for an archive. Raise an exception if the name is among the allowed archive types. + + :param fetcher: + :return: + """ + basename = os.path.basename(resource.fetcher.url) + if not allowed_archive(basename): + raise RuntimeError("%s is not an allowed archive tye" % basename) + return basename + + + def create(path, specs, **kwargs): """Create a directory to be used as a spack mirror, and fill it with package archives. @@ -108,7 +123,7 @@ def create(path, specs, **kwargs): Return Value: Returns a tuple of lists: (present, mirrored, error) - * present: Package specs that were already prsent. + * present: Package specs that were already present. * mirrored: Package specs that were successfully mirrored. * error: Package specs that failed to mirror due to some error. @@ -140,6 +155,7 @@ def create(path, specs, **kwargs): error = [] # Iterate through packages and download all the safe tarballs for each of them + everything_already_exists = True for spec in version_specs: pkg = spec.package @@ -152,26 +168,47 @@ def create(path, specs, **kwargs): if os.path.exists(archive_path): tty.msg("Already added %s" % spec.format("$_$@")) + else: + everything_already_exists = False + # Set up a stage and a fetcher for the download + unique_fetch_name = spec.format("$_$@") + fetcher = fs.for_package_version(pkg, pkg.version) + stage = Stage(fetcher, name=unique_fetch_name) + fetcher.set_stage(stage) + + # Do the fetch and checksum if necessary + fetcher.fetch() + if not kwargs.get('no_checksum', False): + fetcher.check() + tty.msg("Checksum passed for %s@%s" % (pkg.name, pkg.version)) + + # Fetchers have to know how to archive their files. Use + # that to move/copy/create an archive in the mirror. + fetcher.archive(archive_path) + tty.msg("Added %s." % spec.format("$_$@")) + + # Fetch resources if they are associated with the spec + resources = pkg._get_resources() + for resource in resources: + resource_archive_path = join_path(subdir, suggest_archive_basename(resource)) + if os.path.exists(resource_archive_path): + tty.msg("Already added resource %s (%s@%s)." % (resource.name, pkg.name, pkg.version)) + continue + everything_already_exists = False + resource_stage_folder = pkg._resource_stage(resource) + resource_stage = Stage(resource.fetcher, name=resource_stage_folder) + resource.fetcher.set_stage(resource_stage) + resource.fetcher.fetch() + if not kwargs.get('no_checksum', False): + resource.fetcher.check() + tty.msg("Checksum passed for the resource %s (%s@%s)" % (resource.name, pkg.name, pkg.version)) + resource.fetcher.archive(resource_archive_path) + tty.msg("Added resource %s (%s@%s)." % (resource.name, pkg.name, pkg.version)) + + if everything_already_exists: present.append(spec) - continue - - # Set up a stage and a fetcher for the download - unique_fetch_name = spec.format("$_$@") - fetcher = fs.for_package_version(pkg, pkg.version) - stage = Stage(fetcher, name=unique_fetch_name) - fetcher.set_stage(stage) - - # Do the fetch and checksum if necessary - fetcher.fetch() - if not kwargs.get('no_checksum', False): - fetcher.check() - tty.msg("Checksum passed for %s@%s" % (pkg.name, pkg.version)) - - # Fetchers have to know how to archive their files. Use - # that to move/copy/create an archive in the mirror. - fetcher.archive(archive_path) - tty.msg("Added %s." % spec.format("$_$@")) - mirrored.append(spec) + else: + mirrored.append(spec) except Exception, e: if spack.debug: diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 3f48231c75..dcb514af00 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -644,12 +644,14 @@ class Package(object): # Fetch resources resources = self._get_resources() for resource in resources: - pieces = ['resource', resource.name, self.spec.dag_hash()] - resource_stage_folder = '-'.join(pieces) - stage = Stage(resource.fetcher, name=resource_stage_folder) - resource.fetcher.set_stage(stage) - resource.fetcher.fetch() - resource.fetcher.check() + resource_stage_folder = self._resource_stage(resource) + # FIXME : works only for URLFetchStrategy + resource_mirror = join_path(self.name, os.path.basename(resource.fetcher.url)) + resource_stage = Stage(resource.fetcher, name=resource_stage_folder, mirror_path=resource_mirror) + resource.fetcher.set_stage(resource_stage) + # Delegate to stage object to trigger mirror logic + resource_stage.fetch() + resource_stage.check() ########## self._fetch_time = time.time() - start_time @@ -766,6 +768,11 @@ class Package(object): resources.extend(resource_list) return resources + def _resource_stage(self, resource): + pieces = ['resource', resource.name, self.spec.dag_hash()] + resource_stage_folder = '-'.join(pieces) + return resource_stage_folder + def _build_logger(self, log_path): """Create a context manager to log build output.""" -- cgit v1.2.3-70-g09d2 From 530e049d4a3f50481e7d55e8b61a9e840f48ad76 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 4 Dec 2015 17:55:20 -0800 Subject: Added nose 1.3.7 source along with with the lgpl license (renamed from lgpl.txt to LICENSE) --- lib/spack/external/nose/LICENSE | 504 +++++ lib/spack/external/nose/__init__.py | 15 + lib/spack/external/nose/__main__.py | 8 + lib/spack/external/nose/case.py | 397 ++++ lib/spack/external/nose/commands.py | 172 ++ lib/spack/external/nose/config.py | 661 +++++++ lib/spack/external/nose/core.py | 341 ++++ lib/spack/external/nose/exc.py | 9 + lib/spack/external/nose/ext/__init__.py | 3 + lib/spack/external/nose/ext/dtcompat.py | 2272 ++++++++++++++++++++++ lib/spack/external/nose/failure.py | 42 + lib/spack/external/nose/importer.py | 167 ++ lib/spack/external/nose/inspector.py | 207 ++ lib/spack/external/nose/loader.py | 623 ++++++ lib/spack/external/nose/plugins/__init__.py | 190 ++ lib/spack/external/nose/plugins/allmodules.py | 45 + lib/spack/external/nose/plugins/attrib.py | 286 +++ lib/spack/external/nose/plugins/base.py | 725 +++++++ lib/spack/external/nose/plugins/builtin.py | 34 + lib/spack/external/nose/plugins/capture.py | 115 ++ lib/spack/external/nose/plugins/collect.py | 94 + lib/spack/external/nose/plugins/cover.py | 271 +++ lib/spack/external/nose/plugins/debug.py | 67 + lib/spack/external/nose/plugins/deprecated.py | 45 + lib/spack/external/nose/plugins/doctests.py | 455 +++++ lib/spack/external/nose/plugins/errorclass.py | 210 ++ lib/spack/external/nose/plugins/failuredetail.py | 49 + lib/spack/external/nose/plugins/isolate.py | 103 + lib/spack/external/nose/plugins/logcapture.py | 245 +++ lib/spack/external/nose/plugins/manager.py | 460 +++++ lib/spack/external/nose/plugins/multiprocess.py | 835 ++++++++ lib/spack/external/nose/plugins/plugintest.py | 416 ++++ lib/spack/external/nose/plugins/prof.py | 154 ++ lib/spack/external/nose/plugins/skip.py | 63 + lib/spack/external/nose/plugins/testid.py | 311 +++ lib/spack/external/nose/plugins/xunit.py | 341 ++++ lib/spack/external/nose/proxy.py | 188 ++ lib/spack/external/nose/pyversion.py | 215 ++ lib/spack/external/nose/result.py | 200 ++ lib/spack/external/nose/selector.py | 251 +++ lib/spack/external/nose/sphinx/__init__.py | 1 + lib/spack/external/nose/sphinx/pluginopts.py | 189 ++ lib/spack/external/nose/suite.py | 609 ++++++ lib/spack/external/nose/tools/__init__.py | 15 + lib/spack/external/nose/tools/nontrivial.py | 151 ++ lib/spack/external/nose/tools/trivial.py | 54 + lib/spack/external/nose/twistedtools.py | 173 ++ lib/spack/external/nose/usage.txt | 115 ++ lib/spack/external/nose/util.py | 668 +++++++ 49 files changed, 13764 insertions(+) create mode 100644 lib/spack/external/nose/LICENSE create mode 100644 lib/spack/external/nose/__init__.py create mode 100644 lib/spack/external/nose/__main__.py create mode 100644 lib/spack/external/nose/case.py create mode 100644 lib/spack/external/nose/commands.py create mode 100644 lib/spack/external/nose/config.py create mode 100644 lib/spack/external/nose/core.py create mode 100644 lib/spack/external/nose/exc.py create mode 100644 lib/spack/external/nose/ext/__init__.py create mode 100644 lib/spack/external/nose/ext/dtcompat.py create mode 100644 lib/spack/external/nose/failure.py create mode 100644 lib/spack/external/nose/importer.py create mode 100644 lib/spack/external/nose/inspector.py create mode 100644 lib/spack/external/nose/loader.py create mode 100644 lib/spack/external/nose/plugins/__init__.py create mode 100644 lib/spack/external/nose/plugins/allmodules.py create mode 100644 lib/spack/external/nose/plugins/attrib.py create mode 100644 lib/spack/external/nose/plugins/base.py create mode 100644 lib/spack/external/nose/plugins/builtin.py create mode 100644 lib/spack/external/nose/plugins/capture.py create mode 100644 lib/spack/external/nose/plugins/collect.py create mode 100644 lib/spack/external/nose/plugins/cover.py create mode 100644 lib/spack/external/nose/plugins/debug.py create mode 100644 lib/spack/external/nose/plugins/deprecated.py create mode 100644 lib/spack/external/nose/plugins/doctests.py create mode 100644 lib/spack/external/nose/plugins/errorclass.py create mode 100644 lib/spack/external/nose/plugins/failuredetail.py create mode 100644 lib/spack/external/nose/plugins/isolate.py create mode 100644 lib/spack/external/nose/plugins/logcapture.py create mode 100644 lib/spack/external/nose/plugins/manager.py create mode 100644 lib/spack/external/nose/plugins/multiprocess.py create mode 100644 lib/spack/external/nose/plugins/plugintest.py create mode 100644 lib/spack/external/nose/plugins/prof.py create mode 100644 lib/spack/external/nose/plugins/skip.py create mode 100644 lib/spack/external/nose/plugins/testid.py create mode 100644 lib/spack/external/nose/plugins/xunit.py create mode 100644 lib/spack/external/nose/proxy.py create mode 100644 lib/spack/external/nose/pyversion.py create mode 100644 lib/spack/external/nose/result.py create mode 100644 lib/spack/external/nose/selector.py create mode 100644 lib/spack/external/nose/sphinx/__init__.py create mode 100644 lib/spack/external/nose/sphinx/pluginopts.py create mode 100644 lib/spack/external/nose/suite.py create mode 100644 lib/spack/external/nose/tools/__init__.py create mode 100644 lib/spack/external/nose/tools/nontrivial.py create mode 100644 lib/spack/external/nose/tools/trivial.py create mode 100644 lib/spack/external/nose/twistedtools.py create mode 100644 lib/spack/external/nose/usage.txt create mode 100644 lib/spack/external/nose/util.py (limited to 'lib') diff --git a/lib/spack/external/nose/LICENSE b/lib/spack/external/nose/LICENSE new file mode 100644 index 0000000000..8add30ad59 --- /dev/null +++ b/lib/spack/external/nose/LICENSE @@ -0,0 +1,504 @@ + GNU LESSER GENERAL PUBLIC LICENSE + Version 2.1, February 1999 + + Copyright (C) 1991, 1999 Free Software Foundation, Inc. + 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + +[This is the first released version of the Lesser GPL. It also counts + as the successor of the GNU Library Public License, version 2, hence + the version number 2.1.] + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +Licenses are intended to guarantee your freedom to share and change +free software--to make sure the software is free for all its users. + + This license, the Lesser General Public License, applies to some +specially designated software packages--typically libraries--of the +Free Software Foundation and other authors who decide to use it. You +can use it too, but we suggest you first think carefully about whether +this license or the ordinary General Public License is the better +strategy to use in any particular case, based on the explanations below. + + When we speak of free software, we are referring to freedom of use, +not price. Our General Public Licenses are designed to make sure that +you have the freedom to distribute copies of free software (and charge +for this service if you wish); that you receive source code or can get +it if you want it; that you can change the software and use pieces of +it in new free programs; and that you are informed that you can do +these things. + + To protect your rights, we need to make restrictions that forbid +distributors to deny you these rights or to ask you to surrender these +rights. These restrictions translate to certain responsibilities for +you if you distribute copies of the library or if you modify it. + + For example, if you distribute copies of the library, whether gratis +or for a fee, you must give the recipients all the rights that we gave +you. You must make sure that they, too, receive or can get the source +code. If you link other code with the library, you must provide +complete object files to the recipients, so that they can relink them +with the library after making changes to the library and recompiling +it. And you must show them these terms so they know their rights. + + We protect your rights with a two-step method: (1) we copyright the +library, and (2) we offer you this license, which gives you legal +permission to copy, distribute and/or modify the library. + + To protect each distributor, we want to make it very clear that +there is no warranty for the free library. Also, if the library is +modified by someone else and passed on, the recipients should know +that what they have is not the original version, so that the original +author's reputation will not be affected by problems that might be +introduced by others. + + Finally, software patents pose a constant threat to the existence of +any free program. We wish to make sure that a company cannot +effectively restrict the users of a free program by obtaining a +restrictive license from a patent holder. Therefore, we insist that +any patent license obtained for a version of the library must be +consistent with the full freedom of use specified in this license. + + Most GNU software, including some libraries, is covered by the +ordinary GNU General Public License. This license, the GNU Lesser +General Public License, applies to certain designated libraries, and +is quite different from the ordinary General Public License. We use +this license for certain libraries in order to permit linking those +libraries into non-free programs. + + When a program is linked with a library, whether statically or using +a shared library, the combination of the two is legally speaking a +combined work, a derivative of the original library. The ordinary +General Public License therefore permits such linking only if the +entire combination fits its criteria of freedom. The Lesser General +Public License permits more lax criteria for linking other code with +the library. + + We call this license the "Lesser" General Public License because it +does Less to protect the user's freedom than the ordinary General +Public License. It also provides other free software developers Less +of an advantage over competing non-free programs. These disadvantages +are the reason we use the ordinary General Public License for many +libraries. However, the Lesser license provides advantages in certain +special circumstances. + + For example, on rare occasions, there may be a special need to +encourage the widest possible use of a certain library, so that it becomes +a de-facto standard. To achieve this, non-free programs must be +allowed to use the library. A more frequent case is that a free +library does the same job as widely used non-free libraries. In this +case, there is little to gain by limiting the free library to free +software only, so we use the Lesser General Public License. + + In other cases, permission to use a particular library in non-free +programs enables a greater number of people to use a large body of +free software. For example, permission to use the GNU C Library in +non-free programs enables many more people to use the whole GNU +operating system, as well as its variant, the GNU/Linux operating +system. + + Although the Lesser General Public License is Less protective of the +users' freedom, it does ensure that the user of a program that is +linked with the Library has the freedom and the wherewithal to run +that program using a modified version of the Library. + + The precise terms and conditions for copying, distribution and +modification follow. Pay close attention to the difference between a +"work based on the library" and a "work that uses the library". The +former contains code derived from the library, whereas the latter must +be combined with the library in order to run. + + GNU LESSER GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License Agreement applies to any software library or other +program which contains a notice placed by the copyright holder or +other authorized party saying it may be distributed under the terms of +this Lesser General Public License (also called "this License"). +Each licensee is addressed as "you". + + A "library" means a collection of software functions and/or data +prepared so as to be conveniently linked with application programs +(which use some of those functions and data) to form executables. + + The "Library", below, refers to any such software library or work +which has been distributed under these terms. A "work based on the +Library" means either the Library or any derivative work under +copyright law: that is to say, a work containing the Library or a +portion of it, either verbatim or with modifications and/or translated +straightforwardly into another language. (Hereinafter, translation is +included without limitation in the term "modification".) + + "Source code" for a work means the preferred form of the work for +making modifications to it. For a library, complete source code means +all the source code for all modules it contains, plus any associated +interface definition files, plus the scripts used to control compilation +and installation of the library. + + Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running a program using the Library is not restricted, and output from +such a program is covered only if its contents constitute a work based +on the Library (independent of the use of the Library in a tool for +writing it). Whether that is true depends on what the Library does +and what the program that uses the Library does. + + 1. You may copy and distribute verbatim copies of the Library's +complete source code as you receive it, in any medium, provided that +you conspicuously and appropriately publish on each copy an +appropriate copyright notice and disclaimer of warranty; keep intact +all the notices that refer to this License and to the absence of any +warranty; and distribute a copy of this License along with the +Library. + + You may charge a fee for the physical act of transferring a copy, +and you may at your option offer warranty protection in exchange for a +fee. + + 2. You may modify your copy or copies of the Library or any portion +of it, thus forming a work based on the Library, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) The modified work must itself be a software library. + + b) You must cause the files modified to carry prominent notices + stating that you changed the files and the date of any change. + + c) You must cause the whole of the work to be licensed at no + charge to all third parties under the terms of this License. + + d) If a facility in the modified Library refers to a function or a + table of data to be supplied by an application program that uses + the facility, other than as an argument passed when the facility + is invoked, then you must make a good faith effort to ensure that, + in the event an application does not supply such function or + table, the facility still operates, and performs whatever part of + its purpose remains meaningful. + + (For example, a function in a library to compute square roots has + a purpose that is entirely well-defined independent of the + application. Therefore, Subsection 2d requires that any + application-supplied function or table used by this function must + be optional: if the application does not supply it, the square + root function must still compute square roots.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Library, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Library, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote +it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Library. + +In addition, mere aggregation of another work not based on the Library +with the Library (or with a work based on the Library) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may opt to apply the terms of the ordinary GNU General Public +License instead of this License to a given copy of the Library. To do +this, you must alter all the notices that refer to this License, so +that they refer to the ordinary GNU General Public License, version 2, +instead of to this License. (If a newer version than version 2 of the +ordinary GNU General Public License has appeared, then you can specify +that version instead if you wish.) Do not make any other change in +these notices. + + Once this change is made in a given copy, it is irreversible for +that copy, so the ordinary GNU General Public License applies to all +subsequent copies and derivative works made from that copy. + + This option is useful when you wish to copy part of the code of +the Library into a program that is not a library. + + 4. You may copy and distribute the Library (or a portion or +derivative of it, under Section 2) in object code or executable form +under the terms of Sections 1 and 2 above provided that you accompany +it with the complete corresponding machine-readable source code, which +must be distributed under the terms of Sections 1 and 2 above on a +medium customarily used for software interchange. + + If distribution of object code is made by offering access to copy +from a designated place, then offering equivalent access to copy the +source code from the same place satisfies the requirement to +distribute the source code, even though third parties are not +compelled to copy the source along with the object code. + + 5. A program that contains no derivative of any portion of the +Library, but is designed to work with the Library by being compiled or +linked with it, is called a "work that uses the Library". Such a +work, in isolation, is not a derivative work of the Library, and +therefore falls outside the scope of this License. + + However, linking a "work that uses the Library" with the Library +creates an executable that is a derivative of the Library (because it +contains portions of the Library), rather than a "work that uses the +library". The executable is therefore covered by this License. +Section 6 states terms for distribution of such executables. + + When a "work that uses the Library" uses material from a header file +that is part of the Library, the object code for the work may be a +derivative work of the Library even though the source code is not. +Whether this is true is especially significant if the work can be +linked without the Library, or if the work is itself a library. The +threshold for this to be true is not precisely defined by law. + + If such an object file uses only numerical parameters, data +structure layouts and accessors, and small macros and small inline +functions (ten lines or less in length), then the use of the object +file is unrestricted, regardless of whether it is legally a derivative +work. (Executables containing this object code plus portions of the +Library will still fall under Section 6.) + + Otherwise, if the work is a derivative of the Library, you may +distribute the object code for the work under the terms of Section 6. +Any executables containing that work also fall under Section 6, +whether or not they are linked directly with the Library itself. + + 6. As an exception to the Sections above, you may also combine or +link a "work that uses the Library" with the Library to produce a +work containing portions of the Library, and distribute that work +under terms of your choice, provided that the terms permit +modification of the work for the customer's own use and reverse +engineering for debugging such modifications. + + You must give prominent notice with each copy of the work that the +Library is used in it and that the Library and its use are covered by +this License. You must supply a copy of this License. If the work +during execution displays copyright notices, you must include the +copyright notice for the Library among them, as well as a reference +directing the user to the copy of this License. Also, you must do one +of these things: + + a) Accompany the work with the complete corresponding + machine-readable source code for the Library including whatever + changes were used in the work (which must be distributed under + Sections 1 and 2 above); and, if the work is an executable linked + with the Library, with the complete machine-readable "work that + uses the Library", as object code and/or source code, so that the + user can modify the Library and then relink to produce a modified + executable containing the modified Library. (It is understood + that the user who changes the contents of definitions files in the + Library will not necessarily be able to recompile the application + to use the modified definitions.) + + b) Use a suitable shared library mechanism for linking with the + Library. A suitable mechanism is one that (1) uses at run time a + copy of the library already present on the user's computer system, + rather than copying library functions into the executable, and (2) + will operate properly with a modified version of the library, if + the user installs one, as long as the modified version is + interface-compatible with the version that the work was made with. + + c) Accompany the work with a written offer, valid for at + least three years, to give the same user the materials + specified in Subsection 6a, above, for a charge no more + than the cost of performing this distribution. + + d) If distribution of the work is made by offering access to copy + from a designated place, offer equivalent access to copy the above + specified materials from the same place. + + e) Verify that the user has already received a copy of these + materials or that you have already sent this user a copy. + + For an executable, the required form of the "work that uses the +Library" must include any data and utility programs needed for +reproducing the executable from it. However, as a special exception, +the materials to be distributed need not include anything that is +normally distributed (in either source or binary form) with the major +components (compiler, kernel, and so on) of the operating system on +which the executable runs, unless that component itself accompanies +the executable. + + It may happen that this requirement contradicts the license +restrictions of other proprietary libraries that do not normally +accompany the operating system. Such a contradiction means you cannot +use both them and the Library together in an executable that you +distribute. + + 7. You may place library facilities that are a work based on the +Library side-by-side in a single library together with other library +facilities not covered by this License, and distribute such a combined +library, provided that the separate distribution of the work based on +the Library and of the other library facilities is otherwise +permitted, and provided that you do these two things: + + a) Accompany the combined library with a copy of the same work + based on the Library, uncombined with any other library + facilities. This must be distributed under the terms of the + Sections above. + + b) Give prominent notice with the combined library of the fact + that part of it is a work based on the Library, and explaining + where to find the accompanying uncombined form of the same work. + + 8. You may not copy, modify, sublicense, link with, or distribute +the Library except as expressly provided under this License. Any +attempt otherwise to copy, modify, sublicense, link with, or +distribute the Library is void, and will automatically terminate your +rights under this License. However, parties who have received copies, +or rights, from you under this License will not have their licenses +terminated so long as such parties remain in full compliance. + + 9. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Library or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Library (or any work based on the +Library), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Library or works based on it. + + 10. Each time you redistribute the Library (or any work based on the +Library), the recipient automatically receives a license from the +original licensor to copy, distribute, link with or modify the Library +subject to these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties with +this License. + + 11. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Library at all. For example, if a patent +license would not permit royalty-free redistribution of the Library by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Library. + +If any portion of this section is held invalid or unenforceable under any +particular circumstance, the balance of the section is intended to apply, +and the section as a whole is intended to apply in other circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 12. If the distribution and/or use of the Library is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Library under this License may add +an explicit geographical distribution limitation excluding those countries, +so that distribution is permitted only in or among countries not thus +excluded. In such case, this License incorporates the limitation as if +written in the body of this License. + + 13. The Free Software Foundation may publish revised and/or new +versions of the Lesser General Public License from time to time. +Such new versions will be similar in spirit to the present version, +but may differ in detail to address new problems or concerns. + +Each version is given a distinguishing version number. If the Library +specifies a version number of this License which applies to it and +"any later version", you have the option of following the terms and +conditions either of that version or of any later version published by +the Free Software Foundation. If the Library does not specify a +license version number, you may choose any version ever published by +the Free Software Foundation. + + 14. If you wish to incorporate parts of the Library into other free +programs whose distribution conditions are incompatible with these, +write to the author to ask for permission. For software which is +copyrighted by the Free Software Foundation, write to the Free +Software Foundation; we sometimes make exceptions for this. Our +decision will be guided by the two goals of preserving the free status +of all derivatives of our free software and of promoting the sharing +and reuse of software generally. + + NO WARRANTY + + 15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO +WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW. +EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR +OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY +KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE +LIBRARY IS WITH YOU. SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME +THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN +WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY +AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU +FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR +CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE +LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING +RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A +FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF +SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH +DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Libraries + + If you develop a new library, and you want it to be of the greatest +possible use to the public, we recommend making it free software that +everyone can redistribute and change. You can do so by permitting +redistribution under these terms (or, alternatively, under the terms of the +ordinary General Public License). + + To apply these terms, attach the following notices to the library. It is +safest to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least the +"copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +Also add information on how to contact you by electronic and paper mail. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the library, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the + library `Frob' (a library for tweaking knobs) written by James Random Hacker. + + , 1 April 1990 + Ty Coon, President of Vice + +That's all there is to it! + + diff --git a/lib/spack/external/nose/__init__.py b/lib/spack/external/nose/__init__.py new file mode 100644 index 0000000000..1ae1362b7a --- /dev/null +++ b/lib/spack/external/nose/__init__.py @@ -0,0 +1,15 @@ +from nose.core import collector, main, run, run_exit, runmodule +# backwards compatibility +from nose.exc import SkipTest, DeprecatedTest +from nose.tools import with_setup + +__author__ = 'Jason Pellerin' +__versioninfo__ = (1, 3, 7) +__version__ = '.'.join(map(str, __versioninfo__)) + +__all__ = [ + 'main', 'run', 'run_exit', 'runmodule', 'with_setup', + 'SkipTest', 'DeprecatedTest', 'collector' + ] + + diff --git a/lib/spack/external/nose/__main__.py b/lib/spack/external/nose/__main__.py new file mode 100644 index 0000000000..b402d9df12 --- /dev/null +++ b/lib/spack/external/nose/__main__.py @@ -0,0 +1,8 @@ +import sys + +from nose.core import run_exit + +if sys.argv[0].endswith('__main__.py'): + sys.argv[0] = '%s -m nose' % sys.executable + +run_exit() diff --git a/lib/spack/external/nose/case.py b/lib/spack/external/nose/case.py new file mode 100644 index 0000000000..cffa4ab4c9 --- /dev/null +++ b/lib/spack/external/nose/case.py @@ -0,0 +1,397 @@ +"""nose unittest.TestCase subclasses. It is not necessary to subclass these +classes when writing tests; they are used internally by nose.loader.TestLoader +to create test cases from test functions and methods in test classes. +""" +import logging +import sys +import unittest +from inspect import isfunction +from nose.config import Config +from nose.failure import Failure # for backwards compatibility +from nose.util import resolve_name, test_address, try_run + +log = logging.getLogger(__name__) + + +__all__ = ['Test'] + + +class Test(unittest.TestCase): + """The universal test case wrapper. + + When a plugin sees a test, it will always see an instance of this + class. To access the actual test case that will be run, access the + test property of the nose.case.Test instance. + """ + __test__ = False # do not collect + def __init__(self, test, config=None, resultProxy=None): + # sanity check + if not callable(test): + raise TypeError("nose.case.Test called with argument %r that " + "is not callable. A callable is required." + % test) + self.test = test + if config is None: + config = Config() + self.config = config + self.tbinfo = None + self.capturedOutput = None + self.resultProxy = resultProxy + self.plugins = config.plugins + self.passed = None + unittest.TestCase.__init__(self) + + def __call__(self, *arg, **kwarg): + return self.run(*arg, **kwarg) + + def __str__(self): + name = self.plugins.testName(self) + if name is not None: + return name + return str(self.test) + + def __repr__(self): + return "Test(%r)" % self.test + + def afterTest(self, result): + """Called after test is complete (after result.stopTest) + """ + try: + afterTest = result.afterTest + except AttributeError: + pass + else: + afterTest(self.test) + + def beforeTest(self, result): + """Called before test is run (before result.startTest) + """ + try: + beforeTest = result.beforeTest + except AttributeError: + pass + else: + beforeTest(self.test) + + def exc_info(self): + """Extract exception info. + """ + exc, exv, tb = sys.exc_info() + return (exc, exv, tb) + + def id(self): + """Get a short(er) description of the test + """ + return self.test.id() + + def address(self): + """Return a round-trip name for this test, a name that can be + fed back as input to loadTestByName and (assuming the same + plugin configuration) result in the loading of this test. + """ + if hasattr(self.test, 'address'): + return self.test.address() + else: + # not a nose case + return test_address(self.test) + + def _context(self): + try: + return self.test.context + except AttributeError: + pass + try: + return self.test.__class__ + except AttributeError: + pass + try: + return resolve_name(self.test.__module__) + except AttributeError: + pass + return None + context = property(_context, None, None, + """Get the context object of this test (if any).""") + + def run(self, result): + """Modified run for the test wrapper. + + From here we don't call result.startTest or stopTest or + addSuccess. The wrapper calls addError/addFailure only if its + own setup or teardown fails, or running the wrapped test fails + (eg, if the wrapped "test" is not callable). + + Two additional methods are called, beforeTest and + afterTest. These give plugins a chance to modify the wrapped + test before it is called and do cleanup after it is + called. They are called unconditionally. + """ + if self.resultProxy: + result = self.resultProxy(result, self) + try: + try: + self.beforeTest(result) + self.runTest(result) + except KeyboardInterrupt: + raise + except: + err = sys.exc_info() + result.addError(self, err) + finally: + self.afterTest(result) + + def runTest(self, result): + """Run the test. Plugins may alter the test by returning a + value from prepareTestCase. The value must be callable and + must accept one argument, the result instance. + """ + test = self.test + plug_test = self.config.plugins.prepareTestCase(self) + if plug_test is not None: + test = plug_test + test(result) + + def shortDescription(self): + desc = self.plugins.describeTest(self) + if desc is not None: + return desc + # work around bug in unittest.TestCase.shortDescription + # with multiline docstrings. + test = self.test + try: + test._testMethodDoc = test._testMethodDoc.strip()# 2.5 + except AttributeError: + try: + # 2.4 and earlier + test._TestCase__testMethodDoc = \ + test._TestCase__testMethodDoc.strip() + except AttributeError: + pass + # 2.7 compat: shortDescription() always returns something + # which is a change from 2.6 and below, and breaks the + # testName plugin call. + try: + desc = self.test.shortDescription() + except Exception: + # this is probably caused by a problem in test.__str__() and is + # only triggered by python 3.1's unittest! + pass + try: + if desc == str(self.test): + return + except Exception: + # If str() triggers an exception then ignore it. + # see issue 422 + pass + return desc + + +class TestBase(unittest.TestCase): + """Common functionality for FunctionTestCase and MethodTestCase. + """ + __test__ = False # do not collect + + def id(self): + return str(self) + + def runTest(self): + self.test(*self.arg) + + def shortDescription(self): + if hasattr(self.test, 'description'): + return self.test.description + func, arg = self._descriptors() + doc = getattr(func, '__doc__', None) + if not doc: + doc = str(self) + return doc.strip().split("\n")[0].strip() + + +class FunctionTestCase(TestBase): + """TestCase wrapper for test functions. + + Don't use this class directly; it is used internally in nose to + create test cases for test functions. + """ + __test__ = False # do not collect + + def __init__(self, test, setUp=None, tearDown=None, arg=tuple(), + descriptor=None): + """Initialize the MethodTestCase. + + Required argument: + + * test -- the test function to call. + + Optional arguments: + + * setUp -- function to run at setup. + + * tearDown -- function to run at teardown. + + * arg -- arguments to pass to the test function. This is to support + generator functions that yield arguments. + + * descriptor -- the function, other than the test, that should be used + to construct the test name. This is to support generator functions. + """ + + self.test = test + self.setUpFunc = setUp + self.tearDownFunc = tearDown + self.arg = arg + self.descriptor = descriptor + TestBase.__init__(self) + + def address(self): + """Return a round-trip name for this test, a name that can be + fed back as input to loadTestByName and (assuming the same + plugin configuration) result in the loading of this test. + """ + if self.descriptor is not None: + return test_address(self.descriptor) + else: + return test_address(self.test) + + def _context(self): + return resolve_name(self.test.__module__) + context = property(_context, None, None, + """Get context (module) of this test""") + + def setUp(self): + """Run any setup function attached to the test function + """ + if self.setUpFunc: + self.setUpFunc() + else: + names = ('setup', 'setUp', 'setUpFunc') + try_run(self.test, names) + + def tearDown(self): + """Run any teardown function attached to the test function + """ + if self.tearDownFunc: + self.tearDownFunc() + else: + names = ('teardown', 'tearDown', 'tearDownFunc') + try_run(self.test, names) + + def __str__(self): + func, arg = self._descriptors() + if hasattr(func, 'compat_func_name'): + name = func.compat_func_name + else: + name = func.__name__ + name = "%s.%s" % (func.__module__, name) + if arg: + name = "%s%s" % (name, arg) + # FIXME need to include the full dir path to disambiguate + # in cases where test module of the same name was seen in + # another directory (old fromDirectory) + return name + __repr__ = __str__ + + def _descriptors(self): + """Get the descriptors of the test function: the function and + arguments that will be used to construct the test name. In + most cases, this is the function itself and no arguments. For + tests generated by generator functions, the original + (generator) function and args passed to the generated function + are returned. + """ + if self.descriptor: + return self.descriptor, self.arg + else: + return self.test, self.arg + + +class MethodTestCase(TestBase): + """Test case wrapper for test methods. + + Don't use this class directly; it is used internally in nose to + create test cases for test methods. + """ + __test__ = False # do not collect + + def __init__(self, method, test=None, arg=tuple(), descriptor=None): + """Initialize the MethodTestCase. + + Required argument: + + * method -- the method to call, may be bound or unbound. In either + case, a new instance of the method's class will be instantiated to + make the call. Note: In Python 3.x, if using an unbound method, you + must wrap it using pyversion.unbound_method. + + Optional arguments: + + * test -- the test function to call. If this is passed, it will be + called instead of getting a new bound method of the same name as the + desired method from the test instance. This is to support generator + methods that yield inline functions. + + * arg -- arguments to pass to the test function. This is to support + generator methods that yield arguments. + + * descriptor -- the function, other than the test, that should be used + to construct the test name. This is to support generator methods. + """ + self.method = method + self.test = test + self.arg = arg + self.descriptor = descriptor + if isfunction(method): + raise ValueError("Unbound methods must be wrapped using pyversion.unbound_method before passing to MethodTestCase") + self.cls = method.im_class + self.inst = self.cls() + if self.test is None: + method_name = self.method.__name__ + self.test = getattr(self.inst, method_name) + TestBase.__init__(self) + + def __str__(self): + func, arg = self._descriptors() + if hasattr(func, 'compat_func_name'): + name = func.compat_func_name + else: + name = func.__name__ + name = "%s.%s.%s" % (self.cls.__module__, + self.cls.__name__, + name) + if arg: + name = "%s%s" % (name, arg) + return name + __repr__ = __str__ + + def address(self): + """Return a round-trip name for this test, a name that can be + fed back as input to loadTestByName and (assuming the same + plugin configuration) result in the loading of this test. + """ + if self.descriptor is not None: + return test_address(self.descriptor) + else: + return test_address(self.method) + + def _context(self): + return self.cls + context = property(_context, None, None, + """Get context (class) of this test""") + + def setUp(self): + try_run(self.inst, ('setup', 'setUp')) + + def tearDown(self): + try_run(self.inst, ('teardown', 'tearDown')) + + def _descriptors(self): + """Get the descriptors of the test method: the method and + arguments that will be used to construct the test name. In + most cases, this is the method itself and no arguments. For + tests generated by generator methods, the original + (generator) method and args passed to the generated method + or function are returned. + """ + if self.descriptor: + return self.descriptor, self.arg + else: + return self.method, self.arg diff --git a/lib/spack/external/nose/commands.py b/lib/spack/external/nose/commands.py new file mode 100644 index 0000000000..ef0e9caed4 --- /dev/null +++ b/lib/spack/external/nose/commands.py @@ -0,0 +1,172 @@ +""" +nosetests setuptools command +---------------------------- + +The easiest way to run tests with nose is to use the `nosetests` setuptools +command:: + + python setup.py nosetests + +This command has one *major* benefit over the standard `test` command: *all +nose plugins are supported*. + +To configure the `nosetests` command, add a [nosetests] section to your +setup.cfg. The [nosetests] section can contain any command line arguments that +nosetests supports. The differences between issuing an option on the command +line and adding it to setup.cfg are: + +* In setup.cfg, the -- prefix must be excluded +* In setup.cfg, command line flags that take no arguments must be given an + argument flag (1, T or TRUE for active, 0, F or FALSE for inactive) + +Here's an example [nosetests] setup.cfg section:: + + [nosetests] + verbosity=1 + detailed-errors=1 + with-coverage=1 + cover-package=nose + debug=nose.loader + pdb=1 + pdb-failures=1 + +If you commonly run nosetests with a large number of options, using +the nosetests setuptools command and configuring with setup.cfg can +make running your tests much less tedious. (Note that the same options +and format supported in setup.cfg are supported in all other config +files, and the nosetests script will also load config files.) + +Another reason to run tests with the command is that the command will +install packages listed in your `tests_require`, as well as doing a +complete build of your package before running tests. For packages with +dependencies or that build C extensions, using the setuptools command +can be more convenient than building by hand and running the nosetests +script. + +Bootstrapping +------------- + +If you are distributing your project and want users to be able to run tests +without having to install nose themselves, add nose to the setup_requires +section of your setup():: + + setup( + # ... + setup_requires=['nose>=1.0'] + ) + +This will direct setuptools to download and activate nose during the setup +process, making the ``nosetests`` command available. + +""" +try: + from setuptools import Command +except ImportError: + Command = nosetests = None +else: + from nose.config import Config, option_blacklist, user_config_files, \ + flag, _bool + from nose.core import TestProgram + from nose.plugins import DefaultPluginManager + + + def get_user_options(parser): + """convert a optparse option list into a distutils option tuple list""" + opt_list = [] + for opt in parser.option_list: + if opt._long_opts[0][2:] in option_blacklist: + continue + long_name = opt._long_opts[0][2:] + if opt.action not in ('store_true', 'store_false'): + long_name = long_name + "=" + short_name = None + if opt._short_opts: + short_name = opt._short_opts[0][1:] + opt_list.append((long_name, short_name, opt.help or "")) + return opt_list + + + class nosetests(Command): + description = "Run unit tests using nosetests" + __config = Config(files=user_config_files(), + plugins=DefaultPluginManager()) + __parser = __config.getParser() + user_options = get_user_options(__parser) + + def initialize_options(self): + """create the member variables, but change hyphens to + underscores + """ + + self.option_to_cmds = {} + for opt in self.__parser.option_list: + cmd_name = opt._long_opts[0][2:] + option_name = cmd_name.replace('-', '_') + self.option_to_cmds[option_name] = cmd_name + setattr(self, option_name, None) + self.attr = None + + def finalize_options(self): + """nothing to do here""" + pass + + def run(self): + """ensure tests are capable of being run, then + run nose.main with a reconstructed argument list""" + if getattr(self.distribution, 'use_2to3', False): + # If we run 2to3 we can not do this inplace: + + # Ensure metadata is up-to-date + build_py = self.get_finalized_command('build_py') + build_py.inplace = 0 + build_py.run() + bpy_cmd = self.get_finalized_command("build_py") + build_path = bpy_cmd.build_lib + + # Build extensions + egg_info = self.get_finalized_command('egg_info') + egg_info.egg_base = build_path + egg_info.run() + + build_ext = self.get_finalized_command('build_ext') + build_ext.inplace = 0 + build_ext.run() + else: + self.run_command('egg_info') + + # Build extensions in-place + build_ext = self.get_finalized_command('build_ext') + build_ext.inplace = 1 + build_ext.run() + + if self.distribution.install_requires: + self.distribution.fetch_build_eggs( + self.distribution.install_requires) + if self.distribution.tests_require: + self.distribution.fetch_build_eggs( + self.distribution.tests_require) + + ei_cmd = self.get_finalized_command("egg_info") + argv = ['nosetests', '--where', ei_cmd.egg_base] + for (option_name, cmd_name) in self.option_to_cmds.items(): + if option_name in option_blacklist: + continue + value = getattr(self, option_name) + if value is not None: + argv.extend( + self.cfgToArg(option_name.replace('_', '-'), value)) + TestProgram(argv=argv, config=self.__config) + + def cfgToArg(self, optname, value): + argv = [] + long_optname = '--' + optname + opt = self.__parser.get_option(long_optname) + if opt.action in ('store_true', 'store_false'): + if not flag(value): + raise ValueError("Invalid value '%s' for '%s'" % ( + value, optname)) + if _bool(value): + argv.append(long_optname) + else: + argv.extend([long_optname, value]) + return argv diff --git a/lib/spack/external/nose/config.py b/lib/spack/external/nose/config.py new file mode 100644 index 0000000000..125eb5579d --- /dev/null +++ b/lib/spack/external/nose/config.py @@ -0,0 +1,661 @@ +import logging +import optparse +import os +import re +import sys +import ConfigParser +from optparse import OptionParser +from nose.util import absdir, tolist +from nose.plugins.manager import NoPlugins +from warnings import warn, filterwarnings + +log = logging.getLogger(__name__) + +# not allowed in config files +option_blacklist = ['help', 'verbose'] + +config_files = [ + # Linux users will prefer this + "~/.noserc", + # Windows users will prefer this + "~/nose.cfg" + ] + +# plaforms on which the exe check defaults to off +# Windows and IronPython +exe_allowed_platforms = ('win32', 'cli') + +filterwarnings("always", category=DeprecationWarning, + module=r'(.*\.)?nose\.config') + +class NoSuchOptionError(Exception): + def __init__(self, name): + Exception.__init__(self, name) + self.name = name + + +class ConfigError(Exception): + pass + + +class ConfiguredDefaultsOptionParser(object): + """ + Handler for options from commandline and config files. + """ + def __init__(self, parser, config_section, error=None, file_error=None): + self._parser = parser + self._config_section = config_section + if error is None: + error = self._parser.error + self._error = error + if file_error is None: + file_error = lambda msg, **kw: error(msg) + self._file_error = file_error + + def _configTuples(self, cfg, filename): + config = [] + if self._config_section in cfg.sections(): + for name, value in cfg.items(self._config_section): + config.append((name, value, filename)) + return config + + def _readFromFilenames(self, filenames): + config = [] + for filename in filenames: + cfg = ConfigParser.RawConfigParser() + try: + cfg.read(filename) + except ConfigParser.Error, exc: + raise ConfigError("Error reading config file %r: %s" % + (filename, str(exc))) + config.extend(self._configTuples(cfg, filename)) + return config + + def _readFromFileObject(self, fh): + cfg = ConfigParser.RawConfigParser() + try: + filename = fh.name + except AttributeError: + filename = '' + try: + cfg.readfp(fh) + except ConfigParser.Error, exc: + raise ConfigError("Error reading config file %r: %s" % + (filename, str(exc))) + return self._configTuples(cfg, filename) + + def _readConfiguration(self, config_files): + try: + config_files.readline + except AttributeError: + filename_or_filenames = config_files + if isinstance(filename_or_filenames, basestring): + filenames = [filename_or_filenames] + else: + filenames = filename_or_filenames + config = self._readFromFilenames(filenames) + else: + fh = config_files + config = self._readFromFileObject(fh) + return config + + def _processConfigValue(self, name, value, values, parser): + opt_str = '--' + name + option = parser.get_option(opt_str) + if option is None: + raise NoSuchOptionError(name) + else: + option.process(opt_str, value, values, parser) + + def _applyConfigurationToValues(self, parser, config, values): + for name, value, filename in config: + if name in option_blacklist: + continue + try: + self._processConfigValue(name, value, values, parser) + except NoSuchOptionError, exc: + self._file_error( + "Error reading config file %r: " + "no such option %r" % (filename, exc.name), + name=name, filename=filename) + except optparse.OptionValueError, exc: + msg = str(exc).replace('--' + name, repr(name), 1) + self._file_error("Error reading config file %r: " + "%s" % (filename, msg), + name=name, filename=filename) + + def parseArgsAndConfigFiles(self, args, config_files): + values = self._parser.get_default_values() + try: + config = self._readConfiguration(config_files) + except ConfigError, exc: + self._error(str(exc)) + else: + try: + self._applyConfigurationToValues(self._parser, config, values) + except ConfigError, exc: + self._error(str(exc)) + return self._parser.parse_args(args, values) + + +class Config(object): + """nose configuration. + + Instances of Config are used throughout nose to configure + behavior, including plugin lists. Here are the default values for + all config keys:: + + self.env = env = kw.pop('env', {}) + self.args = () + self.testMatch = re.compile(r'(?:^|[\\b_\\.%s-])[Tt]est' % os.sep) + self.addPaths = not env.get('NOSE_NOPATH', False) + self.configSection = 'nosetests' + self.debug = env.get('NOSE_DEBUG') + self.debugLog = env.get('NOSE_DEBUG_LOG') + self.exclude = None + self.getTestCaseNamesCompat = False + self.includeExe = env.get('NOSE_INCLUDE_EXE', + sys.platform in exe_allowed_platforms) + self.ignoreFiles = (re.compile(r'^\.'), + re.compile(r'^_'), + re.compile(r'^setup\.py$') + ) + self.include = None + self.loggingConfig = None + self.logStream = sys.stderr + self.options = NoOptions() + self.parser = None + self.plugins = NoPlugins() + self.srcDirs = ('lib', 'src') + self.runOnInit = True + self.stopOnError = env.get('NOSE_STOP', False) + self.stream = sys.stderr + self.testNames = () + self.verbosity = int(env.get('NOSE_VERBOSE', 1)) + self.where = () + self.py3where = () + self.workingDir = None + """ + + def __init__(self, **kw): + self.env = env = kw.pop('env', {}) + self.args = () + self.testMatchPat = env.get('NOSE_TESTMATCH', + r'(?:^|[\b_\.%s-])[Tt]est' % os.sep) + self.testMatch = re.compile(self.testMatchPat) + self.addPaths = not env.get('NOSE_NOPATH', False) + self.configSection = 'nosetests' + self.debug = env.get('NOSE_DEBUG') + self.debugLog = env.get('NOSE_DEBUG_LOG') + self.exclude = None + self.getTestCaseNamesCompat = False + self.includeExe = env.get('NOSE_INCLUDE_EXE', + sys.platform in exe_allowed_platforms) + self.ignoreFilesDefaultStrings = [r'^\.', + r'^_', + r'^setup\.py$', + ] + self.ignoreFiles = map(re.compile, self.ignoreFilesDefaultStrings) + self.include = None + self.loggingConfig = None + self.logStream = sys.stderr + self.options = NoOptions() + self.parser = None + self.plugins = NoPlugins() + self.srcDirs = ('lib', 'src') + self.runOnInit = True + self.stopOnError = env.get('NOSE_STOP', False) + self.stream = sys.stderr + self.testNames = [] + self.verbosity = int(env.get('NOSE_VERBOSE', 1)) + self.where = () + self.py3where = () + self.workingDir = os.getcwd() + self.traverseNamespace = False + self.firstPackageWins = False + self.parserClass = OptionParser + self.worker = False + + self._default = self.__dict__.copy() + self.update(kw) + self._orig = self.__dict__.copy() + + def __getstate__(self): + state = self.__dict__.copy() + del state['stream'] + del state['_orig'] + del state['_default'] + del state['env'] + del state['logStream'] + # FIXME remove plugins, have only plugin manager class + state['plugins'] = self.plugins.__class__ + return state + + def __setstate__(self, state): + plugincls = state.pop('plugins') + self.update(state) + self.worker = True + # FIXME won't work for static plugin lists + self.plugins = plugincls() + self.plugins.loadPlugins() + # needed so .can_configure gets set appropriately + dummy_parser = self.parserClass() + self.plugins.addOptions(dummy_parser, {}) + self.plugins.configure(self.options, self) + + def __repr__(self): + d = self.__dict__.copy() + # don't expose env, could include sensitive info + d['env'] = {} + keys = [ k for k in d.keys() + if not k.startswith('_') ] + keys.sort() + return "Config(%s)" % ', '.join([ '%s=%r' % (k, d[k]) + for k in keys ]) + __str__ = __repr__ + + def _parseArgs(self, argv, cfg_files): + def warn_sometimes(msg, name=None, filename=None): + if (hasattr(self.plugins, 'excludedOption') and + self.plugins.excludedOption(name)): + msg = ("Option %r in config file %r ignored: " + "excluded by runtime environment" % + (name, filename)) + warn(msg, RuntimeWarning) + else: + raise ConfigError(msg) + parser = ConfiguredDefaultsOptionParser( + self.getParser(), self.configSection, file_error=warn_sometimes) + return parser.parseArgsAndConfigFiles(argv[1:], cfg_files) + + def configure(self, argv=None, doc=None): + """Configure the nose running environment. Execute configure before + collecting tests with nose.TestCollector to enable output capture and + other features. + """ + env = self.env + if argv is None: + argv = sys.argv + + cfg_files = getattr(self, 'files', []) + options, args = self._parseArgs(argv, cfg_files) + # If -c --config has been specified on command line, + # load those config files and reparse + if getattr(options, 'files', []): + options, args = self._parseArgs(argv, options.files) + + self.options = options + if args: + self.testNames = args + if options.testNames is not None: + self.testNames.extend(tolist(options.testNames)) + + if options.py3where is not None: + if sys.version_info >= (3,): + options.where = options.py3where + + # `where` is an append action, so it can't have a default value + # in the parser, or that default will always be in the list + if not options.where: + options.where = env.get('NOSE_WHERE', None) + + # include and exclude also + if not options.ignoreFiles: + options.ignoreFiles = env.get('NOSE_IGNORE_FILES', []) + if not options.include: + options.include = env.get('NOSE_INCLUDE', []) + if not options.exclude: + options.exclude = env.get('NOSE_EXCLUDE', []) + + self.addPaths = options.addPaths + self.stopOnError = options.stopOnError + self.verbosity = options.verbosity + self.includeExe = options.includeExe + self.traverseNamespace = options.traverseNamespace + self.debug = options.debug + self.debugLog = options.debugLog + self.loggingConfig = options.loggingConfig + self.firstPackageWins = options.firstPackageWins + self.configureLogging() + + if not options.byteCompile: + sys.dont_write_bytecode = True + + if options.where is not None: + self.configureWhere(options.where) + + if options.testMatch: + self.testMatch = re.compile(options.testMatch) + + if options.ignoreFiles: + self.ignoreFiles = map(re.compile, tolist(options.ignoreFiles)) + log.info("Ignoring files matching %s", options.ignoreFiles) + else: + log.info("Ignoring files matching %s", self.ignoreFilesDefaultStrings) + + if options.include: + self.include = map(re.compile, tolist(options.include)) + log.info("Including tests matching %s", options.include) + + if options.exclude: + self.exclude = map(re.compile, tolist(options.exclude)) + log.info("Excluding tests matching %s", options.exclude) + + # When listing plugins we don't want to run them + if not options.showPlugins: + self.plugins.configure(options, self) + self.plugins.begin() + + def configureLogging(self): + """Configure logging for nose, or optionally other packages. Any logger + name may be set with the debug option, and that logger will be set to + debug level and be assigned the same handler as the nose loggers, unless + it already has a handler. + """ + if self.loggingConfig: + from logging.config import fileConfig + fileConfig(self.loggingConfig) + return + + format = logging.Formatter('%(name)s: %(levelname)s: %(message)s') + if self.debugLog: + handler = logging.FileHandler(self.debugLog) + else: + handler = logging.StreamHandler(self.logStream) + handler.setFormatter(format) + + logger = logging.getLogger('nose') + logger.propagate = 0 + + # only add our default handler if there isn't already one there + # this avoids annoying duplicate log messages. + found = False + if self.debugLog: + debugLogAbsPath = os.path.abspath(self.debugLog) + for h in logger.handlers: + if type(h) == logging.FileHandler and \ + h.baseFilename == debugLogAbsPath: + found = True + else: + for h in logger.handlers: + if type(h) == logging.StreamHandler and \ + h.stream == self.logStream: + found = True + if not found: + logger.addHandler(handler) + + # default level + lvl = logging.WARNING + if self.verbosity >= 5: + lvl = 0 + elif self.verbosity >= 4: + lvl = logging.DEBUG + elif self.verbosity >= 3: + lvl = logging.INFO + logger.setLevel(lvl) + + # individual overrides + if self.debug: + # no blanks + debug_loggers = [ name for name in self.debug.split(',') + if name ] + for logger_name in debug_loggers: + l = logging.getLogger(logger_name) + l.setLevel(logging.DEBUG) + if not l.handlers and not logger_name.startswith('nose'): + l.addHandler(handler) + + def configureWhere(self, where): + """Configure the working directory or directories for the test run. + """ + from nose.importer import add_path + self.workingDir = None + where = tolist(where) + warned = False + for path in where: + if not self.workingDir: + abs_path = absdir(path) + if abs_path is None: + raise ValueError("Working directory '%s' not found, or " + "not a directory" % path) + log.info("Set working dir to %s", abs_path) + self.workingDir = abs_path + if self.addPaths and \ + os.path.exists(os.path.join(abs_path, '__init__.py')): + log.info("Working directory %s is a package; " + "adding to sys.path" % abs_path) + add_path(abs_path) + continue + if not warned: + warn("Use of multiple -w arguments is deprecated and " + "support may be removed in a future release. You can " + "get the same behavior by passing directories without " + "the -w argument on the command line, or by using the " + "--tests argument in a configuration file.", + DeprecationWarning) + warned = True + self.testNames.append(path) + + def default(self): + """Reset all config values to defaults. + """ + self.__dict__.update(self._default) + + def getParser(self, doc=None): + """Get the command line option parser. + """ + if self.parser: + return self.parser + env = self.env + parser = self.parserClass(doc) + parser.add_option( + "-V","--version", action="store_true", + dest="version", default=False, + help="Output nose version and exit") + parser.add_option( + "-p", "--plugins", action="store_true", + dest="showPlugins", default=False, + help="Output list of available plugins and exit. Combine with " + "higher verbosity for greater detail") + parser.add_option( + "-v", "--verbose", + action="count", dest="verbosity", + default=self.verbosity, + help="Be more verbose. [NOSE_VERBOSE]") + parser.add_option( + "--verbosity", action="store", dest="verbosity", + metavar='VERBOSITY', + type="int", help="Set verbosity; --verbosity=2 is " + "the same as -v") + parser.add_option( + "-q", "--quiet", action="store_const", const=0, dest="verbosity", + help="Be less verbose") + parser.add_option( + "-c", "--config", action="append", dest="files", + metavar="FILES", + help="Load configuration from config file(s). May be specified " + "multiple times; in that case, all config files will be " + "loaded and combined") + parser.add_option( + "-w", "--where", action="append", dest="where", + metavar="WHERE", + help="Look for tests in this directory. " + "May be specified multiple times. The first directory passed " + "will be used as the working directory, in place of the current " + "working directory, which is the default. Others will be added " + "to the list of tests to execute. [NOSE_WHERE]" + ) + parser.add_option( + "--py3where", action="append", dest="py3where", + metavar="PY3WHERE", + help="Look for tests in this directory under Python 3.x. " + "Functions the same as 'where', but only applies if running under " + "Python 3.x or above. Note that, if present under 3.x, this " + "option completely replaces any directories specified with " + "'where', so the 'where' option becomes ineffective. " + "[NOSE_PY3WHERE]" + ) + parser.add_option( + "-m", "--match", "--testmatch", action="store", + dest="testMatch", metavar="REGEX", + help="Files, directories, function names, and class names " + "that match this regular expression are considered tests. " + "Default: %s [NOSE_TESTMATCH]" % self.testMatchPat, + default=self.testMatchPat) + parser.add_option( + "--tests", action="store", dest="testNames", default=None, + metavar='NAMES', + help="Run these tests (comma-separated list). This argument is " + "useful mainly from configuration files; on the command line, " + "just pass the tests to run as additional arguments with no " + "switch.") + parser.add_option( + "-l", "--debug", action="store", + dest="debug", default=self.debug, + help="Activate debug logging for one or more systems. " + "Available debug loggers: nose, nose.importer, " + "nose.inspector, nose.plugins, nose.result and " + "nose.selector. Separate multiple names with a comma.") + parser.add_option( + "--debug-log", dest="debugLog", action="store", + default=self.debugLog, metavar="FILE", + help="Log debug messages to this file " + "(default: sys.stderr)") + parser.add_option( + "--logging-config", "--log-config", + dest="loggingConfig", action="store", + default=self.loggingConfig, metavar="FILE", + help="Load logging config from this file -- bypasses all other" + " logging config settings.") + parser.add_option( + "-I", "--ignore-files", action="append", dest="ignoreFiles", + metavar="REGEX", + help="Completely ignore any file that matches this regular " + "expression. Takes precedence over any other settings or " + "plugins. " + "Specifying this option will replace the default setting. " + "Specify this option multiple times " + "to add more regular expressions [NOSE_IGNORE_FILES]") + parser.add_option( + "-e", "--exclude", action="append", dest="exclude", + metavar="REGEX", + help="Don't run tests that match regular " + "expression [NOSE_EXCLUDE]") + parser.add_option( + "-i", "--include", action="append", dest="include", + metavar="REGEX", + help="This regular expression will be applied to files, " + "directories, function names, and class names for a chance " + "to include additional tests that do not match TESTMATCH. " + "Specify this option multiple times " + "to add more regular expressions [NOSE_INCLUDE]") + parser.add_option( + "-x", "--stop", action="store_true", dest="stopOnError", + default=self.stopOnError, + help="Stop running tests after the first error or failure") + parser.add_option( + "-P", "--no-path-adjustment", action="store_false", + dest="addPaths", + default=self.addPaths, + help="Don't make any changes to sys.path when " + "loading tests [NOSE_NOPATH]") + parser.add_option( + "--exe", action="store_true", dest="includeExe", + default=self.includeExe, + help="Look for tests in python modules that are " + "executable. Normal behavior is to exclude executable " + "modules, since they may not be import-safe " + "[NOSE_INCLUDE_EXE]") + parser.add_option( + "--noexe", action="store_false", dest="includeExe", + help="DO NOT look for tests in python modules that are " + "executable. (The default on the windows platform is to " + "do so.)") + parser.add_option( + "--traverse-namespace", action="store_true", + default=self.traverseNamespace, dest="traverseNamespace", + help="Traverse through all path entries of a namespace package") + parser.add_option( + "--first-package-wins", "--first-pkg-wins", "--1st-pkg-wins", + action="store_true", default=False, dest="firstPackageWins", + help="nose's importer will normally evict a package from sys." + "modules if it sees a package with the same name in a different " + "location. Set this option to disable that behavior.") + parser.add_option( + "--no-byte-compile", + action="store_false", default=True, dest="byteCompile", + help="Prevent nose from byte-compiling the source into .pyc files " + "while nose is scanning for and running tests.") + + self.plugins.loadPlugins() + self.pluginOpts(parser) + + self.parser = parser + return parser + + def help(self, doc=None): + """Return the generated help message + """ + return self.getParser(doc).format_help() + + def pluginOpts(self, parser): + self.plugins.addOptions(parser, self.env) + + def reset(self): + self.__dict__.update(self._orig) + + def todict(self): + return self.__dict__.copy() + + def update(self, d): + self.__dict__.update(d) + + +class NoOptions(object): + """Options container that returns None for all options. + """ + def __getstate__(self): + return {} + + def __setstate__(self, state): + pass + + def __getnewargs__(self): + return () + + def __nonzero__(self): + return False + + +def user_config_files(): + """Return path to any existing user config files + """ + return filter(os.path.exists, + map(os.path.expanduser, config_files)) + + +def all_config_files(): + """Return path to any existing user config files, plus any setup.cfg + in the current working directory. + """ + user = user_config_files() + if os.path.exists('setup.cfg'): + return user + ['setup.cfg'] + return user + + +# used when parsing config files +def flag(val): + """Does the value look like an on/off flag?""" + if val == 1: + return True + elif val == 0: + return False + val = str(val) + if len(val) > 5: + return False + return val.upper() in ('1', '0', 'F', 'T', 'TRUE', 'FALSE', 'ON', 'OFF') + + +def _bool(val): + return str(val).upper() in ('1', 'T', 'TRUE', 'ON') diff --git a/lib/spack/external/nose/core.py b/lib/spack/external/nose/core.py new file mode 100644 index 0000000000..49e7939b98 --- /dev/null +++ b/lib/spack/external/nose/core.py @@ -0,0 +1,341 @@ +"""Implements nose test program and collector. +""" +from __future__ import generators + +import logging +import os +import sys +import time +import unittest + +from nose.config import Config, all_config_files +from nose.loader import defaultTestLoader +from nose.plugins.manager import PluginManager, DefaultPluginManager, \ + RestrictedPluginManager +from nose.result import TextTestResult +from nose.suite import FinalizingSuiteWrapper +from nose.util import isclass, tolist + + +log = logging.getLogger('nose.core') +compat_24 = sys.version_info >= (2, 4) + +__all__ = ['TestProgram', 'main', 'run', 'run_exit', 'runmodule', 'collector', + 'TextTestRunner'] + + +class TextTestRunner(unittest.TextTestRunner): + """Test runner that uses nose's TextTestResult to enable errorClasses, + as well as providing hooks for plugins to override or replace the test + output stream, results, and the test case itself. + """ + def __init__(self, stream=sys.stderr, descriptions=1, verbosity=1, + config=None): + if config is None: + config = Config() + self.config = config + unittest.TextTestRunner.__init__(self, stream, descriptions, verbosity) + + + def _makeResult(self): + return TextTestResult(self.stream, + self.descriptions, + self.verbosity, + self.config) + + def run(self, test): + """Overrides to provide plugin hooks and defer all output to + the test result class. + """ + wrapper = self.config.plugins.prepareTest(test) + if wrapper is not None: + test = wrapper + + # plugins can decorate or capture the output stream + wrapped = self.config.plugins.setOutputStream(self.stream) + if wrapped is not None: + self.stream = wrapped + + result = self._makeResult() + start = time.time() + try: + test(result) + except KeyboardInterrupt: + pass + stop = time.time() + result.printErrors() + result.printSummary(start, stop) + self.config.plugins.finalize(result) + return result + + +class TestProgram(unittest.TestProgram): + """Collect and run tests, returning success or failure. + + The arguments to TestProgram() are the same as to + :func:`main()` and :func:`run()`: + + * module: All tests are in this module (default: None) + * defaultTest: Tests to load (default: '.') + * argv: Command line arguments (default: None; sys.argv is read) + * testRunner: Test runner instance (default: None) + * testLoader: Test loader instance (default: None) + * env: Environment; ignored if config is provided (default: None; + os.environ is read) + * config: :class:`nose.config.Config` instance (default: None) + * suite: Suite or list of tests to run (default: None). Passing a + suite or lists of tests will bypass all test discovery and + loading. *ALSO NOTE* that if you pass a unittest.TestSuite + instance as the suite, context fixtures at the class, module and + package level will not be used, and many plugin hooks will not + be called. If you want normal nose behavior, either pass a list + of tests, or a fully-configured :class:`nose.suite.ContextSuite`. + * exit: Exit after running tests and printing report (default: True) + * plugins: List of plugins to use; ignored if config is provided + (default: load plugins with DefaultPluginManager) + * addplugins: List of **extra** plugins to use. Pass a list of plugin + instances in this argument to make custom plugins available while + still using the DefaultPluginManager. + """ + verbosity = 1 + + def __init__(self, module=None, defaultTest='.', argv=None, + testRunner=None, testLoader=None, env=None, config=None, + suite=None, exit=True, plugins=None, addplugins=None): + if env is None: + env = os.environ + if config is None: + config = self.makeConfig(env, plugins) + if addplugins: + config.plugins.addPlugins(extraplugins=addplugins) + self.config = config + self.suite = suite + self.exit = exit + extra_args = {} + version = sys.version_info[0:2] + if version >= (2,7) and version != (3,0): + extra_args['exit'] = exit + unittest.TestProgram.__init__( + self, module=module, defaultTest=defaultTest, + argv=argv, testRunner=testRunner, testLoader=testLoader, + **extra_args) + + def getAllConfigFiles(self, env=None): + env = env or {} + if env.get('NOSE_IGNORE_CONFIG_FILES', False): + return [] + else: + return all_config_files() + + def makeConfig(self, env, plugins=None): + """Load a Config, pre-filled with user config files if any are + found. + """ + cfg_files = self.getAllConfigFiles(env) + if plugins: + manager = PluginManager(plugins=plugins) + else: + manager = DefaultPluginManager() + return Config( + env=env, files=cfg_files, plugins=manager) + + def parseArgs(self, argv): + """Parse argv and env and configure running environment. + """ + self.config.configure(argv, doc=self.usage()) + log.debug("configured %s", self.config) + + # quick outs: version, plugins (optparse would have already + # caught and exited on help) + if self.config.options.version: + from nose import __version__ + sys.stdout = sys.__stdout__ + print "%s version %s" % (os.path.basename(sys.argv[0]), __version__) + sys.exit(0) + + if self.config.options.showPlugins: + self.showPlugins() + sys.exit(0) + + if self.testLoader is None: + self.testLoader = defaultTestLoader(config=self.config) + elif isclass(self.testLoader): + self.testLoader = self.testLoader(config=self.config) + plug_loader = self.config.plugins.prepareTestLoader(self.testLoader) + if plug_loader is not None: + self.testLoader = plug_loader + log.debug("test loader is %s", self.testLoader) + + # FIXME if self.module is a string, add it to self.testNames? not sure + + if self.config.testNames: + self.testNames = self.config.testNames + else: + self.testNames = tolist(self.defaultTest) + log.debug('defaultTest %s', self.defaultTest) + log.debug('Test names are %s', self.testNames) + if self.config.workingDir is not None: + os.chdir(self.config.workingDir) + self.createTests() + + def createTests(self): + """Create the tests to run. If a self.suite + is set, then that suite will be used. Otherwise, tests will be + loaded from the given test names (self.testNames) using the + test loader. + """ + log.debug("createTests called with %s", self.suite) + if self.suite is not None: + # We were given an explicit suite to run. Make sure it's + # loaded and wrapped correctly. + self.test = self.testLoader.suiteClass(self.suite) + else: + self.test = self.testLoader.loadTestsFromNames(self.testNames) + + def runTests(self): + """Run Tests. Returns true on success, false on failure, and sets + self.success to the same value. + """ + log.debug("runTests called") + if self.testRunner is None: + self.testRunner = TextTestRunner(stream=self.config.stream, + verbosity=self.config.verbosity, + config=self.config) + plug_runner = self.config.plugins.prepareTestRunner(self.testRunner) + if plug_runner is not None: + self.testRunner = plug_runner + result = self.testRunner.run(self.test) + self.success = result.wasSuccessful() + if self.exit: + sys.exit(not self.success) + return self.success + + def showPlugins(self): + """Print list of available plugins. + """ + import textwrap + + class DummyParser: + def __init__(self): + self.options = [] + def add_option(self, *arg, **kw): + self.options.append((arg, kw.pop('help', ''))) + + v = self.config.verbosity + self.config.plugins.sort() + for p in self.config.plugins: + print "Plugin %s" % p.name + if v >= 2: + print " score: %s" % p.score + print '\n'.join(textwrap.wrap(p.help().strip(), + initial_indent=' ', + subsequent_indent=' ')) + if v >= 3: + parser = DummyParser() + p.addOptions(parser) + if len(parser.options): + print + print " Options:" + for opts, help in parser.options: + print ' %s' % (', '.join(opts)) + if help: + print '\n'.join( + textwrap.wrap(help.strip(), + initial_indent=' ', + subsequent_indent=' ')) + print + + def usage(cls): + import nose + try: + ld = nose.__loader__ + text = ld.get_data(os.path.join( + os.path.dirname(__file__), 'usage.txt')) + except AttributeError: + f = open(os.path.join( + os.path.dirname(__file__), 'usage.txt'), 'r') + try: + text = f.read() + finally: + f.close() + # Ensure that we return str, not bytes. + if not isinstance(text, str): + text = text.decode('utf-8') + return text + usage = classmethod(usage) + +# backwards compatibility +run_exit = main = TestProgram + + +def run(*arg, **kw): + """Collect and run tests, returning success or failure. + + The arguments to `run()` are the same as to `main()`: + + * module: All tests are in this module (default: None) + * defaultTest: Tests to load (default: '.') + * argv: Command line arguments (default: None; sys.argv is read) + * testRunner: Test runner instance (default: None) + * testLoader: Test loader instance (default: None) + * env: Environment; ignored if config is provided (default: None; + os.environ is read) + * config: :class:`nose.config.Config` instance (default: None) + * suite: Suite or list of tests to run (default: None). Passing a + suite or lists of tests will bypass all test discovery and + loading. *ALSO NOTE* that if you pass a unittest.TestSuite + instance as the suite, context fixtures at the class, module and + package level will not be used, and many plugin hooks will not + be called. If you want normal nose behavior, either pass a list + of tests, or a fully-configured :class:`nose.suite.ContextSuite`. + * plugins: List of plugins to use; ignored if config is provided + (default: load plugins with DefaultPluginManager) + * addplugins: List of **extra** plugins to use. Pass a list of plugin + instances in this argument to make custom plugins available while + still using the DefaultPluginManager. + + With the exception that the ``exit`` argument is always set + to False. + """ + kw['exit'] = False + return TestProgram(*arg, **kw).success + + +def runmodule(name='__main__', **kw): + """Collect and run tests in a single module only. Defaults to running + tests in __main__. Additional arguments to TestProgram may be passed + as keyword arguments. + """ + main(defaultTest=name, **kw) + + +def collector(): + """TestSuite replacement entry point. Use anywhere you might use a + unittest.TestSuite. The collector will, by default, load options from + all config files and execute loader.loadTestsFromNames() on the + configured testNames, or '.' if no testNames are configured. + """ + # plugins that implement any of these methods are disabled, since + # we don't control the test runner and won't be able to run them + # finalize() is also not called, but plugins that use it aren't disabled, + # because capture needs it. + setuptools_incompat = ('report', 'prepareTest', + 'prepareTestLoader', 'prepareTestRunner', + 'setOutputStream') + + plugins = RestrictedPluginManager(exclude=setuptools_incompat) + conf = Config(files=all_config_files(), + plugins=plugins) + conf.configure(argv=['collector']) + loader = defaultTestLoader(conf) + + if conf.testNames: + suite = loader.loadTestsFromNames(conf.testNames) + else: + suite = loader.loadTestsFromNames(('.',)) + return FinalizingSuiteWrapper(suite, plugins.finalize) + + + +if __name__ == '__main__': + main() diff --git a/lib/spack/external/nose/exc.py b/lib/spack/external/nose/exc.py new file mode 100644 index 0000000000..8b780db0d4 --- /dev/null +++ b/lib/spack/external/nose/exc.py @@ -0,0 +1,9 @@ +"""Exceptions for marking tests as skipped or deprecated. + +This module exists to provide backwards compatibility with previous +versions of nose where skipped and deprecated tests were core +functionality, rather than being provided by plugins. It may be +removed in a future release. +""" +from nose.plugins.skip import SkipTest +from nose.plugins.deprecated import DeprecatedTest diff --git a/lib/spack/external/nose/ext/__init__.py b/lib/spack/external/nose/ext/__init__.py new file mode 100644 index 0000000000..5fd1516a09 --- /dev/null +++ b/lib/spack/external/nose/ext/__init__.py @@ -0,0 +1,3 @@ +""" +External or vendor files +""" diff --git a/lib/spack/external/nose/ext/dtcompat.py b/lib/spack/external/nose/ext/dtcompat.py new file mode 100644 index 0000000000..332cf08c12 --- /dev/null +++ b/lib/spack/external/nose/ext/dtcompat.py @@ -0,0 +1,2272 @@ +# Module doctest. +# Released to the public domain 16-Jan-2001, by Tim Peters (tim@python.org). +# Major enhancements and refactoring by: +# Jim Fulton +# Edward Loper + +# Provided as-is; use at your own risk; no warranty; no promises; enjoy! +# +# Modified for inclusion in nose to provide support for DocFileTest in +# python 2.3: +# +# - all doctests removed from module (they fail under 2.3 and 2.5) +# - now handles the $py.class extension when ran under Jython + +r"""Module doctest -- a framework for running examples in docstrings. + +In simplest use, end each module M to be tested with: + +def _test(): + import doctest + doctest.testmod() + +if __name__ == "__main__": + _test() + +Then running the module as a script will cause the examples in the +docstrings to get executed and verified: + +python M.py + +This won't display anything unless an example fails, in which case the +failing example(s) and the cause(s) of the failure(s) are printed to stdout +(why not stderr? because stderr is a lame hack <0.2 wink>), and the final +line of output is "Test failed.". + +Run it with the -v switch instead: + +python M.py -v + +and a detailed report of all examples tried is printed to stdout, along +with assorted summaries at the end. + +You can force verbose mode by passing "verbose=True" to testmod, or prohibit +it by passing "verbose=False". In either of those cases, sys.argv is not +examined by testmod. + +There are a variety of other ways to run doctests, including integration +with the unittest framework, and support for running non-Python text +files containing doctests. There are also many ways to override parts +of doctest's default behaviors. See the Library Reference Manual for +details. +""" + +__docformat__ = 'reStructuredText en' + +__all__ = [ + # 0, Option Flags + 'register_optionflag', + 'DONT_ACCEPT_TRUE_FOR_1', + 'DONT_ACCEPT_BLANKLINE', + 'NORMALIZE_WHITESPACE', + 'ELLIPSIS', + 'IGNORE_EXCEPTION_DETAIL', + 'COMPARISON_FLAGS', + 'REPORT_UDIFF', + 'REPORT_CDIFF', + 'REPORT_NDIFF', + 'REPORT_ONLY_FIRST_FAILURE', + 'REPORTING_FLAGS', + # 1. Utility Functions + 'is_private', + # 2. Example & DocTest + 'Example', + 'DocTest', + # 3. Doctest Parser + 'DocTestParser', + # 4. Doctest Finder + 'DocTestFinder', + # 5. Doctest Runner + 'DocTestRunner', + 'OutputChecker', + 'DocTestFailure', + 'UnexpectedException', + 'DebugRunner', + # 6. Test Functions + 'testmod', + 'testfile', + 'run_docstring_examples', + # 7. Tester + 'Tester', + # 8. Unittest Support + 'DocTestSuite', + 'DocFileSuite', + 'set_unittest_reportflags', + # 9. Debugging Support + 'script_from_examples', + 'testsource', + 'debug_src', + 'debug', +] + +import __future__ + +import sys, traceback, inspect, linecache, os, re +import unittest, difflib, pdb, tempfile +import warnings +from StringIO import StringIO + +# Don't whine about the deprecated is_private function in this +# module's tests. +warnings.filterwarnings("ignore", "is_private", DeprecationWarning, + __name__, 0) + +# There are 4 basic classes: +# - Example: a pair, plus an intra-docstring line number. +# - DocTest: a collection of examples, parsed from a docstring, plus +# info about where the docstring came from (name, filename, lineno). +# - DocTestFinder: extracts DocTests from a given object's docstring and +# its contained objects' docstrings. +# - DocTestRunner: runs DocTest cases, and accumulates statistics. +# +# So the basic picture is: +# +# list of: +# +------+ +---------+ +-------+ +# |object| --DocTestFinder-> | DocTest | --DocTestRunner-> |results| +# +------+ +---------+ +-------+ +# | Example | +# | ... | +# | Example | +# +---------+ + +# Option constants. + +OPTIONFLAGS_BY_NAME = {} +def register_optionflag(name): + # Create a new flag unless `name` is already known. + return OPTIONFLAGS_BY_NAME.setdefault(name, 1 << len(OPTIONFLAGS_BY_NAME)) + +DONT_ACCEPT_TRUE_FOR_1 = register_optionflag('DONT_ACCEPT_TRUE_FOR_1') +DONT_ACCEPT_BLANKLINE = register_optionflag('DONT_ACCEPT_BLANKLINE') +NORMALIZE_WHITESPACE = register_optionflag('NORMALIZE_WHITESPACE') +ELLIPSIS = register_optionflag('ELLIPSIS') +IGNORE_EXCEPTION_DETAIL = register_optionflag('IGNORE_EXCEPTION_DETAIL') + +COMPARISON_FLAGS = (DONT_ACCEPT_TRUE_FOR_1 | + DONT_ACCEPT_BLANKLINE | + NORMALIZE_WHITESPACE | + ELLIPSIS | + IGNORE_EXCEPTION_DETAIL) + +REPORT_UDIFF = register_optionflag('REPORT_UDIFF') +REPORT_CDIFF = register_optionflag('REPORT_CDIFF') +REPORT_NDIFF = register_optionflag('REPORT_NDIFF') +REPORT_ONLY_FIRST_FAILURE = register_optionflag('REPORT_ONLY_FIRST_FAILURE') + +REPORTING_FLAGS = (REPORT_UDIFF | + REPORT_CDIFF | + REPORT_NDIFF | + REPORT_ONLY_FIRST_FAILURE) + +# Special string markers for use in `want` strings: +BLANKLINE_MARKER = '' +ELLIPSIS_MARKER = '...' + +###################################################################### +## Table of Contents +###################################################################### +# 1. Utility Functions +# 2. Example & DocTest -- store test cases +# 3. DocTest Parser -- extracts examples from strings +# 4. DocTest Finder -- extracts test cases from objects +# 5. DocTest Runner -- runs test cases +# 6. Test Functions -- convenient wrappers for testing +# 7. Tester Class -- for backwards compatibility +# 8. Unittest Support +# 9. Debugging Support +# 10. Example Usage + +###################################################################### +## 1. Utility Functions +###################################################################### + +def is_private(prefix, base): + """prefix, base -> true iff name prefix + "." + base is "private". + + Prefix may be an empty string, and base does not contain a period. + Prefix is ignored (although functions you write conforming to this + protocol may make use of it). + Return true iff base begins with an (at least one) underscore, but + does not both begin and end with (at least) two underscores. + """ + warnings.warn("is_private is deprecated; it wasn't useful; " + "examine DocTestFinder.find() lists instead", + DeprecationWarning, stacklevel=2) + return base[:1] == "_" and not base[:2] == "__" == base[-2:] + +def _extract_future_flags(globs): + """ + Return the compiler-flags associated with the future features that + have been imported into the given namespace (globs). + """ + flags = 0 + for fname in __future__.all_feature_names: + feature = globs.get(fname, None) + if feature is getattr(__future__, fname): + flags |= feature.compiler_flag + return flags + +def _normalize_module(module, depth=2): + """ + Return the module specified by `module`. In particular: + - If `module` is a module, then return module. + - If `module` is a string, then import and return the + module with that name. + - If `module` is None, then return the calling module. + The calling module is assumed to be the module of + the stack frame at the given depth in the call stack. + """ + if inspect.ismodule(module): + return module + elif isinstance(module, (str, unicode)): + return __import__(module, globals(), locals(), ["*"]) + elif module is None: + return sys.modules[sys._getframe(depth).f_globals['__name__']] + else: + raise TypeError("Expected a module, string, or None") + +def _indent(s, indent=4): + """ + Add the given number of space characters to the beginning every + non-blank line in `s`, and return the result. + """ + # This regexp matches the start of non-blank lines: + return re.sub('(?m)^(?!$)', indent*' ', s) + +def _exception_traceback(exc_info): + """ + Return a string containing a traceback message for the given + exc_info tuple (as returned by sys.exc_info()). + """ + # Get a traceback message. + excout = StringIO() + exc_type, exc_val, exc_tb = exc_info + traceback.print_exception(exc_type, exc_val, exc_tb, file=excout) + return excout.getvalue() + +# Override some StringIO methods. +class _SpoofOut(StringIO): + def getvalue(self): + result = StringIO.getvalue(self) + # If anything at all was written, make sure there's a trailing + # newline. There's no way for the expected output to indicate + # that a trailing newline is missing. + if result and not result.endswith("\n"): + result += "\n" + # Prevent softspace from screwing up the next test case, in + # case they used print with a trailing comma in an example. + if hasattr(self, "softspace"): + del self.softspace + return result + + def truncate(self, size=None): + StringIO.truncate(self, size) + if hasattr(self, "softspace"): + del self.softspace + +# Worst-case linear-time ellipsis matching. +def _ellipsis_match(want, got): + if ELLIPSIS_MARKER not in want: + return want == got + + # Find "the real" strings. + ws = want.split(ELLIPSIS_MARKER) + assert len(ws) >= 2 + + # Deal with exact matches possibly needed at one or both ends. + startpos, endpos = 0, len(got) + w = ws[0] + if w: # starts with exact match + if got.startswith(w): + startpos = len(w) + del ws[0] + else: + return False + w = ws[-1] + if w: # ends with exact match + if got.endswith(w): + endpos -= len(w) + del ws[-1] + else: + return False + + if startpos > endpos: + # Exact end matches required more characters than we have, as in + # _ellipsis_match('aa...aa', 'aaa') + return False + + # For the rest, we only need to find the leftmost non-overlapping + # match for each piece. If there's no overall match that way alone, + # there's no overall match period. + for w in ws: + # w may be '' at times, if there are consecutive ellipses, or + # due to an ellipsis at the start or end of `want`. That's OK. + # Search for an empty string succeeds, and doesn't change startpos. + startpos = got.find(w, startpos, endpos) + if startpos < 0: + return False + startpos += len(w) + + return True + +def _comment_line(line): + "Return a commented form of the given line" + line = line.rstrip() + if line: + return '# '+line + else: + return '#' + +class _OutputRedirectingPdb(pdb.Pdb): + """ + A specialized version of the python debugger that redirects stdout + to a given stream when interacting with the user. Stdout is *not* + redirected when traced code is executed. + """ + def __init__(self, out): + self.__out = out + pdb.Pdb.__init__(self) + + def trace_dispatch(self, *args): + # Redirect stdout to the given stream. + save_stdout = sys.stdout + sys.stdout = self.__out + # Call Pdb's trace dispatch method. + try: + return pdb.Pdb.trace_dispatch(self, *args) + finally: + sys.stdout = save_stdout + +# [XX] Normalize with respect to os.path.pardir? +def _module_relative_path(module, path): + if not inspect.ismodule(module): + raise TypeError, 'Expected a module: %r' % module + if path.startswith('/'): + raise ValueError, 'Module-relative files may not have absolute paths' + + # Find the base directory for the path. + if hasattr(module, '__file__'): + # A normal module/package + basedir = os.path.split(module.__file__)[0] + elif module.__name__ == '__main__': + # An interactive session. + if len(sys.argv)>0 and sys.argv[0] != '': + basedir = os.path.split(sys.argv[0])[0] + else: + basedir = os.curdir + else: + # A module w/o __file__ (this includes builtins) + raise ValueError("Can't resolve paths relative to the module " + + module + " (it has no __file__)") + + # Combine the base directory and the path. + return os.path.join(basedir, *(path.split('/'))) + +###################################################################### +## 2. Example & DocTest +###################################################################### +## - An "example" is a pair, where "source" is a +## fragment of source code, and "want" is the expected output for +## "source." The Example class also includes information about +## where the example was extracted from. +## +## - A "doctest" is a collection of examples, typically extracted from +## a string (such as an object's docstring). The DocTest class also +## includes information about where the string was extracted from. + +class Example: + """ + A single doctest example, consisting of source code and expected + output. `Example` defines the following attributes: + + - source: A single Python statement, always ending with a newline. + The constructor adds a newline if needed. + + - want: The expected output from running the source code (either + from stdout, or a traceback in case of exception). `want` ends + with a newline unless it's empty, in which case it's an empty + string. The constructor adds a newline if needed. + + - exc_msg: The exception message generated by the example, if + the example is expected to generate an exception; or `None` if + it is not expected to generate an exception. This exception + message is compared against the return value of + `traceback.format_exception_only()`. `exc_msg` ends with a + newline unless it's `None`. The constructor adds a newline + if needed. + + - lineno: The line number within the DocTest string containing + this Example where the Example begins. This line number is + zero-based, with respect to the beginning of the DocTest. + + - indent: The example's indentation in the DocTest string. + I.e., the number of space characters that preceed the + example's first prompt. + + - options: A dictionary mapping from option flags to True or + False, which is used to override default options for this + example. Any option flags not contained in this dictionary + are left at their default value (as specified by the + DocTestRunner's optionflags). By default, no options are set. + """ + def __init__(self, source, want, exc_msg=None, lineno=0, indent=0, + options=None): + # Normalize inputs. + if not source.endswith('\n'): + source += '\n' + if want and not want.endswith('\n'): + want += '\n' + if exc_msg is not None and not exc_msg.endswith('\n'): + exc_msg += '\n' + # Store properties. + self.source = source + self.want = want + self.lineno = lineno + self.indent = indent + if options is None: options = {} + self.options = options + self.exc_msg = exc_msg + +class DocTest: + """ + A collection of doctest examples that should be run in a single + namespace. Each `DocTest` defines the following attributes: + + - examples: the list of examples. + + - globs: The namespace (aka globals) that the examples should + be run in. + + - name: A name identifying the DocTest (typically, the name of + the object whose docstring this DocTest was extracted from). + + - filename: The name of the file that this DocTest was extracted + from, or `None` if the filename is unknown. + + - lineno: The line number within filename where this DocTest + begins, or `None` if the line number is unavailable. This + line number is zero-based, with respect to the beginning of + the file. + + - docstring: The string that the examples were extracted from, + or `None` if the string is unavailable. + """ + def __init__(self, examples, globs, name, filename, lineno, docstring): + """ + Create a new DocTest containing the given examples. The + DocTest's globals are initialized with a copy of `globs`. + """ + assert not isinstance(examples, basestring), \ + "DocTest no longer accepts str; use DocTestParser instead" + self.examples = examples + self.docstring = docstring + self.globs = globs.copy() + self.name = name + self.filename = filename + self.lineno = lineno + + def __repr__(self): + if len(self.examples) == 0: + examples = 'no examples' + elif len(self.examples) == 1: + examples = '1 example' + else: + examples = '%d examples' % len(self.examples) + return ('' % + (self.name, self.filename, self.lineno, examples)) + + + # This lets us sort tests by name: + def __cmp__(self, other): + if not isinstance(other, DocTest): + return -1 + return cmp((self.name, self.filename, self.lineno, id(self)), + (other.name, other.filename, other.lineno, id(other))) + +###################################################################### +## 3. DocTestParser +###################################################################### + +class DocTestParser: + """ + A class used to parse strings containing doctest examples. + """ + # This regular expression is used to find doctest examples in a + # string. It defines three groups: `source` is the source code + # (including leading indentation and prompts); `indent` is the + # indentation of the first (PS1) line of the source code; and + # `want` is the expected output (including leading indentation). + _EXAMPLE_RE = re.compile(r''' + # Source consists of a PS1 line followed by zero or more PS2 lines. + (?P + (?:^(?P [ ]*) >>> .*) # PS1 line + (?:\n [ ]* \.\.\. .*)*) # PS2 lines + \n? + # Want consists of any non-blank lines that do not start with PS1. + (?P (?:(?![ ]*$) # Not a blank line + (?![ ]*>>>) # Not a line starting with PS1 + .*$\n? # But any other line + )*) + ''', re.MULTILINE | re.VERBOSE) + + # A regular expression for handling `want` strings that contain + # expected exceptions. It divides `want` into three pieces: + # - the traceback header line (`hdr`) + # - the traceback stack (`stack`) + # - the exception message (`msg`), as generated by + # traceback.format_exception_only() + # `msg` may have multiple lines. We assume/require that the + # exception message is the first non-indented line starting with a word + # character following the traceback header line. + _EXCEPTION_RE = re.compile(r""" + # Grab the traceback header. Different versions of Python have + # said different things on the first traceback line. + ^(?P Traceback\ \( + (?: most\ recent\ call\ last + | innermost\ last + ) \) : + ) + \s* $ # toss trailing whitespace on the header. + (?P .*?) # don't blink: absorb stuff until... + ^ (?P \w+ .*) # a line *starts* with alphanum. + """, re.VERBOSE | re.MULTILINE | re.DOTALL) + + # A callable returning a true value iff its argument is a blank line + # or contains a single comment. + _IS_BLANK_OR_COMMENT = re.compile(r'^[ ]*(#.*)?$').match + + def parse(self, string, name=''): + """ + Divide the given string into examples and intervening text, + and return them as a list of alternating Examples and strings. + Line numbers for the Examples are 0-based. The optional + argument `name` is a name identifying this string, and is only + used for error messages. + """ + string = string.expandtabs() + # If all lines begin with the same indentation, then strip it. + min_indent = self._min_indent(string) + if min_indent > 0: + string = '\n'.join([l[min_indent:] for l in string.split('\n')]) + + output = [] + charno, lineno = 0, 0 + # Find all doctest examples in the string: + for m in self._EXAMPLE_RE.finditer(string): + # Add the pre-example text to `output`. + output.append(string[charno:m.start()]) + # Update lineno (lines before this example) + lineno += string.count('\n', charno, m.start()) + # Extract info from the regexp match. + (source, options, want, exc_msg) = \ + self._parse_example(m, name, lineno) + # Create an Example, and add it to the list. + if not self._IS_BLANK_OR_COMMENT(source): + output.append( Example(source, want, exc_msg, + lineno=lineno, + indent=min_indent+len(m.group('indent')), + options=options) ) + # Update lineno (lines inside this example) + lineno += string.count('\n', m.start(), m.end()) + # Update charno. + charno = m.end() + # Add any remaining post-example text to `output`. + output.append(string[charno:]) + return output + + def get_doctest(self, string, globs, name, filename, lineno): + """ + Extract all doctest examples from the given string, and + collect them into a `DocTest` object. + + `globs`, `name`, `filename`, and `lineno` are attributes for + the new `DocTest` object. See the documentation for `DocTest` + for more information. + """ + return DocTest(self.get_examples(string, name), globs, + name, filename, lineno, string) + + def get_examples(self, string, name=''): + """ + Extract all doctest examples from the given string, and return + them as a list of `Example` objects. Line numbers are + 0-based, because it's most common in doctests that nothing + interesting appears on the same line as opening triple-quote, + and so the first interesting line is called \"line 1\" then. + + The optional argument `name` is a name identifying this + string, and is only used for error messages. + """ + return [x for x in self.parse(string, name) + if isinstance(x, Example)] + + def _parse_example(self, m, name, lineno): + """ + Given a regular expression match from `_EXAMPLE_RE` (`m`), + return a pair `(source, want)`, where `source` is the matched + example's source code (with prompts and indentation stripped); + and `want` is the example's expected output (with indentation + stripped). + + `name` is the string's name, and `lineno` is the line number + where the example starts; both are used for error messages. + """ + # Get the example's indentation level. + indent = len(m.group('indent')) + + # Divide source into lines; check that they're properly + # indented; and then strip their indentation & prompts. + source_lines = m.group('source').split('\n') + self._check_prompt_blank(source_lines, indent, name, lineno) + self._check_prefix(source_lines[1:], ' '*indent + '.', name, lineno) + source = '\n'.join([sl[indent+4:] for sl in source_lines]) + + # Divide want into lines; check that it's properly indented; and + # then strip the indentation. Spaces before the last newline should + # be preserved, so plain rstrip() isn't good enough. + want = m.group('want') + want_lines = want.split('\n') + if len(want_lines) > 1 and re.match(r' *$', want_lines[-1]): + del want_lines[-1] # forget final newline & spaces after it + self._check_prefix(want_lines, ' '*indent, name, + lineno + len(source_lines)) + want = '\n'.join([wl[indent:] for wl in want_lines]) + + # If `want` contains a traceback message, then extract it. + m = self._EXCEPTION_RE.match(want) + if m: + exc_msg = m.group('msg') + else: + exc_msg = None + + # Extract options from the source. + options = self._find_options(source, name, lineno) + + return source, options, want, exc_msg + + # This regular expression looks for option directives in the + # source code of an example. Option directives are comments + # starting with "doctest:". Warning: this may give false + # positives for string-literals that contain the string + # "#doctest:". Eliminating these false positives would require + # actually parsing the string; but we limit them by ignoring any + # line containing "#doctest:" that is *followed* by a quote mark. + _OPTION_DIRECTIVE_RE = re.compile(r'#\s*doctest:\s*([^\n\'"]*)$', + re.MULTILINE) + + def _find_options(self, source, name, lineno): + """ + Return a dictionary containing option overrides extracted from + option directives in the given source string. + + `name` is the string's name, and `lineno` is the line number + where the example starts; both are used for error messages. + """ + options = {} + # (note: with the current regexp, this will match at most once:) + for m in self._OPTION_DIRECTIVE_RE.finditer(source): + option_strings = m.group(1).replace(',', ' ').split() + for option in option_strings: + if (option[0] not in '+-' or + option[1:] not in OPTIONFLAGS_BY_NAME): + raise ValueError('line %r of the doctest for %s ' + 'has an invalid option: %r' % + (lineno+1, name, option)) + flag = OPTIONFLAGS_BY_NAME[option[1:]] + options[flag] = (option[0] == '+') + if options and self._IS_BLANK_OR_COMMENT(source): + raise ValueError('line %r of the doctest for %s has an option ' + 'directive on a line with no example: %r' % + (lineno, name, source)) + return options + + # This regular expression finds the indentation of every non-blank + # line in a string. + _INDENT_RE = re.compile('^([ ]*)(?=\S)', re.MULTILINE) + + def _min_indent(self, s): + "Return the minimum indentation of any non-blank line in `s`" + indents = [len(indent) for indent in self._INDENT_RE.findall(s)] + if len(indents) > 0: + return min(indents) + else: + return 0 + + def _check_prompt_blank(self, lines, indent, name, lineno): + """ + Given the lines of a source string (including prompts and + leading indentation), check to make sure that every prompt is + followed by a space character. If any line is not followed by + a space character, then raise ValueError. + """ + for i, line in enumerate(lines): + if len(line) >= indent+4 and line[indent+3] != ' ': + raise ValueError('line %r of the docstring for %s ' + 'lacks blank after %s: %r' % + (lineno+i+1, name, + line[indent:indent+3], line)) + + def _check_prefix(self, lines, prefix, name, lineno): + """ + Check that every line in the given list starts with the given + prefix; if any line does not, then raise a ValueError. + """ + for i, line in enumerate(lines): + if line and not line.startswith(prefix): + raise ValueError('line %r of the docstring for %s has ' + 'inconsistent leading whitespace: %r' % + (lineno+i+1, name, line)) + + +###################################################################### +## 4. DocTest Finder +###################################################################### + +class DocTestFinder: + """ + A class used to extract the DocTests that are relevant to a given + object, from its docstring and the docstrings of its contained + objects. Doctests can currently be extracted from the following + object types: modules, functions, classes, methods, staticmethods, + classmethods, and properties. + """ + + def __init__(self, verbose=False, parser=DocTestParser(), + recurse=True, _namefilter=None, exclude_empty=True): + """ + Create a new doctest finder. + + The optional argument `parser` specifies a class or + function that should be used to create new DocTest objects (or + objects that implement the same interface as DocTest). The + signature for this factory function should match the signature + of the DocTest constructor. + + If the optional argument `recurse` is false, then `find` will + only examine the given object, and not any contained objects. + + If the optional argument `exclude_empty` is false, then `find` + will include tests for objects with empty docstrings. + """ + self._parser = parser + self._verbose = verbose + self._recurse = recurse + self._exclude_empty = exclude_empty + # _namefilter is undocumented, and exists only for temporary backward- + # compatibility support of testmod's deprecated isprivate mess. + self._namefilter = _namefilter + + def find(self, obj, name=None, module=None, globs=None, + extraglobs=None): + """ + Return a list of the DocTests that are defined by the given + object's docstring, or by any of its contained objects' + docstrings. + + The optional parameter `module` is the module that contains + the given object. If the module is not specified or is None, then + the test finder will attempt to automatically determine the + correct module. The object's module is used: + + - As a default namespace, if `globs` is not specified. + - To prevent the DocTestFinder from extracting DocTests + from objects that are imported from other modules. + - To find the name of the file containing the object. + - To help find the line number of the object within its + file. + + Contained objects whose module does not match `module` are ignored. + + If `module` is False, no attempt to find the module will be made. + This is obscure, of use mostly in tests: if `module` is False, or + is None but cannot be found automatically, then all objects are + considered to belong to the (non-existent) module, so all contained + objects will (recursively) be searched for doctests. + + The globals for each DocTest is formed by combining `globs` + and `extraglobs` (bindings in `extraglobs` override bindings + in `globs`). A new copy of the globals dictionary is created + for each DocTest. If `globs` is not specified, then it + defaults to the module's `__dict__`, if specified, or {} + otherwise. If `extraglobs` is not specified, then it defaults + to {}. + + """ + # If name was not specified, then extract it from the object. + if name is None: + name = getattr(obj, '__name__', None) + if name is None: + raise ValueError("DocTestFinder.find: name must be given " + "when obj.__name__ doesn't exist: %r" % + (type(obj),)) + + # Find the module that contains the given object (if obj is + # a module, then module=obj.). Note: this may fail, in which + # case module will be None. + if module is False: + module = None + elif module is None: + module = inspect.getmodule(obj) + + # Read the module's source code. This is used by + # DocTestFinder._find_lineno to find the line number for a + # given object's docstring. + try: + file = inspect.getsourcefile(obj) or inspect.getfile(obj) + source_lines = linecache.getlines(file) + if not source_lines: + source_lines = None + except TypeError: + source_lines = None + + # Initialize globals, and merge in extraglobs. + if globs is None: + if module is None: + globs = {} + else: + globs = module.__dict__.copy() + else: + globs = globs.copy() + if extraglobs is not None: + globs.update(extraglobs) + + # Recursively expore `obj`, extracting DocTests. + tests = [] + self._find(tests, obj, name, module, source_lines, globs, {}) + # Sort the tests by alpha order of names, for consistency in + # verbose-mode output. This was a feature of doctest in Pythons + # <= 2.3 that got lost by accident in 2.4. It was repaired in + # 2.4.4 and 2.5. + tests.sort() + return tests + + def _filter(self, obj, prefix, base): + """ + Return true if the given object should not be examined. + """ + return (self._namefilter is not None and + self._namefilter(prefix, base)) + + def _from_module(self, module, object): + """ + Return true if the given object is defined in the given + module. + """ + if module is None: + return True + elif inspect.isfunction(object): + return module.__dict__ is object.func_globals + elif inspect.isclass(object): + # Some jython classes don't set __module__ + return module.__name__ == getattr(object, '__module__', None) + elif inspect.getmodule(object) is not None: + return module is inspect.getmodule(object) + elif hasattr(object, '__module__'): + return module.__name__ == object.__module__ + elif isinstance(object, property): + return True # [XX] no way not be sure. + else: + raise ValueError("object must be a class or function") + + def _find(self, tests, obj, name, module, source_lines, globs, seen): + """ + Find tests for the given object and any contained objects, and + add them to `tests`. + """ + if self._verbose: + print 'Finding tests in %s' % name + + # If we've already processed this object, then ignore it. + if id(obj) in seen: + return + seen[id(obj)] = 1 + + # Find a test for this object, and add it to the list of tests. + test = self._get_test(obj, name, module, globs, source_lines) + if test is not None: + tests.append(test) + + # Look for tests in a module's contained objects. + if inspect.ismodule(obj) and self._recurse: + for valname, val in obj.__dict__.items(): + # Check if this contained object should be ignored. + if self._filter(val, name, valname): + continue + valname = '%s.%s' % (name, valname) + # Recurse to functions & classes. + if ((inspect.isfunction(val) or inspect.isclass(val)) and + self._from_module(module, val)): + self._find(tests, val, valname, module, source_lines, + globs, seen) + + # Look for tests in a module's __test__ dictionary. + if inspect.ismodule(obj) and self._recurse: + for valname, val in getattr(obj, '__test__', {}).items(): + if not isinstance(valname, basestring): + raise ValueError("DocTestFinder.find: __test__ keys " + "must be strings: %r" % + (type(valname),)) + if not (inspect.isfunction(val) or inspect.isclass(val) or + inspect.ismethod(val) or inspect.ismodule(val) or + isinstance(val, basestring)): + raise ValueError("DocTestFinder.find: __test__ values " + "must be strings, functions, methods, " + "classes, or modules: %r" % + (type(val),)) + valname = '%s.__test__.%s' % (name, valname) + self._find(tests, val, valname, module, source_lines, + globs, seen) + + # Look for tests in a class's contained objects. + if inspect.isclass(obj) and self._recurse: + for valname, val in obj.__dict__.items(): + # Check if this contained object should be ignored. + if self._filter(val, name, valname): + continue + # Special handling for staticmethod/classmethod. + if isinstance(val, staticmethod): + val = getattr(obj, valname) + if isinstance(val, classmethod): + val = getattr(obj, valname).im_func + + # Recurse to methods, properties, and nested classes. + if ((inspect.isfunction(val) or inspect.isclass(val) or + isinstance(val, property)) and + self._from_module(module, val)): + valname = '%s.%s' % (name, valname) + self._find(tests, val, valname, module, source_lines, + globs, seen) + + def _get_test(self, obj, name, module, globs, source_lines): + """ + Return a DocTest for the given object, if it defines a docstring; + otherwise, return None. + """ + # Extract the object's docstring. If it doesn't have one, + # then return None (no test for this object). + if isinstance(obj, basestring): + docstring = obj + else: + try: + if obj.__doc__ is None: + docstring = '' + else: + docstring = obj.__doc__ + if not isinstance(docstring, basestring): + docstring = str(docstring) + except (TypeError, AttributeError): + docstring = '' + + # Find the docstring's location in the file. + lineno = self._find_lineno(obj, source_lines) + + # Don't bother if the docstring is empty. + if self._exclude_empty and not docstring: + return None + + # Return a DocTest for this object. + if module is None: + filename = None + else: + filename = getattr(module, '__file__', module.__name__) + if filename[-4:] in (".pyc", ".pyo"): + filename = filename[:-1] + elif sys.platform.startswith('java') and \ + filename.endswith('$py.class'): + filename = '%s.py' % filename[:-9] + return self._parser.get_doctest(docstring, globs, name, + filename, lineno) + + def _find_lineno(self, obj, source_lines): + """ + Return a line number of the given object's docstring. Note: + this method assumes that the object has a docstring. + """ + lineno = None + + # Find the line number for modules. + if inspect.ismodule(obj): + lineno = 0 + + # Find the line number for classes. + # Note: this could be fooled if a class is defined multiple + # times in a single file. + if inspect.isclass(obj): + if source_lines is None: + return None + pat = re.compile(r'^\s*class\s*%s\b' % + getattr(obj, '__name__', '-')) + for i, line in enumerate(source_lines): + if pat.match(line): + lineno = i + break + + # Find the line number for functions & methods. + if inspect.ismethod(obj): obj = obj.im_func + if inspect.isfunction(obj): obj = obj.func_code + if inspect.istraceback(obj): obj = obj.tb_frame + if inspect.isframe(obj): obj = obj.f_code + if inspect.iscode(obj): + lineno = getattr(obj, 'co_firstlineno', None)-1 + + # Find the line number where the docstring starts. Assume + # that it's the first line that begins with a quote mark. + # Note: this could be fooled by a multiline function + # signature, where a continuation line begins with a quote + # mark. + if lineno is not None: + if source_lines is None: + return lineno+1 + pat = re.compile('(^|.*:)\s*\w*("|\')') + for lineno in range(lineno, len(source_lines)): + if pat.match(source_lines[lineno]): + return lineno + + # We couldn't find the line number. + return None + +###################################################################### +## 5. DocTest Runner +###################################################################### + +class DocTestRunner: + # This divider string is used to separate failure messages, and to + # separate sections of the summary. + DIVIDER = "*" * 70 + + def __init__(self, checker=None, verbose=None, optionflags=0): + """ + Create a new test runner. + + Optional keyword arg `checker` is the `OutputChecker` that + should be used to compare the expected outputs and actual + outputs of doctest examples. + + Optional keyword arg 'verbose' prints lots of stuff if true, + only failures if false; by default, it's true iff '-v' is in + sys.argv. + + Optional argument `optionflags` can be used to control how the + test runner compares expected output to actual output, and how + it displays failures. See the documentation for `testmod` for + more information. + """ + self._checker = checker or OutputChecker() + if verbose is None: + verbose = '-v' in sys.argv + self._verbose = verbose + self.optionflags = optionflags + self.original_optionflags = optionflags + + # Keep track of the examples we've run. + self.tries = 0 + self.failures = 0 + self._name2ft = {} + + # Create a fake output target for capturing doctest output. + self._fakeout = _SpoofOut() + + #///////////////////////////////////////////////////////////////// + # Reporting methods + #///////////////////////////////////////////////////////////////// + + def report_start(self, out, test, example): + """ + Report that the test runner is about to process the given + example. (Only displays a message if verbose=True) + """ + if self._verbose: + if example.want: + out('Trying:\n' + _indent(example.source) + + 'Expecting:\n' + _indent(example.want)) + else: + out('Trying:\n' + _indent(example.source) + + 'Expecting nothing\n') + + def report_success(self, out, test, example, got): + """ + Report that the given example ran successfully. (Only + displays a message if verbose=True) + """ + if self._verbose: + out("ok\n") + + def report_failure(self, out, test, example, got): + """ + Report that the given example failed. + """ + out(self._failure_header(test, example) + + self._checker.output_difference(example, got, self.optionflags)) + + def report_unexpected_exception(self, out, test, example, exc_info): + """ + Report that the given example raised an unexpected exception. + """ + out(self._failure_header(test, example) + + 'Exception raised:\n' + _indent(_exception_traceback(exc_info))) + + def _failure_header(self, test, example): + out = [self.DIVIDER] + if test.filename: + if test.lineno is not None and example.lineno is not None: + lineno = test.lineno + example.lineno + 1 + else: + lineno = '?' + out.append('File "%s", line %s, in %s' % + (test.filename, lineno, test.name)) + else: + out.append('Line %s, in %s' % (example.lineno+1, test.name)) + out.append('Failed example:') + source = example.source + out.append(_indent(source)) + return '\n'.join(out) + + #///////////////////////////////////////////////////////////////// + # DocTest Running + #///////////////////////////////////////////////////////////////// + + def __run(self, test, compileflags, out): + """ + Run the examples in `test`. Write the outcome of each example + with one of the `DocTestRunner.report_*` methods, using the + writer function `out`. `compileflags` is the set of compiler + flags that should be used to execute examples. Return a tuple + `(f, t)`, where `t` is the number of examples tried, and `f` + is the number of examples that failed. The examples are run + in the namespace `test.globs`. + """ + # Keep track of the number of failures and tries. + failures = tries = 0 + + # Save the option flags (since option directives can be used + # to modify them). + original_optionflags = self.optionflags + + SUCCESS, FAILURE, BOOM = range(3) # `outcome` state + + check = self._checker.check_output + + # Process each example. + for examplenum, example in enumerate(test.examples): + + # If REPORT_ONLY_FIRST_FAILURE is set, then supress + # reporting after the first failure. + quiet = (self.optionflags & REPORT_ONLY_FIRST_FAILURE and + failures > 0) + + # Merge in the example's options. + self.optionflags = original_optionflags + if example.options: + for (optionflag, val) in example.options.items(): + if val: + self.optionflags |= optionflag + else: + self.optionflags &= ~optionflag + + # Record that we started this example. + tries += 1 + if not quiet: + self.report_start(out, test, example) + + # Use a special filename for compile(), so we can retrieve + # the source code during interactive debugging (see + # __patched_linecache_getlines). + filename = '' % (test.name, examplenum) + + # Run the example in the given context (globs), and record + # any exception that gets raised. (But don't intercept + # keyboard interrupts.) + try: + # Don't blink! This is where the user's code gets run. + exec compile(example.source, filename, "single", + compileflags, 1) in test.globs + self.debugger.set_continue() # ==== Example Finished ==== + exception = None + except KeyboardInterrupt: + raise + except: + exception = sys.exc_info() + self.debugger.set_continue() # ==== Example Finished ==== + + got = self._fakeout.getvalue() # the actual output + self._fakeout.truncate(0) + outcome = FAILURE # guilty until proved innocent or insane + + # If the example executed without raising any exceptions, + # verify its output. + if exception is None: + if check(example.want, got, self.optionflags): + outcome = SUCCESS + + # The example raised an exception: check if it was expected. + else: + exc_info = sys.exc_info() + exc_msg = traceback.format_exception_only(*exc_info[:2])[-1] + if not quiet: + got += _exception_traceback(exc_info) + + # If `example.exc_msg` is None, then we weren't expecting + # an exception. + if example.exc_msg is None: + outcome = BOOM + + # We expected an exception: see whether it matches. + elif check(example.exc_msg, exc_msg, self.optionflags): + outcome = SUCCESS + + # Another chance if they didn't care about the detail. + elif self.optionflags & IGNORE_EXCEPTION_DETAIL: + m1 = re.match(r'[^:]*:', example.exc_msg) + m2 = re.match(r'[^:]*:', exc_msg) + if m1 and m2 and check(m1.group(0), m2.group(0), + self.optionflags): + outcome = SUCCESS + + # Report the outcome. + if outcome is SUCCESS: + if not quiet: + self.report_success(out, test, example, got) + elif outcome is FAILURE: + if not quiet: + self.report_failure(out, test, example, got) + failures += 1 + elif outcome is BOOM: + if not quiet: + self.report_unexpected_exception(out, test, example, + exc_info) + failures += 1 + else: + assert False, ("unknown outcome", outcome) + + # Restore the option flags (in case they were modified) + self.optionflags = original_optionflags + + # Record and return the number of failures and tries. + self.__record_outcome(test, failures, tries) + return failures, tries + + def __record_outcome(self, test, f, t): + """ + Record the fact that the given DocTest (`test`) generated `f` + failures out of `t` tried examples. + """ + f2, t2 = self._name2ft.get(test.name, (0,0)) + self._name2ft[test.name] = (f+f2, t+t2) + self.failures += f + self.tries += t + + __LINECACHE_FILENAME_RE = re.compile(r'[\w\.]+)' + r'\[(?P\d+)\]>$') + def __patched_linecache_getlines(self, filename): + m = self.__LINECACHE_FILENAME_RE.match(filename) + if m and m.group('name') == self.test.name: + example = self.test.examples[int(m.group('examplenum'))] + return example.source.splitlines(True) + else: + return self.save_linecache_getlines(filename) + + def run(self, test, compileflags=None, out=None, clear_globs=True): + """ + Run the examples in `test`, and display the results using the + writer function `out`. + + The examples are run in the namespace `test.globs`. If + `clear_globs` is true (the default), then this namespace will + be cleared after the test runs, to help with garbage + collection. If you would like to examine the namespace after + the test completes, then use `clear_globs=False`. + + `compileflags` gives the set of flags that should be used by + the Python compiler when running the examples. If not + specified, then it will default to the set of future-import + flags that apply to `globs`. + + The output of each example is checked using + `DocTestRunner.check_output`, and the results are formatted by + the `DocTestRunner.report_*` methods. + """ + self.test = test + + if compileflags is None: + compileflags = _extract_future_flags(test.globs) + + save_stdout = sys.stdout + if out is None: + out = save_stdout.write + sys.stdout = self._fakeout + + # Patch pdb.set_trace to restore sys.stdout during interactive + # debugging (so it's not still redirected to self._fakeout). + # Note that the interactive output will go to *our* + # save_stdout, even if that's not the real sys.stdout; this + # allows us to write test cases for the set_trace behavior. + save_set_trace = pdb.set_trace + self.debugger = _OutputRedirectingPdb(save_stdout) + self.debugger.reset() + pdb.set_trace = self.debugger.set_trace + + # Patch linecache.getlines, so we can see the example's source + # when we're inside the debugger. + self.save_linecache_getlines = linecache.getlines + linecache.getlines = self.__patched_linecache_getlines + + try: + return self.__run(test, compileflags, out) + finally: + sys.stdout = save_stdout + pdb.set_trace = save_set_trace + linecache.getlines = self.save_linecache_getlines + if clear_globs: + test.globs.clear() + + #///////////////////////////////////////////////////////////////// + # Summarization + #///////////////////////////////////////////////////////////////// + def summarize(self, verbose=None): + """ + Print a summary of all the test cases that have been run by + this DocTestRunner, and return a tuple `(f, t)`, where `f` is + the total number of failed examples, and `t` is the total + number of tried examples. + + The optional `verbose` argument controls how detailed the + summary is. If the verbosity is not specified, then the + DocTestRunner's verbosity is used. + """ + if verbose is None: + verbose = self._verbose + notests = [] + passed = [] + failed = [] + totalt = totalf = 0 + for x in self._name2ft.items(): + name, (f, t) = x + assert f <= t + totalt += t + totalf += f + if t == 0: + notests.append(name) + elif f == 0: + passed.append( (name, t) ) + else: + failed.append(x) + if verbose: + if notests: + print len(notests), "items had no tests:" + notests.sort() + for thing in notests: + print " ", thing + if passed: + print len(passed), "items passed all tests:" + passed.sort() + for thing, count in passed: + print " %3d tests in %s" % (count, thing) + if failed: + print self.DIVIDER + print len(failed), "items had failures:" + failed.sort() + for thing, (f, t) in failed: + print " %3d of %3d in %s" % (f, t, thing) + if verbose: + print totalt, "tests in", len(self._name2ft), "items." + print totalt - totalf, "passed and", totalf, "failed." + if totalf: + print "***Test Failed***", totalf, "failures." + elif verbose: + print "Test passed." + return totalf, totalt + + #///////////////////////////////////////////////////////////////// + # Backward compatibility cruft to maintain doctest.master. + #///////////////////////////////////////////////////////////////// + def merge(self, other): + d = self._name2ft + for name, (f, t) in other._name2ft.items(): + if name in d: + print "*** DocTestRunner.merge: '" + name + "' in both" \ + " testers; summing outcomes." + f2, t2 = d[name] + f = f + f2 + t = t + t2 + d[name] = f, t + +class OutputChecker: + """ + A class used to check the whether the actual output from a doctest + example matches the expected output. `OutputChecker` defines two + methods: `check_output`, which compares a given pair of outputs, + and returns true if they match; and `output_difference`, which + returns a string describing the differences between two outputs. + """ + def check_output(self, want, got, optionflags): + """ + Return True iff the actual output from an example (`got`) + matches the expected output (`want`). These strings are + always considered to match if they are identical; but + depending on what option flags the test runner is using, + several non-exact match types are also possible. See the + documentation for `TestRunner` for more information about + option flags. + """ + # Handle the common case first, for efficiency: + # if they're string-identical, always return true. + if got == want: + return True + + # The values True and False replaced 1 and 0 as the return + # value for boolean comparisons in Python 2.3. + if not (optionflags & DONT_ACCEPT_TRUE_FOR_1): + if (got,want) == ("True\n", "1\n"): + return True + if (got,want) == ("False\n", "0\n"): + return True + + # can be used as a special sequence to signify a + # blank line, unless the DONT_ACCEPT_BLANKLINE flag is used. + if not (optionflags & DONT_ACCEPT_BLANKLINE): + # Replace in want with a blank line. + want = re.sub('(?m)^%s\s*?$' % re.escape(BLANKLINE_MARKER), + '', want) + # If a line in got contains only spaces, then remove the + # spaces. + got = re.sub('(?m)^\s*?$', '', got) + if got == want: + return True + + # This flag causes doctest to ignore any differences in the + # contents of whitespace strings. Note that this can be used + # in conjunction with the ELLIPSIS flag. + if optionflags & NORMALIZE_WHITESPACE: + got = ' '.join(got.split()) + want = ' '.join(want.split()) + if got == want: + return True + + # The ELLIPSIS flag says to let the sequence "..." in `want` + # match any substring in `got`. + if optionflags & ELLIPSIS: + if _ellipsis_match(want, got): + return True + + # We didn't find any match; return false. + return False + + # Should we do a fancy diff? + def _do_a_fancy_diff(self, want, got, optionflags): + # Not unless they asked for a fancy diff. + if not optionflags & (REPORT_UDIFF | + REPORT_CDIFF | + REPORT_NDIFF): + return False + + # If expected output uses ellipsis, a meaningful fancy diff is + # too hard ... or maybe not. In two real-life failures Tim saw, + # a diff was a major help anyway, so this is commented out. + # [todo] _ellipsis_match() knows which pieces do and don't match, + # and could be the basis for a kick-ass diff in this case. + ##if optionflags & ELLIPSIS and ELLIPSIS_MARKER in want: + ## return False + + # ndiff does intraline difference marking, so can be useful even + # for 1-line differences. + if optionflags & REPORT_NDIFF: + return True + + # The other diff types need at least a few lines to be helpful. + return want.count('\n') > 2 and got.count('\n') > 2 + + def output_difference(self, example, got, optionflags): + """ + Return a string describing the differences between the + expected output for a given example (`example`) and the actual + output (`got`). `optionflags` is the set of option flags used + to compare `want` and `got`. + """ + want = example.want + # If s are being used, then replace blank lines + # with in the actual output string. + if not (optionflags & DONT_ACCEPT_BLANKLINE): + got = re.sub('(?m)^[ ]*(?=\n)', BLANKLINE_MARKER, got) + + # Check if we should use diff. + if self._do_a_fancy_diff(want, got, optionflags): + # Split want & got into lines. + want_lines = want.splitlines(True) # True == keep line ends + got_lines = got.splitlines(True) + # Use difflib to find their differences. + if optionflags & REPORT_UDIFF: + diff = difflib.unified_diff(want_lines, got_lines, n=2) + diff = list(diff)[2:] # strip the diff header + kind = 'unified diff with -expected +actual' + elif optionflags & REPORT_CDIFF: + diff = difflib.context_diff(want_lines, got_lines, n=2) + diff = list(diff)[2:] # strip the diff header + kind = 'context diff with expected followed by actual' + elif optionflags & REPORT_NDIFF: + engine = difflib.Differ(charjunk=difflib.IS_CHARACTER_JUNK) + diff = list(engine.compare(want_lines, got_lines)) + kind = 'ndiff with -expected +actual' + else: + assert 0, 'Bad diff option' + # Remove trailing whitespace on diff output. + diff = [line.rstrip() + '\n' for line in diff] + return 'Differences (%s):\n' % kind + _indent(''.join(diff)) + + # If we're not using diff, then simply list the expected + # output followed by the actual output. + if want and got: + return 'Expected:\n%sGot:\n%s' % (_indent(want), _indent(got)) + elif want: + return 'Expected:\n%sGot nothing\n' % _indent(want) + elif got: + return 'Expected nothing\nGot:\n%s' % _indent(got) + else: + return 'Expected nothing\nGot nothing\n' + +class DocTestFailure(Exception): + """A DocTest example has failed in debugging mode. + + The exception instance has variables: + + - test: the DocTest object being run + + - excample: the Example object that failed + + - got: the actual output + """ + def __init__(self, test, example, got): + self.test = test + self.example = example + self.got = got + + def __str__(self): + return str(self.test) + +class UnexpectedException(Exception): + """A DocTest example has encountered an unexpected exception + + The exception instance has variables: + + - test: the DocTest object being run + + - excample: the Example object that failed + + - exc_info: the exception info + """ + def __init__(self, test, example, exc_info): + self.test = test + self.example = example + self.exc_info = exc_info + + def __str__(self): + return str(self.test) + +class DebugRunner(DocTestRunner): + + def run(self, test, compileflags=None, out=None, clear_globs=True): + r = DocTestRunner.run(self, test, compileflags, out, False) + if clear_globs: + test.globs.clear() + return r + + def report_unexpected_exception(self, out, test, example, exc_info): + raise UnexpectedException(test, example, exc_info) + + def report_failure(self, out, test, example, got): + raise DocTestFailure(test, example, got) + +###################################################################### +## 6. Test Functions +###################################################################### +# These should be backwards compatible. + +# For backward compatibility, a global instance of a DocTestRunner +# class, updated by testmod. +master = None + +def testmod(m=None, name=None, globs=None, verbose=None, isprivate=None, + report=True, optionflags=0, extraglobs=None, + raise_on_error=False, exclude_empty=False): + """m=None, name=None, globs=None, verbose=None, isprivate=None, + report=True, optionflags=0, extraglobs=None, raise_on_error=False, + exclude_empty=False + + Test examples in docstrings in functions and classes reachable + from module m (or the current module if m is not supplied), starting + with m.__doc__. Unless isprivate is specified, private names + are not skipped. + + Also test examples reachable from dict m.__test__ if it exists and is + not None. m.__test__ maps names to functions, classes and strings; + function and class docstrings are tested even if the name is private; + strings are tested directly, as if they were docstrings. + + Return (#failures, #tests). + + See doctest.__doc__ for an overview. + + Optional keyword arg "name" gives the name of the module; by default + use m.__name__. + + Optional keyword arg "globs" gives a dict to be used as the globals + when executing examples; by default, use m.__dict__. A copy of this + dict is actually used for each docstring, so that each docstring's + examples start with a clean slate. + + Optional keyword arg "extraglobs" gives a dictionary that should be + merged into the globals that are used to execute examples. By + default, no extra globals are used. This is new in 2.4. + + Optional keyword arg "verbose" prints lots of stuff if true, prints + only failures if false; by default, it's true iff "-v" is in sys.argv. + + Optional keyword arg "report" prints a summary at the end when true, + else prints nothing at the end. In verbose mode, the summary is + detailed, else very brief (in fact, empty if all tests passed). + + Optional keyword arg "optionflags" or's together module constants, + and defaults to 0. This is new in 2.3. Possible values (see the + docs for details): + + DONT_ACCEPT_TRUE_FOR_1 + DONT_ACCEPT_BLANKLINE + NORMALIZE_WHITESPACE + ELLIPSIS + IGNORE_EXCEPTION_DETAIL + REPORT_UDIFF + REPORT_CDIFF + REPORT_NDIFF + REPORT_ONLY_FIRST_FAILURE + + Optional keyword arg "raise_on_error" raises an exception on the + first unexpected exception or failure. This allows failures to be + post-mortem debugged. + + Deprecated in Python 2.4: + Optional keyword arg "isprivate" specifies a function used to + determine whether a name is private. The default function is + treat all functions as public. Optionally, "isprivate" can be + set to doctest.is_private to skip over functions marked as private + using the underscore naming convention; see its docs for details. + + Advanced tomfoolery: testmod runs methods of a local instance of + class doctest.Tester, then merges the results into (or creates) + global Tester instance doctest.master. Methods of doctest.master + can be called directly too, if you want to do something unusual. + Passing report=0 to testmod is especially useful then, to delay + displaying a summary. Invoke doctest.master.summarize(verbose) + when you're done fiddling. + """ + global master + + if isprivate is not None: + warnings.warn("the isprivate argument is deprecated; " + "examine DocTestFinder.find() lists instead", + DeprecationWarning) + + # If no module was given, then use __main__. + if m is None: + # DWA - m will still be None if this wasn't invoked from the command + # line, in which case the following TypeError is about as good an error + # as we should expect + m = sys.modules.get('__main__') + + # Check that we were actually given a module. + if not inspect.ismodule(m): + raise TypeError("testmod: module required; %r" % (m,)) + + # If no name was given, then use the module's name. + if name is None: + name = m.__name__ + + # Find, parse, and run all tests in the given module. + finder = DocTestFinder(_namefilter=isprivate, exclude_empty=exclude_empty) + + if raise_on_error: + runner = DebugRunner(verbose=verbose, optionflags=optionflags) + else: + runner = DocTestRunner(verbose=verbose, optionflags=optionflags) + + for test in finder.find(m, name, globs=globs, extraglobs=extraglobs): + runner.run(test) + + if report: + runner.summarize() + + if master is None: + master = runner + else: + master.merge(runner) + + return runner.failures, runner.tries + +def testfile(filename, module_relative=True, name=None, package=None, + globs=None, verbose=None, report=True, optionflags=0, + extraglobs=None, raise_on_error=False, parser=DocTestParser()): + """ + Test examples in the given file. Return (#failures, #tests). + + Optional keyword arg "module_relative" specifies how filenames + should be interpreted: + + - If "module_relative" is True (the default), then "filename" + specifies a module-relative path. By default, this path is + relative to the calling module's directory; but if the + "package" argument is specified, then it is relative to that + package. To ensure os-independence, "filename" should use + "/" characters to separate path segments, and should not + be an absolute path (i.e., it may not begin with "/"). + + - If "module_relative" is False, then "filename" specifies an + os-specific path. The path may be absolute or relative (to + the current working directory). + + Optional keyword arg "name" gives the name of the test; by default + use the file's basename. + + Optional keyword argument "package" is a Python package or the + name of a Python package whose directory should be used as the + base directory for a module relative filename. If no package is + specified, then the calling module's directory is used as the base + directory for module relative filenames. It is an error to + specify "package" if "module_relative" is False. + + Optional keyword arg "globs" gives a dict to be used as the globals + when executing examples; by default, use {}. A copy of this dict + is actually used for each docstring, so that each docstring's + examples start with a clean slate. + + Optional keyword arg "extraglobs" gives a dictionary that should be + merged into the globals that are used to execute examples. By + default, no extra globals are used. + + Optional keyword arg "verbose" prints lots of stuff if true, prints + only failures if false; by default, it's true iff "-v" is in sys.argv. + + Optional keyword arg "report" prints a summary at the end when true, + else prints nothing at the end. In verbose mode, the summary is + detailed, else very brief (in fact, empty if all tests passed). + + Optional keyword arg "optionflags" or's together module constants, + and defaults to 0. Possible values (see the docs for details): + + DONT_ACCEPT_TRUE_FOR_1 + DONT_ACCEPT_BLANKLINE + NORMALIZE_WHITESPACE + ELLIPSIS + IGNORE_EXCEPTION_DETAIL + REPORT_UDIFF + REPORT_CDIFF + REPORT_NDIFF + REPORT_ONLY_FIRST_FAILURE + + Optional keyword arg "raise_on_error" raises an exception on the + first unexpected exception or failure. This allows failures to be + post-mortem debugged. + + Optional keyword arg "parser" specifies a DocTestParser (or + subclass) that should be used to extract tests from the files. + + Advanced tomfoolery: testmod runs methods of a local instance of + class doctest.Tester, then merges the results into (or creates) + global Tester instance doctest.master. Methods of doctest.master + can be called directly too, if you want to do something unusual. + Passing report=0 to testmod is especially useful then, to delay + displaying a summary. Invoke doctest.master.summarize(verbose) + when you're done fiddling. + """ + global master + + if package and not module_relative: + raise ValueError("Package may only be specified for module-" + "relative paths.") + + # Relativize the path + if module_relative: + package = _normalize_module(package) + filename = _module_relative_path(package, filename) + + # If no name was given, then use the file's name. + if name is None: + name = os.path.basename(filename) + + # Assemble the globals. + if globs is None: + globs = {} + else: + globs = globs.copy() + if extraglobs is not None: + globs.update(extraglobs) + + if raise_on_error: + runner = DebugRunner(verbose=verbose, optionflags=optionflags) + else: + runner = DocTestRunner(verbose=verbose, optionflags=optionflags) + + # Read the file, convert it to a test, and run it. + s = open(filename).read() + test = parser.get_doctest(s, globs, name, filename, 0) + runner.run(test) + + if report: + runner.summarize() + + if master is None: + master = runner + else: + master.merge(runner) + + return runner.failures, runner.tries + +def run_docstring_examples(f, globs, verbose=False, name="NoName", + compileflags=None, optionflags=0): + """ + Test examples in the given object's docstring (`f`), using `globs` + as globals. Optional argument `name` is used in failure messages. + If the optional argument `verbose` is true, then generate output + even if there are no failures. + + `compileflags` gives the set of flags that should be used by the + Python compiler when running the examples. If not specified, then + it will default to the set of future-import flags that apply to + `globs`. + + Optional keyword arg `optionflags` specifies options for the + testing and output. See the documentation for `testmod` for more + information. + """ + # Find, parse, and run all tests in the given module. + finder = DocTestFinder(verbose=verbose, recurse=False) + runner = DocTestRunner(verbose=verbose, optionflags=optionflags) + for test in finder.find(f, name, globs=globs): + runner.run(test, compileflags=compileflags) + +###################################################################### +## 7. Tester +###################################################################### +# This is provided only for backwards compatibility. It's not +# actually used in any way. + +class Tester: + def __init__(self, mod=None, globs=None, verbose=None, + isprivate=None, optionflags=0): + + warnings.warn("class Tester is deprecated; " + "use class doctest.DocTestRunner instead", + DeprecationWarning, stacklevel=2) + if mod is None and globs is None: + raise TypeError("Tester.__init__: must specify mod or globs") + if mod is not None and not inspect.ismodule(mod): + raise TypeError("Tester.__init__: mod must be a module; %r" % + (mod,)) + if globs is None: + globs = mod.__dict__ + self.globs = globs + + self.verbose = verbose + self.isprivate = isprivate + self.optionflags = optionflags + self.testfinder = DocTestFinder(_namefilter=isprivate) + self.testrunner = DocTestRunner(verbose=verbose, + optionflags=optionflags) + + def runstring(self, s, name): + test = DocTestParser().get_doctest(s, self.globs, name, None, None) + if self.verbose: + print "Running string", name + (f,t) = self.testrunner.run(test) + if self.verbose: + print f, "of", t, "examples failed in string", name + return (f,t) + + def rundoc(self, object, name=None, module=None): + f = t = 0 + tests = self.testfinder.find(object, name, module=module, + globs=self.globs) + for test in tests: + (f2, t2) = self.testrunner.run(test) + (f,t) = (f+f2, t+t2) + return (f,t) + + def rundict(self, d, name, module=None): + import new + m = new.module(name) + m.__dict__.update(d) + if module is None: + module = False + return self.rundoc(m, name, module) + + def run__test__(self, d, name): + import new + m = new.module(name) + m.__test__ = d + return self.rundoc(m, name) + + def summarize(self, verbose=None): + return self.testrunner.summarize(verbose) + + def merge(self, other): + self.testrunner.merge(other.testrunner) + +###################################################################### +## 8. Unittest Support +###################################################################### + +_unittest_reportflags = 0 + +def set_unittest_reportflags(flags): + global _unittest_reportflags + + if (flags & REPORTING_FLAGS) != flags: + raise ValueError("Only reporting flags allowed", flags) + old = _unittest_reportflags + _unittest_reportflags = flags + return old + + +class DocTestCase(unittest.TestCase): + + def __init__(self, test, optionflags=0, setUp=None, tearDown=None, + checker=None): + + unittest.TestCase.__init__(self) + self._dt_optionflags = optionflags + self._dt_checker = checker + self._dt_test = test + self._dt_setUp = setUp + self._dt_tearDown = tearDown + + def setUp(self): + test = self._dt_test + + if self._dt_setUp is not None: + self._dt_setUp(test) + + def tearDown(self): + test = self._dt_test + + if self._dt_tearDown is not None: + self._dt_tearDown(test) + + test.globs.clear() + + def runTest(self): + test = self._dt_test + old = sys.stdout + new = StringIO() + optionflags = self._dt_optionflags + + if not (optionflags & REPORTING_FLAGS): + # The option flags don't include any reporting flags, + # so add the default reporting flags + optionflags |= _unittest_reportflags + + runner = DocTestRunner(optionflags=optionflags, + checker=self._dt_checker, verbose=False) + + try: + runner.DIVIDER = "-"*70 + failures, tries = runner.run( + test, out=new.write, clear_globs=False) + finally: + sys.stdout = old + + if failures: + raise self.failureException(self.format_failure(new.getvalue())) + + def format_failure(self, err): + test = self._dt_test + if test.lineno is None: + lineno = 'unknown line number' + else: + lineno = '%s' % test.lineno + lname = '.'.join(test.name.split('.')[-1:]) + return ('Failed doctest test for %s\n' + ' File "%s", line %s, in %s\n\n%s' + % (test.name, test.filename, lineno, lname, err) + ) + + def debug(self): + self.setUp() + runner = DebugRunner(optionflags=self._dt_optionflags, + checker=self._dt_checker, verbose=False) + runner.run(self._dt_test) + self.tearDown() + + def id(self): + return self._dt_test.name + + def __repr__(self): + name = self._dt_test.name.split('.') + return "%s (%s)" % (name[-1], '.'.join(name[:-1])) + + __str__ = __repr__ + + def shortDescription(self): + return "Doctest: " + self._dt_test.name + +def DocTestSuite(module=None, globs=None, extraglobs=None, test_finder=None, + **options): + """ + Convert doctest tests for a module to a unittest test suite. + + This converts each documentation string in a module that + contains doctest tests to a unittest test case. If any of the + tests in a doc string fail, then the test case fails. An exception + is raised showing the name of the file containing the test and a + (sometimes approximate) line number. + + The `module` argument provides the module to be tested. The argument + can be either a module or a module name. + + If no argument is given, the calling module is used. + + A number of options may be provided as keyword arguments: + + setUp + A set-up function. This is called before running the + tests in each file. The setUp function will be passed a DocTest + object. The setUp function can access the test globals as the + globs attribute of the test passed. + + tearDown + A tear-down function. This is called after running the + tests in each file. The tearDown function will be passed a DocTest + object. The tearDown function can access the test globals as the + globs attribute of the test passed. + + globs + A dictionary containing initial global variables for the tests. + + optionflags + A set of doctest option flags expressed as an integer. + """ + + if test_finder is None: + test_finder = DocTestFinder() + + module = _normalize_module(module) + tests = test_finder.find(module, globs=globs, extraglobs=extraglobs) + if globs is None: + globs = module.__dict__ + if not tests: + # Why do we want to do this? Because it reveals a bug that might + # otherwise be hidden. + raise ValueError(module, "has no tests") + + tests.sort() + suite = unittest.TestSuite() + for test in tests: + if len(test.examples) == 0: + continue + if not test.filename: + filename = module.__file__ + if filename[-4:] in (".pyc", ".pyo"): + filename = filename[:-1] + elif sys.platform.startswith('java') and \ + filename.endswith('$py.class'): + filename = '%s.py' % filename[:-9] + test.filename = filename + suite.addTest(DocTestCase(test, **options)) + + return suite + +class DocFileCase(DocTestCase): + + def id(self): + return '_'.join(self._dt_test.name.split('.')) + + def __repr__(self): + return self._dt_test.filename + __str__ = __repr__ + + def format_failure(self, err): + return ('Failed doctest test for %s\n File "%s", line 0\n\n%s' + % (self._dt_test.name, self._dt_test.filename, err) + ) + +def DocFileTest(path, module_relative=True, package=None, + globs=None, parser=DocTestParser(), **options): + if globs is None: + globs = {} + + if package and not module_relative: + raise ValueError("Package may only be specified for module-" + "relative paths.") + + # Relativize the path. + if module_relative: + package = _normalize_module(package) + path = _module_relative_path(package, path) + + # Find the file and read it. + name = os.path.basename(path) + doc = open(path).read() + + # Convert it to a test, and wrap it in a DocFileCase. + test = parser.get_doctest(doc, globs, name, path, 0) + return DocFileCase(test, **options) + +def DocFileSuite(*paths, **kw): + """A unittest suite for one or more doctest files. + + The path to each doctest file is given as a string; the + interpretation of that string depends on the keyword argument + "module_relative". + + A number of options may be provided as keyword arguments: + + module_relative + If "module_relative" is True, then the given file paths are + interpreted as os-independent module-relative paths. By + default, these paths are relative to the calling module's + directory; but if the "package" argument is specified, then + they are relative to that package. To ensure os-independence, + "filename" should use "/" characters to separate path + segments, and may not be an absolute path (i.e., it may not + begin with "/"). + + If "module_relative" is False, then the given file paths are + interpreted as os-specific paths. These paths may be absolute + or relative (to the current working directory). + + package + A Python package or the name of a Python package whose directory + should be used as the base directory for module relative paths. + If "package" is not specified, then the calling module's + directory is used as the base directory for module relative + filenames. It is an error to specify "package" if + "module_relative" is False. + + setUp + A set-up function. This is called before running the + tests in each file. The setUp function will be passed a DocTest + object. The setUp function can access the test globals as the + globs attribute of the test passed. + + tearDown + A tear-down function. This is called after running the + tests in each file. The tearDown function will be passed a DocTest + object. The tearDown function can access the test globals as the + globs attribute of the test passed. + + globs + A dictionary containing initial global variables for the tests. + + optionflags + A set of doctest option flags expressed as an integer. + + parser + A DocTestParser (or subclass) that should be used to extract + tests from the files. + """ + suite = unittest.TestSuite() + + # We do this here so that _normalize_module is called at the right + # level. If it were called in DocFileTest, then this function + # would be the caller and we might guess the package incorrectly. + if kw.get('module_relative', True): + kw['package'] = _normalize_module(kw.get('package')) + + for path in paths: + suite.addTest(DocFileTest(path, **kw)) + + return suite + +###################################################################### +## 9. Debugging Support +###################################################################### + +def script_from_examples(s): + output = [] + for piece in DocTestParser().parse(s): + if isinstance(piece, Example): + # Add the example's source code (strip trailing NL) + output.append(piece.source[:-1]) + # Add the expected output: + want = piece.want + if want: + output.append('# Expected:') + output += ['## '+l for l in want.split('\n')[:-1]] + else: + # Add non-example text. + output += [_comment_line(l) + for l in piece.split('\n')[:-1]] + + # Trim junk on both ends. + while output and output[-1] == '#': + output.pop() + while output and output[0] == '#': + output.pop(0) + # Combine the output, and return it. + # Add a courtesy newline to prevent exec from choking (see bug #1172785) + return '\n'.join(output) + '\n' + +def testsource(module, name): + """Extract the test sources from a doctest docstring as a script. + + Provide the module (or dotted name of the module) containing the + test to be debugged and the name (within the module) of the object + with the doc string with tests to be debugged. + """ + module = _normalize_module(module) + tests = DocTestFinder().find(module) + test = [t for t in tests if t.name == name] + if not test: + raise ValueError(name, "not found in tests") + test = test[0] + testsrc = script_from_examples(test.docstring) + return testsrc + +def debug_src(src, pm=False, globs=None): + """Debug a single doctest docstring, in argument `src`'""" + testsrc = script_from_examples(src) + debug_script(testsrc, pm, globs) + +def debug_script(src, pm=False, globs=None): + "Debug a test script. `src` is the script, as a string." + import pdb + + # Note that tempfile.NameTemporaryFile() cannot be used. As the + # docs say, a file so created cannot be opened by name a second time + # on modern Windows boxes, and execfile() needs to open it. + srcfilename = tempfile.mktemp(".py", "doctestdebug") + f = open(srcfilename, 'w') + f.write(src) + f.close() + + try: + if globs: + globs = globs.copy() + else: + globs = {} + + if pm: + try: + execfile(srcfilename, globs, globs) + except: + print sys.exc_info()[1] + pdb.post_mortem(sys.exc_info()[2]) + else: + # Note that %r is vital here. '%s' instead can, e.g., cause + # backslashes to get treated as metacharacters on Windows. + pdb.run("execfile(%r)" % srcfilename, globs, globs) + + finally: + os.remove(srcfilename) + +def debug(module, name, pm=False): + """Debug a single doctest docstring. + + Provide the module (or dotted name of the module) containing the + test to be debugged and the name (within the module) of the object + with the docstring with tests to be debugged. + """ + module = _normalize_module(module) + testsrc = testsource(module, name) + debug_script(testsrc, pm, module.__dict__) + + +__test__ = {} diff --git a/lib/spack/external/nose/failure.py b/lib/spack/external/nose/failure.py new file mode 100644 index 0000000000..c5fabfda5e --- /dev/null +++ b/lib/spack/external/nose/failure.py @@ -0,0 +1,42 @@ +import logging +import unittest +from traceback import format_tb +from nose.pyversion import is_base_exception + +log = logging.getLogger(__name__) + + +__all__ = ['Failure'] + + +class Failure(unittest.TestCase): + """Unloadable or unexecutable test. + + A Failure case is placed in a test suite to indicate the presence of a + test that could not be loaded or executed. A common example is a test + module that fails to import. + + """ + __test__ = False # do not collect + def __init__(self, exc_class, exc_val, tb=None, address=None): + log.debug("A failure! %s %s %s", exc_class, exc_val, format_tb(tb)) + self.exc_class = exc_class + self.exc_val = exc_val + self.tb = tb + self._address = address + unittest.TestCase.__init__(self) + + def __str__(self): + return "Failure: %s (%s)" % ( + getattr(self.exc_class, '__name__', self.exc_class), self.exc_val) + + def address(self): + return self._address + + def runTest(self): + if self.tb is not None: + if is_base_exception(self.exc_val): + raise self.exc_val, None, self.tb + raise self.exc_class, self.exc_val, self.tb + else: + raise self.exc_class(self.exc_val) diff --git a/lib/spack/external/nose/importer.py b/lib/spack/external/nose/importer.py new file mode 100644 index 0000000000..e677658ce6 --- /dev/null +++ b/lib/spack/external/nose/importer.py @@ -0,0 +1,167 @@ +"""Implements an importer that looks only in specific path (ignoring +sys.path), and uses a per-path cache in addition to sys.modules. This is +necessary because test modules in different directories frequently have the +same names, which means that the first loaded would mask the rest when using +the builtin importer. +""" +import logging +import os +import sys +from nose.config import Config + +from imp import find_module, load_module, acquire_lock, release_lock + +log = logging.getLogger(__name__) + +try: + _samefile = os.path.samefile +except AttributeError: + def _samefile(src, dst): + return (os.path.normcase(os.path.realpath(src)) == + os.path.normcase(os.path.realpath(dst))) + + +class Importer(object): + """An importer class that does only path-specific imports. That + is, the given module is not searched for on sys.path, but only at + the path or in the directory specified. + """ + def __init__(self, config=None): + if config is None: + config = Config() + self.config = config + + def importFromPath(self, path, fqname): + """Import a dotted-name package whose tail is at path. In other words, + given foo.bar and path/to/foo/bar.py, import foo from path/to/foo then + bar from path/to/foo/bar, returning bar. + """ + # find the base dir of the package + path_parts = os.path.normpath(os.path.abspath(path)).split(os.sep) + name_parts = fqname.split('.') + if path_parts[-1] == '__init__.py': + path_parts.pop() + path_parts = path_parts[:-(len(name_parts))] + dir_path = os.sep.join(path_parts) + # then import fqname starting from that dir + return self.importFromDir(dir_path, fqname) + + def importFromDir(self, dir, fqname): + """Import a module *only* from path, ignoring sys.path and + reloading if the version in sys.modules is not the one we want. + """ + dir = os.path.normpath(os.path.abspath(dir)) + log.debug("Import %s from %s", fqname, dir) + + # FIXME reimplement local per-dir cache? + + # special case for __main__ + if fqname == '__main__': + return sys.modules[fqname] + + if self.config.addPaths: + add_path(dir, self.config) + + path = [dir] + parts = fqname.split('.') + part_fqname = '' + mod = parent = fh = None + + for part in parts: + if part_fqname == '': + part_fqname = part + else: + part_fqname = "%s.%s" % (part_fqname, part) + try: + acquire_lock() + log.debug("find module part %s (%s) in %s", + part, part_fqname, path) + fh, filename, desc = find_module(part, path) + old = sys.modules.get(part_fqname) + if old is not None: + # test modules frequently have name overlap; make sure + # we get a fresh copy of anything we are trying to load + # from a new path + log.debug("sys.modules has %s as %s", part_fqname, old) + if (self.sameModule(old, filename) + or (self.config.firstPackageWins and + getattr(old, '__path__', None))): + mod = old + else: + del sys.modules[part_fqname] + mod = load_module(part_fqname, fh, filename, desc) + else: + mod = load_module(part_fqname, fh, filename, desc) + finally: + if fh: + fh.close() + release_lock() + if parent: + setattr(parent, part, mod) + if hasattr(mod, '__path__'): + path = mod.__path__ + parent = mod + return mod + + def _dirname_if_file(self, filename): + # We only take the dirname if we have a path to a non-dir, + # because taking the dirname of a symlink to a directory does not + # give the actual directory parent. + if os.path.isdir(filename): + return filename + else: + return os.path.dirname(filename) + + def sameModule(self, mod, filename): + mod_paths = [] + if hasattr(mod, '__path__'): + for path in mod.__path__: + mod_paths.append(self._dirname_if_file(path)) + elif hasattr(mod, '__file__'): + mod_paths.append(self._dirname_if_file(mod.__file__)) + else: + # builtin or other module-like object that + # doesn't have __file__; must be new + return False + new_path = self._dirname_if_file(filename) + for mod_path in mod_paths: + log.debug( + "module already loaded? mod: %s new: %s", + mod_path, new_path) + if _samefile(mod_path, new_path): + return True + return False + + +def add_path(path, config=None): + """Ensure that the path, or the root of the current package (if + path is in a package), is in sys.path. + """ + + # FIXME add any src-looking dirs seen too... need to get config for that + + log.debug('Add path %s' % path) + if not path: + return [] + added = [] + parent = os.path.dirname(path) + if (parent + and os.path.exists(os.path.join(path, '__init__.py'))): + added.extend(add_path(parent, config)) + elif not path in sys.path: + log.debug("insert %s into sys.path", path) + sys.path.insert(0, path) + added.append(path) + if config and config.srcDirs: + for dirname in config.srcDirs: + dirpath = os.path.join(path, dirname) + if os.path.isdir(dirpath): + sys.path.insert(0, dirpath) + added.append(dirpath) + return added + + +def remove_path(path): + log.debug('Remove path %s' % path) + if path in sys.path: + sys.path.remove(path) diff --git a/lib/spack/external/nose/inspector.py b/lib/spack/external/nose/inspector.py new file mode 100644 index 0000000000..a6c4a3e3b6 --- /dev/null +++ b/lib/spack/external/nose/inspector.py @@ -0,0 +1,207 @@ +"""Simple traceback introspection. Used to add additional information to +AssertionErrors in tests, so that failure messages may be more informative. +""" +import inspect +import logging +import re +import sys +import textwrap +import tokenize + +try: + from cStringIO import StringIO +except ImportError: + from StringIO import StringIO + +log = logging.getLogger(__name__) + +def inspect_traceback(tb): + """Inspect a traceback and its frame, returning source for the expression + where the exception was raised, with simple variable replacement performed + and the line on which the exception was raised marked with '>>' + """ + log.debug('inspect traceback %s', tb) + + # we only want the innermost frame, where the exception was raised + while tb.tb_next: + tb = tb.tb_next + + frame = tb.tb_frame + lines, exc_line = tbsource(tb) + + # figure out the set of lines to grab. + inspect_lines, mark_line = find_inspectable_lines(lines, exc_line) + src = StringIO(textwrap.dedent(''.join(inspect_lines))) + exp = Expander(frame.f_locals, frame.f_globals) + + while inspect_lines: + try: + for tok in tokenize.generate_tokens(src.readline): + exp(*tok) + except tokenize.TokenError, e: + # this can happen if our inspectable region happens to butt up + # against the end of a construct like a docstring with the closing + # """ on separate line + log.debug("Tokenizer error: %s", e) + inspect_lines.pop(0) + mark_line -= 1 + src = StringIO(textwrap.dedent(''.join(inspect_lines))) + exp = Expander(frame.f_locals, frame.f_globals) + continue + break + padded = [] + if exp.expanded_source: + exp_lines = exp.expanded_source.split('\n') + ep = 0 + for line in exp_lines: + if ep == mark_line: + padded.append('>> ' + line) + else: + padded.append(' ' + line) + ep += 1 + return '\n'.join(padded) + + +def tbsource(tb, context=6): + """Get source from a traceback object. + + A tuple of two things is returned: a list of lines of context from + the source code, and the index of the current line within that list. + The optional second argument specifies the number of lines of context + to return, which are centered around the current line. + + .. Note :: + This is adapted from inspect.py in the python 2.4 standard library, + since a bug in the 2.3 version of inspect prevents it from correctly + locating source lines in a traceback frame. + """ + + lineno = tb.tb_lineno + frame = tb.tb_frame + + if context > 0: + start = lineno - 1 - context//2 + log.debug("lineno: %s start: %s", lineno, start) + + try: + lines, dummy = inspect.findsource(frame) + except IOError: + lines, index = [''], 0 + else: + all_lines = lines + start = max(start, 1) + start = max(0, min(start, len(lines) - context)) + lines = lines[start:start+context] + index = lineno - 1 - start + + # python 2.5 compat: if previous line ends in a continuation, + # decrement start by 1 to match 2.4 behavior + if sys.version_info >= (2, 5) and index > 0: + while lines[index-1].strip().endswith('\\'): + start -= 1 + lines = all_lines[start:start+context] + else: + lines, index = [''], 0 + log.debug("tbsource lines '''%s''' around index %s", lines, index) + return (lines, index) + + +def find_inspectable_lines(lines, pos): + """Find lines in home that are inspectable. + + Walk back from the err line up to 3 lines, but don't walk back over + changes in indent level. + + Walk forward up to 3 lines, counting \ separated lines as 1. Don't walk + over changes in indent level (unless part of an extended line) + """ + cnt = re.compile(r'\\[\s\n]*$') + df = re.compile(r':[\s\n]*$') + ind = re.compile(r'^(\s*)') + toinspect = [] + home = lines[pos] + home_indent = ind.match(home).groups()[0] + + before = lines[max(pos-3, 0):pos] + before.reverse() + after = lines[pos+1:min(pos+4, len(lines))] + + for line in before: + if ind.match(line).groups()[0] == home_indent: + toinspect.append(line) + else: + break + toinspect.reverse() + toinspect.append(home) + home_pos = len(toinspect)-1 + continued = cnt.search(home) + for line in after: + if ((continued or ind.match(line).groups()[0] == home_indent) + and not df.search(line)): + toinspect.append(line) + continued = cnt.search(line) + else: + break + log.debug("Inspecting lines '''%s''' around %s", toinspect, home_pos) + return toinspect, home_pos + + +class Expander: + """Simple expression expander. Uses tokenize to find the names and + expands any that can be looked up in the frame. + """ + def __init__(self, locals, globals): + self.locals = locals + self.globals = globals + self.lpos = None + self.expanded_source = '' + + def __call__(self, ttype, tok, start, end, line): + # TODO + # deal with unicode properly + + # TODO + # Dealing with instance members + # always keep the last thing seen + # if the current token is a dot, + # get ready to getattr(lastthing, this thing) on the + # next call. + + if self.lpos is not None: + if start[1] >= self.lpos: + self.expanded_source += ' ' * (start[1]-self.lpos) + elif start[1] < self.lpos: + # newline, indent correctly + self.expanded_source += ' ' * start[1] + self.lpos = end[1] + + if ttype == tokenize.INDENT: + pass + elif ttype == tokenize.NAME: + # Clean this junk up + try: + val = self.locals[tok] + if callable(val): + val = tok + else: + val = repr(val) + except KeyError: + try: + val = self.globals[tok] + if callable(val): + val = tok + else: + val = repr(val) + + except KeyError: + val = tok + # FIXME... not sure how to handle things like funcs, classes + # FIXME this is broken for some unicode strings + self.expanded_source += val + else: + self.expanded_source += tok + # if this is the end of the line and the line ends with + # \, then tack a \ and newline onto the output + # print line[end[1]:] + if re.match(r'\s+\\\n', line[end[1]:]): + self.expanded_source += ' \\\n' diff --git a/lib/spack/external/nose/loader.py b/lib/spack/external/nose/loader.py new file mode 100644 index 0000000000..3744e54ff6 --- /dev/null +++ b/lib/spack/external/nose/loader.py @@ -0,0 +1,623 @@ +""" +Test Loader +----------- + +nose's test loader implements the same basic functionality as its +superclass, unittest.TestLoader, but extends it by more liberal +interpretations of what may be a test and how a test may be named. +""" +from __future__ import generators + +import logging +import os +import sys +import unittest +import types +from inspect import isfunction +from nose.pyversion import unbound_method, ismethod +from nose.case import FunctionTestCase, MethodTestCase +from nose.failure import Failure +from nose.config import Config +from nose.importer import Importer, add_path, remove_path +from nose.selector import defaultSelector, TestAddress +from nose.util import func_lineno, getpackage, isclass, isgenerator, \ + ispackage, regex_last_key, resolve_name, transplant_func, \ + transplant_class, test_address +from nose.suite import ContextSuiteFactory, ContextList, LazySuite +from nose.pyversion import sort_list, cmp_to_key + + +log = logging.getLogger(__name__) +#log.setLevel(logging.DEBUG) + +# for efficiency and easier mocking +op_normpath = os.path.normpath +op_abspath = os.path.abspath +op_join = os.path.join +op_isdir = os.path.isdir +op_isfile = os.path.isfile + + +__all__ = ['TestLoader', 'defaultTestLoader'] + + +class TestLoader(unittest.TestLoader): + """Test loader that extends unittest.TestLoader to: + + * Load tests from test-like functions and classes that are not + unittest.TestCase subclasses + * Find and load test modules in a directory + * Support tests that are generators + * Support easy extensions of or changes to that behavior through plugins + """ + config = None + importer = None + workingDir = None + selector = None + suiteClass = None + + def __init__(self, config=None, importer=None, workingDir=None, + selector=None): + """Initialize a test loader. + + Parameters (all optional): + + * config: provide a `nose.config.Config`_ or other config class + instance; if not provided a `nose.config.Config`_ with + default values is used. + * importer: provide an importer instance that implements + `importFromPath`. If not provided, a + `nose.importer.Importer`_ is used. + * workingDir: the directory to which file and module names are + relative. If not provided, assumed to be the current working + directory. + * selector: a selector class or instance. If a class is + provided, it will be instantiated with one argument, the + current config. If not provided, a `nose.selector.Selector`_ + is used. + """ + if config is None: + config = Config() + if importer is None: + importer = Importer(config=config) + if workingDir is None: + workingDir = config.workingDir + if selector is None: + selector = defaultSelector(config) + elif isclass(selector): + selector = selector(config) + self.config = config + self.importer = importer + self.workingDir = op_normpath(op_abspath(workingDir)) + self.selector = selector + if config.addPaths: + add_path(workingDir, config) + self.suiteClass = ContextSuiteFactory(config=config) + + self._visitedPaths = set([]) + + unittest.TestLoader.__init__(self) + + def getTestCaseNames(self, testCaseClass): + """Override to select with selector, unless + config.getTestCaseNamesCompat is True + """ + if self.config.getTestCaseNamesCompat: + return unittest.TestLoader.getTestCaseNames(self, testCaseClass) + + def wanted(attr, cls=testCaseClass, sel=self.selector): + item = getattr(cls, attr, None) + if isfunction(item): + item = unbound_method(cls, item) + elif not ismethod(item): + return False + return sel.wantMethod(item) + + cases = filter(wanted, dir(testCaseClass)) + + # add runTest if nothing else picked + if not cases and hasattr(testCaseClass, 'runTest'): + cases = ['runTest'] + if self.sortTestMethodsUsing: + sort_list(cases, cmp_to_key(self.sortTestMethodsUsing)) + return cases + + def _haveVisited(self, path): + # For cases where path is None, we always pretend we haven't visited + # them. + if path is None: + return False + + return path in self._visitedPaths + + def _addVisitedPath(self, path): + if path is not None: + self._visitedPaths.add(path) + + def loadTestsFromDir(self, path): + """Load tests from the directory at path. This is a generator + -- each suite of tests from a module or other file is yielded + and is expected to be executed before the next file is + examined. + """ + log.debug("load from dir %s", path) + plugins = self.config.plugins + plugins.beforeDirectory(path) + if self.config.addPaths: + paths_added = add_path(path, self.config) + + entries = os.listdir(path) + sort_list(entries, regex_last_key(self.config.testMatch)) + for entry in entries: + # this hard-coded initial-dot test will be removed: + # http://code.google.com/p/python-nose/issues/detail?id=82 + if entry.startswith('.'): + continue + entry_path = op_abspath(op_join(path, entry)) + is_file = op_isfile(entry_path) + wanted = False + if is_file: + is_dir = False + wanted = self.selector.wantFile(entry_path) + else: + is_dir = op_isdir(entry_path) + if is_dir: + # this hard-coded initial-underscore test will be removed: + # http://code.google.com/p/python-nose/issues/detail?id=82 + if entry.startswith('_'): + continue + wanted = self.selector.wantDirectory(entry_path) + is_package = ispackage(entry_path) + + # Python 3.3 now implements PEP 420: Implicit Namespace Packages. + # As a result, it's now possible that parent paths that have a + # segment with the same basename as our package ends up + # in module.__path__. So we have to keep track of what we've + # visited, and not-revisit them again. + if wanted and not self._haveVisited(entry_path): + self._addVisitedPath(entry_path) + if is_file: + plugins.beforeContext() + if entry.endswith('.py'): + yield self.loadTestsFromName( + entry_path, discovered=True) + else: + yield self.loadTestsFromFile(entry_path) + plugins.afterContext() + elif is_package: + # Load the entry as a package: given the full path, + # loadTestsFromName() will figure it out + yield self.loadTestsFromName( + entry_path, discovered=True) + else: + # Another test dir in this one: recurse lazily + yield self.suiteClass( + lambda: self.loadTestsFromDir(entry_path)) + tests = [] + for test in plugins.loadTestsFromDir(path): + tests.append(test) + # TODO: is this try/except needed? + try: + if tests: + yield self.suiteClass(tests) + except (KeyboardInterrupt, SystemExit): + raise + except: + yield self.suiteClass([Failure(*sys.exc_info())]) + + # pop paths + if self.config.addPaths: + for p in paths_added: + remove_path(p) + plugins.afterDirectory(path) + + def loadTestsFromFile(self, filename): + """Load tests from a non-module file. Default is to raise a + ValueError; plugins may implement `loadTestsFromFile` to + provide a list of tests loaded from the file. + """ + log.debug("Load from non-module file %s", filename) + try: + tests = [test for test in + self.config.plugins.loadTestsFromFile(filename)] + if tests: + # Plugins can yield False to indicate that they were + # unable to load tests from a file, but it was not an + # error -- the file just had no tests to load. + tests = filter(None, tests) + return self.suiteClass(tests) + else: + # Nothing was able to even try to load from this file + open(filename, 'r').close() # trigger os error + raise ValueError("Unable to load tests from file %s" + % filename) + except (KeyboardInterrupt, SystemExit): + raise + except: + exc = sys.exc_info() + return self.suiteClass( + [Failure(exc[0], exc[1], exc[2], + address=(filename, None, None))]) + + def loadTestsFromGenerator(self, generator, module): + """Lazy-load tests from a generator function. The generator function + may yield either: + + * a callable, or + * a function name resolvable within the same module + """ + def generate(g=generator, m=module): + try: + for test in g(): + test_func, arg = self.parseGeneratedTest(test) + if not callable(test_func): + test_func = getattr(m, test_func) + yield FunctionTestCase(test_func, arg=arg, descriptor=g) + except KeyboardInterrupt: + raise + except: + exc = sys.exc_info() + yield Failure(exc[0], exc[1], exc[2], + address=test_address(generator)) + return self.suiteClass(generate, context=generator, can_split=False) + + def loadTestsFromGeneratorMethod(self, generator, cls): + """Lazy-load tests from a generator method. + + This is more complicated than loading from a generator function, + since a generator method may yield: + + * a function + * a bound or unbound method, or + * a method name + """ + # convert the unbound generator method + # into a bound method so it can be called below + if hasattr(generator, 'im_class'): + cls = generator.im_class + inst = cls() + method = generator.__name__ + generator = getattr(inst, method) + + def generate(g=generator, c=cls): + try: + for test in g(): + test_func, arg = self.parseGeneratedTest(test) + if not callable(test_func): + test_func = unbound_method(c, getattr(c, test_func)) + if ismethod(test_func): + yield MethodTestCase(test_func, arg=arg, descriptor=g) + elif callable(test_func): + # In this case we're forcing the 'MethodTestCase' + # to run the inline function as its test call, + # but using the generator method as the 'method of + # record' (so no need to pass it as the descriptor) + yield MethodTestCase(g, test=test_func, arg=arg) + else: + yield Failure( + TypeError, + "%s is not a callable or method" % test_func) + except KeyboardInterrupt: + raise + except: + exc = sys.exc_info() + yield Failure(exc[0], exc[1], exc[2], + address=test_address(generator)) + return self.suiteClass(generate, context=generator, can_split=False) + + def loadTestsFromModule(self, module, path=None, discovered=False): + """Load all tests from module and return a suite containing + them. If the module has been discovered and is not test-like, + the suite will be empty by default, though plugins may add + their own tests. + """ + log.debug("Load from module %s", module) + tests = [] + test_classes = [] + test_funcs = [] + # For *discovered* modules, we only load tests when the module looks + # testlike. For modules we've been directed to load, we always + # look for tests. (discovered is set to True by loadTestsFromDir) + if not discovered or self.selector.wantModule(module): + for item in dir(module): + test = getattr(module, item, None) + # print "Check %s (%s) in %s" % (item, test, module.__name__) + if isclass(test): + if self.selector.wantClass(test): + test_classes.append(test) + elif isfunction(test) and self.selector.wantFunction(test): + test_funcs.append(test) + sort_list(test_classes, lambda x: x.__name__) + sort_list(test_funcs, func_lineno) + tests = map(lambda t: self.makeTest(t, parent=module), + test_classes + test_funcs) + + # Now, descend into packages + # FIXME can or should this be lazy? + # is this syntax 2.2 compatible? + module_paths = getattr(module, '__path__', []) + + if path: + path = os.path.normcase(os.path.realpath(path)) + + for module_path in module_paths: + log.debug("Load tests from module path %s?", module_path) + log.debug("path: %s os.path.realpath(%s): %s", + path, os.path.normcase(module_path), + os.path.realpath(os.path.normcase(module_path))) + if (self.config.traverseNamespace or not path) or \ + os.path.realpath( + os.path.normcase(module_path)).startswith(path): + # Egg files can be on sys.path, so make sure the path is a + # directory before trying to load from it. + if os.path.isdir(module_path): + tests.extend(self.loadTestsFromDir(module_path)) + + for test in self.config.plugins.loadTestsFromModule(module, path): + tests.append(test) + + return self.suiteClass(ContextList(tests, context=module)) + + def loadTestsFromName(self, name, module=None, discovered=False): + """Load tests from the entity with the given name. + + The name may indicate a file, directory, module, or any object + within a module. See `nose.util.split_test_name` for details on + test name parsing. + """ + # FIXME refactor this method into little bites? + log.debug("load from %s (%s)", name, module) + + suite = self.suiteClass + + # give plugins first crack + plug_tests = self.config.plugins.loadTestsFromName(name, module) + if plug_tests: + return suite(plug_tests) + + addr = TestAddress(name, workingDir=self.workingDir) + if module: + # Two cases: + # name is class.foo + # The addr will be incorrect, since it thinks class.foo is + # a dotted module name. It's actually a dotted attribute + # name. In this case we want to use the full submitted + # name as the name to load from the module. + # name is module:class.foo + # The addr will be correct. The part we want is the part after + # the :, which is in addr.call. + if addr.call: + name = addr.call + parent, obj = self.resolve(name, module) + if (isclass(parent) + and getattr(parent, '__module__', None) != module.__name__ + and not isinstance(obj, Failure)): + parent = transplant_class(parent, module.__name__) + obj = getattr(parent, obj.__name__) + log.debug("parent %s obj %s module %s", parent, obj, module) + if isinstance(obj, Failure): + return suite([obj]) + else: + return suite(ContextList([self.makeTest(obj, parent)], + context=parent)) + else: + if addr.module: + try: + if addr.filename is None: + module = resolve_name(addr.module) + else: + self.config.plugins.beforeImport( + addr.filename, addr.module) + # FIXME: to support module.name names, + # do what resolve-name does and keep trying to + # import, popping tail of module into addr.call, + # until we either get an import or run out of + # module parts + try: + module = self.importer.importFromPath( + addr.filename, addr.module) + finally: + self.config.plugins.afterImport( + addr.filename, addr.module) + except (KeyboardInterrupt, SystemExit): + raise + except: + exc = sys.exc_info() + return suite([Failure(exc[0], exc[1], exc[2], + address=addr.totuple())]) + if addr.call: + return self.loadTestsFromName(addr.call, module) + else: + return self.loadTestsFromModule( + module, addr.filename, + discovered=discovered) + elif addr.filename: + path = addr.filename + if addr.call: + package = getpackage(path) + if package is None: + return suite([ + Failure(ValueError, + "Can't find callable %s in file %s: " + "file is not a python module" % + (addr.call, path), + address=addr.totuple())]) + return self.loadTestsFromName(addr.call, module=package) + else: + if op_isdir(path): + # In this case we *can* be lazy since we know + # that each module in the dir will be fully + # loaded before its tests are executed; we + # also know that we're not going to be asked + # to load from . and ./some_module.py *as part + # of this named test load* + return LazySuite( + lambda: self.loadTestsFromDir(path)) + elif op_isfile(path): + return self.loadTestsFromFile(path) + else: + return suite([ + Failure(OSError, "No such file %s" % path, + address=addr.totuple())]) + else: + # just a function? what to do? I think it can only be + # handled when module is not None + return suite([ + Failure(ValueError, "Unresolvable test name %s" % name, + address=addr.totuple())]) + + def loadTestsFromNames(self, names, module=None): + """Load tests from all names, returning a suite containing all + tests. + """ + plug_res = self.config.plugins.loadTestsFromNames(names, module) + if plug_res: + suite, names = plug_res + if suite: + return self.suiteClass([ + self.suiteClass(suite), + unittest.TestLoader.loadTestsFromNames(self, names, module) + ]) + return unittest.TestLoader.loadTestsFromNames(self, names, module) + + def loadTestsFromTestCase(self, testCaseClass): + """Load tests from a unittest.TestCase subclass. + """ + cases = [] + plugins = self.config.plugins + for case in plugins.loadTestsFromTestCase(testCaseClass): + cases.append(case) + # For efficiency in the most common case, just call and return from + # super. This avoids having to extract cases and rebuild a context + # suite when there are no plugin-contributed cases. + if not cases: + return super(TestLoader, self).loadTestsFromTestCase(testCaseClass) + cases.extend( + [case for case in + super(TestLoader, self).loadTestsFromTestCase(testCaseClass)]) + return self.suiteClass(cases) + + def loadTestsFromTestClass(self, cls): + """Load tests from a test class that is *not* a unittest.TestCase + subclass. + + In this case, we can't depend on the class's `__init__` taking method + name arguments, so we have to compose a MethodTestCase for each + method in the class that looks testlike. + """ + def wanted(attr, cls=cls, sel=self.selector): + item = getattr(cls, attr, None) + if isfunction(item): + item = unbound_method(cls, item) + elif not ismethod(item): + return False + return sel.wantMethod(item) + cases = [self.makeTest(getattr(cls, case), cls) + for case in filter(wanted, dir(cls))] + for test in self.config.plugins.loadTestsFromTestClass(cls): + cases.append(test) + return self.suiteClass(ContextList(cases, context=cls)) + + def makeTest(self, obj, parent=None): + try: + return self._makeTest(obj, parent) + except (KeyboardInterrupt, SystemExit): + raise + except: + exc = sys.exc_info() + try: + addr = test_address(obj) + except KeyboardInterrupt: + raise + except: + addr = None + return Failure(exc[0], exc[1], exc[2], address=addr) + + def _makeTest(self, obj, parent=None): + """Given a test object and its parent, return a test case + or test suite. + """ + plug_tests = [] + try: + addr = test_address(obj) + except KeyboardInterrupt: + raise + except: + addr = None + for test in self.config.plugins.makeTest(obj, parent): + plug_tests.append(test) + # TODO: is this try/except needed? + try: + if plug_tests: + return self.suiteClass(plug_tests) + except (KeyboardInterrupt, SystemExit): + raise + except: + exc = sys.exc_info() + return Failure(exc[0], exc[1], exc[2], address=addr) + + if isfunction(obj) and parent and not isinstance(parent, types.ModuleType): + # This is a Python 3.x 'unbound method'. Wrap it with its + # associated class.. + obj = unbound_method(parent, obj) + + if isinstance(obj, unittest.TestCase): + return obj + elif isclass(obj): + if parent and obj.__module__ != parent.__name__: + obj = transplant_class(obj, parent.__name__) + if issubclass(obj, unittest.TestCase): + return self.loadTestsFromTestCase(obj) + else: + return self.loadTestsFromTestClass(obj) + elif ismethod(obj): + if parent is None: + parent = obj.__class__ + if issubclass(parent, unittest.TestCase): + return parent(obj.__name__) + else: + if isgenerator(obj): + return self.loadTestsFromGeneratorMethod(obj, parent) + else: + return MethodTestCase(obj) + elif isfunction(obj): + if parent and obj.__module__ != parent.__name__: + obj = transplant_func(obj, parent.__name__) + if isgenerator(obj): + return self.loadTestsFromGenerator(obj, parent) + else: + return FunctionTestCase(obj) + else: + return Failure(TypeError, + "Can't make a test from %s" % obj, + address=addr) + + def resolve(self, name, module): + """Resolve name within module + """ + obj = module + parts = name.split('.') + for part in parts: + parent, obj = obj, getattr(obj, part, None) + if obj is None: + # no such test + obj = Failure(ValueError, "No such test %s" % name) + return parent, obj + + def parseGeneratedTest(self, test): + """Given the yield value of a test generator, return a func and args. + + This is used in the two loadTestsFromGenerator* methods. + + """ + if not isinstance(test, tuple): # yield test + test_func, arg = (test, tuple()) + elif len(test) == 1: # yield (test,) + test_func, arg = (test[0], tuple()) + else: # yield test, foo, bar, ... + assert len(test) > 1 # sanity check + test_func, arg = (test[0], test[1:]) + return test_func, arg + +defaultTestLoader = TestLoader + diff --git a/lib/spack/external/nose/plugins/__init__.py b/lib/spack/external/nose/plugins/__init__.py new file mode 100644 index 0000000000..08ee8f3230 --- /dev/null +++ b/lib/spack/external/nose/plugins/__init__.py @@ -0,0 +1,190 @@ +""" +Writing Plugins +--------------- + +nose supports plugins for test collection, selection, observation and +reporting. There are two basic rules for plugins: + +* Plugin classes should subclass :class:`nose.plugins.Plugin`. + +* Plugins may implement any of the methods described in the class + :doc:`IPluginInterface ` in nose.plugins.base. Please note that + this class is for documentary purposes only; plugins may not subclass + IPluginInterface. + +Hello World +=========== + +Here's a basic plugin. It doesn't do much so read on for more ideas or dive +into the :doc:`IPluginInterface ` to see all available hooks. + +.. code-block:: python + + import logging + import os + + from nose.plugins import Plugin + + log = logging.getLogger('nose.plugins.helloworld') + + class HelloWorld(Plugin): + name = 'helloworld' + + def options(self, parser, env=os.environ): + super(HelloWorld, self).options(parser, env=env) + + def configure(self, options, conf): + super(HelloWorld, self).configure(options, conf) + if not self.enabled: + return + + def finalize(self, result): + log.info('Hello pluginized world!') + +Registering +=========== + +.. Note:: + Important note: the following applies only to the default + plugin manager. Other plugin managers may use different means to + locate and load plugins. + +For nose to find a plugin, it must be part of a package that uses +setuptools_, and the plugin must be included in the entry points defined +in the setup.py for the package: + +.. code-block:: python + + setup(name='Some plugin', + # ... + entry_points = { + 'nose.plugins.0.10': [ + 'someplugin = someplugin:SomePlugin' + ] + }, + # ... + ) + +Once the package is installed with install or develop, nose will be able +to load the plugin. + +.. _setuptools: http://peak.telecommunity.com/DevCenter/setuptools + +Registering a plugin without setuptools +======================================= + +It is currently possible to register a plugin programmatically by +creating a custom nose runner like this : + +.. code-block:: python + + import nose + from yourplugin import YourPlugin + + if __name__ == '__main__': + nose.main(addplugins=[YourPlugin()]) + +Defining options +================ + +All plugins must implement the methods ``options(self, parser, env)`` +and ``configure(self, options, conf)``. Subclasses of nose.plugins.Plugin +that want the standard options should call the superclass methods. + +nose uses optparse.OptionParser from the standard library to parse +arguments. A plugin's ``options()`` method receives a parser +instance. It's good form for a plugin to use that instance only to add +additional arguments that take only long arguments (--like-this). Most +of nose's built-in arguments get their default value from an environment +variable. + +A plugin's ``configure()`` method receives the parsed ``OptionParser`` options +object, as well as the current config object. Plugins should configure their +behavior based on the user-selected settings, and may raise exceptions +if the configured behavior is nonsensical. + +Logging +======= + +nose uses the logging classes from the standard library. To enable users +to view debug messages easily, plugins should use ``logging.getLogger()`` to +acquire a logger in the ``nose.plugins`` namespace. + +Recipes +======= + +* Writing a plugin that monitors or controls test result output + + Implement any or all of ``addError``, ``addFailure``, etc., to monitor test + results. If you also want to monitor output, implement + ``setOutputStream`` and keep a reference to the output stream. If you + want to prevent the builtin ``TextTestResult`` output, implement + ``setOutputSteam`` and *return a dummy stream*. The default output will go + to the dummy stream, while you send your desired output to the real stream. + + Example: `examples/html_plugin/htmlplug.py`_ + +* Writing a plugin that handles exceptions + + Subclass :doc:`ErrorClassPlugin `. + + Examples: :doc:`nose.plugins.deprecated `, + :doc:`nose.plugins.skip ` + +* Writing a plugin that adds detail to error reports + + Implement ``formatError`` and/or ``formatFailure``. The error tuple + you return (error class, error message, traceback) will replace the + original error tuple. + + Examples: :doc:`nose.plugins.capture `, + :doc:`nose.plugins.failuredetail ` + +* Writing a plugin that loads tests from files other than python modules + + Implement ``wantFile`` and ``loadTestsFromFile``. In ``wantFile``, + return True for files that you want to examine for tests. In + ``loadTestsFromFile``, for those files, return an iterable + containing TestCases (or yield them as you find them; + ``loadTestsFromFile`` may also be a generator). + + Example: :doc:`nose.plugins.doctests ` + +* Writing a plugin that prints a report + + Implement ``begin`` if you need to perform setup before testing + begins. Implement ``report`` and output your report to the provided stream. + + Examples: :doc:`nose.plugins.cover `, :doc:`nose.plugins.prof ` + +* Writing a plugin that selects or rejects tests + + Implement any or all ``want*`` methods. Return False to reject the test + candidate, True to accept it -- which means that the test candidate + will pass through the rest of the system, so you must be prepared to + load tests from it if tests can't be loaded by the core loader or + another plugin -- and None if you don't care. + + Examples: :doc:`nose.plugins.attrib `, + :doc:`nose.plugins.doctests `, :doc:`nose.plugins.testid ` + + +More Examples +============= + +See any builtin plugin or example plugin in the examples_ directory in +the nose source distribution. There is a list of third-party plugins +`on jottit`_. + +.. _examples/html_plugin/htmlplug.py: http://python-nose.googlecode.com/svn/trunk/examples/html_plugin/htmlplug.py +.. _examples: http://python-nose.googlecode.com/svn/trunk/examples +.. _on jottit: http://nose-plugins.jottit.com/ + +""" +from nose.plugins.base import Plugin +from nose.plugins.manager import * +from nose.plugins.plugintest import PluginTester + +if __name__ == '__main__': + import doctest + doctest.testmod() diff --git a/lib/spack/external/nose/plugins/allmodules.py b/lib/spack/external/nose/plugins/allmodules.py new file mode 100644 index 0000000000..1ccd7773a7 --- /dev/null +++ b/lib/spack/external/nose/plugins/allmodules.py @@ -0,0 +1,45 @@ +"""Use the AllModules plugin by passing ``--all-modules`` or setting the +NOSE_ALL_MODULES environment variable to enable collection and execution of +tests in all python modules. Normal nose behavior is to look for tests only in +modules that match testMatch. + +More information: :doc:`../doc_tests/test_allmodules/test_allmodules` + +.. warning :: + + This plugin can have surprising interactions with plugins that load tests + from what nose normally considers non-test modules, such as + the :doc:`doctest plugin `. This is because any given + object in a module can't be loaded both by a plugin and the normal nose + :class:`test loader `. Also, if you have functions + or classes in non-test modules that look like tests but aren't, you will + likely see errors as nose attempts to run them as tests. + +""" + +import os +from nose.plugins.base import Plugin + +class AllModules(Plugin): + """Collect tests from all python modules. + """ + def options(self, parser, env): + """Register commandline options. + """ + env_opt = 'NOSE_ALL_MODULES' + parser.add_option('--all-modules', + action="store_true", + dest=self.enableOpt, + default=env.get(env_opt), + help="Enable plugin %s: %s [%s]" % + (self.__class__.__name__, self.help(), env_opt)) + + def wantFile(self, file): + """Override to return True for all files ending with .py""" + # always want .py files + if file.endswith('.py'): + return True + + def wantModule(self, module): + """Override return True for all modules""" + return True diff --git a/lib/spack/external/nose/plugins/attrib.py b/lib/spack/external/nose/plugins/attrib.py new file mode 100644 index 0000000000..3d4422a23a --- /dev/null +++ b/lib/spack/external/nose/plugins/attrib.py @@ -0,0 +1,286 @@ +"""Attribute selector plugin. + +Oftentimes when testing you will want to select tests based on +criteria rather then simply by filename. For example, you might want +to run all tests except for the slow ones. You can do this with the +Attribute selector plugin by setting attributes on your test methods. +Here is an example: + +.. code-block:: python + + def test_big_download(): + import urllib + # commence slowness... + + test_big_download.slow = 1 + +Once you've assigned an attribute ``slow = 1`` you can exclude that +test and all other tests having the slow attribute by running :: + + $ nosetests -a '!slow' + +There is also a decorator available for you that will set attributes. +Here's how to set ``slow=1`` like above with the decorator: + +.. code-block:: python + + from nose.plugins.attrib import attr + @attr('slow') + def test_big_download(): + import urllib + # commence slowness... + +And here's how to set an attribute with a specific value: + +.. code-block:: python + + from nose.plugins.attrib import attr + @attr(speed='slow') + def test_big_download(): + import urllib + # commence slowness... + +This test could be run with :: + + $ nosetests -a speed=slow + +In Python 2.6 and higher, ``@attr`` can be used on a class to set attributes +on all its test methods at once. For example: + +.. code-block:: python + + from nose.plugins.attrib import attr + @attr(speed='slow') + class MyTestCase: + def test_long_integration(self): + pass + def test_end_to_end_something(self): + pass + +Below is a reference to the different syntaxes available. + +Simple syntax +------------- + +Examples of using the ``-a`` and ``--attr`` options: + +* ``nosetests -a status=stable`` + Only runs tests with attribute "status" having value "stable" + +* ``nosetests -a priority=2,status=stable`` + Runs tests having both attributes and values + +* ``nosetests -a priority=2 -a slow`` + Runs tests that match either attribute + +* ``nosetests -a tags=http`` + If a test's ``tags`` attribute was a list and it contained the value + ``http`` then it would be run + +* ``nosetests -a slow`` + Runs tests with the attribute ``slow`` if its value does not equal False + (False, [], "", etc...) + +* ``nosetests -a '!slow'`` + Runs tests that do NOT have the attribute ``slow`` or have a ``slow`` + attribute that is equal to False + **NOTE**: + if your shell (like bash) interprets '!' as a special character make sure to + put single quotes around it. + +Expression Evaluation +--------------------- + +Examples using the ``-A`` and ``--eval-attr`` options: + +* ``nosetests -A "not slow"`` + Evaluates the Python expression "not slow" and runs the test if True + +* ``nosetests -A "(priority > 5) and not slow"`` + Evaluates a complex Python expression and runs the test if True + +""" +import inspect +import logging +import os +import sys +from inspect import isfunction +from nose.plugins.base import Plugin +from nose.util import tolist + +log = logging.getLogger('nose.plugins.attrib') +compat_24 = sys.version_info >= (2, 4) + +def attr(*args, **kwargs): + """Decorator that adds attributes to classes or functions + for use with the Attribute (-a) plugin. + """ + def wrap_ob(ob): + for name in args: + setattr(ob, name, True) + for name, value in kwargs.iteritems(): + setattr(ob, name, value) + return ob + return wrap_ob + +def get_method_attr(method, cls, attr_name, default = False): + """Look up an attribute on a method/ function. + If the attribute isn't found there, looking it up in the + method's class, if any. + """ + Missing = object() + value = getattr(method, attr_name, Missing) + if value is Missing and cls is not None: + value = getattr(cls, attr_name, Missing) + if value is Missing: + return default + return value + + +class ContextHelper: + """Object that can act as context dictionary for eval and looks up + names as attributes on a method/ function and its class. + """ + def __init__(self, method, cls): + self.method = method + self.cls = cls + + def __getitem__(self, name): + return get_method_attr(self.method, self.cls, name) + + +class AttributeSelector(Plugin): + """Selects test cases to be run based on their attributes. + """ + + def __init__(self): + Plugin.__init__(self) + self.attribs = [] + + def options(self, parser, env): + """Register command line options""" + parser.add_option("-a", "--attr", + dest="attr", action="append", + default=env.get('NOSE_ATTR'), + metavar="ATTR", + help="Run only tests that have attributes " + "specified by ATTR [NOSE_ATTR]") + # disable in < 2.4: eval can't take needed args + if compat_24: + parser.add_option("-A", "--eval-attr", + dest="eval_attr", metavar="EXPR", action="append", + default=env.get('NOSE_EVAL_ATTR'), + help="Run only tests for whose attributes " + "the Python expression EXPR evaluates " + "to True [NOSE_EVAL_ATTR]") + + def configure(self, options, config): + """Configure the plugin and system, based on selected options. + + attr and eval_attr may each be lists. + + self.attribs will be a list of lists of tuples. In that list, each + list is a group of attributes, all of which must match for the rule to + match. + """ + self.attribs = [] + + # handle python eval-expression parameter + if compat_24 and options.eval_attr: + eval_attr = tolist(options.eval_attr) + for attr in eval_attr: + # "" + # -> eval(expr) in attribute context must be True + def eval_in_context(expr, obj, cls): + return eval(expr, None, ContextHelper(obj, cls)) + self.attribs.append([(attr, eval_in_context)]) + + # attribute requirements are a comma separated list of + # 'key=value' pairs + if options.attr: + std_attr = tolist(options.attr) + for attr in std_attr: + # all attributes within an attribute group must match + attr_group = [] + for attrib in attr.strip().split(","): + # don't die on trailing comma + if not attrib: + continue + items = attrib.split("=", 1) + if len(items) > 1: + # "name=value" + # -> 'str(obj.name) == value' must be True + key, value = items + else: + key = items[0] + if key[0] == "!": + # "!name" + # 'bool(obj.name)' must be False + key = key[1:] + value = False + else: + # "name" + # -> 'bool(obj.name)' must be True + value = True + attr_group.append((key, value)) + self.attribs.append(attr_group) + if self.attribs: + self.enabled = True + + def validateAttrib(self, method, cls = None): + """Verify whether a method has the required attributes + The method is considered a match if it matches all attributes + for any attribute group. + .""" + # TODO: is there a need for case-sensitive value comparison? + any = False + for group in self.attribs: + match = True + for key, value in group: + attr = get_method_attr(method, cls, key) + if callable(value): + if not value(key, method, cls): + match = False + break + elif value is True: + # value must exist and be True + if not bool(attr): + match = False + break + elif value is False: + # value must not exist or be False + if bool(attr): + match = False + break + elif type(attr) in (list, tuple): + # value must be found in the list attribute + if not str(value).lower() in [str(x).lower() + for x in attr]: + match = False + break + else: + # value must match, convert to string and compare + if (value != attr + and str(value).lower() != str(attr).lower()): + match = False + break + any = any or match + if any: + # not True because we don't want to FORCE the selection of the + # item, only say that it is acceptable + return None + return False + + def wantFunction(self, function): + """Accept the function if its attributes match. + """ + return self.validateAttrib(function) + + def wantMethod(self, method): + """Accept the method if its attributes match. + """ + try: + cls = method.im_class + except AttributeError: + return False + return self.validateAttrib(method, cls) diff --git a/lib/spack/external/nose/plugins/base.py b/lib/spack/external/nose/plugins/base.py new file mode 100644 index 0000000000..f09beb696f --- /dev/null +++ b/lib/spack/external/nose/plugins/base.py @@ -0,0 +1,725 @@ +import os +import textwrap +from optparse import OptionConflictError +from warnings import warn +from nose.util import tolist + +class Plugin(object): + """Base class for nose plugins. It's recommended but not *necessary* to + subclass this class to create a plugin, but all plugins *must* implement + `options(self, parser, env)` and `configure(self, options, conf)`, and + must have the attributes `enabled`, `name` and `score`. The `name` + attribute may contain hyphens ('-'). + + Plugins should not be enabled by default. + + Subclassing Plugin (and calling the superclass methods in + __init__, configure, and options, if you override them) will give + your plugin some friendly default behavior: + + * A --with-$name option will be added to the command line interface + to enable the plugin, and a corresponding environment variable + will be used as the default value. The plugin class's docstring + will be used as the help for this option. + * The plugin will not be enabled unless this option is selected by + the user. + """ + can_configure = False + enabled = False + enableOpt = None + name = None + score = 100 + + def __init__(self): + if self.name is None: + self.name = self.__class__.__name__.lower() + if self.enableOpt is None: + self.enableOpt = "enable_plugin_%s" % self.name.replace('-', '_') + + def addOptions(self, parser, env=None): + """Add command-line options for this plugin. + + The base plugin class adds --with-$name by default, used to enable the + plugin. + + .. warning :: Don't implement addOptions unless you want to override + all default option handling behavior, including + warnings for conflicting options. Implement + :meth:`options + ` + instead. + """ + self.add_options(parser, env) + + def add_options(self, parser, env=None): + """Non-camel-case version of func name for backwards compatibility. + + .. warning :: + + DEPRECATED: Do not use this method, + use :meth:`options ` + instead. + + """ + # FIXME raise deprecation warning if wasn't called by wrapper + if env is None: + env = os.environ + try: + self.options(parser, env) + self.can_configure = True + except OptionConflictError, e: + warn("Plugin %s has conflicting option string: %s and will " + "be disabled" % (self, e), RuntimeWarning) + self.enabled = False + self.can_configure = False + + def options(self, parser, env): + """Register commandline options. + + Implement this method for normal options behavior with protection from + OptionConflictErrors. If you override this method and want the default + --with-$name option to be registered, be sure to call super(). + """ + env_opt = 'NOSE_WITH_%s' % self.name.upper() + env_opt = env_opt.replace('-', '_') + parser.add_option("--with-%s" % self.name, + action="store_true", + dest=self.enableOpt, + default=env.get(env_opt), + help="Enable plugin %s: %s [%s]" % + (self.__class__.__name__, self.help(), env_opt)) + + def configure(self, options, conf): + """Configure the plugin and system, based on selected options. + + The base plugin class sets the plugin to enabled if the enable option + for the plugin (self.enableOpt) is true. + """ + if not self.can_configure: + return + self.conf = conf + if hasattr(options, self.enableOpt): + self.enabled = getattr(options, self.enableOpt) + + def help(self): + """Return help for this plugin. This will be output as the help + section of the --with-$name option that enables the plugin. + """ + if self.__class__.__doc__: + # doc sections are often indented; compress the spaces + return textwrap.dedent(self.__class__.__doc__) + return "(no help available)" + + # Compatiblity shim + def tolist(self, val): + warn("Plugin.tolist is deprecated. Use nose.util.tolist instead", + DeprecationWarning) + return tolist(val) + + +class IPluginInterface(object): + """ + IPluginInterface describes the plugin API. Do not subclass or use this + class directly. + """ + def __new__(cls, *arg, **kw): + raise TypeError("IPluginInterface class is for documentation only") + + def addOptions(self, parser, env): + """Called to allow plugin to register command-line options with the + parser. DO NOT return a value from this method unless you want to stop + all other plugins from setting their options. + + .. warning :: + + DEPRECATED -- implement + :meth:`options ` instead. + """ + pass + add_options = addOptions + add_options.deprecated = True + + def addDeprecated(self, test): + """Called when a deprecated test is seen. DO NOT return a value + unless you want to stop other plugins from seeing the deprecated + test. + + .. warning :: DEPRECATED -- check error class in addError instead + """ + pass + addDeprecated.deprecated = True + + def addError(self, test, err): + """Called when a test raises an uncaught exception. DO NOT return a + value unless you want to stop other plugins from seeing that the + test has raised an error. + + :param test: the test case + :type test: :class:`nose.case.Test` + :param err: sys.exc_info() tuple + :type err: 3-tuple + """ + pass + addError.changed = True + + def addFailure(self, test, err): + """Called when a test fails. DO NOT return a value unless you + want to stop other plugins from seeing that the test has failed. + + :param test: the test case + :type test: :class:`nose.case.Test` + :param err: 3-tuple + :type err: sys.exc_info() tuple + """ + pass + addFailure.changed = True + + def addSkip(self, test): + """Called when a test is skipped. DO NOT return a value unless + you want to stop other plugins from seeing the skipped test. + + .. warning:: DEPRECATED -- check error class in addError instead + """ + pass + addSkip.deprecated = True + + def addSuccess(self, test): + """Called when a test passes. DO NOT return a value unless you + want to stop other plugins from seeing the passing test. + + :param test: the test case + :type test: :class:`nose.case.Test` + """ + pass + addSuccess.changed = True + + def afterContext(self): + """Called after a context (generally a module) has been + lazy-loaded, imported, setup, had its tests loaded and + executed, and torn down. + """ + pass + afterContext._new = True + + def afterDirectory(self, path): + """Called after all tests have been loaded from directory at path + and run. + + :param path: the directory that has finished processing + :type path: string + """ + pass + afterDirectory._new = True + + def afterImport(self, filename, module): + """Called after module is imported from filename. afterImport + is called even if the import failed. + + :param filename: The file that was loaded + :type filename: string + :param module: The name of the module + :type module: string + """ + pass + afterImport._new = True + + def afterTest(self, test): + """Called after the test has been run and the result recorded + (after stopTest). + + :param test: the test case + :type test: :class:`nose.case.Test` + """ + pass + afterTest._new = True + + def beforeContext(self): + """Called before a context (generally a module) is + examined. Because the context is not yet loaded, plugins don't + get to know what the context is; so any context operations + should use a stack that is pushed in `beforeContext` and popped + in `afterContext` to ensure they operate symmetrically. + + `beforeContext` and `afterContext` are mainly useful for tracking + and restoring global state around possible changes from within a + context, whatever the context may be. If you need to operate on + contexts themselves, see `startContext` and `stopContext`, which + are passed the context in question, but are called after + it has been loaded (imported in the module case). + """ + pass + beforeContext._new = True + + def beforeDirectory(self, path): + """Called before tests are loaded from directory at path. + + :param path: the directory that is about to be processed + """ + pass + beforeDirectory._new = True + + def beforeImport(self, filename, module): + """Called before module is imported from filename. + + :param filename: The file that will be loaded + :param module: The name of the module found in file + :type module: string + """ + beforeImport._new = True + + def beforeTest(self, test): + """Called before the test is run (before startTest). + + :param test: the test case + :type test: :class:`nose.case.Test` + """ + pass + beforeTest._new = True + + def begin(self): + """Called before any tests are collected or run. Use this to + perform any setup needed before testing begins. + """ + pass + + def configure(self, options, conf): + """Called after the command line has been parsed, with the + parsed options and the config container. Here, implement any + config storage or changes to state or operation that are set + by command line options. + + DO NOT return a value from this method unless you want to + stop all other plugins from being configured. + """ + pass + + def finalize(self, result): + """Called after all report output, including output from all + plugins, has been sent to the stream. Use this to print final + test results or perform final cleanup. Return None to allow + other plugins to continue printing, or any other value to stop + them. + + :param result: test result object + + .. Note:: When tests are run under a test runner other than + :class:`nose.core.TextTestRunner`, such as + via ``python setup.py test``, this method may be called + **before** the default report output is sent. + """ + pass + + def describeTest(self, test): + """Return a test description. + + Called by :meth:`nose.case.Test.shortDescription`. + + :param test: the test case + :type test: :class:`nose.case.Test` + """ + pass + describeTest._new = True + + def formatError(self, test, err): + """Called in result.addError, before plugin.addError. If you + want to replace or modify the error tuple, return a new error + tuple, otherwise return err, the original error tuple. + + :param test: the test case + :type test: :class:`nose.case.Test` + :param err: sys.exc_info() tuple + :type err: 3-tuple + """ + pass + formatError._new = True + formatError.chainable = True + # test arg is not chainable + formatError.static_args = (True, False) + + def formatFailure(self, test, err): + """Called in result.addFailure, before plugin.addFailure. If you + want to replace or modify the error tuple, return a new error + tuple, otherwise return err, the original error tuple. + + :param test: the test case + :type test: :class:`nose.case.Test` + :param err: sys.exc_info() tuple + :type err: 3-tuple + """ + pass + formatFailure._new = True + formatFailure.chainable = True + # test arg is not chainable + formatFailure.static_args = (True, False) + + def handleError(self, test, err): + """Called on addError. To handle the error yourself and prevent normal + error processing, return a true value. + + :param test: the test case + :type test: :class:`nose.case.Test` + :param err: sys.exc_info() tuple + :type err: 3-tuple + """ + pass + handleError._new = True + + def handleFailure(self, test, err): + """Called on addFailure. To handle the failure yourself and + prevent normal failure processing, return a true value. + + :param test: the test case + :type test: :class:`nose.case.Test` + :param err: sys.exc_info() tuple + :type err: 3-tuple + """ + pass + handleFailure._new = True + + def loadTestsFromDir(self, path): + """Return iterable of tests from a directory. May be a + generator. Each item returned must be a runnable + unittest.TestCase (or subclass) instance or suite instance. + Return None if your plugin cannot collect any tests from + directory. + + :param path: The path to the directory. + """ + pass + loadTestsFromDir.generative = True + loadTestsFromDir._new = True + + def loadTestsFromModule(self, module, path=None): + """Return iterable of tests in a module. May be a + generator. Each item returned must be a runnable + unittest.TestCase (or subclass) instance. + Return None if your plugin cannot + collect any tests from module. + + :param module: The module object + :type module: python module + :param path: the path of the module to search, to distinguish from + namespace package modules + + .. note:: + + NEW. The ``path`` parameter will only be passed by nose 0.11 + or above. + """ + pass + loadTestsFromModule.generative = True + + def loadTestsFromName(self, name, module=None, importPath=None): + """Return tests in this file or module. Return None if you are not able + to load any tests, or an iterable if you are. May be a + generator. + + :param name: The test name. May be a file or module name plus a test + callable. Use split_test_name to split into parts. Or it might + be some crazy name of your own devising, in which case, do + whatever you want. + :param module: Module from which the name is to be loaded + :param importPath: Path from which file (must be a python module) was + found + + .. warning:: DEPRECATED: this argument will NOT be passed. + """ + pass + loadTestsFromName.generative = True + + def loadTestsFromNames(self, names, module=None): + """Return a tuple of (tests loaded, remaining names). Return + None if you are not able to load any tests. Multiple plugins + may implement loadTestsFromNames; the remaining name list from + each will be passed to the next as input. + + :param names: List of test names. + :type names: iterable + :param module: Module from which the names are to be loaded + """ + pass + loadTestsFromNames._new = True + loadTestsFromNames.chainable = True + + def loadTestsFromFile(self, filename): + """Return tests in this file. Return None if you are not + interested in loading any tests, or an iterable if you are and + can load some. May be a generator. *If you are interested in + loading tests from the file and encounter no errors, but find + no tests, yield False or return [False].* + + .. Note:: This method replaces loadTestsFromPath from the 0.9 + API. + + :param filename: The full path to the file or directory. + """ + pass + loadTestsFromFile.generative = True + loadTestsFromFile._new = True + + def loadTestsFromPath(self, path): + """ + .. warning:: DEPRECATED -- use loadTestsFromFile instead + """ + pass + loadTestsFromPath.deprecated = True + + def loadTestsFromTestCase(self, cls): + """Return tests in this test case class. Return None if you are + not able to load any tests, or an iterable if you are. May be a + generator. + + :param cls: The test case class. Must be subclass of + :class:`unittest.TestCase`. + """ + pass + loadTestsFromTestCase.generative = True + + def loadTestsFromTestClass(self, cls): + """Return tests in this test class. Class will *not* be a + unittest.TestCase subclass. Return None if you are not able to + load any tests, an iterable if you are. May be a generator. + + :param cls: The test case class. Must be **not** be subclass of + :class:`unittest.TestCase`. + """ + pass + loadTestsFromTestClass._new = True + loadTestsFromTestClass.generative = True + + def makeTest(self, obj, parent): + """Given an object and its parent, return or yield one or more + test cases. Each test must be a unittest.TestCase (or subclass) + instance. This is called before default test loading to allow + plugins to load an alternate test case or cases for an + object. May be a generator. + + :param obj: The object to be made into a test + :param parent: The parent of obj (eg, for a method, the class) + """ + pass + makeTest._new = True + makeTest.generative = True + + def options(self, parser, env): + """Called to allow plugin to register command line + options with the parser. + + DO NOT return a value from this method unless you want to stop + all other plugins from setting their options. + + :param parser: options parser instance + :type parser: :class:`ConfigParser.ConfigParser` + :param env: environment, default is os.environ + """ + pass + options._new = True + + def prepareTest(self, test): + """Called before the test is run by the test runner. Please + note the article *the* in the previous sentence: prepareTest + is called *only once*, and is passed the test case or test + suite that the test runner will execute. It is *not* called + for each individual test case. If you return a non-None value, + that return value will be run as the test. Use this hook to + wrap or decorate the test with another function. If you need + to modify or wrap individual test cases, use `prepareTestCase` + instead. + + :param test: the test case + :type test: :class:`nose.case.Test` + """ + pass + + def prepareTestCase(self, test): + """Prepare or wrap an individual test case. Called before + execution of the test. The test passed here is a + nose.case.Test instance; the case to be executed is in the + test attribute of the passed case. To modify the test to be + run, you should return a callable that takes one argument (the + test result object) -- it is recommended that you *do not* + side-effect the nose.case.Test instance you have been passed. + + Keep in mind that when you replace the test callable you are + replacing the run() method of the test case -- including the + exception handling and result calls, etc. + + :param test: the test case + :type test: :class:`nose.case.Test` + """ + pass + prepareTestCase._new = True + + def prepareTestLoader(self, loader): + """Called before tests are loaded. To replace the test loader, + return a test loader. To allow other plugins to process the + test loader, return None. Only one plugin may replace the test + loader. Only valid when using nose.TestProgram. + + :param loader: :class:`nose.loader.TestLoader` + (or other loader) instance + """ + pass + prepareTestLoader._new = True + + def prepareTestResult(self, result): + """Called before the first test is run. To use a different + test result handler for all tests than the given result, + return a test result handler. NOTE however that this handler + will only be seen by tests, that is, inside of the result + proxy system. The TestRunner and TestProgram -- whether nose's + or other -- will continue to see the original result + handler. For this reason, it is usually better to monkeypatch + the result (for instance, if you want to handle some + exceptions in a unique way). Only one plugin may replace the + result, but many may monkeypatch it. If you want to + monkeypatch and stop other plugins from doing so, monkeypatch + and return the patched result. + + :param result: :class:`nose.result.TextTestResult` + (or other result) instance + """ + pass + prepareTestResult._new = True + + def prepareTestRunner(self, runner): + """Called before tests are run. To replace the test runner, + return a test runner. To allow other plugins to process the + test runner, return None. Only valid when using nose.TestProgram. + + :param runner: :class:`nose.core.TextTestRunner` + (or other runner) instance + """ + pass + prepareTestRunner._new = True + + def report(self, stream): + """Called after all error output has been printed. Print your + plugin's report to the provided stream. Return None to allow + other plugins to print reports, any other value to stop them. + + :param stream: stream object; send your output here + :type stream: file-like object + """ + pass + + def setOutputStream(self, stream): + """Called before test output begins. To direct test output to a + new stream, return a stream object, which must implement a + `write(msg)` method. If you only want to note the stream, not + capture or redirect it, then return None. + + :param stream: stream object; send your output here + :type stream: file-like object + """ + + def startContext(self, context): + """Called before context setup and the running of tests in the + context. Note that tests have already been *loaded* from the + context before this call. + + :param context: the context about to be setup. May be a module or + class, or any other object that contains tests. + """ + pass + startContext._new = True + + def startTest(self, test): + """Called before each test is run. DO NOT return a value unless + you want to stop other plugins from seeing the test start. + + :param test: the test case + :type test: :class:`nose.case.Test` + """ + pass + + def stopContext(self, context): + """Called after the tests in a context have run and the + context has been torn down. + + :param context: the context that has been torn down. May be a module or + class, or any other object that contains tests. + """ + pass + stopContext._new = True + + def stopTest(self, test): + """Called after each test is run. DO NOT return a value unless + you want to stop other plugins from seeing that the test has stopped. + + :param test: the test case + :type test: :class:`nose.case.Test` + """ + pass + + def testName(self, test): + """Return a short test name. Called by `nose.case.Test.__str__`. + + :param test: the test case + :type test: :class:`nose.case.Test` + """ + pass + testName._new = True + + def wantClass(self, cls): + """Return true if you want the main test selector to collect + tests from this class, false if you don't, and None if you don't + care. + + :param cls: The class being examined by the selector + """ + pass + + def wantDirectory(self, dirname): + """Return true if you want test collection to descend into this + directory, false if you do not, and None if you don't care. + + :param dirname: Full path to directory being examined by the selector + """ + pass + + def wantFile(self, file): + """Return true if you want to collect tests from this file, + false if you do not and None if you don't care. + + Change from 0.9: The optional package parameter is no longer passed. + + :param file: Full path to file being examined by the selector + """ + pass + + def wantFunction(self, function): + """Return true to collect this function as a test, false to + prevent it from being collected, and None if you don't care. + + :param function: The function object being examined by the selector + """ + pass + + def wantMethod(self, method): + """Return true to collect this method as a test, false to + prevent it from being collected, and None if you don't care. + + :param method: The method object being examined by the selector + :type method: unbound method + """ + pass + + def wantModule(self, module): + """Return true if you want to collection to descend into this + module, false to prevent the collector from descending into the + module, and None if you don't care. + + :param module: The module object being examined by the selector + :type module: python module + """ + pass + + def wantModuleTests(self, module): + """ + .. warning:: DEPRECATED -- this method will not be called, it has + been folded into wantModule. + """ + pass + wantModuleTests.deprecated = True + diff --git a/lib/spack/external/nose/plugins/builtin.py b/lib/spack/external/nose/plugins/builtin.py new file mode 100644 index 0000000000..4fcc0018ad --- /dev/null +++ b/lib/spack/external/nose/plugins/builtin.py @@ -0,0 +1,34 @@ +""" +Lists builtin plugins. +""" +plugins = [] +builtins = ( + ('nose.plugins.attrib', 'AttributeSelector'), + ('nose.plugins.capture', 'Capture'), + ('nose.plugins.logcapture', 'LogCapture'), + ('nose.plugins.cover', 'Coverage'), + ('nose.plugins.debug', 'Pdb'), + ('nose.plugins.deprecated', 'Deprecated'), + ('nose.plugins.doctests', 'Doctest'), + ('nose.plugins.isolate', 'IsolationPlugin'), + ('nose.plugins.failuredetail', 'FailureDetail'), + ('nose.plugins.prof', 'Profile'), + ('nose.plugins.skip', 'Skip'), + ('nose.plugins.testid', 'TestId'), + ('nose.plugins.multiprocess', 'MultiProcess'), + ('nose.plugins.xunit', 'Xunit'), + ('nose.plugins.allmodules', 'AllModules'), + ('nose.plugins.collect', 'CollectOnly'), + ) + +for module, cls in builtins: + try: + plugmod = __import__(module, globals(), locals(), [cls]) + except KeyboardInterrupt: + raise + except: + continue + plug = getattr(plugmod, cls) + plugins.append(plug) + globals()[cls] = plug + diff --git a/lib/spack/external/nose/plugins/capture.py b/lib/spack/external/nose/plugins/capture.py new file mode 100644 index 0000000000..fa4e5dcaaf --- /dev/null +++ b/lib/spack/external/nose/plugins/capture.py @@ -0,0 +1,115 @@ +""" +This plugin captures stdout during test execution. If the test fails +or raises an error, the captured output will be appended to the error +or failure output. It is enabled by default but can be disabled with +the options ``-s`` or ``--nocapture``. + +:Options: + ``--nocapture`` + Don't capture stdout (any stdout output will be printed immediately) + +""" +import logging +import os +import sys +from nose.plugins.base import Plugin +from nose.pyversion import exc_to_unicode, force_unicode +from nose.util import ln +from StringIO import StringIO + + +log = logging.getLogger(__name__) + +class Capture(Plugin): + """ + Output capture plugin. Enabled by default. Disable with ``-s`` or + ``--nocapture``. This plugin captures stdout during test execution, + appending any output captured to the error or failure output, + should the test fail or raise an error. + """ + enabled = True + env_opt = 'NOSE_NOCAPTURE' + name = 'capture' + score = 1600 + + def __init__(self): + self.stdout = [] + self._buf = None + + def options(self, parser, env): + """Register commandline options + """ + parser.add_option( + "-s", "--nocapture", action="store_false", + default=not env.get(self.env_opt), dest="capture", + help="Don't capture stdout (any stdout output " + "will be printed immediately) [NOSE_NOCAPTURE]") + + def configure(self, options, conf): + """Configure plugin. Plugin is enabled by default. + """ + self.conf = conf + if not options.capture: + self.enabled = False + + def afterTest(self, test): + """Clear capture buffer. + """ + self.end() + self._buf = None + + def begin(self): + """Replace sys.stdout with capture buffer. + """ + self.start() # get an early handle on sys.stdout + + def beforeTest(self, test): + """Flush capture buffer. + """ + self.start() + + def formatError(self, test, err): + """Add captured output to error report. + """ + test.capturedOutput = output = self.buffer + self._buf = None + if not output: + # Don't return None as that will prevent other + # formatters from formatting and remove earlier formatters + # formats, instead return the err we got + return err + ec, ev, tb = err + return (ec, self.addCaptureToErr(ev, output), tb) + + def formatFailure(self, test, err): + """Add captured output to failure report. + """ + return self.formatError(test, err) + + def addCaptureToErr(self, ev, output): + ev = exc_to_unicode(ev) + output = force_unicode(output) + return u'\n'.join([ev, ln(u'>> begin captured stdout <<'), + output, ln(u'>> end captured stdout <<')]) + + def start(self): + self.stdout.append(sys.stdout) + self._buf = StringIO() + sys.stdout = self._buf + + def end(self): + if self.stdout: + sys.stdout = self.stdout.pop() + + def finalize(self, result): + """Restore stdout. + """ + while self.stdout: + self.end() + + def _get_buffer(self): + if self._buf is not None: + return self._buf.getvalue() + + buffer = property(_get_buffer, None, None, + """Captured stdout output.""") diff --git a/lib/spack/external/nose/plugins/collect.py b/lib/spack/external/nose/plugins/collect.py new file mode 100644 index 0000000000..6f9f0faa77 --- /dev/null +++ b/lib/spack/external/nose/plugins/collect.py @@ -0,0 +1,94 @@ +""" +This plugin bypasses the actual execution of tests, and instead just collects +test names. Fixtures are also bypassed, so running nosetests with the +collection plugin enabled should be very quick. + +This plugin is useful in combination with the testid plugin (``--with-id``). +Run both together to get an indexed list of all tests, which will enable you to +run individual tests by index number. + +This plugin is also useful for counting tests in a test suite, and making +people watching your demo think all of your tests pass. +""" +from nose.plugins.base import Plugin +from nose.case import Test +import logging +import unittest + +log = logging.getLogger(__name__) + + +class CollectOnly(Plugin): + """ + Collect and output test names only, don't run any tests. + """ + name = "collect-only" + enableOpt = 'collect_only' + + def options(self, parser, env): + """Register commandline options. + """ + parser.add_option('--collect-only', + action='store_true', + dest=self.enableOpt, + default=env.get('NOSE_COLLECT_ONLY'), + help="Enable collect-only: %s [COLLECT_ONLY]" % + (self.help())) + + def prepareTestLoader(self, loader): + """Install collect-only suite class in TestLoader. + """ + # Disable context awareness + log.debug("Preparing test loader") + loader.suiteClass = TestSuiteFactory(self.conf) + + def prepareTestCase(self, test): + """Replace actual test with dummy that always passes. + """ + # Return something that always passes + log.debug("Preparing test case %s", test) + if not isinstance(test, Test): + return + def run(result): + # We need to make these plugin calls because there won't be + # a result proxy, due to using a stripped-down test suite + self.conf.plugins.startTest(test) + result.startTest(test) + self.conf.plugins.addSuccess(test) + result.addSuccess(test) + self.conf.plugins.stopTest(test) + result.stopTest(test) + return run + + +class TestSuiteFactory: + """ + Factory for producing configured test suites. + """ + def __init__(self, conf): + self.conf = conf + + def __call__(self, tests=(), **kw): + return TestSuite(tests, conf=self.conf) + + +class TestSuite(unittest.TestSuite): + """ + Basic test suite that bypasses most proxy and plugin calls, but does + wrap tests in a nose.case.Test so prepareTestCase will be called. + """ + def __init__(self, tests=(), conf=None): + self.conf = conf + # Exec lazy suites: makes discovery depth-first + if callable(tests): + tests = tests() + log.debug("TestSuite(%r)", tests) + unittest.TestSuite.__init__(self, tests) + + def addTest(self, test): + log.debug("Add test %s", test) + if isinstance(test, unittest.TestSuite): + self._tests.append(test) + else: + self._tests.append(Test(test, config=self.conf)) + diff --git a/lib/spack/external/nose/plugins/cover.py b/lib/spack/external/nose/plugins/cover.py new file mode 100644 index 0000000000..fbe2e30dcd --- /dev/null +++ b/lib/spack/external/nose/plugins/cover.py @@ -0,0 +1,271 @@ +"""If you have Ned Batchelder's coverage_ module installed, you may activate a +coverage report with the ``--with-coverage`` switch or NOSE_WITH_COVERAGE +environment variable. The coverage report will cover any python source module +imported after the start of the test run, excluding modules that match +testMatch. If you want to include those modules too, use the ``--cover-tests`` +switch, or set the NOSE_COVER_TESTS environment variable to a true value. To +restrict the coverage report to modules from a particular package or packages, +use the ``--cover-package`` switch or the NOSE_COVER_PACKAGE environment +variable. + +.. _coverage: http://www.nedbatchelder.com/code/modules/coverage.html +""" +import logging +import re +import sys +import StringIO +from nose.plugins.base import Plugin +from nose.util import src, tolist + +log = logging.getLogger(__name__) + + +class Coverage(Plugin): + """ + Activate a coverage report using Ned Batchelder's coverage module. + """ + coverTests = False + coverPackages = None + coverInstance = None + coverErase = False + coverMinPercentage = None + score = 200 + status = {} + + def options(self, parser, env): + """ + Add options to command line. + """ + super(Coverage, self).options(parser, env) + parser.add_option("--cover-package", action="append", + default=env.get('NOSE_COVER_PACKAGE'), + metavar="PACKAGE", + dest="cover_packages", + help="Restrict coverage output to selected packages " + "[NOSE_COVER_PACKAGE]") + parser.add_option("--cover-erase", action="store_true", + default=env.get('NOSE_COVER_ERASE'), + dest="cover_erase", + help="Erase previously collected coverage " + "statistics before run") + parser.add_option("--cover-tests", action="store_true", + dest="cover_tests", + default=env.get('NOSE_COVER_TESTS'), + help="Include test modules in coverage report " + "[NOSE_COVER_TESTS]") + parser.add_option("--cover-min-percentage", action="store", + dest="cover_min_percentage", + default=env.get('NOSE_COVER_MIN_PERCENTAGE'), + help="Minimum percentage of coverage for tests " + "to pass [NOSE_COVER_MIN_PERCENTAGE]") + parser.add_option("--cover-inclusive", action="store_true", + dest="cover_inclusive", + default=env.get('NOSE_COVER_INCLUSIVE'), + help="Include all python files under working " + "directory in coverage report. Useful for " + "discovering holes in test coverage if not all " + "files are imported by the test suite. " + "[NOSE_COVER_INCLUSIVE]") + parser.add_option("--cover-html", action="store_true", + default=env.get('NOSE_COVER_HTML'), + dest='cover_html', + help="Produce HTML coverage information") + parser.add_option('--cover-html-dir', action='store', + default=env.get('NOSE_COVER_HTML_DIR', 'cover'), + dest='cover_html_dir', + metavar='DIR', + help='Produce HTML coverage information in dir') + parser.add_option("--cover-branches", action="store_true", + default=env.get('NOSE_COVER_BRANCHES'), + dest="cover_branches", + help="Include branch coverage in coverage report " + "[NOSE_COVER_BRANCHES]") + parser.add_option("--cover-xml", action="store_true", + default=env.get('NOSE_COVER_XML'), + dest="cover_xml", + help="Produce XML coverage information") + parser.add_option("--cover-xml-file", action="store", + default=env.get('NOSE_COVER_XML_FILE', 'coverage.xml'), + dest="cover_xml_file", + metavar="FILE", + help="Produce XML coverage information in file") + + def configure(self, options, conf): + """ + Configure plugin. + """ + try: + self.status.pop('active') + except KeyError: + pass + super(Coverage, self).configure(options, conf) + if self.enabled: + try: + import coverage + if not hasattr(coverage, 'coverage'): + raise ImportError("Unable to import coverage module") + except ImportError: + log.error("Coverage not available: " + "unable to import coverage module") + self.enabled = False + return + self.conf = conf + self.coverErase = options.cover_erase + self.coverTests = options.cover_tests + self.coverPackages = [] + if options.cover_packages: + if isinstance(options.cover_packages, (list, tuple)): + cover_packages = options.cover_packages + else: + cover_packages = [options.cover_packages] + for pkgs in [tolist(x) for x in cover_packages]: + self.coverPackages.extend(pkgs) + self.coverInclusive = options.cover_inclusive + if self.coverPackages: + log.info("Coverage report will include only packages: %s", + self.coverPackages) + self.coverHtmlDir = None + if options.cover_html: + self.coverHtmlDir = options.cover_html_dir + log.debug('Will put HTML coverage report in %s', self.coverHtmlDir) + self.coverBranches = options.cover_branches + self.coverXmlFile = None + if options.cover_min_percentage: + self.coverMinPercentage = int(options.cover_min_percentage.rstrip('%')) + if options.cover_xml: + self.coverXmlFile = options.cover_xml_file + log.debug('Will put XML coverage report in %s', self.coverXmlFile) + if self.enabled: + self.status['active'] = True + self.coverInstance = coverage.coverage(auto_data=False, + branch=self.coverBranches, data_suffix=conf.worker, + source=self.coverPackages) + self.coverInstance._warn_no_data = False + self.coverInstance.is_worker = conf.worker + self.coverInstance.exclude('#pragma[: ]+[nN][oO] [cC][oO][vV][eE][rR]') + + log.debug("Coverage begin") + self.skipModules = sys.modules.keys()[:] + if self.coverErase: + log.debug("Clearing previously collected coverage statistics") + self.coverInstance.combine() + self.coverInstance.erase() + + if not self.coverInstance.is_worker: + self.coverInstance.load() + self.coverInstance.start() + + + def beforeTest(self, *args, **kwargs): + """ + Begin recording coverage information. + """ + + if self.coverInstance.is_worker: + self.coverInstance.load() + self.coverInstance.start() + + def afterTest(self, *args, **kwargs): + """ + Stop recording coverage information. + """ + + if self.coverInstance.is_worker: + self.coverInstance.stop() + self.coverInstance.save() + + + def report(self, stream): + """ + Output code coverage report. + """ + log.debug("Coverage report") + self.coverInstance.stop() + self.coverInstance.combine() + self.coverInstance.save() + modules = [module + for name, module in sys.modules.items() + if self.wantModuleCoverage(name, module)] + log.debug("Coverage report will cover modules: %s", modules) + self.coverInstance.report(modules, file=stream) + + import coverage + if self.coverHtmlDir: + log.debug("Generating HTML coverage report") + try: + self.coverInstance.html_report(modules, self.coverHtmlDir) + except coverage.misc.CoverageException, e: + log.warning("Failed to generate HTML report: %s" % str(e)) + + if self.coverXmlFile: + log.debug("Generating XML coverage report") + try: + self.coverInstance.xml_report(modules, self.coverXmlFile) + except coverage.misc.CoverageException, e: + log.warning("Failed to generate XML report: %s" % str(e)) + + # make sure we have minimum required coverage + if self.coverMinPercentage: + f = StringIO.StringIO() + self.coverInstance.report(modules, file=f) + + multiPackageRe = (r'-------\s\w+\s+\d+\s+\d+(?:\s+\d+\s+\d+)?' + r'\s+(\d+)%\s+\d*\s{0,1}$') + singlePackageRe = (r'-------\s[\w./]+\s+\d+\s+\d+(?:\s+\d+\s+\d+)?' + r'\s+(\d+)%(?:\s+[-\d, ]+)\s{0,1}$') + + m = re.search(multiPackageRe, f.getvalue()) + if m is None: + m = re.search(singlePackageRe, f.getvalue()) + + if m: + percentage = int(m.groups()[0]) + if percentage < self.coverMinPercentage: + log.error('TOTAL Coverage did not reach minimum ' + 'required: %d%%' % self.coverMinPercentage) + sys.exit(1) + else: + log.error("No total percentage was found in coverage output, " + "something went wrong.") + + + def wantModuleCoverage(self, name, module): + if not hasattr(module, '__file__'): + log.debug("no coverage of %s: no __file__", name) + return False + module_file = src(module.__file__) + if not module_file or not module_file.endswith('.py'): + log.debug("no coverage of %s: not a python file", name) + return False + if self.coverPackages: + for package in self.coverPackages: + if (re.findall(r'^%s\b' % re.escape(package), name) + and (self.coverTests + or not self.conf.testMatch.search(name))): + log.debug("coverage for %s", name) + return True + if name in self.skipModules: + log.debug("no coverage for %s: loaded before coverage start", + name) + return False + if self.conf.testMatch.search(name) and not self.coverTests: + log.debug("no coverage for %s: is a test", name) + return False + # accept any package that passed the previous tests, unless + # coverPackages is on -- in that case, if we wanted this + # module, we would have already returned True + return not self.coverPackages + + def wantFile(self, file, package=None): + """If inclusive coverage enabled, return true for all source files + in wanted packages. + """ + if self.coverInclusive: + if file.endswith(".py"): + if package and self.coverPackages: + for want in self.coverPackages: + if package.startswith(want): + return True + else: + return True + return None diff --git a/lib/spack/external/nose/plugins/debug.py b/lib/spack/external/nose/plugins/debug.py new file mode 100644 index 0000000000..78243e60d0 --- /dev/null +++ b/lib/spack/external/nose/plugins/debug.py @@ -0,0 +1,67 @@ +""" +This plugin provides ``--pdb`` and ``--pdb-failures`` options. The ``--pdb`` +option will drop the test runner into pdb when it encounters an error. To +drop into pdb on failure, use ``--pdb-failures``. +""" + +import pdb +from nose.plugins.base import Plugin + +class Pdb(Plugin): + """ + Provides --pdb and --pdb-failures options that cause the test runner to + drop into pdb if it encounters an error or failure, respectively. + """ + enabled_for_errors = False + enabled_for_failures = False + score = 5 # run last, among builtins + + def options(self, parser, env): + """Register commandline options. + """ + parser.add_option( + "--pdb", action="store_true", dest="debugBoth", + default=env.get('NOSE_PDB', False), + help="Drop into debugger on failures or errors") + parser.add_option( + "--pdb-failures", action="store_true", + dest="debugFailures", + default=env.get('NOSE_PDB_FAILURES', False), + help="Drop into debugger on failures") + parser.add_option( + "--pdb-errors", action="store_true", + dest="debugErrors", + default=env.get('NOSE_PDB_ERRORS', False), + help="Drop into debugger on errors") + + def configure(self, options, conf): + """Configure which kinds of exceptions trigger plugin. + """ + self.conf = conf + self.enabled_for_errors = options.debugErrors or options.debugBoth + self.enabled_for_failures = options.debugFailures or options.debugBoth + self.enabled = self.enabled_for_failures or self.enabled_for_errors + + def addError(self, test, err): + """Enter pdb if configured to debug errors. + """ + if not self.enabled_for_errors: + return + self.debug(err) + + def addFailure(self, test, err): + """Enter pdb if configured to debug failures. + """ + if not self.enabled_for_failures: + return + self.debug(err) + + def debug(self, err): + import sys # FIXME why is this import here? + ec, ev, tb = err + stdout = sys.stdout + sys.stdout = sys.__stdout__ + try: + pdb.post_mortem(tb) + finally: + sys.stdout = stdout diff --git a/lib/spack/external/nose/plugins/deprecated.py b/lib/spack/external/nose/plugins/deprecated.py new file mode 100644 index 0000000000..461a26be63 --- /dev/null +++ b/lib/spack/external/nose/plugins/deprecated.py @@ -0,0 +1,45 @@ +""" +This plugin installs a DEPRECATED error class for the :class:`DeprecatedTest` +exception. When :class:`DeprecatedTest` is raised, the exception will be logged +in the deprecated attribute of the result, ``D`` or ``DEPRECATED`` (verbose) +will be output, and the exception will not be counted as an error or failure. +It is enabled by default, but can be turned off by using ``--no-deprecated``. +""" + +from nose.plugins.errorclass import ErrorClass, ErrorClassPlugin + + +class DeprecatedTest(Exception): + """Raise this exception to mark a test as deprecated. + """ + pass + + +class Deprecated(ErrorClassPlugin): + """ + Installs a DEPRECATED error class for the DeprecatedTest exception. Enabled + by default. + """ + enabled = True + deprecated = ErrorClass(DeprecatedTest, + label='DEPRECATED', + isfailure=False) + + def options(self, parser, env): + """Register commandline options. + """ + env_opt = 'NOSE_WITHOUT_DEPRECATED' + parser.add_option('--no-deprecated', action='store_true', + dest='noDeprecated', default=env.get(env_opt, False), + help="Disable special handling of DeprecatedTest " + "exceptions.") + + def configure(self, options, conf): + """Configure plugin. + """ + if not self.can_configure: + return + self.conf = conf + disable = getattr(options, 'noDeprecated', False) + if disable: + self.enabled = False diff --git a/lib/spack/external/nose/plugins/doctests.py b/lib/spack/external/nose/plugins/doctests.py new file mode 100644 index 0000000000..5ef65799f3 --- /dev/null +++ b/lib/spack/external/nose/plugins/doctests.py @@ -0,0 +1,455 @@ +"""Use the Doctest plugin with ``--with-doctest`` or the NOSE_WITH_DOCTEST +environment variable to enable collection and execution of :mod:`doctests +`. Because doctests are usually included in the tested package +(instead of being grouped into packages or modules of their own), nose only +looks for them in the non-test packages it discovers in the working directory. + +Doctests may also be placed into files other than python modules, in which +case they can be collected and executed by using the ``--doctest-extension`` +switch or NOSE_DOCTEST_EXTENSION environment variable to indicate which file +extension(s) to load. + +When loading doctests from non-module files, use the ``--doctest-fixtures`` +switch to specify how to find modules containing fixtures for the tests. A +module name will be produced by appending the value of that switch to the base +name of each doctest file loaded. For example, a doctest file "widgets.rst" +with the switch ``--doctest_fixtures=_fixt`` will load fixtures from the module +``widgets_fixt.py``. + +A fixtures module may define any or all of the following functions: + +* setup([module]) or setup_module([module]) + + Called before the test runs. You may raise SkipTest to skip all tests. + +* teardown([module]) or teardown_module([module]) + + Called after the test runs, if setup/setup_module did not raise an + unhandled exception. + +* setup_test(test) + + Called before the test. NOTE: the argument passed is a + doctest.DocTest instance, *not* a unittest.TestCase. + +* teardown_test(test) + + Called after the test, if setup_test did not raise an exception. NOTE: the + argument passed is a doctest.DocTest instance, *not* a unittest.TestCase. + +Doctests are run like any other test, with the exception that output +capture does not work; doctest does its own output capture while running a +test. + +.. note :: + + See :doc:`../doc_tests/test_doctest_fixtures/doctest_fixtures` for + additional documentation and examples. + +""" +from __future__ import generators + +import logging +import os +import sys +import unittest +from inspect import getmodule +from nose.plugins.base import Plugin +from nose.suite import ContextList +from nose.util import anyp, getpackage, test_address, resolve_name, \ + src, tolist, isproperty +try: + from cStringIO import StringIO +except ImportError: + from StringIO import StringIO +import sys +import __builtin__ as builtin_mod + +log = logging.getLogger(__name__) + +try: + import doctest + doctest.DocTestCase + # system version of doctest is acceptable, but needs a monkeypatch +except (ImportError, AttributeError): + # system version is too old + import nose.ext.dtcompat as doctest + + +# +# Doctest and coverage don't get along, so we need to create +# a monkeypatch that will replace the part of doctest that +# interferes with coverage reports. +# +# The monkeypatch is based on this zope patch: +# http://svn.zope.org/Zope3/trunk/src/zope/testing/doctest.py?rev=28679&r1=28703&r2=28705 +# +_orp = doctest._OutputRedirectingPdb + +class NoseOutputRedirectingPdb(_orp): + def __init__(self, out): + self.__debugger_used = False + _orp.__init__(self, out) + + def set_trace(self): + self.__debugger_used = True + _orp.set_trace(self, sys._getframe().f_back) + + def set_continue(self): + # Calling set_continue unconditionally would break unit test + # coverage reporting, as Bdb.set_continue calls sys.settrace(None). + if self.__debugger_used: + _orp.set_continue(self) +doctest._OutputRedirectingPdb = NoseOutputRedirectingPdb + + +class DoctestSuite(unittest.TestSuite): + """ + Doctest suites are parallelizable at the module or file level only, + since they may be attached to objects that are not individually + addressable (like properties). This suite subclass is used when + loading doctests from a module to ensure that behavior. + + This class is used only if the plugin is not fully prepared; + in normal use, the loader's suiteClass is used. + + """ + can_split = False + + def __init__(self, tests=(), context=None, can_split=False): + self.context = context + self.can_split = can_split + unittest.TestSuite.__init__(self, tests=tests) + + def address(self): + return test_address(self.context) + + def __iter__(self): + # 2.3 compat + return iter(self._tests) + + def __str__(self): + return str(self._tests) + + +class Doctest(Plugin): + """ + Activate doctest plugin to find and run doctests in non-test modules. + """ + extension = None + suiteClass = DoctestSuite + + def options(self, parser, env): + """Register commmandline options. + """ + Plugin.options(self, parser, env) + parser.add_option('--doctest-tests', action='store_true', + dest='doctest_tests', + default=env.get('NOSE_DOCTEST_TESTS'), + help="Also look for doctests in test modules. " + "Note that classes, methods and functions should " + "have either doctests or non-doctest tests, " + "not both. [NOSE_DOCTEST_TESTS]") + parser.add_option('--doctest-extension', action="append", + dest="doctestExtension", + metavar="EXT", + help="Also look for doctests in files with " + "this extension [NOSE_DOCTEST_EXTENSION]") + parser.add_option('--doctest-result-variable', + dest='doctest_result_var', + default=env.get('NOSE_DOCTEST_RESULT_VAR'), + metavar="VAR", + help="Change the variable name set to the result of " + "the last interpreter command from the default '_'. " + "Can be used to avoid conflicts with the _() " + "function used for text translation. " + "[NOSE_DOCTEST_RESULT_VAR]") + parser.add_option('--doctest-fixtures', action="store", + dest="doctestFixtures", + metavar="SUFFIX", + help="Find fixtures for a doctest file in module " + "with this name appended to the base name " + "of the doctest file") + parser.add_option('--doctest-options', action="append", + dest="doctestOptions", + metavar="OPTIONS", + help="Specify options to pass to doctest. " + + "Eg. '+ELLIPSIS,+NORMALIZE_WHITESPACE'") + # Set the default as a list, if given in env; otherwise + # an additional value set on the command line will cause + # an error. + env_setting = env.get('NOSE_DOCTEST_EXTENSION') + if env_setting is not None: + parser.set_defaults(doctestExtension=tolist(env_setting)) + + def configure(self, options, config): + """Configure plugin. + """ + Plugin.configure(self, options, config) + self.doctest_result_var = options.doctest_result_var + self.doctest_tests = options.doctest_tests + self.extension = tolist(options.doctestExtension) + self.fixtures = options.doctestFixtures + self.finder = doctest.DocTestFinder() + self.optionflags = 0 + if options.doctestOptions: + flags = ",".join(options.doctestOptions).split(',') + for flag in flags: + if not flag or flag[0] not in '+-': + raise ValueError( + "Must specify doctest options with starting " + + "'+' or '-'. Got %s" % (flag,)) + mode, option_name = flag[0], flag[1:] + option_flag = doctest.OPTIONFLAGS_BY_NAME.get(option_name) + if not option_flag: + raise ValueError("Unknown doctest option %s" % + (option_name,)) + if mode == '+': + self.optionflags |= option_flag + elif mode == '-': + self.optionflags &= ~option_flag + + def prepareTestLoader(self, loader): + """Capture loader's suiteClass. + + This is used to create test suites from doctest files. + + """ + self.suiteClass = loader.suiteClass + + def loadTestsFromModule(self, module): + """Load doctests from the module. + """ + log.debug("loading from %s", module) + if not self.matches(module.__name__): + log.debug("Doctest doesn't want module %s", module) + return + try: + tests = self.finder.find(module) + except AttributeError: + log.exception("Attribute error loading from %s", module) + # nose allows module.__test__ = False; doctest does not and throws + # AttributeError + return + if not tests: + log.debug("No tests found in %s", module) + return + tests.sort() + module_file = src(module.__file__) + # FIXME this breaks the id plugin somehow (tests probably don't + # get wrapped in result proxy or something) + cases = [] + for test in tests: + if not test.examples: + continue + if not test.filename: + test.filename = module_file + cases.append(DocTestCase(test, + optionflags=self.optionflags, + result_var=self.doctest_result_var)) + if cases: + yield self.suiteClass(cases, context=module, can_split=False) + + def loadTestsFromFile(self, filename): + """Load doctests from the file. + + Tests are loaded only if filename's extension matches + configured doctest extension. + + """ + if self.extension and anyp(filename.endswith, self.extension): + name = os.path.basename(filename) + dh = open(filename) + try: + doc = dh.read() + finally: + dh.close() + + fixture_context = None + globs = {'__file__': filename} + if self.fixtures: + base, ext = os.path.splitext(name) + dirname = os.path.dirname(filename) + sys.path.append(dirname) + fixt_mod = base + self.fixtures + try: + fixture_context = __import__( + fixt_mod, globals(), locals(), ["nop"]) + except ImportError, e: + log.debug( + "Could not import %s: %s (%s)", fixt_mod, e, sys.path) + log.debug("Fixture module %s resolved to %s", + fixt_mod, fixture_context) + if hasattr(fixture_context, 'globs'): + globs = fixture_context.globs(globs) + parser = doctest.DocTestParser() + test = parser.get_doctest( + doc, globs=globs, name=name, + filename=filename, lineno=0) + if test.examples: + case = DocFileCase( + test, + optionflags=self.optionflags, + setUp=getattr(fixture_context, 'setup_test', None), + tearDown=getattr(fixture_context, 'teardown_test', None), + result_var=self.doctest_result_var) + if fixture_context: + yield ContextList((case,), context=fixture_context) + else: + yield case + else: + yield False # no tests to load + + def makeTest(self, obj, parent): + """Look for doctests in the given object, which will be a + function, method or class. + """ + name = getattr(obj, '__name__', 'Unnammed %s' % type(obj)) + doctests = self.finder.find(obj, module=getmodule(parent), name=name) + if doctests: + for test in doctests: + if len(test.examples) == 0: + continue + yield DocTestCase(test, obj=obj, optionflags=self.optionflags, + result_var=self.doctest_result_var) + + def matches(self, name): + # FIXME this seems wrong -- nothing is ever going to + # fail this test, since we're given a module NAME not FILE + if name == '__init__.py': + return False + # FIXME don't think we need include/exclude checks here? + return ((self.doctest_tests or not self.conf.testMatch.search(name) + or (self.conf.include + and filter(None, + [inc.search(name) + for inc in self.conf.include]))) + and (not self.conf.exclude + or not filter(None, + [exc.search(name) + for exc in self.conf.exclude]))) + + def wantFile(self, file): + """Override to select all modules and any file ending with + configured doctest extension. + """ + # always want .py files + if file.endswith('.py'): + return True + # also want files that match my extension + if (self.extension + and anyp(file.endswith, self.extension) + and (not self.conf.exclude + or not filter(None, + [exc.search(file) + for exc in self.conf.exclude]))): + return True + return None + + +class DocTestCase(doctest.DocTestCase): + """Overrides DocTestCase to + provide an address() method that returns the correct address for + the doctest case. To provide hints for address(), an obj may also + be passed -- this will be used as the test object for purposes of + determining the test address, if it is provided. + """ + def __init__(self, test, optionflags=0, setUp=None, tearDown=None, + checker=None, obj=None, result_var='_'): + self._result_var = result_var + self._nose_obj = obj + super(DocTestCase, self).__init__( + test, optionflags=optionflags, setUp=setUp, tearDown=tearDown, + checker=checker) + + def address(self): + if self._nose_obj is not None: + return test_address(self._nose_obj) + obj = resolve_name(self._dt_test.name) + + if isproperty(obj): + # properties have no connection to the class they are in + # so we can't just look 'em up, we have to first look up + # the class, then stick the prop on the end + parts = self._dt_test.name.split('.') + class_name = '.'.join(parts[:-1]) + cls = resolve_name(class_name) + base_addr = test_address(cls) + return (base_addr[0], base_addr[1], + '.'.join([base_addr[2], parts[-1]])) + else: + return test_address(obj) + + # doctests loaded via find(obj) omit the module name + # so we need to override id, __repr__ and shortDescription + # bonus: this will squash a 2.3 vs 2.4 incompatiblity + def id(self): + name = self._dt_test.name + filename = self._dt_test.filename + if filename is not None: + pk = getpackage(filename) + if pk is None: + return name + if not name.startswith(pk): + name = "%s.%s" % (pk, name) + return name + + def __repr__(self): + name = self.id() + name = name.split('.') + return "%s (%s)" % (name[-1], '.'.join(name[:-1])) + __str__ = __repr__ + + def shortDescription(self): + return 'Doctest: %s' % self.id() + + def setUp(self): + if self._result_var is not None: + self._old_displayhook = sys.displayhook + sys.displayhook = self._displayhook + super(DocTestCase, self).setUp() + + def _displayhook(self, value): + if value is None: + return + setattr(builtin_mod, self._result_var, value) + print repr(value) + + def tearDown(self): + super(DocTestCase, self).tearDown() + if self._result_var is not None: + sys.displayhook = self._old_displayhook + delattr(builtin_mod, self._result_var) + + +class DocFileCase(doctest.DocFileCase): + """Overrides to provide address() method that returns the correct + address for the doc file case. + """ + def __init__(self, test, optionflags=0, setUp=None, tearDown=None, + checker=None, result_var='_'): + self._result_var = result_var + super(DocFileCase, self).__init__( + test, optionflags=optionflags, setUp=setUp, tearDown=tearDown, + checker=None) + + def address(self): + return (self._dt_test.filename, None, None) + + def setUp(self): + if self._result_var is not None: + self._old_displayhook = sys.displayhook + sys.displayhook = self._displayhook + super(DocFileCase, self).setUp() + + def _displayhook(self, value): + if value is None: + return + setattr(builtin_mod, self._result_var, value) + print repr(value) + + def tearDown(self): + super(DocFileCase, self).tearDown() + if self._result_var is not None: + sys.displayhook = self._old_displayhook + delattr(builtin_mod, self._result_var) diff --git a/lib/spack/external/nose/plugins/errorclass.py b/lib/spack/external/nose/plugins/errorclass.py new file mode 100644 index 0000000000..d1540e0070 --- /dev/null +++ b/lib/spack/external/nose/plugins/errorclass.py @@ -0,0 +1,210 @@ +""" +ErrorClass Plugins +------------------ + +ErrorClass plugins provide an easy way to add support for custom +handling of particular classes of exceptions. + +An ErrorClass plugin defines one or more ErrorClasses and how each is +handled and reported on. Each error class is stored in a different +attribute on the result, and reported separately. Each error class must +indicate the exceptions that fall under that class, the label to use +for reporting, and whether exceptions of the class should be +considered as failures for the whole test run. + +ErrorClasses use a declarative syntax. Assign an ErrorClass to the +attribute you wish to add to the result object, defining the +exceptions, label and isfailure attributes. For example, to declare an +ErrorClassPlugin that defines TodoErrors (and subclasses of TodoError) +as an error class with the label 'TODO' that is considered a failure, +do this: + + >>> class Todo(Exception): + ... pass + >>> class TodoError(ErrorClassPlugin): + ... todo = ErrorClass(Todo, label='TODO', isfailure=True) + +The MetaErrorClass metaclass translates the ErrorClass declarations +into the tuples used by the error handling and reporting functions in +the result. This is an internal format and subject to change; you +should always use the declarative syntax for attaching ErrorClasses to +an ErrorClass plugin. + + >>> TodoError.errorClasses # doctest: +ELLIPSIS + ((, ('todo', 'TODO', True)),) + +Let's see the plugin in action. First some boilerplate. + + >>> import sys + >>> import unittest + >>> try: + ... # 2.7+ + ... from unittest.runner import _WritelnDecorator + ... except ImportError: + ... from unittest import _WritelnDecorator + ... + >>> buf = _WritelnDecorator(sys.stdout) + +Now define a test case that raises a Todo. + + >>> class TestTodo(unittest.TestCase): + ... def runTest(self): + ... raise Todo("I need to test something") + >>> case = TestTodo() + +Prepare the result using our plugin. Normally this happens during the +course of test execution within nose -- you won't be doing this +yourself. For the purposes of this testing document, I'm stepping +through the internal process of nose so you can see what happens at +each step. + + >>> plugin = TodoError() + >>> from nose.result import _TextTestResult + >>> result = _TextTestResult(stream=buf, descriptions=0, verbosity=2) + >>> plugin.prepareTestResult(result) + +Now run the test. TODO is printed. + + >>> _ = case(result) # doctest: +ELLIPSIS + runTest (....TestTodo) ... TODO: I need to test something + +Errors and failures are empty, but todo has our test: + + >>> result.errors + [] + >>> result.failures + [] + >>> result.todo # doctest: +ELLIPSIS + [(<....TestTodo testMethod=runTest>, '...Todo: I need to test something\\n')] + >>> result.printErrors() # doctest: +ELLIPSIS + + ====================================================================== + TODO: runTest (....TestTodo) + ---------------------------------------------------------------------- + Traceback (most recent call last): + ... + ...Todo: I need to test something + + +Since we defined a Todo as a failure, the run was not successful. + + >>> result.wasSuccessful() + False +""" + +from nose.pyversion import make_instancemethod +from nose.plugins.base import Plugin +from nose.result import TextTestResult +from nose.util import isclass + +class MetaErrorClass(type): + """Metaclass for ErrorClassPlugins that allows error classes to be + set up in a declarative manner. + """ + def __init__(self, name, bases, attr): + errorClasses = [] + for name, detail in attr.items(): + if isinstance(detail, ErrorClass): + attr.pop(name) + for cls in detail: + errorClasses.append( + (cls, (name, detail.label, detail.isfailure))) + super(MetaErrorClass, self).__init__(name, bases, attr) + self.errorClasses = tuple(errorClasses) + + +class ErrorClass(object): + def __init__(self, *errorClasses, **kw): + self.errorClasses = errorClasses + try: + for key in ('label', 'isfailure'): + setattr(self, key, kw.pop(key)) + except KeyError: + raise TypeError("%r is a required named argument for ErrorClass" + % key) + + def __iter__(self): + return iter(self.errorClasses) + + +class ErrorClassPlugin(Plugin): + """ + Base class for ErrorClass plugins. Subclass this class and declare the + exceptions that you wish to handle as attributes of the subclass. + """ + __metaclass__ = MetaErrorClass + score = 1000 + errorClasses = () + + def addError(self, test, err): + err_cls, a, b = err + if not isclass(err_cls): + return + classes = [e[0] for e in self.errorClasses] + if filter(lambda c: issubclass(err_cls, c), classes): + return True + + def prepareTestResult(self, result): + if not hasattr(result, 'errorClasses'): + self.patchResult(result) + for cls, (storage_attr, label, isfail) in self.errorClasses: + if cls not in result.errorClasses: + storage = getattr(result, storage_attr, []) + setattr(result, storage_attr, storage) + result.errorClasses[cls] = (storage, label, isfail) + + def patchResult(self, result): + result.printLabel = print_label_patch(result) + result._orig_addError, result.addError = \ + result.addError, add_error_patch(result) + result._orig_wasSuccessful, result.wasSuccessful = \ + result.wasSuccessful, wassuccessful_patch(result) + if hasattr(result, 'printErrors'): + result._orig_printErrors, result.printErrors = \ + result.printErrors, print_errors_patch(result) + if hasattr(result, 'addSkip'): + result._orig_addSkip, result.addSkip = \ + result.addSkip, add_skip_patch(result) + result.errorClasses = {} + + +def add_error_patch(result): + """Create a new addError method to patch into a result instance + that recognizes the errorClasses attribute and deals with + errorclasses correctly. + """ + return make_instancemethod(TextTestResult.addError, result) + + +def print_errors_patch(result): + """Create a new printErrors method that prints errorClasses items + as well. + """ + return make_instancemethod(TextTestResult.printErrors, result) + + +def print_label_patch(result): + """Create a new printLabel method that prints errorClasses items + as well. + """ + return make_instancemethod(TextTestResult.printLabel, result) + + +def wassuccessful_patch(result): + """Create a new wasSuccessful method that checks errorClasses for + exceptions that were put into other slots than error or failure + but that still count as not success. + """ + return make_instancemethod(TextTestResult.wasSuccessful, result) + + +def add_skip_patch(result): + """Create a new addSkip method to patch into a result instance + that delegates to addError. + """ + return make_instancemethod(TextTestResult.addSkip, result) + + +if __name__ == '__main__': + import doctest + doctest.testmod() diff --git a/lib/spack/external/nose/plugins/failuredetail.py b/lib/spack/external/nose/plugins/failuredetail.py new file mode 100644 index 0000000000..6462865dd0 --- /dev/null +++ b/lib/spack/external/nose/plugins/failuredetail.py @@ -0,0 +1,49 @@ +""" +This plugin provides assert introspection. When the plugin is enabled +and a test failure occurs, the traceback is displayed with extra context +around the line in which the exception was raised. Simple variable +substitution is also performed in the context output to provide more +debugging information. +""" + +from nose.plugins import Plugin +from nose.pyversion import exc_to_unicode, force_unicode +from nose.inspector import inspect_traceback + +class FailureDetail(Plugin): + """ + Plugin that provides extra information in tracebacks of test failures. + """ + score = 1600 # before capture + + def options(self, parser, env): + """Register commmandline options. + """ + parser.add_option( + "-d", "--detailed-errors", "--failure-detail", + action="store_true", + default=env.get('NOSE_DETAILED_ERRORS'), + dest="detailedErrors", help="Add detail to error" + " output by attempting to evaluate failed" + " asserts [NOSE_DETAILED_ERRORS]") + + def configure(self, options, conf): + """Configure plugin. + """ + if not self.can_configure: + return + self.enabled = options.detailedErrors + self.conf = conf + + def formatFailure(self, test, err): + """Add detail from traceback inspection to error message of a failure. + """ + ec, ev, tb = err + tbinfo, str_ev = None, exc_to_unicode(ev) + + if tb: + tbinfo = force_unicode(inspect_traceback(tb)) + str_ev = '\n'.join([str_ev, tbinfo]) + test.tbinfo = tbinfo + return (ec, str_ev, tb) + diff --git a/lib/spack/external/nose/plugins/isolate.py b/lib/spack/external/nose/plugins/isolate.py new file mode 100644 index 0000000000..13235dfbd1 --- /dev/null +++ b/lib/spack/external/nose/plugins/isolate.py @@ -0,0 +1,103 @@ +"""The isolation plugin resets the contents of sys.modules after running +each test module or package. Use it by setting ``--with-isolation`` or the +NOSE_WITH_ISOLATION environment variable. + +The effects are similar to wrapping the following functions around the +import and execution of each test module:: + + def setup(module): + module._mods = sys.modules.copy() + + def teardown(module): + to_del = [ m for m in sys.modules.keys() if m not in + module._mods ] + for mod in to_del: + del sys.modules[mod] + sys.modules.update(module._mods) + +Isolation works only during lazy loading. In normal use, this is only +during discovery of modules within a directory, where the process of +importing, loading tests and running tests from each module is +encapsulated in a single loadTestsFromName call. This plugin +implements loadTestsFromNames to force the same lazy-loading there, +which allows isolation to work in directed mode as well as discovery, +at the cost of some efficiency: lazy-loading names forces full context +setup and teardown to run for each name, defeating the grouping that +is normally used to ensure that context setup and teardown are run the +fewest possible times for a given set of names. + +.. warning :: + + This plugin should not be used in conjunction with other plugins + that assume that modules, once imported, will stay imported; for + instance, it may cause very odd results when used with the coverage + plugin. + +""" + +import logging +import sys + +from nose.plugins import Plugin + + +log = logging.getLogger('nose.plugins.isolation') + +class IsolationPlugin(Plugin): + """ + Activate the isolation plugin to isolate changes to external + modules to a single test module or package. The isolation plugin + resets the contents of sys.modules after each test module or + package runs to its state before the test. PLEASE NOTE that this + plugin should not be used with the coverage plugin, or in any other case + where module reloading may produce undesirable side-effects. + """ + score = 10 # I want to be last + name = 'isolation' + + def configure(self, options, conf): + """Configure plugin. + """ + Plugin.configure(self, options, conf) + self._mod_stack = [] + + def beforeContext(self): + """Copy sys.modules onto my mod stack + """ + mods = sys.modules.copy() + self._mod_stack.append(mods) + + def afterContext(self): + """Pop my mod stack and restore sys.modules to the state + it was in when mod stack was pushed. + """ + mods = self._mod_stack.pop() + to_del = [ m for m in sys.modules.keys() if m not in mods ] + if to_del: + log.debug('removing sys modules entries: %s', to_del) + for mod in to_del: + del sys.modules[mod] + sys.modules.update(mods) + + def loadTestsFromNames(self, names, module=None): + """Create a lazy suite that calls beforeContext and afterContext + around each name. The side-effect of this is that full context + fixtures will be set up and torn down around each test named. + """ + # Fast path for when we don't care + if not names or len(names) == 1: + return + loader = self.loader + plugins = self.conf.plugins + def lazy(): + for name in names: + plugins.beforeContext() + yield loader.loadTestsFromName(name, module=module) + plugins.afterContext() + return (loader.suiteClass(lazy), []) + + def prepareTestLoader(self, loader): + """Get handle on test loader so we can use it in loadTestsFromNames. + """ + self.loader = loader + diff --git a/lib/spack/external/nose/plugins/logcapture.py b/lib/spack/external/nose/plugins/logcapture.py new file mode 100644 index 0000000000..4c9a79f6fd --- /dev/null +++ b/lib/spack/external/nose/plugins/logcapture.py @@ -0,0 +1,245 @@ +""" +This plugin captures logging statements issued during test execution. When an +error or failure occurs, the captured log messages are attached to the running +test in the test.capturedLogging attribute, and displayed with the error failure +output. It is enabled by default but can be turned off with the option +``--nologcapture``. + +You can filter captured logging statements with the ``--logging-filter`` option. +If set, it specifies which logger(s) will be captured; loggers that do not match +will be passed. Example: specifying ``--logging-filter=sqlalchemy,myapp`` +will ensure that only statements logged via sqlalchemy.engine, myapp +or myapp.foo.bar logger will be logged. + +You can remove other installed logging handlers with the +``--logging-clear-handlers`` option. +""" + +import logging +from logging import Handler +import threading + +from nose.plugins.base import Plugin +from nose.util import anyp, ln, safe_str + +try: + from cStringIO import StringIO +except ImportError: + from StringIO import StringIO + +log = logging.getLogger(__name__) + +class FilterSet(object): + def __init__(self, filter_components): + self.inclusive, self.exclusive = self._partition(filter_components) + + # @staticmethod + def _partition(components): + inclusive, exclusive = [], [] + for component in components: + if component.startswith('-'): + exclusive.append(component[1:]) + else: + inclusive.append(component) + return inclusive, exclusive + _partition = staticmethod(_partition) + + def allow(self, record): + """returns whether this record should be printed""" + if not self: + # nothing to filter + return True + return self._allow(record) and not self._deny(record) + + # @staticmethod + def _any_match(matchers, record): + """return the bool of whether `record` starts with + any item in `matchers`""" + def record_matches_key(key): + return record == key or record.startswith(key + '.') + return anyp(bool, map(record_matches_key, matchers)) + _any_match = staticmethod(_any_match) + + def _allow(self, record): + if not self.inclusive: + return True + return self._any_match(self.inclusive, record) + + def _deny(self, record): + if not self.exclusive: + return False + return self._any_match(self.exclusive, record) + + +class MyMemoryHandler(Handler): + def __init__(self, logformat, logdatefmt, filters): + Handler.__init__(self) + fmt = logging.Formatter(logformat, logdatefmt) + self.setFormatter(fmt) + self.filterset = FilterSet(filters) + self.buffer = [] + def emit(self, record): + self.buffer.append(self.format(record)) + def flush(self): + pass # do nothing + def truncate(self): + self.buffer = [] + def filter(self, record): + if self.filterset.allow(record.name): + return Handler.filter(self, record) + def __getstate__(self): + state = self.__dict__.copy() + del state['lock'] + return state + def __setstate__(self, state): + self.__dict__.update(state) + self.lock = threading.RLock() + + +class LogCapture(Plugin): + """ + Log capture plugin. Enabled by default. Disable with --nologcapture. + This plugin captures logging statements issued during test execution, + appending any output captured to the error or failure output, + should the test fail or raise an error. + """ + enabled = True + env_opt = 'NOSE_NOLOGCAPTURE' + name = 'logcapture' + score = 500 + logformat = '%(name)s: %(levelname)s: %(message)s' + logdatefmt = None + clear = False + filters = ['-nose'] + + def options(self, parser, env): + """Register commandline options. + """ + parser.add_option( + "--nologcapture", action="store_false", + default=not env.get(self.env_opt), dest="logcapture", + help="Disable logging capture plugin. " + "Logging configuration will be left intact." + " [NOSE_NOLOGCAPTURE]") + parser.add_option( + "--logging-format", action="store", dest="logcapture_format", + default=env.get('NOSE_LOGFORMAT') or self.logformat, + metavar="FORMAT", + help="Specify custom format to print statements. " + "Uses the same format as used by standard logging handlers." + " [NOSE_LOGFORMAT]") + parser.add_option( + "--logging-datefmt", action="store", dest="logcapture_datefmt", + default=env.get('NOSE_LOGDATEFMT') or self.logdatefmt, + metavar="FORMAT", + help="Specify custom date/time format to print statements. " + "Uses the same format as used by standard logging handlers." + " [NOSE_LOGDATEFMT]") + parser.add_option( + "--logging-filter", action="store", dest="logcapture_filters", + default=env.get('NOSE_LOGFILTER'), + metavar="FILTER", + help="Specify which statements to filter in/out. " + "By default, everything is captured. If the output is too" + " verbose,\nuse this option to filter out needless output.\n" + "Example: filter=foo will capture statements issued ONLY to\n" + " foo or foo.what.ever.sub but not foobar or other logger.\n" + "Specify multiple loggers with comma: filter=foo,bar,baz.\n" + "If any logger name is prefixed with a minus, eg filter=-foo,\n" + "it will be excluded rather than included. Default: " + "exclude logging messages from nose itself (-nose)." + " [NOSE_LOGFILTER]\n") + parser.add_option( + "--logging-clear-handlers", action="store_true", + default=False, dest="logcapture_clear", + help="Clear all other logging handlers") + parser.add_option( + "--logging-level", action="store", + default='NOTSET', dest="logcapture_level", + help="Set the log level to capture") + + def configure(self, options, conf): + """Configure plugin. + """ + self.conf = conf + # Disable if explicitly disabled, or if logging is + # configured via logging config file + if not options.logcapture or conf.loggingConfig: + self.enabled = False + self.logformat = options.logcapture_format + self.logdatefmt = options.logcapture_datefmt + self.clear = options.logcapture_clear + self.loglevel = options.logcapture_level + if options.logcapture_filters: + self.filters = options.logcapture_filters.split(',') + + def setupLoghandler(self): + # setup our handler with root logger + root_logger = logging.getLogger() + if self.clear: + if hasattr(root_logger, "handlers"): + for handler in root_logger.handlers: + root_logger.removeHandler(handler) + for logger in logging.Logger.manager.loggerDict.values(): + if hasattr(logger, "handlers"): + for handler in logger.handlers: + logger.removeHandler(handler) + # make sure there isn't one already + # you can't simply use "if self.handler not in root_logger.handlers" + # since at least in unit tests this doesn't work -- + # LogCapture() is instantiated for each test case while root_logger + # is module global + # so we always add new MyMemoryHandler instance + for handler in root_logger.handlers[:]: + if isinstance(handler, MyMemoryHandler): + root_logger.handlers.remove(handler) + root_logger.addHandler(self.handler) + # to make sure everything gets captured + loglevel = getattr(self, "loglevel", "NOTSET") + root_logger.setLevel(getattr(logging, loglevel)) + + def begin(self): + """Set up logging handler before test run begins. + """ + self.start() + + def start(self): + self.handler = MyMemoryHandler(self.logformat, self.logdatefmt, + self.filters) + self.setupLoghandler() + + def end(self): + pass + + def beforeTest(self, test): + """Clear buffers and handlers before test. + """ + self.setupLoghandler() + + def afterTest(self, test): + """Clear buffers after test. + """ + self.handler.truncate() + + def formatFailure(self, test, err): + """Add captured log messages to failure output. + """ + return self.formatError(test, err) + + def formatError(self, test, err): + """Add captured log messages to error output. + """ + # logic flow copied from Capture.formatError + test.capturedLogging = records = self.formatLogRecords() + if not records: + return err + ec, ev, tb = err + return (ec, self.addCaptureToErr(ev, records), tb) + + def formatLogRecords(self): + return map(safe_str, self.handler.buffer) + + def addCaptureToErr(self, ev, records): + return '\n'.join([safe_str(ev), ln('>> begin captured logging <<')] + \ + records + \ + [ln('>> end captured logging <<')]) diff --git a/lib/spack/external/nose/plugins/manager.py b/lib/spack/external/nose/plugins/manager.py new file mode 100644 index 0000000000..4d2ed22b6f --- /dev/null +++ b/lib/spack/external/nose/plugins/manager.py @@ -0,0 +1,460 @@ +""" +Plugin Manager +-------------- + +A plugin manager class is used to load plugins, manage the list of +loaded plugins, and proxy calls to those plugins. + +The plugin managers provided with nose are: + +:class:`PluginManager` + This manager doesn't implement loadPlugins, so it can only work + with a static list of plugins. + +:class:`BuiltinPluginManager` + This manager loads plugins referenced in ``nose.plugins.builtin``. + +:class:`EntryPointPluginManager` + This manager uses setuptools entrypoints to load plugins. + +:class:`ExtraPluginsPluginManager` + This manager loads extra plugins specified with the keyword + `addplugins`. + +:class:`DefaultPluginMananger` + This is the manager class that will be used by default. If + setuptools is installed, it is a subclass of + :class:`EntryPointPluginManager` and :class:`BuiltinPluginManager`; + otherwise, an alias to :class:`BuiltinPluginManager`. + +:class:`RestrictedPluginManager` + This manager is for use in test runs where some plugin calls are + not available, such as runs started with ``python setup.py test``, + where the test runner is the default unittest :class:`TextTestRunner`. It + is a subclass of :class:`DefaultPluginManager`. + +Writing a plugin manager +======================== + +If you want to load plugins via some other means, you can write a +plugin manager and pass an instance of your plugin manager class when +instantiating the :class:`nose.config.Config` instance that you pass to +:class:`TestProgram` (or :func:`main` or :func:`run`). + +To implement your plugin loading scheme, implement ``loadPlugins()``, +and in that method, call ``addPlugin()`` with an instance of each plugin +you wish to make available. Make sure to call +``super(self).loadPlugins()`` as well if have subclassed a manager +other than ``PluginManager``. + +""" +import inspect +import logging +import os +import sys +from itertools import chain as iterchain +from warnings import warn +import nose.config +from nose.failure import Failure +from nose.plugins.base import IPluginInterface +from nose.pyversion import sort_list + +try: + import cPickle as pickle +except: + import pickle +try: + from cStringIO import StringIO +except: + from StringIO import StringIO + + +__all__ = ['DefaultPluginManager', 'PluginManager', 'EntryPointPluginManager', + 'BuiltinPluginManager', 'RestrictedPluginManager'] + +log = logging.getLogger(__name__) + + +class PluginProxy(object): + """Proxy for plugin calls. Essentially a closure bound to the + given call and plugin list. + + The plugin proxy also must be bound to a particular plugin + interface specification, so that it knows what calls are available + and any special handling that is required for each call. + """ + interface = IPluginInterface + def __init__(self, call, plugins): + try: + self.method = getattr(self.interface, call) + except AttributeError: + raise AttributeError("%s is not a valid %s method" + % (call, self.interface.__name__)) + self.call = self.makeCall(call) + self.plugins = [] + for p in plugins: + self.addPlugin(p, call) + + def __call__(self, *arg, **kw): + return self.call(*arg, **kw) + + def addPlugin(self, plugin, call): + """Add plugin to my list of plugins to call, if it has the attribute + I'm bound to. + """ + meth = getattr(plugin, call, None) + if meth is not None: + if call == 'loadTestsFromModule' and \ + len(inspect.getargspec(meth)[0]) == 2: + orig_meth = meth + meth = lambda module, path, **kwargs: orig_meth(module) + self.plugins.append((plugin, meth)) + + def makeCall(self, call): + if call == 'loadTestsFromNames': + # special case -- load tests from names behaves somewhat differently + # from other chainable calls, because plugins return a tuple, only + # part of which can be chained to the next plugin. + return self._loadTestsFromNames + + meth = self.method + if getattr(meth, 'generative', False): + # call all plugins and yield a flattened iterator of their results + return lambda *arg, **kw: list(self.generate(*arg, **kw)) + elif getattr(meth, 'chainable', False): + return self.chain + else: + # return a value from the first plugin that returns non-None + return self.simple + + def chain(self, *arg, **kw): + """Call plugins in a chain, where the result of each plugin call is + sent to the next plugin as input. The final output result is returned. + """ + result = None + # extract the static arguments (if any) from arg so they can + # be passed to each plugin call in the chain + static = [a for (static, a) + in zip(getattr(self.method, 'static_args', []), arg) + if static] + for p, meth in self.plugins: + result = meth(*arg, **kw) + arg = static[:] + arg.append(result) + return result + + def generate(self, *arg, **kw): + """Call all plugins, yielding each item in each non-None result. + """ + for p, meth in self.plugins: + result = None + try: + result = meth(*arg, **kw) + if result is not None: + for r in result: + yield r + except (KeyboardInterrupt, SystemExit): + raise + except: + exc = sys.exc_info() + yield Failure(*exc) + continue + + def simple(self, *arg, **kw): + """Call all plugins, returning the first non-None result. + """ + for p, meth in self.plugins: + result = meth(*arg, **kw) + if result is not None: + return result + + def _loadTestsFromNames(self, names, module=None): + """Chainable but not quite normal. Plugins return a tuple of + (tests, names) after processing the names. The tests are added + to a suite that is accumulated throughout the full call, while + names are input for the next plugin in the chain. + """ + suite = [] + for p, meth in self.plugins: + result = meth(names, module=module) + if result is not None: + suite_part, names = result + if suite_part: + suite.extend(suite_part) + return suite, names + + +class NoPlugins(object): + """Null Plugin manager that has no plugins.""" + interface = IPluginInterface + def __init__(self): + self._plugins = self.plugins = () + + def __iter__(self): + return () + + def _doNothing(self, *args, **kwds): + pass + + def _emptyIterator(self, *args, **kwds): + return () + + def __getattr__(self, call): + method = getattr(self.interface, call) + if getattr(method, "generative", False): + return self._emptyIterator + else: + return self._doNothing + + def addPlugin(self, plug): + raise NotImplementedError() + + def addPlugins(self, plugins): + raise NotImplementedError() + + def configure(self, options, config): + pass + + def loadPlugins(self): + pass + + def sort(self): + pass + + +class PluginManager(object): + """Base class for plugin managers. PluginManager is intended to be + used only with a static list of plugins. The loadPlugins() implementation + only reloads plugins from _extraplugins to prevent those from being + overridden by a subclass. + + The basic functionality of a plugin manager is to proxy all unknown + attributes through a ``PluginProxy`` to a list of plugins. + + Note that the list of plugins *may not* be changed after the first plugin + call. + """ + proxyClass = PluginProxy + + def __init__(self, plugins=(), proxyClass=None): + self._plugins = [] + self._extraplugins = () + self._proxies = {} + if plugins: + self.addPlugins(plugins) + if proxyClass is not None: + self.proxyClass = proxyClass + + def __getattr__(self, call): + try: + return self._proxies[call] + except KeyError: + proxy = self.proxyClass(call, self._plugins) + self._proxies[call] = proxy + return proxy + + def __iter__(self): + return iter(self.plugins) + + def addPlugin(self, plug): + # allow, for instance, plugins loaded via entry points to + # supplant builtin plugins. + new_name = getattr(plug, 'name', object()) + self._plugins[:] = [p for p in self._plugins + if getattr(p, 'name', None) != new_name] + self._plugins.append(plug) + + def addPlugins(self, plugins=(), extraplugins=()): + """extraplugins are maintained in a separate list and + re-added by loadPlugins() to prevent their being overwritten + by plugins added by a subclass of PluginManager + """ + self._extraplugins = extraplugins + for plug in iterchain(plugins, extraplugins): + self.addPlugin(plug) + + def configure(self, options, config): + """Configure the set of plugins with the given options + and config instance. After configuration, disabled plugins + are removed from the plugins list. + """ + log.debug("Configuring plugins") + self.config = config + cfg = PluginProxy('configure', self._plugins) + cfg(options, config) + enabled = [plug for plug in self._plugins if plug.enabled] + self.plugins = enabled + self.sort() + log.debug("Plugins enabled: %s", enabled) + + def loadPlugins(self): + for plug in self._extraplugins: + self.addPlugin(plug) + + def sort(self): + return sort_list(self._plugins, lambda x: getattr(x, 'score', 1), reverse=True) + + def _get_plugins(self): + return self._plugins + + def _set_plugins(self, plugins): + self._plugins = [] + self.addPlugins(plugins) + + plugins = property(_get_plugins, _set_plugins, None, + """Access the list of plugins managed by + this plugin manager""") + + +class ZeroNinePlugin: + """Proxy for 0.9 plugins, adapts 0.10 calls to 0.9 standard. + """ + def __init__(self, plugin): + self.plugin = plugin + + def options(self, parser, env=os.environ): + self.plugin.add_options(parser, env) + + def addError(self, test, err): + if not hasattr(self.plugin, 'addError'): + return + # switch off to addSkip, addDeprecated if those types + from nose.exc import SkipTest, DeprecatedTest + ec, ev, tb = err + if issubclass(ec, SkipTest): + if not hasattr(self.plugin, 'addSkip'): + return + return self.plugin.addSkip(test.test) + elif issubclass(ec, DeprecatedTest): + if not hasattr(self.plugin, 'addDeprecated'): + return + return self.plugin.addDeprecated(test.test) + # add capt + capt = test.capturedOutput + return self.plugin.addError(test.test, err, capt) + + def loadTestsFromFile(self, filename): + if hasattr(self.plugin, 'loadTestsFromPath'): + return self.plugin.loadTestsFromPath(filename) + + def addFailure(self, test, err): + if not hasattr(self.plugin, 'addFailure'): + return + # add capt and tbinfo + capt = test.capturedOutput + tbinfo = test.tbinfo + return self.plugin.addFailure(test.test, err, capt, tbinfo) + + def addSuccess(self, test): + if not hasattr(self.plugin, 'addSuccess'): + return + capt = test.capturedOutput + self.plugin.addSuccess(test.test, capt) + + def startTest(self, test): + if not hasattr(self.plugin, 'startTest'): + return + return self.plugin.startTest(test.test) + + def stopTest(self, test): + if not hasattr(self.plugin, 'stopTest'): + return + return self.plugin.stopTest(test.test) + + def __getattr__(self, val): + return getattr(self.plugin, val) + + +class EntryPointPluginManager(PluginManager): + """Plugin manager that loads plugins from the `nose.plugins` and + `nose.plugins.0.10` entry points. + """ + entry_points = (('nose.plugins.0.10', None), + ('nose.plugins', ZeroNinePlugin)) + + def loadPlugins(self): + """Load plugins by iterating the `nose.plugins` entry point. + """ + from pkg_resources import iter_entry_points + loaded = {} + for entry_point, adapt in self.entry_points: + for ep in iter_entry_points(entry_point): + if ep.name in loaded: + continue + loaded[ep.name] = True + log.debug('%s load plugin %s', self.__class__.__name__, ep) + try: + plugcls = ep.load() + except KeyboardInterrupt: + raise + except Exception, e: + # never want a plugin load to kill the test run + # but we can't log here because the logger is not yet + # configured + warn("Unable to load plugin %s: %s" % (ep, e), + RuntimeWarning) + continue + if adapt: + plug = adapt(plugcls()) + else: + plug = plugcls() + self.addPlugin(plug) + super(EntryPointPluginManager, self).loadPlugins() + + +class BuiltinPluginManager(PluginManager): + """Plugin manager that loads plugins from the list in + `nose.plugins.builtin`. + """ + def loadPlugins(self): + """Load plugins in nose.plugins.builtin + """ + from nose.plugins import builtin + for plug in builtin.plugins: + self.addPlugin(plug()) + super(BuiltinPluginManager, self).loadPlugins() + +try: + import pkg_resources + class DefaultPluginManager(EntryPointPluginManager, BuiltinPluginManager): + pass + +except ImportError: + class DefaultPluginManager(BuiltinPluginManager): + pass + +class RestrictedPluginManager(DefaultPluginManager): + """Plugin manager that restricts the plugin list to those not + excluded by a list of exclude methods. Any plugin that implements + an excluded method will be removed from the manager's plugin list + after plugins are loaded. + """ + def __init__(self, plugins=(), exclude=(), load=True): + DefaultPluginManager.__init__(self, plugins) + self.load = load + self.exclude = exclude + self.excluded = [] + self._excludedOpts = None + + def excludedOption(self, name): + if self._excludedOpts is None: + from optparse import OptionParser + self._excludedOpts = OptionParser(add_help_option=False) + for plugin in self.excluded: + plugin.options(self._excludedOpts, env={}) + return self._excludedOpts.get_option('--' + name) + + def loadPlugins(self): + if self.load: + DefaultPluginManager.loadPlugins(self) + allow = [] + for plugin in self.plugins: + ok = True + for method in self.exclude: + if hasattr(plugin, method): + ok = False + self.excluded.append(plugin) + break + if ok: + allow.append(plugin) + self.plugins = allow diff --git a/lib/spack/external/nose/plugins/multiprocess.py b/lib/spack/external/nose/plugins/multiprocess.py new file mode 100644 index 0000000000..2cae744a11 --- /dev/null +++ b/lib/spack/external/nose/plugins/multiprocess.py @@ -0,0 +1,835 @@ +""" +Overview +======== + +The multiprocess plugin enables you to distribute your test run among a set of +worker processes that run tests in parallel. This can speed up CPU-bound test +runs (as long as the number of work processeses is around the number of +processors or cores available), but is mainly useful for IO-bound tests that +spend most of their time waiting for data to arrive from someplace else. + +.. note :: + + See :doc:`../doc_tests/test_multiprocess/multiprocess` for + additional documentation and examples. Use of this plugin on python + 2.5 or earlier requires the multiprocessing_ module, also available + from PyPI. + +.. _multiprocessing : http://code.google.com/p/python-multiprocessing/ + +How tests are distributed +========================= + +The ideal case would be to dispatch each test to a worker process +separately. This ideal is not attainable in all cases, however, because many +test suites depend on context (class, module or package) fixtures. + +The plugin can't know (unless you tell it -- see below!) if a context fixture +can be called many times concurrently (is re-entrant), or if it can be shared +among tests running in different processes. Therefore, if a context has +fixtures, the default behavior is to dispatch the entire suite to a worker as +a unit. + +Controlling distribution +^^^^^^^^^^^^^^^^^^^^^^^^ + +There are two context-level variables that you can use to control this default +behavior. + +If a context's fixtures are re-entrant, set ``_multiprocess_can_split_ = True`` +in the context, and the plugin will dispatch tests in suites bound to that +context as if the context had no fixtures. This means that the fixtures will +execute concurrently and multiple times, typically once per test. + +If a context's fixtures can be shared by tests running in different processes +-- such as a package-level fixture that starts an external http server or +initializes a shared database -- then set ``_multiprocess_shared_ = True`` in +the context. These fixtures will then execute in the primary nose process, and +tests in those contexts will be individually dispatched to run in parallel. + +How results are collected and reported +====================================== + +As each test or suite executes in a worker process, results (failures, errors, +and specially handled exceptions like SkipTest) are collected in that +process. When the worker process finishes, it returns results to the main +nose process. There, any progress output is printed (dots!), and the +results from the test run are combined into a consolidated result +set. When results have been received for all dispatched tests, or all +workers have died, the result summary is output as normal. + +Beware! +======= + +Not all test suites will benefit from, or even operate correctly using, this +plugin. For example, CPU-bound tests will run more slowly if you don't have +multiple processors. There are also some differences in plugin +interactions and behaviors due to the way in which tests are dispatched and +loaded. In general, test loading under this plugin operates as if it were +always in directed mode instead of discovered mode. For instance, doctests +in test modules will always be found when using this plugin with the doctest +plugin. + +But the biggest issue you will face is probably concurrency. Unless you +have kept your tests as religiously pure unit tests, with no side-effects, no +ordering issues, and no external dependencies, chances are you will experience +odd, intermittent and unexplainable failures and errors when using this +plugin. This doesn't necessarily mean the plugin is broken; it may mean that +your test suite is not safe for concurrency. + +New Features in 1.1.0 +===================== + +* functions generated by test generators are now added to the worker queue + making them multi-threaded. +* fixed timeout functionality, now functions will be terminated with a + TimedOutException exception when they exceed their execution time. The + worker processes are not terminated. +* added ``--process-restartworker`` option to restart workers once they are + done, this helps control memory usage. Sometimes memory leaks can accumulate + making long runs very difficult. +* added global _instantiate_plugins to configure which plugins are started + on the worker processes. + +""" + +import logging +import os +import sys +import time +import traceback +import unittest +import pickle +import signal +import nose.case +from nose.core import TextTestRunner +from nose import failure +from nose import loader +from nose.plugins.base import Plugin +from nose.pyversion import bytes_ +from nose.result import TextTestResult +from nose.suite import ContextSuite +from nose.util import test_address +try: + # 2.7+ + from unittest.runner import _WritelnDecorator +except ImportError: + from unittest import _WritelnDecorator +from Queue import Empty +from warnings import warn +try: + from cStringIO import StringIO +except ImportError: + import StringIO + +# this is a list of plugin classes that will be checked for and created inside +# each worker process +_instantiate_plugins = None + +log = logging.getLogger(__name__) + +Process = Queue = Pool = Event = Value = Array = None + +# have to inherit KeyboardInterrupt to it will interrupt process properly +class TimedOutException(KeyboardInterrupt): + def __init__(self, value = "Timed Out"): + self.value = value + def __str__(self): + return repr(self.value) + +def _import_mp(): + global Process, Queue, Pool, Event, Value, Array + try: + from multiprocessing import Manager, Process + #prevent the server process created in the manager which holds Python + #objects and allows other processes to manipulate them using proxies + #to interrupt on SIGINT (keyboardinterrupt) so that the communication + #channel between subprocesses and main process is still usable after + #ctrl+C is received in the main process. + old=signal.signal(signal.SIGINT, signal.SIG_IGN) + m = Manager() + #reset it back so main process will receive a KeyboardInterrupt + #exception on ctrl+c + signal.signal(signal.SIGINT, old) + Queue, Pool, Event, Value, Array = ( + m.Queue, m.Pool, m.Event, m.Value, m.Array + ) + except ImportError: + warn("multiprocessing module is not available, multiprocess plugin " + "cannot be used", RuntimeWarning) + + +class TestLet: + def __init__(self, case): + try: + self._id = case.id() + except AttributeError: + pass + self._short_description = case.shortDescription() + self._str = str(case) + + def id(self): + return self._id + + def shortDescription(self): + return self._short_description + + def __str__(self): + return self._str + +class MultiProcess(Plugin): + """ + Run tests in multiple processes. Requires processing module. + """ + score = 1000 + status = {} + + def options(self, parser, env): + """ + Register command-line options. + """ + parser.add_option("--processes", action="store", + default=env.get('NOSE_PROCESSES', 0), + dest="multiprocess_workers", + metavar="NUM", + help="Spread test run among this many processes. " + "Set a number equal to the number of processors " + "or cores in your machine for best results. " + "Pass a negative number to have the number of " + "processes automatically set to the number of " + "cores. Passing 0 means to disable parallel " + "testing. Default is 0 unless NOSE_PROCESSES is " + "set. " + "[NOSE_PROCESSES]") + parser.add_option("--process-timeout", action="store", + default=env.get('NOSE_PROCESS_TIMEOUT', 10), + dest="multiprocess_timeout", + metavar="SECONDS", + help="Set timeout for return of results from each " + "test runner process. Default is 10. " + "[NOSE_PROCESS_TIMEOUT]") + parser.add_option("--process-restartworker", action="store_true", + default=env.get('NOSE_PROCESS_RESTARTWORKER', False), + dest="multiprocess_restartworker", + help="If set, will restart each worker process once" + " their tests are done, this helps control memory " + "leaks from killing the system. " + "[NOSE_PROCESS_RESTARTWORKER]") + + def configure(self, options, config): + """ + Configure plugin. + """ + try: + self.status.pop('active') + except KeyError: + pass + if not hasattr(options, 'multiprocess_workers'): + self.enabled = False + return + # don't start inside of a worker process + if config.worker: + return + self.config = config + try: + workers = int(options.multiprocess_workers) + except (TypeError, ValueError): + workers = 0 + if workers: + _import_mp() + if Process is None: + self.enabled = False + return + # Negative number of workers will cause multiprocessing to hang. + # Set the number of workers to the CPU count to avoid this. + if workers < 0: + try: + import multiprocessing + workers = multiprocessing.cpu_count() + except NotImplementedError: + self.enabled = False + return + self.enabled = True + self.config.multiprocess_workers = workers + t = float(options.multiprocess_timeout) + self.config.multiprocess_timeout = t + r = int(options.multiprocess_restartworker) + self.config.multiprocess_restartworker = r + self.status['active'] = True + + def prepareTestLoader(self, loader): + """Remember loader class so MultiProcessTestRunner can instantiate + the right loader. + """ + self.loaderClass = loader.__class__ + + def prepareTestRunner(self, runner): + """Replace test runner with MultiProcessTestRunner. + """ + # replace with our runner class + return MultiProcessTestRunner(stream=runner.stream, + verbosity=self.config.verbosity, + config=self.config, + loaderClass=self.loaderClass) + +def signalhandler(sig, frame): + raise TimedOutException() + +class MultiProcessTestRunner(TextTestRunner): + waitkilltime = 5.0 # max time to wait to terminate a process that does not + # respond to SIGILL + def __init__(self, **kw): + self.loaderClass = kw.pop('loaderClass', loader.defaultTestLoader) + super(MultiProcessTestRunner, self).__init__(**kw) + + def collect(self, test, testQueue, tasks, to_teardown, result): + # dispatch and collect results + # put indexes only on queue because tests aren't picklable + for case in self.nextBatch(test): + log.debug("Next batch %s (%s)", case, type(case)) + if (isinstance(case, nose.case.Test) and + isinstance(case.test, failure.Failure)): + log.debug("Case is a Failure") + case(result) # run here to capture the failure + continue + # handle shared fixtures + if isinstance(case, ContextSuite) and case.context is failure.Failure: + log.debug("Case is a Failure") + case(result) # run here to capture the failure + continue + elif isinstance(case, ContextSuite) and self.sharedFixtures(case): + log.debug("%s has shared fixtures", case) + try: + case.setUp() + except (KeyboardInterrupt, SystemExit): + raise + except: + log.debug("%s setup failed", sys.exc_info()) + result.addError(case, sys.exc_info()) + else: + to_teardown.append(case) + if case.factory: + ancestors=case.factory.context.get(case, []) + for an in ancestors[:2]: + #log.debug('reset ancestor %s', an) + if getattr(an, '_multiprocess_shared_', False): + an._multiprocess_can_split_=True + #an._multiprocess_shared_=False + self.collect(case, testQueue, tasks, to_teardown, result) + + else: + test_addr = self.addtask(testQueue,tasks,case) + log.debug("Queued test %s (%s) to %s", + len(tasks), test_addr, testQueue) + + def startProcess(self, iworker, testQueue, resultQueue, shouldStop, result): + currentaddr = Value('c',bytes_('')) + currentstart = Value('d',time.time()) + keyboardCaught = Event() + p = Process(target=runner, + args=(iworker, testQueue, + resultQueue, + currentaddr, + currentstart, + keyboardCaught, + shouldStop, + self.loaderClass, + result.__class__, + pickle.dumps(self.config))) + p.currentaddr = currentaddr + p.currentstart = currentstart + p.keyboardCaught = keyboardCaught + old = signal.signal(signal.SIGILL, signalhandler) + p.start() + signal.signal(signal.SIGILL, old) + return p + + def run(self, test): + """ + Execute the test (which may be a test suite). If the test is a suite, + distribute it out among as many processes as have been configured, at + as fine a level as is possible given the context fixtures defined in + the suite or any sub-suites. + + """ + log.debug("%s.run(%s) (%s)", self, test, os.getpid()) + wrapper = self.config.plugins.prepareTest(test) + if wrapper is not None: + test = wrapper + + # plugins can decorate or capture the output stream + wrapped = self.config.plugins.setOutputStream(self.stream) + if wrapped is not None: + self.stream = wrapped + + testQueue = Queue() + resultQueue = Queue() + tasks = [] + completed = [] + workers = [] + to_teardown = [] + shouldStop = Event() + + result = self._makeResult() + start = time.time() + + self.collect(test, testQueue, tasks, to_teardown, result) + + log.debug("Starting %s workers", self.config.multiprocess_workers) + for i in range(self.config.multiprocess_workers): + p = self.startProcess(i, testQueue, resultQueue, shouldStop, result) + workers.append(p) + log.debug("Started worker process %s", i+1) + + total_tasks = len(tasks) + # need to keep track of the next time to check for timeouts in case + # more than one process times out at the same time. + nexttimeout=self.config.multiprocess_timeout + thrownError = None + + try: + while tasks: + log.debug("Waiting for results (%s/%s tasks), next timeout=%.3fs", + len(completed), total_tasks,nexttimeout) + try: + iworker, addr, newtask_addrs, batch_result = resultQueue.get( + timeout=nexttimeout) + log.debug('Results received for worker %d, %s, new tasks: %d', + iworker,addr,len(newtask_addrs)) + try: + try: + tasks.remove(addr) + except ValueError: + log.warn('worker %s failed to remove from tasks: %s', + iworker,addr) + total_tasks += len(newtask_addrs) + tasks.extend(newtask_addrs) + except KeyError: + log.debug("Got result for unknown task? %s", addr) + log.debug("current: %s",str(list(tasks)[0])) + else: + completed.append([addr,batch_result]) + self.consolidate(result, batch_result) + if (self.config.stopOnError + and not result.wasSuccessful()): + # set the stop condition + shouldStop.set() + break + if self.config.multiprocess_restartworker: + log.debug('joining worker %s',iworker) + # wait for working, but not that important if worker + # cannot be joined in fact, for workers that add to + # testQueue, they will not terminate until all their + # items are read + workers[iworker].join(timeout=1) + if not shouldStop.is_set() and not testQueue.empty(): + log.debug('starting new process on worker %s',iworker) + workers[iworker] = self.startProcess(iworker, testQueue, resultQueue, shouldStop, result) + except Empty: + log.debug("Timed out with %s tasks pending " + "(empty testQueue=%r): %s", + len(tasks),testQueue.empty(),str(tasks)) + any_alive = False + for iworker, w in enumerate(workers): + if w.is_alive(): + worker_addr = bytes_(w.currentaddr.value,'ascii') + timeprocessing = time.time() - w.currentstart.value + if ( len(worker_addr) == 0 + and timeprocessing > self.config.multiprocess_timeout-0.1): + log.debug('worker %d has finished its work item, ' + 'but is not exiting? do we wait for it?', + iworker) + else: + any_alive = True + if (len(worker_addr) > 0 + and timeprocessing > self.config.multiprocess_timeout-0.1): + log.debug('timed out worker %s: %s', + iworker,worker_addr) + w.currentaddr.value = bytes_('') + # If the process is in C++ code, sending a SIGILL + # might not send a python KeybordInterrupt exception + # therefore, send multiple signals until an + # exception is caught. If this takes too long, then + # terminate the process + w.keyboardCaught.clear() + startkilltime = time.time() + while not w.keyboardCaught.is_set() and w.is_alive(): + if time.time()-startkilltime > self.waitkilltime: + # have to terminate... + log.error("terminating worker %s",iworker) + w.terminate() + # there is a small probability that the + # terminated process might send a result, + # which has to be specially handled or + # else processes might get orphaned. + workers[iworker] = w = self.startProcess(iworker, testQueue, resultQueue, shouldStop, result) + break + os.kill(w.pid, signal.SIGILL) + time.sleep(0.1) + if not any_alive and testQueue.empty(): + log.debug("All workers dead") + break + nexttimeout=self.config.multiprocess_timeout + for w in workers: + if w.is_alive() and len(w.currentaddr.value) > 0: + timeprocessing = time.time()-w.currentstart.value + if timeprocessing <= self.config.multiprocess_timeout: + nexttimeout = min(nexttimeout, + self.config.multiprocess_timeout-timeprocessing) + log.debug("Completed %s tasks (%s remain)", len(completed), len(tasks)) + + except (KeyboardInterrupt, SystemExit), e: + log.info('parent received ctrl-c when waiting for test results') + thrownError = e + #resultQueue.get(False) + + result.addError(test, sys.exc_info()) + + try: + for case in to_teardown: + log.debug("Tearing down shared fixtures for %s", case) + try: + case.tearDown() + except (KeyboardInterrupt, SystemExit): + raise + except: + result.addError(case, sys.exc_info()) + + stop = time.time() + + # first write since can freeze on shutting down processes + result.printErrors() + result.printSummary(start, stop) + self.config.plugins.finalize(result) + + if thrownError is None: + log.debug("Tell all workers to stop") + for w in workers: + if w.is_alive(): + testQueue.put('STOP', block=False) + + # wait for the workers to end + for iworker,worker in enumerate(workers): + if worker.is_alive(): + log.debug('joining worker %s',iworker) + worker.join() + if worker.is_alive(): + log.debug('failed to join worker %s',iworker) + except (KeyboardInterrupt, SystemExit): + log.info('parent received ctrl-c when shutting down: stop all processes') + for worker in workers: + if worker.is_alive(): + worker.terminate() + + if thrownError: raise thrownError + else: raise + + return result + + def addtask(testQueue,tasks,case): + arg = None + if isinstance(case,nose.case.Test) and hasattr(case.test,'arg'): + # this removes the top level descriptor and allows real function + # name to be returned + case.test.descriptor = None + arg = case.test.arg + test_addr = MultiProcessTestRunner.address(case) + testQueue.put((test_addr,arg), block=False) + if arg is not None: + test_addr += str(arg) + if tasks is not None: + tasks.append(test_addr) + return test_addr + addtask = staticmethod(addtask) + + def address(case): + if hasattr(case, 'address'): + file, mod, call = case.address() + elif hasattr(case, 'context'): + file, mod, call = test_address(case.context) + else: + raise Exception("Unable to convert %s to address" % case) + parts = [] + if file is None: + if mod is None: + raise Exception("Unaddressable case %s" % case) + else: + parts.append(mod) + else: + # strip __init__.py(c) from end of file part + # if present, having it there confuses loader + dirname, basename = os.path.split(file) + if basename.startswith('__init__'): + file = dirname + parts.append(file) + if call is not None: + parts.append(call) + return ':'.join(map(str, parts)) + address = staticmethod(address) + + def nextBatch(self, test): + # allows tests or suites to mark themselves as not safe + # for multiprocess execution + if hasattr(test, 'context'): + if not getattr(test.context, '_multiprocess_', True): + return + + if ((isinstance(test, ContextSuite) + and test.hasFixtures(self.checkCanSplit)) + or not getattr(test, 'can_split', True) + or not isinstance(test, unittest.TestSuite)): + # regular test case, or a suite with context fixtures + + # special case: when run like nosetests path/to/module.py + # the top-level suite has only one item, and it shares + # the same context as that item. In that case, we want the + # item, not the top-level suite + if isinstance(test, ContextSuite): + contained = list(test) + if (len(contained) == 1 + and getattr(contained[0], + 'context', None) == test.context): + test = contained[0] + yield test + else: + # Suite is without fixtures at this level; but it may have + # fixtures at any deeper level, so we need to examine it all + # the way down to the case level + for case in test: + for batch in self.nextBatch(case): + yield batch + + def checkCanSplit(context, fixt): + """ + Callback that we use to check whether the fixtures found in a + context or ancestor are ones we care about. + + Contexts can tell us that their fixtures are reentrant by setting + _multiprocess_can_split_. So if we see that, we return False to + disregard those fixtures. + """ + if not fixt: + return False + if getattr(context, '_multiprocess_can_split_', False): + return False + return True + checkCanSplit = staticmethod(checkCanSplit) + + def sharedFixtures(self, case): + context = getattr(case, 'context', None) + if not context: + return False + return getattr(context, '_multiprocess_shared_', False) + + def consolidate(self, result, batch_result): + log.debug("batch result is %s" , batch_result) + try: + output, testsRun, failures, errors, errorClasses = batch_result + except ValueError: + log.debug("result in unexpected format %s", batch_result) + failure.Failure(*sys.exc_info())(result) + return + self.stream.write(output) + result.testsRun += testsRun + result.failures.extend(failures) + result.errors.extend(errors) + for key, (storage, label, isfail) in errorClasses.items(): + if key not in result.errorClasses: + # Ordinarily storage is result attribute + # but it's only processed through the errorClasses + # dict, so it's ok to fake it here + result.errorClasses[key] = ([], label, isfail) + mystorage, _junk, _junk = result.errorClasses[key] + mystorage.extend(storage) + log.debug("Ran %s tests (total: %s)", testsRun, result.testsRun) + + +def runner(ix, testQueue, resultQueue, currentaddr, currentstart, + keyboardCaught, shouldStop, loaderClass, resultClass, config): + try: + try: + return __runner(ix, testQueue, resultQueue, currentaddr, currentstart, + keyboardCaught, shouldStop, loaderClass, resultClass, config) + except KeyboardInterrupt: + log.debug('Worker %s keyboard interrupt, stopping',ix) + except Empty: + log.debug("Worker %s timed out waiting for tasks", ix) + +def __runner(ix, testQueue, resultQueue, currentaddr, currentstart, + keyboardCaught, shouldStop, loaderClass, resultClass, config): + + config = pickle.loads(config) + dummy_parser = config.parserClass() + if _instantiate_plugins is not None: + for pluginclass in _instantiate_plugins: + plugin = pluginclass() + plugin.addOptions(dummy_parser,{}) + config.plugins.addPlugin(plugin) + config.plugins.configure(config.options,config) + config.plugins.begin() + log.debug("Worker %s executing, pid=%d", ix,os.getpid()) + loader = loaderClass(config=config) + loader.suiteClass.suiteClass = NoSharedFixtureContextSuite + + def get(): + return testQueue.get(timeout=config.multiprocess_timeout) + + def makeResult(): + stream = _WritelnDecorator(StringIO()) + result = resultClass(stream, descriptions=1, + verbosity=config.verbosity, + config=config) + plug_result = config.plugins.prepareTestResult(result) + if plug_result: + return plug_result + return result + + def batch(result): + failures = [(TestLet(c), err) for c, err in result.failures] + errors = [(TestLet(c), err) for c, err in result.errors] + errorClasses = {} + for key, (storage, label, isfail) in result.errorClasses.items(): + errorClasses[key] = ([(TestLet(c), err) for c, err in storage], + label, isfail) + return ( + result.stream.getvalue(), + result.testsRun, + failures, + errors, + errorClasses) + for test_addr, arg in iter(get, 'STOP'): + if shouldStop.is_set(): + log.exception('Worker %d STOPPED',ix) + break + result = makeResult() + test = loader.loadTestsFromNames([test_addr]) + test.testQueue = testQueue + test.tasks = [] + test.arg = arg + log.debug("Worker %s Test is %s (%s)", ix, test_addr, test) + try: + if arg is not None: + test_addr = test_addr + str(arg) + currentaddr.value = bytes_(test_addr) + currentstart.value = time.time() + test(result) + currentaddr.value = bytes_('') + resultQueue.put((ix, test_addr, test.tasks, batch(result))) + except KeyboardInterrupt, e: #TimedOutException: + timeout = isinstance(e, TimedOutException) + if timeout: + keyboardCaught.set() + if len(currentaddr.value): + if timeout: + msg = 'Worker %s timed out, failing current test %s' + else: + msg = 'Worker %s keyboard interrupt, failing current test %s' + log.exception(msg,ix,test_addr) + currentaddr.value = bytes_('') + failure.Failure(*sys.exc_info())(result) + resultQueue.put((ix, test_addr, test.tasks, batch(result))) + else: + if timeout: + msg = 'Worker %s test %s timed out' + else: + msg = 'Worker %s test %s keyboard interrupt' + log.debug(msg,ix,test_addr) + resultQueue.put((ix, test_addr, test.tasks, batch(result))) + if not timeout: + raise + except SystemExit: + currentaddr.value = bytes_('') + log.exception('Worker %s system exit',ix) + raise + except: + currentaddr.value = bytes_('') + log.exception("Worker %s error running test or returning " + "results",ix) + failure.Failure(*sys.exc_info())(result) + resultQueue.put((ix, test_addr, test.tasks, batch(result))) + if config.multiprocess_restartworker: + break + log.debug("Worker %s ending", ix) + + +class NoSharedFixtureContextSuite(ContextSuite): + """ + Context suite that never fires shared fixtures. + + When a context sets _multiprocess_shared_, fixtures in that context + are executed by the main process. Using this suite class prevents them + from executing in the runner process as well. + + """ + testQueue = None + tasks = None + arg = None + def setupContext(self, context): + if getattr(context, '_multiprocess_shared_', False): + return + super(NoSharedFixtureContextSuite, self).setupContext(context) + + def teardownContext(self, context): + if getattr(context, '_multiprocess_shared_', False): + return + super(NoSharedFixtureContextSuite, self).teardownContext(context) + def run(self, result): + """Run tests in suite inside of suite fixtures. + """ + # proxy the result for myself + log.debug("suite %s (%s) run called, tests: %s", + id(self), self, self._tests) + if self.resultProxy: + result, orig = self.resultProxy(result, self), result + else: + result, orig = result, result + try: + #log.debug('setUp for %s', id(self)); + self.setUp() + except KeyboardInterrupt: + raise + except: + self.error_context = 'setup' + result.addError(self, self._exc_info()) + return + try: + for test in self._tests: + if (isinstance(test,nose.case.Test) + and self.arg is not None): + test.test.arg = self.arg + else: + test.arg = self.arg + test.testQueue = self.testQueue + test.tasks = self.tasks + if result.shouldStop: + log.debug("stopping") + break + # each nose.case.Test will create its own result proxy + # so the cases need the original result, to avoid proxy + # chains + #log.debug('running test %s in suite %s', test, self); + try: + test(orig) + except KeyboardInterrupt, e: + timeout = isinstance(e, TimedOutException) + if timeout: + msg = 'Timeout when running test %s in suite %s' + else: + msg = 'KeyboardInterrupt when running test %s in suite %s' + log.debug(msg, test, self) + err = (TimedOutException,TimedOutException(str(test)), + sys.exc_info()[2]) + test.config.plugins.addError(test,err) + orig.addError(test,err) + if not timeout: + raise + finally: + self.has_run = True + try: + #log.debug('tearDown for %s', id(self)); + self.tearDown() + except KeyboardInterrupt: + raise + except: + self.error_context = 'teardown' + result.addError(self, self._exc_info()) diff --git a/lib/spack/external/nose/plugins/plugintest.py b/lib/spack/external/nose/plugins/plugintest.py new file mode 100644 index 0000000000..76d0d2c48c --- /dev/null +++ b/lib/spack/external/nose/plugins/plugintest.py @@ -0,0 +1,416 @@ +""" +Testing Plugins +=============== + +The plugin interface is well-tested enough to safely unit test your +use of its hooks with some level of confidence. However, there is also +a mixin for unittest.TestCase called PluginTester that's designed to +test plugins in their native runtime environment. + +Here's a simple example with a do-nothing plugin and a composed suite. + + >>> import unittest + >>> from nose.plugins import Plugin, PluginTester + >>> class FooPlugin(Plugin): + ... pass + >>> class TestPluginFoo(PluginTester, unittest.TestCase): + ... activate = '--with-foo' + ... plugins = [FooPlugin()] + ... def test_foo(self): + ... for line in self.output: + ... # i.e. check for patterns + ... pass + ... + ... # or check for a line containing ... + ... assert "ValueError" in self.output + ... def makeSuite(self): + ... class TC(unittest.TestCase): + ... def runTest(self): + ... raise ValueError("I hate foo") + ... return [TC('runTest')] + ... + >>> res = unittest.TestResult() + >>> case = TestPluginFoo('test_foo') + >>> _ = case(res) + >>> res.errors + [] + >>> res.failures + [] + >>> res.wasSuccessful() + True + >>> res.testsRun + 1 + +And here is a more complex example of testing a plugin that has extra +arguments and reads environment variables. + + >>> import unittest, os + >>> from nose.plugins import Plugin, PluginTester + >>> class FancyOutputter(Plugin): + ... name = "fancy" + ... def configure(self, options, conf): + ... Plugin.configure(self, options, conf) + ... if not self.enabled: + ... return + ... self.fanciness = 1 + ... if options.more_fancy: + ... self.fanciness = 2 + ... if 'EVEN_FANCIER' in self.env: + ... self.fanciness = 3 + ... + ... def options(self, parser, env=os.environ): + ... self.env = env + ... parser.add_option('--more-fancy', action='store_true') + ... Plugin.options(self, parser, env=env) + ... + ... def report(self, stream): + ... stream.write("FANCY " * self.fanciness) + ... + >>> class TestFancyOutputter(PluginTester, unittest.TestCase): + ... activate = '--with-fancy' # enables the plugin + ... plugins = [FancyOutputter()] + ... args = ['--more-fancy'] + ... env = {'EVEN_FANCIER': '1'} + ... + ... def test_fancy_output(self): + ... assert "FANCY FANCY FANCY" in self.output, ( + ... "got: %s" % self.output) + ... def makeSuite(self): + ... class TC(unittest.TestCase): + ... def runTest(self): + ... raise ValueError("I hate fancy stuff") + ... return [TC('runTest')] + ... + >>> res = unittest.TestResult() + >>> case = TestFancyOutputter('test_fancy_output') + >>> _ = case(res) + >>> res.errors + [] + >>> res.failures + [] + >>> res.wasSuccessful() + True + >>> res.testsRun + 1 + +""" + +import re +import sys +from warnings import warn + +try: + from cStringIO import StringIO +except ImportError: + from StringIO import StringIO + +__all__ = ['PluginTester', 'run'] + +from os import getpid +class MultiProcessFile(object): + """ + helper for testing multiprocessing + + multiprocessing poses a problem for doctests, since the strategy + of replacing sys.stdout/stderr with file-like objects then + inspecting the results won't work: the child processes will + write to the objects, but the data will not be reflected + in the parent doctest-ing process. + + The solution is to create file-like objects which will interact with + multiprocessing in a more desirable way. + + All processes can write to this object, but only the creator can read. + This allows the testing system to see a unified picture of I/O. + """ + def __init__(self): + # per advice at: + # http://docs.python.org/library/multiprocessing.html#all-platforms + self.__master = getpid() + self.__queue = Manager().Queue() + self.__buffer = StringIO() + self.softspace = 0 + + def buffer(self): + if getpid() != self.__master: + return + + from Queue import Empty + from collections import defaultdict + cache = defaultdict(str) + while True: + try: + pid, data = self.__queue.get_nowait() + except Empty: + break + if pid == (): + #show parent output after children + #this is what users see, usually + pid = ( 1e100, ) # googol! + cache[pid] += data + for pid in sorted(cache): + #self.__buffer.write( '%s wrote: %r\n' % (pid, cache[pid]) ) #DEBUG + self.__buffer.write( cache[pid] ) + def write(self, data): + # note that these pids are in the form of current_process()._identity + # rather than OS pids + from multiprocessing import current_process + pid = current_process()._identity + self.__queue.put((pid, data)) + def __iter__(self): + "getattr doesn't work for iter()" + self.buffer() + return self.__buffer + def seek(self, offset, whence=0): + self.buffer() + return self.__buffer.seek(offset, whence) + def getvalue(self): + self.buffer() + return self.__buffer.getvalue() + def __getattr__(self, attr): + return getattr(self.__buffer, attr) + +try: + from multiprocessing import Manager + Buffer = MultiProcessFile +except ImportError: + Buffer = StringIO + +class PluginTester(object): + """A mixin for testing nose plugins in their runtime environment. + + Subclass this and mix in unittest.TestCase to run integration/functional + tests on your plugin. When setUp() is called, the stub test suite is + executed with your plugin so that during an actual test you can inspect the + artifacts of how your plugin interacted with the stub test suite. + + - activate + + - the argument to send nosetests to activate the plugin + + - suitepath + + - if set, this is the path of the suite to test. Otherwise, you + will need to use the hook, makeSuite() + + - plugins + + - the list of plugins to make available during the run. Note + that this does not mean these plugins will be *enabled* during + the run -- only the plugins enabled by the activate argument + or other settings in argv or env will be enabled. + + - args + + - a list of arguments to add to the nosetests command, in addition to + the activate argument + + - env + + - optional dict of environment variables to send nosetests + + """ + activate = None + suitepath = None + args = None + env = {} + argv = None + plugins = [] + ignoreFiles = None + + def makeSuite(self): + """returns a suite object of tests to run (unittest.TestSuite()) + + If self.suitepath is None, this must be implemented. The returned suite + object will be executed with all plugins activated. It may return + None. + + Here is an example of a basic suite object you can return :: + + >>> import unittest + >>> class SomeTest(unittest.TestCase): + ... def runTest(self): + ... raise ValueError("Now do something, plugin!") + ... + >>> unittest.TestSuite([SomeTest()]) # doctest: +ELLIPSIS + ]> + + """ + raise NotImplementedError + + def _execPlugin(self): + """execute the plugin on the internal test suite. + """ + from nose.config import Config + from nose.core import TestProgram + from nose.plugins.manager import PluginManager + + suite = None + stream = Buffer() + conf = Config(env=self.env, + stream=stream, + plugins=PluginManager(plugins=self.plugins)) + if self.ignoreFiles is not None: + conf.ignoreFiles = self.ignoreFiles + if not self.suitepath: + suite = self.makeSuite() + + self.nose = TestProgram(argv=self.argv, config=conf, suite=suite, + exit=False) + self.output = AccessDecorator(stream) + + def setUp(self): + """runs nosetests with the specified test suite, all plugins + activated. + """ + self.argv = ['nosetests', self.activate] + if self.args: + self.argv.extend(self.args) + if self.suitepath: + self.argv.append(self.suitepath) + + self._execPlugin() + + +class AccessDecorator(object): + stream = None + _buf = None + def __init__(self, stream): + self.stream = stream + stream.seek(0) + self._buf = stream.read() + stream.seek(0) + def __contains__(self, val): + return val in self._buf + def __iter__(self): + return iter(self.stream) + def __str__(self): + return self._buf + + +def blankline_separated_blocks(text): + "a bunch of === characters is also considered a blank line" + block = [] + for line in text.splitlines(True): + block.append(line) + line = line.strip() + if not line or line.startswith('===') and not line.strip('='): + yield "".join(block) + block = [] + if block: + yield "".join(block) + + +def remove_stack_traces(out): + # this regexp taken from Python 2.5's doctest + traceback_re = re.compile(r""" + # Grab the traceback header. Different versions of Python have + # said different things on the first traceback line. + ^(?P Traceback\ \( + (?: most\ recent\ call\ last + | innermost\ last + ) \) : + ) + \s* $ # toss trailing whitespace on the header. + (?P .*?) # don't blink: absorb stuff until... + ^(?=\w) # a line *starts* with alphanum. + .*?(?P \w+ ) # exception name + (?P [:\n] .*) # the rest + """, re.VERBOSE | re.MULTILINE | re.DOTALL) + blocks = [] + for block in blankline_separated_blocks(out): + blocks.append(traceback_re.sub(r"\g\n...\n\g\g", block)) + return "".join(blocks) + + +def simplify_warnings(out): + warn_re = re.compile(r""" + # Cut the file and line no, up to the warning name + ^.*:\d+:\s + (?P\w+): \s+ # warning category + (?P.+) $ \n? # warning message + ^ .* $ # stack frame + """, re.VERBOSE | re.MULTILINE) + return warn_re.sub(r"\g: \g", out) + + +def remove_timings(out): + return re.sub( + r"Ran (\d+ tests?) in [0-9.]+s", r"Ran \1 in ...s", out) + + +def munge_nose_output_for_doctest(out): + """Modify nose output to make it easy to use in doctests.""" + out = remove_stack_traces(out) + out = simplify_warnings(out) + out = remove_timings(out) + return out.strip() + + +def run(*arg, **kw): + """ + Specialized version of nose.run for use inside of doctests that + test test runs. + + This version of run() prints the result output to stdout. Before + printing, the output is processed by replacing the timing + information with an ellipsis (...), removing traceback stacks, and + removing trailing whitespace. + + Use this version of run wherever you are writing a doctest that + tests nose (or unittest) test result output. + + Note: do not use doctest: +ELLIPSIS when testing nose output, + since ellipses ("test_foo ... ok") in your expected test runner + output may match multiple lines of output, causing spurious test + passes! + """ + from nose import run + from nose.config import Config + from nose.plugins.manager import PluginManager + + buffer = Buffer() + if 'config' not in kw: + plugins = kw.pop('plugins', []) + if isinstance(plugins, list): + plugins = PluginManager(plugins=plugins) + env = kw.pop('env', {}) + kw['config'] = Config(env=env, plugins=plugins) + if 'argv' not in kw: + kw['argv'] = ['nosetests', '-v'] + kw['config'].stream = buffer + + # Set up buffering so that all output goes to our buffer, + # or warn user if deprecated behavior is active. If this is not + # done, prints and warnings will either be out of place or + # disappear. + stderr = sys.stderr + stdout = sys.stdout + if kw.pop('buffer_all', False): + sys.stdout = sys.stderr = buffer + restore = True + else: + restore = False + warn("The behavior of nose.plugins.plugintest.run() will change in " + "the next release of nose. The current behavior does not " + "correctly account for output to stdout and stderr. To enable " + "correct behavior, use run_buffered() instead, or pass " + "the keyword argument buffer_all=True to run().", + DeprecationWarning, stacklevel=2) + try: + run(*arg, **kw) + finally: + if restore: + sys.stderr = stderr + sys.stdout = stdout + out = buffer.getvalue() + print munge_nose_output_for_doctest(out) + + +def run_buffered(*arg, **kw): + kw['buffer_all'] = True + run(*arg, **kw) + +if __name__ == '__main__': + import doctest + doctest.testmod() diff --git a/lib/spack/external/nose/plugins/prof.py b/lib/spack/external/nose/plugins/prof.py new file mode 100644 index 0000000000..4d304a934b --- /dev/null +++ b/lib/spack/external/nose/plugins/prof.py @@ -0,0 +1,154 @@ +"""This plugin will run tests using the hotshot profiler, which is part +of the standard library. To turn it on, use the ``--with-profile`` option +or set the NOSE_WITH_PROFILE environment variable. Profiler output can be +controlled with the ``--profile-sort`` and ``--profile-restrict`` options, +and the profiler output file may be changed with ``--profile-stats-file``. + +See the `hotshot documentation`_ in the standard library documentation for +more details on the various output options. + +.. _hotshot documentation: http://docs.python.org/library/hotshot.html +""" + +try: + import hotshot + from hotshot import stats +except ImportError: + hotshot, stats = None, None +import logging +import os +import sys +import tempfile +from nose.plugins.base import Plugin +from nose.util import tolist + +log = logging.getLogger('nose.plugins') + +class Profile(Plugin): + """ + Use this plugin to run tests using the hotshot profiler. + """ + pfile = None + clean_stats_file = False + def options(self, parser, env): + """Register commandline options. + """ + if not self.available(): + return + Plugin.options(self, parser, env) + parser.add_option('--profile-sort', action='store', dest='profile_sort', + default=env.get('NOSE_PROFILE_SORT', 'cumulative'), + metavar="SORT", + help="Set sort order for profiler output") + parser.add_option('--profile-stats-file', action='store', + dest='profile_stats_file', + metavar="FILE", + default=env.get('NOSE_PROFILE_STATS_FILE'), + help='Profiler stats file; default is a new ' + 'temp file on each run') + parser.add_option('--profile-restrict', action='append', + dest='profile_restrict', + metavar="RESTRICT", + default=env.get('NOSE_PROFILE_RESTRICT'), + help="Restrict profiler output. See help for " + "pstats.Stats for details") + + def available(cls): + return hotshot is not None + available = classmethod(available) + + def begin(self): + """Create profile stats file and load profiler. + """ + if not self.available(): + return + self._create_pfile() + self.prof = hotshot.Profile(self.pfile) + + def configure(self, options, conf): + """Configure plugin. + """ + if not self.available(): + self.enabled = False + return + Plugin.configure(self, options, conf) + self.conf = conf + if options.profile_stats_file: + self.pfile = options.profile_stats_file + self.clean_stats_file = False + else: + self.pfile = None + self.clean_stats_file = True + self.fileno = None + self.sort = options.profile_sort + self.restrict = tolist(options.profile_restrict) + + def prepareTest(self, test): + """Wrap entire test run in :func:`prof.runcall`. + """ + if not self.available(): + return + log.debug('preparing test %s' % test) + def run_and_profile(result, prof=self.prof, test=test): + self._create_pfile() + prof.runcall(test, result) + return run_and_profile + + def report(self, stream): + """Output profiler report. + """ + log.debug('printing profiler report') + self.prof.close() + prof_stats = stats.load(self.pfile) + prof_stats.sort_stats(self.sort) + + # 2.5 has completely different stream handling from 2.4 and earlier. + # Before 2.5, stats objects have no stream attribute; in 2.5 and later + # a reference sys.stdout is stored before we can tweak it. + compat_25 = hasattr(prof_stats, 'stream') + if compat_25: + tmp = prof_stats.stream + prof_stats.stream = stream + else: + tmp = sys.stdout + sys.stdout = stream + try: + if self.restrict: + log.debug('setting profiler restriction to %s', self.restrict) + prof_stats.print_stats(*self.restrict) + else: + prof_stats.print_stats() + finally: + if compat_25: + prof_stats.stream = tmp + else: + sys.stdout = tmp + + def finalize(self, result): + """Clean up stats file, if configured to do so. + """ + if not self.available(): + return + try: + self.prof.close() + except AttributeError: + # TODO: is this trying to catch just the case where not + # hasattr(self.prof, "close")? If so, the function call should be + # moved out of the try: suite. + pass + if self.clean_stats_file: + if self.fileno: + try: + os.close(self.fileno) + except OSError: + pass + try: + os.unlink(self.pfile) + except OSError: + pass + return None + + def _create_pfile(self): + if not self.pfile: + self.fileno, self.pfile = tempfile.mkstemp() + self.clean_stats_file = True diff --git a/lib/spack/external/nose/plugins/skip.py b/lib/spack/external/nose/plugins/skip.py new file mode 100644 index 0000000000..9d1ac8f604 --- /dev/null +++ b/lib/spack/external/nose/plugins/skip.py @@ -0,0 +1,63 @@ +""" +This plugin installs a SKIP error class for the SkipTest exception. +When SkipTest is raised, the exception will be logged in the skipped +attribute of the result, 'S' or 'SKIP' (verbose) will be output, and +the exception will not be counted as an error or failure. This plugin +is enabled by default but may be disabled with the ``--no-skip`` option. +""" + +from nose.plugins.errorclass import ErrorClass, ErrorClassPlugin + + +# on SkipTest: +# - unittest SkipTest is first preference, but it's only available +# for >= 2.7 +# - unittest2 SkipTest is second preference for older pythons. This +# mirrors logic for choosing SkipTest exception in testtools +# - if none of the above, provide custom class +try: + from unittest.case import SkipTest +except ImportError: + try: + from unittest2.case import SkipTest + except ImportError: + class SkipTest(Exception): + """Raise this exception to mark a test as skipped. + """ + pass + + +class Skip(ErrorClassPlugin): + """ + Plugin that installs a SKIP error class for the SkipTest + exception. When SkipTest is raised, the exception will be logged + in the skipped attribute of the result, 'S' or 'SKIP' (verbose) + will be output, and the exception will not be counted as an error + or failure. + """ + enabled = True + skipped = ErrorClass(SkipTest, + label='SKIP', + isfailure=False) + + def options(self, parser, env): + """ + Add my options to command line. + """ + env_opt = 'NOSE_WITHOUT_SKIP' + parser.add_option('--no-skip', action='store_true', + dest='noSkip', default=env.get(env_opt, False), + help="Disable special handling of SkipTest " + "exceptions.") + + def configure(self, options, conf): + """ + Configure plugin. Skip plugin is enabled by default. + """ + if not self.can_configure: + return + self.conf = conf + disable = getattr(options, 'noSkip', False) + if disable: + self.enabled = False + diff --git a/lib/spack/external/nose/plugins/testid.py b/lib/spack/external/nose/plugins/testid.py new file mode 100644 index 0000000000..ae8119bd01 --- /dev/null +++ b/lib/spack/external/nose/plugins/testid.py @@ -0,0 +1,311 @@ +""" +This plugin adds a test id (like #1) to each test name output. After +you've run once to generate test ids, you can re-run individual +tests by activating the plugin and passing the ids (with or +without the # prefix) instead of test names. + +For example, if your normal test run looks like:: + + % nosetests -v + tests.test_a ... ok + tests.test_b ... ok + tests.test_c ... ok + +When adding ``--with-id`` you'll see:: + + % nosetests -v --with-id + #1 tests.test_a ... ok + #2 tests.test_b ... ok + #3 tests.test_c ... ok + +Then you can re-run individual tests by supplying just an id number:: + + % nosetests -v --with-id 2 + #2 tests.test_b ... ok + +You can also pass multiple id numbers:: + + % nosetests -v --with-id 2 3 + #2 tests.test_b ... ok + #3 tests.test_c ... ok + +Since most shells consider '#' a special character, you can leave it out when +specifying a test id. + +Note that when run without the -v switch, no special output is displayed, but +the ids file is still written. + +Looping over failed tests +------------------------- + +This plugin also adds a mode that will direct the test runner to record +failed tests. Subsequent test runs will then run only the tests that failed +last time. Activate this mode with the ``--failed`` switch:: + + % nosetests -v --failed + #1 test.test_a ... ok + #2 test.test_b ... ERROR + #3 test.test_c ... FAILED + #4 test.test_d ... ok + +On the second run, only tests #2 and #3 will run:: + + % nosetests -v --failed + #2 test.test_b ... ERROR + #3 test.test_c ... FAILED + +As you correct errors and tests pass, they'll drop out of subsequent runs. + +First:: + + % nosetests -v --failed + #2 test.test_b ... ok + #3 test.test_c ... FAILED + +Second:: + + % nosetests -v --failed + #3 test.test_c ... FAILED + +When all tests pass, the full set will run on the next invocation. + +First:: + + % nosetests -v --failed + #3 test.test_c ... ok + +Second:: + + % nosetests -v --failed + #1 test.test_a ... ok + #2 test.test_b ... ok + #3 test.test_c ... ok + #4 test.test_d ... ok + +.. note :: + + If you expect to use ``--failed`` regularly, it's a good idea to always run + using the ``--with-id`` option. This will ensure that an id file is always + created, allowing you to add ``--failed`` to the command line as soon as + you have failing tests. Otherwise, your first run using ``--failed`` will + (perhaps surprisingly) run *all* tests, because there won't be an id file + containing the record of failed tests from your previous run. + +""" +__test__ = False + +import logging +import os +from nose.plugins import Plugin +from nose.util import src, set + +try: + from cPickle import dump, load +except ImportError: + from pickle import dump, load + +log = logging.getLogger(__name__) + + +class TestId(Plugin): + """ + Activate to add a test id (like #1) to each test name output. Activate + with --failed to rerun failing tests only. + """ + name = 'id' + idfile = None + collecting = True + loopOnFailed = False + + def options(self, parser, env): + """Register commandline options. + """ + Plugin.options(self, parser, env) + parser.add_option('--id-file', action='store', dest='testIdFile', + default='.noseids', metavar="FILE", + help="Store test ids found in test runs in this " + "file. Default is the file .noseids in the " + "working directory.") + parser.add_option('--failed', action='store_true', + dest='failed', default=False, + help="Run the tests that failed in the last " + "test run.") + + def configure(self, options, conf): + """Configure plugin. + """ + Plugin.configure(self, options, conf) + if options.failed: + self.enabled = True + self.loopOnFailed = True + log.debug("Looping on failed tests") + self.idfile = os.path.expanduser(options.testIdFile) + if not os.path.isabs(self.idfile): + self.idfile = os.path.join(conf.workingDir, self.idfile) + self.id = 1 + # Ids and tests are mirror images: ids are {id: test address} and + # tests are {test address: id} + self.ids = {} + self.tests = {} + self.failed = [] + self.source_names = [] + # used to track ids seen when tests is filled from + # loaded ids file + self._seen = {} + self._write_hashes = conf.verbosity >= 2 + + def finalize(self, result): + """Save new ids file, if needed. + """ + if result.wasSuccessful(): + self.failed = [] + if self.collecting: + ids = dict(list(zip(list(self.tests.values()), list(self.tests.keys())))) + else: + ids = self.ids + fh = open(self.idfile, 'wb') + dump({'ids': ids, + 'failed': self.failed, + 'source_names': self.source_names}, fh) + fh.close() + log.debug('Saved test ids: %s, failed %s to %s', + ids, self.failed, self.idfile) + + def loadTestsFromNames(self, names, module=None): + """Translate ids in the list of requested names into their + test addresses, if they are found in my dict of tests. + """ + log.debug('ltfn %s %s', names, module) + try: + fh = open(self.idfile, 'rb') + data = load(fh) + if 'ids' in data: + self.ids = data['ids'] + self.failed = data['failed'] + self.source_names = data['source_names'] + else: + # old ids field + self.ids = data + self.failed = [] + self.source_names = names + if self.ids: + self.id = max(self.ids) + 1 + self.tests = dict(list(zip(list(self.ids.values()), list(self.ids.keys())))) + else: + self.id = 1 + log.debug( + 'Loaded test ids %s tests %s failed %s sources %s from %s', + self.ids, self.tests, self.failed, self.source_names, + self.idfile) + fh.close() + except ValueError, e: + # load() may throw a ValueError when reading the ids file, if it + # was generated with a newer version of Python than we are currently + # running. + log.debug('Error loading %s : %s', self.idfile, str(e)) + except IOError: + log.debug('IO error reading %s', self.idfile) + + if self.loopOnFailed and self.failed: + self.collecting = False + names = self.failed + self.failed = [] + # I don't load any tests myself, only translate names like '#2' + # into the associated test addresses + translated = [] + new_source = [] + really_new = [] + for name in names: + trans = self.tr(name) + if trans != name: + translated.append(trans) + else: + new_source.append(name) + # names that are not ids and that are not in the current + # list of source names go into the list for next time + if new_source: + new_set = set(new_source) + old_set = set(self.source_names) + log.debug("old: %s new: %s", old_set, new_set) + really_new = [s for s in new_source + if not s in old_set] + if really_new: + # remember new sources + self.source_names.extend(really_new) + if not translated: + # new set of source names, no translations + # means "run the requested tests" + names = new_source + else: + # no new names to translate and add to id set + self.collecting = False + log.debug("translated: %s new sources %s names %s", + translated, really_new, names) + return (None, translated + really_new or names) + + def makeName(self, addr): + log.debug("Make name %s", addr) + filename, module, call = addr + if filename is not None: + head = src(filename) + else: + head = module + if call is not None: + return "%s:%s" % (head, call) + return head + + def setOutputStream(self, stream): + """Get handle on output stream so the plugin can print id #s + """ + self.stream = stream + + def startTest(self, test): + """Maybe output an id # before the test name. + + Example output:: + + #1 test.test ... ok + #2 test.test_two ... ok + + """ + adr = test.address() + log.debug('start test %s (%s)', adr, adr in self.tests) + if adr in self.tests: + if adr in self._seen: + self.write(' ') + else: + self.write('#%s ' % self.tests[adr]) + self._seen[adr] = 1 + return + self.tests[adr] = self.id + self.write('#%s ' % self.id) + self.id += 1 + + def afterTest(self, test): + # None means test never ran, False means failed/err + if test.passed is False: + try: + key = str(self.tests[test.address()]) + except KeyError: + # never saw this test -- startTest didn't run + pass + else: + if key not in self.failed: + self.failed.append(key) + + def tr(self, name): + log.debug("tr '%s'", name) + try: + key = int(name.replace('#', '')) + except ValueError: + return name + log.debug("Got key %s", key) + # I'm running tests mapped from the ids file, + # not collecting new ones + if key in self.ids: + return self.makeName(self.ids[key]) + return name + + def write(self, output): + if self._write_hashes: + self.stream.write(output) diff --git a/lib/spack/external/nose/plugins/xunit.py b/lib/spack/external/nose/plugins/xunit.py new file mode 100644 index 0000000000..90b52f5f61 --- /dev/null +++ b/lib/spack/external/nose/plugins/xunit.py @@ -0,0 +1,341 @@ +"""This plugin provides test results in the standard XUnit XML format. + +It's designed for the `Jenkins`_ (previously Hudson) continuous build +system, but will probably work for anything else that understands an +XUnit-formatted XML representation of test results. + +Add this shell command to your builder :: + + nosetests --with-xunit + +And by default a file named nosetests.xml will be written to the +working directory. + +In a Jenkins builder, tick the box named "Publish JUnit test result report" +under the Post-build Actions and enter this value for Test report XMLs:: + + **/nosetests.xml + +If you need to change the name or location of the file, you can set the +``--xunit-file`` option. + +If you need to change the name of the test suite, you can set the +``--xunit-testsuite-name`` option. + +Here is an abbreviated version of what an XML test report might look like:: + + + + + + Traceback (most recent call last): + ... + TypeError: oops, wrong type + + + + +.. _Jenkins: http://jenkins-ci.org/ + +""" +import codecs +import doctest +import os +import sys +import traceback +import re +import inspect +from StringIO import StringIO +from time import time +from xml.sax import saxutils + +from nose.plugins.base import Plugin +from nose.exc import SkipTest +from nose.pyversion import force_unicode, format_exception + +# Invalid XML characters, control characters 0-31 sans \t, \n and \r +CONTROL_CHARACTERS = re.compile(r"[\000-\010\013\014\016-\037]") + +TEST_ID = re.compile(r'^(.*?)(\(.*\))$') + +def xml_safe(value): + """Replaces invalid XML characters with '?'.""" + return CONTROL_CHARACTERS.sub('?', value) + +def escape_cdata(cdata): + """Escape a string for an XML CDATA section.""" + return xml_safe(cdata).replace(']]>', ']]>]]>>> nice_classname(Exception()) # doctest: +ELLIPSIS + '...Exception' + >>> nice_classname(Exception) # doctest: +ELLIPSIS + '...Exception' + + """ + if inspect.isclass(obj): + cls_name = obj.__name__ + else: + cls_name = obj.__class__.__name__ + mod = inspect.getmodule(obj) + if mod: + name = mod.__name__ + # jython + if name.startswith('org.python.core.'): + name = name[len('org.python.core.'):] + return "%s.%s" % (name, cls_name) + else: + return cls_name + +def exc_message(exc_info): + """Return the exception's message.""" + exc = exc_info[1] + if exc is None: + # str exception + result = exc_info[0] + else: + try: + result = str(exc) + except UnicodeEncodeError: + try: + result = unicode(exc) + except UnicodeError: + # Fallback to args as neither str nor + # unicode(Exception(u'\xe6')) work in Python < 2.6 + result = exc.args[0] + result = force_unicode(result, 'UTF-8') + return xml_safe(result) + +class Tee(object): + def __init__(self, encoding, *args): + self._encoding = encoding + self._streams = args + + def write(self, data): + data = force_unicode(data, self._encoding) + for s in self._streams: + s.write(data) + + def writelines(self, lines): + for line in lines: + self.write(line) + + def flush(self): + for s in self._streams: + s.flush() + + def isatty(self): + return False + + +class Xunit(Plugin): + """This plugin provides test results in the standard XUnit XML format.""" + name = 'xunit' + score = 1500 + encoding = 'UTF-8' + error_report_file = None + + def __init__(self): + super(Xunit, self).__init__() + self._capture_stack = [] + self._currentStdout = None + self._currentStderr = None + + def _timeTaken(self): + if hasattr(self, '_timer'): + taken = time() - self._timer + else: + # test died before it ran (probably error in setup()) + # or success/failure added before test started probably + # due to custom TestResult munging + taken = 0.0 + return taken + + def _quoteattr(self, attr): + """Escape an XML attribute. Value can be unicode.""" + attr = xml_safe(attr) + return saxutils.quoteattr(attr) + + def options(self, parser, env): + """Sets additional command line options.""" + Plugin.options(self, parser, env) + parser.add_option( + '--xunit-file', action='store', + dest='xunit_file', metavar="FILE", + default=env.get('NOSE_XUNIT_FILE', 'nosetests.xml'), + help=("Path to xml file to store the xunit report in. " + "Default is nosetests.xml in the working directory " + "[NOSE_XUNIT_FILE]")) + + parser.add_option( + '--xunit-testsuite-name', action='store', + dest='xunit_testsuite_name', metavar="PACKAGE", + default=env.get('NOSE_XUNIT_TESTSUITE_NAME', 'nosetests'), + help=("Name of the testsuite in the xunit xml, generated by plugin. " + "Default test suite name is nosetests.")) + + def configure(self, options, config): + """Configures the xunit plugin.""" + Plugin.configure(self, options, config) + self.config = config + if self.enabled: + self.stats = {'errors': 0, + 'failures': 0, + 'passes': 0, + 'skipped': 0 + } + self.errorlist = [] + self.error_report_file_name = os.path.realpath(options.xunit_file) + self.xunit_testsuite_name = options.xunit_testsuite_name + + def report(self, stream): + """Writes an Xunit-formatted XML file + + The file includes a report of test errors and failures. + + """ + self.error_report_file = codecs.open(self.error_report_file_name, 'w', + self.encoding, 'replace') + self.stats['encoding'] = self.encoding + self.stats['testsuite_name'] = self.xunit_testsuite_name + self.stats['total'] = (self.stats['errors'] + self.stats['failures'] + + self.stats['passes'] + self.stats['skipped']) + self.error_report_file.write( + u'' + u'' % self.stats) + self.error_report_file.write(u''.join([force_unicode(e, self.encoding) + for e in self.errorlist])) + self.error_report_file.write(u'') + self.error_report_file.close() + if self.config.verbosity > 1: + stream.writeln("-" * 70) + stream.writeln("XML: %s" % self.error_report_file.name) + + def _startCapture(self): + self._capture_stack.append((sys.stdout, sys.stderr)) + self._currentStdout = StringIO() + self._currentStderr = StringIO() + sys.stdout = Tee(self.encoding, self._currentStdout, sys.stdout) + sys.stderr = Tee(self.encoding, self._currentStderr, sys.stderr) + + def startContext(self, context): + self._startCapture() + + def stopContext(self, context): + self._endCapture() + + def beforeTest(self, test): + """Initializes a timer before starting a test.""" + self._timer = time() + self._startCapture() + + def _endCapture(self): + if self._capture_stack: + sys.stdout, sys.stderr = self._capture_stack.pop() + + def afterTest(self, test): + self._endCapture() + self._currentStdout = None + self._currentStderr = None + + def finalize(self, test): + while self._capture_stack: + self._endCapture() + + def _getCapturedStdout(self): + if self._currentStdout: + value = self._currentStdout.getvalue() + if value: + return '' % escape_cdata( + value) + return '' + + def _getCapturedStderr(self): + if self._currentStderr: + value = self._currentStderr.getvalue() + if value: + return '' % escape_cdata( + value) + return '' + + def addError(self, test, err, capt=None): + """Add error output to Xunit report. + """ + taken = self._timeTaken() + + if issubclass(err[0], SkipTest): + type = 'skipped' + self.stats['skipped'] += 1 + else: + type = 'error' + self.stats['errors'] += 1 + + tb = format_exception(err, self.encoding) + id = test.id() + + self.errorlist.append( + u'' + u'<%(type)s type=%(errtype)s message=%(message)s>' + u'%(systemout)s%(systemerr)s' % + {'cls': self._quoteattr(id_split(id)[0]), + 'name': self._quoteattr(id_split(id)[-1]), + 'taken': taken, + 'type': type, + 'errtype': self._quoteattr(nice_classname(err[0])), + 'message': self._quoteattr(exc_message(err)), + 'tb': escape_cdata(tb), + 'systemout': self._getCapturedStdout(), + 'systemerr': self._getCapturedStderr(), + }) + + def addFailure(self, test, err, capt=None, tb_info=None): + """Add failure output to Xunit report. + """ + taken = self._timeTaken() + tb = format_exception(err, self.encoding) + self.stats['failures'] += 1 + id = test.id() + + self.errorlist.append( + u'' + u'' + u'%(systemout)s%(systemerr)s' % + {'cls': self._quoteattr(id_split(id)[0]), + 'name': self._quoteattr(id_split(id)[-1]), + 'taken': taken, + 'errtype': self._quoteattr(nice_classname(err[0])), + 'message': self._quoteattr(exc_message(err)), + 'tb': escape_cdata(tb), + 'systemout': self._getCapturedStdout(), + 'systemerr': self._getCapturedStderr(), + }) + + def addSuccess(self, test, capt=None): + """Add success output to Xunit report. + """ + taken = self._timeTaken() + self.stats['passes'] += 1 + id = test.id() + self.errorlist.append( + '%(systemout)s%(systemerr)s' % + {'cls': self._quoteattr(id_split(id)[0]), + 'name': self._quoteattr(id_split(id)[-1]), + 'taken': taken, + 'systemout': self._getCapturedStdout(), + 'systemerr': self._getCapturedStderr(), + }) diff --git a/lib/spack/external/nose/proxy.py b/lib/spack/external/nose/proxy.py new file mode 100644 index 0000000000..c2676cb195 --- /dev/null +++ b/lib/spack/external/nose/proxy.py @@ -0,0 +1,188 @@ +""" +Result Proxy +------------ + +The result proxy wraps the result instance given to each test. It +performs two functions: enabling extended error/failure reporting +and calling plugins. + +As each result event is fired, plugins are called with the same event; +however, plugins are called with the nose.case.Test instance that +wraps the actual test. So when a test fails and calls +result.addFailure(self, err), the result proxy calls +addFailure(self.test, err) for each plugin. This allows plugins to +have a single stable interface for all test types, and also to +manipulate the test object itself by setting the `test` attribute of +the nose.case.Test that they receive. +""" +import logging +from nose.config import Config + + +log = logging.getLogger(__name__) + + +def proxied_attribute(local_attr, proxied_attr, doc): + """Create a property that proxies attribute ``proxied_attr`` through + the local attribute ``local_attr``. + """ + def fget(self): + return getattr(getattr(self, local_attr), proxied_attr) + def fset(self, value): + setattr(getattr(self, local_attr), proxied_attr, value) + def fdel(self): + delattr(getattr(self, local_attr), proxied_attr) + return property(fget, fset, fdel, doc) + + +class ResultProxyFactory(object): + """Factory for result proxies. Generates a ResultProxy bound to each test + and the result passed to the test. + """ + def __init__(self, config=None): + if config is None: + config = Config() + self.config = config + self.__prepared = False + self.__result = None + + def __call__(self, result, test): + """Return a ResultProxy for the current test. + + On first call, plugins are given a chance to replace the + result used for the remaining tests. If a plugin returns a + value from prepareTestResult, that object will be used as the + result for all tests. + """ + if not self.__prepared: + self.__prepared = True + plug_result = self.config.plugins.prepareTestResult(result) + if plug_result is not None: + self.__result = result = plug_result + if self.__result is not None: + result = self.__result + return ResultProxy(result, test, config=self.config) + + +class ResultProxy(object): + """Proxy to TestResults (or other results handler). + + One ResultProxy is created for each nose.case.Test. The result + proxy calls plugins with the nose.case.Test instance (instead of + the wrapped test case) as each result call is made. Finally, the + real result method is called, also with the nose.case.Test + instance as the test parameter. + + """ + def __init__(self, result, test, config=None): + if config is None: + config = Config() + self.config = config + self.plugins = config.plugins + self.result = result + self.test = test + + def __repr__(self): + return repr(self.result) + + def _prepareErr(self, err): + if not isinstance(err[1], Exception) and isinstance(err[0], type): + # Turn value back into an Exception (required in Python 3.x). + # Plugins do all sorts of crazy things with exception values. + # Convert it to a custom subclass of Exception with the same + # name as the actual exception to make it print correctly. + value = type(err[0].__name__, (Exception,), {})(err[1]) + err = (err[0], value, err[2]) + return err + + def assertMyTest(self, test): + # The test I was called with must be my .test or my + # .test's .test. or my .test.test's .case + + case = getattr(self.test, 'test', None) + assert (test is self.test + or test is case + or test is getattr(case, '_nose_case', None)), ( + "ResultProxy for %r (%s) was called with test %r (%s)" + % (self.test, id(self.test), test, id(test))) + + def afterTest(self, test): + self.assertMyTest(test) + self.plugins.afterTest(self.test) + if hasattr(self.result, "afterTest"): + self.result.afterTest(self.test) + + def beforeTest(self, test): + self.assertMyTest(test) + self.plugins.beforeTest(self.test) + if hasattr(self.result, "beforeTest"): + self.result.beforeTest(self.test) + + def addError(self, test, err): + self.assertMyTest(test) + plugins = self.plugins + plugin_handled = plugins.handleError(self.test, err) + if plugin_handled: + return + # test.passed is set in result, to account for error classes + formatted = plugins.formatError(self.test, err) + if formatted is not None: + err = formatted + plugins.addError(self.test, err) + self.result.addError(self.test, self._prepareErr(err)) + if not self.result.wasSuccessful() and self.config.stopOnError: + self.shouldStop = True + + def addFailure(self, test, err): + self.assertMyTest(test) + plugins = self.plugins + plugin_handled = plugins.handleFailure(self.test, err) + if plugin_handled: + return + self.test.passed = False + formatted = plugins.formatFailure(self.test, err) + if formatted is not None: + err = formatted + plugins.addFailure(self.test, err) + self.result.addFailure(self.test, self._prepareErr(err)) + if self.config.stopOnError: + self.shouldStop = True + + def addSkip(self, test, reason): + # 2.7 compat shim + from nose.plugins.skip import SkipTest + self.assertMyTest(test) + plugins = self.plugins + if not isinstance(reason, Exception): + # for Python 3.2+ + reason = Exception(reason) + plugins.addError(self.test, (SkipTest, reason, None)) + self.result.addSkip(self.test, reason) + + def addSuccess(self, test): + self.assertMyTest(test) + self.plugins.addSuccess(self.test) + self.result.addSuccess(self.test) + + def startTest(self, test): + self.assertMyTest(test) + self.plugins.startTest(self.test) + self.result.startTest(self.test) + + def stop(self): + self.result.stop() + + def stopTest(self, test): + self.assertMyTest(test) + self.plugins.stopTest(self.test) + self.result.stopTest(self.test) + + # proxied attributes + shouldStop = proxied_attribute('result', 'shouldStop', + """Should the test run stop?""") + errors = proxied_attribute('result', 'errors', + """Tests that raised an exception""") + failures = proxied_attribute('result', 'failures', + """Tests that failed""") + testsRun = proxied_attribute('result', 'testsRun', + """Number of tests run""") diff --git a/lib/spack/external/nose/pyversion.py b/lib/spack/external/nose/pyversion.py new file mode 100644 index 0000000000..091238da75 --- /dev/null +++ b/lib/spack/external/nose/pyversion.py @@ -0,0 +1,215 @@ +""" +This module contains fixups for using nose under different versions of Python. +""" +import sys +import os +import traceback +import types +import inspect +import nose.util + +__all__ = ['make_instancemethod', 'cmp_to_key', 'sort_list', 'ClassType', + 'TypeType', 'UNICODE_STRINGS', 'unbound_method', 'ismethod', + 'bytes_', 'is_base_exception', 'force_unicode', 'exc_to_unicode', + 'format_exception'] + +# In Python 3.x, all strings are unicode (the call to 'unicode()' in the 2.x +# source will be replaced with 'str()' when running 2to3, so this test will +# then become true) +UNICODE_STRINGS = (type(unicode()) == type(str())) + +if sys.version_info[:2] < (3, 0): + def force_unicode(s, encoding='UTF-8'): + try: + s = unicode(s) + except UnicodeDecodeError: + s = str(s).decode(encoding, 'replace') + + return s +else: + def force_unicode(s, encoding='UTF-8'): + return str(s) + +# new.instancemethod() is obsolete for new-style classes (Python 3.x) +# We need to use descriptor methods instead. +try: + import new + def make_instancemethod(function, instance): + return new.instancemethod(function.im_func, instance, + instance.__class__) +except ImportError: + def make_instancemethod(function, instance): + return function.__get__(instance, instance.__class__) + +# To be forward-compatible, we do all list sorts using keys instead of cmp +# functions. However, part of the unittest.TestLoader API involves a +# user-provideable cmp function, so we need some way to convert that. +def cmp_to_key(mycmp): + 'Convert a cmp= function into a key= function' + class Key(object): + def __init__(self, obj): + self.obj = obj + def __lt__(self, other): + return mycmp(self.obj, other.obj) < 0 + def __gt__(self, other): + return mycmp(self.obj, other.obj) > 0 + def __eq__(self, other): + return mycmp(self.obj, other.obj) == 0 + return Key + +# Python 2.3 also does not support list-sorting by key, so we need to convert +# keys to cmp functions if we're running on old Python.. +if sys.version_info < (2, 4): + def sort_list(l, key, reverse=False): + if reverse: + return l.sort(lambda a, b: cmp(key(b), key(a))) + else: + return l.sort(lambda a, b: cmp(key(a), key(b))) +else: + def sort_list(l, key, reverse=False): + return l.sort(key=key, reverse=reverse) + +# In Python 3.x, all objects are "new style" objects descended from 'type', and +# thus types.ClassType and types.TypeType don't exist anymore. For +# compatibility, we make sure they still work. +if hasattr(types, 'ClassType'): + ClassType = types.ClassType + TypeType = types.TypeType +else: + ClassType = type + TypeType = type + +# The following emulates the behavior (we need) of an 'unbound method' under +# Python 3.x (namely, the ability to have a class associated with a function +# definition so that things can do stuff based on its associated class) +class UnboundMethod: + def __init__(self, cls, func): + # Make sure we have all the same attributes as the original function, + # so that the AttributeSelector plugin will work correctly... + self.__dict__ = func.__dict__.copy() + self._func = func + self.__self__ = UnboundSelf(cls) + if sys.version_info < (3, 0): + self.im_class = cls + self.__doc__ = getattr(func, '__doc__', None) + + def address(self): + cls = self.__self__.cls + modname = cls.__module__ + module = sys.modules[modname] + filename = getattr(module, '__file__', None) + if filename is not None: + filename = os.path.abspath(filename) + return (nose.util.src(filename), modname, "%s.%s" % (cls.__name__, + self._func.__name__)) + + def __call__(self, *args, **kwargs): + return self._func(*args, **kwargs) + + def __getattr__(self, attr): + return getattr(self._func, attr) + + def __repr__(self): + return '' % (self.__self__.cls.__name__, + self._func.__name__) + +class UnboundSelf: + def __init__(self, cls): + self.cls = cls + + # We have to do this hackery because Python won't let us override the + # __class__ attribute... + def __getattribute__(self, attr): + if attr == '__class__': + return self.cls + else: + return object.__getattribute__(self, attr) + +def unbound_method(cls, func): + if inspect.ismethod(func): + return func + if not inspect.isfunction(func): + raise TypeError('%s is not a function' % (repr(func),)) + return UnboundMethod(cls, func) + +def ismethod(obj): + return inspect.ismethod(obj) or isinstance(obj, UnboundMethod) + + +# Make a pseudo-bytes function that can be called without the encoding arg: +if sys.version_info >= (3, 0): + def bytes_(s, encoding='utf8'): + if isinstance(s, bytes): + return s + return bytes(s, encoding) +else: + def bytes_(s, encoding=None): + return str(s) + + +if sys.version_info[:2] >= (2, 6): + def isgenerator(o): + if isinstance(o, UnboundMethod): + o = o._func + return inspect.isgeneratorfunction(o) or inspect.isgenerator(o) +else: + try: + from compiler.consts import CO_GENERATOR + except ImportError: + # IronPython doesn't have a complier module + CO_GENERATOR=0x20 + + def isgenerator(func): + try: + return func.func_code.co_flags & CO_GENERATOR != 0 + except AttributeError: + return False + +# Make a function to help check if an exception is derived from BaseException. +# In Python 2.4, we just use Exception instead. +if sys.version_info[:2] < (2, 5): + def is_base_exception(exc): + return isinstance(exc, Exception) +else: + def is_base_exception(exc): + return isinstance(exc, BaseException) + +if sys.version_info[:2] < (3, 0): + def exc_to_unicode(ev, encoding='utf-8'): + if is_base_exception(ev): + if not hasattr(ev, '__unicode__'): + # 2.5- + if not hasattr(ev, 'message'): + # 2.4 + msg = len(ev.args) and ev.args[0] or '' + else: + msg = ev.message + msg = force_unicode(msg, encoding=encoding) + clsname = force_unicode(ev.__class__.__name__, + encoding=encoding) + ev = u'%s: %s' % (clsname, msg) + elif not isinstance(ev, unicode): + ev = repr(ev) + + return force_unicode(ev, encoding=encoding) +else: + def exc_to_unicode(ev, encoding='utf-8'): + return str(ev) + +def format_exception(exc_info, encoding='UTF-8'): + ec, ev, tb = exc_info + + # Our exception object may have been turned into a string, and Python 3's + # traceback.format_exception() doesn't take kindly to that (it expects an + # actual exception object). So we work around it, by doing the work + # ourselves if ev is not an exception object. + if not is_base_exception(ev): + tb_data = force_unicode( + ''.join(traceback.format_tb(tb)), + encoding) + ev = exc_to_unicode(ev) + return tb_data + ev + else: + return force_unicode( + ''.join(traceback.format_exception(*exc_info)), + encoding) diff --git a/lib/spack/external/nose/result.py b/lib/spack/external/nose/result.py new file mode 100644 index 0000000000..f974a14ae2 --- /dev/null +++ b/lib/spack/external/nose/result.py @@ -0,0 +1,200 @@ +""" +Test Result +----------- + +Provides a TextTestResult that extends unittest's _TextTestResult to +provide support for error classes (such as the builtin skip and +deprecated classes), and hooks for plugins to take over or extend +reporting. +""" + +import logging +try: + # 2.7+ + from unittest.runner import _TextTestResult +except ImportError: + from unittest import _TextTestResult +from nose.config import Config +from nose.util import isclass, ln as _ln # backwards compat + +log = logging.getLogger('nose.result') + + +def _exception_detail(exc): + # this is what stdlib module traceback does + try: + return str(exc) + except: + return '' % type(exc).__name__ + + +class TextTestResult(_TextTestResult): + """Text test result that extends unittest's default test result + support for a configurable set of errorClasses (eg, Skip, + Deprecated, TODO) that extend the errors/failures/success triad. + """ + def __init__(self, stream, descriptions, verbosity, config=None, + errorClasses=None): + if errorClasses is None: + errorClasses = {} + self.errorClasses = errorClasses + if config is None: + config = Config() + self.config = config + _TextTestResult.__init__(self, stream, descriptions, verbosity) + + def addSkip(self, test, reason): + # 2.7 skip compat + from nose.plugins.skip import SkipTest + if SkipTest in self.errorClasses: + storage, label, isfail = self.errorClasses[SkipTest] + storage.append((test, reason)) + self.printLabel(label, (SkipTest, reason, None)) + + def addError(self, test, err): + """Overrides normal addError to add support for + errorClasses. If the exception is a registered class, the + error will be added to the list for that class, not errors. + """ + ec, ev, tb = err + try: + exc_info = self._exc_info_to_string(err, test) + except TypeError: + # 2.3 compat + exc_info = self._exc_info_to_string(err) + for cls, (storage, label, isfail) in self.errorClasses.items(): + #if 'Skip' in cls.__name__ or 'Skip' in ec.__name__: + # from nose.tools import set_trace + # set_trace() + if isclass(ec) and issubclass(ec, cls): + if isfail: + test.passed = False + storage.append((test, exc_info)) + self.printLabel(label, err) + return + self.errors.append((test, exc_info)) + test.passed = False + self.printLabel('ERROR') + + # override to bypass changes in 2.7 + def getDescription(self, test): + if self.descriptions: + return test.shortDescription() or str(test) + else: + return str(test) + + def printLabel(self, label, err=None): + # Might get patched into a streamless result + stream = getattr(self, 'stream', None) + if stream is not None: + if self.showAll: + message = [label] + if err: + detail = _exception_detail(err[1]) + if detail: + message.append(detail) + stream.writeln(": ".join(message)) + elif self.dots: + stream.write(label[:1]) + + def printErrors(self): + """Overrides to print all errorClasses errors as well. + """ + _TextTestResult.printErrors(self) + for cls in self.errorClasses.keys(): + storage, label, isfail = self.errorClasses[cls] + if isfail: + self.printErrorList(label, storage) + # Might get patched into a result with no config + if hasattr(self, 'config'): + self.config.plugins.report(self.stream) + + def printSummary(self, start, stop): + """Called by the test runner to print the final summary of test + run results. + """ + write = self.stream.write + writeln = self.stream.writeln + taken = float(stop - start) + run = self.testsRun + plural = run != 1 and "s" or "" + + writeln(self.separator2) + writeln("Ran %s test%s in %.3fs" % (run, plural, taken)) + writeln() + + summary = {} + eckeys = self.errorClasses.keys() + for cls in eckeys: + storage, label, isfail = self.errorClasses[cls] + count = len(storage) + if not count: + continue + summary[label] = count + if len(self.failures): + summary['failures'] = len(self.failures) + if len(self.errors): + summary['errors'] = len(self.errors) + + if not self.wasSuccessful(): + write("FAILED") + else: + write("OK") + items = summary.items() + if items: + items.sort() + write(" (") + write(", ".join(["%s=%s" % (label, count) for + label, count in items])) + writeln(")") + else: + writeln() + + def wasSuccessful(self): + """Overrides to check that there are no errors in errorClasses + lists that are marked as errors and should cause a run to + fail. + """ + if self.errors or self.failures: + return False + for cls in self.errorClasses.keys(): + storage, label, isfail = self.errorClasses[cls] + if not isfail: + continue + if storage: + return False + return True + + def _addError(self, test, err): + try: + exc_info = self._exc_info_to_string(err, test) + except TypeError: + # 2.3: does not take test arg + exc_info = self._exc_info_to_string(err) + self.errors.append((test, exc_info)) + if self.showAll: + self.stream.write('ERROR') + elif self.dots: + self.stream.write('E') + + def _exc_info_to_string(self, err, test=None): + # 2.7 skip compat + from nose.plugins.skip import SkipTest + if isclass(err[0]) and issubclass(err[0], SkipTest): + return str(err[1]) + # 2.3/2.4 -- 2.4 passes test, 2.3 does not + try: + return _TextTestResult._exc_info_to_string(self, err, test) + except TypeError: + # 2.3: does not take test arg + return _TextTestResult._exc_info_to_string(self, err) + + +def ln(*arg, **kw): + from warnings import warn + warn("ln() has moved to nose.util from nose.result and will be removed " + "from nose.result in a future release. Please update your imports ", + DeprecationWarning) + return _ln(*arg, **kw) + + diff --git a/lib/spack/external/nose/selector.py b/lib/spack/external/nose/selector.py new file mode 100644 index 0000000000..b63f7af0b1 --- /dev/null +++ b/lib/spack/external/nose/selector.py @@ -0,0 +1,251 @@ +""" +Test Selection +-------------- + +Test selection is handled by a Selector. The test loader calls the +appropriate selector method for each object it encounters that it +thinks may be a test. +""" +import logging +import os +import unittest +from nose.config import Config +from nose.util import split_test_name, src, getfilename, getpackage, ispackage, is_executable + +log = logging.getLogger(__name__) + +__all__ = ['Selector', 'defaultSelector', 'TestAddress'] + + +# for efficiency and easier mocking +op_join = os.path.join +op_basename = os.path.basename +op_exists = os.path.exists +op_splitext = os.path.splitext +op_isabs = os.path.isabs +op_abspath = os.path.abspath + + +class Selector(object): + """Core test selector. Examines test candidates and determines whether, + given the specified configuration, the test candidate should be selected + as a test. + """ + def __init__(self, config): + if config is None: + config = Config() + self.configure(config) + + def configure(self, config): + self.config = config + self.exclude = config.exclude + self.ignoreFiles = config.ignoreFiles + self.include = config.include + self.plugins = config.plugins + self.match = config.testMatch + + def matches(self, name): + """Does the name match my requirements? + + To match, a name must match config.testMatch OR config.include + and it must not match config.exclude + """ + return ((self.match.search(name) + or (self.include and + filter(None, + [inc.search(name) for inc in self.include]))) + and ((not self.exclude) + or not filter(None, + [exc.search(name) for exc in self.exclude]) + )) + + def wantClass(self, cls): + """Is the class a wanted test class? + + A class must be a unittest.TestCase subclass, or match test name + requirements. Classes that start with _ are always excluded. + """ + declared = getattr(cls, '__test__', None) + if declared is not None: + wanted = declared + else: + wanted = (not cls.__name__.startswith('_') + and (issubclass(cls, unittest.TestCase) + or self.matches(cls.__name__))) + + plug_wants = self.plugins.wantClass(cls) + if plug_wants is not None: + log.debug("Plugin setting selection of %s to %s", cls, plug_wants) + wanted = plug_wants + log.debug("wantClass %s? %s", cls, wanted) + return wanted + + def wantDirectory(self, dirname): + """Is the directory a wanted test directory? + + All package directories match, so long as they do not match exclude. + All other directories must match test requirements. + """ + tail = op_basename(dirname) + if ispackage(dirname): + wanted = (not self.exclude + or not filter(None, + [exc.search(tail) for exc in self.exclude] + )) + else: + wanted = (self.matches(tail) + or (self.config.srcDirs + and tail in self.config.srcDirs)) + plug_wants = self.plugins.wantDirectory(dirname) + if plug_wants is not None: + log.debug("Plugin setting selection of %s to %s", + dirname, plug_wants) + wanted = plug_wants + log.debug("wantDirectory %s? %s", dirname, wanted) + return wanted + + def wantFile(self, file): + """Is the file a wanted test file? + + The file must be a python source file and match testMatch or + include, and not match exclude. Files that match ignore are *never* + wanted, regardless of plugin, testMatch, include or exclude settings. + """ + # never, ever load files that match anything in ignore + # (.* _* and *setup*.py by default) + base = op_basename(file) + ignore_matches = [ ignore_this for ignore_this in self.ignoreFiles + if ignore_this.search(base) ] + if ignore_matches: + log.debug('%s matches ignoreFiles pattern; skipped', + base) + return False + if not self.config.includeExe and is_executable(file): + log.info('%s is executable; skipped', file) + return False + dummy, ext = op_splitext(base) + pysrc = ext == '.py' + + wanted = pysrc and self.matches(base) + plug_wants = self.plugins.wantFile(file) + if plug_wants is not None: + log.debug("plugin setting want %s to %s", file, plug_wants) + wanted = plug_wants + log.debug("wantFile %s? %s", file, wanted) + return wanted + + def wantFunction(self, function): + """Is the function a test function? + """ + try: + if hasattr(function, 'compat_func_name'): + funcname = function.compat_func_name + else: + funcname = function.__name__ + except AttributeError: + # not a function + return False + declared = getattr(function, '__test__', None) + if declared is not None: + wanted = declared + else: + wanted = not funcname.startswith('_') and self.matches(funcname) + plug_wants = self.plugins.wantFunction(function) + if plug_wants is not None: + wanted = plug_wants + log.debug("wantFunction %s? %s", function, wanted) + return wanted + + def wantMethod(self, method): + """Is the method a test method? + """ + try: + method_name = method.__name__ + except AttributeError: + # not a method + return False + if method_name.startswith('_'): + # never collect 'private' methods + return False + declared = getattr(method, '__test__', None) + if declared is not None: + wanted = declared + else: + wanted = self.matches(method_name) + plug_wants = self.plugins.wantMethod(method) + if plug_wants is not None: + wanted = plug_wants + log.debug("wantMethod %s? %s", method, wanted) + return wanted + + def wantModule(self, module): + """Is the module a test module? + + The tail of the module name must match test requirements. One exception: + we always want __main__. + """ + declared = getattr(module, '__test__', None) + if declared is not None: + wanted = declared + else: + wanted = self.matches(module.__name__.split('.')[-1]) \ + or module.__name__ == '__main__' + plug_wants = self.plugins.wantModule(module) + if plug_wants is not None: + wanted = plug_wants + log.debug("wantModule %s? %s", module, wanted) + return wanted + +defaultSelector = Selector + + +class TestAddress(object): + """A test address represents a user's request to run a particular + test. The user may specify a filename or module (or neither), + and/or a callable (a class, function, or method). The naming + format for test addresses is: + + filename_or_module:callable + + Filenames that are not absolute will be made absolute relative to + the working dir. + + The filename or module part will be considered a module name if it + doesn't look like a file, that is, if it doesn't exist on the file + system and it doesn't contain any directory separators and it + doesn't end in .py. + + Callables may be a class name, function name, method name, or + class.method specification. + """ + def __init__(self, name, workingDir=None): + if workingDir is None: + workingDir = os.getcwd() + self.name = name + self.workingDir = workingDir + self.filename, self.module, self.call = split_test_name(name) + log.debug('Test name %s resolved to file %s, module %s, call %s', + name, self.filename, self.module, self.call) + if self.filename is None: + if self.module is not None: + self.filename = getfilename(self.module, self.workingDir) + if self.filename: + self.filename = src(self.filename) + if not op_isabs(self.filename): + self.filename = op_abspath(op_join(workingDir, + self.filename)) + if self.module is None: + self.module = getpackage(self.filename) + log.debug( + 'Final resolution of test name %s: file %s module %s call %s', + name, self.filename, self.module, self.call) + + def totuple(self): + return (self.filename, self.module, self.call) + + def __str__(self): + return self.name + + def __repr__(self): + return "%s: (%s, %s, %s)" % (self.name, self.filename, + self.module, self.call) diff --git a/lib/spack/external/nose/sphinx/__init__.py b/lib/spack/external/nose/sphinx/__init__.py new file mode 100644 index 0000000000..2ae28399f5 --- /dev/null +++ b/lib/spack/external/nose/sphinx/__init__.py @@ -0,0 +1 @@ +pass diff --git a/lib/spack/external/nose/sphinx/pluginopts.py b/lib/spack/external/nose/sphinx/pluginopts.py new file mode 100644 index 0000000000..d2b284ab27 --- /dev/null +++ b/lib/spack/external/nose/sphinx/pluginopts.py @@ -0,0 +1,189 @@ +""" +Adds a sphinx directive that can be used to automatically document a plugin. + +this:: + + .. autoplugin :: nose.plugins.foo + :plugin: Pluggy + +produces:: + + .. automodule :: nose.plugins.foo + + Options + ------- + + .. cmdoption :: --foo=BAR, --fooble=BAR + + Do the foo thing to the new thing. + + Plugin + ------ + + .. autoclass :: nose.plugins.foo.Pluggy + :members: + + Source + ------ + + .. include :: path/to/nose/plugins/foo.py + :literal: + +""" +import os +try: + from docutils import nodes, utils + from docutils.statemachine import ViewList + from docutils.parsers.rst import directives +except ImportError: + pass # won't run anyway + +from nose.util import resolve_name +from nose.plugins.base import Plugin +from nose.plugins.manager import BuiltinPluginManager +from nose.config import Config +from nose.core import TestProgram +from inspect import isclass + + +def autoplugin_directive(dirname, arguments, options, content, lineno, + content_offset, block_text, state, state_machine): + mod_name = arguments[0] + mod = resolve_name(mod_name) + plug_name = options.get('plugin', None) + if plug_name: + obj = getattr(mod, plug_name) + else: + for entry in dir(mod): + obj = getattr(mod, entry) + if isclass(obj) and issubclass(obj, Plugin) and obj is not Plugin: + plug_name = '%s.%s' % (mod_name, entry) + break + + # mod docstring + rst = ViewList() + rst.append('.. automodule :: %s\n' % mod_name, '') + rst.append('', '') + + # options + rst.append('Options', '') + rst.append('-------', '') + rst.append('', '') + + plug = obj() + opts = OptBucket() + plug.options(opts, {}) + for opt in opts: + rst.append(opt.options(), '') + rst.append(' \n', '') + rst.append(' ' + opt.help + '\n', '') + rst.append('\n', '') + + # plugin class + rst.append('Plugin', '') + rst.append('------', '') + rst.append('', '') + + rst.append('.. autoclass :: %s\n' % plug_name, '') + rst.append(' :members:\n', '') + rst.append(' :show-inheritance:\n', '') + rst.append('', '') + + # source + rst.append('Source', '') + rst.append('------', '') + rst.append( + '.. include :: %s\n' % utils.relative_path( + state_machine.document['source'], + os.path.abspath(mod.__file__.replace('.pyc', '.py'))), + '') + rst.append(' :literal:\n', '') + rst.append('', '') + + node = nodes.section() + node.document = state.document + surrounding_title_styles = state.memo.title_styles + surrounding_section_level = state.memo.section_level + state.memo.title_styles = [] + state.memo.section_level = 0 + state.nested_parse(rst, 0, node, match_titles=1) + state.memo.title_styles = surrounding_title_styles + state.memo.section_level = surrounding_section_level + + return node.children + + +def autohelp_directive(dirname, arguments, options, content, lineno, + content_offset, block_text, state, state_machine): + """produces rst from nose help""" + config = Config(parserClass=OptBucket, + plugins=BuiltinPluginManager()) + parser = config.getParser(TestProgram.usage()) + rst = ViewList() + for line in parser.format_help().split('\n'): + rst.append(line, '') + + rst.append('Options', '') + rst.append('-------', '') + rst.append('', '') + for opt in parser: + rst.append(opt.options(), '') + rst.append(' \n', '') + rst.append(' ' + opt.help + '\n', '') + rst.append('\n', '') + node = nodes.section() + node.document = state.document + surrounding_title_styles = state.memo.title_styles + surrounding_section_level = state.memo.section_level + state.memo.title_styles = [] + state.memo.section_level = 0 + state.nested_parse(rst, 0, node, match_titles=1) + state.memo.title_styles = surrounding_title_styles + state.memo.section_level = surrounding_section_level + + return node.children + + +class OptBucket(object): + def __init__(self, doc=None, prog='nosetests'): + self.opts = [] + self.doc = doc + self.prog = prog + + def __iter__(self): + return iter(self.opts) + + def format_help(self): + return self.doc.replace('%prog', self.prog).replace(':\n', '::\n') + + def add_option(self, *arg, **kw): + self.opts.append(Opt(*arg, **kw)) + + +class Opt(object): + def __init__(self, *arg, **kw): + self.opts = arg + self.action = kw.pop('action', None) + self.default = kw.pop('default', None) + self.metavar = kw.pop('metavar', None) + self.help = kw.pop('help', None) + + def options(self): + buf = [] + for optstring in self.opts: + desc = optstring + if self.action not in ('store_true', 'store_false'): + desc += '=%s' % self.meta(optstring) + buf.append(desc) + return '.. cmdoption :: ' + ', '.join(buf) + + def meta(self, optstring): + # FIXME optparser default metavar? + return self.metavar or 'DEFAULT' + + +def setup(app): + app.add_directive('autoplugin', + autoplugin_directive, 1, (1, 0, 1), + plugin=directives.unchanged) + app.add_directive('autohelp', autohelp_directive, 0, (0, 0, 1)) diff --git a/lib/spack/external/nose/suite.py b/lib/spack/external/nose/suite.py new file mode 100644 index 0000000000..a831105e34 --- /dev/null +++ b/lib/spack/external/nose/suite.py @@ -0,0 +1,609 @@ +""" +Test Suites +----------- + +Provides a LazySuite, which is a suite whose test list is a generator +function, and ContextSuite,which can run fixtures (setup/teardown +functions or methods) for the context that contains its tests. + +""" +from __future__ import generators + +import logging +import sys +import unittest +from nose.case import Test +from nose.config import Config +from nose.proxy import ResultProxyFactory +from nose.util import isclass, resolve_name, try_run + +if sys.platform == 'cli': + if sys.version_info[:2] < (2, 6): + import clr + clr.AddReference("IronPython") + from IronPython.Runtime.Exceptions import StringException + else: + class StringException(Exception): + pass + +log = logging.getLogger(__name__) +#log.setLevel(logging.DEBUG) + +# Singleton for default value -- see ContextSuite.__init__ below +_def = object() + + +def _strclass(cls): + return "%s.%s" % (cls.__module__, cls.__name__) + +class MixedContextError(Exception): + """Error raised when a context suite sees tests from more than + one context. + """ + pass + + +class LazySuite(unittest.TestSuite): + """A suite that may use a generator as its list of tests + """ + def __init__(self, tests=()): + """Initialize the suite. tests may be an iterable or a generator + """ + super(LazySuite, self).__init__() + self._set_tests(tests) + + def __iter__(self): + return iter(self._tests) + + def __repr__(self): + return "<%s tests=generator (%s)>" % ( + _strclass(self.__class__), id(self)) + + def __hash__(self): + return object.__hash__(self) + + __str__ = __repr__ + + def addTest(self, test): + self._precache.append(test) + + # added to bypass run changes in 2.7's unittest + def run(self, result): + for test in self._tests: + if result.shouldStop: + break + test(result) + return result + + def __nonzero__(self): + log.debug("tests in %s?", id(self)) + if self._precache: + return True + if self.test_generator is None: + return False + try: + test = self.test_generator.next() + if test is not None: + self._precache.append(test) + return True + except StopIteration: + pass + return False + + def _get_tests(self): + log.debug("precache is %s", self._precache) + for test in self._precache: + yield test + if self.test_generator is None: + return + for test in self.test_generator: + yield test + + def _set_tests(self, tests): + self._precache = [] + is_suite = isinstance(tests, unittest.TestSuite) + if callable(tests) and not is_suite: + self.test_generator = tests() + elif is_suite: + # Suites need special treatment: they must be called like + # tests for their setup/teardown to run (if any) + self.addTests([tests]) + self.test_generator = None + else: + self.addTests(tests) + self.test_generator = None + + _tests = property(_get_tests, _set_tests, None, + "Access the tests in this suite. Access is through a " + "generator, so iteration may not be repeatable.") + + +class ContextSuite(LazySuite): + """A suite with context. + + A ContextSuite executes fixtures (setup and teardown functions or + methods) for the context containing its tests. + + The context may be explicitly passed. If it is not, a context (or + nested set of contexts) will be constructed by examining the tests + in the suite. + """ + failureException = unittest.TestCase.failureException + was_setup = False + was_torndown = False + classSetup = ('setup_class', 'setup_all', 'setupClass', 'setupAll', + 'setUpClass', 'setUpAll') + classTeardown = ('teardown_class', 'teardown_all', 'teardownClass', + 'teardownAll', 'tearDownClass', 'tearDownAll') + moduleSetup = ('setup_module', 'setupModule', 'setUpModule', 'setup', + 'setUp') + moduleTeardown = ('teardown_module', 'teardownModule', 'tearDownModule', + 'teardown', 'tearDown') + packageSetup = ('setup_package', 'setupPackage', 'setUpPackage') + packageTeardown = ('teardown_package', 'teardownPackage', + 'tearDownPackage') + + def __init__(self, tests=(), context=None, factory=None, + config=None, resultProxy=None, can_split=True): + log.debug("Context suite for %s (%s) (%s)", tests, context, id(self)) + self.context = context + self.factory = factory + if config is None: + config = Config() + self.config = config + self.resultProxy = resultProxy + self.has_run = False + self.can_split = can_split + self.error_context = None + super(ContextSuite, self).__init__(tests) + + def __repr__(self): + return "<%s context=%s>" % ( + _strclass(self.__class__), + getattr(self.context, '__name__', self.context)) + __str__ = __repr__ + + def id(self): + if self.error_context: + return '%s:%s' % (repr(self), self.error_context) + else: + return repr(self) + + def __hash__(self): + return object.__hash__(self) + + # 2.3 compat -- force 2.4 call sequence + def __call__(self, *arg, **kw): + return self.run(*arg, **kw) + + def exc_info(self): + """Hook for replacing error tuple output + """ + return sys.exc_info() + + def _exc_info(self): + """Bottleneck to fix up IronPython string exceptions + """ + e = self.exc_info() + if sys.platform == 'cli': + if isinstance(e[0], StringException): + # IronPython throws these StringExceptions, but + # traceback checks type(etype) == str. Make a real + # string here. + e = (str(e[0]), e[1], e[2]) + + return e + + def run(self, result): + """Run tests in suite inside of suite fixtures. + """ + # proxy the result for myself + log.debug("suite %s (%s) run called, tests: %s", id(self), self, self._tests) + #import pdb + #pdb.set_trace() + if self.resultProxy: + result, orig = self.resultProxy(result, self), result + else: + result, orig = result, result + try: + self.setUp() + except KeyboardInterrupt: + raise + except: + self.error_context = 'setup' + result.addError(self, self._exc_info()) + return + try: + for test in self._tests: + if result.shouldStop: + log.debug("stopping") + break + # each nose.case.Test will create its own result proxy + # so the cases need the original result, to avoid proxy + # chains + test(orig) + finally: + self.has_run = True + try: + self.tearDown() + except KeyboardInterrupt: + raise + except: + self.error_context = 'teardown' + result.addError(self, self._exc_info()) + + def hasFixtures(self, ctx_callback=None): + context = self.context + if context is None: + return False + if self.implementsAnyFixture(context, ctx_callback=ctx_callback): + return True + # My context doesn't have any, but its ancestors might + factory = self.factory + if factory: + ancestors = factory.context.get(self, []) + for ancestor in ancestors: + if self.implementsAnyFixture( + ancestor, ctx_callback=ctx_callback): + return True + return False + + def implementsAnyFixture(self, context, ctx_callback): + if isclass(context): + names = self.classSetup + self.classTeardown + else: + names = self.moduleSetup + self.moduleTeardown + if hasattr(context, '__path__'): + names += self.packageSetup + self.packageTeardown + # If my context has any fixture attribute, I have fixtures + fixt = False + for m in names: + if hasattr(context, m): + fixt = True + break + if ctx_callback is None: + return fixt + return ctx_callback(context, fixt) + + def setUp(self): + log.debug("suite %s setUp called, tests: %s", id(self), self._tests) + if not self: + # I have no tests + log.debug("suite %s has no tests", id(self)) + return + if self.was_setup: + log.debug("suite %s already set up", id(self)) + return + context = self.context + if context is None: + return + # before running my own context's setup, I need to + # ask the factory if my context's contexts' setups have been run + factory = self.factory + if factory: + # get a copy, since we'll be destroying it as we go + ancestors = factory.context.get(self, [])[:] + while ancestors: + ancestor = ancestors.pop() + log.debug("ancestor %s may need setup", ancestor) + if ancestor in factory.was_setup: + continue + log.debug("ancestor %s does need setup", ancestor) + self.setupContext(ancestor) + if not context in factory.was_setup: + self.setupContext(context) + else: + self.setupContext(context) + self.was_setup = True + log.debug("completed suite setup") + + def setupContext(self, context): + self.config.plugins.startContext(context) + log.debug("%s setup context %s", self, context) + if self.factory: + if context in self.factory.was_setup: + return + # note that I ran the setup for this context, so that I'll run + # the teardown in my teardown + self.factory.was_setup[context] = self + if isclass(context): + names = self.classSetup + else: + names = self.moduleSetup + if hasattr(context, '__path__'): + names = self.packageSetup + names + try_run(context, names) + + def shortDescription(self): + if self.context is None: + return "test suite" + return "test suite for %s" % self.context + + def tearDown(self): + log.debug('context teardown') + if not self.was_setup or self.was_torndown: + log.debug( + "No reason to teardown (was_setup? %s was_torndown? %s)" + % (self.was_setup, self.was_torndown)) + return + self.was_torndown = True + context = self.context + if context is None: + log.debug("No context to tear down") + return + + # for each ancestor... if the ancestor was setup + # and I did the setup, I can do teardown + factory = self.factory + if factory: + ancestors = factory.context.get(self, []) + [context] + for ancestor in ancestors: + log.debug('ancestor %s may need teardown', ancestor) + if not ancestor in factory.was_setup: + log.debug('ancestor %s was not setup', ancestor) + continue + if ancestor in factory.was_torndown: + log.debug('ancestor %s already torn down', ancestor) + continue + setup = factory.was_setup[ancestor] + log.debug("%s setup ancestor %s", setup, ancestor) + if setup is self: + self.teardownContext(ancestor) + else: + self.teardownContext(context) + + def teardownContext(self, context): + log.debug("%s teardown context %s", self, context) + if self.factory: + if context in self.factory.was_torndown: + return + self.factory.was_torndown[context] = self + if isclass(context): + names = self.classTeardown + else: + names = self.moduleTeardown + if hasattr(context, '__path__'): + names = self.packageTeardown + names + try_run(context, names) + self.config.plugins.stopContext(context) + + # FIXME the wrapping has to move to the factory? + def _get_wrapped_tests(self): + for test in self._get_tests(): + if isinstance(test, Test) or isinstance(test, unittest.TestSuite): + yield test + else: + yield Test(test, + config=self.config, + resultProxy=self.resultProxy) + + _tests = property(_get_wrapped_tests, LazySuite._set_tests, None, + "Access the tests in this suite. Tests are returned " + "inside of a context wrapper.") + + +class ContextSuiteFactory(object): + """Factory for ContextSuites. Called with a collection of tests, + the factory decides on a hierarchy of contexts by introspecting + the collection or the tests themselves to find the objects + containing the test objects. It always returns one suite, but that + suite may consist of a hierarchy of nested suites. + """ + suiteClass = ContextSuite + def __init__(self, config=None, suiteClass=None, resultProxy=_def): + if config is None: + config = Config() + self.config = config + if suiteClass is not None: + self.suiteClass = suiteClass + # Using a singleton to represent default instead of None allows + # passing resultProxy=None to turn proxying off. + if resultProxy is _def: + resultProxy = ResultProxyFactory(config=config) + self.resultProxy = resultProxy + self.suites = {} + self.context = {} + self.was_setup = {} + self.was_torndown = {} + + def __call__(self, tests, **kw): + """Return ``ContextSuite`` for tests. ``tests`` may either + be a callable (in which case the resulting ContextSuite will + have no parent context and be evaluated lazily) or an + iterable. In that case the tests will wrapped in + nose.case.Test, be examined and the context of each found and a + suite of suites returned, organized into a stack with the + outermost suites belonging to the outermost contexts. + """ + log.debug("Create suite for %s", tests) + context = kw.pop('context', getattr(tests, 'context', None)) + log.debug("tests %s context %s", tests, context) + if context is None: + tests = self.wrapTests(tests) + try: + context = self.findContext(tests) + except MixedContextError: + return self.makeSuite(self.mixedSuites(tests), None, **kw) + return self.makeSuite(tests, context, **kw) + + def ancestry(self, context): + """Return the ancestry of the context (that is, all of the + packages and modules containing the context), in order of + descent with the outermost ancestor last. + This method is a generator. + """ + log.debug("get ancestry %s", context) + if context is None: + return + # Methods include reference to module they are defined in, we + # don't want that, instead want the module the class is in now + # (classes are re-ancestored elsewhere). + if hasattr(context, 'im_class'): + context = context.im_class + elif hasattr(context, '__self__'): + context = context.__self__.__class__ + if hasattr(context, '__module__'): + ancestors = context.__module__.split('.') + elif hasattr(context, '__name__'): + ancestors = context.__name__.split('.')[:-1] + else: + raise TypeError("%s has no ancestors?" % context) + while ancestors: + log.debug(" %s ancestors %s", context, ancestors) + yield resolve_name('.'.join(ancestors)) + ancestors.pop() + + def findContext(self, tests): + if callable(tests) or isinstance(tests, unittest.TestSuite): + return None + context = None + for test in tests: + # Don't look at suites for contexts, only tests + ctx = getattr(test, 'context', None) + if ctx is None: + continue + if context is None: + context = ctx + elif context != ctx: + raise MixedContextError( + "Tests with different contexts in same suite! %s != %s" + % (context, ctx)) + return context + + def makeSuite(self, tests, context, **kw): + suite = self.suiteClass( + tests, context=context, config=self.config, factory=self, + resultProxy=self.resultProxy, **kw) + if context is not None: + self.suites.setdefault(context, []).append(suite) + self.context.setdefault(suite, []).append(context) + log.debug("suite %s has context %s", suite, + getattr(context, '__name__', None)) + for ancestor in self.ancestry(context): + self.suites.setdefault(ancestor, []).append(suite) + self.context[suite].append(ancestor) + log.debug("suite %s has ancestor %s", suite, ancestor.__name__) + return suite + + def mixedSuites(self, tests): + """The complex case where there are tests that don't all share + the same context. Groups tests into suites with common ancestors, + according to the following (essentially tail-recursive) procedure: + + Starting with the context of the first test, if it is not + None, look for tests in the remaining tests that share that + ancestor. If any are found, group into a suite with that + ancestor as the context, and replace the current suite with + that suite. Continue this process for each ancestor of the + first test, until all ancestors have been processed. At this + point if any tests remain, recurse with those tests as the + input, returning a list of the common suite (which may be the + suite or test we started with, if no common tests were found) + plus the results of recursion. + """ + if not tests: + return [] + head = tests.pop(0) + if not tests: + return [head] # short circuit when none are left to combine + suite = head # the common ancestry suite, so far + tail = tests[:] + context = getattr(head, 'context', None) + if context is not None: + ancestors = [context] + [a for a in self.ancestry(context)] + for ancestor in ancestors: + common = [suite] # tests with ancestor in common, so far + remain = [] # tests that remain to be processed + for test in tail: + found_common = False + test_ctx = getattr(test, 'context', None) + if test_ctx is None: + remain.append(test) + continue + if test_ctx is ancestor: + common.append(test) + continue + for test_ancestor in self.ancestry(test_ctx): + if test_ancestor is ancestor: + common.append(test) + found_common = True + break + if not found_common: + remain.append(test) + if common: + suite = self.makeSuite(common, ancestor) + tail = self.mixedSuites(remain) + return [suite] + tail + + def wrapTests(self, tests): + log.debug("wrap %s", tests) + if callable(tests) or isinstance(tests, unittest.TestSuite): + log.debug("I won't wrap") + return tests + wrapped = [] + for test in tests: + log.debug("wrapping %s", test) + if isinstance(test, Test) or isinstance(test, unittest.TestSuite): + wrapped.append(test) + elif isinstance(test, ContextList): + wrapped.append(self.makeSuite(test, context=test.context)) + else: + wrapped.append( + Test(test, config=self.config, resultProxy=self.resultProxy) + ) + return wrapped + + +class ContextList(object): + """Not quite a suite -- a group of tests in a context. This is used + to hint the ContextSuiteFactory about what context the tests + belong to, in cases where it may be ambiguous or missing. + """ + def __init__(self, tests, context=None): + self.tests = tests + self.context = context + + def __iter__(self): + return iter(self.tests) + + +class FinalizingSuiteWrapper(unittest.TestSuite): + """Wraps suite and calls final function after suite has + executed. Used to call final functions in cases (like running in + the standard test runner) where test running is not under nose's + control. + """ + def __init__(self, suite, finalize): + super(FinalizingSuiteWrapper, self).__init__() + self.suite = suite + self.finalize = finalize + + def __call__(self, *arg, **kw): + return self.run(*arg, **kw) + + # 2.7 compat + def __iter__(self): + return iter(self.suite) + + def run(self, *arg, **kw): + try: + return self.suite(*arg, **kw) + finally: + self.finalize(*arg, **kw) + + +# backwards compat -- sort of +class TestDir: + def __init__(*arg, **kw): + raise NotImplementedError( + "TestDir is not usable with nose 0.10. The class is present " + "in nose.suite for backwards compatibility purposes but it " + "may not be used.") + + +class TestModule: + def __init__(*arg, **kw): + raise NotImplementedError( + "TestModule is not usable with nose 0.10. The class is present " + "in nose.suite for backwards compatibility purposes but it " + "may not be used.") diff --git a/lib/spack/external/nose/tools/__init__.py b/lib/spack/external/nose/tools/__init__.py new file mode 100644 index 0000000000..74dab16a74 --- /dev/null +++ b/lib/spack/external/nose/tools/__init__.py @@ -0,0 +1,15 @@ +""" +Tools for testing +----------------- + +nose.tools provides a few convenience functions to make writing tests +easier. You don't have to use them; nothing in the rest of nose depends +on any of these methods. + +""" +from nose.tools.nontrivial import * +from nose.tools.nontrivial import __all__ as nontrivial_all +from nose.tools.trivial import * +from nose.tools.trivial import __all__ as trivial_all + +__all__ = trivial_all + nontrivial_all diff --git a/lib/spack/external/nose/tools/nontrivial.py b/lib/spack/external/nose/tools/nontrivial.py new file mode 100644 index 0000000000..283973245b --- /dev/null +++ b/lib/spack/external/nose/tools/nontrivial.py @@ -0,0 +1,151 @@ +"""Tools not exempt from being descended into in tracebacks""" + +import time + + +__all__ = ['make_decorator', 'raises', 'set_trace', 'timed', 'with_setup', + 'TimeExpired', 'istest', 'nottest'] + + +class TimeExpired(AssertionError): + pass + + +def make_decorator(func): + """ + Wraps a test decorator so as to properly replicate metadata + of the decorated function, including nose's additional stuff + (namely, setup and teardown). + """ + def decorate(newfunc): + if hasattr(func, 'compat_func_name'): + name = func.compat_func_name + else: + name = func.__name__ + newfunc.__dict__ = func.__dict__ + newfunc.__doc__ = func.__doc__ + newfunc.__module__ = func.__module__ + if not hasattr(newfunc, 'compat_co_firstlineno'): + newfunc.compat_co_firstlineno = func.func_code.co_firstlineno + try: + newfunc.__name__ = name + except TypeError: + # can't set func name in 2.3 + newfunc.compat_func_name = name + return newfunc + return decorate + + +def raises(*exceptions): + """Test must raise one of expected exceptions to pass. + + Example use:: + + @raises(TypeError, ValueError) + def test_raises_type_error(): + raise TypeError("This test passes") + + @raises(Exception) + def test_that_fails_by_passing(): + pass + + If you want to test many assertions about exceptions in a single test, + you may want to use `assert_raises` instead. + """ + valid = ' or '.join([e.__name__ for e in exceptions]) + def decorate(func): + name = func.__name__ + def newfunc(*arg, **kw): + try: + func(*arg, **kw) + except exceptions: + pass + except: + raise + else: + message = "%s() did not raise %s" % (name, valid) + raise AssertionError(message) + newfunc = make_decorator(func)(newfunc) + return newfunc + return decorate + + +def set_trace(): + """Call pdb.set_trace in the calling frame, first restoring + sys.stdout to the real output stream. Note that sys.stdout is NOT + reset to whatever it was before the call once pdb is done! + """ + import pdb + import sys + stdout = sys.stdout + sys.stdout = sys.__stdout__ + pdb.Pdb().set_trace(sys._getframe().f_back) + + +def timed(limit): + """Test must finish within specified time limit to pass. + + Example use:: + + @timed(.1) + def test_that_fails(): + time.sleep(.2) + """ + def decorate(func): + def newfunc(*arg, **kw): + start = time.time() + result = func(*arg, **kw) + end = time.time() + if end - start > limit: + raise TimeExpired("Time limit (%s) exceeded" % limit) + return result + newfunc = make_decorator(func)(newfunc) + return newfunc + return decorate + + +def with_setup(setup=None, teardown=None): + """Decorator to add setup and/or teardown methods to a test function:: + + @with_setup(setup, teardown) + def test_something(): + " ... " + + Note that `with_setup` is useful *only* for test functions, not for test + methods or inside of TestCase subclasses. + """ + def decorate(func, setup=setup, teardown=teardown): + if setup: + if hasattr(func, 'setup'): + _old_s = func.setup + def _s(): + setup() + _old_s() + func.setup = _s + else: + func.setup = setup + if teardown: + if hasattr(func, 'teardown'): + _old_t = func.teardown + def _t(): + _old_t() + teardown() + func.teardown = _t + else: + func.teardown = teardown + return func + return decorate + + +def istest(func): + """Decorator to mark a function or method as a test + """ + func.__test__ = True + return func + + +def nottest(func): + """Decorator to mark a function or method as *not* a test + """ + func.__test__ = False + return func diff --git a/lib/spack/external/nose/tools/trivial.py b/lib/spack/external/nose/tools/trivial.py new file mode 100644 index 0000000000..cf83efeda5 --- /dev/null +++ b/lib/spack/external/nose/tools/trivial.py @@ -0,0 +1,54 @@ +"""Tools so trivial that tracebacks should not descend into them + +We define the ``__unittest`` symbol in their module namespace so unittest will +skip them when printing tracebacks, just as it does for their corresponding +methods in ``unittest`` proper. + +""" +import re +import unittest + + +__all__ = ['ok_', 'eq_'] + +# Use the same flag as unittest itself to prevent descent into these functions: +__unittest = 1 + + +def ok_(expr, msg=None): + """Shorthand for assert. Saves 3 whole characters! + """ + if not expr: + raise AssertionError(msg) + + +def eq_(a, b, msg=None): + """Shorthand for 'assert a == b, "%r != %r" % (a, b) + """ + if not a == b: + raise AssertionError(msg or "%r != %r" % (a, b)) + + +# +# Expose assert* from unittest.TestCase +# - give them pep8 style names +# +caps = re.compile('([A-Z])') + +def pep8(name): + return caps.sub(lambda m: '_' + m.groups()[0].lower(), name) + +class Dummy(unittest.TestCase): + def nop(): + pass +_t = Dummy('nop') + +for at in [ at for at in dir(_t) + if at.startswith('assert') and not '_' in at ]: + pepd = pep8(at) + vars()[pepd] = getattr(_t, at) + __all__.append(pepd) + +del Dummy +del _t +del pep8 diff --git a/lib/spack/external/nose/twistedtools.py b/lib/spack/external/nose/twistedtools.py new file mode 100644 index 0000000000..8d9c6ffe9b --- /dev/null +++ b/lib/spack/external/nose/twistedtools.py @@ -0,0 +1,173 @@ +""" +Twisted integration +------------------- + +This module provides a very simple way to integrate your tests with the +Twisted_ event loop. + +You must import this module *before* importing anything from Twisted itself! + +Example:: + + from nose.twistedtools import reactor, deferred + + @deferred() + def test_resolve(): + return reactor.resolve("www.python.org") + +Or, more realistically:: + + @deferred(timeout=5.0) + def test_resolve(): + d = reactor.resolve("www.python.org") + def check_ip(ip): + assert ip == "67.15.36.43" + d.addCallback(check_ip) + return d + +.. _Twisted: http://twistedmatrix.com/trac/ +""" + +import sys +from Queue import Queue, Empty +from nose.tools import make_decorator, TimeExpired + +__all__ = [ + 'threaded_reactor', 'reactor', 'deferred', 'TimeExpired', + 'stop_reactor' +] + +_twisted_thread = None + +def threaded_reactor(): + """ + Start the Twisted reactor in a separate thread, if not already done. + Returns the reactor. + The thread will automatically be destroyed when all the tests are done. + """ + global _twisted_thread + try: + from twisted.internet import reactor + except ImportError: + return None, None + if not _twisted_thread: + from twisted.python import threadable + from threading import Thread + _twisted_thread = Thread(target=lambda: reactor.run( \ + installSignalHandlers=False)) + _twisted_thread.setDaemon(True) + _twisted_thread.start() + return reactor, _twisted_thread + +# Export global reactor variable, as Twisted does +reactor, reactor_thread = threaded_reactor() + + +def stop_reactor(): + """Stop the reactor and join the reactor thread until it stops. + Call this function in teardown at the module or package level to + reset the twisted system after your tests. You *must* do this if + you mix tests using these tools and tests using twisted.trial. + """ + global _twisted_thread + + def stop_reactor(): + '''Helper for calling stop from withing the thread.''' + reactor.stop() + + reactor.callFromThread(stop_reactor) + reactor_thread.join() + for p in reactor.getDelayedCalls(): + if p.active(): + p.cancel() + _twisted_thread = None + + +def deferred(timeout=None): + """ + By wrapping a test function with this decorator, you can return a + twisted Deferred and the test will wait for the deferred to be triggered. + The whole test function will run inside the Twisted event loop. + + The optional timeout parameter specifies the maximum duration of the test. + The difference with timed() is that timed() will still wait for the test + to end, while deferred() will stop the test when its timeout has expired. + The latter is more desireable when dealing with network tests, because + the result may actually never arrive. + + If the callback is triggered, the test has passed. + If the errback is triggered or the timeout expires, the test has failed. + + Example:: + + @deferred(timeout=5.0) + def test_resolve(): + return reactor.resolve("www.python.org") + + Attention! If you combine this decorator with other decorators (like + "raises"), deferred() must be called *first*! + + In other words, this is good:: + + @raises(DNSLookupError) + @deferred() + def test_error(): + return reactor.resolve("xxxjhjhj.biz") + + and this is bad:: + + @deferred() + @raises(DNSLookupError) + def test_error(): + return reactor.resolve("xxxjhjhj.biz") + """ + reactor, reactor_thread = threaded_reactor() + if reactor is None: + raise ImportError("twisted is not available or could not be imported") + # Check for common syntax mistake + # (otherwise, tests can be silently ignored + # if one writes "@deferred" instead of "@deferred()") + try: + timeout is None or timeout + 0 + except TypeError: + raise TypeError("'timeout' argument must be a number or None") + + def decorate(func): + def wrapper(*args, **kargs): + q = Queue() + def callback(value): + q.put(None) + def errback(failure): + # Retrieve and save full exception info + try: + failure.raiseException() + except: + q.put(sys.exc_info()) + def g(): + try: + d = func(*args, **kargs) + try: + d.addCallbacks(callback, errback) + # Check for a common mistake and display a nice error + # message + except AttributeError: + raise TypeError("you must return a twisted Deferred " + "from your test case!") + # Catch exceptions raised in the test body (from the + # Twisted thread) + except: + q.put(sys.exc_info()) + reactor.callFromThread(g) + try: + error = q.get(timeout=timeout) + except Empty: + raise TimeExpired("timeout expired before end of test (%f s.)" + % timeout) + # Re-raise all exceptions + if error is not None: + exc_type, exc_value, tb = error + raise exc_type, exc_value, tb + wrapper = make_decorator(func)(wrapper) + return wrapper + return decorate + diff --git a/lib/spack/external/nose/usage.txt b/lib/spack/external/nose/usage.txt new file mode 100644 index 0000000000..bc96894ab7 --- /dev/null +++ b/lib/spack/external/nose/usage.txt @@ -0,0 +1,115 @@ +nose collects tests automatically from python source files, +directories and packages found in its working directory (which +defaults to the current working directory). Any python source file, +directory or package that matches the testMatch regular expression +(by default: `(?:^|[\b_\.-])[Tt]est)` will be collected as a test (or +source for collection of tests). In addition, all other packages +found in the working directory will be examined for python source files +or directories that match testMatch. Package discovery descends all +the way down the tree, so package.tests and package.sub.tests and +package.sub.sub2.tests will all be collected. + +Within a test directory or package, any python source file matching +testMatch will be examined for test cases. Within a test module, +functions and classes whose names match testMatch and TestCase +subclasses with any name will be loaded and executed as tests. Tests +may use the assert keyword or raise AssertionErrors to indicate test +failure. TestCase subclasses may do the same or use the various +TestCase methods available. + +**It is important to note that the default behavior of nose is to +not include tests from files which are executable.** To include +tests from such files, remove their executable bit or use +the --exe flag (see 'Options' section below). + +Selecting Tests +--------------- + +To specify which tests to run, pass test names on the command line: + + %prog only_test_this.py + +Test names specified may be file or module names, and may optionally +indicate the test case to run by separating the module or file name +from the test case name with a colon. Filenames may be relative or +absolute. Examples: + + %prog test.module + %prog another.test:TestCase.test_method + %prog a.test:TestCase + %prog /path/to/test/file.py:test_function + +You may also change the working directory where nose looks for tests +by using the -w switch: + + %prog -w /path/to/tests + +Note, however, that support for multiple -w arguments is now deprecated +and will be removed in a future release. As of nose 0.10, you can get +the same behavior by specifying the target directories *without* +the -w switch: + + %prog /path/to/tests /another/path/to/tests + +Further customization of test selection and loading is possible +through the use of plugins. + +Test result output is identical to that of unittest, except for +the additional features (error classes, and plugin-supplied +features such as output capture and assert introspection) detailed +in the options below. + +Configuration +------------- + +In addition to passing command-line options, you may also put +configuration options in your project's *setup.cfg* file, or a .noserc +or nose.cfg file in your home directory. In any of these standard +ini-style config files, you put your nosetests configuration in a +``[nosetests]`` section. Options are the same as on the command line, +with the -- prefix removed. For options that are simple switches, you +must supply a value: + + [nosetests] + verbosity=3 + with-doctest=1 + +All configuration files that are found will be loaded and their +options combined. You can override the standard config file loading +with the ``-c`` option. + +Using Plugins +------------- + +There are numerous nose plugins available via easy_install and +elsewhere. To use a plugin, just install it. The plugin will add +command line options to nosetests. To verify that the plugin is installed, +run: + + nosetests --plugins + +You can add -v or -vv to that command to show more information +about each plugin. + +If you are running nose.main() or nose.run() from a script, you +can specify a list of plugins to use by passing a list of plugins +with the plugins keyword argument. + +0.9 plugins +----------- + +nose 1.0 can use SOME plugins that were written for nose 0.9. The +default plugin manager inserts a compatibility wrapper around 0.9 +plugins that adapts the changed plugin api calls. However, plugins +that access nose internals are likely to fail, especially if they +attempt to access test case or test suite classes. For example, +plugins that try to determine if a test passed to startTest is an +individual test or a suite will fail, partly because suites are no +longer passed to startTest and partly because it's likely that the +plugin is trying to find out if the test is an instance of a class +that no longer exists. + +0.10 and 0.11 plugins +--------------------- + +All plugins written for nose 0.10 and 0.11 should work with nose 1.0. diff --git a/lib/spack/external/nose/util.py b/lib/spack/external/nose/util.py new file mode 100644 index 0000000000..bfe16589ea --- /dev/null +++ b/lib/spack/external/nose/util.py @@ -0,0 +1,668 @@ +"""Utility functions and classes used by nose internally. +""" +import inspect +import itertools +import logging +import stat +import os +import re +import sys +import types +import unittest +from nose.pyversion import ClassType, TypeType, isgenerator, ismethod + + +log = logging.getLogger('nose') + +ident_re = re.compile(r'^[A-Za-z_][A-Za-z0-9_.]*$') +class_types = (ClassType, TypeType) +skip_pattern = r"(?:\.svn)|(?:[^.]+\.py[co])|(?:.*~)|(?:.*\$py\.class)|(?:__pycache__)" + +try: + set() + set = set # make from nose.util import set happy +except NameError: + try: + from sets import Set as set + except ImportError: + pass + + +def ls_tree(dir_path="", + skip_pattern=skip_pattern, + indent="|-- ", branch_indent="| ", + last_indent="`-- ", last_branch_indent=" "): + # TODO: empty directories look like non-directory files + return "\n".join(_ls_tree_lines(dir_path, skip_pattern, + indent, branch_indent, + last_indent, last_branch_indent)) + + +def _ls_tree_lines(dir_path, skip_pattern, + indent, branch_indent, last_indent, last_branch_indent): + if dir_path == "": + dir_path = os.getcwd() + + lines = [] + + names = os.listdir(dir_path) + names.sort() + dirs, nondirs = [], [] + for name in names: + if re.match(skip_pattern, name): + continue + if os.path.isdir(os.path.join(dir_path, name)): + dirs.append(name) + else: + nondirs.append(name) + + # list non-directories first + entries = list(itertools.chain([(name, False) for name in nondirs], + [(name, True) for name in dirs])) + def ls_entry(name, is_dir, ind, branch_ind): + if not is_dir: + yield ind + name + else: + path = os.path.join(dir_path, name) + if not os.path.islink(path): + yield ind + name + subtree = _ls_tree_lines(path, skip_pattern, + indent, branch_indent, + last_indent, last_branch_indent) + for x in subtree: + yield branch_ind + x + for name, is_dir in entries[:-1]: + for line in ls_entry(name, is_dir, indent, branch_indent): + yield line + if entries: + name, is_dir = entries[-1] + for line in ls_entry(name, is_dir, last_indent, last_branch_indent): + yield line + + +def absdir(path): + """Return absolute, normalized path to directory, if it exists; None + otherwise. + """ + if not os.path.isabs(path): + path = os.path.normpath(os.path.abspath(os.path.join(os.getcwd(), + path))) + if path is None or not os.path.isdir(path): + return None + return path + + +def absfile(path, where=None): + """Return absolute, normalized path to file (optionally in directory + where), or None if the file can't be found either in where or the current + working directory. + """ + orig = path + if where is None: + where = os.getcwd() + if isinstance(where, list) or isinstance(where, tuple): + for maybe_path in where: + maybe_abs = absfile(path, maybe_path) + if maybe_abs is not None: + return maybe_abs + return None + if not os.path.isabs(path): + path = os.path.normpath(os.path.abspath(os.path.join(where, path))) + if path is None or not os.path.exists(path): + if where != os.getcwd(): + # try the cwd instead + path = os.path.normpath(os.path.abspath(os.path.join(os.getcwd(), + orig))) + if path is None or not os.path.exists(path): + return None + if os.path.isdir(path): + # might want an __init__.py from pacakge + init = os.path.join(path,'__init__.py') + if os.path.isfile(init): + return init + elif os.path.isfile(path): + return path + return None + + +def anyp(predicate, iterable): + for item in iterable: + if predicate(item): + return True + return False + + +def file_like(name): + """A name is file-like if it is a path that exists, or it has a + directory part, or it ends in .py, or it isn't a legal python + identifier. + """ + return (os.path.exists(name) + or os.path.dirname(name) + or name.endswith('.py') + or not ident_re.match(os.path.splitext(name)[0])) + + +def func_lineno(func): + """Get the line number of a function. First looks for + compat_co_firstlineno, then func_code.co_first_lineno. + """ + try: + return func.compat_co_firstlineno + except AttributeError: + try: + return func.func_code.co_firstlineno + except AttributeError: + return -1 + + +def isclass(obj): + """Is obj a class? Inspect's isclass is too liberal and returns True + for objects that can't be subclasses of anything. + """ + obj_type = type(obj) + return obj_type in class_types or issubclass(obj_type, type) + + +# backwards compat (issue #64) +is_generator = isgenerator + + +def ispackage(path): + """ + Is this path a package directory? + + >>> ispackage('nose') + True + >>> ispackage('unit_tests') + False + >>> ispackage('nose/plugins') + True + >>> ispackage('nose/loader.py') + False + """ + if os.path.isdir(path): + # at least the end of the path must be a legal python identifier + # and __init__.py[co] must exist + end = os.path.basename(path) + if ident_re.match(end): + for init in ('__init__.py', '__init__.pyc', '__init__.pyo'): + if os.path.isfile(os.path.join(path, init)): + return True + if sys.platform.startswith('java') and \ + os.path.isfile(os.path.join(path, '__init__$py.class')): + return True + return False + + +def isproperty(obj): + """ + Is this a property? + + >>> class Foo: + ... def got(self): + ... return 2 + ... def get(self): + ... return 1 + ... get = property(get) + + >>> isproperty(Foo.got) + False + >>> isproperty(Foo.get) + True + """ + return type(obj) == property + + +def getfilename(package, relativeTo=None): + """Find the python source file for a package, relative to a + particular directory (defaults to current working directory if not + given). + """ + if relativeTo is None: + relativeTo = os.getcwd() + path = os.path.join(relativeTo, os.sep.join(package.split('.'))) + if os.path.exists(path + '/__init__.py'): + return path + filename = path + '.py' + if os.path.exists(filename): + return filename + return None + + +def getpackage(filename): + """ + Find the full dotted package name for a given python source file + name. Returns None if the file is not a python source file. + + >>> getpackage('foo.py') + 'foo' + >>> getpackage('biff/baf.py') + 'baf' + >>> getpackage('nose/util.py') + 'nose.util' + + Works for directories too. + + >>> getpackage('nose') + 'nose' + >>> getpackage('nose/plugins') + 'nose.plugins' + + And __init__ files stuck onto directories + + >>> getpackage('nose/plugins/__init__.py') + 'nose.plugins' + + Absolute paths also work. + + >>> path = os.path.abspath(os.path.join('nose', 'plugins')) + >>> getpackage(path) + 'nose.plugins' + """ + src_file = src(filename) + if (os.path.isdir(src_file) or not src_file.endswith('.py')) and not ispackage(src_file): + return None + base, ext = os.path.splitext(os.path.basename(src_file)) + if base == '__init__': + mod_parts = [] + else: + mod_parts = [base] + path, part = os.path.split(os.path.split(src_file)[0]) + while part: + if ispackage(os.path.join(path, part)): + mod_parts.append(part) + else: + break + path, part = os.path.split(path) + mod_parts.reverse() + return '.'.join(mod_parts) + + +def ln(label): + """Draw a 70-char-wide divider, with label in the middle. + + >>> ln('hello there') + '---------------------------- hello there -----------------------------' + """ + label_len = len(label) + 2 + chunk = (70 - label_len) // 2 + out = '%s %s %s' % ('-' * chunk, label, '-' * chunk) + pad = 70 - len(out) + if pad > 0: + out = out + ('-' * pad) + return out + + +def resolve_name(name, module=None): + """Resolve a dotted name to a module and its parts. This is stolen + wholesale from unittest.TestLoader.loadTestByName. + + >>> resolve_name('nose.util') #doctest: +ELLIPSIS + + >>> resolve_name('nose.util.resolve_name') #doctest: +ELLIPSIS + + """ + parts = name.split('.') + parts_copy = parts[:] + if module is None: + while parts_copy: + try: + log.debug("__import__ %s", name) + module = __import__('.'.join(parts_copy)) + break + except ImportError: + del parts_copy[-1] + if not parts_copy: + raise + parts = parts[1:] + obj = module + log.debug("resolve: %s, %s, %s, %s", parts, name, obj, module) + for part in parts: + obj = getattr(obj, part) + return obj + + +def split_test_name(test): + """Split a test name into a 3-tuple containing file, module, and callable + names, any of which (but not all) may be blank. + + Test names are in the form: + + file_or_module:callable + + Either side of the : may be dotted. To change the splitting behavior, you + can alter nose.util.split_test_re. + """ + norm = os.path.normpath + file_or_mod = test + fn = None + if not ':' in test: + # only a file or mod part + if file_like(test): + return (norm(test), None, None) + else: + return (None, test, None) + + # could be path|mod:callable, or a : in the file path someplace + head, tail = os.path.split(test) + if not head: + # this is a case like 'foo:bar' -- generally a module + # name followed by a callable, but also may be a windows + # drive letter followed by a path + try: + file_or_mod, fn = test.split(':') + if file_like(fn): + # must be a funny path + file_or_mod, fn = test, None + except ValueError: + # more than one : in the test + # this is a case like c:\some\path.py:a_test + parts = test.split(':') + if len(parts[0]) == 1: + file_or_mod, fn = ':'.join(parts[:-1]), parts[-1] + else: + # nonsense like foo:bar:baz + raise ValueError("Test name '%s' could not be parsed. Please " + "format test names as path:callable or " + "module:callable." % (test,)) + elif not tail: + # this is a case like 'foo:bar/' + # : must be part of the file path, so ignore it + file_or_mod = test + else: + if ':' in tail: + file_part, fn = tail.split(':') + else: + file_part = tail + file_or_mod = os.sep.join([head, file_part]) + if file_or_mod: + if file_like(file_or_mod): + return (norm(file_or_mod), None, fn) + else: + return (None, file_or_mod, fn) + else: + return (None, None, fn) +split_test_name.__test__ = False # do not collect + + +def test_address(test): + """Find the test address for a test, which may be a module, filename, + class, method or function. + """ + if hasattr(test, "address"): + return test.address() + # type-based polymorphism sucks in general, but I believe is + # appropriate here + t = type(test) + file = module = call = None + if t == types.ModuleType: + file = getattr(test, '__file__', None) + module = getattr(test, '__name__', None) + return (src(file), module, call) + if t == types.FunctionType or issubclass(t, type) or t == types.ClassType: + module = getattr(test, '__module__', None) + if module is not None: + m = sys.modules[module] + file = getattr(m, '__file__', None) + if file is not None: + file = os.path.abspath(file) + call = getattr(test, '__name__', None) + return (src(file), module, call) + if t == types.MethodType: + cls_adr = test_address(test.im_class) + return (src(cls_adr[0]), cls_adr[1], + "%s.%s" % (cls_adr[2], test.__name__)) + # handle unittest.TestCase instances + if isinstance(test, unittest.TestCase): + if (hasattr(test, '_FunctionTestCase__testFunc') # pre 2.7 + or hasattr(test, '_testFunc')): # 2.7 + # unittest FunctionTestCase + try: + return test_address(test._FunctionTestCase__testFunc) + except AttributeError: + return test_address(test._testFunc) + # regular unittest.TestCase + cls_adr = test_address(test.__class__) + # 2.5 compat: __testMethodName changed to _testMethodName + try: + method_name = test._TestCase__testMethodName + except AttributeError: + method_name = test._testMethodName + return (src(cls_adr[0]), cls_adr[1], + "%s.%s" % (cls_adr[2], method_name)) + if (hasattr(test, '__class__') and + test.__class__.__module__ not in ('__builtin__', 'builtins')): + return test_address(test.__class__) + raise TypeError("I don't know what %s is (%s)" % (test, t)) +test_address.__test__ = False # do not collect + + +def try_run(obj, names): + """Given a list of possible method names, try to run them with the + provided object. Keep going until something works. Used to run + setup/teardown methods for module, package, and function tests. + """ + for name in names: + func = getattr(obj, name, None) + if func is not None: + if type(obj) == types.ModuleType: + # py.test compatibility + if isinstance(func, types.FunctionType): + args, varargs, varkw, defaults = \ + inspect.getargspec(func) + else: + # Not a function. If it's callable, call it anyway + if hasattr(func, '__call__') and not inspect.ismethod(func): + func = func.__call__ + try: + args, varargs, varkw, defaults = \ + inspect.getargspec(func) + args.pop(0) # pop the self off + except TypeError: + raise TypeError("Attribute %s of %r is not a python " + "function. Only functions or callables" + " may be used as fixtures." % + (name, obj)) + if len(args): + log.debug("call fixture %s.%s(%s)", obj, name, obj) + return func(obj) + log.debug("call fixture %s.%s", obj, name) + return func() + + +def src(filename): + """Find the python source file for a .pyc, .pyo or $py.class file on + jython. Returns the filename provided if it is not a python source + file. + """ + if filename is None: + return filename + if sys.platform.startswith('java') and filename.endswith('$py.class'): + return '.'.join((filename[:-9], 'py')) + base, ext = os.path.splitext(filename) + if ext in ('.pyc', '.pyo', '.py'): + return '.'.join((base, 'py')) + return filename + + +def regex_last_key(regex): + """Sort key function factory that puts items that match a + regular expression last. + + >>> from nose.config import Config + >>> from nose.pyversion import sort_list + >>> c = Config() + >>> regex = c.testMatch + >>> entries = ['.', '..', 'a_test', 'src', 'lib', 'test', 'foo.py'] + >>> sort_list(entries, regex_last_key(regex)) + >>> entries + ['.', '..', 'foo.py', 'lib', 'src', 'a_test', 'test'] + """ + def k(obj): + if regex.search(obj): + return (1, obj) + return (0, obj) + return k + + +def tolist(val): + """Convert a value that may be a list or a (possibly comma-separated) + string into a list. The exception: None is returned as None, not [None]. + + >>> tolist(["one", "two"]) + ['one', 'two'] + >>> tolist("hello") + ['hello'] + >>> tolist("separate,values, with, commas, spaces , are ,ok") + ['separate', 'values', 'with', 'commas', 'spaces', 'are', 'ok'] + """ + if val is None: + return None + try: + # might already be a list + val.extend([]) + return val + except AttributeError: + pass + # might be a string + try: + return re.split(r'\s*,\s*', val) + except TypeError: + # who knows... + return list(val) + + +class odict(dict): + """Simple ordered dict implementation, based on: + + http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/107747 + """ + def __init__(self, *arg, **kw): + self._keys = [] + super(odict, self).__init__(*arg, **kw) + + def __delitem__(self, key): + super(odict, self).__delitem__(key) + self._keys.remove(key) + + def __setitem__(self, key, item): + super(odict, self).__setitem__(key, item) + if key not in self._keys: + self._keys.append(key) + + def __str__(self): + return "{%s}" % ', '.join(["%r: %r" % (k, v) for k, v in self.items()]) + + def clear(self): + super(odict, self).clear() + self._keys = [] + + def copy(self): + d = super(odict, self).copy() + d._keys = self._keys[:] + return d + + def items(self): + return zip(self._keys, self.values()) + + def keys(self): + return self._keys[:] + + def setdefault(self, key, failobj=None): + item = super(odict, self).setdefault(key, failobj) + if key not in self._keys: + self._keys.append(key) + return item + + def update(self, dict): + super(odict, self).update(dict) + for key in dict.keys(): + if key not in self._keys: + self._keys.append(key) + + def values(self): + return map(self.get, self._keys) + + +def transplant_func(func, module): + """ + Make a function imported from module A appear as if it is located + in module B. + + >>> from pprint import pprint + >>> pprint.__module__ + 'pprint' + >>> pp = transplant_func(pprint, __name__) + >>> pp.__module__ + 'nose.util' + + The original function is not modified. + + >>> pprint.__module__ + 'pprint' + + Calling the transplanted function calls the original. + + >>> pp([1, 2]) + [1, 2] + >>> pprint([1,2]) + [1, 2] + + """ + from nose.tools import make_decorator + if isgenerator(func): + def newfunc(*arg, **kw): + for v in func(*arg, **kw): + yield v + else: + def newfunc(*arg, **kw): + return func(*arg, **kw) + + newfunc = make_decorator(func)(newfunc) + newfunc.__module__ = module + return newfunc + + +def transplant_class(cls, module): + """ + Make a class appear to reside in `module`, rather than the module in which + it is actually defined. + + >>> from nose.failure import Failure + >>> Failure.__module__ + 'nose.failure' + >>> Nf = transplant_class(Failure, __name__) + >>> Nf.__module__ + 'nose.util' + >>> Nf.__name__ + 'Failure' + + """ + class C(cls): + pass + C.__module__ = module + C.__name__ = cls.__name__ + return C + + +def safe_str(val, encoding='utf-8'): + try: + return str(val) + except UnicodeEncodeError: + if isinstance(val, Exception): + return ' '.join([safe_str(arg, encoding) + for arg in val]) + return unicode(val).encode(encoding) + + +def is_executable(file): + if not os.path.exists(file): + return False + st = os.stat(file) + return bool(st.st_mode & (stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)) + + +if __name__ == '__main__': + import doctest + doctest.testmod() -- cgit v1.2.3-70-g09d2 From c41b9b7ddc05271e46a86e21e682291420e6e488 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Wed, 9 Dec 2015 01:24:15 -0800 Subject: Change github.com/scalability-llnl to github.com/llnl everywhere. --- bin/spack | 2 +- bin/spack-python | 2 +- lib/spack/docs/conf.py | 2 +- lib/spack/docs/exts/sphinxcontrib/__init__.py | 2 +- lib/spack/docs/exts/sphinxcontrib/programoutput.py | 2 +- lib/spack/docs/getting_started.rst | 6 +++--- lib/spack/docs/index.rst | 4 ++-- lib/spack/env/cc | 2 +- lib/spack/external/__init__.py | 2 +- lib/spack/llnl/util/filesystem.py | 2 +- lib/spack/llnl/util/lang.py | 2 +- lib/spack/llnl/util/link_tree.py | 2 +- lib/spack/llnl/util/lock.py | 2 +- lib/spack/llnl/util/tty/__init__.py | 2 +- lib/spack/llnl/util/tty/colify.py | 2 +- lib/spack/llnl/util/tty/color.py | 2 +- lib/spack/llnl/util/tty/log.py | 2 +- lib/spack/spack/__init__.py | 2 +- lib/spack/spack/architecture.py | 2 +- lib/spack/spack/cmd/__init__.py | 2 +- lib/spack/spack/cmd/activate.py | 2 +- lib/spack/spack/cmd/arch.py | 2 +- lib/spack/spack/cmd/bootstrap.py | 2 +- lib/spack/spack/cmd/cd.py | 2 +- lib/spack/spack/cmd/checksum.py | 2 +- lib/spack/spack/cmd/clean.py | 2 +- lib/spack/spack/cmd/compiler.py | 2 +- lib/spack/spack/cmd/compilers.py | 2 +- lib/spack/spack/cmd/config.py | 2 +- lib/spack/spack/cmd/create.py | 2 +- lib/spack/spack/cmd/deactivate.py | 2 +- lib/spack/spack/cmd/dependents.py | 2 +- lib/spack/spack/cmd/diy.py | 2 +- lib/spack/spack/cmd/doc.py | 2 +- lib/spack/spack/cmd/edit.py | 2 +- lib/spack/spack/cmd/env.py | 2 +- lib/spack/spack/cmd/extensions.py | 2 +- lib/spack/spack/cmd/fetch.py | 2 +- lib/spack/spack/cmd/find.py | 2 +- lib/spack/spack/cmd/graph.py | 2 +- lib/spack/spack/cmd/help.py | 2 +- lib/spack/spack/cmd/info.py | 2 +- lib/spack/spack/cmd/install.py | 2 +- lib/spack/spack/cmd/list.py | 2 +- lib/spack/spack/cmd/load.py | 2 +- lib/spack/spack/cmd/location.py | 2 +- lib/spack/spack/cmd/md5.py | 2 +- lib/spack/spack/cmd/mirror.py | 2 +- lib/spack/spack/cmd/module.py | 2 +- lib/spack/spack/cmd/package-list.py | 4 ++-- lib/spack/spack/cmd/patch.py | 2 +- lib/spack/spack/cmd/pkg.py | 2 +- lib/spack/spack/cmd/providers.py | 2 +- lib/spack/spack/cmd/purge.py | 2 +- lib/spack/spack/cmd/python.py | 2 +- lib/spack/spack/cmd/reindex.py | 2 +- lib/spack/spack/cmd/restage.py | 2 +- lib/spack/spack/cmd/spec.py | 2 +- lib/spack/spack/cmd/stage.py | 2 +- lib/spack/spack/cmd/test-install.py | 2 +- lib/spack/spack/cmd/test.py | 2 +- lib/spack/spack/cmd/uninstall.py | 2 +- lib/spack/spack/cmd/unload.py | 2 +- lib/spack/spack/cmd/unuse.py | 2 +- lib/spack/spack/cmd/urls.py | 2 +- lib/spack/spack/cmd/use.py | 2 +- lib/spack/spack/cmd/versions.py | 2 +- lib/spack/spack/compiler.py | 2 +- lib/spack/spack/compilers/__init__.py | 2 +- lib/spack/spack/compilers/clang.py | 2 +- lib/spack/spack/compilers/gcc.py | 2 +- lib/spack/spack/compilers/intel.py | 2 +- lib/spack/spack/compilers/pgi.py | 2 +- lib/spack/spack/compilers/xl.py | 2 +- lib/spack/spack/concretize.py | 2 +- lib/spack/spack/config.py | 2 +- lib/spack/spack/database.py | 2 +- lib/spack/spack/directives.py | 2 +- lib/spack/spack/directory_layout.py | 2 +- lib/spack/spack/error.py | 2 +- lib/spack/spack/fetch_strategy.py | 2 +- lib/spack/spack/graph.py | 2 +- lib/spack/spack/hooks/__init__.py | 2 +- lib/spack/spack/hooks/dotkit.py | 2 +- lib/spack/spack/hooks/extensions.py | 2 +- lib/spack/spack/hooks/tclmodule.py | 2 +- lib/spack/spack/mirror.py | 2 +- lib/spack/spack/modules.py | 2 +- lib/spack/spack/multimethod.py | 2 +- lib/spack/spack/package.py | 2 +- lib/spack/spack/packages.py | 2 +- lib/spack/spack/parse.py | 2 +- lib/spack/spack/patch.py | 2 +- lib/spack/spack/spec.py | 2 +- lib/spack/spack/stage.py | 2 +- lib/spack/spack/test/__init__.py | 2 +- lib/spack/spack/test/cc.py | 2 +- lib/spack/spack/test/concretize.py | 2 +- lib/spack/spack/test/config.py | 2 +- lib/spack/spack/test/configure_guess.py | 2 +- lib/spack/spack/test/database.py | 2 +- lib/spack/spack/test/directory_layout.py | 2 +- lib/spack/spack/test/git_fetch.py | 2 +- lib/spack/spack/test/hg_fetch.py | 2 +- lib/spack/spack/test/install.py | 2 +- lib/spack/spack/test/link_tree.py | 2 +- lib/spack/spack/test/lock.py | 2 +- lib/spack/spack/test/make_executable.py | 2 +- lib/spack/spack/test/mirror.py | 2 +- lib/spack/spack/test/mock_packages_test.py | 2 +- lib/spack/spack/test/mock_repo.py | 2 +- lib/spack/spack/test/multimethod.py | 2 +- lib/spack/spack/test/optional_deps.py | 2 +- lib/spack/spack/test/package_sanity.py | 2 +- lib/spack/spack/test/packages.py | 2 +- lib/spack/spack/test/python_version.py | 2 +- lib/spack/spack/test/spec_dag.py | 2 +- lib/spack/spack/test/spec_semantics.py | 2 +- lib/spack/spack/test/spec_syntax.py | 2 +- lib/spack/spack/test/spec_yaml.py | 2 +- lib/spack/spack/test/stage.py | 2 +- lib/spack/spack/test/svn_fetch.py | 2 +- lib/spack/spack/test/unit_install.py | 2 +- lib/spack/spack/test/url_extrapolate.py | 2 +- lib/spack/spack/test/url_parse.py | 2 +- lib/spack/spack/test/url_substitution.py | 2 +- lib/spack/spack/test/versions.py | 2 +- lib/spack/spack/url.py | 4 ++-- lib/spack/spack/util/__init__.py | 2 +- lib/spack/spack/util/compression.py | 2 +- lib/spack/spack/util/crypto.py | 2 +- lib/spack/spack/util/debug.py | 2 +- lib/spack/spack/util/environment.py | 2 +- lib/spack/spack/util/executable.py | 2 +- lib/spack/spack/util/multiproc.py | 2 +- lib/spack/spack/util/prefix.py | 2 +- lib/spack/spack/util/string.py | 2 +- lib/spack/spack/util/web.py | 2 +- lib/spack/spack/variant.py | 2 +- lib/spack/spack/version.py | 2 +- lib/spack/spack/virtual.py | 2 +- share/spack/setup-env.csh | 2 +- share/spack/setup-env.sh | 2 +- var/spack/mock_packages/callpath/package.py | 2 +- var/spack/mock_packages/direct_mpich/package.py | 2 +- var/spack/mock_packages/dyninst/package.py | 2 +- var/spack/mock_packages/fake/package.py | 2 +- var/spack/mock_packages/indirect_mpich/package.py | 2 +- var/spack/mock_packages/libdwarf/package.py | 2 +- var/spack/mock_packages/libelf/package.py | 2 +- var/spack/mock_packages/mpich/package.py | 2 +- var/spack/mock_packages/mpich2/package.py | 2 +- var/spack/mock_packages/mpileaks/package.py | 2 +- var/spack/mock_packages/multimethod/package.py | 2 +- var/spack/mock_packages/netlib-blas/package.py | 2 +- var/spack/mock_packages/netlib-lapack/package.py | 2 +- var/spack/mock_packages/openblas/package.py | 2 +- var/spack/mock_packages/trivial_install_test_package/package.py | 2 +- var/spack/mock_packages/zmpi/package.py | 2 +- var/spack/packages/Mitos/package.py | 8 ++++---- var/spack/packages/adept-utils/package.py | 6 +++--- var/spack/packages/automaded/package.py | 6 +++--- var/spack/packages/callpath/package.py | 6 +++--- var/spack/packages/clang/package.py | 2 +- var/spack/packages/cmake/package.py | 2 +- var/spack/packages/cram/package.py | 4 ++-- var/spack/packages/damselfly/package.py | 6 +++--- var/spack/packages/dyninst/package.py | 2 +- var/spack/packages/gcc/package.py | 2 +- var/spack/packages/gmp/package.py | 2 +- var/spack/packages/gperftools/package.py | 2 +- var/spack/packages/launchmon/package.py | 4 ++-- var/spack/packages/libNBC/package.py | 2 +- var/spack/packages/libdwarf/package.py | 2 +- var/spack/packages/libelf/package.py | 2 +- var/spack/packages/libmonitor/package.py | 2 +- var/spack/packages/libunwind/package.py | 2 +- var/spack/packages/llvm-lld/package.py | 2 +- var/spack/packages/llvm/package.py | 2 +- var/spack/packages/memaxes/package.py | 5 ++--- var/spack/packages/mpc/package.py | 2 +- var/spack/packages/mpfr/package.py | 2 +- var/spack/packages/mpich/package.py | 2 +- var/spack/packages/mpileaks/package.py | 2 +- var/spack/packages/muster/package.py | 4 ++-- var/spack/packages/netgauge/package.py | 2 +- var/spack/packages/pmgr_collective/package.py | 2 +- var/spack/packages/ravel/package.py | 4 ++-- var/spack/packages/scr/package.py | 2 +- var/spack/packages/spindle/package.py | 2 +- var/spack/packages/sqlite/package.py | 2 +- var/spack/packages/sundials/package.py | 2 +- var/spack/packages/swig/package.py | 2 +- 193 files changed, 214 insertions(+), 215 deletions(-) (limited to 'lib') diff --git a/bin/spack b/bin/spack index e92d7cc273..ec6a80ff02 100755 --- a/bin/spack +++ b/bin/spack @@ -7,7 +7,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/bin/spack-python b/bin/spack-python index 8a4b9c175d..e0745e8c58 100755 --- a/bin/spack-python +++ b/bin/spack-python @@ -7,7 +7,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/docs/conf.py b/lib/spack/docs/conf.py index bce9ef0e94..e8e170d0a6 100644 --- a/lib/spack/docs/conf.py +++ b/lib/spack/docs/conf.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/docs/exts/sphinxcontrib/__init__.py b/lib/spack/docs/exts/sphinxcontrib/__init__.py index 838d616eb4..298856746c 100644 --- a/lib/spack/docs/exts/sphinxcontrib/__init__.py +++ b/lib/spack/docs/exts/sphinxcontrib/__init__.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/docs/exts/sphinxcontrib/programoutput.py b/lib/spack/docs/exts/sphinxcontrib/programoutput.py index ff006acf72..f0fa045c86 100644 --- a/lib/spack/docs/exts/sphinxcontrib/programoutput.py +++ b/lib/spack/docs/exts/sphinxcontrib/programoutput.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/docs/getting_started.rst b/lib/spack/docs/getting_started.rst index d958d9e74a..67ca18e71a 100644 --- a/lib/spack/docs/getting_started.rst +++ b/lib/spack/docs/getting_started.rst @@ -5,11 +5,11 @@ Download -------------------- Getting spack is easy. You can clone it from the `github repository -`_ using this command: +`_ using this command: .. code-block:: sh - $ git clone https://github.com/scalability-llnl/spack.git + $ git clone https://github.com/llnl/spack.git This will create a directory called ``spack``. We'll assume that the full path to this directory is in the ``SPACK_ROOT`` environment @@ -22,7 +22,7 @@ go: $ spack install libelf For a richer experience, use Spack's `shell support -`_: +`_: .. code-block:: sh diff --git a/lib/spack/docs/index.rst b/lib/spack/docs/index.rst index 97c8361421..79757208c9 100644 --- a/lib/spack/docs/index.rst +++ b/lib/spack/docs/index.rst @@ -24,12 +24,12 @@ maintain a single file for many different builds of the same package. See the :doc:`features` for examples and highlights. Get spack from the `github repository -`_ and install your first +`_ and install your first package: .. code-block:: sh - $ git clone https://github.com/scalability-llnl/spack.git + $ git clone https://github.com/llnl/spack.git $ cd spack/bin $ ./spack install libelf diff --git a/lib/spack/env/cc b/lib/spack/env/cc index fa85bb595e..75a63f6fac 100755 --- a/lib/spack/env/cc +++ b/lib/spack/env/cc @@ -7,7 +7,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/external/__init__.py b/lib/spack/external/__init__.py index 0578022210..7a89a1ac67 100644 --- a/lib/spack/external/__init__.py +++ b/lib/spack/external/__init__.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/llnl/util/filesystem.py b/lib/spack/llnl/util/filesystem.py index 03f25d3dff..24cfbfde71 100644 --- a/lib/spack/llnl/util/filesystem.py +++ b/lib/spack/llnl/util/filesystem.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/llnl/util/lang.py b/lib/spack/llnl/util/lang.py index 156ee34c9e..cc87b7eac2 100644 --- a/lib/spack/llnl/util/lang.py +++ b/lib/spack/llnl/util/lang.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/llnl/util/link_tree.py b/lib/spack/llnl/util/link_tree.py index 583f077b79..6ae8aff75c 100644 --- a/lib/spack/llnl/util/link_tree.py +++ b/lib/spack/llnl/util/link_tree.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/llnl/util/lock.py b/lib/spack/llnl/util/lock.py index ac3684bd55..a7a9bf6b19 100644 --- a/lib/spack/llnl/util/lock.py +++ b/lib/spack/llnl/util/lock.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/llnl/util/tty/__init__.py b/lib/spack/llnl/util/tty/__init__.py index 48368543ff..203f429a48 100644 --- a/lib/spack/llnl/util/tty/__init__.py +++ b/lib/spack/llnl/util/tty/__init__.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/llnl/util/tty/colify.py b/lib/spack/llnl/util/tty/colify.py index db928444c7..5545cf0311 100644 --- a/lib/spack/llnl/util/tty/colify.py +++ b/lib/spack/llnl/util/tty/colify.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/llnl/util/tty/color.py b/lib/spack/llnl/util/tty/color.py index 0d09303da0..167a99d3c2 100644 --- a/lib/spack/llnl/util/tty/color.py +++ b/lib/spack/llnl/util/tty/color.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/llnl/util/tty/log.py b/lib/spack/llnl/util/tty/log.py index 5a52d45bc7..2819cd40df 100644 --- a/lib/spack/llnl/util/tty/log.py +++ b/lib/spack/llnl/util/tty/log.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index 1ecf662178..02eeed01fa 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 6c874e30be..2701fab90c 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/__init__.py b/lib/spack/spack/cmd/__init__.py index 6ce6fa0960..926e7ac14a 100644 --- a/lib/spack/spack/cmd/__init__.py +++ b/lib/spack/spack/cmd/__init__.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/activate.py b/lib/spack/spack/cmd/activate.py index 1e44948d24..bcd01f2a28 100644 --- a/lib/spack/spack/cmd/activate.py +++ b/lib/spack/spack/cmd/activate.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/arch.py b/lib/spack/spack/cmd/arch.py index f0e88d1849..db27544ffd 100644 --- a/lib/spack/spack/cmd/arch.py +++ b/lib/spack/spack/cmd/arch.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/bootstrap.py b/lib/spack/spack/cmd/bootstrap.py index f75b68b00a..e4ec7da35d 100644 --- a/lib/spack/spack/cmd/bootstrap.py +++ b/lib/spack/spack/cmd/bootstrap.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/cd.py b/lib/spack/spack/cmd/cd.py index 24d56db7d0..16cbe6555a 100644 --- a/lib/spack/spack/cmd/cd.py +++ b/lib/spack/spack/cmd/cd.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/checksum.py b/lib/spack/spack/cmd/checksum.py index c940d57781..6b7022a7a1 100644 --- a/lib/spack/spack/cmd/checksum.py +++ b/lib/spack/spack/cmd/checksum.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/clean.py b/lib/spack/spack/cmd/clean.py index e303b3d634..c3409887fb 100644 --- a/lib/spack/spack/cmd/clean.py +++ b/lib/spack/spack/cmd/clean.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/compiler.py b/lib/spack/spack/cmd/compiler.py index 3173d11070..589ca87fb5 100644 --- a/lib/spack/spack/cmd/compiler.py +++ b/lib/spack/spack/cmd/compiler.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/compilers.py b/lib/spack/spack/cmd/compilers.py index 8d046bfd7c..c485a910eb 100644 --- a/lib/spack/spack/cmd/compilers.py +++ b/lib/spack/spack/cmd/compilers.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/config.py b/lib/spack/spack/cmd/config.py index 78972a8be0..a6e914131e 100644 --- a/lib/spack/spack/cmd/config.py +++ b/lib/spack/spack/cmd/config.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/create.py b/lib/spack/spack/cmd/create.py index 05cf170e39..5e42860f3e 100644 --- a/lib/spack/spack/cmd/create.py +++ b/lib/spack/spack/cmd/create.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/deactivate.py b/lib/spack/spack/cmd/deactivate.py index 5a2b353fa2..a0c78bf755 100644 --- a/lib/spack/spack/cmd/deactivate.py +++ b/lib/spack/spack/cmd/deactivate.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/dependents.py b/lib/spack/spack/cmd/dependents.py index 129a4eeb23..de76098d2f 100644 --- a/lib/spack/spack/cmd/dependents.py +++ b/lib/spack/spack/cmd/dependents.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/diy.py b/lib/spack/spack/cmd/diy.py index d6bd1fbb79..ebe8424f09 100644 --- a/lib/spack/spack/cmd/diy.py +++ b/lib/spack/spack/cmd/diy.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/doc.py b/lib/spack/spack/cmd/doc.py index 601ae26e5e..29cadec94f 100644 --- a/lib/spack/spack/cmd/doc.py +++ b/lib/spack/spack/cmd/doc.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/edit.py b/lib/spack/spack/cmd/edit.py index b8764ba391..b168d967b9 100644 --- a/lib/spack/spack/cmd/edit.py +++ b/lib/spack/spack/cmd/edit.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/env.py b/lib/spack/spack/cmd/env.py index ae8e95491e..525e955a00 100644 --- a/lib/spack/spack/cmd/env.py +++ b/lib/spack/spack/cmd/env.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/extensions.py b/lib/spack/spack/cmd/extensions.py index a3db2da394..c2cf288877 100644 --- a/lib/spack/spack/cmd/extensions.py +++ b/lib/spack/spack/cmd/extensions.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/fetch.py b/lib/spack/spack/cmd/fetch.py index 1dd8703daf..6f9e7ab5e2 100644 --- a/lib/spack/spack/cmd/fetch.py +++ b/lib/spack/spack/cmd/fetch.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/find.py b/lib/spack/spack/cmd/find.py index 05fa620c59..c7a376fd8d 100644 --- a/lib/spack/spack/cmd/find.py +++ b/lib/spack/spack/cmd/find.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/graph.py b/lib/spack/spack/cmd/graph.py index 586a02c53b..586a852351 100644 --- a/lib/spack/spack/cmd/graph.py +++ b/lib/spack/spack/cmd/graph.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/help.py b/lib/spack/spack/cmd/help.py index eae3aabd97..841a0d5bcb 100644 --- a/lib/spack/spack/cmd/help.py +++ b/lib/spack/spack/cmd/help.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/info.py b/lib/spack/spack/cmd/info.py index 085e4db44d..8040e23936 100644 --- a/lib/spack/spack/cmd/info.py +++ b/lib/spack/spack/cmd/info.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/install.py b/lib/spack/spack/cmd/install.py index e4338e222f..5ee7bc01b7 100644 --- a/lib/spack/spack/cmd/install.py +++ b/lib/spack/spack/cmd/install.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/list.py b/lib/spack/spack/cmd/list.py index 0b55d9fb7b..b51d5b429a 100644 --- a/lib/spack/spack/cmd/list.py +++ b/lib/spack/spack/cmd/list.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/load.py b/lib/spack/spack/cmd/load.py index 5bc6b15784..30d86c3b01 100644 --- a/lib/spack/spack/cmd/load.py +++ b/lib/spack/spack/cmd/load.py @@ -6,7 +6,7 @@ # Written by David Beckingsale, david@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/location.py b/lib/spack/spack/cmd/location.py index 6fbab9782f..e805cc4012 100644 --- a/lib/spack/spack/cmd/location.py +++ b/lib/spack/spack/cmd/location.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/md5.py b/lib/spack/spack/cmd/md5.py index 8be0a7ad4c..ef1e4f3475 100644 --- a/lib/spack/spack/cmd/md5.py +++ b/lib/spack/spack/cmd/md5.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/mirror.py b/lib/spack/spack/cmd/mirror.py index 4599944f1c..4a1ce00b75 100644 --- a/lib/spack/spack/cmd/mirror.py +++ b/lib/spack/spack/cmd/mirror.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/module.py b/lib/spack/spack/cmd/module.py index c3daed6402..a5a9570eb5 100644 --- a/lib/spack/spack/cmd/module.py +++ b/lib/spack/spack/cmd/module.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/package-list.py b/lib/spack/spack/cmd/package-list.py index f048482845..eca9f918f1 100644 --- a/lib/spack/spack/cmd/package-list.py +++ b/lib/spack/spack/cmd/package-list.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify @@ -34,7 +34,7 @@ description = "Print a list of all packages in reStructuredText." def github_url(pkg): """Link to a package file on github.""" - return ("https://github.com/scalability-llnl/spack/blob/master/var/spack/packages/%s/package.py" % + return ("https://github.com/llnl/spack/blob/master/var/spack/packages/%s/package.py" % pkg.name) diff --git a/lib/spack/spack/cmd/patch.py b/lib/spack/spack/cmd/patch.py index 2356583b07..8fc6f1383e 100644 --- a/lib/spack/spack/cmd/patch.py +++ b/lib/spack/spack/cmd/patch.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/pkg.py b/lib/spack/spack/cmd/pkg.py index ae5efd9d9c..1ebc5ce3ee 100644 --- a/lib/spack/spack/cmd/pkg.py +++ b/lib/spack/spack/cmd/pkg.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/providers.py b/lib/spack/spack/cmd/providers.py index 1a652c82d1..0472f9bbe4 100644 --- a/lib/spack/spack/cmd/providers.py +++ b/lib/spack/spack/cmd/providers.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/purge.py b/lib/spack/spack/cmd/purge.py index 9b96937149..d5d7513c46 100644 --- a/lib/spack/spack/cmd/purge.py +++ b/lib/spack/spack/cmd/purge.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/python.py b/lib/spack/spack/cmd/python.py index 15c45654c2..e26b8d3e79 100644 --- a/lib/spack/spack/cmd/python.py +++ b/lib/spack/spack/cmd/python.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/reindex.py b/lib/spack/spack/cmd/reindex.py index c0008930c4..2b30ef8814 100644 --- a/lib/spack/spack/cmd/reindex.py +++ b/lib/spack/spack/cmd/reindex.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/restage.py b/lib/spack/spack/cmd/restage.py index 9230cf5a1a..703ae30a04 100644 --- a/lib/spack/spack/cmd/restage.py +++ b/lib/spack/spack/cmd/restage.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/spec.py b/lib/spack/spack/cmd/spec.py index af7ec1b36c..43a106ea37 100644 --- a/lib/spack/spack/cmd/spec.py +++ b/lib/spack/spack/cmd/spec.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/stage.py b/lib/spack/spack/cmd/stage.py index 09cf0e1a1c..7638cf31c4 100644 --- a/lib/spack/spack/cmd/stage.py +++ b/lib/spack/spack/cmd/stage.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/test-install.py b/lib/spack/spack/cmd/test-install.py index ba0cb39baf..e37554155f 100644 --- a/lib/spack/spack/cmd/test-install.py +++ b/lib/spack/spack/cmd/test-install.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/test.py b/lib/spack/spack/cmd/test.py index b1418ac2f1..1669ec4cc9 100644 --- a/lib/spack/spack/cmd/test.py +++ b/lib/spack/spack/cmd/test.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/uninstall.py b/lib/spack/spack/cmd/uninstall.py index 93575e005d..191d9d88e8 100644 --- a/lib/spack/spack/cmd/uninstall.py +++ b/lib/spack/spack/cmd/uninstall.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/unload.py b/lib/spack/spack/cmd/unload.py index 24e49b3f24..cfb640ee6f 100644 --- a/lib/spack/spack/cmd/unload.py +++ b/lib/spack/spack/cmd/unload.py @@ -6,7 +6,7 @@ # Written by David Beckingsale, david@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/unuse.py b/lib/spack/spack/cmd/unuse.py index 7f0b384ea0..06176a976b 100644 --- a/lib/spack/spack/cmd/unuse.py +++ b/lib/spack/spack/cmd/unuse.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/urls.py b/lib/spack/spack/cmd/urls.py index 417ce3ab68..a544b6153b 100644 --- a/lib/spack/spack/cmd/urls.py +++ b/lib/spack/spack/cmd/urls.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/use.py b/lib/spack/spack/cmd/use.py index 4990fea2f8..c09695cfd3 100644 --- a/lib/spack/spack/cmd/use.py +++ b/lib/spack/spack/cmd/use.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/cmd/versions.py b/lib/spack/spack/cmd/versions.py index ed16728261..494f13d36d 100644 --- a/lib/spack/spack/cmd/versions.py +++ b/lib/spack/spack/cmd/versions.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index 1e800a8979..b9abf943e8 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index b7b021a1ac..66e608cf79 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/compilers/clang.py b/lib/spack/spack/compilers/clang.py index 790901c86e..b34d1f2f9c 100644 --- a/lib/spack/spack/compilers/clang.py +++ b/lib/spack/spack/compilers/clang.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/compilers/gcc.py b/lib/spack/spack/compilers/gcc.py index f0d27d590e..2886888d57 100644 --- a/lib/spack/spack/compilers/gcc.py +++ b/lib/spack/spack/compilers/gcc.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/compilers/intel.py b/lib/spack/spack/compilers/intel.py index 2a72c4eaea..1298429974 100644 --- a/lib/spack/spack/compilers/intel.py +++ b/lib/spack/spack/compilers/intel.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/compilers/pgi.py b/lib/spack/spack/compilers/pgi.py index d97f24c12e..6999eb50de 100644 --- a/lib/spack/spack/compilers/pgi.py +++ b/lib/spack/spack/compilers/pgi.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/compilers/xl.py b/lib/spack/spack/compilers/xl.py index 562186b865..308f811eb4 100644 --- a/lib/spack/spack/compilers/xl.py +++ b/lib/spack/spack/compilers/xl.py @@ -7,7 +7,7 @@ # Written by François Bissey, francois.bissey@canterbury.ac.nz, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 66002492cb..d9419da784 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 41afe8b232..c127f6a28f 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/database.py b/lib/spack/spack/database.py index 908ffc7fa4..d62a47547d 100644 --- a/lib/spack/spack/database.py +++ b/lib/spack/spack/database.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/directives.py b/lib/spack/spack/directives.py index 78039ac6f9..2a818e8d0c 100644 --- a/lib/spack/spack/directives.py +++ b/lib/spack/spack/directives.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/directory_layout.py b/lib/spack/spack/directory_layout.py index 918405cab6..056606b429 100644 --- a/lib/spack/spack/directory_layout.py +++ b/lib/spack/spack/directory_layout.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/error.py b/lib/spack/spack/error.py index b3b24e6105..0c2e7eb53c 100644 --- a/lib/spack/spack/error.py +++ b/lib/spack/spack/error.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/fetch_strategy.py b/lib/spack/spack/fetch_strategy.py index 5e6850d14b..8c4bd4ed8a 100644 --- a/lib/spack/spack/fetch_strategy.py +++ b/lib/spack/spack/fetch_strategy.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/graph.py b/lib/spack/spack/graph.py index 5fb6a9cd23..e8c5cfb080 100644 --- a/lib/spack/spack/graph.py +++ b/lib/spack/spack/graph.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/hooks/__init__.py b/lib/spack/spack/hooks/__init__.py index 1c44e8abaa..2765f7be39 100644 --- a/lib/spack/spack/hooks/__init__.py +++ b/lib/spack/spack/hooks/__init__.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/hooks/dotkit.py b/lib/spack/spack/hooks/dotkit.py index 4e748ff80a..9123637356 100644 --- a/lib/spack/spack/hooks/dotkit.py +++ b/lib/spack/spack/hooks/dotkit.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/hooks/extensions.py b/lib/spack/spack/hooks/extensions.py index cf87a78c8c..b4847d697f 100644 --- a/lib/spack/spack/hooks/extensions.py +++ b/lib/spack/spack/hooks/extensions.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/hooks/tclmodule.py b/lib/spack/spack/hooks/tclmodule.py index 0b9fd5a67c..8b315f27a2 100644 --- a/lib/spack/spack/hooks/tclmodule.py +++ b/lib/spack/spack/hooks/tclmodule.py @@ -6,7 +6,7 @@ # Written by David Beckingsale, david@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/mirror.py b/lib/spack/spack/mirror.py index 306c8085aa..ee0bf6de11 100644 --- a/lib/spack/spack/mirror.py +++ b/lib/spack/spack/mirror.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py index 56a61adefb..59ed9f893e 100644 --- a/lib/spack/spack/modules.py +++ b/lib/spack/spack/modules.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/multimethod.py b/lib/spack/spack/multimethod.py index 892619c6ac..df9b9b2ab1 100644 --- a/lib/spack/spack/multimethod.py +++ b/lib/spack/spack/multimethod.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index c631a35bf3..daba5cd352 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/packages.py b/lib/spack/spack/packages.py index 2e3e95ca40..f6f4cbf025 100644 --- a/lib/spack/spack/packages.py +++ b/lib/spack/spack/packages.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/parse.py b/lib/spack/spack/parse.py index bc12ec258c..e9467aa685 100644 --- a/lib/spack/spack/parse.py +++ b/lib/spack/spack/parse.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/patch.py b/lib/spack/spack/patch.py index b1b6e07738..42f2105f52 100644 --- a/lib/spack/spack/patch.py +++ b/lib/spack/spack/patch.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index a10edcc6fc..f62182ce76 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index 78930ecb5b..754344fc01 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/test/__init__.py b/lib/spack/spack/test/__init__.py index 0f776bfea4..13cb1d2b78 100644 --- a/lib/spack/spack/test/__init__.py +++ b/lib/spack/spack/test/__init__.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/test/cc.py b/lib/spack/spack/test/cc.py index aa16f9b351..4188b8d550 100644 --- a/lib/spack/spack/test/cc.py +++ b/lib/spack/spack/test/cc.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/test/concretize.py b/lib/spack/spack/test/concretize.py index b3a77d076a..2f8e0c7ec0 100644 --- a/lib/spack/spack/test/concretize.py +++ b/lib/spack/spack/test/concretize.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/test/config.py b/lib/spack/spack/test/config.py index 790b22f3b0..ed11e34c69 100644 --- a/lib/spack/spack/test/config.py +++ b/lib/spack/spack/test/config.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/test/configure_guess.py b/lib/spack/spack/test/configure_guess.py index 766dd51d52..a4e8565b62 100644 --- a/lib/spack/spack/test/configure_guess.py +++ b/lib/spack/spack/test/configure_guess.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/test/database.py b/lib/spack/spack/test/database.py index 8416143f2d..c07d32686e 100644 --- a/lib/spack/spack/test/database.py +++ b/lib/spack/spack/test/database.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/test/directory_layout.py b/lib/spack/spack/test/directory_layout.py index b3ad8efec4..703ac1b867 100644 --- a/lib/spack/spack/test/directory_layout.py +++ b/lib/spack/spack/test/directory_layout.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/test/git_fetch.py b/lib/spack/spack/test/git_fetch.py index 9700bd7533..244680b5d0 100644 --- a/lib/spack/spack/test/git_fetch.py +++ b/lib/spack/spack/test/git_fetch.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/test/hg_fetch.py b/lib/spack/spack/test/hg_fetch.py index 531dfabaa1..f8c6571bda 100644 --- a/lib/spack/spack/test/hg_fetch.py +++ b/lib/spack/spack/test/hg_fetch.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/test/install.py b/lib/spack/spack/test/install.py index 5659e97a4d..1ef4171fb2 100644 --- a/lib/spack/spack/test/install.py +++ b/lib/spack/spack/test/install.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/test/link_tree.py b/lib/spack/spack/test/link_tree.py index 9e887ecc49..886b7ef4c5 100644 --- a/lib/spack/spack/test/link_tree.py +++ b/lib/spack/spack/test/link_tree.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/test/lock.py b/lib/spack/spack/test/lock.py index 5664e71b03..bc68df01db 100644 --- a/lib/spack/spack/test/lock.py +++ b/lib/spack/spack/test/lock.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/test/make_executable.py b/lib/spack/spack/test/make_executable.py index c4bfeb2a03..09efec8580 100644 --- a/lib/spack/spack/test/make_executable.py +++ b/lib/spack/spack/test/make_executable.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/test/mirror.py b/lib/spack/spack/test/mirror.py index 89ab14359e..189a85fb1a 100644 --- a/lib/spack/spack/test/mirror.py +++ b/lib/spack/spack/test/mirror.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/test/mock_packages_test.py b/lib/spack/spack/test/mock_packages_test.py index 00f81114af..e4e1b21b53 100644 --- a/lib/spack/spack/test/mock_packages_test.py +++ b/lib/spack/spack/test/mock_packages_test.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/test/mock_repo.py b/lib/spack/spack/test/mock_repo.py index fd184e64bc..c454b1f106 100644 --- a/lib/spack/spack/test/mock_repo.py +++ b/lib/spack/spack/test/mock_repo.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/test/multimethod.py b/lib/spack/spack/test/multimethod.py index cd5d9e625e..d8d61d14c8 100644 --- a/lib/spack/spack/test/multimethod.py +++ b/lib/spack/spack/test/multimethod.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/test/optional_deps.py b/lib/spack/spack/test/optional_deps.py index fbee0cfa8f..ebd7281999 100644 --- a/lib/spack/spack/test/optional_deps.py +++ b/lib/spack/spack/test/optional_deps.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/test/package_sanity.py b/lib/spack/spack/test/package_sanity.py index 6222e7b5f8..370cf676ef 100644 --- a/lib/spack/spack/test/package_sanity.py +++ b/lib/spack/spack/test/package_sanity.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/test/packages.py b/lib/spack/spack/test/packages.py index a8183cf6a6..b2daea7b7b 100644 --- a/lib/spack/spack/test/packages.py +++ b/lib/spack/spack/test/packages.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/test/python_version.py b/lib/spack/spack/test/python_version.py index ba7bab6f4b..ba570b416f 100644 --- a/lib/spack/spack/test/python_version.py +++ b/lib/spack/spack/test/python_version.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/test/spec_dag.py b/lib/spack/spack/test/spec_dag.py index 549f829d3e..94438b6a0c 100644 --- a/lib/spack/spack/test/spec_dag.py +++ b/lib/spack/spack/test/spec_dag.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/test/spec_semantics.py b/lib/spack/spack/test/spec_semantics.py index 64220e5893..1381556aad 100644 --- a/lib/spack/spack/test/spec_semantics.py +++ b/lib/spack/spack/test/spec_semantics.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/test/spec_syntax.py b/lib/spack/spack/test/spec_syntax.py index 404f38906e..1daaa4be8f 100644 --- a/lib/spack/spack/test/spec_syntax.py +++ b/lib/spack/spack/test/spec_syntax.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/test/spec_yaml.py b/lib/spack/spack/test/spec_yaml.py index 869befc02a..11987ea1b3 100644 --- a/lib/spack/spack/test/spec_yaml.py +++ b/lib/spack/spack/test/spec_yaml.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/test/stage.py b/lib/spack/spack/test/stage.py index 8cff8f7960..c1b2a2a573 100644 --- a/lib/spack/spack/test/stage.py +++ b/lib/spack/spack/test/stage.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/test/svn_fetch.py b/lib/spack/spack/test/svn_fetch.py index 184fe8faa1..9229af76d4 100644 --- a/lib/spack/spack/test/svn_fetch.py +++ b/lib/spack/spack/test/svn_fetch.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/test/unit_install.py b/lib/spack/spack/test/unit_install.py index c4b9092f05..41c76a6dfa 100644 --- a/lib/spack/spack/test/unit_install.py +++ b/lib/spack/spack/test/unit_install.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/test/url_extrapolate.py b/lib/spack/spack/test/url_extrapolate.py index 00d8216020..87adf89401 100644 --- a/lib/spack/spack/test/url_extrapolate.py +++ b/lib/spack/spack/test/url_extrapolate.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/test/url_parse.py b/lib/spack/spack/test/url_parse.py index ae1d559f7c..efde7c0c73 100644 --- a/lib/spack/spack/test/url_parse.py +++ b/lib/spack/spack/test/url_parse.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/test/url_substitution.py b/lib/spack/spack/test/url_substitution.py index db7ddd251d..8b90ee086a 100644 --- a/lib/spack/spack/test/url_substitution.py +++ b/lib/spack/spack/test/url_substitution.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/test/versions.py b/lib/spack/spack/test/versions.py index 20e946e90e..108450e098 100644 --- a/lib/spack/spack/test/versions.py +++ b/lib/spack/spack/test/versions.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/url.py b/lib/spack/spack/url.py index 6adbfe156d..02c0b83e26 100644 --- a/lib/spack/spack/url.py +++ b/lib/spack/spack/url.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify @@ -70,7 +70,7 @@ def find_list_url(url): """ url_types = [ - # e.g. https://github.com/scalability-llnl/callpath/archive/v1.0.1.tar.gz + # e.g. https://github.com/llnl/callpath/archive/v1.0.1.tar.gz (r'^(https://github.com/[^/]+/[^/]+)/archive/', lambda m: m.group(1) + '/releases') ] diff --git a/lib/spack/spack/util/__init__.py b/lib/spack/spack/util/__init__.py index 1c388f31c5..b54691b67c 100644 --- a/lib/spack/spack/util/__init__.py +++ b/lib/spack/spack/util/__init__.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/util/compression.py b/lib/spack/spack/util/compression.py index fd17785ad0..ea1f233bce 100644 --- a/lib/spack/spack/util/compression.py +++ b/lib/spack/spack/util/compression.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/util/crypto.py b/lib/spack/spack/util/crypto.py index 8a8574cd3d..5269260284 100644 --- a/lib/spack/spack/util/crypto.py +++ b/lib/spack/spack/util/crypto.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/util/debug.py b/lib/spack/spack/util/debug.py index 37985eccdd..7930753f6f 100644 --- a/lib/spack/spack/util/debug.py +++ b/lib/spack/spack/util/debug.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/util/environment.py b/lib/spack/spack/util/environment.py index 7a4ff919ad..cd413dcfbc 100644 --- a/lib/spack/spack/util/environment.py +++ b/lib/spack/spack/util/environment.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/util/executable.py b/lib/spack/spack/util/executable.py index d1dfb62ffb..4f9958062b 100644 --- a/lib/spack/spack/util/executable.py +++ b/lib/spack/spack/util/executable.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/util/multiproc.py b/lib/spack/spack/util/multiproc.py index 21cd6f543d..8ca82df011 100644 --- a/lib/spack/spack/util/multiproc.py +++ b/lib/spack/spack/util/multiproc.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/util/prefix.py b/lib/spack/spack/util/prefix.py index 7bd63c16ca..c613ca5182 100644 --- a/lib/spack/spack/util/prefix.py +++ b/lib/spack/spack/util/prefix.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/util/string.py b/lib/spack/spack/util/string.py index 234163bf52..1556ce6d29 100644 --- a/lib/spack/spack/util/string.py +++ b/lib/spack/spack/util/string.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/util/web.py b/lib/spack/spack/util/web.py index 1420d62a77..94384e9c86 100644 --- a/lib/spack/spack/util/web.py +++ b/lib/spack/spack/util/web.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/variant.py b/lib/spack/spack/variant.py index 3d3e2b0f6d..8959e76684 100644 --- a/lib/spack/spack/variant.py +++ b/lib/spack/spack/variant.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/version.py b/lib/spack/spack/version.py index ffce2d1ff8..e8a0a261c9 100644 --- a/lib/spack/spack/version.py +++ b/lib/spack/spack/version.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/lib/spack/spack/virtual.py b/lib/spack/spack/virtual.py index f92cc4509c..d16aea8642 100644 --- a/lib/spack/spack/virtual.py +++ b/lib/spack/spack/virtual.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/share/spack/setup-env.csh b/share/spack/setup-env.csh index 5f91670a60..42d8c42726 100755 --- a/share/spack/setup-env.csh +++ b/share/spack/setup-env.csh @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/share/spack/setup-env.sh b/share/spack/setup-env.sh index b90846d28f..c96a94195d 100755 --- a/share/spack/setup-env.sh +++ b/share/spack/setup-env.sh @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/mock_packages/callpath/package.py b/var/spack/mock_packages/callpath/package.py index 5b6b70ba2a..abc576f78f 100644 --- a/var/spack/mock_packages/callpath/package.py +++ b/var/spack/mock_packages/callpath/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/mock_packages/direct_mpich/package.py b/var/spack/mock_packages/direct_mpich/package.py index 2ced82521b..efe7fc2afc 100644 --- a/var/spack/mock_packages/direct_mpich/package.py +++ b/var/spack/mock_packages/direct_mpich/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/mock_packages/dyninst/package.py b/var/spack/mock_packages/dyninst/package.py index 7998578da1..ea57950865 100644 --- a/var/spack/mock_packages/dyninst/package.py +++ b/var/spack/mock_packages/dyninst/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/mock_packages/fake/package.py b/var/spack/mock_packages/fake/package.py index fb3c2bdd2e..5f81ef20fc 100644 --- a/var/spack/mock_packages/fake/package.py +++ b/var/spack/mock_packages/fake/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/mock_packages/indirect_mpich/package.py b/var/spack/mock_packages/indirect_mpich/package.py index daf8b4b166..0b1773a27b 100644 --- a/var/spack/mock_packages/indirect_mpich/package.py +++ b/var/spack/mock_packages/indirect_mpich/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/mock_packages/libdwarf/package.py b/var/spack/mock_packages/libdwarf/package.py index 0b8df04cfb..e486a5de03 100644 --- a/var/spack/mock_packages/libdwarf/package.py +++ b/var/spack/mock_packages/libdwarf/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/mock_packages/libelf/package.py b/var/spack/mock_packages/libelf/package.py index 94c8f942cd..5e5b0b7143 100644 --- a/var/spack/mock_packages/libelf/package.py +++ b/var/spack/mock_packages/libelf/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/mock_packages/mpich/package.py b/var/spack/mock_packages/mpich/package.py index f77d3efc5d..55bf97f2cf 100644 --- a/var/spack/mock_packages/mpich/package.py +++ b/var/spack/mock_packages/mpich/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/mock_packages/mpich2/package.py b/var/spack/mock_packages/mpich2/package.py index 827b94c8a4..90f969b898 100644 --- a/var/spack/mock_packages/mpich2/package.py +++ b/var/spack/mock_packages/mpich2/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/mock_packages/mpileaks/package.py b/var/spack/mock_packages/mpileaks/package.py index 3989f1b452..9a18c5e1f2 100644 --- a/var/spack/mock_packages/mpileaks/package.py +++ b/var/spack/mock_packages/mpileaks/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/mock_packages/multimethod/package.py b/var/spack/mock_packages/multimethod/package.py index 75b1606ffc..ea103fe175 100644 --- a/var/spack/mock_packages/multimethod/package.py +++ b/var/spack/mock_packages/multimethod/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/mock_packages/netlib-blas/package.py b/var/spack/mock_packages/netlib-blas/package.py index 199327812e..39f2c92ae5 100644 --- a/var/spack/mock_packages/netlib-blas/package.py +++ b/var/spack/mock_packages/netlib-blas/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/mock_packages/netlib-lapack/package.py b/var/spack/mock_packages/netlib-lapack/package.py index 8f7f236f1b..331844e544 100644 --- a/var/spack/mock_packages/netlib-lapack/package.py +++ b/var/spack/mock_packages/netlib-lapack/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/mock_packages/openblas/package.py b/var/spack/mock_packages/openblas/package.py index 9c2fb30573..c7771b92a3 100644 --- a/var/spack/mock_packages/openblas/package.py +++ b/var/spack/mock_packages/openblas/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/mock_packages/trivial_install_test_package/package.py b/var/spack/mock_packages/trivial_install_test_package/package.py index c4db9f5f07..fec5849e67 100644 --- a/var/spack/mock_packages/trivial_install_test_package/package.py +++ b/var/spack/mock_packages/trivial_install_test_package/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/mock_packages/zmpi/package.py b/var/spack/mock_packages/zmpi/package.py index 8c6ceda6d3..201fac2fbf 100644 --- a/var/spack/mock_packages/zmpi/package.py +++ b/var/spack/mock_packages/zmpi/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/packages/Mitos/package.py b/var/spack/packages/Mitos/package.py index 0377887943..ea131872dd 100644 --- a/var/spack/packages/Mitos/package.py +++ b/var/spack/packages/Mitos/package.py @@ -4,15 +4,15 @@ class Mitos(Package): """Mitos is a library and a tool for collecting sampled memory performance data to view with MemAxes""" - homepage = "https://github.com/scalability-llnl/Mitos" - url = "https://github.com/scalability-llnl/Mitos" + homepage = "https://github.com/llnl/Mitos" + url = "https://github.com/llnl/Mitos" version('0.9.2', - git='https://github.com/scalability-llnl/Mitos.git', + git='https://github.com/llnl/Mitos.git', commit='8cb143a2e8c00353ff531a781a9ca0992b0aaa3d') version('0.9.1', - git='https://github.com/scalability-llnl/Mitos.git', + git='https://github.com/llnl/Mitos.git', tag='v0.9.1') depends_on('dyninst@8.2.1:') diff --git a/var/spack/packages/adept-utils/package.py b/var/spack/packages/adept-utils/package.py index e4a2e1523f..fb59576c21 100644 --- a/var/spack/packages/adept-utils/package.py +++ b/var/spack/packages/adept-utils/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify @@ -27,8 +27,8 @@ from spack import * class AdeptUtils(Package): """Utility libraries for LLNL performance tools.""" - homepage = "https://github.com/scalability-llnl/adept-utils" - url = "https://github.com/scalability-llnl/adept-utils/archive/v1.0.tar.gz" + homepage = "https://github.com/llnl/adept-utils" + url = "https://github.com/llnl/adept-utils/archive/v1.0.tar.gz" version('1.0.1', '731a310717adcb004d9d195130efee7d') version('1.0', '5c6cd9badce56c945ac8551e34804397') diff --git a/var/spack/packages/automaded/package.py b/var/spack/packages/automaded/package.py index 9fbd93e3b3..e0bc7efb8b 100644 --- a/var/spack/packages/automaded/package.py +++ b/var/spack/packages/automaded/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify @@ -36,8 +36,8 @@ class Automaded(Package): finding the process (or group of processes) that caused the hang. """ - homepage = "https://github.com/scalability-llnl/AutomaDeD" - url = "https://github.com/scalability-llnl/AutomaDeD/archive/v1.0.tar.gz" + homepage = "https://github.com/llnl/AutomaDeD" + url = "https://github.com/llnl/AutomaDeD/archive/v1.0.tar.gz" version('1.0', '16a3d4def2c4c77d0bc4b21de8b3ab03') diff --git a/var/spack/packages/callpath/package.py b/var/spack/packages/callpath/package.py index f8a1eab9f7..3d2d96249e 100644 --- a/var/spack/packages/callpath/package.py +++ b/var/spack/packages/callpath/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify @@ -28,8 +28,8 @@ class Callpath(Package): """Library for representing callpaths consistently in distributed-memory performance tools.""" - homepage = "https://github.com/scalability-llnl/callpath" - url = "https://github.com/scalability-llnl/callpath/archive/v1.0.1.tar.gz" + homepage = "https://github.com/llnl/callpath" + url = "https://github.com/llnl/callpath/archive/v1.0.1.tar.gz" version('1.0.2', 'b1994d5ee7c7db9d27586fc2dcf8f373') version('1.0.1', '0047983d2a52c5c335f8ba7f5bab2325') diff --git a/var/spack/packages/clang/package.py b/var/spack/packages/clang/package.py index 4f977bf9a4..20a5ac2c94 100644 --- a/var/spack/packages/clang/package.py +++ b/var/spack/packages/clang/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/packages/cmake/package.py b/var/spack/packages/cmake/package.py index fcfbca0705..c24a80748c 100644 --- a/var/spack/packages/cmake/package.py +++ b/var/spack/packages/cmake/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/packages/cram/package.py b/var/spack/packages/cram/package.py index 1f1722993f..b19422b8a8 100644 --- a/var/spack/packages/cram/package.py +++ b/var/spack/packages/cram/package.py @@ -2,8 +2,8 @@ from spack import * class Cram(Package): """Cram runs many small MPI jobs inside one large MPI job.""" - homepage = "https://github.com/scalability-llnl/cram" - url = "http://github.com/scalability-llnl/cram/archive/v1.0.1.tar.gz" + homepage = "https://github.com/llnl/cram" + url = "http://github.com/llnl/cram/archive/v1.0.1.tar.gz" version('1.0.1', 'c73711e945cf5dc603e44395f6647f5e') diff --git a/var/spack/packages/damselfly/package.py b/var/spack/packages/damselfly/package.py index 54df528c4a..96666d1abe 100644 --- a/var/spack/packages/damselfly/package.py +++ b/var/spack/packages/damselfly/package.py @@ -2,10 +2,10 @@ from spack import * class Damselfly(Package): """Damselfly is a model-based parallel network simulator.""" - homepage = "https://github.com/scalability-llnl/damselfly" - url = "https://github.com/scalability-llnl/damselfly" + homepage = "https://github.com/llnl/damselfly" + url = "https://github.com/llnl/damselfly" - version('1.0', '05cf7e2d8ece4408c0f2abb7ab63fd74c0d62895', git='https://github.com/scalability-llnl/damselfly.git', tag='v1.0') + version('1.0', '05cf7e2d8ece4408c0f2abb7ab63fd74c0d62895', git='https://github.com/llnl/damselfly.git', tag='v1.0') def install(self, spec, prefix): with working_dir('spack-build', create=True): diff --git a/var/spack/packages/dyninst/package.py b/var/spack/packages/dyninst/package.py index 41ec57dd2f..7f8598e0e5 100644 --- a/var/spack/packages/dyninst/package.py +++ b/var/spack/packages/dyninst/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/packages/gcc/package.py b/var/spack/packages/gcc/package.py index b5dc585be7..7ec160d595 100644 --- a/var/spack/packages/gcc/package.py +++ b/var/spack/packages/gcc/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/packages/gmp/package.py b/var/spack/packages/gmp/package.py index 6cd501dbf2..fe13de3b95 100644 --- a/var/spack/packages/gmp/package.py +++ b/var/spack/packages/gmp/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/packages/gperftools/package.py b/var/spack/packages/gperftools/package.py index 8900462324..0ba44c9329 100644 --- a/var/spack/packages/gperftools/package.py +++ b/var/spack/packages/gperftools/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/packages/launchmon/package.py b/var/spack/packages/launchmon/package.py index 6fbe6a68d0..f97384a249 100644 --- a/var/spack/packages/launchmon/package.py +++ b/var/spack/packages/launchmon/package.py @@ -6,7 +6,7 @@ # Written by Matthew LeGendre, legendre1@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify @@ -31,7 +31,7 @@ class Launchmon(Package): url = "http://downloads.sourceforge.net/project/launchmon/launchmon/1.0.1%20release/launchmon-1.0.1.tar.gz" version('1.0.1', '2f12465803409fd07f91174a4389eb2b') - version('1.0.1-2', git='https://github.com/scalability-llnl/launchmon.git', commit='ff7e22424b8f375318951eb1c9282fcbbfa8aadf') + version('1.0.1-2', git='https://github.com/llnl/launchmon.git', commit='ff7e22424b8f375318951eb1c9282fcbbfa8aadf') depends_on('autoconf') depends_on('automake') diff --git a/var/spack/packages/libNBC/package.py b/var/spack/packages/libNBC/package.py index 6d08f3219c..550568e97d 100644 --- a/var/spack/packages/libNBC/package.py +++ b/var/spack/packages/libNBC/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/packages/libdwarf/package.py b/var/spack/packages/libdwarf/package.py index 099a974e93..addb557519 100644 --- a/var/spack/packages/libdwarf/package.py +++ b/var/spack/packages/libdwarf/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/packages/libelf/package.py b/var/spack/packages/libelf/package.py index 9338b8f393..29bc21b65c 100644 --- a/var/spack/packages/libelf/package.py +++ b/var/spack/packages/libelf/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/packages/libmonitor/package.py b/var/spack/packages/libmonitor/package.py index 3b95b86ddf..c75e1a7947 100644 --- a/var/spack/packages/libmonitor/package.py +++ b/var/spack/packages/libmonitor/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/packages/libunwind/package.py b/var/spack/packages/libunwind/package.py index 239fcbcfd5..6f162f7b08 100644 --- a/var/spack/packages/libunwind/package.py +++ b/var/spack/packages/libunwind/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/packages/llvm-lld/package.py b/var/spack/packages/llvm-lld/package.py index f229211396..cb91aa22a5 100644 --- a/var/spack/packages/llvm-lld/package.py +++ b/var/spack/packages/llvm-lld/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/packages/llvm/package.py b/var/spack/packages/llvm/package.py index 80c44e4d5f..b3ca488809 100644 --- a/var/spack/packages/llvm/package.py +++ b/var/spack/packages/llvm/package.py @@ -6,7 +6,7 @@ # Written by David Beckingsale, david@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/packages/memaxes/package.py b/var/spack/packages/memaxes/package.py index 76d5d3f831..4b1da558a2 100644 --- a/var/spack/packages/memaxes/package.py +++ b/var/spack/packages/memaxes/package.py @@ -3,10 +3,10 @@ from spack import * class Memaxes(Package): """MemAxes is a visualizer for sampled memory trace data.""" - homepage = "https://github.com/scalability-llnl/MemAxes" + homepage = "https://github.com/llnl/MemAxes" version('0.5', '5874f3fda9fd2d313c0ff9684f915ab5', - url='https://github.com/scalability-llnl/MemAxes/archive/v0.5.tar.gz') + url='https://github.com/llnl/MemAxes/archive/v0.5.tar.gz') depends_on("cmake@2.8.9:") depends_on("qt@5:") @@ -16,4 +16,3 @@ class Memaxes(Package): cmake('..', *std_cmake_args) make() make("install") - diff --git a/var/spack/packages/mpc/package.py b/var/spack/packages/mpc/package.py index 33706f043f..50477a0ccb 100644 --- a/var/spack/packages/mpc/package.py +++ b/var/spack/packages/mpc/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/packages/mpfr/package.py b/var/spack/packages/mpfr/package.py index 9c744a22df..0f2baac004 100644 --- a/var/spack/packages/mpfr/package.py +++ b/var/spack/packages/mpfr/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/packages/mpich/package.py b/var/spack/packages/mpich/package.py index d48bf878f6..e018a08201 100644 --- a/var/spack/packages/mpich/package.py +++ b/var/spack/packages/mpich/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/packages/mpileaks/package.py b/var/spack/packages/mpileaks/package.py index 4ef866588c..661d9d66bf 100644 --- a/var/spack/packages/mpileaks/package.py +++ b/var/spack/packages/mpileaks/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/packages/muster/package.py b/var/spack/packages/muster/package.py index 722daf3d7f..0dc2e5e086 100644 --- a/var/spack/packages/muster/package.py +++ b/var/spack/packages/muster/package.py @@ -7,8 +7,8 @@ class Muster(Package): for performance data analysis on systems with very large numbers of processes. """ - homepage = "https://github.com/scalability-llnl/muster" - url = "https://github.com/scalability-llnl/muster/archive/v1.0.tar.gz" + homepage = "https://github.com/llnl/muster" + url = "https://github.com/llnl/muster/archive/v1.0.tar.gz" version('1.0.1', 'd709787db7e080447afb6571ac17723c') version('1.0', '2eec6979a4a36d3a65a792d12969be16') diff --git a/var/spack/packages/netgauge/package.py b/var/spack/packages/netgauge/package.py index c2378b0718..0ea42175c6 100644 --- a/var/spack/packages/netgauge/package.py +++ b/var/spack/packages/netgauge/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/packages/pmgr_collective/package.py b/var/spack/packages/pmgr_collective/package.py index 5d9b02acc3..1fc47c658f 100644 --- a/var/spack/packages/pmgr_collective/package.py +++ b/var/spack/packages/pmgr_collective/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/packages/ravel/package.py b/var/spack/packages/ravel/package.py index 01fa941cfe..d774a0ab86 100644 --- a/var/spack/packages/ravel/package.py +++ b/var/spack/packages/ravel/package.py @@ -4,8 +4,8 @@ class Ravel(Package): """Ravel is a parallel communication trace visualization tool that orders events according to logical time.""" - homepage = "https://github.com/scalability-llnl/ravel" - url = 'https://github.com/scalability-llnl/ravel/archive/v1.0.0.tar.gz' + homepage = "https://github.com/llnl/ravel" + url = 'https://github.com/llnl/ravel/archive/v1.0.0.tar.gz' version('1.0.0', 'b25fece58331c2adfcce76c5036485c2') diff --git a/var/spack/packages/scr/package.py b/var/spack/packages/scr/package.py index 9fb758f072..1408dce678 100644 --- a/var/spack/packages/scr/package.py +++ b/var/spack/packages/scr/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/packages/spindle/package.py b/var/spack/packages/spindle/package.py index 06a1e14284..a20753458a 100644 --- a/var/spack/packages/spindle/package.py +++ b/var/spack/packages/spindle/package.py @@ -6,7 +6,7 @@ # Written by Matthew LeGendre, legendre1@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/packages/sqlite/package.py b/var/spack/packages/sqlite/package.py index 734b0b6cb6..1cf2d30239 100644 --- a/var/spack/packages/sqlite/package.py +++ b/var/spack/packages/sqlite/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/packages/sundials/package.py b/var/spack/packages/sundials/package.py index 8b784c8c3c..7e025a8244 100644 --- a/var/spack/packages/sundials/package.py +++ b/var/spack/packages/sundials/package.py @@ -6,7 +6,7 @@ # Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify diff --git a/var/spack/packages/swig/package.py b/var/spack/packages/swig/package.py index ee536d7063..8d46c4fe46 100644 --- a/var/spack/packages/swig/package.py +++ b/var/spack/packages/swig/package.py @@ -6,7 +6,7 @@ # Written by Matthew LeGendre, legendre1@llnl.gov, All rights reserved. # LLNL-CODE-647188 # -# For details, see https://scalability-llnl.github.io/spack +# For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify -- cgit v1.2.3-70-g09d2 From 537451f0b1ea057e723daa9bb81bc950730583cb Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Wed, 9 Dec 2015 01:32:12 -0800 Subject: bugfix: doc build needed additional sys.path due to externals change. --- lib/spack/docs/conf.py | 1 + 1 file changed, 1 insertion(+) (limited to 'lib') diff --git a/lib/spack/docs/conf.py b/lib/spack/docs/conf.py index e8e170d0a6..3d2a8251aa 100644 --- a/lib/spack/docs/conf.py +++ b/lib/spack/docs/conf.py @@ -43,6 +43,7 @@ import subprocess # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. sys.path.insert(0, os.path.abspath('exts')) +sys.path.insert(0, os.path.abspath('../external')) # Add the Spack bin directory to the path so that we can use its output in docs. spack_root = '../../..' -- cgit v1.2.3-70-g09d2 From 20e67bc5e6c4a4ac759bb068ff78c13bfc17fb0f Mon Sep 17 00:00:00 2001 From: alalazo Date: Wed, 9 Dec 2015 13:06:39 +0100 Subject: clang : solve the issue with missing default include paths for OpenMP and libc++ resource : support for finer grained linking of resources --- lib/spack/spack/directives.py | 17 +++++++++-------- lib/spack/spack/package.py | 16 +++++++++++----- lib/spack/spack/resource.py | 6 +++--- var/spack/packages/clang/package.py | 31 +++++++++++++++++++++++++++++++ var/spack/packages/llvm/package.py | 4 ++-- 5 files changed, 56 insertions(+), 18 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/directives.py b/lib/spack/spack/directives.py index 07ef32294b..22b262b57c 100644 --- a/lib/spack/spack/directives.py +++ b/lib/spack/spack/directives.py @@ -269,19 +269,20 @@ def variant(pkg, name, default=False, description=""): def resource(pkg, **kwargs): """ Define an external resource to be fetched and staged when building the package. Based on the keywords present in the - dictionary the appropriate FetchStrategy will be used for the resource. + dictionary the appropriate FetchStrategy will be used for the resource. Resources are fetched and staged in their + own folder inside spack stage area, and then linked into the stage area of the package that needs them. List of recognized keywords: - * 'when' : represents the condition upon which the resource is needed (optional) - * 'destination' : path where to extract / checkout the resource (optional). This path must be a relative path, - and it must fall inside the stage area of the main package. - * 'basename' : basename of the resource source folder within destination (optional). - + * 'when' : (optional) represents the condition upon which the resource is needed + * 'destination' : (optional) path where to link the resource. This path must be relative to the main package stage + area. + * 'placement' : (optional) gives the possibility to fine tune how the resource is linked into the main package stage + area. """ when = kwargs.get('when', pkg.name) destination = kwargs.get('destination', "") - basename = kwargs.get('basename', None) + placement = kwargs.get('placement', None) # Check if the path is relative if os.path.isabs(destination): message = "The destination keyword of a resource directive can't be an absolute path.\n" @@ -298,7 +299,7 @@ def resource(pkg, **kwargs): resources = pkg.resources.setdefault(when_spec, []) fetcher = from_kwargs(**kwargs) name = kwargs.get('name') - resources.append(Resource(name, fetcher, destination, basename)) + resources.append(Resource(name, fetcher, destination, placement)) class DirectiveError(spack.error.SpackError): diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index dcb514af00..b386f8f6a8 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -683,11 +683,17 @@ class Package(object): for resource in resources: stage = resource.fetcher.stage _expand_archive(stage, resource.name) - basename = os.path.basename(stage.source_path) if resource.basename is None else resource.basename - link_path = join_path(self.stage.source_path, resource.destination, basename) - if not os.path.exists(link_path): - # Create a symlink - os.symlink(stage.source_path, link_path) + # Turn placement into a dict with relative paths + placement = os.path.basename(stage.source_path) if resource.placement is None else resource.placement + if not isinstance(placement, dict): + placement = {'': placement} + # Make the paths in the dictionary absolute and link + for key, value in placement.iteritems(): + link_path = join_path(self.stage.source_path, resource.destination, value) + source_path = join_path(stage.source_path, key) + if not os.path.exists(link_path): + # Create a symlink + os.symlink(source_path, link_path) ########## self.stage.chdir_to_source() diff --git a/lib/spack/spack/resource.py b/lib/spack/spack/resource.py index b20612686d..8d081b45c9 100644 --- a/lib/spack/spack/resource.py +++ b/lib/spack/spack/resource.py @@ -30,10 +30,10 @@ package to enable optional features. class Resource(object): """ - Represents an optional resource. Aggregates a name, a fetcher and a destination. + Represents an optional resource. Aggregates a name, a fetcher, a destination and a placement """ - def __init__(self, name, fetcher, destination, basename): + def __init__(self, name, fetcher, destination, placement): self.name = name self.fetcher = fetcher self.destination = destination - self.basename = basename + self.placement = placement diff --git a/var/spack/packages/clang/package.py b/var/spack/packages/clang/package.py index eac1863b97..ca368b3074 100644 --- a/var/spack/packages/clang/package.py +++ b/var/spack/packages/clang/package.py @@ -22,8 +22,13 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## + + from spack import * +import os +import os.path + class Clang(Package): """The goal of the Clang project is to create a new C, C++, Objective C and Objective C++ front-end for the LLVM compiler. @@ -62,3 +67,29 @@ class Clang(Package): *options) make() make("install") + # CLang doesn't look in llvm folders for system headers... + self.link_llvm_directories(spec) + + def link_llvm_directories(self, spec): + + def clang_include_dir_at(root): + return join_path(root, 'include') + + def clang_lib_dir_at(root): + return join_path(root, 'lib/clang/', str(self.version), 'include') + + def do_link(source_dir, destination_dir): + if os.path.exists(source_dir): + for name in os.listdir(source_dir): + source = join_path(source_dir, name) + link = join_path(destination_dir, name) + os.symlink(source, link) + + # Link folder and files in include + llvm_dir = clang_include_dir_at(spec['llvm'].prefix) + clang_dir = clang_include_dir_at(self.prefix) + do_link(llvm_dir, clang_dir) + # Link folder and files in lib + llvm_dir = clang_lib_dir_at(spec['llvm'].prefix) + clang_dir = clang_lib_dir_at(self.prefix) + do_link(llvm_dir, clang_dir) \ No newline at end of file diff --git a/var/spack/packages/llvm/package.py b/var/spack/packages/llvm/package.py index 872a6c082b..d7ae3390be 100644 --- a/var/spack/packages/llvm/package.py +++ b/var/spack/packages/llvm/package.py @@ -53,10 +53,10 @@ class Llvm(Package): destination='projects', when='@3.7.0') resource(name='libcxx', url='http://llvm.org/releases/3.7.0/libcxx-3.7.0.src.tar.xz', md5='46aa5175cbe1ad42d6e9c995968e56dd', - destination='projects', basename='libcxx', when='+libcxx@3.7.0') + destination='projects', placement='libcxx', when='+libcxx@3.7.0') resource(name='libcxxabi', url='http://llvm.org/releases/3.7.0/libcxxabi-3.7.0.src.tar.xz', md5='5aa769e2fca79fa5335cfae8f6258772', - destination='projects', basename='libcxxabi', when='+libcxx@3.7.0') + destination='projects', placement='libcxxabi', when='+libcxx@3.7.0') ########## def install(self, spec, prefix): -- cgit v1.2.3-70-g09d2 From 25f2b01a3cbf74fa86f349287e8026ea422fbeb3 Mon Sep 17 00:00:00 2001 From: Tom Scogland Date: Thu, 10 Dec 2015 07:27:06 -0800 Subject: fetch: add options to fetch missing or all deps Small additions to fetch to make it easier to fetch all files necessary for a build on a system without network connectivity. --- lib/spack/spack/cmd/fetch.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/cmd/fetch.py b/lib/spack/spack/cmd/fetch.py index 6f9e7ab5e2..04cad80bd2 100644 --- a/lib/spack/spack/cmd/fetch.py +++ b/lib/spack/spack/cmd/fetch.py @@ -33,10 +33,13 @@ def setup_parser(subparser): subparser.add_argument( '-n', '--no-checksum', action='store_true', dest='no_checksum', help="Do not check packages against checksum") + subparser.add_argument( + '-m', '--missing', action='store_true', help="Also fetch all missing dependencies") + subparser.add_argument( + '-d', '--dependencies', action='store_true', help="Also fetch all dependencies") subparser.add_argument( 'packages', nargs=argparse.REMAINDER, help="specs of packages to fetch") - def fetch(parser, args): if not args.packages: tty.die("fetch requires at least one package argument") @@ -46,5 +49,13 @@ def fetch(parser, args): specs = spack.cmd.parse_specs(args.packages, concretize=True) for spec in specs: + if args.missing or args.dependencies: + to_fetch = set() + for s in spec.traverse(): + package = spack.db.get(s) + if args.missing and package.installed: + continue + package.do_fetch() + package = spack.db.get(spec) package.do_fetch() -- cgit v1.2.3-70-g09d2 From 786f4cd2c2cc492f25f8886223efaa397ac97185 Mon Sep 17 00:00:00 2001 From: Tom Scogland Date: Thu, 10 Dec 2015 08:32:51 -0800 Subject: add dependency fetching to mirror creation --- lib/spack/spack/cmd/fetch.py | 2 +- lib/spack/spack/cmd/mirror.py | 11 ++++++++++- var/spack/packages/py-cffi/package.py | 2 +- 3 files changed, 12 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/cmd/fetch.py b/lib/spack/spack/cmd/fetch.py index 04cad80bd2..57d6f6b63b 100644 --- a/lib/spack/spack/cmd/fetch.py +++ b/lib/spack/spack/cmd/fetch.py @@ -36,7 +36,7 @@ def setup_parser(subparser): subparser.add_argument( '-m', '--missing', action='store_true', help="Also fetch all missing dependencies") subparser.add_argument( - '-d', '--dependencies', action='store_true', help="Also fetch all dependencies") + '-D', '--dependencies', action='store_true', help="Also fetch all dependencies") subparser.add_argument( 'packages', nargs=argparse.REMAINDER, help="specs of packages to fetch") diff --git a/lib/spack/spack/cmd/mirror.py b/lib/spack/spack/cmd/mirror.py index 4a1ce00b75..10348e5b99 100644 --- a/lib/spack/spack/cmd/mirror.py +++ b/lib/spack/spack/cmd/mirror.py @@ -54,6 +54,8 @@ def setup_parser(subparser): 'specs', nargs=argparse.REMAINDER, help="Specs of packages to put in mirror") create_parser.add_argument( '-f', '--file', help="File with specs of packages to put in mirror.") + create_parser.add_argument( + '-D', '--dependencies', action='store_true', help="Also fetch all dependencies") create_parser.add_argument( '-o', '--one-version-per-spec', action='store_const', const=1, default=0, help="Only fetch one 'preferred' version per spec, not all known versions.") @@ -118,7 +120,7 @@ def mirror_create(args): """Create a directory to be used as a spack mirror, and fill it with package archives.""" # try to parse specs from the command line first. - specs = spack.cmd.parse_specs(args.specs) + specs = spack.cmd.parse_specs(args.specs, concretize=True) # If there is a file, parse each line as a spec and add it to the list. if args.file: @@ -131,6 +133,13 @@ def mirror_create(args): specs = [Spec(n) for n in spack.db.all_package_names()] specs.sort(key=lambda s: s.format("$_$@").lower()) + if args.dependencies: + new_specs = set() + for spec in specs: + for s in spec.traverse(): + new_specs.add(s) + specs = list(new_specs) + # Default name for directory is spack-mirror- directory = args.directory if not directory: diff --git a/var/spack/packages/py-cffi/package.py b/var/spack/packages/py-cffi/package.py index a4d37483fe..909049a67c 100644 --- a/var/spack/packages/py-cffi/package.py +++ b/var/spack/packages/py-cffi/package.py @@ -4,7 +4,7 @@ class PyCffi(Package): """Foreign Function Interface for Python calling C code""" homepage = "http://cffi.readthedocs.org/en/latest/" # base https://pypi.python.org/pypi/cffi - url = "https://pypi.python.org/packages/source/c/cffi/cffi-1.1.2.tar.gz#md5=" + url = "https://pypi.python.org/packages/source/c/cffi/cffi-1.1.2.tar.gz" version('1.1.2', 'ca6e6c45b45caa87aee9adc7c796eaea') -- cgit v1.2.3-70-g09d2 From 281a869ef699c354f31bbb069ab28012b65c9b4c Mon Sep 17 00:00:00 2001 From: Tom Scogland Date: Thu, 10 Dec 2015 09:24:01 -0800 Subject: fix path resolution for mirror packages, especially with dependency fetching --- lib/spack/spack/cmd/mirror.py | 1 + lib/spack/spack/mirror.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/cmd/mirror.py b/lib/spack/spack/cmd/mirror.py index 10348e5b99..89d51bbe04 100644 --- a/lib/spack/spack/cmd/mirror.py +++ b/lib/spack/spack/cmd/mirror.py @@ -136,6 +136,7 @@ def mirror_create(args): if args.dependencies: new_specs = set() for spec in specs: + spec.concretize() for s in spec.traverse(): new_specs.add(s) specs = list(new_specs) diff --git a/lib/spack/spack/mirror.py b/lib/spack/spack/mirror.py index ee0bf6de11..6fbf82de14 100644 --- a/lib/spack/spack/mirror.py +++ b/lib/spack/spack/mirror.py @@ -146,7 +146,7 @@ def create(path, specs, **kwargs): stage = None try: # create a subdirectory for the current package@version - archive_path = os.path.abspath(join_path(path, mirror_archive_path(spec))) + archive_path = os.path.abspath(join_path(mirror_root, mirror_archive_path(spec))) subdir = os.path.dirname(archive_path) mkdirp(subdir) -- cgit v1.2.3-70-g09d2 From 7526a89463c6f4c61cd4ae725fc096e14fc063f3 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Fri, 11 Dec 2015 02:16:24 -0800 Subject: Fix #217: Make package cache use DAG hash instead of sorted deps. - Gets rid of last vestige of old-style specs. - Uses new hashing for lookup --- lib/spack/spack/packages.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/packages.py b/lib/spack/spack/packages.py index f6f4cbf025..9d87c3a94b 100644 --- a/lib/spack/spack/packages.py +++ b/lib/spack/spack/packages.py @@ -67,27 +67,28 @@ class PackageDB(object): if spec.virtual: raise UnknownPackageError(spec.name) + dhash = spec.dag_hash() if kwargs.get('new', False): - if spec in self.instances: - del self.instances[spec] + if dhash in self.instances: + del self.instances[dhash] - if not spec in self.instances: + if not dhash in self.instances: package_class = self.get_class_for_package_name(spec.name) try: - copy = spec.copy() - self.instances[copy] = package_class(copy) + copy = spec.copy() # defensive copy. Package owns its spec. + self.instances[dhash] = package_class(copy) except Exception, e: if spack.debug: sys.excepthook(*sys.exc_info()) raise FailedConstructorError(spec.name, e) - return self.instances[spec] + return self.instances[dhash] @_autospec def delete(self, spec): """Force a package to be recreated.""" - del self.instances[spec] + del self.instances[spec.dag_hash()] def purge(self): -- cgit v1.2.3-70-g09d2 From 6ee2eb21dd2c3c32d115ebf28becc735311bb1a9 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Fri, 11 Dec 2015 12:29:32 -0800 Subject: Fix #217: Use MUCH faster hashing, reduce number of DAG copies. This changes the hash algorithm so that it does much less object allocation and copying, and so that it is correct. The old version of `_cmp_key()` would call `sorted_deps`, which would call `flat_dependencies` to get a list of dependencies so that it could sort them in alphabetical order. This isn't necessary in the `_cmp_key()`, and in fact we want more DAG structure than that to be included in the `_cmp_key()`. The new version constructs a tuple without copying the Spec DAG, and the tuple contains hashes of sub-DAGs that are computed recursively in-place. This is way faster than the previous algorithm and reduces the numebr of copies significantly. It is also a correct DAG hash. Example timing and copy counts for the different hashing algorithms we've tried: Original (wrong) Spec hash: ``` 106,170 copies real 0m5.024s user 0m4.949s sys 0m0.104s ``` Spec hash using YAML `dag_hash()`: ``` 3,794 copies real 0m5.024s user 0m4.949s sys 0m0.104s New no-copy, no-YAML hash: ``` 3,594 copies real 0m2.543s user 0m2.435s sys 0m0.104s ``` So now we have a hash that is correct AND faster. The remaining ~3k copies happen mostly during concretization, and as all packages are initially loaded. I believe this is because Spack currently has to load all packages to figure out virtual dependency information; it could also be becasue there ar a lot of lookups of partial specs in concretize. I can investigate this further. --- lib/spack/spack/packages.py | 12 ++++++------ lib/spack/spack/spec.py | 18 ++++++++++++------ 2 files changed, 18 insertions(+), 12 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/packages.py b/lib/spack/spack/packages.py index 9d87c3a94b..080644fb90 100644 --- a/lib/spack/spack/packages.py +++ b/lib/spack/spack/packages.py @@ -67,22 +67,22 @@ class PackageDB(object): if spec.virtual: raise UnknownPackageError(spec.name) - dhash = spec.dag_hash() + key = hash(spec) if kwargs.get('new', False): - if dhash in self.instances: - del self.instances[dhash] + if key in self.instances: + del self.instances[key] - if not dhash in self.instances: + if not key in self.instances: package_class = self.get_class_for_package_name(spec.name) try: copy = spec.copy() # defensive copy. Package owns its spec. - self.instances[dhash] = package_class(copy) + self.instances[key] = package_class(copy) except Exception, e: if spack.debug: sys.excepthook(*sys.exc_info()) raise FailedConstructorError(spec.name, e) - return self.instances[dhash] + return self.instances[key] @_autospec diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index f62182ce76..483da03c47 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -1481,8 +1481,11 @@ class Spec(object): def _cmp_node(self): """Comparison key for just *this node* and not its deps.""" - return (self.name, self.versions, self.variants, - self.architecture, self.compiler) + return (self.name, + self.versions, + self.variants, + self.architecture, + self.compiler) def eq_node(self, other): @@ -1496,11 +1499,14 @@ class Spec(object): def _cmp_key(self): - """Comparison key for this node and all dependencies *without* - considering structure. This is the default, as - normalization will restore structure. + """This returns a key for the spec *including* DAG structure. + + The key is the concatenation of: + 1. A tuple describing this node in the DAG. + 2. The hash of each of this node's dependencies' cmp_keys. """ - return self._cmp_node() + (self.sorted_deps(),) + return self._cmp_node() + ( + tuple(sorted(hash(d) for d in self.dependencies.values())),) def colorized(self): -- cgit v1.2.3-70-g09d2 From 3dd6cbc556458ff5776988d12927b9d93c326b64 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Fri, 11 Dec 2015 13:07:20 -0800 Subject: Fix #217: update spec_dag test for new `_cmp_key`. --- lib/spack/spack/test/spec_dag.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/test/spec_dag.py b/lib/spack/spack/test/spec_dag.py index 94438b6a0c..d3a4d77b32 100644 --- a/lib/spack/spack/test/spec_dag.py +++ b/lib/spack/spack/test/spec_dag.py @@ -340,16 +340,18 @@ class SpecDagTest(MockPackagesTest): self.assertEqual(spec, expected_flat) self.assertTrue(spec.eq_dag(expected_flat)) - self.assertEqual(spec, expected_normalized) + # Normalized has different DAG structure, so NOT equal. + self.assertNotEqual(spec, expected_normalized) self.assertFalse(spec.eq_dag(expected_normalized)) - self.assertEqual(spec, non_unique_nodes) + # Again, different DAG structure so not equal. + self.assertNotEqual(spec, non_unique_nodes) self.assertFalse(spec.eq_dag(non_unique_nodes)) spec.normalize() # After normalizing, spec_dag_equal should match the normalized spec. - self.assertEqual(spec, expected_flat) + self.assertNotEqual(spec, expected_flat) self.assertFalse(spec.eq_dag(expected_flat)) self.assertEqual(spec, expected_normalized) -- cgit v1.2.3-70-g09d2 From afbd0e77d0754227b799ae3872d8775ceec0ccd0 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Fri, 11 Dec 2015 16:47:34 -0800 Subject: Make internal hash dep sort order match external one. --- lib/spack/spack/spec.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 483da03c47..fb5ea5d02b 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -1506,7 +1506,8 @@ class Spec(object): 2. The hash of each of this node's dependencies' cmp_keys. """ return self._cmp_node() + ( - tuple(sorted(hash(d) for d in self.dependencies.values())),) + tuple(hash(self.dependencies[name]) + for name in sorted(self.dependencies)),) def colorized(self): -- cgit v1.2.3-70-g09d2 From 6e074a196a25f5737e50c57bf64b1d452245ef50 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 14 Dec 2015 21:06:32 -0800 Subject: Fix #206: need to make deps AND root concrete when read in. --- lib/spack/spack/directory_layout.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/directory_layout.py b/lib/spack/spack/directory_layout.py index 056606b429..a434dad5c4 100644 --- a/lib/spack/spack/directory_layout.py +++ b/lib/spack/spack/directory_layout.py @@ -212,8 +212,10 @@ class YamlDirectoryLayout(DirectoryLayout): spec = Spec.from_yaml(f) # Specs read from actual installations are always concrete - spec._normal = True - spec._concrete = True + for s in spec.traverse(): + s._normal = True + s._concrete = True + return spec -- cgit v1.2.3-70-g09d2 From 2f9ee884087005d90a9d63d29667cca1634fafca Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 15 Dec 2015 17:15:37 -0800 Subject: Fix #238: Database specs need to be concrete when read in as well. --- lib/spack/spack/database.py | 4 ++++ lib/spack/spack/directory_layout.py | 4 +--- lib/spack/spack/spec.py | 13 ++++++++++++- 3 files changed, 17 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/database.py b/lib/spack/spack/database.py index d62a47547d..bf54055a24 100644 --- a/lib/spack/spack/database.py +++ b/lib/spack/spack/database.py @@ -211,6 +211,10 @@ class Database(object): child = self._read_spec_from_yaml(dep_hash, installs, hash_key) spec._add_dependency(child) + # Specs from the database need to be marked concrete because + # they represent actual installations. + spec._mark_concrete() + return spec diff --git a/lib/spack/spack/directory_layout.py b/lib/spack/spack/directory_layout.py index a434dad5c4..d91fbe9f4e 100644 --- a/lib/spack/spack/directory_layout.py +++ b/lib/spack/spack/directory_layout.py @@ -212,9 +212,7 @@ class YamlDirectoryLayout(DirectoryLayout): spec = Spec.from_yaml(f) # Specs read from actual installations are always concrete - for s in spec.traverse(): - s._normal = True - s._concrete = True + spec._mark_concrete() return spec diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index fb5ea5d02b..037ec97a5e 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -833,7 +833,18 @@ class Spec(object): changed = any(changes) force=True - self._concrete = True + self._mark_concrete() + + + def _mark_concrete(self): + """Mark this spec and its dependencies as concrete. + + Only for internal use -- client code should use "concretize" + unless there is a need to force a spec to be concrete. + """ + for s in self.traverse(): + s._normal = True + s._concrete = True def concretized(self): -- cgit v1.2.3-70-g09d2 From 82aa366adfd5b8e0acd6d320c620bf33dd8e6a68 Mon Sep 17 00:00:00 2001 From: Mike Nolta Date: Wed, 16 Dec 2015 11:43:07 -0500 Subject: minor comment tweak in modules.py --- lib/spack/spack/modules.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py index 59ed9f893e..7036626e29 100644 --- a/lib/spack/spack/modules.py +++ b/lib/spack/spack/modules.py @@ -29,11 +29,11 @@ The various types of modules are installed by post-install hooks and removed after an uninstall by post-uninstall hooks. This class consolidates the logic for creating an abstract description of the information that module systems need. Currently that includes a -number directories to be appended to paths in the user's environment: +number of directories to be appended to paths in the user's environment: * /bin directories to be appended to PATH * /lib* directories for LD_LIBRARY_PATH - * /man* and /share/man* directories for LD_LIBRARY_PATH + * /man* and /share/man* directories for MANPATH * the package prefix for CMAKE_PREFIX_PATH This module also includes logic for coming up with unique names for -- cgit v1.2.3-70-g09d2 From abd3d3946d8a4d5b73f26a682c1728cf9d4d05df Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Fri, 18 Dec 2015 19:23:45 -0800 Subject: Format to 80 char width. --- lib/spack/spack/cmd/uninstall.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/cmd/uninstall.py b/lib/spack/spack/cmd/uninstall.py index 191d9d88e8..03873bb5f8 100644 --- a/lib/spack/spack/cmd/uninstall.py +++ b/lib/spack/spack/cmd/uninstall.py @@ -42,9 +42,9 @@ def setup_parser(subparser): help="Remove regardless of whether other packages depend on this one.") subparser.add_argument( '-a', '--all', action='store_true', dest='all', - help="USE CAREFULLY. Remove ALL installed packages that match each supplied spec. " + - "i.e., if you say uninstall libelf, ALL versions of libelf are uninstalled. " + - "This is both useful and dangerous, like rm -r.") + help="USE CAREFULLY. Remove ALL installed packages that match each " + + "supplied spec. i.e., if you say uninstall libelf, ALL versions of " + + "libelf are uninstalled. This is both useful and dangerous, like rm -r.") subparser.add_argument( 'packages', nargs=argparse.REMAINDER, help="specs of packages to uninstall") @@ -81,7 +81,8 @@ def uninstall(parser, args): pkgs.append(s.package) except spack.packages.UnknownPackageError, e: - # The package.py file has gone away -- but still want to uninstall. + # The package.py file has gone away -- but still want to + # uninstall. spack.Package(s).do_uninstall(force=True) # Sort packages to be uninstalled by the number of installed dependents -- cgit v1.2.3-70-g09d2 From ce8bd2094403ca5844f164e65b22ad4cfdab4a9a Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Fri, 18 Dec 2015 22:23:51 -0800 Subject: Fix import error. --- lib/spack/spack/database.py | 1 + 1 file changed, 1 insertion(+) (limited to 'lib') diff --git a/lib/spack/spack/database.py b/lib/spack/spack/database.py index bf54055a24..a6f1cc5077 100644 --- a/lib/spack/spack/database.py +++ b/lib/spack/spack/database.py @@ -54,6 +54,7 @@ import spack.spec from spack.version import Version from spack.spec import Spec from spack.error import SpackError +from spack.packages import UnknownPackageError # DB goes in this directory underneath the root _db_dirname = '.spack-db' -- cgit v1.2.3-70-g09d2 From b7e926eef6a248ec6f67cf30bdb2ce7dea09b8a2 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Fri, 18 Dec 2015 22:24:22 -0800 Subject: minor formatting. --- lib/spack/spack/cmd/extensions.py | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'lib') diff --git a/lib/spack/spack/cmd/extensions.py b/lib/spack/spack/cmd/extensions.py index c2cf288877..2ce6f406ca 100644 --- a/lib/spack/spack/cmd/extensions.py +++ b/lib/spack/spack/cmd/extensions.py @@ -54,7 +54,9 @@ def extensions(parser, args): if not args.spec: tty.die("extensions requires a package spec.") + # # Checks + # spec = spack.cmd.parse_specs(args.spec) if len(spec) > 1: tty.die("Can only list extensions for one package.") @@ -70,7 +72,9 @@ def extensions(parser, args): if not args.mode: args.mode = 'short' + # # List package names of extensions + # extensions = spack.db.extensions_for(spec) if not extensions: tty.msg("%s has no extensions." % spec.cshort_spec) @@ -79,7 +83,9 @@ def extensions(parser, args): tty.msg("%d extensions:" % len(extensions)) colify(ext.name for ext in extensions) + # # List specs of installed extensions. + # installed = [s.spec for s in spack.installed_db.installed_extensions_for(spec)] print if not installed: @@ -88,7 +94,9 @@ def extensions(parser, args): tty.msg("%d installed:" % len(installed)) spack.cmd.find.display_specs(installed, mode=args.mode) + # # List specs of activated extensions. + # activated = spack.install_layout.extension_map(spec) print if not activated: -- cgit v1.2.3-70-g09d2 From 52e3364efa4fa18d392316371e7164bf3899a638 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Fri, 18 Dec 2015 22:24:35 -0800 Subject: Fix #244: errors on uninstall - Extension logic didn't take conditional deps into account. - Extension methods now check for whether the extnesion is in the extendee map AND whether the dependency is actually present in the spec yet. --- lib/spack/spack/hooks/extensions.py | 4 +--- lib/spack/spack/package.py | 28 ++++++++++++++++++++++------ 2 files changed, 23 insertions(+), 9 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/hooks/extensions.py b/lib/spack/spack/hooks/extensions.py index b4847d697f..627184cabd 100644 --- a/lib/spack/spack/hooks/extensions.py +++ b/lib/spack/spack/hooks/extensions.py @@ -27,9 +27,7 @@ import spack def pre_uninstall(pkg): - # Need to do this b/c uninstall does not automatically do it. - # TODO: store full graph info in stored .spec file. - pkg.spec.normalize() + assert(pkg.spec.concrete) if pkg.is_extension: if pkg.activated: diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index daba5cd352..4d75726e06 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -487,9 +487,15 @@ class Package(object): if name == dep.name: return dep - # Otherwise return the spec from the extends() directive - spec, kwargs = self.extendees[name] - return spec + # if the spec is concrete already, then it extends something + # that is an *optional* dependency, and the dep isn't there. + if self.spec._concrete: + return None + else: + # If it's not concrete, then return the spec from the + # extends() directive since that is all we know so far. + spec, kwargs = self.extendees[name] + return spec @property @@ -497,18 +503,28 @@ class Package(object): """Spec of the extendee of this package, or None if it is not an extension.""" if not self.extendees: return None + + # TODO: allow multiple extendees. name = next(iter(self.extendees)) return self.extendees[name][1] @property def is_extension(self): - return len(self.extendees) > 0 + # if it is concrete, it's only an extension if it actually + # dependes on the extendee. + if self.spec._concrete: + return self.extendee_spec is not None + else: + # If not, then it's an extension if it *could* be an extension + return bool(self.extendees) def extends(self, spec): - return (spec.name in self.extendees and - spec.satisfies(self.extendees[spec.name][0])) + if not spec.name in self.extendees: + return False + s = self.extendee_spec + return s and s.satisfies(spec) @property -- cgit v1.2.3-70-g09d2 From 7cc4d88726501d6d93adf86f8080e65523e2400f Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sat, 19 Dec 2015 02:06:47 -0800 Subject: Fix #254: libtool & distutils want certain compiler names. This changes the compiler wrappers so that they are called by the same name as the wrapped compiler. Many builds make assumptions about compiler names, and we need the spack compilers to be recognizable so that build systems will get their flags right. This adds per-compiler subdirectories to lib/spack/spack/env directory that contain symlinks to cc for the C, C++, F77, and F90 compilers. The build now sets CC, CXX, F77, and F90 to point to these links instead of to the generically named cc, c++, f77, and f90 wrappers. --- lib/spack/env/cc | 14 +++++++------- lib/spack/env/clang/clang | 1 + lib/spack/env/clang/clang++ | 1 + lib/spack/env/gcc/g++ | 1 + lib/spack/env/gcc/gcc | 1 + lib/spack/env/gcc/gfortran | 1 + lib/spack/env/intel/icc | 1 + lib/spack/env/intel/icpc | 1 + lib/spack/env/intel/ifort | 1 + lib/spack/env/pgi/case-insensitive/pgCC | 1 + lib/spack/env/pgi/pgcc | 1 + lib/spack/env/pgi/pgf77 | 1 + lib/spack/env/pgi/pgf90 | 1 + lib/spack/env/xl/xlc | 1 + lib/spack/env/xl/xlc++ | 1 + lib/spack/env/xl/xlf | 1 + lib/spack/env/xl/xlf90 | 1 + lib/spack/spack/build_environment.py | 32 ++++++++++++++++++++++++-------- lib/spack/spack/compilers/clang.py | 6 ++++++ lib/spack/spack/compilers/gcc.py | 6 ++++++ lib/spack/spack/compilers/intel.py | 6 ++++++ lib/spack/spack/compilers/pgi.py | 6 ++++++ lib/spack/spack/compilers/xl.py | 6 ++++++ 23 files changed, 77 insertions(+), 15 deletions(-) create mode 120000 lib/spack/env/clang/clang create mode 120000 lib/spack/env/clang/clang++ create mode 120000 lib/spack/env/gcc/g++ create mode 120000 lib/spack/env/gcc/gcc create mode 120000 lib/spack/env/gcc/gfortran create mode 120000 lib/spack/env/intel/icc create mode 120000 lib/spack/env/intel/icpc create mode 120000 lib/spack/env/intel/ifort create mode 120000 lib/spack/env/pgi/case-insensitive/pgCC create mode 120000 lib/spack/env/pgi/pgcc create mode 120000 lib/spack/env/pgi/pgf77 create mode 120000 lib/spack/env/pgi/pgf90 create mode 120000 lib/spack/env/xl/xlc create mode 120000 lib/spack/env/xl/xlc++ create mode 120000 lib/spack/env/xl/xlf create mode 120000 lib/spack/env/xl/xlf90 (limited to 'lib') diff --git a/lib/spack/env/cc b/lib/spack/env/cc index 75a63f6fac..0966277a91 100755 --- a/lib/spack/env/cc +++ b/lib/spack/env/cc @@ -86,22 +86,22 @@ done # command=$(basename "$0") case "$command" in - cc|gcc|c89|c99|clang|xlc) + cc|c89|c99|gcc|clang|icc|pgcc|xlc) command="$SPACK_CC" language="C" ;; - c++|CC|g++|clang++|xlC) + c++|CC|g++|clang++|icpc|pgCC|xlc++) command="$SPACK_CXX" language="C++" ;; - f77|xlf) - command="$SPACK_F77" - language="Fortran 77" - ;; - fc|f90|f95|xlf90) + f90|fc|f95|gfortran|ifort|pgf90|xlf90) command="$SPACK_FC" language="Fortran 90" ;; + f77|gfortran|ifort|pgf77|xlf) + command="$SPACK_F77" + language="Fortran 77" + ;; cpp) mode=cpp ;; diff --git a/lib/spack/env/clang/clang b/lib/spack/env/clang/clang new file mode 120000 index 0000000000..82c2b8e90a --- /dev/null +++ b/lib/spack/env/clang/clang @@ -0,0 +1 @@ +../cc \ No newline at end of file diff --git a/lib/spack/env/clang/clang++ b/lib/spack/env/clang/clang++ new file mode 120000 index 0000000000..82c2b8e90a --- /dev/null +++ b/lib/spack/env/clang/clang++ @@ -0,0 +1 @@ +../cc \ No newline at end of file diff --git a/lib/spack/env/gcc/g++ b/lib/spack/env/gcc/g++ new file mode 120000 index 0000000000..82c2b8e90a --- /dev/null +++ b/lib/spack/env/gcc/g++ @@ -0,0 +1 @@ +../cc \ No newline at end of file diff --git a/lib/spack/env/gcc/gcc b/lib/spack/env/gcc/gcc new file mode 120000 index 0000000000..82c2b8e90a --- /dev/null +++ b/lib/spack/env/gcc/gcc @@ -0,0 +1 @@ +../cc \ No newline at end of file diff --git a/lib/spack/env/gcc/gfortran b/lib/spack/env/gcc/gfortran new file mode 120000 index 0000000000..82c2b8e90a --- /dev/null +++ b/lib/spack/env/gcc/gfortran @@ -0,0 +1 @@ +../cc \ No newline at end of file diff --git a/lib/spack/env/intel/icc b/lib/spack/env/intel/icc new file mode 120000 index 0000000000..82c2b8e90a --- /dev/null +++ b/lib/spack/env/intel/icc @@ -0,0 +1 @@ +../cc \ No newline at end of file diff --git a/lib/spack/env/intel/icpc b/lib/spack/env/intel/icpc new file mode 120000 index 0000000000..82c2b8e90a --- /dev/null +++ b/lib/spack/env/intel/icpc @@ -0,0 +1 @@ +../cc \ No newline at end of file diff --git a/lib/spack/env/intel/ifort b/lib/spack/env/intel/ifort new file mode 120000 index 0000000000..82c2b8e90a --- /dev/null +++ b/lib/spack/env/intel/ifort @@ -0,0 +1 @@ +../cc \ No newline at end of file diff --git a/lib/spack/env/pgi/case-insensitive/pgCC b/lib/spack/env/pgi/case-insensitive/pgCC new file mode 120000 index 0000000000..e2deb67f3b --- /dev/null +++ b/lib/spack/env/pgi/case-insensitive/pgCC @@ -0,0 +1 @@ +../../cc \ No newline at end of file diff --git a/lib/spack/env/pgi/pgcc b/lib/spack/env/pgi/pgcc new file mode 120000 index 0000000000..82c2b8e90a --- /dev/null +++ b/lib/spack/env/pgi/pgcc @@ -0,0 +1 @@ +../cc \ No newline at end of file diff --git a/lib/spack/env/pgi/pgf77 b/lib/spack/env/pgi/pgf77 new file mode 120000 index 0000000000..82c2b8e90a --- /dev/null +++ b/lib/spack/env/pgi/pgf77 @@ -0,0 +1 @@ +../cc \ No newline at end of file diff --git a/lib/spack/env/pgi/pgf90 b/lib/spack/env/pgi/pgf90 new file mode 120000 index 0000000000..82c2b8e90a --- /dev/null +++ b/lib/spack/env/pgi/pgf90 @@ -0,0 +1 @@ +../cc \ No newline at end of file diff --git a/lib/spack/env/xl/xlc b/lib/spack/env/xl/xlc new file mode 120000 index 0000000000..82c2b8e90a --- /dev/null +++ b/lib/spack/env/xl/xlc @@ -0,0 +1 @@ +../cc \ No newline at end of file diff --git a/lib/spack/env/xl/xlc++ b/lib/spack/env/xl/xlc++ new file mode 120000 index 0000000000..82c2b8e90a --- /dev/null +++ b/lib/spack/env/xl/xlc++ @@ -0,0 +1 @@ +../cc \ No newline at end of file diff --git a/lib/spack/env/xl/xlf b/lib/spack/env/xl/xlf new file mode 120000 index 0000000000..82c2b8e90a --- /dev/null +++ b/lib/spack/env/xl/xlf @@ -0,0 +1 @@ +../cc \ No newline at end of file diff --git a/lib/spack/env/xl/xlf90 b/lib/spack/env/xl/xlf90 new file mode 120000 index 0000000000..82c2b8e90a --- /dev/null +++ b/lib/spack/env/xl/xlf90 @@ -0,0 +1 @@ +../cc \ No newline at end of file diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index dac25d9940..20154a55b3 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -88,10 +88,14 @@ def set_compiler_environment_variables(pkg): compiler = pkg.compiler # Set compiler variables used by CMake and autotools - os.environ['CC'] = join_path(spack.build_env_path, 'cc') - os.environ['CXX'] = join_path(spack.build_env_path, 'c++') - os.environ['F77'] = join_path(spack.build_env_path, 'f77') - os.environ['FC'] = join_path(spack.build_env_path, 'f90') + assert all(key in pkg.compiler.link_paths + for key in ('cc', 'cxx', 'f77', 'fc')) + + link_dir = spack.build_env_path + os.environ['CC'] = join_path(link_dir, pkg.compiler.link_paths['cc']) + os.environ['CXX'] = join_path(link_dir, pkg.compiler.link_paths['cxx']) + os.environ['F77'] = join_path(link_dir, pkg.compiler.link_paths['f77']) + os.environ['FC'] = join_path(link_dir, pkg.compiler.link_paths['fc']) # Set SPACK compiler variables so that our wrapper knows what to call if compiler.cc: @@ -110,11 +114,23 @@ def set_build_environment_variables(pkg): """This ensures a clean install environment when we build packages. """ # Add spack build environment path with compiler wrappers first in - # the path. We handle case sensitivity conflicts like "CC" and - # "cc" by putting one in the /case-insensitive + # the path. We add both spack.env_path, which includes default + # wrappers (cc, c++, f77, f90), AND a subdirectory containing + # compiler-specific symlinks. The latter ensures that builds that + # are sensitive to the *name* of the compiler see the right name + # when we're building wtih the wrappers. + # + # Conflicts on case-insensitive systems (like "CC" and "cc") are + # handled by putting one in the /case-insensitive # directory. Add that to the path too. - env_paths = [spack.build_env_path, - join_path(spack.build_env_path, 'case-insensitive')] + env_paths = [] + def add_env_path(path): + env_paths.append(path) + ci = join_path(path, 'case-insensitive') + if os.path.isdir(ci): env_paths.append(ci) + add_env_path(spack.build_env_path) + add_env_path(join_path(spack.build_env_path, pkg.compiler.name)) + path_put_first("PATH", env_paths) path_set(SPACK_ENV_PATH, env_paths) diff --git a/lib/spack/spack/compilers/clang.py b/lib/spack/spack/compilers/clang.py index b34d1f2f9c..340051019c 100644 --- a/lib/spack/spack/compilers/clang.py +++ b/lib/spack/spack/compilers/clang.py @@ -37,6 +37,12 @@ class Clang(Compiler): # Subclasses use possible names of Fortran 90 compiler fc_names = [] + # Named wrapper links within spack.build_env_path + link_paths = { 'cc' : 'clang/clang', + 'cxx' : 'clang/clang++', + # Use default wrappers for fortran, in case provided in compilers.yaml + 'f77' : 'f77', + 'fc' : 'f90' } @classmethod def default_version(self, comp): diff --git a/lib/spack/spack/compilers/gcc.py b/lib/spack/spack/compilers/gcc.py index 2886888d57..495b638a3a 100644 --- a/lib/spack/spack/compilers/gcc.py +++ b/lib/spack/spack/compilers/gcc.py @@ -42,6 +42,12 @@ class Gcc(Compiler): # MacPorts builds gcc versions with prefixes and -mp-X.Y suffixes. suffixes = [r'-mp-\d\.\d'] + # Named wrapper links within spack.build_env_path + link_paths = {'cc' : 'gcc/gcc', + 'cxx' : 'gcc/g++', + 'f77' : 'gcc/gfortran', + 'fc' : 'gcc/gfortran' } + @property def cxx11_flag(self): if self.version < ver('4.3'): diff --git a/lib/spack/spack/compilers/intel.py b/lib/spack/spack/compilers/intel.py index 1298429974..69e9764790 100644 --- a/lib/spack/spack/compilers/intel.py +++ b/lib/spack/spack/compilers/intel.py @@ -37,6 +37,12 @@ class Intel(Compiler): # Subclasses use possible names of Fortran 90 compiler fc_names = ['ifort'] + # Named wrapper links within spack.build_env_path + link_paths = { 'cc' : 'intel/icc', + 'cxx' : 'intel/icpc', + 'f77' : 'intel/ifort', + 'fc' : 'intel/ifort' } + @property def cxx11_flag(self): if self.version < ver('11.1'): diff --git a/lib/spack/spack/compilers/pgi.py b/lib/spack/spack/compilers/pgi.py index 6999eb50de..9ac74cfbdb 100644 --- a/lib/spack/spack/compilers/pgi.py +++ b/lib/spack/spack/compilers/pgi.py @@ -37,6 +37,12 @@ class Pgi(Compiler): # Subclasses use possible names of Fortran 90 compiler fc_names = ['pgf95', 'pgf90'] + # Named wrapper links within spack.build_env_path + link_paths = { 'cc' : 'pgi/pgcc', + 'cxx' : 'pgi/case-insensitive/pgCC', + 'f77' : 'pgi/pgf77', + 'fc' : 'pgi/pgf90' } + @classmethod def default_version(cls, comp): """The '-V' option works for all the PGI compilers. diff --git a/lib/spack/spack/compilers/xl.py b/lib/spack/spack/compilers/xl.py index 308f811eb4..c1d55109a3 100644 --- a/lib/spack/spack/compilers/xl.py +++ b/lib/spack/spack/compilers/xl.py @@ -38,6 +38,12 @@ class Xl(Compiler): # Subclasses use possible names of Fortran 90 compiler fc_names = ['xlf90','xlf90_r','xlf95','xlf95_r','xlf2003','xlf2003_r','xlf2008','xlf2008_r'] + # Named wrapper links within spack.build_env_path + link_paths = { 'cc' : 'xl/xlc', + 'cxx' : 'xl/xlc++', + 'f77' : 'xl/xlf', + 'fc' : 'xl/xlf90' } + @property def cxx11_flag(self): if self.version < ver('13.1'): -- cgit v1.2.3-70-g09d2 From 429e15c4a64d29926f1b7a7168c23592e7dec0b7 Mon Sep 17 00:00:00 2001 From: Mike Nolta Date: Mon, 14 Dec 2015 15:22:51 -0500 Subject: spack python: add -c option Allows passing program in as a string. Example: $ spack python -c 'print 2+3' 5 Also imports spack module by default into the environment. --- lib/spack/spack/cmd/python.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/cmd/python.py b/lib/spack/spack/cmd/python.py index e26b8d3e79..5325e8fd9a 100644 --- a/lib/spack/spack/cmd/python.py +++ b/lib/spack/spack/cmd/python.py @@ -31,6 +31,8 @@ import platform import spack def setup_parser(subparser): + subparser.add_argument( + '-c', dest='python_command', help='Command to execute.') subparser.add_argument( 'python_args', nargs=argparse.REMAINDER, help="File to run plus arguments.") @@ -38,7 +40,8 @@ description = "Launch an interpreter as spack would launch a command" def python(parser, args): # Fake a main python shell by setting __name__ to __main__. - console = code.InteractiveConsole({'__name__' : '__main__'}) + console = code.InteractiveConsole({'__name__' : '__main__', + 'spack' : spack}) if "PYTHONSTARTUP" in os.environ: startup_file = os.environ["PYTHONSTARTUP"] @@ -47,7 +50,10 @@ def python(parser, args): console.runsource(startup.read(), startup_file, 'exec') python_args = args.python_args - if python_args: + python_command = args.python_command + if python_command: + console.runsource(python_command) + elif python_args: sys.argv = python_args with open(python_args[0]) as file: console.runsource(file.read(), python_args[0], 'exec') -- cgit v1.2.3-70-g09d2 From 32f96b5c73ff25d2a206423b514c831c2ab074fa Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sat, 19 Dec 2015 20:04:00 -0800 Subject: Fix stale code in python version test so that it actually works. --- lib/spack/external/pyqver2.py | 3 ++- lib/spack/spack/test/python_version.py | 4 ---- 2 files changed, 2 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/spack/external/pyqver2.py b/lib/spack/external/pyqver2.py index 4a16e2811e..4690239748 100755 --- a/lib/spack/external/pyqver2.py +++ b/lib/spack/external/pyqver2.py @@ -30,7 +30,8 @@ import sys StandardModules = { "__future__": (2, 1), "abc": (2, 6), - "argparse": (2, 7), +# skip argparse now that it's in lib/spack/external +# "argparse": (2, 7), "ast": (2, 6), "atexit": (2, 0), "bz2": (2, 3), diff --git a/lib/spack/spack/test/python_version.py b/lib/spack/spack/test/python_version.py index ba570b416f..2ea5febb11 100644 --- a/lib/spack/spack/test/python_version.py +++ b/lib/spack/spack/test/python_version.py @@ -63,10 +63,6 @@ class PythonVersionTest(unittest.TestCase): all_issues = {} for fn in files: - if fn != '/Users/gamblin2/src/spack/var/spack/packages/vim/package.py': - continue - print fn - with open(fn) as pyfile: versions = pyqver2.get_versions(pyfile.read()) for ver, reasons in versions.items(): -- cgit v1.2.3-70-g09d2 From fe0fdf60b46716fa20318c627ea80ff4e3398263 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 21 Dec 2015 15:35:47 -0800 Subject: Add ability to prefer particular versions in packages. - Adding `preferred=True` to a version directive will change its sort order in concretization. - This provides us a rudimentary ability to keep the Spack stack stable as new versions are added. - Having multiple stacks will come next, but this at least allows us to specify default versions of things instead of always taking the newest. --- lib/spack/spack/concretize.py | 11 +++++++++-- var/spack/packages/python/package.py | 2 +- 2 files changed, 10 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index d9419da784..85cdb202d5 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -40,7 +40,6 @@ import spack.error from spack.version import * - class DefaultConcretizer(object): """This class doesn't have any state, it just provides some methods for concretization. You can subclass it to override just some of the @@ -68,9 +67,17 @@ class DefaultConcretizer(object): # If there are known available versions, return the most recent # version that satisfies the spec pkg = spec.package + + # Key function to sort versions first by whether they were + # marked `preferred=True`, then by most recent. + def preferred_key(v): + prefer = pkg.versions[v].get('preferred', False) + return (prefer, v) + valid_versions = sorted( [v for v in pkg.versions - if any(v.satisfies(sv) for sv in spec.versions)]) + if any(v.satisfies(sv) for sv in spec.versions)], + key=preferred_key) if valid_versions: spec.versions = ver([valid_versions[-1]]) diff --git a/var/spack/packages/python/package.py b/var/spack/packages/python/package.py index 30e6ef6bb8..0da7f4ce89 100644 --- a/var/spack/packages/python/package.py +++ b/var/spack/packages/python/package.py @@ -17,7 +17,7 @@ class Python(Package): version('2.7.8', 'd235bdfa75b8396942e360a70487ee00') version('2.7.10', 'c685ef0b8e9f27b5e3db5db12b268ac6') - version('2.7.11', '1dbcc848b4cd8399a8199d000f9f823c') + version('2.7.11', '1dbcc848b4cd8399a8199d000f9f823c', preferred=True) version('3.5.0', 'd149d2812f10cbe04c042232e7964171') version('3.5.1', 'e9ea6f2623fffcdd871b7b19113fde80') -- cgit v1.2.3-70-g09d2 From 2ba6bb21fbac14b6eec9b4ba4fb7ea1f11e9df6f Mon Sep 17 00:00:00 2001 From: Mike Nolta Date: Mon, 21 Dec 2015 18:48:30 -0500 Subject: fix #249 (tmp spack-stage directories conflict) --- lib/spack/spack/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index 02eeed01fa..ccd7dc25c8 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -112,8 +112,9 @@ use_tmp_stage = True tmp_dirs = [] _default_tmp = tempfile.gettempdir() if _default_tmp != os.getcwd(): - tmp_dirs.append(os.path.join(_default_tmp, 'spack-stage')) -tmp_dirs.append('/nfs/tmp2/%u/spack-stage') + tmp_dirs = [ join_path(_default_tmp, '%u', 'spack-stage'), + join_path(_default_tmp, 'spack-stage') ] +tmp_dirs.append('/nfs/tmp2/%u/spack-stage') # TODO: remove # Whether spack should allow installation of unsafe versions of # software. "Unsafe" versions are ones it doesn't have a checksum -- cgit v1.2.3-70-g09d2 From 61f07b0c4d50c831d4723341c1a896c54d82c221 Mon Sep 17 00:00:00 2001 From: Mike Nolta Date: Sun, 20 Dec 2015 20:15:02 -0500 Subject: fix a few comment typos --- bin/spack | 2 +- lib/spack/spack/build_environment.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/bin/spack b/bin/spack index a0b260ccaf..cd46cf6180 100755 --- a/bin/spack +++ b/bin/spack @@ -150,7 +150,7 @@ def main(): sys.stderr.write('\n') tty.die("Keyboard interrupt.") - # Allow commands to return values if they want to exit with some ohter code. + # Allow commands to return values if they want to exit with some other code. if return_val is None: sys.exit(0) elif isinstance(return_val, int): diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 20154a55b3..84c2f6015b 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -145,7 +145,7 @@ def set_build_environment_variables(pkg): # Install root prefix os.environ[SPACK_INSTALL] = spack.install_path - # Remove these vars from the environment during build becaus they + # Remove these vars from the environment during build because they # can affect how some packages find libraries. We want to make # sure that builds never pull in unintended external dependencies. pop_keys(os.environ, "LD_LIBRARY_PATH", "LD_RUN_PATH", "DYLD_LIBRARY_PATH") -- cgit v1.2.3-70-g09d2 From 792b03325562e5577d018b188577915ea4dd5e7d Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 22 Dec 2015 07:54:25 -0800 Subject: Slightly more robust tmp directory search. - remove getcwd() check (seems arbitrary -- if users set their TMPDIR to this why stop them?) - try a number of common locations and try per-user directories in them first. --- lib/spack/spack/__init__.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index ccd7dc25c8..92cb417a85 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -24,6 +24,7 @@ ############################################################################## import os import tempfile +import getpass from llnl.util.filesystem import * # This lives in $prefix/lib/spack/spack/__file__ @@ -111,10 +112,17 @@ use_tmp_stage = True # that it can create. tmp_dirs = [] _default_tmp = tempfile.gettempdir() -if _default_tmp != os.getcwd(): - tmp_dirs = [ join_path(_default_tmp, '%u', 'spack-stage'), - join_path(_default_tmp, 'spack-stage') ] -tmp_dirs.append('/nfs/tmp2/%u/spack-stage') # TODO: remove +_tmp_user = getpass.getuser() + +_tmp_candidates = (_default_tmp, '/nfs/tmp2', '/tmp', '/var/tmp') +for path in _tmp_candidates: + # don't add a second username if it's already unique by user. + if not _tmp_user in path: + tmp_dirs.append(join_path(path, '%u', 'spack-stage')) + +for path in _tmp_candidates: + if not path in tmp_dirs: + tmp_dirs.append(join_path(path, 'spack-stage')) # Whether spack should allow installation of unsafe versions of # software. "Unsafe" versions are ones it doesn't have a checksum -- cgit v1.2.3-70-g09d2 From 2b89d9b1db620c3b4a0e391ffc6621717aa03382 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 22 Dec 2015 16:54:41 -0800 Subject: More consistent URL parsing when finding versions. Versions found by wildcard URLs are different from versions found by parse_version, etc. The wildcards are constructed more haphazardly than the very specific URL patterns in url.py, so they can get things wrong. e.g., for this URL: https://software.lanl.gov/MeshTools/trac/attachment/wiki/WikiStart/mstk-2.25rc1.tgz We miss the 'rc' and only return 2.25r as the version if we ONLY use URL wildcards. Future: Maybe use the regexes from url.py to scrape web pages, and then compare them for similarity with the original URL, instead of trying to make a structured wildcard URL pattern? This might yield better results. --- lib/spack/spack/package.py | 12 +++++++++--- lib/spack/spack/url.py | 2 +- 2 files changed, 10 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index b95afb073d..2e7678b4b0 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -1200,6 +1200,8 @@ def find_versions_of_archive(*archive_urls, **kwargs): for aurl in archive_urls: list_urls.add(spack.url.find_list_url(aurl)) + print list_urls + # Grab some web pages to scrape. page_map = {} for lurl in list_urls: @@ -1224,9 +1226,13 @@ def find_versions_of_archive(*archive_urls, **kwargs): for page_url, content in page_map.iteritems(): # extract versions from matches. for regex in regexes: - versions.update( - (Version(m.group(1)), urljoin(page_url, m.group(0))) - for m in re.finditer(regex, content)) + print regex + print + + for m in re.finditer(regex, content): + url = urljoin(page_url, m.group(0)) + ver = spack.url.parse_version(url) + versions[ver] = url return versions diff --git a/lib/spack/spack/url.py b/lib/spack/spack/url.py index 02c0b83e26..ed5a6d2a44 100644 --- a/lib/spack/spack/url.py +++ b/lib/spack/spack/url.py @@ -210,7 +210,7 @@ def parse_version_offset(path): (r'-((\d+\.)*\d+)$', stem), # e.g. foobar-4.5.1b, foobar4.5RC, foobar.v4.5.1b - (r'[-._]?v?((\d+\.)*\d+[-._]?([a-z]|rc|RC|tp|TP?)\d*)$', stem), + (r'[-._]?v?((\d+\.)*\d+[-._]?([a-z]|rc|RC|tp|TP)?\d*)$', stem), # e.g. foobar-4.5.0-beta1, or foobar-4.50-beta (r'-((\d+\.)*\d+-beta(\d+)?)$', stem), -- cgit v1.2.3-70-g09d2 From 0d4b77a24b883d15ee2c297a983b8a8dc807de6f Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 22 Dec 2015 17:02:14 -0800 Subject: revert accidental print statements. --- lib/spack/spack/package.py | 5 ----- 1 file changed, 5 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 2e7678b4b0..6673e4f392 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -1200,8 +1200,6 @@ def find_versions_of_archive(*archive_urls, **kwargs): for aurl in archive_urls: list_urls.add(spack.url.find_list_url(aurl)) - print list_urls - # Grab some web pages to scrape. page_map = {} for lurl in list_urls: @@ -1226,9 +1224,6 @@ def find_versions_of_archive(*archive_urls, **kwargs): for page_url, content in page_map.iteritems(): # extract versions from matches. for regex in regexes: - print regex - print - for m in re.finditer(regex, content): url = urljoin(page_url, m.group(0)) ver = spack.url.parse_version(url) -- cgit v1.2.3-70-g09d2 From d63cb8b537a9fc12a2a1ee5b22f2b20e19d90dc1 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 22 Dec 2015 17:05:23 -0800 Subject: Fix bug in URL regex. --- lib/spack/spack/url.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/url.py b/lib/spack/spack/url.py index ed5a6d2a44..02c0b83e26 100644 --- a/lib/spack/spack/url.py +++ b/lib/spack/spack/url.py @@ -210,7 +210,7 @@ def parse_version_offset(path): (r'-((\d+\.)*\d+)$', stem), # e.g. foobar-4.5.1b, foobar4.5RC, foobar.v4.5.1b - (r'[-._]?v?((\d+\.)*\d+[-._]?([a-z]|rc|RC|tp|TP)?\d*)$', stem), + (r'[-._]?v?((\d+\.)*\d+[-._]?([a-z]|rc|RC|tp|TP?)\d*)$', stem), # e.g. foobar-4.5.0-beta1, or foobar-4.50-beta (r'-((\d+\.)*\d+-beta(\d+)?)$', stem), -- cgit v1.2.3-70-g09d2 From 5ca5884ad6285fd766a2f704bceb40b1cf63750f Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Wed, 23 Dec 2015 13:13:07 -0800 Subject: Fix #235: Smarter web spidering -- use parsed links instead of reconstructing. - Move `find_versions_of_archive` from spack.package to `spack.util.web`. - `spider` funciton now just uses the link parsing it already does to return links. We evaluate actual links found in the scraped pages instead of trying to reconstruct them naively. - Add `spack url-parse` command, which you can use to show how Spack interprets the name and version in a URL. --- lib/spack/spack/cmd/create.py | 4 +- lib/spack/spack/cmd/url-parse.py | 75 +++++++++++++++++++++ lib/spack/spack/package.py | 46 +------------ lib/spack/spack/util/web.py | 140 +++++++++++++++++++++++++++++---------- 4 files changed, 183 insertions(+), 82 deletions(-) create mode 100644 lib/spack/spack/cmd/url-parse.py (limited to 'lib') diff --git a/lib/spack/spack/cmd/create.py b/lib/spack/spack/cmd/create.py index 5e42860f3e..9ecb709110 100644 --- a/lib/spack/spack/cmd/create.py +++ b/lib/spack/spack/cmd/create.py @@ -34,8 +34,8 @@ from llnl.util.filesystem import mkdirp import spack import spack.cmd import spack.cmd.checksum -import spack.package import spack.url +import spack.util.web from spack.util.naming import * import spack.util.crypto as crypto @@ -166,7 +166,7 @@ def create(parser, args): tty.msg("This looks like a URL for %s version %s." % (name, version)) tty.msg("Creating template for package %s" % name) - versions = spack.package.find_versions_of_archive(url) + versions = spack.util.web.find_versions_of_archive(url) rkeys = sorted(versions.keys(), reverse=True) versions = OrderedDict(zip(rkeys, (versions[v] for v in rkeys))) diff --git a/lib/spack/spack/cmd/url-parse.py b/lib/spack/spack/cmd/url-parse.py new file mode 100644 index 0000000000..077c793d2e --- /dev/null +++ b/lib/spack/spack/cmd/url-parse.py @@ -0,0 +1,75 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +import sys + +import llnl.util.tty as tty + +import spack +import spack.url +from spack.util.web import find_versions_of_archive + +description = "Show parsing of a URL, optionally spider web for other versions." + +def setup_parser(subparser): + subparser.add_argument('url', help="url of a package archive") + subparser.add_argument( + '-s', '--spider', action='store_true', help="Spider the source page for versions.") + + +def print_name_and_version(url): + name, ns, nl, ntup, ver, vs, vl, vtup = spack.url.substitution_offsets(url) + underlines = [" "] * max(ns+nl, vs+vl) + for i in range(ns, ns+nl): + underlines[i] = '-' + for i in range(vs, vs+vl): + underlines[i] = '~' + + print " %s" % url + print " %s" % ''.join(underlines) + + +def url_parse(parser, args): + url = args.url + + ver, vs, vl = spack.url.parse_version_offset(url) + name, ns, nl = spack.url.parse_name_offset(url, ver) + + tty.msg("Parsing URL:") + try: + print_name_and_version(url) + except spack.url.UrlParseError as e: + tty.error(str(e)) + + print + tty.msg("Substituting version 9.9.9b:") + newurl = spack.url.substitute_version(url, '9.9.9b') + print_name_and_version(newurl) + + if args.spider: + print + tty.msg("Spidering for versions:") + versions = find_versions_of_archive(url) + for v in sorted(versions): + print "%-20s%s" % (v, versions[v]) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 6673e4f392..b44554e418 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -1164,7 +1164,7 @@ class Package(object): raise VersionFetchError(self.__class__) try: - return find_versions_of_archive( + return spack.util.web.find_versions_of_archive( *self.all_urls, list_url=self.list_url, list_depth=self.list_depth) except spack.error.NoNetworkConnectionError, e: tty.die("Package.fetch_versions couldn't connect to:", @@ -1188,50 +1188,6 @@ class Package(object): return " ".join("-Wl,-rpath=%s" % p for p in self.rpath) -def find_versions_of_archive(*archive_urls, **kwargs): - list_url = kwargs.get('list_url', None) - list_depth = kwargs.get('list_depth', 1) - - # Generate a list of list_urls based on archive urls and any - # explicitly listed list_url in the package - list_urls = set() - if list_url: - list_urls.add(list_url) - for aurl in archive_urls: - list_urls.add(spack.url.find_list_url(aurl)) - - # Grab some web pages to scrape. - page_map = {} - for lurl in list_urls: - pages = spack.util.web.get_pages(lurl, depth=list_depth) - page_map.update(pages) - - # Scrape them for archive URLs - regexes = [] - for aurl in archive_urls: - # This creates a regex from the URL with a capture group for - # the version part of the URL. The capture group is converted - # to a generic wildcard, so we can use this to extract things - # on a page that look like archive URLs. - url_regex = spack.url.wildcard_version(aurl) - - # We'll be a bit more liberal and just look for the archive - # part, not the full path. - regexes.append(os.path.basename(url_regex)) - - # Build a version list from all the matches we find - versions = {} - for page_url, content in page_map.iteritems(): - # extract versions from matches. - for regex in regexes: - for m in re.finditer(regex, content): - url = urljoin(page_url, m.group(0)) - ver = spack.url.parse_version(url) - versions[ver] = url - - return versions - - def validate_package_url(url_string): """Determine whether spack can handle a particular URL or not.""" url = urlparse(url_string) diff --git a/lib/spack/spack/util/web.py b/lib/spack/spack/util/web.py index 94384e9c86..e26daef296 100644 --- a/lib/spack/spack/util/web.py +++ b/lib/spack/spack/util/web.py @@ -23,6 +23,7 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import re +import os import sys import subprocess import urllib2, cookielib @@ -70,7 +71,9 @@ def _spider(args): """ url, visited, root, opener, depth, max_depth, raise_on_error = args - pages = {} + pages = {} # dict from page URL -> text content. + links = set() # set of all links seen on visited pages. + try: # Make a HEAD request first to check the content type. This lets # us ignore tarballs and gigantic files. @@ -99,42 +102,45 @@ def _spider(args): page = response.read() pages[response_url] = page - # If we're not at max depth, parse out the links in the page - if depth < max_depth: - link_parser = LinkParser() - subcalls = [] - link_parser.feed(page) - - while link_parser.links: - raw_link = link_parser.links.pop() + # Parse out the links in the page + link_parser = LinkParser() + subcalls = [] + link_parser.feed(page) - # Skip stuff that looks like an archive - if any(raw_link.endswith(suf) for suf in ALLOWED_ARCHIVE_TYPES): - continue + while link_parser.links: + raw_link = link_parser.links.pop() + abs_link = urlparse.urljoin(response_url, raw_link) - # Evaluate the link relative to the page it came from. - abs_link = urlparse.urljoin(response_url, raw_link) + links.add(abs_link) - # Skip things outside the root directory - if not abs_link.startswith(root): - continue + # Skip stuff that looks like an archive + if any(raw_link.endswith(suf) for suf in ALLOWED_ARCHIVE_TYPES): + continue - # Skip already-visited links - if abs_link in visited: - continue + # Skip things outside the root directory + if not abs_link.startswith(root): + continue - subcalls.append((abs_link, visited, root, None, depth+1, max_depth, raise_on_error)) - visited.add(abs_link) + # Skip already-visited links + if abs_link in visited: + continue - if subcalls: - try: - pool = Pool(processes=len(subcalls)) - dicts = pool.map(_spider, subcalls) - for d in dicts: - pages.update(d) - finally: - pool.terminate() - pool.join() + # If we're not at max depth, follow links. + if depth < max_depth: + subcalls.append((abs_link, visited, root, None, + depth+1, max_depth, raise_on_error)) + visited.add(abs_link) + + if subcalls: + try: + pool = Pool(processes=len(subcalls)) + results = pool.map(_spider, subcalls) + for sub_pages, sub_links in results: + pages.update(sub_pages) + links.update(sub_links) + finally: + pool.terminate() + pool.join() except urllib2.URLError, e: tty.debug(e) @@ -155,10 +161,10 @@ def _spider(args): # Other types of errors are completely ignored, except in debug mode. tty.debug("Error in _spider: %s" % e) - return pages + return pages, links -def get_pages(root_url, **kwargs): +def spider(root_url, **kwargs): """Gets web pages from a root URL. If depth is specified (e.g., depth=2), then this will also fetches pages linked from the root and its children up to depth. @@ -167,5 +173,69 @@ def get_pages(root_url, **kwargs): performance over a sequential fetch. """ max_depth = kwargs.setdefault('depth', 1) - pages = _spider((root_url, set(), root_url, None, 1, max_depth, False)) - return pages + pages, links = _spider((root_url, set(), root_url, None, 1, max_depth, False)) + return pages, links + + +def find_versions_of_archive(*archive_urls, **kwargs): + """Scrape web pages for new versions of a tarball. + + Arguments: + archive_urls: + URLs for different versions of a package. Typically these + are just the tarballs from the package file itself. By + default, this searches the parent directories of archives. + + Keyword Arguments: + list_url: + + URL for a listing of archives. Spack wills scrape these + pages for download links that look like the archive URL. + + list_depth: + Max depth to follow links on list_url pages. + + """ + list_url = kwargs.get('list_url', None) + list_depth = kwargs.get('list_depth', 1) + + # Generate a list of list_urls based on archive urls and any + # explicitly listed list_url in the package + list_urls = set() + if list_url: + list_urls.add(list_url) + for aurl in archive_urls: + list_urls.add(spack.url.find_list_url(aurl)) + + # Grab some web pages to scrape. + pages = {} + links = set() + for lurl in list_urls: + p, l = spider(lurl, depth=list_depth) + pages.update(p) + links.update(l) + + # Scrape them for archive URLs + regexes = [] + for aurl in archive_urls: + # This creates a regex from the URL with a capture group for + # the version part of the URL. The capture group is converted + # to a generic wildcard, so we can use this to extract things + # on a page that look like archive URLs. + url_regex = spack.url.wildcard_version(aurl) + + # We'll be a bit more liberal and just look for the archive + # part, not the full path. + regexes.append(os.path.basename(url_regex)) + + # Build a dict version -> URL from any links that match the wildcards. + versions = {} + for url in links: + if any(re.search(r, url) for r in regexes): + try: + ver = spack.url.parse_version(url) + versions[ver] = url + except spack.url.UndetectableVersionError as e: + continue + + return versions -- cgit v1.2.3-70-g09d2 From f37f872e5f4920f9dc22237deed5895b94700136 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Wed, 23 Dec 2015 16:23:58 -0800 Subject: Fix #85 and #228: errors fetching VCS packages from a mirror. - Stage and fetcher were not being set up properly when fetching using a different fetch strategy than the default one for the package. - This is fixed but fetch/stage/mirror logic is still too complicated and long-term needs a rethink. - Spack will now print a warning when fetching a checksum-less tarball from a mirror -- users should be careful to use https or local filesystem mirrors for this. --- lib/spack/spack/fetch_strategy.py | 4 +-- lib/spack/spack/stage.py | 53 ++++++++++++++++++++++++++++----------- 2 files changed, 41 insertions(+), 16 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/fetch_strategy.py b/lib/spack/spack/fetch_strategy.py index a9374fb34b..0657146bf6 100644 --- a/lib/spack/spack/fetch_strategy.py +++ b/lib/spack/spack/fetch_strategy.py @@ -687,7 +687,7 @@ def for_package_version(pkg, version): class FetchError(spack.error.SpackError): - def __init__(self, msg, long_msg): + def __init__(self, msg, long_msg=None): super(FetchError, self).__init__(msg, long_msg) @@ -705,7 +705,7 @@ class NoArchiveFileError(FetchError): class NoDigestError(FetchError): - def __init__(self, msg, long_msg): + def __init__(self, msg, long_msg=None): super(NoDigestError, self).__init__(msg, long_msg) diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index 754344fc01..76ca7273cb 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -82,14 +82,18 @@ class Stage(object): stage object later). If name is not provided, then this stage will be given a unique name automatically. """ + # TODO: fetch/stage coupling needs to be reworked -- the logic + # TODO: here is convoluted and not modular enough. if isinstance(url_or_fetch_strategy, basestring): self.fetcher = fs.from_url(url_or_fetch_strategy) elif isinstance(url_or_fetch_strategy, fs.FetchStrategy): self.fetcher = url_or_fetch_strategy else: raise ValueError("Can't construct Stage without url or fetch strategy") - self.fetcher.set_stage(self) + self.default_fetcher = self.fetcher # self.fetcher can change with mirrors. + self.skip_checksum_for_mirror = True # used for mirrored archives of repositories. + self.name = kwargs.get('name') self.mirror_path = kwargs.get('mirror_path') @@ -198,17 +202,18 @@ class Stage(object): @property def archive_file(self): """Path to the source archive within this stage directory.""" - if not isinstance(self.fetcher, fs.URLFetchStrategy): - return None + paths = [] + if isinstance(self.fetcher, fs.URLFetchStrategy): + paths.append(os.path.join(self.path, os.path.basename(self.fetcher.url))) - paths = [os.path.join(self.path, os.path.basename(self.fetcher.url))] if self.mirror_path: paths.append(os.path.join(self.path, os.path.basename(self.mirror_path))) for path in paths: if os.path.exists(path): return path - return None + else: + return None @property @@ -238,23 +243,34 @@ class Stage(object): """Downloads an archive or checks out code from a repository.""" self.chdir() - fetchers = [self.fetcher] + fetchers = [self.default_fetcher] # TODO: move mirror logic out of here and clean it up! + # TODO: Or @alalazo may have some ideas about how to use a + # TODO: CompositeFetchStrategy here. + self.skip_checksum_for_mirror = True if self.mirror_path: urls = ["%s/%s" % (m, self.mirror_path) for m in _get_mirrors()] + # If this archive is normally fetched from a tarball URL, + # then use the same digest. `spack mirror` ensures that + # the checksum will be the same. digest = None - if isinstance(self.fetcher, fs.URLFetchStrategy): - digest = self.fetcher.digest - fetchers = [fs.URLFetchStrategy(url, digest) - for url in urls] + fetchers - for f in fetchers: - f.set_stage(self) + if isinstance(self.default_fetcher, fs.URLFetchStrategy): + digest = self.default_fetcher.digest + + # Have to skip the checkesum for things archived from + # repositories. How can this be made safer? + self.skip_checksum_for_mirror = not bool(digest) + + for url in urls: + fetchers.insert(0, fs.URLFetchStrategy(url, digest)) for fetcher in fetchers: try: - fetcher.fetch() + fetcher.set_stage(self) + self.fetcher = fetcher + self.fetcher.fetch() break except spack.error.SpackError, e: tty.msg("Fetching from %s failed." % fetcher) @@ -262,13 +278,22 @@ class Stage(object): continue else: errMessage = "All fetchers failed for %s" % self.name + self.fetcher = self.default_fetcher raise fs.FetchError(errMessage, None) def check(self): """Check the downloaded archive against a checksum digest. No-op if this stage checks code out of a repository.""" - self.fetcher.check() + if self.fetcher is not self.default_fetcher and self.skip_checksum_for_mirror: + tty.warn("Fetching from mirror without a checksum!", + "This package is normally checked out from a version " + "control system, but it has been archived on a spack " + "mirror. This means we cannot know a checksum for the " + "tarball in advance. Be sure that your connection to " + "this mirror is secure!.") + else: + self.fetcher.check() def expand_archive(self): -- cgit v1.2.3-70-g09d2 From 52625de52c396b995716c0451ba169df0f3a4b1f Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 29 Nov 2015 16:02:51 -0800 Subject: Fix #154 -- better log messages for do_patch() --- lib/spack/spack/package.py | 40 +++++++++++++++++++++++++++++----------- 1 file changed, 29 insertions(+), 11 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index b44554e418..84bcb15f7f 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -733,9 +733,10 @@ class Package(object): # Construct paths to special files in the archive dir used to # keep track of whether patches were successfully applied. - archive_dir = self.stage.source_path - good_file = join_path(archive_dir, '.spack_patched') - bad_file = join_path(archive_dir, '.spack_patch_failed') + archive_dir = self.stage.source_path + good_file = join_path(archive_dir, '.spack_patched') + no_patches_file = join_path(archive_dir, '.spack_no_patches') + bad_file = join_path(archive_dir, '.spack_patch_failed') # If we encounter an archive that failed to patch, restage it # so that we can apply all the patches again. @@ -749,29 +750,46 @@ class Package(object): if os.path.isfile(good_file): tty.msg("Already patched %s" % self.name) return + elif os.path.isfile(no_patches_file): + tty.msg("No patches needed for %s." % self.name) + return # Apply all the patches for specs that match this one + patched = False for spec, patch_list in self.patches.items(): if self.spec.satisfies(spec): for patch in patch_list: - tty.msg('Applying patch %s' % patch.path_or_url) try: patch.apply(self.stage) + tty.msg('Applied patch %s' % patch.path_or_url) + patched = True except: # Touch bad file if anything goes wrong. + tty.msg('Patch %s failed.' % patch.path_or_url) touch(bad_file) raise - # patch succeeded. Get rid of failed file & touch good file so we - # don't try to patch again again next time. + if has_patch_fun: + try: + self.patch() + tty.msg("Ran patch() for %s." % self.name) + patched = True + except: + tty.msg("patch() function failed for %s." % self.name) + touch(bad_file) + raise + + # Get rid of any old failed file -- patches have either succeeded + # or are not needed. This is mostly defensive -- it's needed + # if the restage() method doesn't clean *everything* (e.g., for a repo) if os.path.isfile(bad_file): os.remove(bad_file) - touch(good_file) - if has_patch_fun: - self.patch() - - tty.msg("Patched %s" % self.name) + # touch good or no patches file so that we skip next time. + if patched: + touch(good_file) + else: + touch(no_patches_file) def do_fake_install(self): -- cgit v1.2.3-70-g09d2 From 34401cf0c3ee019fd6b8fb739e2a511c6de0870c Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Fri, 25 Dec 2015 14:00:33 -0800 Subject: Rework Spack config: keep user & site config in memory. - User and site config are now kept separately in memory. - Merging is done on demand when client code requests the configuration. - Allows user/site config to be updated independently of each other by commands. - simplifies config logic (no more tracking merged files) --- .gitignore | 1 + lib/spack/spack/cmd/compiler.py | 7 +- lib/spack/spack/cmd/config.py | 12 +- lib/spack/spack/cmd/mirror.py | 4 +- lib/spack/spack/compilers/__init__.py | 76 +++--- lib/spack/spack/config.py | 417 +++++++++++++++-------------- lib/spack/spack/spec.py | 1 + lib/spack/spack/stage.py | 5 +- lib/spack/spack/test/config.py | 21 +- lib/spack/spack/test/database.py | 3 +- lib/spack/spack/test/mock_packages_test.py | 6 +- 11 files changed, 285 insertions(+), 268 deletions(-) (limited to 'lib') diff --git a/.gitignore b/.gitignore index 1c6ca4c99e..4b97de5d50 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ /etc/spackconfig /share/spack/dotkit /share/spack/modules +/TAGS diff --git a/lib/spack/spack/cmd/compiler.py b/lib/spack/spack/cmd/compiler.py index 2a64dc914e..6efc9a3347 100644 --- a/lib/spack/spack/cmd/compiler.py +++ b/lib/spack/spack/cmd/compiler.py @@ -65,10 +65,11 @@ def compiler_add(args): if c.spec not in spack.compilers.all_compilers()] if compilers: - spack.compilers.add_compilers_to_config('user', *compilers) + spack.compilers.add_compilers_to_config('user', compilers) n = len(compilers) - tty.msg("Added %d new compiler%s to %s" % ( - n, 's' if n > 1 else '', spack.config.get_config_scope_filename('user', 'compilers'))) + s = 's' if n > 1 else '' + filename = spack.config.get_config_filename('user', 'compilers') + tty.msg("Added %d new compiler%s to %s" % (n, s, filename)) colify(reversed(sorted(c.spec for c in compilers)), indent=4) else: tty.msg("Found no new compilers") diff --git a/lib/spack/spack/cmd/config.py b/lib/spack/spack/cmd/config.py index 8c18f88b64..603023d891 100644 --- a/lib/spack/spack/cmd/config.py +++ b/lib/spack/spack/cmd/config.py @@ -44,22 +44,22 @@ def setup_parser(subparser): sp = subparser.add_subparsers(metavar='SUBCOMMAND', dest='config_command') get_parser = sp.add_parser('get', help='Print configuration values.') - get_parser.add_argument('category', help="Configuration category to print.") + get_parser.add_argument('section', help="Configuration section to print.") edit_parser = sp.add_parser('edit', help='Edit configuration file.') - edit_parser.add_argument('category', help="Configuration category to edit") + edit_parser.add_argument('section', help="Configuration section to edit") def config_get(args): - spack.config.print_category(args.category) + spack.config.print_section(args.section) def config_edit(args): if not args.scope: args.scope = 'user' - if not args.category: - args.category = None - config_file = spack.config.get_config_scope_filename(args.scope, args.category) + if not args.section: + args.section = None + config_file = spack.config.get_config_filename(args.scope, args.section) spack.editor(config_file) diff --git a/lib/spack/spack/cmd/mirror.py b/lib/spack/spack/cmd/mirror.py index 9a507e69db..2b25793927 100644 --- a/lib/spack/spack/cmd/mirror.py +++ b/lib/spack/spack/cmd/mirror.py @@ -76,7 +76,7 @@ def mirror_add(args): url = 'file://' + url mirror_dict = { args.name : url } - spack.config.add_to_mirror_config({ args.name : url }) + spack.config.update_config('mirrors', { args.name : url }, 'user') def mirror_remove(args): @@ -90,7 +90,7 @@ def mirror_remove(args): def mirror_list(args): """Print out available mirrors to the console.""" - sec_names = spack.config.get_mirror_config() + sec_names = spack.config.get_config('mirrors') if not sec_names: tty.msg("No mirrors configured.") return diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index b7b021a1ac..a1980f1cdf 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -35,6 +35,7 @@ import spack import spack.error import spack.spec import spack.config +import spack.architecture from spack.util.multiproc import parmap from spack.compiler import Compiler @@ -55,23 +56,48 @@ def _auto_compiler_spec(function): return converter -def _get_config(): - """Get a Spack config, but make sure it has compiler configuration - first.""" +def _to_dict(compiler): + """Return a dict version of compiler suitable to insert in YAML.""" + return { + str(compiler.spec) : dict( + (attr, getattr(compiler, attr, None)) + for attr in _required_instance_vars) + } + + +def get_compiler_config(arch=None): + """Return the compiler configuration for the specified architecture. + + If the compiler configuration designates some compilers for + 'all' architectures, those are merged into the result, as well. + + """ # If any configuration file has compilers, just stick with the # ones already configured. - config = spack.config.get_compilers_config() - existing = [spack.spec.CompilerSpec(s) - for s in config] - if existing: - return config + config = spack.config.get_config('compilers') + + if arch is None: + arch = spack.architecture.sys_type() - compilers = find_compilers(*get_path('PATH')) - add_compilers_to_config('user', *compilers) + if arch not in config: + config[arch] = {} + compilers = find_compilers(*get_path('PATH')) + for compiler in compilers: + config[arch].update(_to_dict(compiler)) + spack.config.update_config('compilers', config, 'user') - # After writing compilers to the user config, return a full config - # from all files. - return spack.config.get_compilers_config() + # Merge 'all' compilers with arch-specific ones. + merged_config = config.get('all', {}) + merged_config = spack.config._merge_yaml(merged_config, config[arch]) + + return merged_config + + +def all_compilers(arch=None): + """Return a set of specs for all the compiler versions currently + available to build with. These are instances of CompilerSpec. + """ + return [spack.spec.CompilerSpec(s) for s in get_compiler_config(arch)] _cached_default_compiler = None @@ -123,20 +149,6 @@ def find_compilers(*path): return clist -def add_compilers_to_config(scope, *compilers): - compiler_config_tree = {} - for compiler in compilers: - compiler_entry = {} - for c in _required_instance_vars: - val = getattr(compiler, c) - if not val: - val = "None" - compiler_entry[c] = val - compiler_config_tree[str(compiler.spec)] = compiler_entry - spack.config.add_to_compiler_config(compiler_config_tree, scope) - - - def supported_compilers(): """Return a set of names of compilers supported by Spack. @@ -152,14 +164,6 @@ def supported(compiler_spec): return compiler_spec.name in supported_compilers() -def all_compilers(): - """Return a set of specs for all the compiler versions currently - available to build with. These are instances of CompilerSpec. - """ - configuration = _get_config() - return [spack.spec.CompilerSpec(s) for s in configuration] - - @_auto_compiler_spec def find(compiler_spec): """Return specs of available compilers that match the supplied @@ -172,7 +176,7 @@ def compilers_for_spec(compiler_spec): """This gets all compilers that satisfy the supplied CompilerSpec. Returns an empty list if none are found. """ - config = _get_config() + config = get_compiler_config() def get_compiler(cspec): items = config[str(cspec)] diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 36bf8a7fc3..7d7a87c7dc 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -67,25 +67,54 @@ In this example, entries like ''compilers'' and ''xlc@12.1'' are used to categorize entries beneath them in the tree. At the root of the tree, entries like ''cc'' and ''cxx'' are specified as name/value pairs. -Spack returns these trees as nested dicts. The dict for the above example -would looks like: - - { 'compilers' : - { 'chaos_5_x86_64_ib' : - { 'gcc@4.4.7' : - { 'cc' : '/usr/bin/gcc', - 'cxx' : '/usr/bin/g++' - 'f77' : '/usr/bin/gfortran' - 'fc' : '/usr/bin/gfortran' } - } - { 'bgqos_0' : - { 'cc' : '/usr/local/bin/mpixlc' } - } - } - -Some convenience functions, like get_mirrors_config and -``get_compilers_config`` may strip off the top-levels of the tree and -return subtrees. +``config.get_config()`` returns these trees as nested dicts, but it +strips the first level off. So, ``config.get_config('compilers')`` +would return something like this for the above example: + + { 'chaos_5_x86_64_ib' : + { 'gcc@4.4.7' : + { 'cc' : '/usr/bin/gcc', + 'cxx' : '/usr/bin/g++' + 'f77' : '/usr/bin/gfortran' + 'fc' : '/usr/bin/gfortran' } + } + { 'bgqos_0' : + { 'cc' : '/usr/local/bin/mpixlc' } } + +Likewise, the ``mirrors.yaml`` file's first line must be ``mirrors:``, +but ``get_config()`` strips that off too. + +Precedence +=============================== + +``config.py`` routines attempt to recursively merge configuration +across scopes. So if there are ``compilers.py`` files in both the +site scope and the user scope, ``get_config('compilers')`` will return +merged dictionaries of *all* the compilers available. If a user +compiler conflicts with a site compiler, Spack will overwrite the site +configuration wtih the user configuration. If both the user and site +``mirrors.yaml`` files contain lists of mirrors, then ``get_config()`` +will return a concatenated list of mirrors, with the user config items +first. + +Sometimes, it is useful to *completely* override a site setting with a +user one. To accomplish this, you can use *two* colons at the end of +a key in a configuration file. For example, this: + + compilers:: + chaos_5_x86_64_ib: + gcc@4.4.7: + cc: /usr/bin/gcc + cxx: /usr/bin/g++ + f77: /usr/bin/gfortran + fc: /usr/bin/gfortran + bgqos_0: + xlc@12.1: + cc: /usr/local/bin/mpixlc + ... + +Will make Spack take compilers *only* from the user configuration, and +the site configuration will be ignored. """ import os @@ -96,80 +125,132 @@ from external.yaml.error import MarkedYAMLError import llnl.util.tty as tty from llnl.util.filesystem import mkdirp -from llnl.util.lang import memoized import spack +from spack.error import SpackError +"""List of valid config sections.""" +valid_sections = ('compilers', 'mirrors', 'repos') -_config_sections = {} -class _ConfigCategory: - name = None - filename = None - merge = True - def __init__(self, name, filename, merge, strip): - self.name = name - self.filename = filename - self.merge = merge - self.strip = strip - self.files_read_from = [] - self.result_dict = {} - _config_sections[name] = self - -_ConfigCategory('repos', 'repos.yaml', True, True) -_ConfigCategory('compilers', 'compilers.yaml', True, True) -_ConfigCategory('mirrors', 'mirrors.yaml', True, True) -_ConfigCategory('view', 'views.yaml', True, True) -_ConfigCategory('order', 'orders.yaml', True, True) - -"""Names of scopes and their corresponding configuration files.""" -config_scopes = [('site', os.path.join(spack.etc_path, 'spack')), - ('user', os.path.expanduser('~/.spack'))] - -_compiler_by_arch = {} - -@memoized -def _read_config_file(filename): - """Read a YAML configuration file""" +def check_section(section): + """Raise a ValueError if the section is not a valid section.""" + if section not in valid_sections: + raise ValueError("Invalid config section: '%s'. Options are %s." + % (section, valid_sections)) + + +class ConfigScope(object): + """This class represents a configuration scope. + + A scope is one directory containing named configuration files. + Each file is a config "section" (e.g., mirrors, compilers, etc). + """ + def __init__(self, name, path): + self.name = name # scope name. + self.path = path # path to directory containing configs. + self.sections = {} # sections read from config files. + + + def get_section_filename(self, section): + check_section(section) + return os.path.join(self.path, "%s.yaml" % section) + + + def get_section(self, section): + if not section in self.sections: + path = self.get_section_filename(section) + data = _read_config_file(path) + self.sections[section] = {} if data is None else data + return self.sections[section] + + + def write_section(self, section): + filename = self.get_section_filename(section) + data = self.get_section(section) + try: + mkdirp(self.path) + with open(filename, 'w') as f: + yaml.dump(data, stream=f, default_flow_style=False) + except (yaml.YAMLError, IOError) as e: + raise ConfigFileError("Error writing to config file: '%s'" % str(e)) + + + def clear(self): + """Empty cached config information.""" + self.sections = {} + + +"""List of config scopes by name. + Later scopes in the list will override earlier scopes. +""" +config_scopes = [ + ConfigScope('site', os.path.join(spack.etc_path, 'spack')), + ConfigScope('user', os.path.expanduser('~/.spack'))] + +"""List of valid scopes, for convenience.""" +valid_scopes = (s.name for s in config_scopes) + + +def check_scope(scope): + if scope is None: + return 'user' + elif scope not in valid_scopes: + raise ValueError("Invalid config scope: '%s'. Must be one of %s." + % (scope, valid_scopes)) + return scope + + +def get_scope(scope): + scope = check_scope(scope) + return next(s for s in config_scopes if s.name == scope) + + +def _read_config_file(filename): + """Read a YAML configuration file.""" # Ignore nonexisting files. if not os.path.exists(filename): return None elif not os.path.isfile(filename): - tty.die("Invlaid configuration. %s exists but is not a file." % filename) + raise ConfigFileError( + "Invlaid configuration. %s exists but is not a file." % filename) elif not os.access(filename, os.R_OK): - tty.die("Configuration file %s is not readable." % filename) + raise ConfigFileError("Config file is not readable: %s." % filename) try: with open(filename) as f: return yaml.load(f) except MarkedYAMLError, e: - tty.die("Error parsing yaml%s: %s" % (str(e.context_mark), e.problem)) + raise ConfigFileError( + "Error parsing yaml%s: %s" % (str(e.context_mark), e.problem)) except IOError, e: - tty.die("Error reading configuration file %s: %s" % (filename, str(e))) + raise ConfigFileError( + "Error reading configuration file %s: %s" % (filename, str(e))) def clear_config_caches(): """Clears the caches for configuration files, which will cause them to be re-read upon the next request""" - for key,s in _config_sections.iteritems(): - s.files_read_from = [] - s.result_dict = {} - - _read_config_file.clear() - spack.config._compiler_by_arch = {} - spack.compilers._cached_default_compiler = None + for scope in config_scopes: + scope.clear() def _merge_yaml(dest, source): """Merges source into dest; entries in source take precedence over dest. + This routine may modify dest and should be assigned to dest, in + case dest was None to begin with, e.g.: + + dest = _merge_yaml(dest, source) + Config file authors can optionally end any attribute in a dict with `::` instead of `:`, and the key will override that of the parent instead of merging. + """ def they_are(t): return isinstance(dest, t) and isinstance(source, t) @@ -212,61 +293,31 @@ def substitute_spack_prefix(path): return path.replace('$spack', spack.prefix) -def get_config(category): - """Get the confguration tree for a category. +def get_config(section): + """Get configuration settings for a section. - Strips off the top-level category entry from the dict + Strips off the top-level section name from the YAML dict. """ - category = _config_sections[category] - if category.result_dict: - return category.result_dict - - category.result_dict = {} - for scope, scope_path in config_scopes: - path = os.path.join(scope_path, category.filename) - result = _read_config_file(path) - if not result: + check_section(section) + merged_section = {} + + for scope in config_scopes: + # read potentially cached data from the scope. + data = scope.get_section(section) + if not data or not section in data: continue - if category.strip: - if not category.name in result: - continue - result = result[category.name] - - # ignore empty sections for easy commenting of single-line configs. - if result is None: - continue - - category.files_read_from.insert(0, path) - if category.merge: - category.result_dict = _merge_yaml(category.result_dict, result) - else: - category.result_dict = result - - return category.result_dict - - -def get_compilers_config(arch=None): - """Get the compiler configuration from config files for the given - architecture. Strips off the architecture component of the - configuration""" - global _compiler_by_arch - if not arch: - arch = spack.architecture.sys_type() - if arch in _compiler_by_arch: - return _compiler_by_arch[arch] - - cc_config = get_config('compilers') - if arch in cc_config and 'all' in cc_config: - arch_compiler = dict(cc_config[arch]) - _compiler_by_arch[arch] = _merge_yaml(arch_compiler, cc_config['all']) - elif arch in cc_config: - _compiler_by_arch[arch] = cc_config[arch] - elif 'all' in cc_config: - _compiler_by_arch[arch] = cc_config['all'] - else: - _compiler_by_arch[arch] = {} - return _compiler_by_arch[arch] + # extract data under the section name header + data = data[section] + + # ignore empty sections for easy commenting of single-line configs. + if not data: + continue + + # merge config data from scopes. + merged_section = _merge_yaml(merged_section, data) + + return merged_section def get_repos_config(): @@ -284,119 +335,71 @@ def get_repos_config(): return [expand_repo_path(repo) for repo in repo_list] -def get_mirror_config(): - """Get the mirror configuration from config files""" - return get_config('mirrors') - +def get_config_filename(scope, section): + """For some scope and section, get the name of the configuration file""" + scope = get_scope(scope) + return scope.get_section_filename(section) -def get_config_scope_dirname(scope): - """For a scope return the config directory""" - for s,p in config_scopes: - if s == scope: - return p - tty.die("Unknown scope %s. Valid options are %s" % - (scope, ", ".join([s for s,p in config_scopes]))) +def update_config(section, update_data, scope=None): + """Update the configuration file for a particular scope. -def get_config_scope_filename(scope, category_name): - """For some scope and category, get the name of the configuration file""" - if not category_name in _config_sections: - tty.die("Unknown config category %s. Valid options are: %s" % - (category_name, ", ".join([s for s in _config_sections]))) - return os.path.join(get_config_scope_dirname(scope), _config_sections[category_name].filename) + Merges contents of update_data into the scope's data for the + specified section, then writes out the config file. + update_data shoudl contain only the section's data, with the + top-level name stripped off. This can be a list, dict, or any + other yaml-ish structure. -def add_to_config(category_name, addition_dict, scope=None): - """Merge a new dict into a configuration tree and write the new - configuration to disk""" - get_config(category_name) - category = _config_sections[category_name] - - # If scope is specified, use it. Otherwise use the last config scope that - # we successfully parsed data from. - file = None - path = None - if not scope and not category.files_read_from: - scope = 'user' - - if scope: - try: - dir = get_config_scope_dirname(scope) - if not os.path.exists(dir): - mkdirp(dir) - path = os.path.join(dir, category.filename) - file = open(path, 'w') - except IOError, e: - pass - else: - for p in category.files_read_from: - try: - file = open(p, 'w') - except IOError, e: - pass - if file: - path = p - break; + """ + # read in the config to ensure we've got current data + get_config(section) - if not file: - tty.die('Unable to write to config file %s' % path) + check_section(section) # validate section name + scope = get_scope(scope) # get ConfigScope object from string. - # Merge the new information into the existing file info, then write to disk - new_dict = _read_config_file(path) + # read only the requested section's data. + data = scope.get_section(section) + data = _merge_yaml(data, { section : update_data }) + scope.write_section(section) - if new_dict and category_name in new_dict: - new_dict = new_dict[category_name] - new_dict = _merge_yaml(new_dict, addition_dict) - new_dict = { category_name : new_dict } +def remove_from_config(section, key_to_rm, scope=None): + """Remove a configuration key and write updated configuration to disk. - # Install new dict as memoized value, and dump to disk - _read_config_file.cache[path] = new_dict - yaml.dump(new_dict, stream=file, default_flow_style=False) - file.close() + Return True if something was removed, False otherwise. - # Merge the new information into the cached results - category.result_dict = _merge_yaml(category.result_dict, addition_dict) + """ + # ensure configs are current by reading in. + get_config(section) + # check args and get the objects we need. + scope = get_scope(scope) + data = scope.get_section(section) + filename = scope.get_section_filename(section) -def add_to_mirror_config(addition_dict, scope=None): - """Add mirrors to the configuration files""" - add_to_config('mirrors', addition_dict, scope) + # do some checks + if not data: + return False + if not section in data: + raise ConfigFileError("Invalid configuration file: '%s'" % filename) -def add_to_compiler_config(addition_dict, scope=None, arch=None): - """Add compilerss to the configuration files""" - if not arch: - arch = spack.architecture.sys_type() - add_to_config('compilers', { arch : addition_dict }, scope) - clear_config_caches() + if key_to_rm not in section[section]: + return False + # remove the key from the section's configuration + del data[section][key_to_rm] + scope.write_section(section) -def remove_from_config(category_name, key_to_rm, scope=None): - """Remove a configuration key and write a new configuration to disk""" - get_config(category_name) - scopes_to_rm_from = [scope] if scope else [s for s,p in config_scopes] - category = _config_sections[category_name] - rmd_something = False - for s in scopes_to_rm_from: - path = get_config_scope_filename(scope, category_name) - result = _read_config_file(path) - if not result: - continue - if not key_to_rm in result[category_name]: - continue - with open(path, 'w') as f: - result[category_name].pop(key_to_rm, None) - yaml.dump(result, stream=f, default_flow_style=False) - category.result_dict.pop(key_to_rm, None) - rmd_something = True - return rmd_something +"""Print a configuration to stdout""" +def print_section(section): + try: + yaml.dump(get_config(section), stream=sys.stdout, default_flow_style=False) + except (yaml.YAMLError, IOError) as e: + raise ConfigError("Error reading configuration: %s" % section) -"""Print a configuration to stdout""" -def print_category(category_name): - if not category_name in _config_sections: - tty.die("Unknown config category %s. Valid options are: %s" % - (category_name, ", ".join([s for s in _config_sections]))) - yaml.dump(get_config(category_name), stream=sys.stdout, default_flow_style=False) +class ConfigError(SpackError): pass +class ConfigFileError(ConfigError): pass diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 303df6df38..2f1b6e29ea 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -531,6 +531,7 @@ class Spec(object): and self.architecture and self.compiler and self.compiler.concrete and self.dependencies.concrete) + return self._concrete diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index 78930ecb5b..da85bd6f21 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -242,7 +242,8 @@ class Stage(object): # TODO: move mirror logic out of here and clean it up! if self.mirror_path: - urls = ["%s/%s" % (m, self.mirror_path) for m in _get_mirrors()] + urls = ["%s/%s" % (u, self.mirror_path) + for name, u in spack.config.get_config('mirrors')] digest = None if isinstance(self.fetcher, fs.URLFetchStrategy): @@ -345,7 +346,7 @@ class DIYStage(object): def _get_mirrors(): """Get mirrors from spack configuration.""" - config = spack.config.get_mirror_config() + config = spack.config.get_config('mirrors') return [val for name, val in config.iteritems()] diff --git a/lib/spack/spack/test/config.py b/lib/spack/spack/test/config.py index b1195dfe4e..fe6cec82fe 100644 --- a/lib/spack/spack/test/config.py +++ b/lib/spack/spack/test/config.py @@ -27,6 +27,7 @@ import shutil import os from tempfile import mkdtemp import spack +import spack.config from spack.test.mock_packages_test import * # Some sample compiler config data @@ -72,9 +73,9 @@ class ConfigTest(MockPackagesTest): super(ConfigTest, self).setUp() self.tmp_dir = mkdtemp('.tmp', 'spack-config-test-') spack.config.config_scopes = [ - ('test_low_priority', os.path.join(self.tmp_dir, 'low')), - ('test_high_priority', os.path.join(self.tmp_dir, 'high'))] - + spack.config.ConfigScope('test_low_priority', os.path.join(self.tmp_dir, 'low')), + spack.config.ConfigScope('test_high_priority', os.path.join(self.tmp_dir, 'high'))] + spack.config.valid_scopes = ('test_low_priority', 'test_high_priority') def tearDown(self): super(ConfigTest, self).tearDown() @@ -83,17 +84,19 @@ class ConfigTest(MockPackagesTest): def check_config(self, comps, *compiler_names): """Check that named compilers in comps match Spack's config.""" - config = spack.config.get_compilers_config() + config = spack.config.get_config('compilers') compiler_list = ['cc', 'cxx', 'f77', 'f90'] for key in compiler_names: for c in compiler_list: - self.assertEqual(comps[key][c], config[key][c]) + expected = comps[key][c] + actual = config[key][c] + self.assertEqual(expected, actual) def test_write_key_in_memory(self): # Write b_comps "on top of" a_comps. - spack.config.add_to_compiler_config(a_comps, 'test_low_priority') - spack.config.add_to_compiler_config(b_comps, 'test_high_priority') + spack.config.update_config('compilers', a_comps, 'test_low_priority') + spack.config.update_config('compilers', b_comps, 'test_high_priority') # Make sure the config looks how we expect. self.check_config(a_comps, 'gcc@4.7.3', 'gcc@4.5.0') @@ -102,8 +105,8 @@ class ConfigTest(MockPackagesTest): def test_write_key_to_disk(self): # Write b_comps "on top of" a_comps. - spack.config.add_to_compiler_config(a_comps, 'test_low_priority') - spack.config.add_to_compiler_config(b_comps, 'test_high_priority') + spack.config.update_config('compilers', a_comps, 'test_low_priority') + spack.config.update_config('compilers', b_comps, 'test_high_priority') # Clear caches so we're forced to read from disk. spack.config.clear_config_caches() diff --git a/lib/spack/spack/test/database.py b/lib/spack/spack/test/database.py index 5ce010ae8f..e1f7961bed 100644 --- a/lib/spack/spack/test/database.py +++ b/lib/spack/spack/test/database.py @@ -79,7 +79,8 @@ class DatabaseTest(MockPackagesTest): def _mock_install(self, spec): s = Spec(spec) - pkg = spack.repo.get(s.concretized()) + s.concretize() + pkg = spack.repo.get(s) pkg.do_install(fake=True) diff --git a/lib/spack/spack/test/mock_packages_test.py b/lib/spack/spack/test/mock_packages_test.py index 2150b40876..6d92aacab9 100644 --- a/lib/spack/spack/test/mock_packages_test.py +++ b/lib/spack/spack/test/mock_packages_test.py @@ -41,9 +41,10 @@ class MockPackagesTest(unittest.TestCase): spack.config.clear_config_caches() self.real_scopes = spack.config.config_scopes + self.real_valid_scopes = spack.config.valid_scopes spack.config.config_scopes = [ - ('site', spack.mock_site_config), - ('user', spack.mock_user_config)] + spack.config.ConfigScope('site', spack.mock_site_config), + spack.config.ConfigScope('user', spack.mock_user_config)] # Store changes to the package's dependencies so we can # restore later. @@ -71,6 +72,7 @@ class MockPackagesTest(unittest.TestCase): """Restore the real packages path after any test.""" spack.repo.swap(self.db) spack.config.config_scopes = self.real_scopes + spack.config.valid_scopes = self.real_valid_scopes spack.config.clear_config_caches() # Restore dependency changes that happened during the test -- cgit v1.2.3-70-g09d2 From bef52570aee9d9e287eb46b7461fc2fbcda8a033 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Fri, 25 Dec 2015 18:42:06 -0800 Subject: Default to scope with highest precedence instead of user scope, - Generalizes config scopes a bit more: nothing assumes there is a 'user' scope (this would break testing sometimes). --- lib/spack/spack/compilers/__init__.py | 2 +- lib/spack/spack/config.py | 10 +++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index 321452fddb..a1b6d978df 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -84,7 +84,7 @@ def get_compiler_config(arch=None): compilers = find_compilers(*get_path('PATH')) for compiler in compilers: config[arch].update(_to_dict(compiler)) - spack.config.update_config('compilers', config, 'user') + spack.config.update_config('compilers', config) # Merge 'all' compilers with arch-specific ones. merged_config = config.get('all', {}) diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index b401f59d7f..aa6afd183e 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -193,8 +193,16 @@ valid_scopes = (s.name for s in config_scopes) def check_scope(scope): + """Ensure that scope is valid, and return a valid scope if it is None. + + This should be used by routines in ``config.py`` to validate + scope name arguments, and to determine a default scope where no + scope is specified. + + """ if scope is None: - return 'user' + # default to the scope with highest precedence. + return config_scopes[-1] elif scope not in valid_scopes: raise ValueError("Invalid config scope: '%s'. Must be one of %s." % (scope, valid_scopes)) -- cgit v1.2.3-70-g09d2 From 4e8e298eb3242cb25abe0cca315a74a100514dbf Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 27 Dec 2015 17:40:31 -0800 Subject: Uncomment disabled tests. --- lib/spack/spack/test/package_sanity.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/test/package_sanity.py b/lib/spack/spack/test/package_sanity.py index 0b075d135d..ee09040d0d 100644 --- a/lib/spack/spack/test/package_sanity.py +++ b/lib/spack/spack/test/package_sanity.py @@ -44,7 +44,7 @@ class PackageSanityTest(unittest.TestCase): self.check_db() - def ztest_get_all_mock_packages(self): + def test_get_all_mock_packages(self): """Get the mock packages once each too.""" db = RepoPath(spack.mock_packages_path) spack.repo.swap(db) @@ -52,7 +52,7 @@ class PackageSanityTest(unittest.TestCase): spack.repo.swap(db) - def ztest_url_versions(self): + def test_url_versions(self): """Check URLs for regular packages, if they are explicitly defined.""" for pkg in spack.repo.all_packages(): for v, vattrs in pkg.versions.items(): -- cgit v1.2.3-70-g09d2 From 39e360f93a374314c28716ded3b1533d66cd62be Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 27 Dec 2015 17:51:11 -0800 Subject: Add custom YAML loader & dumper to track lines & maintain dict order. - Configs are now parsed with `spack.util.spack_yaml.load/dump` - Parser annotates returned data with `_start_mark` and `_end_mark` properties, so that we can recover what lines/files they came from. - Parser uses `OrderedDict` instead of `dict`. This will help maintain some sanity when round-tripping config files. --- lib/spack/spack/config.py | 77 ++++++----- lib/spack/spack/test/__init__.py | 3 +- lib/spack/spack/test/config.py | 8 +- lib/spack/spack/test/mock_packages_test.py | 12 +- lib/spack/spack/test/yaml.py | 93 +++++++++++++ lib/spack/spack/util/spack_yaml.py | 201 +++++++++++++++++++++++++++++ 6 files changed, 351 insertions(+), 43 deletions(-) create mode 100644 lib/spack/spack/test/yaml.py create mode 100644 lib/spack/spack/util/spack_yaml.py (limited to 'lib') diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index aa6afd183e..9e3b44085f 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -1,5 +1,5 @@ ############################################################################## -# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Copyright (c) 2013-2015, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory. # # This file is part of Spack. @@ -120,8 +120,10 @@ the site configuration will be ignored. import os import sys import copy + import yaml from yaml.error import MarkedYAMLError +from ordereddict_backport import OrderedDict import llnl.util.tty as tty from llnl.util.filesystem import mkdirp @@ -129,11 +131,20 @@ from llnl.util.filesystem import mkdirp import spack from spack.error import SpackError -"""List of valid config sections.""" -valid_sections = ('compilers', 'mirrors', 'repos') +# Hacked yaml for configuration files preserves line numbers. +import spack.util.spack_yaml as syaml + + +"""Dict from section names -> function to check section YAML format.""" +valid_sections = ['compilers', 'mirrors', 'repos'] + +"""OrderedDict of config scopes keyed by name. + Later scopes will override earlier scopes. +""" +config_scopes = OrderedDict() -def check_section(section): +def validate_section(section): """Raise a ValueError if the section is not a valid section.""" if section not in valid_sections: raise ValueError("Invalid config section: '%s'. Options are %s." @@ -146,14 +157,19 @@ class ConfigScope(object): A scope is one directory containing named configuration files. Each file is a config "section" (e.g., mirrors, compilers, etc). """ + def __init__(self, name, path): self.name = name # scope name. self.path = path # path to directory containing configs. self.sections = {} # sections read from config files. + # Register in a dict of all ConfigScopes + # TODO: make this cleaner. Mocking up for testing is brittle. + global config_scopes + config_scopes[name] = self def get_section_filename(self, section): - check_section(section) + validate_section(section) return os.path.join(self.path, "%s.yaml" % section) @@ -161,7 +177,10 @@ class ConfigScope(object): if not section in self.sections: path = self.get_section_filename(section) data = _read_config_file(path) - self.sections[section] = {} if data is None else data + if data is None: + self.sections[section] = {} + else: + self.sections[section] = data return self.sections[section] @@ -171,7 +190,7 @@ class ConfigScope(object): try: mkdirp(self.path) with open(filename, 'w') as f: - yaml.dump(data, stream=f, default_flow_style=False) + syaml.dump(data, stream=f, default_flow_style=False) except (yaml.YAMLError, IOError) as e: raise ConfigFileError("Error writing to config file: '%s'" % str(e)) @@ -181,18 +200,11 @@ class ConfigScope(object): self.sections = {} -"""List of config scopes by name. - Later scopes in the list will override earlier scopes. -""" -config_scopes = [ - ConfigScope('site', os.path.join(spack.etc_path, 'spack')), - ConfigScope('user', os.path.expanduser('~/.spack'))] - -"""List of valid scopes, for convenience.""" -valid_scopes = (s.name for s in config_scopes) +ConfigScope('site', os.path.join(spack.etc_path, 'spack')), +ConfigScope('user', os.path.expanduser('~/.spack')) -def check_scope(scope): +def validate_scope(scope): """Ensure that scope is valid, and return a valid scope if it is None. This should be used by routines in ``config.py`` to validate @@ -202,16 +214,14 @@ def check_scope(scope): """ if scope is None: # default to the scope with highest precedence. - return config_scopes[-1] - elif scope not in valid_scopes: - raise ValueError("Invalid config scope: '%s'. Must be one of %s." - % (scope, valid_scopes)) - return scope + return config_scopes.values()[-1] + elif scope in config_scopes: + return config_scopes[scope] -def get_scope(scope): - scope = check_scope(scope) - return next(s for s in config_scopes if s.name == scope) + else: + raise ValueError("Invalid config scope: '%s'. Must be one of %s." + % (scope, config_scopes.keys())) def _read_config_file(filename): @@ -229,7 +239,7 @@ def _read_config_file(filename): try: with open(filename) as f: - return yaml.load(f) + return syaml.load(f) except MarkedYAMLError, e: raise ConfigFileError( @@ -243,7 +253,7 @@ def _read_config_file(filename): def clear_config_caches(): """Clears the caches for configuration files, which will cause them to be re-read upon the next request""" - for scope in config_scopes: + for scope in config_scopes.values(): scope.clear() @@ -306,10 +316,10 @@ def get_config(section): Strips off the top-level section name from the YAML dict. """ - check_section(section) + validate_section(section) merged_section = {} - for scope in config_scopes: + for scope in config_scopes.values(): # read potentially cached data from the scope. data = scope.get_section(section) if not data or not section in data: @@ -345,7 +355,7 @@ def get_repos_config(): def get_config_filename(scope, section): """For some scope and section, get the name of the configuration file""" - scope = get_scope(scope) + scope = validate_scope(scope) return scope.get_section_filename(section) @@ -363,8 +373,8 @@ def update_config(section, update_data, scope=None): # read in the config to ensure we've got current data get_config(section) - check_section(section) # validate section name - scope = get_scope(scope) # get ConfigScope object from string. + validate_section(section) # validate section name + scope = validate_scope(scope) # get ConfigScope object from string. # read only the requested section's data. data = scope.get_section(section) @@ -382,7 +392,7 @@ def remove_from_config(section, key_to_rm, scope=None): get_config(section) # check args and get the objects we need. - scope = get_scope(scope) + scope = validate_scope(scope) data = scope.get_section(section) filename = scope.get_section_filename(section) @@ -411,3 +421,4 @@ def print_section(section): class ConfigError(SpackError): pass class ConfigFileError(ConfigError): pass +class ConfigFormatError(ConfigError): pass diff --git a/lib/spack/spack/test/__init__.py b/lib/spack/spack/test/__init__.py index 081e6c7b06..9609fd2f36 100644 --- a/lib/spack/spack/test/__init__.py +++ b/lib/spack/spack/test/__init__.py @@ -60,7 +60,8 @@ test_names = ['versions', 'unit_install', 'lock', 'database', - 'namespace_trie'] + 'namespace_trie', + 'yaml'] def list_tests(): diff --git a/lib/spack/spack/test/config.py b/lib/spack/spack/test/config.py index f56287aa98..5f99dcb903 100644 --- a/lib/spack/spack/test/config.py +++ b/lib/spack/spack/test/config.py @@ -26,6 +26,7 @@ import unittest import shutil import os from tempfile import mkdtemp +from ordereddict_backport import OrderedDict import spack import spack.config from spack.test.mock_packages_test import * @@ -72,10 +73,9 @@ class ConfigTest(MockPackagesTest): def setUp(self): super(ConfigTest, self).setUp() self.tmp_dir = mkdtemp('.tmp', 'spack-config-test-') - spack.config.config_scopes = [ - spack.config.ConfigScope('test_low_priority', os.path.join(self.tmp_dir, 'low')), - spack.config.ConfigScope('test_high_priority', os.path.join(self.tmp_dir, 'high'))] - spack.config.valid_scopes = ('test_low_priority', 'test_high_priority') + spack.config.config_scopes = OrderedDict() + spack.config.ConfigScope('test_low_priority', os.path.join(self.tmp_dir, 'low')) + spack.config.ConfigScope('test_high_priority', os.path.join(self.tmp_dir, 'high')) def tearDown(self): super(ConfigTest, self).tearDown() diff --git a/lib/spack/spack/test/mock_packages_test.py b/lib/spack/spack/test/mock_packages_test.py index d000271960..320c2921b0 100644 --- a/lib/spack/spack/test/mock_packages_test.py +++ b/lib/spack/spack/test/mock_packages_test.py @@ -24,6 +24,7 @@ ############################################################################## import sys import unittest +from ordereddict_backport import OrderedDict import spack import spack.config @@ -41,10 +42,12 @@ class MockPackagesTest(unittest.TestCase): spack.config.clear_config_caches() self.real_scopes = spack.config.config_scopes - self.real_valid_scopes = spack.config.valid_scopes - spack.config.config_scopes = [ - spack.config.ConfigScope('site', spack.mock_site_config), - spack.config.ConfigScope('user', spack.mock_user_config)] + + # TODO: Mocking this up is kind of brittle b/c ConfigScope + # TODO: constructor modifies config_scopes. Make it cleaner. + spack.config.config_scopes = OrderedDict() + spack.config.ConfigScope('site', spack.mock_site_config) + spack.config.ConfigScope('user', spack.mock_user_config) # Store changes to the package's dependencies so we can # restore later. @@ -72,7 +75,6 @@ class MockPackagesTest(unittest.TestCase): """Restore the real packages path after any test.""" spack.repo.swap(self.db) spack.config.config_scopes = self.real_scopes - spack.config.valid_scopes = self.real_valid_scopes spack.config.clear_config_caches() # Restore dependency changes that happened during the test diff --git a/lib/spack/spack/test/yaml.py b/lib/spack/spack/test/yaml.py new file mode 100644 index 0000000000..5a357b8e69 --- /dev/null +++ b/lib/spack/spack/test/yaml.py @@ -0,0 +1,93 @@ +############################################################################## +# Copyright (c) 2013-2015, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +""" +Test Spack's custom YAML format. +""" +import unittest +import spack.util.spack_yaml as syaml + +test_file = """\ +config_file: + x86_64: + foo: /path/to/foo + bar: /path/to/bar + baz: /path/to/baz + some_list: + - item 1 + - item 2 + - item 3 + another_list: + [ 1, 2, 3 ] + some_key: some_string +""" + +test_data = { + 'config_file' : syaml.syaml_dict([ + ('x86_64', syaml.syaml_dict([ + ('foo', '/path/to/foo'), + ('bar', '/path/to/bar'), + ('baz', '/path/to/baz' )])), + ('some_list', [ 'item 1', 'item 2', 'item 3' ]), + ('another_list', [ 1, 2, 3 ]), + ('some_key', 'some_string') + ])} + +class YamlTest(unittest.TestCase): + + def setUp(self): + self.data = syaml.load(test_file) + + + def test_parse(self): + self.assertEqual(test_data, self.data) + + + def test_dict_order(self): + self.assertEqual( + ['x86_64', 'some_list', 'another_list', 'some_key'], + self.data['config_file'].keys()) + + self.assertEqual( + ['foo', 'bar', 'baz'], + self.data['config_file']['x86_64'].keys()) + + + def test_line_numbers(self): + def check(obj, start_line, end_line): + self.assertEqual(obj._start_mark.line, start_line) + self.assertEqual(obj._end_mark.line, end_line) + + check(self.data, 0, 12) + check(self.data['config_file'], 1, 12) + check(self.data['config_file']['x86_64'], 2, 5) + check(self.data['config_file']['x86_64']['foo'], 2, 2) + check(self.data['config_file']['x86_64']['bar'], 3, 3) + check(self.data['config_file']['x86_64']['baz'], 4, 4) + check(self.data['config_file']['some_list'], 6, 9) + check(self.data['config_file']['some_list'][0], 6, 6) + check(self.data['config_file']['some_list'][1], 7, 7) + check(self.data['config_file']['some_list'][2], 8, 8) + check(self.data['config_file']['another_list'], 10, 10) + check(self.data['config_file']['some_key'], 11, 11) diff --git a/lib/spack/spack/util/spack_yaml.py b/lib/spack/spack/util/spack_yaml.py new file mode 100644 index 0000000000..728e86b8ee --- /dev/null +++ b/lib/spack/spack/util/spack_yaml.py @@ -0,0 +1,201 @@ +############################################################################## +# Copyright (c) 2013-2015, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +"""Enhanced YAML parsing for Spack. + +- ``load()`` preserves YAML Marks on returned objects -- this allows + us to access file and line information later. + +- ``Our load methods use ``OrderedDict`` class instead of YAML's + default unorderd dict. + +""" +import yaml +from yaml.nodes import * +from yaml.constructor import ConstructorError +from yaml.representer import SafeRepresenter +from ordereddict_backport import OrderedDict + +# Only export load and dump +__all__ = ['load', 'dump'] + +# Make new classes so we can add custom attributes. +# Also, use OrderedDict instead of just dict. +class syaml_dict(OrderedDict): + def __repr__(self): + mappings = ('%r: %r' % (k,v) for k,v in self.items()) + return '{%s}' % ', '.join(mappings) +class syaml_list(list): + __repr__ = list.__repr__ +class syaml_str(str): + __repr__ = str.__repr__ + +def mark(obj, node): + """Add start and end markers to an object.""" + obj._start_mark = node.start_mark + obj._end_mark = node.end_mark + + +class OrderedLineLoader(yaml.Loader): + """YAML loader that preserves order and line numbers. + + Mappings read in by this loader behave like an ordered dict. + Sequences, mappings, and strings also have new attributes, + ``_start_mark`` and ``_end_mark``, that preserve YAML line + information in the output data. + + """ + # + # Override construct_yaml_* so that they build our derived types, + # which allows us to add new attributes to them. + # + # The standard YAML constructors return empty instances and fill + # in with mappings later. We preserve this behavior. + # + def construct_yaml_str(self, node): + value = self.construct_scalar(node) + try: + value = value.encode('ascii') + except UnicodeEncodeError: + pass + value = syaml_str(value) + mark(value, node) + return value + + + def construct_yaml_seq(self, node): + data = syaml_list() + mark(data, node) + yield data + data.extend(self.construct_sequence(node)) + + + def construct_yaml_map(self, node): + data = syaml_dict() + mark(data, node) + yield data + value = self.construct_mapping(node) + data.update(value) + + # + # Override the ``construct_*`` routines. These fill in empty + # objects after yielded by the above ``construct_yaml_*`` methods. + # + def construct_sequence(self, node, deep=False): + if not isinstance(node, SequenceNode): + raise ConstructorError(None, None, + "expected a sequence node, but found %s" % node.id, + node.start_mark) + value = syaml_list(self.construct_object(child, deep=deep) + for child in node.value) + mark(value, node) + return value + + + def construct_mapping(self, node, deep=False): + """Store mappings as OrderedDicts instead of as regular python + dictionaries to preserve file ordering.""" + if not isinstance(node, MappingNode): + raise ConstructorError(None, None, + "expected a mapping node, but found %s" % node.id, + node.start_mark) + + mapping = syaml_dict() + for key_node, value_node in node.value: + key = self.construct_object(key_node, deep=deep) + try: + hash(key) + except TypeError, exc: + raise ConstructorError("while constructing a mapping", node.start_mark, + "found unacceptable key (%s)" % exc, key_node.start_mark) + value = self.construct_object(value_node, deep=deep) + if key in mapping: + raise ConstructorError("while constructing a mapping", node.start_mark, + "found already in-use key (%s)" % key, key_node.start_mark) + mapping[key] = value + + mark(mapping, node) + return mapping + +# register above new constructors +OrderedLineLoader.add_constructor(u'tag:yaml.org,2002:map', OrderedLineLoader.construct_yaml_map) +OrderedLineLoader.add_constructor(u'tag:yaml.org,2002:seq', OrderedLineLoader.construct_yaml_seq) +OrderedLineLoader.add_constructor(u'tag:yaml.org,2002:str', OrderedLineLoader.construct_yaml_str) + + + +class OrderedLineDumper(yaml.Dumper): + """Dumper that preserves ordering and formats ``syaml_*`` objects. + + This dumper preserves insertion ordering ``syaml_dict`` objects + when they're written out. It also has some custom formatters + for ``syaml_*`` objects so that they are formatted like their + regular Python equivalents, instead of ugly YAML pyobjects. + + """ + def represent_mapping(self, tag, mapping, flow_style=None): + value = [] + node = MappingNode(tag, value, flow_style=flow_style) + if self.alias_key is not None: + self.represented_objects[self.alias_key] = node + best_style = True + if hasattr(mapping, 'items'): + # if it's a syaml_dict, preserve OrderedDict order. + # Otherwise do the default thing. + sort = not isinstance(mapping, syaml_dict) + mapping = mapping.items() + if sort: + mapping.sort() + + for item_key, item_value in mapping: + node_key = self.represent_data(item_key) + node_value = self.represent_data(item_value) + if not (isinstance(node_key, ScalarNode) and not node_key.style): + best_style = False + if not (isinstance(node_value, ScalarNode) and not node_value.style): + best_style = False + value.append((node_key, node_value)) + if flow_style is None: + if self.default_flow_style is not None: + node.flow_style = self.default_flow_style + else: + node.flow_style = best_style + return node + +# Make our special objects look like normal YAML ones. +OrderedLineDumper.add_representer(syaml_dict, OrderedLineDumper.represent_dict) +OrderedLineDumper.add_representer(syaml_list, OrderedLineDumper.represent_list) +OrderedLineDumper.add_representer(syaml_str, OrderedLineDumper.represent_str) + + +def load(*args, **kwargs): + """Load but modify the loader instance so that it will add __line__ + atrributes to the returned object.""" + kwargs['Loader'] = OrderedLineLoader + return yaml.load(*args, **kwargs) + + +def dump(*args, **kwargs): + kwargs['Dumper'] = OrderedLineDumper + return yaml.dump(*args, **kwargs) -- cgit v1.2.3-70-g09d2 From ff0d871612c05d803fdabb8a5b870b7af961cdc2 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 27 Dec 2015 21:13:18 -0800 Subject: Remove mock_configs; tests no longer modify spack home directory. --- lib/spack/spack/__init__.py | 4 --- lib/spack/spack/test/mock_packages_test.py | 32 ++++++++++++++++++++-- .../mock_configs/site_spackconfig/compilers.yaml | 12 -------- 3 files changed, 30 insertions(+), 18 deletions(-) delete mode 100644 var/spack/mock_configs/site_spackconfig/compilers.yaml (limited to 'lib') diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index de1a98d092..973ba64b96 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -78,10 +78,6 @@ installed_db = Database(install_path) packages_path = join_path(repos_path, "builtin") mock_packages_path = join_path(repos_path, "builtin.mock") -mock_config_path = join_path(var_path, "mock_configs") -mock_site_config = join_path(mock_config_path, "site_spackconfig") -mock_user_config = join_path(mock_config_path, "user_spackconfig") - # # This controls how spack lays out install prefixes and # stage directories. diff --git a/lib/spack/spack/test/mock_packages_test.py b/lib/spack/spack/test/mock_packages_test.py index 320c2921b0..7642edcf4b 100644 --- a/lib/spack/spack/test/mock_packages_test.py +++ b/lib/spack/spack/test/mock_packages_test.py @@ -23,14 +23,32 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import sys +import os import unittest +import tempfile from ordereddict_backport import OrderedDict +from llnl.util.filesystem import mkdirp + import spack import spack.config from spack.repository import RepoPath from spack.spec import Spec +mock_compiler_config = """\ +compilers: + all: + clang@3.3: + cc: /path/to/clang + cxx: /path/to/clang++ + f77: None + fc: None + gcc@4.5.0: + cc: /path/to/gcc + cxx: /path/to/g++ + f77: /path/to/gfortran + fc: /path/to/gfortran +""" class MockPackagesTest(unittest.TestCase): def initmock(self): @@ -43,11 +61,21 @@ class MockPackagesTest(unittest.TestCase): spack.config.clear_config_caches() self.real_scopes = spack.config.config_scopes + # Mock up temporary configuration directories + self.temp_config = tempfile.mkdtemp() + self.mock_site_config = os.path.join(self.temp_config, 'site') + self.mock_user_config = os.path.join(self.temp_config, 'user') + mkdirp(self.mock_site_config) + mkdirp(self.mock_user_config) + comp_yaml = os.path.join(self.mock_site_config, 'compilers.yaml') + with open(comp_yaml, 'w') as f: + f.write(mock_compiler_config) + # TODO: Mocking this up is kind of brittle b/c ConfigScope # TODO: constructor modifies config_scopes. Make it cleaner. spack.config.config_scopes = OrderedDict() - spack.config.ConfigScope('site', spack.mock_site_config) - spack.config.ConfigScope('user', spack.mock_user_config) + spack.config.ConfigScope('site', self.mock_site_config) + spack.config.ConfigScope('user', self.mock_user_config) # Store changes to the package's dependencies so we can # restore later. diff --git a/var/spack/mock_configs/site_spackconfig/compilers.yaml b/var/spack/mock_configs/site_spackconfig/compilers.yaml deleted file mode 100644 index 0a2dc893e2..0000000000 --- a/var/spack/mock_configs/site_spackconfig/compilers.yaml +++ /dev/null @@ -1,12 +0,0 @@ -compilers: - all: - clang@3.3: - cc: /path/to/clang - cxx: /path/to/clang++ - f77: None - fc: None - gcc@4.5.0: - cc: /path/to/gcc - cxx: /path/to/g++ - f77: /path/to/gfortran - fc: /path/to/gfortran -- cgit v1.2.3-70-g09d2 From 1f8ba53ca7767ca452f553f71d49eaf90fa19db8 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 28 Dec 2015 00:33:17 -0800 Subject: Rework compiler configuration and simplify config.py logic. - `spack compiler` subcommands now take an optional --scope argument. - no more `remove_from_config` in `config.py` -- `update` just overwrites b/c it's easier to just call `get_config`, modify YAML structures directly, and then call `update`. - Implemented `spack compiler remove`. --- lib/spack/spack/cmd/compiler.py | 49 ++++++++++++++----- lib/spack/spack/cmd/compilers.py | 5 ++ lib/spack/spack/compilers/__init__.py | 92 ++++++++++++++++++++++++++--------- lib/spack/spack/config.py | 51 +++++-------------- 4 files changed, 124 insertions(+), 73 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/cmd/compiler.py b/lib/spack/spack/cmd/compiler.py index af1a22c9dd..a3860abf76 100644 --- a/lib/spack/spack/cmd/compiler.py +++ b/lib/spack/spack/cmd/compiler.py @@ -22,6 +22,7 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## +import sys import argparse import llnl.util.tty as tty @@ -41,17 +42,26 @@ def setup_parser(subparser): sp = subparser.add_subparsers( metavar='SUBCOMMAND', dest='compiler_command') - update_parser = sp.add_parser( - 'add', help='Add compilers to the Spack configuration.') - update_parser.add_argument('add_paths', nargs=argparse.REMAINDER) + add_parser = sp.add_parser('add', help='Add compilers to the Spack configuration.') + add_parser.add_argument('add_paths', nargs=argparse.REMAINDER) + add_parser.add_argument('--scope', choices=spack.config.config_scopes, + help="Configuration scope to modify.") - remove_parser = sp.add_parser('remove', help='remove compiler') - remove_parser.add_argument('path') + remove_parser = sp.add_parser('remove', help='Remove compiler by spec.') + remove_parser.add_argument( + '-a', '--all', action='store_true', help='Remove ALL compilers that match spec.') + remove_parser.add_argument('compiler_spec') + remove_parser.add_argument('--scope', choices=spack.config.config_scopes, + help="Configuration scope to modify.") - list_parser = sp.add_parser('list', help='list available compilers') + list_parser = sp.add_parser('list', help='list available compilers') + list_parser.add_argument('--scope', choices=spack.config.config_scopes, + help="Configuration scope to read from.") - info_parser = sp.add_parser('info', help='Show compiler paths.') + info_parser = sp.add_parser('info', help='Show compiler paths.') info_parser.add_argument('compiler_spec') + info_parser.add_argument('--scope', choices=spack.config.config_scopes, + help="Configuration scope to read from.") def compiler_add(args): @@ -62,13 +72,13 @@ def compiler_add(args): paths = get_path('PATH') compilers = [c for c in spack.compilers.find_compilers(*args.add_paths) - if c.spec not in spack.compilers.all_compilers()] + if c.spec not in spack.compilers.all_compilers(scope=args.scope)] if compilers: - spack.compilers.add_compilers_to_config('user', compilers) + spack.compilers.add_compilers_to_config(compilers, scope=args.scope) n = len(compilers) s = 's' if n > 1 else '' - filename = spack.config.get_config_filename('user', 'compilers') + filename = spack.config.get_config_filename(args.scope, 'compilers') tty.msg("Added %d new compiler%s to %s" % (n, s, filename)) colify(reversed(sorted(c.spec for c in compilers)), indent=4) else: @@ -76,13 +86,26 @@ def compiler_add(args): def compiler_remove(args): - pass + cspec = CompilerSpec(args.compiler_spec) + compilers = spack.compilers.compilers_for_spec(cspec, scope=args.scope) + + if not compilers: + tty.die("No compilers match spec %s." % cspec) + elif not args.all and len(compilers) > 1: + tty.error("Multiple compilers match spec %s. Choose one:" % cspec) + colify(reversed(sorted([c.spec for c in compilers])), indent=4) + tty.msg("Or, you can use `spack compiler remove -a` to remove all of them.") + sys.exit(1) + + for compiler in compilers: + spack.compilers.remove_compiler_from_config(compiler.spec, scope=args.scope) + tty.msg("Removed compiler %s." % compiler.spec) def compiler_info(args): """Print info about all compilers matching a spec.""" cspec = CompilerSpec(args.compiler_spec) - compilers = spack.compilers.compilers_for_spec(cspec) + compilers = spack.compilers.compilers_for_spec(cspec, scope=args.scope) if not compilers: tty.error("No compilers match spec %s." % cspec) @@ -97,7 +120,7 @@ def compiler_info(args): def compiler_list(args): tty.msg("Available compilers") - index = index_by(spack.compilers.all_compilers(), 'name') + index = index_by(spack.compilers.all_compilers(scope=args.scope), 'name') for i, (name, compilers) in enumerate(index.items()): if i >= 1: print diff --git a/lib/spack/spack/cmd/compilers.py b/lib/spack/spack/cmd/compilers.py index c485a910eb..7e09016f2d 100644 --- a/lib/spack/spack/cmd/compilers.py +++ b/lib/spack/spack/cmd/compilers.py @@ -26,9 +26,14 @@ import llnl.util.tty as tty from llnl.util.tty.colify import colify from llnl.util.lang import index_by +import spack from spack.cmd.compiler import compiler_list description = "List available compilers. Same as 'spack compiler list'." +def setup_parser(subparser): + subparser.add_argument('--scope', choices=spack.config.config_scopes, + help="Configuration scope to read/modify.") + def compilers(parser, args): compiler_list(args) diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index a1b6d978df..67dfaa3ac9 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -49,10 +49,10 @@ _required_instance_vars = ['cc', 'cxx', 'f77', 'fc'] _default_order = ['gcc', 'intel', 'pgi', 'clang', 'xlc'] def _auto_compiler_spec(function): - def converter(cspec_like): + def converter(cspec_like, *args, **kwargs): if not isinstance(cspec_like, spack.spec.CompilerSpec): cspec_like = spack.spec.CompilerSpec(cspec_like) - return function(cspec_like) + return function(cspec_like, *args, **kwargs) return converter @@ -65,39 +65,87 @@ def _to_dict(compiler): } -def get_compiler_config(arch=None): +def get_compiler_config(arch=None, scope=None): """Return the compiler configuration for the specified architecture. - - If the compiler configuration designates some compilers for - 'all' architectures, those are merged into the result, as well. - """ # If any configuration file has compilers, just stick with the # ones already configured. - config = spack.config.get_config('compilers') + config = spack.config.get_config('compilers', scope=scope) + my_arch = spack.architecture.sys_type() if arch is None: - arch = spack.architecture.sys_type() + arch = my_arch - if arch not in config: + if arch in config: + return config[arch] + + # Only for the current arch in *highest* scope: automatically try to + # find compilers if none are configured yet. + if arch == my_arch and scope == 'user': config[arch] = {} compilers = find_compilers(*get_path('PATH')) for compiler in compilers: config[arch].update(_to_dict(compiler)) - spack.config.update_config('compilers', config) + spack.config.update_config('compilers', config, scope=scope) + return config[arch] - # Merge 'all' compilers with arch-specific ones. - merged_config = config.get('all', {}) - merged_config = spack.config._merge_yaml(merged_config, config[arch]) + return {} + + +def add_compilers_to_config(compilers, arch=None, scope=None): + """Add compilers to the config for the specified architecture. + + Arguments: + - compilers: a list of Compiler objects. + - arch: arch to add compilers for. + - scope: configuration scope to modify. + """ + if arch is None: + arch = spack.architecture.sys_type() + + compiler_config = get_compiler_config(arch, scope) + for compiler in compilers: + compiler_config[str(compiler.spec)] = dict( + (c, getattr(compiler, c, "None")) + for c in _required_instance_vars) - return merged_config + update = { arch : compiler_config } + spack.config.update_config('compilers', update, scope) -def all_compilers(arch=None): +@_auto_compiler_spec +def remove_compiler_from_config(compiler_spec, arch=None, scope=None): + """Remove compilers from the config, by spec. + + Arguments: + - compiler_specs: a list of CompilerSpec objects. + - arch: arch to add compilers for. + - scope: configuration scope to modify. + """ + if arch is None: + arch = spack.architecture.sys_type() + + compiler_config = get_compiler_config(arch, scope) + del compiler_config[str(compiler_spec)] + update = { arch : compiler_config } + + spack.config.update_config('compilers', update, scope) + + +def all_compilers(arch=None, scope=None): """Return a set of specs for all the compiler versions currently available to build with. These are instances of CompilerSpec. """ - return [spack.spec.CompilerSpec(s) for s in get_compiler_config(arch)] + # Get compilers for this architecture. + arch_config = get_compiler_config(arch, scope) + + # Merge 'all' compilers with arch-specific ones. + # Arch-specific compilers have higher precedence. + merged_config = get_compiler_config('all', scope=scope) + merged_config = spack.config._merge_yaml(merged_config, arch_config) + + # Return compiler specs for the result. + return [spack.spec.CompilerSpec(s) for s in merged_config] _cached_default_compiler = None @@ -165,18 +213,18 @@ def supported(compiler_spec): @_auto_compiler_spec -def find(compiler_spec): +def find(compiler_spec, arch=None, scope=None): """Return specs of available compilers that match the supplied compiler spec. Return an list if nothing found.""" - return [c for c in all_compilers() if c.satisfies(compiler_spec)] + return [c for c in all_compilers(arch, scope) if c.satisfies(compiler_spec)] @_auto_compiler_spec -def compilers_for_spec(compiler_spec): +def compilers_for_spec(compiler_spec, arch=None, scope=None): """This gets all compilers that satisfy the supplied CompilerSpec. Returns an empty list if none are found. """ - config = get_compiler_config() + config = get_compiler_config(arch, scope) def get_compiler(cspec): items = config[str(cspec)] @@ -195,7 +243,7 @@ def compilers_for_spec(compiler_spec): return cls(cspec, *compiler_paths) - matches = find(compiler_spec) + matches = find(compiler_spec, arch, scope) return [get_compiler(cspec) for cspec in matches] diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 9e3b44085f..d266d2e23f 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -311,7 +311,7 @@ def substitute_spack_prefix(path): return path.replace('$spack', spack.prefix) -def get_config(section): +def get_config(section, scope=None): """Get configuration settings for a section. Strips off the top-level section name from the YAML dict. @@ -319,7 +319,12 @@ def get_config(section): validate_section(section) merged_section = {} - for scope in config_scopes.values(): + if scope is None: + scopes = config_scopes.values() + else: + scopes = [validate_scope(scope)] + + for scope in scopes: # read potentially cached data from the scope. data = scope.get_section(section) if not data or not section in data: @@ -362,52 +367,22 @@ def get_config_filename(scope, section): def update_config(section, update_data, scope=None): """Update the configuration file for a particular scope. - Merges contents of update_data into the scope's data for the - specified section, then writes out the config file. + Overwrites contents of a section in a scope with update_data, + then writes out the config file. - update_data shoudl contain only the section's data, with the - top-level name stripped off. This can be a list, dict, or any + update_data should have the top-level section name stripped off + (it will be re-added). Data itself can be a list, dict, or any other yaml-ish structure. """ # read in the config to ensure we've got current data get_config(section) - validate_section(section) # validate section name + validate_section(section) # validate section name scope = validate_scope(scope) # get ConfigScope object from string. # read only the requested section's data. - data = scope.get_section(section) - data = _merge_yaml(data, { section : update_data }) - scope.write_section(section) - - -def remove_from_config(section, key_to_rm, scope=None): - """Remove a configuration key and write updated configuration to disk. - - Return True if something was removed, False otherwise. - - """ - # ensure configs are current by reading in. - get_config(section) - - # check args and get the objects we need. - scope = validate_scope(scope) - data = scope.get_section(section) - filename = scope.get_section_filename(section) - - # do some checks - if not data: - return False - - if not section in data: - raise ConfigFileError("Invalid configuration file: '%s'" % filename) - - if key_to_rm not in section[section]: - return False - - # remove the key from the section's configuration - del data[section][key_to_rm] + scope.sections[section] = { section : update_data } scope.write_section(section) -- cgit v1.2.3-70-g09d2 From e8e6368cc8ab03be4b635731ed43b0fcae47712a Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 28 Dec 2015 01:14:41 -0800 Subject: Rework mirror configuration. - All of these work: - `spack mirror add` - `spack mirror remove` - `spack mirror list` - `spack mirror` subcommands (except create) now have their own --scope argument. - Mirror config is now stored sanely as an ordered list. --- lib/spack/spack/cmd/mirror.py | 50 ++++++++++++++++++++++++++++++++++--------- lib/spack/spack/stage.py | 6 ++++-- 2 files changed, 44 insertions(+), 12 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/cmd/mirror.py b/lib/spack/spack/cmd/mirror.py index a2d2e80f5e..946b50350b 100644 --- a/lib/spack/spack/cmd/mirror.py +++ b/lib/spack/spack/cmd/mirror.py @@ -60,15 +60,22 @@ def setup_parser(subparser): '-o', '--one-version-per-spec', action='store_const', const=1, default=0, help="Only fetch one 'preferred' version per spec, not all known versions.") + add_parser = sp.add_parser('add', help=mirror_add.__doc__) add_parser.add_argument('name', help="Mnemonic name for mirror.") add_parser.add_argument( 'url', help="URL of mirror directory created by 'spack mirror create'.") + add_parser.add_argument('--scope', choices=spack.config.config_scopes, + help="Configuration scope to modify.") remove_parser = sp.add_parser('remove', help=mirror_remove.__doc__) remove_parser.add_argument('name') + remove_parser.add_argument('--scope', choices=spack.config.config_scopes, + help="Configuration scope to modify.") list_parser = sp.add_parser('list', help=mirror_list.__doc__) + list_parser.add_argument('--scope', choices=spack.config.config_scopes, + help="Configuration scope to read from.") def mirror_add(args): @@ -77,31 +84,54 @@ def mirror_add(args): if url.startswith('/'): url = 'file://' + url - mirror_dict = { args.name : url } - spack.config.update_config('mirrors', { args.name : url }, 'user') + mirrors = spack.config.get_config('mirrors', scope=args.scope) + if not mirrors: + mirrors = [] + + for m in mirrors: + for name, u in m.items(): + if name == args.name: + tty.die("Mirror with name %s already exists." % name) + if u == url: + tty.die("Mirror with url %s already exists." % url) + # should only be one item per mirror dict. + + mirrors.insert(0, { args.name : url }) + spack.config.update_config('mirrors', mirrors, scope=args.scope) def mirror_remove(args): """Remove a mirror by name.""" name = args.name - rmd_something = spack.config.remove_from_config('mirrors', name) - if not rmd_something: - tty.die("No such mirror: %s" % name) + mirrors = spack.config.get_config('mirrors', scope=args.scope) + if not mirrors: + mirrors = [] + + names = [n for m in mirrors for n,u in m.items()] + if not name in names: + tty.die("No mirror with name %s" % name) + + old_mirror = mirrors.pop(names.index(name)) + spack.config.update_config('mirrors', mirrors, scope=args.scope) + tty.msg("Removed mirror %s with url %s." % old_mirror.popitem()) def mirror_list(args): """Print out available mirrors to the console.""" - sec_names = spack.config.get_config('mirrors') - if not sec_names: + mirrors = spack.config.get_config('mirrors', scope=args.scope) + if not mirrors: tty.msg("No mirrors configured.") return - max_len = max(len(s) for s in sec_names) + names = [n for m in mirrors for n,u in m.items()] + max_len = max(len(n) for n in names) fmt = "%%-%ds%%s" % (max_len + 4) - for name, val in sec_names.iteritems(): - print fmt % (name, val) + for m in mirrors: + for name, url in m.items(): + print fmt % (name, url) + # should only be one item per mirror dict. def _read_specs_from_file(filename): diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index 543a3a6223..a9631d4b62 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -26,6 +26,7 @@ import os import re import shutil import tempfile +from urlparse import urljoin import llnl.util.tty as tty from llnl.util.filesystem import * @@ -242,8 +243,9 @@ class Stage(object): # TODO: move mirror logic out of here and clean it up! if self.mirror_path: - urls = ["%s/%s" % (u, self.mirror_path) - for name, u in spack.config.get_config('mirrors')] + mirrors = spack.config.get_config('mirrors') + mirrors = [(n,u) for m in mirrors for n,u in m.items()] + urls = [urljoin(u, self.mirror_path) for name, u in mirrors] digest = None if isinstance(self.fetcher, fs.URLFetchStrategy): -- cgit v1.2.3-70-g09d2 From 30f5ccb80d62608b3006c0f2521c37d3bff6ec32 Mon Sep 17 00:00:00 2001 From: Tom Scogland Date: Tue, 29 Dec 2015 12:14:52 -0800 Subject: create leading directories for resources This allows resources to be placed into subdirectory trees that may not exist in the base package, and may depend on other resources to be staged later. --- lib/spack/spack/package.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 84bcb15f7f..69b263db0c 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -705,10 +705,17 @@ class Package(object): placement = {'': placement} # Make the paths in the dictionary absolute and link for key, value in placement.iteritems(): - link_path = join_path(self.stage.source_path, resource.destination, value) + target_path = join_path(self.stage.source_path, resource.destination) + link_path = join_path(target_path, value) source_path = join_path(stage.source_path, key) if not os.path.exists(link_path): # Create a symlink + try: + os.makedirs(target_path) + except OSError as err: + if err.errno == errno.EEXIST and os.path.isdir(target_path): + pass + else: raise os.symlink(source_path, link_path) ########## self.stage.chdir_to_source() -- cgit v1.2.3-70-g09d2 From cc8c783bb29e037e3343698e7a958bce4c17add5 Mon Sep 17 00:00:00 2001 From: Tom Scogland Date: Tue, 29 Dec 2015 12:27:35 -0800 Subject: adding errno import for the deep directory creation patch --- lib/spack/spack/package.py | 1 + 1 file changed, 1 insertion(+) (limited to 'lib') diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 69b263db0c..999b5e125e 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -34,6 +34,7 @@ rundown on spack and how it differs from homebrew, look at the README. """ import os +import errno import re import time import itertools -- cgit v1.2.3-70-g09d2 From fcdf08e4d7da824afe95445496d8b14f7bc08724 Mon Sep 17 00:00:00 2001 From: Tom Scogland Date: Wed, 30 Dec 2015 12:26:55 -0800 Subject: allow nested resources This solution doesn't really make me happy, but does seem to work. It sorts the resources by the length of the string representing their destination. Since any nested resource must contain another resource's name in its path, it seems that should work, but there should be a better way to do this. --- lib/spack/spack/package.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 999b5e125e..118069a0a7 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -697,7 +697,9 @@ class Package(object): ########## # Stage resources in appropriate path resources = self._get_resources() - for resource in resources: + # TODO: this is to allow nested resources, a better solution would be + # good + for resource in sorted(resources, key=lambda res: len(res.destination)): stage = resource.fetcher.stage _expand_archive(stage, resource.name) # Turn placement into a dict with relative paths -- cgit v1.2.3-70-g09d2 From 4ae98f8b214ea491a50aa2ddf3d995625d1b35f7 Mon Sep 17 00:00:00 2001 From: Tom Scogland Date: Wed, 30 Dec 2015 15:37:06 -0800 Subject: significant llvm update This update significantly reworks the llvm and clang packages. The llvm package now includes variants allowing it to build and install any and all of: * clang * lldb * llvm's libunwind (why, WHY did they name it this?!?) * polly (including building it directly into the clang tools, 3.7.0 only) * clang extra tools * compiler-rt (sanitizers) * clang lto (the gold linker plugin that allows same to work) * libcxx/libcxxabi * libopenmp, also setting the default openmp runtime to same, when parameters happen this shoudl be an option of libomp or libgomp Ideally, this should have rpath setup like the gcc package does, but clang's driver has no support for specs as such, and no clearly equivalent mechanism either. If anyone has ideas on this, they would be welcome. One significant note related to gcc though, if you test this on LLNL systems, or anywhere that has multiple GCCs straddling the dwarf2 boundary and sharing a libstdc++, build a gcc with spack and use that to build clang. If you use a gcc4.8+ to build this with an older libstdc++ it will fail on missing unwind symbols because of the discrepancy. Resource handling has been changed slightly to move the unpacked archive into the target rather than use symlinks, because symlinks break certain kinds of relative paths, and orders resource staging such that nested resources are unpacked after outer ones. --- lib/spack/spack/package.py | 22 +++-- var/spack/packages/clang/package.py | 95 ------------------ var/spack/packages/llvm/package.py | 186 +++++++++++++++++++++++++++++++----- 3 files changed, 176 insertions(+), 127 deletions(-) delete mode 100644 var/spack/packages/clang/package.py (limited to 'lib') diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 118069a0a7..fe82d58394 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -36,6 +36,7 @@ README. import os import errno import re +import shutil import time import itertools import subprocess @@ -711,15 +712,20 @@ class Package(object): target_path = join_path(self.stage.source_path, resource.destination) link_path = join_path(target_path, value) source_path = join_path(stage.source_path, key) + + try: + os.makedirs(target_path) + except OSError as err: + if err.errno == errno.EEXIST and os.path.isdir(target_path): + pass + else: raise + + # NOTE: a reasonable fix for the TODO above might be to have + # these expand in place, but expand_archive does not offer + # this + if not os.path.exists(link_path): - # Create a symlink - try: - os.makedirs(target_path) - except OSError as err: - if err.errno == errno.EEXIST and os.path.isdir(target_path): - pass - else: raise - os.symlink(source_path, link_path) + shutil.move(source_path, link_path) ########## self.stage.chdir_to_source() diff --git a/var/spack/packages/clang/package.py b/var/spack/packages/clang/package.py deleted file mode 100644 index e46e08d5f1..0000000000 --- a/var/spack/packages/clang/package.py +++ /dev/null @@ -1,95 +0,0 @@ -############################################################################## -# Copyright (c) 2013, Lawrence Livermore National Security, LLC. -# Produced at the Lawrence Livermore National Laboratory. -# -# This file is part of Spack. -# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. -# LLNL-CODE-647188 -# -# For details, see https://github.com/llnl/spack -# Please also see the LICENSE file for our notice and the LGPL. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License (as published by -# the Free Software Foundation) version 2.1 dated February 1999. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and -# conditions of the GNU General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -############################################################################## - - -from spack import * - -import os -import os.path - -class Clang(Package): - """The goal of the Clang project is to create a new C, C++, - Objective C and Objective C++ front-end for the LLVM compiler. - """ - homepage = 'http://clang.llvm.org' - url = 'http://llvm.org/releases/3.7.0/cfe-3.7.0.src.tar.xz' - - depends_on('llvm@3.7.0', when='@3.7.0') - depends_on('llvm@3.6.2', when='@3.6.2') - depends_on('llvm@3.5.1', when='@3.5.1') - - version('3.7.0', '8f9d27335e7331cf0a4711e952f21f01', url='http://llvm.org/releases/3.7.0/cfe-3.7.0.src.tar.xz') - version('3.6.2', 'ff862793682f714bb7862325b9c06e20', url='http://llvm.org/releases/3.6.2/cfe-3.6.2.src.tar.xz') - version('3.5.1', '93f9532f8f7e6f1d8e5c1116907051cb', url='http://llvm.org/releases/3.5.1/cfe-3.5.1.src.tar.xz') - - ########## - # @3.7.0 - resource(name='clang-tools-extra', - url='http://llvm.org/releases/3.7.0/clang-tools-extra-3.7.0.src.tar.xz', - md5='d5a87dacb65d981a427a536f6964642e', destination='tools', when='@3.7.0') - ########## - - def install(self, spec, prefix): - env['CXXFLAGS'] = self.compiler.cxx11_flag - - with working_dir('spack-build', create=True): - - options = [] - if '@3.7.0:' in spec: - options.append('-DCLANG_DEFAULT_OPENMP_RUNTIME:STRING=libomp') - options.extend(std_cmake_args) - - cmake('..', - '-DCLANG_PATH_TO_LLVM_BUILD:PATH=%s' % spec['llvm'].prefix, - '-DLLVM_MAIN_SRC_DIR:PATH=%s' % spec['llvm'].prefix, - *options) - make() - make("install") - # CLang doesn't look in llvm folders for system headers... - self.link_llvm_directories(spec) - - def link_llvm_directories(self, spec): - - def clang_include_dir_at(root): - return join_path(root, 'include') - - def clang_lib_dir_at(root): - return join_path(root, 'lib/clang/', str(self.version), 'include') - - def do_link(source_dir, destination_dir): - if os.path.exists(source_dir): - for name in os.listdir(source_dir): - source = join_path(source_dir, name) - link = join_path(destination_dir, name) - os.symlink(source, link) - - # Link folder and files in include - llvm_dir = clang_include_dir_at(spec['llvm'].prefix) - clang_dir = clang_include_dir_at(self.prefix) - do_link(llvm_dir, clang_dir) - # Link folder and files in lib - llvm_dir = clang_lib_dir_at(spec['llvm'].prefix) - clang_dir = clang_lib_dir_at(self.prefix) - do_link(llvm_dir, clang_dir) \ No newline at end of file diff --git a/var/spack/packages/llvm/package.py b/var/spack/packages/llvm/package.py index a3307584e0..eeb2c11fe2 100644 --- a/var/spack/packages/llvm/package.py +++ b/var/spack/packages/llvm/package.py @@ -35,38 +35,176 @@ class Llvm(Package): homepage = 'http://llvm.org/' url = 'http://llvm.org/releases/3.7.0/llvm-3.7.0.src.tar.xz' - version('3.7.0', 'b98b9495e5655a672d6cb83e1a180f8e', url='http://llvm.org/releases/3.7.0/llvm-3.7.0.src.tar.xz') - version('3.6.2', '0c1ee3597d75280dee603bae9cbf5cc2', url='http://llvm.org/releases/3.6.2/llvm-3.6.2.src.tar.xz') - version('3.5.1', '2d3d8004f38852aa679e5945b8ce0b14', url='http://llvm.org/releases/3.5.1/llvm-3.5.1.src.tar.xz') version('3.0', 'a8e5f5f1c1adebae7b4a654c376a6005', url='http://llvm.org/releases/3.0/llvm-3.0.tar.gz') # currently required by mesa package + variant('debug', default=False, description="Build a debug version of LLVM, this increases binary size by an order of magnitude, make sure you have 20-30gb of space available to build this") + variant('clang', default=True, description="Build the LLVM C/C++/Objective-C compiler frontend") + variant('lldb', default=True, description="Build the LLVM debugger") + variant('internal_unwind', default=True, description="Build the libcxxabi libunwind") + variant('polly', default=True, description="Build the LLVM polyhedral optimization plugin, only builds for 3.7.0+") + variant('libcxx', default=True, description="Build the LLVM C++ standard library") + variant('compiler-rt', default=True, description="Build the LLVM compiler runtime, including sanitizers") + variant('lto', default=True, description="Add support for LTO with the gold linker plugin") + + + # Universal dependency depends_on('python@2.7:') - variant('libcxx', default=False, description="Builds the LLVM Standard C++ library targeting C++11") - - ########## - # @3.7.0 - resource(name='compiler-rt', - url='http://llvm.org/releases/3.7.0/compiler-rt-3.7.0.src.tar.xz', md5='383c10affd513026f08936b5525523f5', - destination='projects', when='@3.7.0') - resource(name='openmp', - url='http://llvm.org/releases/3.7.0/openmp-3.7.0.src.tar.xz', md5='f482c86fdead50ba246a1a2b0bbf206f', - destination='projects', when='@3.7.0') - resource(name='libcxx', - url='http://llvm.org/releases/3.7.0/libcxx-3.7.0.src.tar.xz', md5='46aa5175cbe1ad42d6e9c995968e56dd', - destination='projects', placement='libcxx', when='+libcxx@3.7.0') - resource(name='libcxxabi', - url='http://llvm.org/releases/3.7.0/libcxxabi-3.7.0.src.tar.xz', md5='5aa769e2fca79fa5335cfae8f6258772', - destination='projects', placement='libcxxabi', when='+libcxx@3.7.0') - ########## + # lldb dependencies + depends_on('ncurses', when='+lldb') + depends_on('swig', when='+lldb') + depends_on('libedit', when='+lldb') + + # gold support + depends_on('binutils+gold', when='+lto') + + # polly plugin + depends_on('gmp', when='+polly') + depends_on('isl', when='+polly') + + base_url = 'http://llvm.org/releases/%%(version)s/%(pkg)s-%%(version)s.src.tar.xz' + llvm_url = base_url % { 'pkg' : 'llvm'} + + resources = { + 'compiler-rt' : { + 'url' : base_url % { 'pkg' : 'compiler-rt'}, + 'destination' : 'projects', + 'placement' : 'compiler-rt', + }, + 'openmp' : { + 'url' : base_url % { 'pkg' : 'openmp'}, + 'destination' : 'projects', + 'placement' : 'openmp', + }, + 'libcxx' : { + 'url' : base_url % { 'pkg' : 'libcxx'}, + 'destination' : 'projects', + 'placement' : 'libcxx', + }, + 'libcxxabi' : { + 'url' : base_url % { 'pkg' : 'libcxxabi'}, + 'destination' : 'projects', + 'placement' : 'libcxxabi', + }, + 'clang' : { + 'url' : base_url % { 'pkg' : 'cfe'}, + 'destination' : 'tools', + 'placement' : 'clang', + }, + 'clang-tools-extra' : { + 'url' : base_url % { 'pkg' : 'clang-tools-extra'}, + 'destination' : 'tools/clang/tools', + 'placement' : 'extra', + }, + 'lldb' : { + 'url' : base_url % { 'pkg' : 'lldb'}, + 'destination' : 'tools', + 'placement' : 'lldb', + }, + 'polly' : { + 'url' : base_url % { 'pkg' : 'polly'}, + 'destination' : 'tools', + 'placement' : 'polly', + }, + 'llvm-libunwind' : { + 'url' : base_url % { 'pkg' : 'libunwind'}, + 'destination' : 'projects', + 'placement' : 'libunwind', + }, + } + releases = [ + { + 'version' : '3.7.0', + 'md5':'b98b9495e5655a672d6cb83e1a180f8e', + 'resources' : { + 'compiler-rt' : '383c10affd513026f08936b5525523f5', + 'openmp' : 'f482c86fdead50ba246a1a2b0bbf206f', + 'polly' : '32f93ffc9cc7e042df22089761558f8b', + 'libcxx' : '46aa5175cbe1ad42d6e9c995968e56dd', + 'libcxxabi' : '5aa769e2fca79fa5335cfae8f6258772', + 'clang' : '8f9d27335e7331cf0a4711e952f21f01', + 'clang-tools-extra' : 'd5a87dacb65d981a427a536f6964642e', + 'lldb' : 'e5931740400d1dc3e7db4c7ba2ceff68', + 'llvm-libunwind' : '9a75392eb7eb8ed5c0840007e212baf5', + } + }, + { + 'version' : '3.6.2', + 'md5':'0c1ee3597d75280dee603bae9cbf5cc2', + 'resources' : { + 'compiler-rt' : 'e3bc4eb7ba8c39a6fe90d6c988927f3c', + 'openmp' : '65dd5863b9b270960a96817e9152b123', + 'libcxx' : '22214c90697636ef960a49aef7c1823a', + 'libcxxabi' : '17518e361e4e228f193dd91e8ef54ba2', + 'clang' : 'ff862793682f714bb7862325b9c06e20', + 'clang-tools-extra' : '3ebc1dc41659fcec3db1b47d81575e06', + 'lldb' : '51e5eb552f777b950bb0ff326e60d5f0', + } + }, + { + 'version' : '3.5.1', + 'md5':'2d3d8004f38852aa679e5945b8ce0b14', + 'resources' : { + 'compiler-rt' : 'd626cfb8a9712cb92b820798ab5bc1f8', + 'openmp' : '121ddb10167d7fc38b1f7e4b029cf059', + 'libcxx' : '406f09b1dab529f3f7879f4d548329d2', + 'libcxxabi' : 'b22c707e8d474a99865ad3c521c3d464', + 'clang' : '93f9532f8f7e6f1d8e5c1116907051cb', + 'clang-tools-extra' : 'f13f31ed3038acadc6fa63fef812a246', + 'lldb' : 'cc5ea8a414c62c33e760517f8929a204', + } + }, + ] + + for release in releases: + version(release['version'], release['md5'], url=llvm_url % release) + + for name, md5 in release['resources'].items(): + resource(name=name, + url=resources[name]['url'] % release, + md5=md5, + destination=resources[name]['destination'], + when='@%(version)s' % release, + placement=resources[name].get('placement', None)) def install(self, spec, prefix): env['CXXFLAGS'] = self.compiler.cxx11_flag + cmake_args = [ arg for arg in std_cmake_args if 'BUILD_TYPE' not in arg ] + + build_type = 'RelWithDebInfo' if '+debug' in spec else 'Release' + cmake_args.extend([ + '..', + '-DCMAKE_BUILD_TYPE=' + build_type, + '-DLLVM_REQUIRES_RTTI:BOOL=ON', + '-DCLANG_DEFAULT_OPENMP_RUNTIME:STRING=libomp', + '-DPYTHON_EXECUTABLE:PATH=%s/bin/python' % spec['python'].prefix ]) + + if '+lto' in spec: + cmake_args.append('-DLLVM_BINUTILS_INCDIR=' + os.path.join( spec['binutils'].prefix, 'include')) + if '+polly' in spec: + cmake_args.append('-DPOLLY_LINK_INTO_TOOLS:Bool=ON') + else: + cmake_args.append('-DLLVM_EXTERNAL_POLLY_BUILD:Bool=OFF') + + if '+clang' not in spec: + cmake_args.append('-DLLVM_EXTERNAL_CLANG_BUILD:Bool=OFF') + if '+lldb' not in spec: + cmake_args.append('-DLLVM_EXTERNAL_LLDB_BUILD:Bool=OFF') + if '+internal_unwind' not in spec: + cmake_args.append('-DLLVM_EXTERNAL_LIBUNWIND_BUILD:Bool=OFF') + if '+libcxx' not in spec: + cmake_args.append('-DLLVM_EXTERNAL_LIBCXX_BUILD:Bool=OFF') + cmake_args.append('-DLLVM_EXTERNAL_LIBCXXABI_BUILD:Bool=OFF') + if '+compiler-rt' not in spec: + cmake_args.append('-DLLVM_EXTERNAL_COMPILER_RT_BUILD:Bool=OFF') + + if '+clang' not in spec: + if '+clang_extra' in spec: + raise SpackException('The clang_extra variant requires the clang variant to be selected') + if '+lldb' in spec: + raise SpackException('The lldb variant requires the clang variant to be selected') with working_dir('spack-build', create=True): - cmake('..', - '-DLLVM_REQUIRES_RTTI:BOOL=ON', - '-DPYTHON_EXECUTABLE:PATH=%s/bin/python' % spec['python'].prefix, - *std_cmake_args) + cmake(*cmake_args) make() make("install") -- cgit v1.2.3-70-g09d2 From 20b7f8a8e0ea186cde1e641c154566f279e67cd6 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sat, 2 Jan 2016 14:25:10 -0800 Subject: Fix bug in tests. --- lib/spack/spack/compilers/__init__.py | 29 +++++++++++++++-------------- lib/spack/spack/test/mock_packages_test.py | 2 ++ 2 files changed, 17 insertions(+), 14 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index 67dfaa3ac9..2a3d67c731 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -46,6 +46,7 @@ from spack.util.environment import get_path _imported_compilers_module = 'spack.compilers' _required_instance_vars = ['cc', 'cxx', 'f77', 'fc'] +# TODO: customize order in config file _default_order = ['gcc', 'intel', 'pgi', 'clang', 'xlc'] def _auto_compiler_spec(function): @@ -132,7 +133,7 @@ def remove_compiler_from_config(compiler_spec, arch=None, scope=None): spack.config.update_config('compilers', update, scope) -def all_compilers(arch=None, scope=None): +def all_compilers_config(arch=None, scope=None): """Return a set of specs for all the compiler versions currently available to build with. These are instances of CompilerSpec. """ @@ -144,25 +145,25 @@ def all_compilers(arch=None, scope=None): merged_config = get_compiler_config('all', scope=scope) merged_config = spack.config._merge_yaml(merged_config, arch_config) - # Return compiler specs for the result. - return [spack.spec.CompilerSpec(s) for s in merged_config] + return merged_config + + +def all_compilers(arch=None, scope=None): + # Return compiler specs from the merged config. + return [spack.spec.CompilerSpec(s) + for s in all_compilers_config(arch, scope)] -_cached_default_compiler = None def default_compiler(): - global _cached_default_compiler - if _cached_default_compiler: - return _cached_default_compiler versions = [] - for name in _default_order: # TODO: customize order. + for name in _default_order: versions = find(name) - if versions: break - - if not versions: + if versions: + break + else: raise NoCompilersError() - _cached_default_compiler = sorted(versions)[-1] - return _cached_default_compiler + return sorted(versions)[-1] def find_compilers(*path): @@ -224,7 +225,7 @@ def compilers_for_spec(compiler_spec, arch=None, scope=None): """This gets all compilers that satisfy the supplied CompilerSpec. Returns an empty list if none are found. """ - config = get_compiler_config(arch, scope) + config = all_compilers_config(arch, scope) def get_compiler(cspec): items = config[str(cspec)] diff --git a/lib/spack/spack/test/mock_packages_test.py b/lib/spack/spack/test/mock_packages_test.py index 7642edcf4b..e9f1f95df5 100644 --- a/lib/spack/spack/test/mock_packages_test.py +++ b/lib/spack/spack/test/mock_packages_test.py @@ -24,6 +24,7 @@ ############################################################################## import sys import os +import shutil import unittest import tempfile from ordereddict_backport import OrderedDict @@ -103,6 +104,7 @@ class MockPackagesTest(unittest.TestCase): """Restore the real packages path after any test.""" spack.repo.swap(self.db) spack.config.config_scopes = self.real_scopes + shutil.rmtree(self.temp_config, ignore_errors=True) spack.config.clear_config_caches() # Restore dependency changes that happened during the test -- cgit v1.2.3-70-g09d2 From 56937416a92787ba712bbc73c0089cc7eb0c5f78 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sat, 2 Jan 2016 18:18:00 -0800 Subject: All tests that call concretize() need to be MockPackagesTests. --- lib/spack/spack/test/directory_layout.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/test/directory_layout.py b/lib/spack/spack/test/directory_layout.py index 107a7a6412..925cb648ed 100644 --- a/lib/spack/spack/test/directory_layout.py +++ b/lib/spack/spack/test/directory_layout.py @@ -36,21 +36,25 @@ import spack from spack.spec import Spec from spack.repository import RepoPath from spack.directory_layout import YamlDirectoryLayout +from spack.test.mock_packages_test import * + # number of packages to test (to reduce test time) max_packages = 10 -class DirectoryLayoutTest(unittest.TestCase): +class DirectoryLayoutTest(MockPackagesTest): """Tests that a directory layout works correctly and produces a consistent install path.""" def setUp(self): + super(DirectoryLayoutTest, self).setUp() self.tmpdir = tempfile.mkdtemp() self.layout = YamlDirectoryLayout(self.tmpdir) def tearDown(self): + super(DirectoryLayoutTest, self).tearDown() shutil.rmtree(self.tmpdir, ignore_errors=True) self.layout = None -- cgit v1.2.3-70-g09d2 From 21fae634a54b3aa0f0d8f22fcf1e9d1429636c5f Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 3 Jan 2016 01:19:03 -0800 Subject: Add Python 3 aliases to our argparse backport. --- lib/spack/external/argparse.py | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/spack/external/argparse.py b/lib/spack/external/argparse.py index 394e5da152..ec9a9ee738 100644 --- a/lib/spack/external/argparse.py +++ b/lib/spack/external/argparse.py @@ -1067,9 +1067,13 @@ class _SubParsersAction(Action): class _ChoicesPseudoAction(Action): - def __init__(self, name, help): + def __init__(self, name, aliases, help): + metavar = dest = name + if aliases: + metavar += ' (%s)' % ', '.join(aliases) sup = super(_SubParsersAction._ChoicesPseudoAction, self) - sup.__init__(option_strings=[], dest=name, help=help) + sup.__init__(option_strings=[], dest=dest, help=help, + metavar=metavar) def __init__(self, option_strings, @@ -1097,15 +1101,22 @@ class _SubParsersAction(Action): if kwargs.get('prog') is None: kwargs['prog'] = '%s %s' % (self._prog_prefix, name) + aliases = kwargs.pop('aliases', ()) + # create a pseudo-action to hold the choice help if 'help' in kwargs: help = kwargs.pop('help') - choice_action = self._ChoicesPseudoAction(name, help) + choice_action = self._ChoicesPseudoAction(name, aliases, help) self._choices_actions.append(choice_action) # create the parser and add it to the map parser = self._parser_class(**kwargs) self._name_parser_map[name] = parser + + # make parser available under aliases also + for alias in aliases: + self._name_parser_map[alias] = parser + return parser def _get_subactions(self): @@ -1123,8 +1134,9 @@ class _SubParsersAction(Action): try: parser = self._name_parser_map[parser_name] except KeyError: - tup = parser_name, ', '.join(self._name_parser_map) - msg = _('unknown parser %r (choices: %s)' % tup) + args = {'parser_name': parser_name, + 'choices': ', '.join(self._name_parser_map)} + msg = _('unknown parser %(parser_name)r (choices: %(choices)s)') % args raise ArgumentError(self, msg) # parse all the remaining options into the namespace -- cgit v1.2.3-70-g09d2 From b02faf56411c734b91a7f51b60f8921a31e12c16 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 3 Jan 2016 02:27:50 -0800 Subject: add/remove/list working for new config format. - mirrors.yaml now uses dict order for precedence, instead of lists of dicts. - spack.cmd now specifies default scope for add/remove and for list with `default_modify_scope` and `default_list_scope`. - commands that only read or list default to all scopes (merged) - commands that modify configs modify user scope (highest precedence) by default - These vars are used in setup_paraser for mirror/repo/compiler. - Spack's argparse supports aliases now. - added 'rm' alias for `spack [repo|compiler|mirror] remove` --- lib/spack/spack/__init__.py | 11 ++--- lib/spack/spack/cmd/__init__.py | 9 ++++ lib/spack/spack/cmd/compiler.py | 17 +++++-- lib/spack/spack/cmd/mirror.py | 65 ++++++++++++++----------- lib/spack/spack/cmd/repo.py | 105 ++++++++++++++++++++++++++++++++++++---- lib/spack/spack/config.py | 57 ++++++++-------------- lib/spack/spack/repository.py | 63 ++++++++++++++++-------- lib/spack/spack/stage.py | 3 +- 8 files changed, 221 insertions(+), 109 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index 973ba64b96..6c4a15aaab 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -56,15 +56,12 @@ etc_path = join_path(prefix, "etc") # Set up the default packages database. # import spack.repository -_repo_paths = spack.config.get_repos_config() -if not _repo_paths: - tty.die("Spack configuration contains no package repositories.") - try: - repo = spack.repository.RepoPath(*_repo_paths) + repo = spack.repository.RepoPath() sys.meta_path.append(repo) -except spack.repository.BadRepoError, e: - tty.die('Bad repository. %s' % e.message) +except spack.repository.RepoError, e: + tty.error('while initializing Spack RepoPath:') + tty.die(e.message) # # Set up the installed packages database diff --git a/lib/spack/spack/cmd/__init__.py b/lib/spack/spack/cmd/__init__.py index 926e7ac14a..6c635a1e6c 100644 --- a/lib/spack/spack/cmd/__init__.py +++ b/lib/spack/spack/cmd/__init__.py @@ -31,6 +31,15 @@ from llnl.util.lang import attr_setdefault import spack import spack.spec +import spack.config + +# +# Settings for commands that modify configuration +# +# Commands that modify confguration By default modify the *highest* priority scope. +default_modify_scope = spack.config.highest_precedence_scope().name +# Commands that list confguration list *all* scopes by default. +default_list_scope = None # cmd has a submodule called "list" so preserve the python list module python_list = list diff --git a/lib/spack/spack/cmd/compiler.py b/lib/spack/spack/cmd/compiler.py index a3860abf76..75b51f6b49 100644 --- a/lib/spack/spack/cmd/compiler.py +++ b/lib/spack/spack/cmd/compiler.py @@ -42,25 +42,31 @@ def setup_parser(subparser): sp = subparser.add_subparsers( metavar='SUBCOMMAND', dest='compiler_command') + scopes = spack.config.config_scopes + + # Add add_parser = sp.add_parser('add', help='Add compilers to the Spack configuration.') add_parser.add_argument('add_paths', nargs=argparse.REMAINDER) - add_parser.add_argument('--scope', choices=spack.config.config_scopes, + add_parser.add_argument('--scope', choices=scopes, default=spack.cmd.default_modify_scope, help="Configuration scope to modify.") - remove_parser = sp.add_parser('remove', help='Remove compiler by spec.') + # Remove + remove_parser = sp.add_parser('remove', aliases=['rm'], help='Remove compiler by spec.') remove_parser.add_argument( '-a', '--all', action='store_true', help='Remove ALL compilers that match spec.') remove_parser.add_argument('compiler_spec') - remove_parser.add_argument('--scope', choices=spack.config.config_scopes, + remove_parser.add_argument('--scope', choices=scopes, default=spack.cmd.default_modify_scope, help="Configuration scope to modify.") + # List list_parser = sp.add_parser('list', help='list available compilers') - list_parser.add_argument('--scope', choices=spack.config.config_scopes, + list_parser.add_argument('--scope', choices=scopes, default=spack.cmd.default_list_scope, help="Configuration scope to read from.") + # Info info_parser = sp.add_parser('info', help='Show compiler paths.') info_parser.add_argument('compiler_spec') - info_parser.add_argument('--scope', choices=spack.config.config_scopes, + info_parser.add_argument('--scope', choices=scopes, default=spack.cmd.default_list_scope, help="Configuration scope to read from.") @@ -132,6 +138,7 @@ def compiler_list(args): def compiler(parser, args): action = { 'add' : compiler_add, 'remove' : compiler_remove, + 'rm' : compiler_remove, 'info' : compiler_info, 'list' : compiler_list } action[args.compiler_command](args) diff --git a/lib/spack/spack/cmd/mirror.py b/lib/spack/spack/cmd/mirror.py index 946b50350b..885483a840 100644 --- a/lib/spack/spack/cmd/mirror.py +++ b/lib/spack/spack/cmd/mirror.py @@ -36,6 +36,7 @@ import spack.config import spack.mirror from spack.spec import Spec from spack.error import SpackError +from spack.util.spack_yaml import syaml_dict description = "Manage mirrors." @@ -47,6 +48,7 @@ def setup_parser(subparser): sp = subparser.add_subparsers( metavar='SUBCOMMAND', dest='mirror_command') + # Create create_parser = sp.add_parser('create', help=mirror_create.__doc__) create_parser.add_argument('-d', '--directory', default=None, help="Directory in which to create mirror.") @@ -60,22 +62,29 @@ def setup_parser(subparser): '-o', '--one-version-per-spec', action='store_const', const=1, default=0, help="Only fetch one 'preferred' version per spec, not all known versions.") + scopes = spack.config.config_scopes + # Add add_parser = sp.add_parser('add', help=mirror_add.__doc__) add_parser.add_argument('name', help="Mnemonic name for mirror.") add_parser.add_argument( 'url', help="URL of mirror directory created by 'spack mirror create'.") - add_parser.add_argument('--scope', choices=spack.config.config_scopes, - help="Configuration scope to modify.") + add_parser.add_argument( + '--scope', choices=scopes, default=spack.cmd.default_modify_scope, + help="Configuration scope to modify.") - remove_parser = sp.add_parser('remove', help=mirror_remove.__doc__) + # Remove + remove_parser = sp.add_parser('remove', aliases=['rm'], help=mirror_remove.__doc__) remove_parser.add_argument('name') - remove_parser.add_argument('--scope', choices=spack.config.config_scopes, - help="Configuration scope to modify.") + remove_parser.add_argument( + '--scope', choices=scopes, default=spack.cmd.default_modify_scope, + help="Configuration scope to modify.") + # List list_parser = sp.add_parser('list', help=mirror_list.__doc__) - list_parser.add_argument('--scope', choices=spack.config.config_scopes, - help="Configuration scope to read from.") + list_parser.add_argument( + '--scope', choices=scopes, default=spack.cmd.default_list_scope, + help="Configuration scope to read from.") def mirror_add(args): @@ -86,17 +95,18 @@ def mirror_add(args): mirrors = spack.config.get_config('mirrors', scope=args.scope) if not mirrors: - mirrors = [] - - for m in mirrors: - for name, u in m.items(): - if name == args.name: - tty.die("Mirror with name %s already exists." % name) - if u == url: - tty.die("Mirror with url %s already exists." % url) - # should only be one item per mirror dict. - - mirrors.insert(0, { args.name : url }) + mirrors = syaml_dict() + + for name, u in mirrors.items(): + if name == args.name: + tty.die("Mirror with name %s already exists." % name) + if u == url: + tty.die("Mirror with url %s already exists." % url) + # should only be one item per mirror dict. + + items = [(n,u) for n,u in mirrors.items()] + items.insert(0, (args.name, url)) + mirrors = syaml_dict(items) spack.config.update_config('mirrors', mirrors, scope=args.scope) @@ -106,15 +116,14 @@ def mirror_remove(args): mirrors = spack.config.get_config('mirrors', scope=args.scope) if not mirrors: - mirrors = [] + mirrors = syaml_dict() - names = [n for m in mirrors for n,u in m.items()] - if not name in names: + if not name in mirrors: tty.die("No mirror with name %s" % name) - old_mirror = mirrors.pop(names.index(name)) + old_value = mirrors.pop(name) spack.config.update_config('mirrors', mirrors, scope=args.scope) - tty.msg("Removed mirror %s with url %s." % old_mirror.popitem()) + tty.msg("Removed mirror %s with url %s." % (name, old_value)) def mirror_list(args): @@ -124,14 +133,11 @@ def mirror_list(args): tty.msg("No mirrors configured.") return - names = [n for m in mirrors for n,u in m.items()] - max_len = max(len(n) for n in names) + max_len = max(len(n) for n in mirrors.keys()) fmt = "%%-%ds%%s" % (max_len + 4) - for m in mirrors: - for name, url in m.items(): - print fmt % (name, url) - # should only be one item per mirror dict. + for name in mirrors: + print fmt % (name, mirrors[name]) def _read_specs_from_file(filename): @@ -205,6 +211,7 @@ def mirror(parser, args): action = { 'create' : mirror_create, 'add' : mirror_add, 'remove' : mirror_remove, + 'rm' : mirror_remove, 'list' : mirror_list } action[args.mirror_command](args) diff --git a/lib/spack/spack/cmd/repo.py b/lib/spack/spack/cmd/repo.py index 991d306c04..ebe42d0138 100644 --- a/lib/spack/spack/cmd/repo.py +++ b/lib/spack/spack/cmd/repo.py @@ -1,5 +1,5 @@ ############################################################################## -# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Copyright (c) 2013-2015, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory. # # This file is part of Spack. @@ -33,12 +33,13 @@ from llnl.util.filesystem import join_path, mkdirp import spack.spec import spack.config from spack.util.environment import get_path -from spack.repository import packages_dir_name, repo_config_name, Repo +from spack.repository import * description = "Manage package source repositories." def setup_parser(subparser): sp = subparser.add_subparsers(metavar='SUBCOMMAND', dest='repo_command') + scopes = spack.config.config_scopes # Create create_parser = sp.add_parser('create', help=repo_create.__doc__) @@ -49,6 +50,25 @@ def setup_parser(subparser): # List list_parser = sp.add_parser('list', help=repo_list.__doc__) + list_parser.add_argument( + '--scope', choices=scopes, default=spack.cmd.default_list_scope, + help="Configuration scope to read from.") + + # Add + add_parser = sp.add_parser('add', help=repo_add.__doc__) + add_parser.add_argument('path', help="Path to a Spack package repository directory.") + add_parser.add_argument( + '--scope', choices=scopes, default=spack.cmd.default_modify_scope, + help="Configuration scope to modify.") + + # Remove + remove_parser = sp.add_parser('remove', help=repo_remove.__doc__, aliases=['rm']) + remove_parser.add_argument( + 'path_or_namespace', + help="Path or namespace of a Spack package repository.") + remove_parser.add_argument( + '--scope', choices=scopes, default=spack.cmd.default_modify_scope, + help="Configuration scope to modify.") def repo_create(args): @@ -104,25 +124,87 @@ def repo_create(args): def repo_add(args): - """Remove a package source from the Spack configuration""" - # FIXME: how to deal with this with the current config architecture? - # FIXME: Repos do not have mnemonics, which I assumed would be simpler... should they have them after all? + """Add a package source to the Spack configuration""" + path = args.path + + # check if the path is relative to the spack directory. + real_path = path + if path.startswith('$spack'): + real_path = spack.repository.substitute_spack_prefix(path) + elif not os.path.isabs(real_path): + real_path = os.path.abspath(real_path) + path = real_path + + # check if the path exists + if not os.path.exists(real_path): + tty.die("No such file or directory: '%s'." % path) + + # Make sure the path is a directory. + if not os.path.isdir(real_path): + tty.die("Not a Spack repository: '%s'." % path) + + # Make sure it's actually a spack repository by constructing it. + repo = Repo(real_path) + + # If that succeeds, finally add it to the configuration. + repos = spack.config.get_config('repos', args.scope) + if not repos: repos = [] + + if repo.root in repos or path in repos: + tty.die("Repository is already registered with Spack: '%s'" % path) + + repos.insert(0, path) + spack.config.update_config('repos', repos, args.scope) + tty.msg("Created repo with namespace '%s'." % repo.namespace) def repo_remove(args): - """Remove a package source from the Spack configuration""" - # FIXME: see above. + """Remove a repository from the Spack configuration.""" + repos = spack.config.get_config('repos', args.scope) + path_or_namespace = args.path_or_namespace + + # If the argument is a path, remove that repository from config. + path = os.path.abspath(path_or_namespace) + if path in repos: + repos.remove(path) + spack.config.update_config('repos', repos, args.scope) + tty.msg("Removed repository '%s'." % path) + return + + # If it is a namespace, remove corresponding repo + for path in repos: + try: + repo = Repo(path) + if repo.namespace == path_or_namespace: + repos.remove(repo.root) + spack.config.update_config('repos', repos, args.scope) + tty.msg("Removed repository '%s' with namespace %s." + % (repo.root, repo.namespace)) + return + except RepoError as e: + continue + + tty.die("No repository with path or namespace: '%s'" + % path_or_namespace) def repo_list(args): """List package sources and their mnemoics""" - roots = spack.config.get_repos_config() - repos = [Repo(r) for r in roots] + roots = spack.config.get_config('repos', args.scope) + repos = [] + for r in roots: + try: + repos.append(Repo(r)) + except RepoError as e: + continue msg = "%d package repositor" % len(repos) msg += "y." if len(repos) == 1 else "ies." tty.msg(msg) + if not repos: + return + max_ns_len = max(len(r.namespace) for r in repos) for repo in repos: fmt = "%%-%ds%%s" % (max_ns_len + 4) @@ -131,5 +213,8 @@ def repo_list(args): def repo(parser, args): action = { 'create' : repo_create, - 'list' : repo_list } + 'list' : repo_list, + 'add' : repo_add, + 'remove' : repo_remove, + 'rm' : repo_remove} action[args.repo_command](args) diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index d266d2e23f..c53dcbc405 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -204,6 +204,11 @@ ConfigScope('site', os.path.join(spack.etc_path, 'spack')), ConfigScope('user', os.path.expanduser('~/.spack')) +def highest_precedence_scope(): + """Get the scope with highest precedence (prefs will override others).""" + return config_scopes.values()[-1] + + def validate_scope(scope): """Ensure that scope is valid, and return a valid scope if it is None. @@ -214,7 +219,7 @@ def validate_scope(scope): """ if scope is None: # default to the scope with highest precedence. - return config_scopes.values()[-1] + return highest_precedence_scope() elif scope in config_scopes: return config_scopes[scope] @@ -287,15 +292,10 @@ def _merge_yaml(dest, source): dest[:] = source + [x for x in dest if x not in seen] return dest - # Source dict is merged into dest. Extra ':' means overwrite. + # Source dict is merged into dest. elif they_are(dict): for sk, sv in source.iteritems(): - # allow total override with, e.g., repos:: - override = sk.endswith(':') - if override: - sk = sk.rstrip(':') - - if override or not sk in dest: + if not sk in dest: dest[sk] = copy.copy(sv) else: dest[sk] = _merge_yaml(dest[sk], source[sk]) @@ -306,18 +306,13 @@ def _merge_yaml(dest, source): return copy.copy(source) -def substitute_spack_prefix(path): - """Replaces instances of $spack with Spack's prefix.""" - return path.replace('$spack', spack.prefix) - - def get_config(section, scope=None): """Get configuration settings for a section. Strips off the top-level section name from the YAML dict. """ validate_section(section) - merged_section = {} + merged_section = syaml.syaml_dict() if scope is None: scopes = config_scopes.values() @@ -327,37 +322,25 @@ def get_config(section, scope=None): for scope in scopes: # read potentially cached data from the scope. data = scope.get_section(section) - if not data or not section in data: - continue - # extract data under the section name header - data = data[section] + # Skip empty configs + if not data or not isinstance(data, dict): + continue - # ignore empty sections for easy commenting of single-line configs. - if not data: + # Allow complete override of site config with '
::' + override_key = section + ':' + if not (section in data or override_key in data): + tty.warn("Skipping bad configuration file: '%s'" % scope.path) continue - # merge config data from scopes. - merged_section = _merge_yaml(merged_section, data) + if override_key in data: + merged_section = data[override_key] + else: + merged_section = _merge_yaml(merged_section, data[section]) return merged_section -def get_repos_config(): - repo_list = get_config('repos') - if repo_list is None: - return [] - - if not isinstance(repo_list, list): - tty.die("Bad repository configuration. 'repos' element does not contain a list.") - - def expand_repo_path(path): - path = substitute_spack_prefix(path) - path = os.path.expanduser(path) - return path - return [expand_repo_path(repo) for repo in repo_list] - - def get_config_filename(scope, section): """For some scope and section, get the name of the configuration file""" scope = validate_scope(scope) diff --git a/lib/spack/spack/repository.py b/lib/spack/spack/repository.py index 4e91855db0..3367572ef5 100644 --- a/lib/spack/spack/repository.py +++ b/lib/spack/spack/repository.py @@ -36,6 +36,7 @@ import llnl.util.tty as tty from llnl.util.filesystem import join_path import spack.error +import spack.config import spack.spec from spack.virtual import ProviderIndex from spack.util.naming import * @@ -53,6 +54,7 @@ repo_config_name = 'repo.yaml' # Top-level filename for repo config. packages_dir_name = 'packages' # Top-level repo directory containing pkgs. package_file_name = 'package.py' # Filename for packages in a repository. + def _autospec(function): """Decorator that automatically converts the argument of a single-arg function to a Spec.""" @@ -71,6 +73,11 @@ def _make_namespace_module(ns): return module +def substitute_spack_prefix(path): + """Replaces instances of $spack with Spack's prefix.""" + return path.replace('$spack', spack.prefix) + + class RepoPath(object): """A RepoPath is a list of repos that function as one. @@ -89,10 +96,20 @@ class RepoPath(object): self._all_package_names = [] self._provider_index = None + # If repo_dirs is empty, just use the configuration + if not repo_dirs: + repo_dirs = spack.config.get_config('repos') + if not repo_dirs: + raise NoRepoConfiguredError( + "Spack configuration contains no package repositories.") + # Add each repo to this path. for root in repo_dirs: - repo = Repo(root, self.super_namespace) - self.put_last(repo) + try: + repo = Repo(root, self.super_namespace) + self.put_last(repo) + except RepoError as e: + tty.warn("Failed to initialize repository at '%s'." % root, e.message) def swap(self, other): @@ -121,12 +138,12 @@ class RepoPath(object): """ if repo.root in self.by_path: - raise DuplicateRepoError("Package repos are the same", - repo, self.by_path[repo.root]) + raise DuplicateRepoError("Duplicate repository: '%s'" % repo.root) if repo.namespace in self.by_namespace: - raise DuplicateRepoError("Package repos cannot provide the same namespace", - repo, self.by_namespace[repo.namespace]) + raise DuplicateRepoError( + "Package repos '%s' and '%s' both provide namespace %s." + % (repo.root, self.by_namespace[repo.namespace].root, repo.namespace)) # Add repo to the pkg indexes self.by_namespace[repo.full_namespace] = repo @@ -292,7 +309,8 @@ class Repo(object): """ # Root directory, containing _repo.yaml and package dirs - self.root = root + # Allow roots to by spack-relative by starting with '$spack' + self.root = substitute_spack_prefix(root) # super-namespace for all packages in the Repo self.super_namespace = namespace @@ -629,13 +647,27 @@ class Repo(object): return self.exists(pkg_name) -class BadRepoError(spack.error.SpackError): +class RepoError(spack.error.SpackError): + """Superclass for repository-related errors.""" + + +class NoRepoConfiguredError(RepoError): + """Raised when there are no repositories configured.""" + + +class BadRepoError(RepoError): """Raised when repo layout is invalid.""" - def __init__(self, msg): - super(BadRepoError, self).__init__(msg) -class UnknownPackageError(spack.error.SpackError): +class DuplicateRepoError(RepoError): + """Raised when duplicate repos are added to a RepoPath.""" + + +class PackageLoadError(spack.error.SpackError): + """Superclass for errors related to loading packages.""" + + +class UnknownPackageError(PackageLoadError): """Raised when we encounter a package spack doesn't have.""" def __init__(self, name, repo=None): msg = None @@ -647,14 +679,7 @@ class UnknownPackageError(spack.error.SpackError): self.name = name -class DuplicateRepoError(spack.error.SpackError): - """Raised when duplicate repos are added to a RepoPath.""" - def __init__(self, msg, repo1, repo2): - super(UnknownPackageError, self).__init__( - "%s: %s, %s" % (msg, repo1, repo2)) - - -class FailedConstructorError(spack.error.SpackError): +class FailedConstructorError(PackageLoadError): """Raised when a package's class constructor fails.""" def __init__(self, name, exc_type, exc_obj, exc_tb): super(FailedConstructorError, self).__init__( diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index a9631d4b62..61f9846835 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -244,8 +244,7 @@ class Stage(object): # TODO: move mirror logic out of here and clean it up! if self.mirror_path: mirrors = spack.config.get_config('mirrors') - mirrors = [(n,u) for m in mirrors for n,u in m.items()] - urls = [urljoin(u, self.mirror_path) for name, u in mirrors] + urls = [urljoin(u, self.mirror_path) for name, u in mirrors.items()] digest = None if isinstance(self.fetcher, fs.URLFetchStrategy): -- cgit v1.2.3-70-g09d2 From f9fbb57d3182bf0e9c39e4b7c0a958c6ba87a2fa Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Wed, 13 Jan 2016 12:32:04 -0600 Subject: Add NAG Fortran Compiler support --- lib/spack/env/cc | 4 ++-- lib/spack/spack/compilers/__init__.py | 2 +- lib/spack/spack/compilers/nag.py | 24 ++++++++++++++++++++++++ 3 files changed, 27 insertions(+), 3 deletions(-) create mode 100644 lib/spack/spack/compilers/nag.py (limited to 'lib') diff --git a/lib/spack/env/cc b/lib/spack/env/cc index 0966277a91..aacba996b3 100755 --- a/lib/spack/env/cc +++ b/lib/spack/env/cc @@ -94,11 +94,11 @@ case "$command" in command="$SPACK_CXX" language="C++" ;; - f90|fc|f95|gfortran|ifort|pgf90|xlf90) + f90|fc|f95|gfortran|ifort|pgf90|xlf90|nagfor) command="$SPACK_FC" language="Fortran 90" ;; - f77|gfortran|ifort|pgf77|xlf) + f77|gfortran|ifort|pgf77|xlf|nagfor) command="$SPACK_F77" language="Fortran 77" ;; diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index 66e608cf79..a6198fc1ce 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -45,7 +45,7 @@ from spack.util.environment import get_path _imported_compilers_module = 'spack.compilers' _required_instance_vars = ['cc', 'cxx', 'f77', 'fc'] -_default_order = ['gcc', 'intel', 'pgi', 'clang', 'xlc'] +_default_order = ['gcc', 'intel', 'pgi', 'clang', 'xlc', 'nag'] def _auto_compiler_spec(function): def converter(cspec_like): diff --git a/lib/spack/spack/compilers/nag.py b/lib/spack/spack/compilers/nag.py new file mode 100644 index 0000000000..2dfc97af73 --- /dev/null +++ b/lib/spack/spack/compilers/nag.py @@ -0,0 +1,24 @@ +class Nag(Compiler): + # Subclasses use possible names of C compiler + cc_names = [] + + # Subclasses use possible names of C++ compiler + cxx_names = [] + + # Subclasses use possible names of Fortran 77 compiler + f77_names = ['nagfor'] + + # Subclasses use possible names of Fortran 90 compiler + fc_names = ['nagfor'] + + @classmethod + def default_version(self, comp): + """The '-V' option works for nag compilers. + Output looks like this:: + + NAG Fortran Compiler Release 6.0(Hibiya) Build 1037 + Product NPL6A60NA for x86-64 Linux + Copyright 1990-2015 The Numerical Algorithms Group Ltd., Oxford, U.K. + """ + return get_compiler_version( + comp, '-V', r'NAG Fortran Compiler Release ([0-9.]+)') -- cgit v1.2.3-70-g09d2 From 05b30bf83e0bc8723472b6d609c692c3b83b486c Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Thu, 14 Jan 2016 10:26:31 -0800 Subject: Make text wrapping off by default in tty, add a kwarg for it. --- lib/spack/llnl/util/tty/__init__.py | 39 +++++++++++++++++++++++------------- lib/spack/spack/config.py | 40 +++++++++++++++++++++++++++++++------ lib/spack/spack/package.py | 2 +- 3 files changed, 60 insertions(+), 21 deletions(-) (limited to 'lib') diff --git a/lib/spack/llnl/util/tty/__init__.py b/lib/spack/llnl/util/tty/__init__.py index 203f429a48..3ecd3a4ac2 100644 --- a/lib/spack/llnl/util/tty/__init__.py +++ b/lib/spack/llnl/util/tty/__init__.py @@ -63,35 +63,46 @@ def msg(message, *args): def info(message, *args, **kwargs): format = kwargs.get('format', '*b') stream = kwargs.get('stream', sys.stdout) + wrap = kwargs.get('wrap', False) cprint("@%s{==>} %s" % (format, cescape(str(message))), stream=stream) for arg in args: - lines = textwrap.wrap( - str(arg), initial_indent=indent, subsequent_indent=indent) - for line in lines: - stream.write(line + '\n') + if wrap: + lines = textwrap.wrap( + str(arg), initial_indent=indent, subsequent_indent=indent) + for line in lines: + stream.write(line + '\n') + else: + stream.write(indent + str(arg) + '\n') -def verbose(message, *args): +def verbose(message, *args, **kwargs): if _verbose: - info(message, *args, format='c') + kwargs.setdefault('format', 'c') + info(message, *args, **kwargs) -def debug(message, *args): +def debug(message, *args, **kwargs): if _debug: - info(message, *args, format='g', stream=sys.stderr) + kwargs.setdefault('format', 'g') + kwargs.setdefault('stream', sys.stderr) + info(message, *args, **kwargs) -def error(message, *args): - info("Error: " + str(message), *args, format='*r', stream=sys.stderr) +def error(message, *args, **kwargs): + kwargs.setdefault('format', '*r') + kwargs.setdefault('stream', sys.stderr) + info("Error: " + str(message), *args, **kwargs) -def warn(message, *args): - info("Warning: " + str(message), *args, format='*Y', stream=sys.stderr) +def warn(message, *args, **kwargs): + kwargs.setdefault('format', '*Y') + kwargs.setdefault('stream', sys.stderr) + info("Warning: " + str(message), *args, **kwargs) -def die(message, *args): - error(message, *args) +def die(message, *args, **kwargs): + error(message, *args, **kwargs) sys.exit(1) diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index c53dcbc405..3ff83ae529 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -135,8 +135,34 @@ from spack.error import SpackError import spack.util.spack_yaml as syaml -"""Dict from section names -> function to check section YAML format.""" -valid_sections = ['compilers', 'mirrors', 'repos'] +"""Dict from section names -> schema for that section.""" +section_schemas = { + 'compilers' : { + '$schema': 'http://json-schema.org/schema#', + 'title' : 'Spack compiler configuration file schema', + 'type' : 'object', + 'properties' : { + 'compilers' : { + 'type' : 'map', + }, + }, + }, + + 'mirrors' : { + '$schema': 'http://json-schema.org/schema#', + 'title' : 'Spack mirror configuration file schema', + 'type' : 'map', + 'properties' : { + 'mirrors' : { + + } + }, + }, + + 'repos' : { + '$schema': 'http://json-schema.org/schema#', + 'title' : 'Spack repository configuration file schema', + }} """OrderedDict of config scopes keyed by name. Later scopes will override earlier scopes. @@ -146,9 +172,9 @@ config_scopes = OrderedDict() def validate_section(section): """Raise a ValueError if the section is not a valid section.""" - if section not in valid_sections: + if section not in section_schemas: raise ValueError("Invalid config section: '%s'. Options are %s." - % (section, valid_sections)) + % (section, section_schemas)) class ConfigScope(object): @@ -369,10 +395,12 @@ def update_config(section, update_data, scope=None): scope.write_section(section) -"""Print a configuration to stdout""" def print_section(section): + """Print a configuration to stdout.""" try: - yaml.dump(get_config(section), stream=sys.stdout, default_flow_style=False) + data = syaml.syaml_dict() + data[section] = get_config(section) + syaml.dump(data, stream=sys.stdout, default_flow_style=False) except (yaml.YAMLError, IOError) as e: raise ConfigError("Error reading configuration: %s" % section) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 5db83064b5..f926a14206 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -858,7 +858,7 @@ class Package(object): tty.warn("Keeping install prefix in place despite error.", "Spack will think this package is installed." + "Manually remove this directory to fix:", - self.prefix) + self.prefix, wrap=True) def real_work(): -- cgit v1.2.3-70-g09d2 From b567cb57e1c74dcb876b0ed935295a848ec45438 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Thu, 14 Jan 2016 10:40:34 -0800 Subject: Add jsonschema validation for config files. --- lib/spack/external/jsonschema/COPYING | 19 + lib/spack/external/jsonschema/README.rst | 104 +++ lib/spack/external/jsonschema/__init__.py | 26 + lib/spack/external/jsonschema/__main__.py | 2 + lib/spack/external/jsonschema/_format.py | 240 +++++++ lib/spack/external/jsonschema/_reflect.py | 155 ++++ lib/spack/external/jsonschema/_utils.py | 213 ++++++ lib/spack/external/jsonschema/_validators.py | 358 ++++++++++ lib/spack/external/jsonschema/cli.py | 72 ++ lib/spack/external/jsonschema/compat.py | 53 ++ lib/spack/external/jsonschema/exceptions.py | 264 +++++++ lib/spack/external/jsonschema/schemas/draft3.json | 201 ++++++ lib/spack/external/jsonschema/schemas/draft4.json | 221 ++++++ lib/spack/external/jsonschema/tests/__init__.py | 0 lib/spack/external/jsonschema/tests/compat.py | 15 + lib/spack/external/jsonschema/tests/test_cli.py | 110 +++ .../external/jsonschema/tests/test_exceptions.py | 382 ++++++++++ lib/spack/external/jsonschema/tests/test_format.py | 63 ++ .../jsonschema/tests/test_jsonschema_test_suite.py | 290 ++++++++ .../external/jsonschema/tests/test_validators.py | 786 +++++++++++++++++++++ lib/spack/external/jsonschema/validators.py | 428 +++++++++++ lib/spack/spack/__init__.py | 5 +- lib/spack/spack/cmd/repo.py | 5 +- lib/spack/spack/config.py | 199 +++++- 24 files changed, 4168 insertions(+), 43 deletions(-) create mode 100644 lib/spack/external/jsonschema/COPYING create mode 100644 lib/spack/external/jsonschema/README.rst create mode 100644 lib/spack/external/jsonschema/__init__.py create mode 100644 lib/spack/external/jsonschema/__main__.py create mode 100644 lib/spack/external/jsonschema/_format.py create mode 100644 lib/spack/external/jsonschema/_reflect.py create mode 100644 lib/spack/external/jsonschema/_utils.py create mode 100644 lib/spack/external/jsonschema/_validators.py create mode 100644 lib/spack/external/jsonschema/cli.py create mode 100644 lib/spack/external/jsonschema/compat.py create mode 100644 lib/spack/external/jsonschema/exceptions.py create mode 100644 lib/spack/external/jsonschema/schemas/draft3.json create mode 100644 lib/spack/external/jsonschema/schemas/draft4.json create mode 100644 lib/spack/external/jsonschema/tests/__init__.py create mode 100644 lib/spack/external/jsonschema/tests/compat.py create mode 100644 lib/spack/external/jsonschema/tests/test_cli.py create mode 100644 lib/spack/external/jsonschema/tests/test_exceptions.py create mode 100644 lib/spack/external/jsonschema/tests/test_format.py create mode 100644 lib/spack/external/jsonschema/tests/test_jsonschema_test_suite.py create mode 100644 lib/spack/external/jsonschema/tests/test_validators.py create mode 100644 lib/spack/external/jsonschema/validators.py (limited to 'lib') diff --git a/lib/spack/external/jsonschema/COPYING b/lib/spack/external/jsonschema/COPYING new file mode 100644 index 0000000000..af9cfbdb13 --- /dev/null +++ b/lib/spack/external/jsonschema/COPYING @@ -0,0 +1,19 @@ +Copyright (c) 2013 Julian Berman + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/lib/spack/external/jsonschema/README.rst b/lib/spack/external/jsonschema/README.rst new file mode 100644 index 0000000000..20c2fe6266 --- /dev/null +++ b/lib/spack/external/jsonschema/README.rst @@ -0,0 +1,104 @@ +========== +jsonschema +========== + +``jsonschema`` is an implementation of `JSON Schema `_ +for Python (supporting 2.6+ including Python 3). + +.. code-block:: python + + >>> from jsonschema import validate + + >>> # A sample schema, like what we'd get from json.load() + >>> schema = { + ... "type" : "object", + ... "properties" : { + ... "price" : {"type" : "number"}, + ... "name" : {"type" : "string"}, + ... }, + ... } + + >>> # If no exception is raised by validate(), the instance is valid. + >>> validate({"name" : "Eggs", "price" : 34.99}, schema) + + >>> validate( + ... {"name" : "Eggs", "price" : "Invalid"}, schema + ... ) # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + ... + ValidationError: 'Invalid' is not of type 'number' + + +Features +-------- + +* Full support for + `Draft 3 `_ + **and** `Draft 4 `_ + of the schema. + +* `Lazy validation `_ + that can iteratively report *all* validation errors. + +* Small and extensible + +* `Programmatic querying `_ + of which properties or items failed validation. + + +Release Notes +------------- + +* A simple CLI was added for validation +* Validation errors now keep full absolute paths and absolute schema paths in + their ``absolute_path`` and ``absolute_schema_path`` attributes. The ``path`` + and ``schema_path`` attributes are deprecated in favor of ``relative_path`` + and ``relative_schema_path``\ . + +*Note:* Support for Python 3.2 was dropped in this release, and installation +now uses setuptools. + + +Running the Test Suite +---------------------- + +``jsonschema`` uses the wonderful `Tox `_ for its +test suite. (It really is wonderful, if for some reason you haven't heard of +it, you really should use it for your projects). + +Assuming you have ``tox`` installed (perhaps via ``pip install tox`` or your +package manager), just run ``tox`` in the directory of your source checkout to +run ``jsonschema``'s test suite on all of the versions of Python ``jsonschema`` +supports. Note that you'll need to have all of those versions installed in +order to run the tests on each of them, otherwise ``tox`` will skip (and fail) +the tests on that version. + +Of course you're also free to just run the tests on a single version with your +favorite test runner. The tests live in the ``jsonschema.tests`` package. + + +Community +--------- + +There's a `mailing list `_ +for this implementation on Google Groups. + +Please join, and feel free to send questions there. + + +Contributing +------------ + +I'm Julian Berman. + +``jsonschema`` is on `GitHub `_. + +Get in touch, via GitHub or otherwise, if you've got something to contribute, +it'd be most welcome! + +You can also generally find me on Freenode (nick: ``tos9``) in various +channels, including ``#python``. + +If you feel overwhelmingly grateful, you can woo me with beer money on +`Gittip `_ or via Google Wallet with the email +in my GitHub profile. diff --git a/lib/spack/external/jsonschema/__init__.py b/lib/spack/external/jsonschema/__init__.py new file mode 100644 index 0000000000..6c099f1d8b --- /dev/null +++ b/lib/spack/external/jsonschema/__init__.py @@ -0,0 +1,26 @@ +""" +An implementation of JSON Schema for Python + +The main functionality is provided by the validator classes for each of the +supported JSON Schema versions. + +Most commonly, :func:`validate` is the quickest way to simply validate a given +instance under a schema, and will create a validator for you. + +""" + +from jsonschema.exceptions import ( + ErrorTree, FormatError, RefResolutionError, SchemaError, ValidationError +) +from jsonschema._format import ( + FormatChecker, draft3_format_checker, draft4_format_checker, +) +from jsonschema.validators import ( + Draft3Validator, Draft4Validator, RefResolver, validate +) + + +__version__ = "2.4.0" + + +# flake8: noqa diff --git a/lib/spack/external/jsonschema/__main__.py b/lib/spack/external/jsonschema/__main__.py new file mode 100644 index 0000000000..82c29fd39e --- /dev/null +++ b/lib/spack/external/jsonschema/__main__.py @@ -0,0 +1,2 @@ +from jsonschema.cli import main +main() diff --git a/lib/spack/external/jsonschema/_format.py b/lib/spack/external/jsonschema/_format.py new file mode 100644 index 0000000000..bb52d183ad --- /dev/null +++ b/lib/spack/external/jsonschema/_format.py @@ -0,0 +1,240 @@ +import datetime +import re +import socket + +from jsonschema.compat import str_types +from jsonschema.exceptions import FormatError + + +class FormatChecker(object): + """ + A ``format`` property checker. + + JSON Schema does not mandate that the ``format`` property actually do any + validation. If validation is desired however, instances of this class can + be hooked into validators to enable format validation. + + :class:`FormatChecker` objects always return ``True`` when asked about + formats that they do not know how to validate. + + To check a custom format using a function that takes an instance and + returns a ``bool``, use the :meth:`FormatChecker.checks` or + :meth:`FormatChecker.cls_checks` decorators. + + :argument iterable formats: the known formats to validate. This argument + can be used to limit which formats will be used + during validation. + + """ + + checkers = {} + + def __init__(self, formats=None): + if formats is None: + self.checkers = self.checkers.copy() + else: + self.checkers = dict((k, self.checkers[k]) for k in formats) + + def checks(self, format, raises=()): + """ + Register a decorated function as validating a new format. + + :argument str format: the format that the decorated function will check + :argument Exception raises: the exception(s) raised by the decorated + function when an invalid instance is found. The exception object + will be accessible as the :attr:`ValidationError.cause` attribute + of the resulting validation error. + + """ + + def _checks(func): + self.checkers[format] = (func, raises) + return func + return _checks + + cls_checks = classmethod(checks) + + def check(self, instance, format): + """ + Check whether the instance conforms to the given format. + + :argument instance: the instance to check + :type: any primitive type (str, number, bool) + :argument str format: the format that instance should conform to + :raises: :exc:`FormatError` if instance does not conform to format + + """ + + if format not in self.checkers: + return + + func, raises = self.checkers[format] + result, cause = None, None + try: + result = func(instance) + except raises as e: + cause = e + if not result: + raise FormatError( + "%r is not a %r" % (instance, format), cause=cause, + ) + + def conforms(self, instance, format): + """ + Check whether the instance conforms to the given format. + + :argument instance: the instance to check + :type: any primitive type (str, number, bool) + :argument str format: the format that instance should conform to + :rtype: bool + + """ + + try: + self.check(instance, format) + except FormatError: + return False + else: + return True + + +_draft_checkers = {"draft3": [], "draft4": []} + + +def _checks_drafts(both=None, draft3=None, draft4=None, raises=()): + draft3 = draft3 or both + draft4 = draft4 or both + + def wrap(func): + if draft3: + _draft_checkers["draft3"].append(draft3) + func = FormatChecker.cls_checks(draft3, raises)(func) + if draft4: + _draft_checkers["draft4"].append(draft4) + func = FormatChecker.cls_checks(draft4, raises)(func) + return func + return wrap + + +@_checks_drafts("email") +def is_email(instance): + if not isinstance(instance, str_types): + return True + return "@" in instance + + +_ipv4_re = re.compile(r"^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$") + +@_checks_drafts(draft3="ip-address", draft4="ipv4") +def is_ipv4(instance): + if not isinstance(instance, str_types): + return True + if not _ipv4_re.match(instance): + return False + return all(0 <= int(component) <= 255 for component in instance.split(".")) + + +if hasattr(socket, "inet_pton"): + @_checks_drafts("ipv6", raises=socket.error) + def is_ipv6(instance): + if not isinstance(instance, str_types): + return True + return socket.inet_pton(socket.AF_INET6, instance) + + +_host_name_re = re.compile(r"^[A-Za-z0-9][A-Za-z0-9\.\-]{1,255}$") + +@_checks_drafts(draft3="host-name", draft4="hostname") +def is_host_name(instance): + if not isinstance(instance, str_types): + return True + if not _host_name_re.match(instance): + return False + components = instance.split(".") + for component in components: + if len(component) > 63: + return False + return True + + +try: + import rfc3987 +except ImportError: + pass +else: + @_checks_drafts("uri", raises=ValueError) + def is_uri(instance): + if not isinstance(instance, str_types): + return True + return rfc3987.parse(instance, rule="URI") + + +try: + import strict_rfc3339 +except ImportError: + try: + import isodate + except ImportError: + pass + else: + @_checks_drafts("date-time", raises=(ValueError, isodate.ISO8601Error)) + def is_date(instance): + if not isinstance(instance, str_types): + return True + return isodate.parse_datetime(instance) +else: + @_checks_drafts("date-time") + def is_date(instance): + if not isinstance(instance, str_types): + return True + return strict_rfc3339.validate_rfc3339(instance) + + +@_checks_drafts("regex", raises=re.error) +def is_regex(instance): + if not isinstance(instance, str_types): + return True + return re.compile(instance) + + +@_checks_drafts(draft3="date", raises=ValueError) +def is_date(instance): + if not isinstance(instance, str_types): + return True + return datetime.datetime.strptime(instance, "%Y-%m-%d") + + +@_checks_drafts(draft3="time", raises=ValueError) +def is_time(instance): + if not isinstance(instance, str_types): + return True + return datetime.datetime.strptime(instance, "%H:%M:%S") + + +try: + import webcolors +except ImportError: + pass +else: + def is_css_color_code(instance): + return webcolors.normalize_hex(instance) + + + @_checks_drafts(draft3="color", raises=(ValueError, TypeError)) + def is_css21_color(instance): + if ( + not isinstance(instance, str_types) or + instance.lower() in webcolors.css21_names_to_hex + ): + return True + return is_css_color_code(instance) + + + def is_css3_color(instance): + if instance.lower() in webcolors.css3_names_to_hex: + return True + return is_css_color_code(instance) + + +draft3_format_checker = FormatChecker(_draft_checkers["draft3"]) +draft4_format_checker = FormatChecker(_draft_checkers["draft4"]) diff --git a/lib/spack/external/jsonschema/_reflect.py b/lib/spack/external/jsonschema/_reflect.py new file mode 100644 index 0000000000..d09e38fbdc --- /dev/null +++ b/lib/spack/external/jsonschema/_reflect.py @@ -0,0 +1,155 @@ +# -*- test-case-name: twisted.test.test_reflect -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Standardized versions of various cool and/or strange things that you can do +with Python's reflection capabilities. +""" + +import sys + +from jsonschema.compat import PY3 + + +class _NoModuleFound(Exception): + """ + No module was found because none exists. + """ + + + +class InvalidName(ValueError): + """ + The given name is not a dot-separated list of Python objects. + """ + + + +class ModuleNotFound(InvalidName): + """ + The module associated with the given name doesn't exist and it can't be + imported. + """ + + + +class ObjectNotFound(InvalidName): + """ + The object associated with the given name doesn't exist and it can't be + imported. + """ + + + +if PY3: + def reraise(exception, traceback): + raise exception.with_traceback(traceback) +else: + exec("""def reraise(exception, traceback): + raise exception.__class__, exception, traceback""") + +reraise.__doc__ = """ +Re-raise an exception, with an optional traceback, in a way that is compatible +with both Python 2 and Python 3. + +Note that on Python 3, re-raised exceptions will be mutated, with their +C{__traceback__} attribute being set. + +@param exception: The exception instance. +@param traceback: The traceback to use, or C{None} indicating a new traceback. +""" + + +def _importAndCheckStack(importName): + """ + Import the given name as a module, then walk the stack to determine whether + the failure was the module not existing, or some code in the module (for + example a dependent import) failing. This can be helpful to determine + whether any actual application code was run. For example, to distiguish + administrative error (entering the wrong module name), from programmer + error (writing buggy code in a module that fails to import). + + @param importName: The name of the module to import. + @type importName: C{str} + @raise Exception: if something bad happens. This can be any type of + exception, since nobody knows what loading some arbitrary code might + do. + @raise _NoModuleFound: if no module was found. + """ + try: + return __import__(importName) + except ImportError: + excType, excValue, excTraceback = sys.exc_info() + while excTraceback: + execName = excTraceback.tb_frame.f_globals["__name__"] + # in Python 2 execName is None when an ImportError is encountered, + # where in Python 3 execName is equal to the importName. + if execName is None or execName == importName: + reraise(excValue, excTraceback) + excTraceback = excTraceback.tb_next + raise _NoModuleFound() + + + +def namedAny(name): + """ + Retrieve a Python object by its fully qualified name from the global Python + module namespace. The first part of the name, that describes a module, + will be discovered and imported. Each subsequent part of the name is + treated as the name of an attribute of the object specified by all of the + name which came before it. For example, the fully-qualified name of this + object is 'twisted.python.reflect.namedAny'. + + @type name: L{str} + @param name: The name of the object to return. + + @raise InvalidName: If the name is an empty string, starts or ends with + a '.', or is otherwise syntactically incorrect. + + @raise ModuleNotFound: If the name is syntactically correct but the + module it specifies cannot be imported because it does not appear to + exist. + + @raise ObjectNotFound: If the name is syntactically correct, includes at + least one '.', but the module it specifies cannot be imported because + it does not appear to exist. + + @raise AttributeError: If an attribute of an object along the way cannot be + accessed, or a module along the way is not found. + + @return: the Python object identified by 'name'. + """ + if not name: + raise InvalidName('Empty module name') + + names = name.split('.') + + # if the name starts or ends with a '.' or contains '..', the __import__ + # will raise an 'Empty module name' error. This will provide a better error + # message. + if '' in names: + raise InvalidName( + "name must be a string giving a '.'-separated list of Python " + "identifiers, not %r" % (name,)) + + topLevelPackage = None + moduleNames = names[:] + while not topLevelPackage: + if moduleNames: + trialname = '.'.join(moduleNames) + try: + topLevelPackage = _importAndCheckStack(trialname) + except _NoModuleFound: + moduleNames.pop() + else: + if len(names) == 1: + raise ModuleNotFound("No module named %r" % (name,)) + else: + raise ObjectNotFound('%r does not name an object' % (name,)) + + obj = topLevelPackage + for n in names[1:]: + obj = getattr(obj, n) + + return obj diff --git a/lib/spack/external/jsonschema/_utils.py b/lib/spack/external/jsonschema/_utils.py new file mode 100644 index 0000000000..2262f3305d --- /dev/null +++ b/lib/spack/external/jsonschema/_utils.py @@ -0,0 +1,213 @@ +import itertools +import json +import pkgutil +import re + +from jsonschema.compat import str_types, MutableMapping, urlsplit + + +class URIDict(MutableMapping): + """ + Dictionary which uses normalized URIs as keys. + + """ + + def normalize(self, uri): + return urlsplit(uri).geturl() + + def __init__(self, *args, **kwargs): + self.store = dict() + self.store.update(*args, **kwargs) + + def __getitem__(self, uri): + return self.store[self.normalize(uri)] + + def __setitem__(self, uri, value): + self.store[self.normalize(uri)] = value + + def __delitem__(self, uri): + del self.store[self.normalize(uri)] + + def __iter__(self): + return iter(self.store) + + def __len__(self): + return len(self.store) + + def __repr__(self): + return repr(self.store) + + +class Unset(object): + """ + An as-of-yet unset attribute or unprovided default parameter. + + """ + + def __repr__(self): + return "" + + +def load_schema(name): + """ + Load a schema from ./schemas/``name``.json and return it. + + """ + + data = pkgutil.get_data(__package__, "schemas/{0}.json".format(name)) + return json.loads(data.decode("utf-8")) + + +def indent(string, times=1): + """ + A dumb version of :func:`textwrap.indent` from Python 3.3. + + """ + + return "\n".join(" " * (4 * times) + line for line in string.splitlines()) + + +def format_as_index(indices): + """ + Construct a single string containing indexing operations for the indices. + + For example, [1, 2, "foo"] -> [1][2]["foo"] + + :type indices: sequence + + """ + + if not indices: + return "" + return "[%s]" % "][".join(repr(index) for index in indices) + + +def find_additional_properties(instance, schema): + """ + Return the set of additional properties for the given ``instance``. + + Weeds out properties that should have been validated by ``properties`` and + / or ``patternProperties``. + + Assumes ``instance`` is dict-like already. + + """ + + properties = schema.get("properties", {}) + patterns = "|".join(schema.get("patternProperties", {})) + for property in instance: + if property not in properties: + if patterns and re.search(patterns, property): + continue + yield property + + +def extras_msg(extras): + """ + Create an error message for extra items or properties. + + """ + + if len(extras) == 1: + verb = "was" + else: + verb = "were" + return ", ".join(repr(extra) for extra in extras), verb + + +def types_msg(instance, types): + """ + Create an error message for a failure to match the given types. + + If the ``instance`` is an object and contains a ``name`` property, it will + be considered to be a description of that object and used as its type. + + Otherwise the message is simply the reprs of the given ``types``. + + """ + + reprs = [] + for type in types: + try: + reprs.append(repr(type["name"])) + except Exception: + reprs.append(repr(type)) + return "%r is not of type %s" % (instance, ", ".join(reprs)) + + +def flatten(suitable_for_isinstance): + """ + isinstance() can accept a bunch of really annoying different types: + * a single type + * a tuple of types + * an arbitrary nested tree of tuples + + Return a flattened tuple of the given argument. + + """ + + types = set() + + if not isinstance(suitable_for_isinstance, tuple): + suitable_for_isinstance = (suitable_for_isinstance,) + for thing in suitable_for_isinstance: + if isinstance(thing, tuple): + types.update(flatten(thing)) + else: + types.add(thing) + return tuple(types) + + +def ensure_list(thing): + """ + Wrap ``thing`` in a list if it's a single str. + + Otherwise, return it unchanged. + + """ + + if isinstance(thing, str_types): + return [thing] + return thing + + +def unbool(element, true=object(), false=object()): + """ + A hack to make True and 1 and False and 0 unique for ``uniq``. + + """ + + if element is True: + return true + elif element is False: + return false + return element + + +def uniq(container): + """ + Check if all of a container's elements are unique. + + Successively tries first to rely that the elements are hashable, then + falls back on them being sortable, and finally falls back on brute + force. + + """ + + try: + return len(set(unbool(i) for i in container)) == len(container) + except TypeError: + try: + sort = sorted(unbool(i) for i in container) + sliced = itertools.islice(sort, 1, None) + for i, j in zip(sort, sliced): + if i == j: + return False + except (NotImplementedError, TypeError): + seen = [] + for e in container: + e = unbool(e) + if e in seen: + return False + seen.append(e) + return True diff --git a/lib/spack/external/jsonschema/_validators.py b/lib/spack/external/jsonschema/_validators.py new file mode 100644 index 0000000000..c6e801ccb2 --- /dev/null +++ b/lib/spack/external/jsonschema/_validators.py @@ -0,0 +1,358 @@ +import re + +from jsonschema import _utils +from jsonschema.exceptions import FormatError, ValidationError +from jsonschema.compat import iteritems + + +FLOAT_TOLERANCE = 10 ** -15 + + +def patternProperties(validator, patternProperties, instance, schema): + if not validator.is_type(instance, "object"): + return + + for pattern, subschema in iteritems(patternProperties): + for k, v in iteritems(instance): + if re.search(pattern, k): + for error in validator.descend( + v, subschema, path=k, schema_path=pattern, + ): + yield error + + +def additionalProperties(validator, aP, instance, schema): + if not validator.is_type(instance, "object"): + return + + extras = set(_utils.find_additional_properties(instance, schema)) + + if validator.is_type(aP, "object"): + for extra in extras: + for error in validator.descend(instance[extra], aP, path=extra): + yield error + elif not aP and extras: + error = "Additional properties are not allowed (%s %s unexpected)" + yield ValidationError(error % _utils.extras_msg(extras)) + + +def items(validator, items, instance, schema): + if not validator.is_type(instance, "array"): + return + + if validator.is_type(items, "object"): + for index, item in enumerate(instance): + for error in validator.descend(item, items, path=index): + yield error + else: + for (index, item), subschema in zip(enumerate(instance), items): + for error in validator.descend( + item, subschema, path=index, schema_path=index, + ): + yield error + + +def additionalItems(validator, aI, instance, schema): + if ( + not validator.is_type(instance, "array") or + validator.is_type(schema.get("items", {}), "object") + ): + return + + len_items = len(schema.get("items", [])) + if validator.is_type(aI, "object"): + for index, item in enumerate(instance[len_items:], start=len_items): + for error in validator.descend(item, aI, path=index): + yield error + elif not aI and len(instance) > len(schema.get("items", [])): + error = "Additional items are not allowed (%s %s unexpected)" + yield ValidationError( + error % + _utils.extras_msg(instance[len(schema.get("items", [])):]) + ) + + +def minimum(validator, minimum, instance, schema): + if not validator.is_type(instance, "number"): + return + + if schema.get("exclusiveMinimum", False): + failed = float(instance) <= minimum + cmp = "less than or equal to" + else: + failed = float(instance) < minimum + cmp = "less than" + + if failed: + yield ValidationError( + "%r is %s the minimum of %r" % (instance, cmp, minimum) + ) + + +def maximum(validator, maximum, instance, schema): + if not validator.is_type(instance, "number"): + return + + if schema.get("exclusiveMaximum", False): + failed = instance >= maximum + cmp = "greater than or equal to" + else: + failed = instance > maximum + cmp = "greater than" + + if failed: + yield ValidationError( + "%r is %s the maximum of %r" % (instance, cmp, maximum) + ) + + +def multipleOf(validator, dB, instance, schema): + if not validator.is_type(instance, "number"): + return + + if isinstance(dB, float): + mod = instance % dB + failed = (mod > FLOAT_TOLERANCE) and (dB - mod) > FLOAT_TOLERANCE + else: + failed = instance % dB + + if failed: + yield ValidationError("%r is not a multiple of %r" % (instance, dB)) + + +def minItems(validator, mI, instance, schema): + if validator.is_type(instance, "array") and len(instance) < mI: + yield ValidationError("%r is too short" % (instance,)) + + +def maxItems(validator, mI, instance, schema): + if validator.is_type(instance, "array") and len(instance) > mI: + yield ValidationError("%r is too long" % (instance,)) + + +def uniqueItems(validator, uI, instance, schema): + if ( + uI and + validator.is_type(instance, "array") and + not _utils.uniq(instance) + ): + yield ValidationError("%r has non-unique elements" % instance) + + +def pattern(validator, patrn, instance, schema): + if ( + validator.is_type(instance, "string") and + not re.search(patrn, instance) + ): + yield ValidationError("%r does not match %r" % (instance, patrn)) + + +def format(validator, format, instance, schema): + if validator.format_checker is not None: + try: + validator.format_checker.check(instance, format) + except FormatError as error: + yield ValidationError(error.message, cause=error.cause) + + +def minLength(validator, mL, instance, schema): + if validator.is_type(instance, "string") and len(instance) < mL: + yield ValidationError("%r is too short" % (instance,)) + + +def maxLength(validator, mL, instance, schema): + if validator.is_type(instance, "string") and len(instance) > mL: + yield ValidationError("%r is too long" % (instance,)) + + +def dependencies(validator, dependencies, instance, schema): + if not validator.is_type(instance, "object"): + return + + for property, dependency in iteritems(dependencies): + if property not in instance: + continue + + if validator.is_type(dependency, "object"): + for error in validator.descend( + instance, dependency, schema_path=property, + ): + yield error + else: + dependencies = _utils.ensure_list(dependency) + for dependency in dependencies: + if dependency not in instance: + yield ValidationError( + "%r is a dependency of %r" % (dependency, property) + ) + + +def enum(validator, enums, instance, schema): + if instance not in enums: + yield ValidationError("%r is not one of %r" % (instance, enums)) + + +def ref(validator, ref, instance, schema): + with validator.resolver.resolving(ref) as resolved: + for error in validator.descend(instance, resolved): + yield error + + +def type_draft3(validator, types, instance, schema): + types = _utils.ensure_list(types) + + all_errors = [] + for index, type in enumerate(types): + if type == "any": + return + if validator.is_type(type, "object"): + errors = list(validator.descend(instance, type, schema_path=index)) + if not errors: + return + all_errors.extend(errors) + else: + if validator.is_type(instance, type): + return + else: + yield ValidationError( + _utils.types_msg(instance, types), context=all_errors, + ) + + +def properties_draft3(validator, properties, instance, schema): + if not validator.is_type(instance, "object"): + return + + for property, subschema in iteritems(properties): + if property in instance: + for error in validator.descend( + instance[property], + subschema, + path=property, + schema_path=property, + ): + yield error + elif subschema.get("required", False): + error = ValidationError("%r is a required property" % property) + error._set( + validator="required", + validator_value=subschema["required"], + instance=instance, + schema=schema, + ) + error.path.appendleft(property) + error.schema_path.extend([property, "required"]) + yield error + + +def disallow_draft3(validator, disallow, instance, schema): + for disallowed in _utils.ensure_list(disallow): + if validator.is_valid(instance, {"type" : [disallowed]}): + yield ValidationError( + "%r is disallowed for %r" % (disallowed, instance) + ) + + +def extends_draft3(validator, extends, instance, schema): + if validator.is_type(extends, "object"): + for error in validator.descend(instance, extends): + yield error + return + for index, subschema in enumerate(extends): + for error in validator.descend(instance, subschema, schema_path=index): + yield error + + +def type_draft4(validator, types, instance, schema): + types = _utils.ensure_list(types) + + if not any(validator.is_type(instance, type) for type in types): + yield ValidationError(_utils.types_msg(instance, types)) + + +def properties_draft4(validator, properties, instance, schema): + if not validator.is_type(instance, "object"): + return + + for property, subschema in iteritems(properties): + if property in instance: + for error in validator.descend( + instance[property], + subschema, + path=property, + schema_path=property, + ): + yield error + + +def required_draft4(validator, required, instance, schema): + if not validator.is_type(instance, "object"): + return + for property in required: + if property not in instance: + yield ValidationError("%r is a required property" % property) + + +def minProperties_draft4(validator, mP, instance, schema): + if validator.is_type(instance, "object") and len(instance) < mP: + yield ValidationError( + "%r does not have enough properties" % (instance,) + ) + + +def maxProperties_draft4(validator, mP, instance, schema): + if not validator.is_type(instance, "object"): + return + if validator.is_type(instance, "object") and len(instance) > mP: + yield ValidationError("%r has too many properties" % (instance,)) + + +def allOf_draft4(validator, allOf, instance, schema): + for index, subschema in enumerate(allOf): + for error in validator.descend(instance, subschema, schema_path=index): + yield error + + +def oneOf_draft4(validator, oneOf, instance, schema): + subschemas = enumerate(oneOf) + all_errors = [] + for index, subschema in subschemas: + errs = list(validator.descend(instance, subschema, schema_path=index)) + if not errs: + first_valid = subschema + break + all_errors.extend(errs) + else: + yield ValidationError( + "%r is not valid under any of the given schemas" % (instance,), + context=all_errors, + ) + + more_valid = [s for i, s in subschemas if validator.is_valid(instance, s)] + if more_valid: + more_valid.append(first_valid) + reprs = ", ".join(repr(schema) for schema in more_valid) + yield ValidationError( + "%r is valid under each of %s" % (instance, reprs) + ) + + +def anyOf_draft4(validator, anyOf, instance, schema): + all_errors = [] + for index, subschema in enumerate(anyOf): + errs = list(validator.descend(instance, subschema, schema_path=index)) + if not errs: + break + all_errors.extend(errs) + else: + yield ValidationError( + "%r is not valid under any of the given schemas" % (instance,), + context=all_errors, + ) + + +def not_draft4(validator, not_schema, instance, schema): + if validator.is_valid(instance, not_schema): + yield ValidationError( + "%r is not allowed for %r" % (not_schema, instance) + ) diff --git a/lib/spack/external/jsonschema/cli.py b/lib/spack/external/jsonschema/cli.py new file mode 100644 index 0000000000..0126564f46 --- /dev/null +++ b/lib/spack/external/jsonschema/cli.py @@ -0,0 +1,72 @@ +from __future__ import absolute_import +import argparse +import json +import sys + +from jsonschema._reflect import namedAny +from jsonschema.validators import validator_for + + +def _namedAnyWithDefault(name): + if "." not in name: + name = "jsonschema." + name + return namedAny(name) + + +def _json_file(path): + with open(path) as file: + return json.load(file) + + +parser = argparse.ArgumentParser( + description="JSON Schema Validation CLI", +) +parser.add_argument( + "-i", "--instance", + action="append", + dest="instances", + type=_json_file, + help="a path to a JSON instance to validate " + "(may be specified multiple times)", +) +parser.add_argument( + "-F", "--error-format", + default="{error.instance}: {error.message}\n", + help="the format to use for each error output message, specified in " + "a form suitable for passing to str.format, which will be called " + "with 'error' for each error", +) +parser.add_argument( + "-V", "--validator", + type=_namedAnyWithDefault, + help="the fully qualified object name of a validator to use, or, for " + "validators that are registered with jsonschema, simply the name " + "of the class.", +) +parser.add_argument( + "schema", + help="the JSON Schema to validate with", + type=_json_file, +) + + +def parse_args(args): + arguments = vars(parser.parse_args(args=args or ["--help"])) + if arguments["validator"] is None: + arguments["validator"] = validator_for(arguments["schema"]) + return arguments + + +def main(args=sys.argv[1:]): + sys.exit(run(arguments=parse_args(args=args))) + + +def run(arguments, stdout=sys.stdout, stderr=sys.stderr): + error_format = arguments["error_format"] + validator = arguments["validator"](schema=arguments["schema"]) + errored = False + for instance in arguments["instances"] or (): + for error in validator.iter_errors(instance): + stderr.write(error_format.format(error=error)) + errored = True + return errored diff --git a/lib/spack/external/jsonschema/compat.py b/lib/spack/external/jsonschema/compat.py new file mode 100644 index 0000000000..6ca49ab6be --- /dev/null +++ b/lib/spack/external/jsonschema/compat.py @@ -0,0 +1,53 @@ +from __future__ import unicode_literals +import sys +import operator + +try: + from collections import MutableMapping, Sequence # noqa +except ImportError: + from collections.abc import MutableMapping, Sequence # noqa + +PY3 = sys.version_info[0] >= 3 + +if PY3: + zip = zip + from io import StringIO + from urllib.parse import ( + unquote, urljoin, urlunsplit, SplitResult, urlsplit as _urlsplit + ) + from urllib.request import urlopen + str_types = str, + int_types = int, + iteritems = operator.methodcaller("items") +else: + from itertools import izip as zip # noqa + from StringIO import StringIO + from urlparse import ( + urljoin, urlunsplit, SplitResult, urlsplit as _urlsplit # noqa + ) + from urllib import unquote # noqa + from urllib2 import urlopen # noqa + str_types = basestring + int_types = int, long + iteritems = operator.methodcaller("iteritems") + + +# On python < 3.3 fragments are not handled properly with unknown schemes +def urlsplit(url): + scheme, netloc, path, query, fragment = _urlsplit(url) + if "#" in path: + path, fragment = path.split("#", 1) + return SplitResult(scheme, netloc, path, query, fragment) + + +def urldefrag(url): + if "#" in url: + s, n, p, q, frag = urlsplit(url) + defrag = urlunsplit((s, n, p, q, '')) + else: + defrag = url + frag = '' + return defrag, frag + + +# flake8: noqa diff --git a/lib/spack/external/jsonschema/exceptions.py b/lib/spack/external/jsonschema/exceptions.py new file mode 100644 index 0000000000..478e59c531 --- /dev/null +++ b/lib/spack/external/jsonschema/exceptions.py @@ -0,0 +1,264 @@ +from collections import defaultdict, deque +import itertools +import pprint +import textwrap + +from jsonschema import _utils +from jsonschema.compat import PY3, iteritems + + +WEAK_MATCHES = frozenset(["anyOf", "oneOf"]) +STRONG_MATCHES = frozenset() + +_unset = _utils.Unset() + + +class _Error(Exception): + def __init__( + self, + message, + validator=_unset, + path=(), + cause=None, + context=(), + validator_value=_unset, + instance=_unset, + schema=_unset, + schema_path=(), + parent=None, + ): + self.message = message + self.path = self.relative_path = deque(path) + self.schema_path = self.relative_schema_path = deque(schema_path) + self.context = list(context) + self.cause = self.__cause__ = cause + self.validator = validator + self.validator_value = validator_value + self.instance = instance + self.schema = schema + self.parent = parent + + for error in context: + error.parent = self + + def __repr__(self): + return "<%s: %r>" % (self.__class__.__name__, self.message) + + def __str__(self): + return unicode(self).encode("utf-8") + + def __unicode__(self): + essential_for_verbose = ( + self.validator, self.validator_value, self.instance, self.schema, + ) + if any(m is _unset for m in essential_for_verbose): + return self.message + + pschema = pprint.pformat(self.schema, width=72) + pinstance = pprint.pformat(self.instance, width=72) + return self.message + textwrap.dedent(""" + + Failed validating %r in schema%s: + %s + + On instance%s: + %s + """.rstrip() + ) % ( + self.validator, + _utils.format_as_index(list(self.relative_schema_path)[:-1]), + _utils.indent(pschema), + _utils.format_as_index(self.relative_path), + _utils.indent(pinstance), + ) + + if PY3: + __str__ = __unicode__ + + @classmethod + def create_from(cls, other): + return cls(**other._contents()) + + @property + def absolute_path(self): + parent = self.parent + if parent is None: + return self.relative_path + + path = deque(self.relative_path) + path.extendleft(parent.absolute_path) + return path + + @property + def absolute_schema_path(self): + parent = self.parent + if parent is None: + return self.relative_schema_path + + path = deque(self.relative_schema_path) + path.extendleft(parent.absolute_schema_path) + return path + + def _set(self, **kwargs): + for k, v in iteritems(kwargs): + if getattr(self, k) is _unset: + setattr(self, k, v) + + def _contents(self): + attrs = ( + "message", "cause", "context", "validator", "validator_value", + "path", "schema_path", "instance", "schema", "parent", + ) + return dict((attr, getattr(self, attr)) for attr in attrs) + + +class ValidationError(_Error): + pass + + +class SchemaError(_Error): + pass + + +class RefResolutionError(Exception): + pass + + +class UnknownType(Exception): + def __init__(self, type, instance, schema): + self.type = type + self.instance = instance + self.schema = schema + + def __str__(self): + return unicode(self).encode("utf-8") + + def __unicode__(self): + pschema = pprint.pformat(self.schema, width=72) + pinstance = pprint.pformat(self.instance, width=72) + return textwrap.dedent(""" + Unknown type %r for validator with schema: + %s + + While checking instance: + %s + """.rstrip() + ) % (self.type, _utils.indent(pschema), _utils.indent(pinstance)) + + if PY3: + __str__ = __unicode__ + + + +class FormatError(Exception): + def __init__(self, message, cause=None): + super(FormatError, self).__init__(message, cause) + self.message = message + self.cause = self.__cause__ = cause + + def __str__(self): + return self.message.encode("utf-8") + + def __unicode__(self): + return self.message + + if PY3: + __str__ = __unicode__ + + +class ErrorTree(object): + """ + ErrorTrees make it easier to check which validations failed. + + """ + + _instance = _unset + + def __init__(self, errors=()): + self.errors = {} + self._contents = defaultdict(self.__class__) + + for error in errors: + container = self + for element in error.path: + container = container[element] + container.errors[error.validator] = error + + self._instance = error.instance + + def __contains__(self, index): + """ + Check whether ``instance[index]`` has any errors. + + """ + + return index in self._contents + + def __getitem__(self, index): + """ + Retrieve the child tree one level down at the given ``index``. + + If the index is not in the instance that this tree corresponds to and + is not known by this tree, whatever error would be raised by + ``instance.__getitem__`` will be propagated (usually this is some + subclass of :class:`LookupError`. + + """ + + if self._instance is not _unset and index not in self: + self._instance[index] + return self._contents[index] + + def __setitem__(self, index, value): + self._contents[index] = value + + def __iter__(self): + """ + Iterate (non-recursively) over the indices in the instance with errors. + + """ + + return iter(self._contents) + + def __len__(self): + """ + Same as :attr:`total_errors`. + + """ + + return self.total_errors + + def __repr__(self): + return "<%s (%s total errors)>" % (self.__class__.__name__, len(self)) + + @property + def total_errors(self): + """ + The total number of errors in the entire tree, including children. + + """ + + child_errors = sum(len(tree) for _, tree in iteritems(self._contents)) + return len(self.errors) + child_errors + + +def by_relevance(weak=WEAK_MATCHES, strong=STRONG_MATCHES): + def relevance(error): + validator = error.validator + return -len(error.path), validator not in weak, validator in strong + return relevance + + +relevance = by_relevance() + + +def best_match(errors, key=relevance): + errors = iter(errors) + best = next(errors, None) + if best is None: + return + best = max(itertools.chain([best], errors), key=key) + + while best.context: + best = min(best.context, key=key) + return best diff --git a/lib/spack/external/jsonschema/schemas/draft3.json b/lib/spack/external/jsonschema/schemas/draft3.json new file mode 100644 index 0000000000..5bcefe30d5 --- /dev/null +++ b/lib/spack/external/jsonschema/schemas/draft3.json @@ -0,0 +1,201 @@ +{ + "$schema": "http://json-schema.org/draft-03/schema#", + "dependencies": { + "exclusiveMaximum": "maximum", + "exclusiveMinimum": "minimum" + }, + "id": "http://json-schema.org/draft-03/schema#", + "properties": { + "$ref": { + "format": "uri", + "type": "string" + }, + "$schema": { + "format": "uri", + "type": "string" + }, + "additionalItems": { + "default": {}, + "type": [ + { + "$ref": "#" + }, + "boolean" + ] + }, + "additionalProperties": { + "default": {}, + "type": [ + { + "$ref": "#" + }, + "boolean" + ] + }, + "default": { + "type": "any" + }, + "dependencies": { + "additionalProperties": { + "items": { + "type": "string" + }, + "type": [ + "string", + "array", + { + "$ref": "#" + } + ] + }, + "default": {}, + "type": [ + "string", + "array", + "object" + ] + }, + "description": { + "type": "string" + }, + "disallow": { + "items": { + "type": [ + "string", + { + "$ref": "#" + } + ] + }, + "type": [ + "string", + "array" + ], + "uniqueItems": true + }, + "divisibleBy": { + "default": 1, + "exclusiveMinimum": true, + "minimum": 0, + "type": "number" + }, + "enum": { + "minItems": 1, + "type": "array", + "uniqueItems": true + }, + "exclusiveMaximum": { + "default": false, + "type": "boolean" + }, + "exclusiveMinimum": { + "default": false, + "type": "boolean" + }, + "extends": { + "default": {}, + "items": { + "$ref": "#" + }, + "type": [ + { + "$ref": "#" + }, + "array" + ] + }, + "format": { + "type": "string" + }, + "id": { + "format": "uri", + "type": "string" + }, + "items": { + "default": {}, + "items": { + "$ref": "#" + }, + "type": [ + { + "$ref": "#" + }, + "array" + ] + }, + "maxDecimal": { + "minimum": 0, + "type": "number" + }, + "maxItems": { + "minimum": 0, + "type": "integer" + }, + "maxLength": { + "type": "integer" + }, + "maximum": { + "type": "number" + }, + "minItems": { + "default": 0, + "minimum": 0, + "type": "integer" + }, + "minLength": { + "default": 0, + "minimum": 0, + "type": "integer" + }, + "minimum": { + "type": "number" + }, + "pattern": { + "format": "regex", + "type": "string" + }, + "patternProperties": { + "additionalProperties": { + "$ref": "#" + }, + "default": {}, + "type": "object" + }, + "properties": { + "additionalProperties": { + "$ref": "#", + "type": "object" + }, + "default": {}, + "type": "object" + }, + "required": { + "default": false, + "type": "boolean" + }, + "title": { + "type": "string" + }, + "type": { + "default": "any", + "items": { + "type": [ + "string", + { + "$ref": "#" + } + ] + }, + "type": [ + "string", + "array" + ], + "uniqueItems": true + }, + "uniqueItems": { + "default": false, + "type": "boolean" + } + }, + "type": "object" +} diff --git a/lib/spack/external/jsonschema/schemas/draft4.json b/lib/spack/external/jsonschema/schemas/draft4.json new file mode 100644 index 0000000000..fead5cefab --- /dev/null +++ b/lib/spack/external/jsonschema/schemas/draft4.json @@ -0,0 +1,221 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "default": {}, + "definitions": { + "positiveInteger": { + "minimum": 0, + "type": "integer" + }, + "positiveIntegerDefault0": { + "allOf": [ + { + "$ref": "#/definitions/positiveInteger" + }, + { + "default": 0 + } + ] + }, + "schemaArray": { + "items": { + "$ref": "#" + }, + "minItems": 1, + "type": "array" + }, + "simpleTypes": { + "enum": [ + "array", + "boolean", + "integer", + "null", + "number", + "object", + "string" + ] + }, + "stringArray": { + "items": { + "type": "string" + }, + "minItems": 1, + "type": "array", + "uniqueItems": true + } + }, + "dependencies": { + "exclusiveMaximum": [ + "maximum" + ], + "exclusiveMinimum": [ + "minimum" + ] + }, + "description": "Core schema meta-schema", + "id": "http://json-schema.org/draft-04/schema#", + "properties": { + "$schema": { + "format": "uri", + "type": "string" + }, + "additionalItems": { + "anyOf": [ + { + "type": "boolean" + }, + { + "$ref": "#" + } + ], + "default": {} + }, + "additionalProperties": { + "anyOf": [ + { + "type": "boolean" + }, + { + "$ref": "#" + } + ], + "default": {} + }, + "allOf": { + "$ref": "#/definitions/schemaArray" + }, + "anyOf": { + "$ref": "#/definitions/schemaArray" + }, + "default": {}, + "definitions": { + "additionalProperties": { + "$ref": "#" + }, + "default": {}, + "type": "object" + }, + "dependencies": { + "additionalProperties": { + "anyOf": [ + { + "$ref": "#" + }, + { + "$ref": "#/definitions/stringArray" + } + ] + }, + "type": "object" + }, + "description": { + "type": "string" + }, + "enum": { + "minItems": 1, + "type": "array", + "uniqueItems": true + }, + "exclusiveMaximum": { + "default": false, + "type": "boolean" + }, + "exclusiveMinimum": { + "default": false, + "type": "boolean" + }, + "id": { + "format": "uri", + "type": "string" + }, + "items": { + "anyOf": [ + { + "$ref": "#" + }, + { + "$ref": "#/definitions/schemaArray" + } + ], + "default": {} + }, + "maxItems": { + "$ref": "#/definitions/positiveInteger" + }, + "maxLength": { + "$ref": "#/definitions/positiveInteger" + }, + "maxProperties": { + "$ref": "#/definitions/positiveInteger" + }, + "maximum": { + "type": "number" + }, + "minItems": { + "$ref": "#/definitions/positiveIntegerDefault0" + }, + "minLength": { + "$ref": "#/definitions/positiveIntegerDefault0" + }, + "minProperties": { + "$ref": "#/definitions/positiveIntegerDefault0" + }, + "minimum": { + "type": "number" + }, + "multipleOf": { + "exclusiveMinimum": true, + "minimum": 0, + "type": "number" + }, + "not": { + "$ref": "#" + }, + "oneOf": { + "$ref": "#/definitions/schemaArray" + }, + "pattern": { + "format": "regex", + "type": "string" + }, + "patternProperties": { + "additionalProperties": { + "$ref": "#" + }, + "default": {}, + "type": "object" + }, + "properties": { + "additionalProperties": { + "$ref": "#" + }, + "default": {}, + "type": "object" + }, + "required": { + "$ref": "#/definitions/stringArray" + }, + "title": { + "type": "string" + }, + "type": { + "anyOf": [ + { + "$ref": "#/definitions/simpleTypes" + }, + { + "items": { + "$ref": "#/definitions/simpleTypes" + }, + "minItems": 1, + "type": "array", + "uniqueItems": true + } + ] + }, + "uniqueItems": { + "default": false, + "type": "boolean" + } + }, + "type": "object" +} diff --git a/lib/spack/external/jsonschema/tests/__init__.py b/lib/spack/external/jsonschema/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/lib/spack/external/jsonschema/tests/compat.py b/lib/spack/external/jsonschema/tests/compat.py new file mode 100644 index 0000000000..b37483f5dd --- /dev/null +++ b/lib/spack/external/jsonschema/tests/compat.py @@ -0,0 +1,15 @@ +import sys + + +if sys.version_info[:2] < (2, 7): # pragma: no cover + import unittest2 as unittest +else: + import unittest + +try: + from unittest import mock +except ImportError: + import mock + + +# flake8: noqa diff --git a/lib/spack/external/jsonschema/tests/test_cli.py b/lib/spack/external/jsonschema/tests/test_cli.py new file mode 100644 index 0000000000..f625ca989d --- /dev/null +++ b/lib/spack/external/jsonschema/tests/test_cli.py @@ -0,0 +1,110 @@ +from jsonschema import Draft4Validator, ValidationError, cli +from jsonschema.compat import StringIO +from jsonschema.tests.compat import mock, unittest + + +def fake_validator(*errors): + errors = list(reversed(errors)) + + class FakeValidator(object): + def __init__(self, *args, **kwargs): + pass + + def iter_errors(self, instance): + if errors: + return errors.pop() + return [] + return FakeValidator + + +class TestParser(unittest.TestCase): + FakeValidator = fake_validator() + + def setUp(self): + mock_open = mock.mock_open() + patch_open = mock.patch.object(cli, "open", mock_open, create=True) + patch_open.start() + self.addCleanup(patch_open.stop) + + mock_json_load = mock.Mock() + mock_json_load.return_value = {} + patch_json_load = mock.patch("json.load") + patch_json_load.start() + self.addCleanup(patch_json_load.stop) + + def test_find_validator_by_fully_qualified_object_name(self): + arguments = cli.parse_args( + [ + "--validator", + "jsonschema.tests.test_cli.TestParser.FakeValidator", + "--instance", "foo.json", + "schema.json", + ] + ) + self.assertIs(arguments["validator"], self.FakeValidator) + + def test_find_validator_in_jsonschema(self): + arguments = cli.parse_args( + [ + "--validator", "Draft4Validator", + "--instance", "foo.json", + "schema.json", + ] + ) + self.assertIs(arguments["validator"], Draft4Validator) + + +class TestCLI(unittest.TestCase): + def test_successful_validation(self): + stdout, stderr = StringIO(), StringIO() + exit_code = cli.run( + { + "validator": fake_validator(), + "schema": {}, + "instances": [1], + "error_format": "{error.message}", + }, + stdout=stdout, + stderr=stderr, + ) + self.assertFalse(stdout.getvalue()) + self.assertFalse(stderr.getvalue()) + self.assertEqual(exit_code, 0) + + def test_unsuccessful_validation(self): + error = ValidationError("I am an error!", instance=1) + stdout, stderr = StringIO(), StringIO() + exit_code = cli.run( + { + "validator": fake_validator([error]), + "schema": {}, + "instances": [1], + "error_format": "{error.instance} - {error.message}", + }, + stdout=stdout, + stderr=stderr, + ) + self.assertFalse(stdout.getvalue()) + self.assertEqual(stderr.getvalue(), "1 - I am an error!") + self.assertEqual(exit_code, 1) + + def test_unsuccessful_validation_multiple_instances(self): + first_errors = [ + ValidationError("9", instance=1), + ValidationError("8", instance=1), + ] + second_errors = [ValidationError("7", instance=2)] + stdout, stderr = StringIO(), StringIO() + exit_code = cli.run( + { + "validator": fake_validator(first_errors, second_errors), + "schema": {}, + "instances": [1, 2], + "error_format": "{error.instance} - {error.message}\t", + }, + stdout=stdout, + stderr=stderr, + ) + self.assertFalse(stdout.getvalue()) + self.assertEqual(stderr.getvalue(), "1 - 9\t1 - 8\t2 - 7\t") + self.assertEqual(exit_code, 1) diff --git a/lib/spack/external/jsonschema/tests/test_exceptions.py b/lib/spack/external/jsonschema/tests/test_exceptions.py new file mode 100644 index 0000000000..9e5793c628 --- /dev/null +++ b/lib/spack/external/jsonschema/tests/test_exceptions.py @@ -0,0 +1,382 @@ +import textwrap + +from jsonschema import Draft4Validator, exceptions +from jsonschema.compat import PY3 +from jsonschema.tests.compat import mock, unittest + + +class TestBestMatch(unittest.TestCase): + def best_match(self, errors): + errors = list(errors) + best = exceptions.best_match(errors) + reversed_best = exceptions.best_match(reversed(errors)) + self.assertEqual( + best, + reversed_best, + msg="Didn't return a consistent best match!\n" + "Got: {0}\n\nThen: {1}".format(best, reversed_best), + ) + return best + + def test_shallower_errors_are_better_matches(self): + validator = Draft4Validator( + { + "properties" : { + "foo" : { + "minProperties" : 2, + "properties" : {"bar" : {"type" : "object"}}, + } + } + } + ) + best = self.best_match(validator.iter_errors({"foo" : {"bar" : []}})) + self.assertEqual(best.validator, "minProperties") + + def test_oneOf_and_anyOf_are_weak_matches(self): + """ + A property you *must* match is probably better than one you have to + match a part of. + + """ + + validator = Draft4Validator( + { + "minProperties" : 2, + "anyOf" : [{"type" : "string"}, {"type" : "number"}], + "oneOf" : [{"type" : "string"}, {"type" : "number"}], + } + ) + best = self.best_match(validator.iter_errors({})) + self.assertEqual(best.validator, "minProperties") + + def test_if_the_most_relevant_error_is_anyOf_it_is_traversed(self): + """ + If the most relevant error is an anyOf, then we traverse its context + and select the otherwise *least* relevant error, since in this case + that means the most specific, deep, error inside the instance. + + I.e. since only one of the schemas must match, we look for the most + relevant one. + + """ + + validator = Draft4Validator( + { + "properties" : { + "foo" : { + "anyOf" : [ + {"type" : "string"}, + {"properties" : {"bar" : {"type" : "array"}}}, + ], + }, + }, + }, + ) + best = self.best_match(validator.iter_errors({"foo" : {"bar" : 12}})) + self.assertEqual(best.validator_value, "array") + + def test_if_the_most_relevant_error_is_oneOf_it_is_traversed(self): + """ + If the most relevant error is an oneOf, then we traverse its context + and select the otherwise *least* relevant error, since in this case + that means the most specific, deep, error inside the instance. + + I.e. since only one of the schemas must match, we look for the most + relevant one. + + """ + + validator = Draft4Validator( + { + "properties" : { + "foo" : { + "oneOf" : [ + {"type" : "string"}, + {"properties" : {"bar" : {"type" : "array"}}}, + ], + }, + }, + }, + ) + best = self.best_match(validator.iter_errors({"foo" : {"bar" : 12}})) + self.assertEqual(best.validator_value, "array") + + def test_if_the_most_relevant_error_is_allOf_it_is_traversed(self): + """ + Now, if the error is allOf, we traverse but select the *most* relevant + error from the context, because all schemas here must match anyways. + + """ + + validator = Draft4Validator( + { + "properties" : { + "foo" : { + "allOf" : [ + {"type" : "string"}, + {"properties" : {"bar" : {"type" : "array"}}}, + ], + }, + }, + }, + ) + best = self.best_match(validator.iter_errors({"foo" : {"bar" : 12}})) + self.assertEqual(best.validator_value, "string") + + def test_nested_context_for_oneOf(self): + validator = Draft4Validator( + { + "properties" : { + "foo" : { + "oneOf" : [ + {"type" : "string"}, + { + "oneOf" : [ + {"type" : "string"}, + { + "properties" : { + "bar" : {"type" : "array"} + }, + }, + ], + }, + ], + }, + }, + }, + ) + best = self.best_match(validator.iter_errors({"foo" : {"bar" : 12}})) + self.assertEqual(best.validator_value, "array") + + def test_one_error(self): + validator = Draft4Validator({"minProperties" : 2}) + error, = validator.iter_errors({}) + self.assertEqual( + exceptions.best_match(validator.iter_errors({})).validator, + "minProperties", + ) + + def test_no_errors(self): + validator = Draft4Validator({}) + self.assertIsNone(exceptions.best_match(validator.iter_errors({}))) + + +class TestByRelevance(unittest.TestCase): + def test_short_paths_are_better_matches(self): + shallow = exceptions.ValidationError("Oh no!", path=["baz"]) + deep = exceptions.ValidationError("Oh yes!", path=["foo", "bar"]) + match = max([shallow, deep], key=exceptions.relevance) + self.assertIs(match, shallow) + + match = max([deep, shallow], key=exceptions.relevance) + self.assertIs(match, shallow) + + def test_global_errors_are_even_better_matches(self): + shallow = exceptions.ValidationError("Oh no!", path=[]) + deep = exceptions.ValidationError("Oh yes!", path=["foo"]) + + errors = sorted([shallow, deep], key=exceptions.relevance) + self.assertEqual( + [list(error.path) for error in errors], + [["foo"], []], + ) + + errors = sorted([deep, shallow], key=exceptions.relevance) + self.assertEqual( + [list(error.path) for error in errors], + [["foo"], []], + ) + + def test_weak_validators_are_lower_priority(self): + weak = exceptions.ValidationError("Oh no!", path=[], validator="a") + normal = exceptions.ValidationError("Oh yes!", path=[], validator="b") + + best_match = exceptions.by_relevance(weak="a") + + match = max([weak, normal], key=best_match) + self.assertIs(match, normal) + + match = max([normal, weak], key=best_match) + self.assertIs(match, normal) + + def test_strong_validators_are_higher_priority(self): + weak = exceptions.ValidationError("Oh no!", path=[], validator="a") + normal = exceptions.ValidationError("Oh yes!", path=[], validator="b") + strong = exceptions.ValidationError("Oh fine!", path=[], validator="c") + + best_match = exceptions.by_relevance(weak="a", strong="c") + + match = max([weak, normal, strong], key=best_match) + self.assertIs(match, strong) + + match = max([strong, normal, weak], key=best_match) + self.assertIs(match, strong) + + +class TestErrorTree(unittest.TestCase): + def test_it_knows_how_many_total_errors_it_contains(self): + errors = [mock.MagicMock() for _ in range(8)] + tree = exceptions.ErrorTree(errors) + self.assertEqual(tree.total_errors, 8) + + def test_it_contains_an_item_if_the_item_had_an_error(self): + errors = [exceptions.ValidationError("a message", path=["bar"])] + tree = exceptions.ErrorTree(errors) + self.assertIn("bar", tree) + + def test_it_does_not_contain_an_item_if_the_item_had_no_error(self): + errors = [exceptions.ValidationError("a message", path=["bar"])] + tree = exceptions.ErrorTree(errors) + self.assertNotIn("foo", tree) + + def test_validators_that_failed_appear_in_errors_dict(self): + error = exceptions.ValidationError("a message", validator="foo") + tree = exceptions.ErrorTree([error]) + self.assertEqual(tree.errors, {"foo" : error}) + + def test_it_creates_a_child_tree_for_each_nested_path(self): + errors = [ + exceptions.ValidationError("a bar message", path=["bar"]), + exceptions.ValidationError("a bar -> 0 message", path=["bar", 0]), + ] + tree = exceptions.ErrorTree(errors) + self.assertIn(0, tree["bar"]) + self.assertNotIn(1, tree["bar"]) + + def test_children_have_their_errors_dicts_built(self): + e1, e2 = ( + exceptions.ValidationError("1", validator="foo", path=["bar", 0]), + exceptions.ValidationError("2", validator="quux", path=["bar", 0]), + ) + tree = exceptions.ErrorTree([e1, e2]) + self.assertEqual(tree["bar"][0].errors, {"foo" : e1, "quux" : e2}) + + def test_it_does_not_contain_subtrees_that_are_not_in_the_instance(self): + error = exceptions.ValidationError("123", validator="foo", instance=[]) + tree = exceptions.ErrorTree([error]) + + with self.assertRaises(IndexError): + tree[0] + + def test_if_its_in_the_tree_anyhow_it_does_not_raise_an_error(self): + """ + If a validator is dumb (like :validator:`required` in draft 3) and + refers to a path that isn't in the instance, the tree still properly + returns a subtree for that path. + + """ + + error = exceptions.ValidationError( + "a message", validator="foo", instance={}, path=["foo"], + ) + tree = exceptions.ErrorTree([error]) + self.assertIsInstance(tree["foo"], exceptions.ErrorTree) + + +class TestErrorReprStr(unittest.TestCase): + def make_error(self, **kwargs): + defaults = dict( + message=u"hello", + validator=u"type", + validator_value=u"string", + instance=5, + schema={u"type": u"string"}, + ) + defaults.update(kwargs) + return exceptions.ValidationError(**defaults) + + def assertShows(self, expected, **kwargs): + if PY3: + expected = expected.replace("u'", "'") + expected = textwrap.dedent(expected).rstrip("\n") + + error = self.make_error(**kwargs) + message_line, _, rest = str(error).partition("\n") + self.assertEqual(message_line, error.message) + self.assertEqual(rest, expected) + + def test_repr(self): + self.assertEqual( + repr(exceptions.ValidationError(message="Hello!")), + "" % "Hello!", + ) + + def test_unset_error(self): + error = exceptions.ValidationError("message") + self.assertEqual(str(error), "message") + + kwargs = { + "validator": "type", + "validator_value": "string", + "instance": 5, + "schema": {"type": "string"} + } + # Just the message should show if any of the attributes are unset + for attr in kwargs: + k = dict(kwargs) + del k[attr] + error = exceptions.ValidationError("message", **k) + self.assertEqual(str(error), "message") + + def test_empty_paths(self): + self.assertShows( + """ + Failed validating u'type' in schema: + {u'type': u'string'} + + On instance: + 5 + """, + path=[], + schema_path=[], + ) + + def test_one_item_paths(self): + self.assertShows( + """ + Failed validating u'type' in schema: + {u'type': u'string'} + + On instance[0]: + 5 + """, + path=[0], + schema_path=["items"], + ) + + def test_multiple_item_paths(self): + self.assertShows( + """ + Failed validating u'type' in schema[u'items'][0]: + {u'type': u'string'} + + On instance[0][u'a']: + 5 + """, + path=[0, u"a"], + schema_path=[u"items", 0, 1], + ) + + def test_uses_pprint(self): + with mock.patch("pprint.pformat") as pformat: + str(self.make_error()) + self.assertEqual(pformat.call_count, 2) # schema + instance + + def test_str_works_with_instances_having_overriden_eq_operator(self): + """ + Check for https://github.com/Julian/jsonschema/issues/164 which + rendered exceptions unusable when a `ValidationError` involved + instances with an `__eq__` method that returned truthy values. + + """ + + instance = mock.MagicMock() + error = exceptions.ValidationError( + "a message", + validator="foo", + instance=instance, + validator_value="some", + schema="schema", + ) + str(error) + self.assertFalse(instance.__eq__.called) diff --git a/lib/spack/external/jsonschema/tests/test_format.py b/lib/spack/external/jsonschema/tests/test_format.py new file mode 100644 index 0000000000..8392ca1de3 --- /dev/null +++ b/lib/spack/external/jsonschema/tests/test_format.py @@ -0,0 +1,63 @@ +""" +Tests for the parts of jsonschema related to the :validator:`format` property. + +""" + +from jsonschema.tests.compat import mock, unittest + +from jsonschema import FormatError, ValidationError, FormatChecker +from jsonschema.validators import Draft4Validator + + +class TestFormatChecker(unittest.TestCase): + def setUp(self): + self.fn = mock.Mock() + + def test_it_can_validate_no_formats(self): + checker = FormatChecker(formats=()) + self.assertFalse(checker.checkers) + + def test_it_raises_a_key_error_for_unknown_formats(self): + with self.assertRaises(KeyError): + FormatChecker(formats=["o noes"]) + + def test_it_can_register_cls_checkers(self): + with mock.patch.dict(FormatChecker.checkers, clear=True): + FormatChecker.cls_checks("new")(self.fn) + self.assertEqual(FormatChecker.checkers, {"new" : (self.fn, ())}) + + def test_it_can_register_checkers(self): + checker = FormatChecker() + checker.checks("new")(self.fn) + self.assertEqual( + checker.checkers, + dict(FormatChecker.checkers, new=(self.fn, ())) + ) + + def test_it_catches_registered_errors(self): + checker = FormatChecker() + cause = self.fn.side_effect = ValueError() + + checker.checks("foo", raises=ValueError)(self.fn) + + with self.assertRaises(FormatError) as cm: + checker.check("bar", "foo") + + self.assertIs(cm.exception.cause, cause) + self.assertIs(cm.exception.__cause__, cause) + + # Unregistered errors should not be caught + self.fn.side_effect = AttributeError + with self.assertRaises(AttributeError): + checker.check("bar", "foo") + + def test_format_error_causes_become_validation_error_causes(self): + checker = FormatChecker() + checker.checks("foo", raises=ValueError)(self.fn) + cause = self.fn.side_effect = ValueError() + validator = Draft4Validator({"format" : "foo"}, format_checker=checker) + + with self.assertRaises(ValidationError) as cm: + validator.validate("bar") + + self.assertIs(cm.exception.__cause__, cause) diff --git a/lib/spack/external/jsonschema/tests/test_jsonschema_test_suite.py b/lib/spack/external/jsonschema/tests/test_jsonschema_test_suite.py new file mode 100644 index 0000000000..75c6857bc0 --- /dev/null +++ b/lib/spack/external/jsonschema/tests/test_jsonschema_test_suite.py @@ -0,0 +1,290 @@ +""" +Test runner for the JSON Schema official test suite + +Tests comprehensive correctness of each draft's validator. + +See https://github.com/json-schema/JSON-Schema-Test-Suite for details. + +""" + +from contextlib import closing +from decimal import Decimal +import glob +import json +import io +import itertools +import os +import re +import subprocess +import sys + +try: + from sys import pypy_version_info +except ImportError: + pypy_version_info = None + +from jsonschema import ( + FormatError, SchemaError, ValidationError, Draft3Validator, + Draft4Validator, FormatChecker, draft3_format_checker, + draft4_format_checker, validate, +) +from jsonschema.compat import PY3 +from jsonschema.tests.compat import mock, unittest +import jsonschema + + +REPO_ROOT = os.path.join(os.path.dirname(jsonschema.__file__), os.path.pardir) +SUITE = os.getenv("JSON_SCHEMA_TEST_SUITE", os.path.join(REPO_ROOT, "json")) + +if not os.path.isdir(SUITE): + raise ValueError( + "Can't find the JSON-Schema-Test-Suite directory. Set the " + "'JSON_SCHEMA_TEST_SUITE' environment variable or run the tests from " + "alongside a checkout of the suite." + ) + +TESTS_DIR = os.path.join(SUITE, "tests") +JSONSCHEMA_SUITE = os.path.join(SUITE, "bin", "jsonschema_suite") + +remotes_stdout = subprocess.Popen( + ["python", JSONSCHEMA_SUITE, "remotes"], stdout=subprocess.PIPE, +).stdout + +with closing(remotes_stdout): + if PY3: + remotes_stdout = io.TextIOWrapper(remotes_stdout) + REMOTES = json.load(remotes_stdout) + + +def make_case(schema, data, valid, name): + if valid: + def test_case(self): + kwargs = getattr(self, "validator_kwargs", {}) + validate(data, schema, cls=self.validator_class, **kwargs) + else: + def test_case(self): + kwargs = getattr(self, "validator_kwargs", {}) + with self.assertRaises(ValidationError): + validate(data, schema, cls=self.validator_class, **kwargs) + + if not PY3: + name = name.encode("utf-8") + test_case.__name__ = name + + return test_case + + +def maybe_skip(skip, test_case, case, test): + if skip is not None: + reason = skip(case, test) + if reason is not None: + test_case = unittest.skip(reason)(test_case) + return test_case + + +def load_json_cases(tests_glob, ignore_glob="", basedir=TESTS_DIR, skip=None): + if ignore_glob: + ignore_glob = os.path.join(basedir, ignore_glob) + + def add_test_methods(test_class): + ignored = set(glob.iglob(ignore_glob)) + + for filename in glob.iglob(os.path.join(basedir, tests_glob)): + if filename in ignored: + continue + + validating, _ = os.path.splitext(os.path.basename(filename)) + id = itertools.count(1) + + with open(filename) as test_file: + for case in json.load(test_file): + for test in case["tests"]: + name = "test_%s_%s_%s" % ( + validating, + next(id), + re.sub(r"[\W ]+", "_", test["description"]), + ) + assert not hasattr(test_class, name), name + + test_case = make_case( + data=test["data"], + schema=case["schema"], + valid=test["valid"], + name=name, + ) + test_case = maybe_skip(skip, test_case, case, test) + setattr(test_class, name, test_case) + + return test_class + return add_test_methods + + +class TypesMixin(object): + @unittest.skipIf(PY3, "In Python 3 json.load always produces unicode") + def test_string_a_bytestring_is_a_string(self): + self.validator_class({"type" : "string"}).validate(b"foo") + + +class DecimalMixin(object): + def test_it_can_validate_with_decimals(self): + schema = {"type" : "number"} + validator = self.validator_class( + schema, types={"number" : (int, float, Decimal)} + ) + + for valid in [1, 1.1, Decimal(1) / Decimal(8)]: + validator.validate(valid) + + for invalid in ["foo", {}, [], True, None]: + with self.assertRaises(ValidationError): + validator.validate(invalid) + + +def missing_format(checker): + def missing_format(case, test): + format = case["schema"].get("format") + if format not in checker.checkers: + return "Format checker {0!r} not found.".format(format) + elif ( + format == "date-time" and + pypy_version_info is not None and + pypy_version_info[:2] <= (1, 9) + ): + # datetime.datetime is overzealous about typechecking in <=1.9 + return "datetime.datetime is broken on this version of PyPy." + return missing_format + + +class FormatMixin(object): + def test_it_returns_true_for_formats_it_does_not_know_about(self): + validator = self.validator_class( + {"format" : "carrot"}, format_checker=FormatChecker(), + ) + validator.validate("bugs") + + def test_it_does_not_validate_formats_by_default(self): + validator = self.validator_class({}) + self.assertIsNone(validator.format_checker) + + def test_it_validates_formats_if_a_checker_is_provided(self): + checker = mock.Mock(spec=FormatChecker) + validator = self.validator_class( + {"format" : "foo"}, format_checker=checker, + ) + + validator.validate("bar") + + checker.check.assert_called_once_with("bar", "foo") + + cause = ValueError() + checker.check.side_effect = FormatError('aoeu', cause=cause) + + with self.assertRaises(ValidationError) as cm: + validator.validate("bar") + # Make sure original cause is attached + self.assertIs(cm.exception.cause, cause) + + def test_it_validates_formats_of_any_type(self): + checker = mock.Mock(spec=FormatChecker) + validator = self.validator_class( + {"format" : "foo"}, format_checker=checker, + ) + + validator.validate([1, 2, 3]) + + checker.check.assert_called_once_with([1, 2, 3], "foo") + + cause = ValueError() + checker.check.side_effect = FormatError('aoeu', cause=cause) + + with self.assertRaises(ValidationError) as cm: + validator.validate([1, 2, 3]) + # Make sure original cause is attached + self.assertIs(cm.exception.cause, cause) + + +if sys.maxunicode == 2 ** 16 - 1: # This is a narrow build. + def narrow_unicode_build(case, test): + if "supplementary Unicode" in test["description"]: + return "Not running surrogate Unicode case, this Python is narrow." +else: + def narrow_unicode_build(case, test): # This isn't, skip nothing. + return + + +@load_json_cases( + "draft3/*.json", + skip=narrow_unicode_build, + ignore_glob="draft3/refRemote.json", +) +@load_json_cases( + "draft3/optional/format.json", skip=missing_format(draft3_format_checker) +) +@load_json_cases("draft3/optional/bignum.json") +@load_json_cases("draft3/optional/zeroTerminatedFloats.json") +class TestDraft3(unittest.TestCase, TypesMixin, DecimalMixin, FormatMixin): + validator_class = Draft3Validator + validator_kwargs = {"format_checker" : draft3_format_checker} + + def test_any_type_is_valid_for_type_any(self): + validator = self.validator_class({"type" : "any"}) + validator.validate(mock.Mock()) + + # TODO: we're in need of more meta schema tests + def test_invalid_properties(self): + with self.assertRaises(SchemaError): + validate({}, {"properties": {"test": True}}, + cls=self.validator_class) + + def test_minItems_invalid_string(self): + with self.assertRaises(SchemaError): + # needs to be an integer + validate([1], {"minItems" : "1"}, cls=self.validator_class) + + +@load_json_cases( + "draft4/*.json", + skip=narrow_unicode_build, + ignore_glob="draft4/refRemote.json", +) +@load_json_cases( + "draft4/optional/format.json", skip=missing_format(draft4_format_checker) +) +@load_json_cases("draft4/optional/bignum.json") +@load_json_cases("draft4/optional/zeroTerminatedFloats.json") +class TestDraft4(unittest.TestCase, TypesMixin, DecimalMixin, FormatMixin): + validator_class = Draft4Validator + validator_kwargs = {"format_checker" : draft4_format_checker} + + # TODO: we're in need of more meta schema tests + def test_invalid_properties(self): + with self.assertRaises(SchemaError): + validate({}, {"properties": {"test": True}}, + cls=self.validator_class) + + def test_minItems_invalid_string(self): + with self.assertRaises(SchemaError): + # needs to be an integer + validate([1], {"minItems" : "1"}, cls=self.validator_class) + + +class RemoteRefResolutionMixin(object): + def setUp(self): + patch = mock.patch("jsonschema.validators.requests") + requests = patch.start() + requests.get.side_effect = self.resolve + self.addCleanup(patch.stop) + + def resolve(self, reference): + _, _, reference = reference.partition("http://localhost:1234/") + return mock.Mock(**{"json.return_value" : REMOTES.get(reference)}) + + +@load_json_cases("draft3/refRemote.json") +class Draft3RemoteResolution(RemoteRefResolutionMixin, unittest.TestCase): + validator_class = Draft3Validator + + +@load_json_cases("draft4/refRemote.json") +class Draft4RemoteResolution(RemoteRefResolutionMixin, unittest.TestCase): + validator_class = Draft4Validator diff --git a/lib/spack/external/jsonschema/tests/test_validators.py b/lib/spack/external/jsonschema/tests/test_validators.py new file mode 100644 index 0000000000..f8692388ea --- /dev/null +++ b/lib/spack/external/jsonschema/tests/test_validators.py @@ -0,0 +1,786 @@ +from collections import deque +from contextlib import contextmanager +import json + +from jsonschema import FormatChecker, ValidationError +from jsonschema.tests.compat import mock, unittest +from jsonschema.validators import ( + RefResolutionError, UnknownType, Draft3Validator, + Draft4Validator, RefResolver, create, extend, validator_for, validate, +) + + +class TestCreateAndExtend(unittest.TestCase): + def setUp(self): + self.meta_schema = {u"properties" : {u"smelly" : {}}} + self.smelly = mock.MagicMock() + self.validators = {u"smelly" : self.smelly} + self.types = {u"dict" : dict} + self.Validator = create( + meta_schema=self.meta_schema, + validators=self.validators, + default_types=self.types, + ) + + self.validator_value = 12 + self.schema = {u"smelly" : self.validator_value} + self.validator = self.Validator(self.schema) + + def test_attrs(self): + self.assertEqual(self.Validator.VALIDATORS, self.validators) + self.assertEqual(self.Validator.META_SCHEMA, self.meta_schema) + self.assertEqual(self.Validator.DEFAULT_TYPES, self.types) + + def test_init(self): + self.assertEqual(self.validator.schema, self.schema) + + def test_iter_errors(self): + instance = "hello" + + self.smelly.return_value = [] + self.assertEqual(list(self.validator.iter_errors(instance)), []) + + error = mock.Mock() + self.smelly.return_value = [error] + self.assertEqual(list(self.validator.iter_errors(instance)), [error]) + + self.smelly.assert_called_with( + self.validator, self.validator_value, instance, self.schema, + ) + + def test_if_a_version_is_provided_it_is_registered(self): + with mock.patch("jsonschema.validators.validates") as validates: + validates.side_effect = lambda version : lambda cls : cls + Validator = create(meta_schema={u"id" : ""}, version="my version") + validates.assert_called_once_with("my version") + self.assertEqual(Validator.__name__, "MyVersionValidator") + + def test_if_a_version_is_not_provided_it_is_not_registered(self): + with mock.patch("jsonschema.validators.validates") as validates: + create(meta_schema={u"id" : "id"}) + self.assertFalse(validates.called) + + def test_extend(self): + validators = dict(self.Validator.VALIDATORS) + new = mock.Mock() + + Extended = extend(self.Validator, validators={u"a new one" : new}) + + validators.update([(u"a new one", new)]) + self.assertEqual(Extended.VALIDATORS, validators) + self.assertNotIn(u"a new one", self.Validator.VALIDATORS) + + self.assertEqual(Extended.META_SCHEMA, self.Validator.META_SCHEMA) + self.assertEqual(Extended.DEFAULT_TYPES, self.Validator.DEFAULT_TYPES) + + +class TestIterErrors(unittest.TestCase): + def setUp(self): + self.validator = Draft3Validator({}) + + def test_iter_errors(self): + instance = [1, 2] + schema = { + u"disallow" : u"array", + u"enum" : [["a", "b", "c"], ["d", "e", "f"]], + u"minItems" : 3 + } + + got = (e.message for e in self.validator.iter_errors(instance, schema)) + expected = [ + "%r is disallowed for [1, 2]" % (schema["disallow"],), + "[1, 2] is too short", + "[1, 2] is not one of %r" % (schema["enum"],), + ] + self.assertEqual(sorted(got), sorted(expected)) + + def test_iter_errors_multiple_failures_one_validator(self): + instance = {"foo" : 2, "bar" : [1], "baz" : 15, "quux" : "spam"} + schema = { + u"properties" : { + "foo" : {u"type" : "string"}, + "bar" : {u"minItems" : 2}, + "baz" : {u"maximum" : 10, u"enum" : [2, 4, 6, 8]}, + } + } + + errors = list(self.validator.iter_errors(instance, schema)) + self.assertEqual(len(errors), 4) + + +class TestValidationErrorMessages(unittest.TestCase): + def message_for(self, instance, schema, *args, **kwargs): + kwargs.setdefault("cls", Draft3Validator) + with self.assertRaises(ValidationError) as e: + validate(instance, schema, *args, **kwargs) + return e.exception.message + + def test_single_type_failure(self): + message = self.message_for(instance=1, schema={u"type" : u"string"}) + self.assertEqual(message, "1 is not of type %r" % u"string") + + def test_single_type_list_failure(self): + message = self.message_for(instance=1, schema={u"type" : [u"string"]}) + self.assertEqual(message, "1 is not of type %r" % u"string") + + def test_multiple_type_failure(self): + types = u"string", u"object" + message = self.message_for(instance=1, schema={u"type" : list(types)}) + self.assertEqual(message, "1 is not of type %r, %r" % types) + + def test_object_without_title_type_failure(self): + type = {u"type" : [{u"minimum" : 3}]} + message = self.message_for(instance=1, schema={u"type" : [type]}) + self.assertEqual(message, "1 is not of type %r" % (type,)) + + def test_object_with_name_type_failure(self): + name = "Foo" + schema = {u"type" : [{u"name" : name, u"minimum" : 3}]} + message = self.message_for(instance=1, schema=schema) + self.assertEqual(message, "1 is not of type %r" % (name,)) + + def test_minimum(self): + message = self.message_for(instance=1, schema={"minimum" : 2}) + self.assertEqual(message, "1 is less than the minimum of 2") + + def test_maximum(self): + message = self.message_for(instance=1, schema={"maximum" : 0}) + self.assertEqual(message, "1 is greater than the maximum of 0") + + def test_dependencies_failure_has_single_element_not_list(self): + depend, on = "bar", "foo" + schema = {u"dependencies" : {depend : on}} + message = self.message_for({"bar" : 2}, schema) + self.assertEqual(message, "%r is a dependency of %r" % (on, depend)) + + def test_additionalItems_single_failure(self): + message = self.message_for( + [2], {u"items" : [], u"additionalItems" : False}, + ) + self.assertIn("(2 was unexpected)", message) + + def test_additionalItems_multiple_failures(self): + message = self.message_for( + [1, 2, 3], {u"items" : [], u"additionalItems" : False} + ) + self.assertIn("(1, 2, 3 were unexpected)", message) + + def test_additionalProperties_single_failure(self): + additional = "foo" + schema = {u"additionalProperties" : False} + message = self.message_for({additional : 2}, schema) + self.assertIn("(%r was unexpected)" % (additional,), message) + + def test_additionalProperties_multiple_failures(self): + schema = {u"additionalProperties" : False} + message = self.message_for(dict.fromkeys(["foo", "bar"]), schema) + + self.assertIn(repr("foo"), message) + self.assertIn(repr("bar"), message) + self.assertIn("were unexpected)", message) + + def test_invalid_format_default_message(self): + checker = FormatChecker(formats=()) + check_fn = mock.Mock(return_value=False) + checker.checks(u"thing")(check_fn) + + schema = {u"format" : u"thing"} + message = self.message_for("bla", schema, format_checker=checker) + + self.assertIn(repr("bla"), message) + self.assertIn(repr("thing"), message) + self.assertIn("is not a", message) + + +class TestValidationErrorDetails(unittest.TestCase): + # TODO: These really need unit tests for each individual validator, rather + # than just these higher level tests. + def test_anyOf(self): + instance = 5 + schema = { + "anyOf": [ + {"minimum": 20}, + {"type": "string"} + ] + } + + validator = Draft4Validator(schema) + errors = list(validator.iter_errors(instance)) + self.assertEqual(len(errors), 1) + e = errors[0] + + self.assertEqual(e.validator, "anyOf") + self.assertEqual(e.validator_value, schema["anyOf"]) + self.assertEqual(e.instance, instance) + self.assertEqual(e.schema, schema) + self.assertIsNone(e.parent) + + self.assertEqual(e.path, deque([])) + self.assertEqual(e.relative_path, deque([])) + self.assertEqual(e.absolute_path, deque([])) + + self.assertEqual(e.schema_path, deque(["anyOf"])) + self.assertEqual(e.relative_schema_path, deque(["anyOf"])) + self.assertEqual(e.absolute_schema_path, deque(["anyOf"])) + + self.assertEqual(len(e.context), 2) + + e1, e2 = sorted_errors(e.context) + + self.assertEqual(e1.validator, "minimum") + self.assertEqual(e1.validator_value, schema["anyOf"][0]["minimum"]) + self.assertEqual(e1.instance, instance) + self.assertEqual(e1.schema, schema["anyOf"][0]) + self.assertIs(e1.parent, e) + + self.assertEqual(e1.path, deque([])) + self.assertEqual(e1.absolute_path, deque([])) + self.assertEqual(e1.relative_path, deque([])) + + self.assertEqual(e1.schema_path, deque([0, "minimum"])) + self.assertEqual(e1.relative_schema_path, deque([0, "minimum"])) + self.assertEqual( + e1.absolute_schema_path, deque(["anyOf", 0, "minimum"]), + ) + + self.assertFalse(e1.context) + + self.assertEqual(e2.validator, "type") + self.assertEqual(e2.validator_value, schema["anyOf"][1]["type"]) + self.assertEqual(e2.instance, instance) + self.assertEqual(e2.schema, schema["anyOf"][1]) + self.assertIs(e2.parent, e) + + self.assertEqual(e2.path, deque([])) + self.assertEqual(e2.relative_path, deque([])) + self.assertEqual(e2.absolute_path, deque([])) + + self.assertEqual(e2.schema_path, deque([1, "type"])) + self.assertEqual(e2.relative_schema_path, deque([1, "type"])) + self.assertEqual(e2.absolute_schema_path, deque(["anyOf", 1, "type"])) + + self.assertEqual(len(e2.context), 0) + + def test_type(self): + instance = {"foo": 1} + schema = { + "type": [ + {"type": "integer"}, + { + "type": "object", + "properties": { + "foo": {"enum": [2]} + } + } + ] + } + + validator = Draft3Validator(schema) + errors = list(validator.iter_errors(instance)) + self.assertEqual(len(errors), 1) + e = errors[0] + + self.assertEqual(e.validator, "type") + self.assertEqual(e.validator_value, schema["type"]) + self.assertEqual(e.instance, instance) + self.assertEqual(e.schema, schema) + self.assertIsNone(e.parent) + + self.assertEqual(e.path, deque([])) + self.assertEqual(e.relative_path, deque([])) + self.assertEqual(e.absolute_path, deque([])) + + self.assertEqual(e.schema_path, deque(["type"])) + self.assertEqual(e.relative_schema_path, deque(["type"])) + self.assertEqual(e.absolute_schema_path, deque(["type"])) + + self.assertEqual(len(e.context), 2) + + e1, e2 = sorted_errors(e.context) + + self.assertEqual(e1.validator, "type") + self.assertEqual(e1.validator_value, schema["type"][0]["type"]) + self.assertEqual(e1.instance, instance) + self.assertEqual(e1.schema, schema["type"][0]) + self.assertIs(e1.parent, e) + + self.assertEqual(e1.path, deque([])) + self.assertEqual(e1.relative_path, deque([])) + self.assertEqual(e1.absolute_path, deque([])) + + self.assertEqual(e1.schema_path, deque([0, "type"])) + self.assertEqual(e1.relative_schema_path, deque([0, "type"])) + self.assertEqual(e1.absolute_schema_path, deque(["type", 0, "type"])) + + self.assertFalse(e1.context) + + self.assertEqual(e2.validator, "enum") + self.assertEqual(e2.validator_value, [2]) + self.assertEqual(e2.instance, 1) + self.assertEqual(e2.schema, {u"enum" : [2]}) + self.assertIs(e2.parent, e) + + self.assertEqual(e2.path, deque(["foo"])) + self.assertEqual(e2.relative_path, deque(["foo"])) + self.assertEqual(e2.absolute_path, deque(["foo"])) + + self.assertEqual( + e2.schema_path, deque([1, "properties", "foo", "enum"]), + ) + self.assertEqual( + e2.relative_schema_path, deque([1, "properties", "foo", "enum"]), + ) + self.assertEqual( + e2.absolute_schema_path, + deque(["type", 1, "properties", "foo", "enum"]), + ) + + self.assertFalse(e2.context) + + def test_single_nesting(self): + instance = {"foo" : 2, "bar" : [1], "baz" : 15, "quux" : "spam"} + schema = { + "properties" : { + "foo" : {"type" : "string"}, + "bar" : {"minItems" : 2}, + "baz" : {"maximum" : 10, "enum" : [2, 4, 6, 8]}, + } + } + + validator = Draft3Validator(schema) + errors = validator.iter_errors(instance) + e1, e2, e3, e4 = sorted_errors(errors) + + self.assertEqual(e1.path, deque(["bar"])) + self.assertEqual(e2.path, deque(["baz"])) + self.assertEqual(e3.path, deque(["baz"])) + self.assertEqual(e4.path, deque(["foo"])) + + self.assertEqual(e1.relative_path, deque(["bar"])) + self.assertEqual(e2.relative_path, deque(["baz"])) + self.assertEqual(e3.relative_path, deque(["baz"])) + self.assertEqual(e4.relative_path, deque(["foo"])) + + self.assertEqual(e1.absolute_path, deque(["bar"])) + self.assertEqual(e2.absolute_path, deque(["baz"])) + self.assertEqual(e3.absolute_path, deque(["baz"])) + self.assertEqual(e4.absolute_path, deque(["foo"])) + + self.assertEqual(e1.validator, "minItems") + self.assertEqual(e2.validator, "enum") + self.assertEqual(e3.validator, "maximum") + self.assertEqual(e4.validator, "type") + + def test_multiple_nesting(self): + instance = [1, {"foo" : 2, "bar" : {"baz" : [1]}}, "quux"] + schema = { + "type" : "string", + "items" : { + "type" : ["string", "object"], + "properties" : { + "foo" : {"enum" : [1, 3]}, + "bar" : { + "type" : "array", + "properties" : { + "bar" : {"required" : True}, + "baz" : {"minItems" : 2}, + } + } + } + } + } + + validator = Draft3Validator(schema) + errors = validator.iter_errors(instance) + e1, e2, e3, e4, e5, e6 = sorted_errors(errors) + + self.assertEqual(e1.path, deque([])) + self.assertEqual(e2.path, deque([0])) + self.assertEqual(e3.path, deque([1, "bar"])) + self.assertEqual(e4.path, deque([1, "bar", "bar"])) + self.assertEqual(e5.path, deque([1, "bar", "baz"])) + self.assertEqual(e6.path, deque([1, "foo"])) + + self.assertEqual(e1.schema_path, deque(["type"])) + self.assertEqual(e2.schema_path, deque(["items", "type"])) + self.assertEqual( + list(e3.schema_path), ["items", "properties", "bar", "type"], + ) + self.assertEqual( + list(e4.schema_path), + ["items", "properties", "bar", "properties", "bar", "required"], + ) + self.assertEqual( + list(e5.schema_path), + ["items", "properties", "bar", "properties", "baz", "minItems"] + ) + self.assertEqual( + list(e6.schema_path), ["items", "properties", "foo", "enum"], + ) + + self.assertEqual(e1.validator, "type") + self.assertEqual(e2.validator, "type") + self.assertEqual(e3.validator, "type") + self.assertEqual(e4.validator, "required") + self.assertEqual(e5.validator, "minItems") + self.assertEqual(e6.validator, "enum") + + def test_additionalProperties(self): + instance = {"bar": "bar", "foo": 2} + schema = { + "additionalProperties" : {"type": "integer", "minimum": 5} + } + + validator = Draft3Validator(schema) + errors = validator.iter_errors(instance) + e1, e2 = sorted_errors(errors) + + self.assertEqual(e1.path, deque(["bar"])) + self.assertEqual(e2.path, deque(["foo"])) + + self.assertEqual(e1.validator, "type") + self.assertEqual(e2.validator, "minimum") + + def test_patternProperties(self): + instance = {"bar": 1, "foo": 2} + schema = { + "patternProperties" : { + "bar": {"type": "string"}, + "foo": {"minimum": 5} + } + } + + validator = Draft3Validator(schema) + errors = validator.iter_errors(instance) + e1, e2 = sorted_errors(errors) + + self.assertEqual(e1.path, deque(["bar"])) + self.assertEqual(e2.path, deque(["foo"])) + + self.assertEqual(e1.validator, "type") + self.assertEqual(e2.validator, "minimum") + + def test_additionalItems(self): + instance = ["foo", 1] + schema = { + "items": [], + "additionalItems" : {"type": "integer", "minimum": 5} + } + + validator = Draft3Validator(schema) + errors = validator.iter_errors(instance) + e1, e2 = sorted_errors(errors) + + self.assertEqual(e1.path, deque([0])) + self.assertEqual(e2.path, deque([1])) + + self.assertEqual(e1.validator, "type") + self.assertEqual(e2.validator, "minimum") + + def test_additionalItems_with_items(self): + instance = ["foo", "bar", 1] + schema = { + "items": [{}], + "additionalItems" : {"type": "integer", "minimum": 5} + } + + validator = Draft3Validator(schema) + errors = validator.iter_errors(instance) + e1, e2 = sorted_errors(errors) + + self.assertEqual(e1.path, deque([1])) + self.assertEqual(e2.path, deque([2])) + + self.assertEqual(e1.validator, "type") + self.assertEqual(e2.validator, "minimum") + + +class ValidatorTestMixin(object): + def setUp(self): + self.instance = mock.Mock() + self.schema = {} + self.resolver = mock.Mock() + self.validator = self.validator_class(self.schema) + + def test_valid_instances_are_valid(self): + errors = iter([]) + + with mock.patch.object( + self.validator, "iter_errors", return_value=errors, + ): + self.assertTrue( + self.validator.is_valid(self.instance, self.schema) + ) + + def test_invalid_instances_are_not_valid(self): + errors = iter([mock.Mock()]) + + with mock.patch.object( + self.validator, "iter_errors", return_value=errors, + ): + self.assertFalse( + self.validator.is_valid(self.instance, self.schema) + ) + + def test_non_existent_properties_are_ignored(self): + instance, my_property, my_value = mock.Mock(), mock.Mock(), mock.Mock() + validate(instance=instance, schema={my_property : my_value}) + + def test_it_creates_a_ref_resolver_if_not_provided(self): + self.assertIsInstance(self.validator.resolver, RefResolver) + + def test_it_delegates_to_a_ref_resolver(self): + resolver = RefResolver("", {}) + schema = {"$ref" : mock.Mock()} + + @contextmanager + def resolving(): + yield {"type": "integer"} + + with mock.patch.object(resolver, "resolving") as resolve: + resolve.return_value = resolving() + with self.assertRaises(ValidationError): + self.validator_class(schema, resolver=resolver).validate(None) + + resolve.assert_called_once_with(schema["$ref"]) + + def test_is_type_is_true_for_valid_type(self): + self.assertTrue(self.validator.is_type("foo", "string")) + + def test_is_type_is_false_for_invalid_type(self): + self.assertFalse(self.validator.is_type("foo", "array")) + + def test_is_type_evades_bool_inheriting_from_int(self): + self.assertFalse(self.validator.is_type(True, "integer")) + self.assertFalse(self.validator.is_type(True, "number")) + + def test_is_type_raises_exception_for_unknown_type(self): + with self.assertRaises(UnknownType): + self.validator.is_type("foo", object()) + + +class TestDraft3Validator(ValidatorTestMixin, unittest.TestCase): + validator_class = Draft3Validator + + def test_is_type_is_true_for_any_type(self): + self.assertTrue(self.validator.is_valid(mock.Mock(), {"type": "any"})) + + def test_is_type_does_not_evade_bool_if_it_is_being_tested(self): + self.assertTrue(self.validator.is_type(True, "boolean")) + self.assertTrue(self.validator.is_valid(True, {"type": "any"})) + + def test_non_string_custom_types(self): + schema = {'type': [None]} + cls = self.validator_class(schema, types={None: type(None)}) + cls.validate(None, schema) + + +class TestDraft4Validator(ValidatorTestMixin, unittest.TestCase): + validator_class = Draft4Validator + + +class TestBuiltinFormats(unittest.TestCase): + """ + The built-in (specification-defined) formats do not raise type errors. + + If an instance or value is not a string, it should be ignored. + + """ + + +for format in FormatChecker.checkers: + def test(self, format=format): + v = Draft4Validator({"format": format}, format_checker=FormatChecker()) + v.validate(123) + + name = "test_{0}_ignores_non_strings".format(format) + test.__name__ = name + setattr(TestBuiltinFormats, name, test) + del test # Ugh py.test. Stop discovering top level tests. + + +class TestValidatorFor(unittest.TestCase): + def test_draft_3(self): + schema = {"$schema" : "http://json-schema.org/draft-03/schema"} + self.assertIs(validator_for(schema), Draft3Validator) + + schema = {"$schema" : "http://json-schema.org/draft-03/schema#"} + self.assertIs(validator_for(schema), Draft3Validator) + + def test_draft_4(self): + schema = {"$schema" : "http://json-schema.org/draft-04/schema"} + self.assertIs(validator_for(schema), Draft4Validator) + + schema = {"$schema" : "http://json-schema.org/draft-04/schema#"} + self.assertIs(validator_for(schema), Draft4Validator) + + def test_custom_validator(self): + Validator = create(meta_schema={"id" : "meta schema id"}, version="12") + schema = {"$schema" : "meta schema id"} + self.assertIs(validator_for(schema), Validator) + + def test_validator_for_jsonschema_default(self): + self.assertIs(validator_for({}), Draft4Validator) + + def test_validator_for_custom_default(self): + self.assertIs(validator_for({}, default=None), None) + + +class TestValidate(unittest.TestCase): + def test_draft3_validator_is_chosen(self): + schema = {"$schema" : "http://json-schema.org/draft-03/schema#"} + with mock.patch.object(Draft3Validator, "check_schema") as chk_schema: + validate({}, schema) + chk_schema.assert_called_once_with(schema) + # Make sure it works without the empty fragment + schema = {"$schema" : "http://json-schema.org/draft-03/schema"} + with mock.patch.object(Draft3Validator, "check_schema") as chk_schema: + validate({}, schema) + chk_schema.assert_called_once_with(schema) + + def test_draft4_validator_is_chosen(self): + schema = {"$schema" : "http://json-schema.org/draft-04/schema#"} + with mock.patch.object(Draft4Validator, "check_schema") as chk_schema: + validate({}, schema) + chk_schema.assert_called_once_with(schema) + + def test_draft4_validator_is_the_default(self): + with mock.patch.object(Draft4Validator, "check_schema") as chk_schema: + validate({}, {}) + chk_schema.assert_called_once_with({}) + + +class TestRefResolver(unittest.TestCase): + + base_uri = "" + stored_uri = "foo://stored" + stored_schema = {"stored" : "schema"} + + def setUp(self): + self.referrer = {} + self.store = {self.stored_uri : self.stored_schema} + self.resolver = RefResolver(self.base_uri, self.referrer, self.store) + + def test_it_does_not_retrieve_schema_urls_from_the_network(self): + ref = Draft3Validator.META_SCHEMA["id"] + with mock.patch.object(self.resolver, "resolve_remote") as remote: + with self.resolver.resolving(ref) as resolved: + self.assertEqual(resolved, Draft3Validator.META_SCHEMA) + self.assertFalse(remote.called) + + def test_it_resolves_local_refs(self): + ref = "#/properties/foo" + self.referrer["properties"] = {"foo" : object()} + with self.resolver.resolving(ref) as resolved: + self.assertEqual(resolved, self.referrer["properties"]["foo"]) + + def test_it_resolves_local_refs_with_id(self): + schema = {"id": "foo://bar/schema#", "a": {"foo": "bar"}} + resolver = RefResolver.from_schema(schema) + with resolver.resolving("#/a") as resolved: + self.assertEqual(resolved, schema["a"]) + with resolver.resolving("foo://bar/schema#/a") as resolved: + self.assertEqual(resolved, schema["a"]) + + def test_it_retrieves_stored_refs(self): + with self.resolver.resolving(self.stored_uri) as resolved: + self.assertIs(resolved, self.stored_schema) + + self.resolver.store["cached_ref"] = {"foo" : 12} + with self.resolver.resolving("cached_ref#/foo") as resolved: + self.assertEqual(resolved, 12) + + def test_it_retrieves_unstored_refs_via_requests(self): + ref = "http://bar#baz" + schema = {"baz" : 12} + + with mock.patch("jsonschema.validators.requests") as requests: + requests.get.return_value.json.return_value = schema + with self.resolver.resolving(ref) as resolved: + self.assertEqual(resolved, 12) + requests.get.assert_called_once_with("http://bar") + + def test_it_retrieves_unstored_refs_via_urlopen(self): + ref = "http://bar#baz" + schema = {"baz" : 12} + + with mock.patch("jsonschema.validators.requests", None): + with mock.patch("jsonschema.validators.urlopen") as urlopen: + urlopen.return_value.read.return_value = ( + json.dumps(schema).encode("utf8")) + with self.resolver.resolving(ref) as resolved: + self.assertEqual(resolved, 12) + urlopen.assert_called_once_with("http://bar") + + def test_it_can_construct_a_base_uri_from_a_schema(self): + schema = {"id" : "foo"} + resolver = RefResolver.from_schema(schema) + self.assertEqual(resolver.base_uri, "foo") + with resolver.resolving("") as resolved: + self.assertEqual(resolved, schema) + with resolver.resolving("#") as resolved: + self.assertEqual(resolved, schema) + with resolver.resolving("foo") as resolved: + self.assertEqual(resolved, schema) + with resolver.resolving("foo#") as resolved: + self.assertEqual(resolved, schema) + + def test_it_can_construct_a_base_uri_from_a_schema_without_id(self): + schema = {} + resolver = RefResolver.from_schema(schema) + self.assertEqual(resolver.base_uri, "") + with resolver.resolving("") as resolved: + self.assertEqual(resolved, schema) + with resolver.resolving("#") as resolved: + self.assertEqual(resolved, schema) + + def test_custom_uri_scheme_handlers(self): + schema = {"foo": "bar"} + ref = "foo://bar" + foo_handler = mock.Mock(return_value=schema) + resolver = RefResolver("", {}, handlers={"foo": foo_handler}) + with resolver.resolving(ref) as resolved: + self.assertEqual(resolved, schema) + foo_handler.assert_called_once_with(ref) + + def test_cache_remote_on(self): + ref = "foo://bar" + foo_handler = mock.Mock() + resolver = RefResolver( + "", {}, cache_remote=True, handlers={"foo" : foo_handler}, + ) + with resolver.resolving(ref): + pass + with resolver.resolving(ref): + pass + foo_handler.assert_called_once_with(ref) + + def test_cache_remote_off(self): + ref = "foo://bar" + foo_handler = mock.Mock() + resolver = RefResolver( + "", {}, cache_remote=False, handlers={"foo" : foo_handler}, + ) + with resolver.resolving(ref): + pass + with resolver.resolving(ref): + pass + self.assertEqual(foo_handler.call_count, 2) + + def test_if_you_give_it_junk_you_get_a_resolution_error(self): + ref = "foo://bar" + foo_handler = mock.Mock(side_effect=ValueError("Oh no! What's this?")) + resolver = RefResolver("", {}, handlers={"foo" : foo_handler}) + with self.assertRaises(RefResolutionError) as err: + with resolver.resolving(ref): + pass + self.assertEqual(str(err.exception), "Oh no! What's this?") + + +def sorted_errors(errors): + def key(error): + return ( + [str(e) for e in error.path], + [str(e) for e in error.schema_path] + ) + return sorted(errors, key=key) diff --git a/lib/spack/external/jsonschema/validators.py b/lib/spack/external/jsonschema/validators.py new file mode 100644 index 0000000000..3e326844f4 --- /dev/null +++ b/lib/spack/external/jsonschema/validators.py @@ -0,0 +1,428 @@ +from __future__ import division + +import contextlib +import json +import numbers + +try: + import requests +except ImportError: + requests = None + +from jsonschema import _utils, _validators +from jsonschema.compat import ( + Sequence, urljoin, urlsplit, urldefrag, unquote, urlopen, + str_types, int_types, iteritems, +) +from jsonschema.exceptions import ErrorTree # Backwards compatibility # noqa +from jsonschema.exceptions import RefResolutionError, SchemaError, UnknownType + + +_unset = _utils.Unset() + +validators = {} +meta_schemas = _utils.URIDict() + + +def validates(version): + """ + Register the decorated validator for a ``version`` of the specification. + + Registered validators and their meta schemas will be considered when + parsing ``$schema`` properties' URIs. + + :argument str version: an identifier to use as the version's name + :returns: a class decorator to decorate the validator with the version + + """ + + def _validates(cls): + validators[version] = cls + if u"id" in cls.META_SCHEMA: + meta_schemas[cls.META_SCHEMA[u"id"]] = cls + return cls + return _validates + + +def create(meta_schema, validators=(), version=None, default_types=None): # noqa + if default_types is None: + default_types = { + u"array" : list, u"boolean" : bool, u"integer" : int_types, + u"null" : type(None), u"number" : numbers.Number, u"object" : dict, + u"string" : str_types, + } + + class Validator(object): + VALIDATORS = dict(validators) + META_SCHEMA = dict(meta_schema) + DEFAULT_TYPES = dict(default_types) + + def __init__( + self, schema, types=(), resolver=None, format_checker=None, + ): + self._types = dict(self.DEFAULT_TYPES) + self._types.update(types) + + if resolver is None: + resolver = RefResolver.from_schema(schema) + + self.resolver = resolver + self.format_checker = format_checker + self.schema = schema + + @classmethod + def check_schema(cls, schema): + for error in cls(cls.META_SCHEMA).iter_errors(schema): + raise SchemaError.create_from(error) + + def iter_errors(self, instance, _schema=None): + if _schema is None: + _schema = self.schema + + with self.resolver.in_scope(_schema.get(u"id", u"")): + ref = _schema.get(u"$ref") + if ref is not None: + validators = [(u"$ref", ref)] + else: + validators = iteritems(_schema) + + for k, v in validators: + validator = self.VALIDATORS.get(k) + if validator is None: + continue + + errors = validator(self, v, instance, _schema) or () + for error in errors: + # set details if not already set by the called fn + error._set( + validator=k, + validator_value=v, + instance=instance, + schema=_schema, + ) + if k != u"$ref": + error.schema_path.appendleft(k) + yield error + + def descend(self, instance, schema, path=None, schema_path=None): + for error in self.iter_errors(instance, schema): + if path is not None: + error.path.appendleft(path) + if schema_path is not None: + error.schema_path.appendleft(schema_path) + yield error + + def validate(self, *args, **kwargs): + for error in self.iter_errors(*args, **kwargs): + raise error + + def is_type(self, instance, type): + if type not in self._types: + raise UnknownType(type, instance, self.schema) + pytypes = self._types[type] + + # bool inherits from int, so ensure bools aren't reported as ints + if isinstance(instance, bool): + pytypes = _utils.flatten(pytypes) + is_number = any( + issubclass(pytype, numbers.Number) for pytype in pytypes + ) + if is_number and bool not in pytypes: + return False + return isinstance(instance, pytypes) + + def is_valid(self, instance, _schema=None): + error = next(self.iter_errors(instance, _schema), None) + return error is None + + if version is not None: + Validator = validates(version)(Validator) + Validator.__name__ = version.title().replace(" ", "") + "Validator" + + return Validator + + +def extend(validator, validators, version=None): + all_validators = dict(validator.VALIDATORS) + all_validators.update(validators) + return create( + meta_schema=validator.META_SCHEMA, + validators=all_validators, + version=version, + default_types=validator.DEFAULT_TYPES, + ) + + +Draft3Validator = create( + meta_schema=_utils.load_schema("draft3"), + validators={ + u"$ref" : _validators.ref, + u"additionalItems" : _validators.additionalItems, + u"additionalProperties" : _validators.additionalProperties, + u"dependencies" : _validators.dependencies, + u"disallow" : _validators.disallow_draft3, + u"divisibleBy" : _validators.multipleOf, + u"enum" : _validators.enum, + u"extends" : _validators.extends_draft3, + u"format" : _validators.format, + u"items" : _validators.items, + u"maxItems" : _validators.maxItems, + u"maxLength" : _validators.maxLength, + u"maximum" : _validators.maximum, + u"minItems" : _validators.minItems, + u"minLength" : _validators.minLength, + u"minimum" : _validators.minimum, + u"multipleOf" : _validators.multipleOf, + u"pattern" : _validators.pattern, + u"patternProperties" : _validators.patternProperties, + u"properties" : _validators.properties_draft3, + u"type" : _validators.type_draft3, + u"uniqueItems" : _validators.uniqueItems, + }, + version="draft3", +) + +Draft4Validator = create( + meta_schema=_utils.load_schema("draft4"), + validators={ + u"$ref" : _validators.ref, + u"additionalItems" : _validators.additionalItems, + u"additionalProperties" : _validators.additionalProperties, + u"allOf" : _validators.allOf_draft4, + u"anyOf" : _validators.anyOf_draft4, + u"dependencies" : _validators.dependencies, + u"enum" : _validators.enum, + u"format" : _validators.format, + u"items" : _validators.items, + u"maxItems" : _validators.maxItems, + u"maxLength" : _validators.maxLength, + u"maxProperties" : _validators.maxProperties_draft4, + u"maximum" : _validators.maximum, + u"minItems" : _validators.minItems, + u"minLength" : _validators.minLength, + u"minProperties" : _validators.minProperties_draft4, + u"minimum" : _validators.minimum, + u"multipleOf" : _validators.multipleOf, + u"not" : _validators.not_draft4, + u"oneOf" : _validators.oneOf_draft4, + u"pattern" : _validators.pattern, + u"patternProperties" : _validators.patternProperties, + u"properties" : _validators.properties_draft4, + u"required" : _validators.required_draft4, + u"type" : _validators.type_draft4, + u"uniqueItems" : _validators.uniqueItems, + }, + version="draft4", +) + + +class RefResolver(object): + """ + Resolve JSON References. + + :argument str base_uri: URI of the referring document + :argument referrer: the actual referring document + :argument dict store: a mapping from URIs to documents to cache + :argument bool cache_remote: whether remote refs should be cached after + first resolution + :argument dict handlers: a mapping from URI schemes to functions that + should be used to retrieve them + + """ + + def __init__( + self, base_uri, referrer, store=(), cache_remote=True, handlers=(), + ): + self.base_uri = base_uri + self.resolution_scope = base_uri + # This attribute is not used, it is for backwards compatibility + self.referrer = referrer + self.cache_remote = cache_remote + self.handlers = dict(handlers) + + self.store = _utils.URIDict( + (id, validator.META_SCHEMA) + for id, validator in iteritems(meta_schemas) + ) + self.store.update(store) + self.store[base_uri] = referrer + + @classmethod + def from_schema(cls, schema, *args, **kwargs): + """ + Construct a resolver from a JSON schema object. + + :argument schema schema: the referring schema + :rtype: :class:`RefResolver` + + """ + + return cls(schema.get(u"id", u""), schema, *args, **kwargs) + + @contextlib.contextmanager + def in_scope(self, scope): + old_scope = self.resolution_scope + self.resolution_scope = urljoin(old_scope, scope) + try: + yield + finally: + self.resolution_scope = old_scope + + @contextlib.contextmanager + def resolving(self, ref): + """ + Context manager which resolves a JSON ``ref`` and enters the + resolution scope of this ref. + + :argument str ref: reference to resolve + + """ + + full_uri = urljoin(self.resolution_scope, ref) + uri, fragment = urldefrag(full_uri) + if not uri: + uri = self.base_uri + + if uri in self.store: + document = self.store[uri] + else: + try: + document = self.resolve_remote(uri) + except Exception as exc: + raise RefResolutionError(exc) + + old_base_uri, self.base_uri = self.base_uri, uri + try: + with self.in_scope(uri): + yield self.resolve_fragment(document, fragment) + finally: + self.base_uri = old_base_uri + + def resolve_fragment(self, document, fragment): + """ + Resolve a ``fragment`` within the referenced ``document``. + + :argument document: the referrant document + :argument str fragment: a URI fragment to resolve within it + + """ + + fragment = fragment.lstrip(u"/") + parts = unquote(fragment).split(u"/") if fragment else [] + + for part in parts: + part = part.replace(u"~1", u"/").replace(u"~0", u"~") + + if isinstance(document, Sequence): + # Array indexes should be turned into integers + try: + part = int(part) + except ValueError: + pass + try: + document = document[part] + except (TypeError, LookupError): + raise RefResolutionError( + "Unresolvable JSON pointer: %r" % fragment + ) + + return document + + def resolve_remote(self, uri): + """ + Resolve a remote ``uri``. + + Does not check the store first, but stores the retrieved document in + the store if :attr:`RefResolver.cache_remote` is True. + + .. note:: + + If the requests_ library is present, ``jsonschema`` will use it to + request the remote ``uri``, so that the correct encoding is + detected and used. + + If it isn't, or if the scheme of the ``uri`` is not ``http`` or + ``https``, UTF-8 is assumed. + + :argument str uri: the URI to resolve + :returns: the retrieved document + + .. _requests: http://pypi.python.org/pypi/requests/ + + """ + + scheme = urlsplit(uri).scheme + + if scheme in self.handlers: + result = self.handlers[scheme](uri) + elif ( + scheme in [u"http", u"https"] and + requests and + getattr(requests.Response, "json", None) is not None + ): + # Requests has support for detecting the correct encoding of + # json over http + if callable(requests.Response.json): + result = requests.get(uri).json() + else: + result = requests.get(uri).json + else: + # Otherwise, pass off to urllib and assume utf-8 + result = json.loads(urlopen(uri).read().decode("utf-8")) + + if self.cache_remote: + self.store[uri] = result + return result + + +def validator_for(schema, default=_unset): + if default is _unset: + default = Draft4Validator + return meta_schemas.get(schema.get(u"$schema", u""), default) + + +def validate(instance, schema, cls=None, *args, **kwargs): + """ + Validate an instance under the given schema. + + >>> validate([2, 3, 4], {"maxItems" : 2}) + Traceback (most recent call last): + ... + ValidationError: [2, 3, 4] is too long + + :func:`validate` will first verify that the provided schema is itself + valid, since not doing so can lead to less obvious error messages and fail + in less obvious or consistent ways. If you know you have a valid schema + already or don't care, you might prefer using the + :meth:`~IValidator.validate` method directly on a specific validator + (e.g. :meth:`Draft4Validator.validate`). + + + :argument instance: the instance to validate + :argument schema: the schema to validate with + :argument cls: an :class:`IValidator` class that will be used to validate + the instance. + + If the ``cls`` argument is not provided, two things will happen in + accordance with the specification. First, if the schema has a + :validator:`$schema` property containing a known meta-schema [#]_ then the + proper validator will be used. The specification recommends that all + schemas contain :validator:`$schema` properties for this reason. If no + :validator:`$schema` property is found, the default validator class is + :class:`Draft4Validator`. + + Any other provided positional and keyword arguments will be passed on when + instantiating the ``cls``. + + :raises: + :exc:`ValidationError` if the instance is invalid + + :exc:`SchemaError` if the schema itself is invalid + + .. rubric:: Footnotes + .. [#] known by a validator registered with :func:`validates` + """ + if cls is None: + cls = validator_for(schema) + cls.check_schema(schema) + cls(schema, *args, **kwargs).validate(instance) diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index 6c4a15aaab..ab78ecef30 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -59,9 +59,8 @@ import spack.repository try: repo = spack.repository.RepoPath() sys.meta_path.append(repo) -except spack.repository.RepoError, e: - tty.error('while initializing Spack RepoPath:') - tty.die(e.message) +except spack.error.SpackError, e: + tty.die('while initializing Spack RepoPath:', e.message) # # Set up the installed packages database diff --git a/lib/spack/spack/cmd/repo.py b/lib/spack/spack/cmd/repo.py index ebe42d0138..a792f04cfd 100644 --- a/lib/spack/spack/cmd/repo.py +++ b/lib/spack/spack/cmd/repo.py @@ -118,9 +118,8 @@ def repo_create(args): shutil.rmtree(root, ignore_errors=True) tty.msg("Created repo with namespace '%s'." % namespace) - tty.msg("To register it with Spack, add a line like this to ~/.spack/repos.yaml:", - 'repos:', - ' - ' + full_path) + tty.msg("To register it with spack, run this command:", + 'spack repo add %s' % full_path) def repo_add(args): diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 3ff83ae529..e57aa75f8e 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -118,9 +118,11 @@ the site configuration will be ignored. """ import os +import re import sys import copy - +import jsonschema +from jsonschema import Draft4Validator, validators import yaml from yaml.error import MarkedYAMLError from ordereddict_backport import OrderedDict @@ -137,32 +139,62 @@ import spack.util.spack_yaml as syaml """Dict from section names -> schema for that section.""" section_schemas = { - 'compilers' : { + 'compilers': { '$schema': 'http://json-schema.org/schema#', - 'title' : 'Spack compiler configuration file schema', - 'type' : 'object', - 'properties' : { - 'compilers' : { - 'type' : 'map', - }, - }, - }, - - 'mirrors' : { + 'title': 'Spack compiler configuration file schema', + 'type': 'object', + 'additionalProperties': False, + 'patternProperties': { + 'compilers:?': { # optional colon for overriding site config. + 'type': 'object', + 'default': {}, + 'additionalProperties': False, + 'patternProperties': { + r'\w[\w-]*': { # architecture + 'type': 'object', + 'additionalProperties': False, + 'patternProperties': { + r'\w[\w-]*@\w[\w-]*': { # compiler spec + 'type': 'object', + 'additionalProperties': False, + 'required': ['cc', 'cxx', 'f77', 'fc'], + 'properties': { + 'cc': { 'anyOf': [ {'type' : 'string' }, + {'type' : 'null' }]}, + 'cxx': { 'anyOf': [ {'type' : 'string' }, + {'type' : 'null' }]}, + 'f77': { 'anyOf': [ {'type' : 'string' }, + {'type' : 'null' }]}, + 'fc': { 'anyOf': [ {'type' : 'string' }, + {'type' : 'null' }]}, + },},},},},},},}, + + 'mirrors': { '$schema': 'http://json-schema.org/schema#', - 'title' : 'Spack mirror configuration file schema', - 'type' : 'map', - 'properties' : { - 'mirrors' : { - - } - }, - }, - - 'repos' : { + 'title': 'Spack mirror configuration file schema', + 'type': 'object', + 'additionalProperties': False, + 'patternProperties': { + r'mirrors:?': { + 'type': 'object', + 'default': {}, + 'additionalProperties': False, + 'patternProperties': { + r'\w[\w-]*': { + 'type': 'string'},},},},}, + + 'repos': { '$schema': 'http://json-schema.org/schema#', - 'title' : 'Spack repository configuration file schema', - }} + 'title': 'Spack repository configuration file schema', + 'type': 'object', + 'additionalProperties': False, + 'patternProperties': { + r'repos:?': { + 'type': 'array', + 'default': [], + 'items': { + 'type': 'string'},},},}, +} """OrderedDict of config scopes keyed by name. Later scopes will override earlier scopes. @@ -170,13 +202,64 @@ section_schemas = { config_scopes = OrderedDict() -def validate_section(section): +def validate_section_name(section): """Raise a ValueError if the section is not a valid section.""" if section not in section_schemas: raise ValueError("Invalid config section: '%s'. Options are %s." % (section, section_schemas)) +def extend_with_default(validator_class): + """Add support for the 'default' attribute for properties and patternProperties. + + jsonschema does not handle this out of the box -- it only + validates. This allows us to set default values for configs + where certain fields are `None` b/c they're deleted or + commented out. + + """ + validate_properties = validator_class.VALIDATORS["properties"] + validate_pattern_properties = validator_class.VALIDATORS["patternProperties"] + + def set_defaults(validator, properties, instance, schema): + for property, subschema in properties.iteritems(): + if "default" in subschema: + instance.setdefault(property, subschema["default"]) + for err in validate_properties(validator, properties, instance, schema): + yield err + + def set_pp_defaults(validator, properties, instance, schema): + for property, subschema in properties.iteritems(): + if "default" in subschema: + if isinstance(instance, dict): + for key, val in instance.iteritems(): + if re.match(property, key) and val is None: + instance[key] = subschema["default"] + + for err in validate_pattern_properties(validator, properties, instance, schema): + yield err + + return validators.extend(validator_class, { + "properties" : set_defaults, + "patternProperties" : set_pp_defaults + }) + + +DefaultSettingValidator = extend_with_default(Draft4Validator) + +def validate_section(data, schema): + """Validate data read in from a Spack YAML file. + + This leverages the line information (start_mark, end_mark) stored + on Spack YAML structures. + + """ + try: + DefaultSettingValidator(schema).validate(data) + except jsonschema.ValidationError as e: + raise ConfigFormatError(e, data) + + class ConfigScope(object): """This class represents a configuration scope. @@ -195,18 +278,16 @@ class ConfigScope(object): config_scopes[name] = self def get_section_filename(self, section): - validate_section(section) + validate_section_name(section) return os.path.join(self.path, "%s.yaml" % section) def get_section(self, section): if not section in self.sections: - path = self.get_section_filename(section) - data = _read_config_file(path) - if data is None: - self.sections[section] = {} - else: - self.sections[section] = data + path = self.get_section_filename(section) + schema = section_schemas[section] + data = _read_config_file(path, schema) + self.sections[section] = data return self.sections[section] @@ -255,7 +336,7 @@ def validate_scope(scope): % (scope, config_scopes.keys())) -def _read_config_file(filename): +def _read_config_file(filename, schema): """Read a YAML configuration file.""" # Ignore nonexisting files. if not os.path.exists(filename): @@ -269,8 +350,13 @@ def _read_config_file(filename): raise ConfigFileError("Config file is not readable: %s." % filename) try: + tty.debug("Reading config file %s" % filename) with open(filename) as f: - return syaml.load(f) + data = syaml.load(f) + + validate_section(data, schema) + + return data except MarkedYAMLError, e: raise ConfigFileError( @@ -337,7 +423,7 @@ def get_config(section, scope=None): Strips off the top-level section name from the YAML dict. """ - validate_section(section) + validate_section_name(section) merged_section = syaml.syaml_dict() if scope is None: @@ -387,7 +473,7 @@ def update_config(section, update_data, scope=None): # read in the config to ensure we've got current data get_config(section) - validate_section(section) # validate section name + validate_section_name(section) # validate section name scope = validate_scope(scope) # get ConfigScope object from string. # read only the requested section's data. @@ -407,4 +493,43 @@ def print_section(section): class ConfigError(SpackError): pass class ConfigFileError(ConfigError): pass -class ConfigFormatError(ConfigError): pass + +def get_path(path, data): + if path: + return get_path(path[1:], data[path[0]]) + else: + return data + +class ConfigFormatError(ConfigError): + """Raised when a configuration format does not match its schema.""" + def __init__(self, validation_error, data): + # Try to get line number from erroneous instance and its parent + instance_mark = getattr(validation_error.instance, '_start_mark', None) + parent_mark = getattr(validation_error.parent, '_start_mark', None) + path = getattr(validation_error, 'path', None) + + # Try really hard to get the parent (which sometimes is not + # set) This digs it out of the validated structure if it's not + # on the validation_error. + if not parent_mark: + parent_path = list(path)[:-1] + parent = get_path(parent_path, data) + if path[-1] in parent: + if isinstance(parent, dict): + keylist = parent.keys() + elif isinstance(parent, list): + keylist = parent + idx = keylist.index(path[-1]) + parent_mark = getattr(keylist[idx], '_start_mark', None) + + if instance_mark: + location = '%s:%d' % (instance_mark.name, instance_mark.line + 1) + elif parent_mark: + location = '%s:%d' % (parent_mark.name, parent_mark.line + 1) + elif path: + location = 'At ' + ':'.join(path) + else: + location = '' + + message = '%s: %s' % (location, validation_error.message) + super(ConfigError, self).__init__(message) -- cgit v1.2.3-70-g09d2 From d869d8e86fcabfd04121806c8c1554a00cc5339c Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Thu, 14 Jan 2016 12:00:33 -0800 Subject: add sanity check on config write as well as validation on read. --- lib/spack/spack/config.py | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'lib') diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index e57aa75f8e..193b311434 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -297,7 +297,10 @@ class ConfigScope(object): try: mkdirp(self.path) with open(filename, 'w') as f: + validate_section(data, section_schemas[section]) syaml.dump(data, stream=f, default_flow_style=False) + except jsonschema.ValidationError as e: + raise ConfigSanityError(e, data) except (yaml.YAMLError, IOError) as e: raise ConfigFileError("Error writing to config file: '%s'" % str(e)) @@ -533,3 +536,6 @@ class ConfigFormatError(ConfigError): message = '%s: %s' % (location, validation_error.message) super(ConfigError, self).__init__(message) + +class ConfigSanityError(ConfigFormatError): + """Same as ConfigFormatError, raised when config is written by Spack.""" -- cgit v1.2.3-70-g09d2 From aa00999fcacf6ba19987388c3998cdd4d1c6e65c Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Thu, 14 Jan 2016 12:00:56 -0800 Subject: update compiler config test. --- lib/spack/spack/test/config.py | 70 ++++++++++++++++++++++-------------------- 1 file changed, 37 insertions(+), 33 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/test/config.py b/lib/spack/spack/test/config.py index 5f99dcb903..d8be5a855b 100644 --- a/lib/spack/spack/test/config.py +++ b/lib/spack/spack/test/config.py @@ -33,39 +33,43 @@ from spack.test.mock_packages_test import * # Some sample compiler config data a_comps = { - "gcc@4.7.3" : { - "cc" : "/gcc473", - "cxx" : "/g++473", - "f77" : None, - "f90" : None }, - "gcc@4.5.0" : { - "cc" : "/gcc450", - "cxx" : "/g++450", - "f77" : "/gfortran", - "f90" : "/gfortran" }, - "clang@3.3" : { - "cc" : "", - "cxx" : "", - "f77" : "", - "f90" : "" } + "all": { + "gcc@4.7.3" : { + "cc" : "/gcc473", + "cxx": "/g++473", + "f77": None, + "fc" : None }, + "gcc@4.5.0" : { + "cc" : "/gcc450", + "cxx": "/g++450", + "f77": "/gfortran", + "fc" : "/gfortran" }, + "clang@3.3" : { + "cc" : "", + "cxx": "", + "f77": "", + "fc" : "" } + } } b_comps = { - "icc@10.0" : { - "cc" : "/icc100", - "cxx" : "/icc100", - "f77" : None, - "f90" : None }, - "icc@11.1" : { - "cc" : "/icc111", - "cxx" : "/icp111", - "f77" : "/ifort", - "f90" : "/ifort" }, - "clang@3.3" : { - "cc" : "/clang", - "cxx" : "/clang++", - "f77" : None, - "f90" : None} + "all": { + "icc@10.0" : { + "cc" : "/icc100", + "cxx": "/icc100", + "f77": None, + "fc" : None }, + "icc@11.1" : { + "cc" : "/icc111", + "cxx": "/icp111", + "f77": "/ifort", + "fc" : "/ifort" }, + "clang@3.3" : { + "cc" : "/clang", + "cxx": "/clang++", + "f77": None, + "fc" : None} + } } class ConfigTest(MockPackagesTest): @@ -85,11 +89,11 @@ class ConfigTest(MockPackagesTest): def check_config(self, comps, *compiler_names): """Check that named compilers in comps match Spack's config.""" config = spack.config.get_config('compilers') - compiler_list = ['cc', 'cxx', 'f77', 'f90'] + compiler_list = ['cc', 'cxx', 'f77', 'fc'] for key in compiler_names: for c in compiler_list: - expected = comps[key][c] - actual = config[key][c] + expected = comps['all'][key][c] + actual = config['all'][key][c] self.assertEqual(expected, actual) -- cgit v1.2.3-70-g09d2 From 358b2ab4ba9d08bc47b59bf39811b6472acbc53a Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 17 Jan 2016 18:12:08 -0800 Subject: Temporary fix: Clang is the default compiler on Mac OS X. --- lib/spack/spack/compilers/__init__.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index 2a3d67c731..facc9c338b 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -27,6 +27,7 @@ system and configuring Spack to use multiple compilers. """ import imp import os +import platform from llnl.util.lang import memoized, list_modules from llnl.util.filesystem import join_path @@ -47,7 +48,11 @@ _imported_compilers_module = 'spack.compilers' _required_instance_vars = ['cc', 'cxx', 'f77', 'fc'] # TODO: customize order in config file -_default_order = ['gcc', 'intel', 'pgi', 'clang', 'xlc'] +if platform.system() == 'Darwin': + _default_order = ['clang', 'gcc', 'intel'] +else: + _default_order = ['gcc', 'intel', 'pgi', 'clang', 'xlc'] + def _auto_compiler_spec(function): def converter(cspec_like, *args, **kwargs): -- cgit v1.2.3-70-g09d2 From 5984bc2ad3b3d161ac8e8c37ec1dc3dfa9250241 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 17 Jan 2016 18:12:53 -0800 Subject: Add namespace option to find command. --- lib/spack/spack/cmd/find.py | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/cmd/find.py b/lib/spack/spack/cmd/find.py index d951c37fe0..714f1d514b 100644 --- a/lib/spack/spack/cmd/find.py +++ b/lib/spack/spack/cmd/find.py @@ -40,6 +40,9 @@ description ="Find installed spack packages" def setup_parser(subparser): format_group = subparser.add_mutually_exclusive_group() + format_group.add_argument( + '-s', '--short', action='store_const', dest='mode', const='short', + help='Show only specs (default)') format_group.add_argument( '-p', '--paths', action='store_const', dest='mode', const='paths', help='Show paths to package install directories') @@ -48,21 +51,24 @@ def setup_parser(subparser): help='Show full dependency DAG of installed packages') subparser.add_argument( - '-l', '--long', action='store_true', dest='long', + '-l', '--long', action='store_true', help='Show dependency hashes as well as versions.') subparser.add_argument( - '-L', '--very-long', action='store_true', dest='very_long', + '-L', '--very-long', action='store_true', help='Show dependency hashes as well as versions.') subparser.add_argument( - '-u', '--unknown', action='store_true', dest='unknown', + '-u', '--unknown', action='store_true', help='Show only specs Spack does not have a package for.') subparser.add_argument( - '-m', '--missing', action='store_true', dest='missing', + '-m', '--missing', action='store_true', help='Show missing dependencies as well as installed specs.') subparser.add_argument( - '-M', '--only-missing', action='store_true', dest='only_missing', + '-M', '--only-missing', action='store_true', help='Show only missing dependencies.') + subparser.add_argument( + '-N', '--namespace', action='store_true', + help='Show fully qualified package names.') subparser.add_argument( 'query_specs', nargs=argparse.REMAINDER, @@ -76,6 +82,7 @@ def gray_hash(spec, length): def display_specs(specs, **kwargs): mode = kwargs.get('mode', 'short') hashes = kwargs.get('long', False) + namespace = kwargs.get('namespace', False) hlen = 7 if kwargs.get('very_long', False): @@ -97,7 +104,8 @@ def display_specs(specs, **kwargs): specs = index[(architecture,compiler)] specs.sort() - abbreviated = [s.format('$_$@$+', color=True) for s in specs] + nfmt = '.' if namespace else '_' + abbreviated = [s.format('$%s$@$+' % nfmt, color=True) for s in specs] if mode == 'paths': # Print one spec per line along with prefix path width = max(len(s) for s in abbreviated) @@ -112,7 +120,7 @@ def display_specs(specs, **kwargs): elif mode == 'deps': for spec in specs: print spec.tree( - format='$_$@$+', + format='$%s$@$+' % nfmt, color=True, indent=4, prefix=(lambda s: gray_hash(s, hlen)) if hashes else None) @@ -122,7 +130,7 @@ def display_specs(specs, **kwargs): string = "" if hashes: string += gray_hash(s, hlen) + ' ' - string += s.format('$-_$@$+', color=True) + string += s.format('$-%s$@$+' % nfmt, color=True) return string colify(fmt(s) for s in specs) @@ -171,4 +179,5 @@ def find(parser, args): tty.msg("%d installed packages." % len(specs)) display_specs(specs, mode=args.mode, long=args.long, - very_long=args.very_long) + very_long=args.very_long, + namespace=args.namespace) -- cgit v1.2.3-70-g09d2 From 97b492756acce93dbd5f1c305504f07df7582ba0 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 17 Jan 2016 18:14:35 -0800 Subject: Fix create, diy, edit, and repo commands to use multiple repos. --- lib/spack/spack/cmd/create.py | 117 +++++++++++++++++++++++++++++++---------- lib/spack/spack/cmd/diy.py | 2 +- lib/spack/spack/cmd/edit.py | 35 ++++++++---- lib/spack/spack/cmd/repo.py | 57 ++++++++++---------- lib/spack/spack/repository.py | 103 ++++++++++++++++++++++++++++-------- lib/spack/spack/util/naming.py | 25 ++++++++- 6 files changed, 247 insertions(+), 92 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/cmd/create.py b/lib/spack/spack/cmd/create.py index 1a60875de8..7cea39cb55 100644 --- a/lib/spack/spack/cmd/create.py +++ b/lib/spack/spack/cmd/create.py @@ -36,7 +36,9 @@ import spack.cmd import spack.cmd.checksum import spack.url import spack.util.web +from spack.spec import Spec from spack.util.naming import * +from spack.repository import Repo, RepoError import spack.util.crypto as crypto from spack.util.executable import which @@ -85,21 +87,34 @@ ${versions} """) +def make_version_calls(ver_hash_tuples): + """Adds a version() call to the package for each version found.""" + max_len = max(len(str(v)) for v, h in ver_hash_tuples) + format = " version(%%-%ds, '%%s')" % (max_len + 2) + return '\n'.join(format % ("'%s'" % v, h) for v, h in ver_hash_tuples) + + def setup_parser(subparser): subparser.add_argument('url', nargs='?', help="url of package archive") subparser.add_argument( - '--keep-stage', action='store_true', dest='keep_stage', + '--keep-stage', action='store_true', help="Don't clean up staging area when command completes.") subparser.add_argument( - '-n', '--name', dest='alternate_name', default=None, + '-n', '--name', dest='alternate_name', default=None, metavar='NAME', help="Override the autodetected name for the created package.") subparser.add_argument( - '-p', '--package-repo', dest='package_repo', default=None, - help="Create the package in the specified packagerepo.") + '-r', '--repo', default=None, + help="Path to a repository where the package should be created.") + subparser.add_argument( + '-N', '--namespace', + help="Specify a namespace for the package. Must be the namespace of " + "a repository registered with Spack.") subparser.add_argument( '-f', '--force', action='store_true', dest='force', help="Overwrite any existing package file with the same name.") + setup_parser.subparser = subparser + class ConfigureGuesser(object): def __call__(self, stage): @@ -137,16 +152,7 @@ class ConfigureGuesser(object): self.build_system = build_system -def make_version_calls(ver_hash_tuples): - """Adds a version() call to the package for each version found.""" - max_len = max(len(str(v)) for v, h in ver_hash_tuples) - format = " version(%%-%ds, '%%s')" % (max_len + 2) - return '\n'.join(format % ("'%s'" % v, h) for v, h in ver_hash_tuples) - - -def create(parser, args): - url = args.url - +def guess_name_and_version(url, args): # Try to deduce name and version of the new package from the URL version = spack.url.parse_version(url) if not version: @@ -163,21 +169,52 @@ def create(parser, args): tty.die("Couldn't guess a name for this package. Try running:", "", "spack create --name ") - package_repo = args.package_repo - - if not valid_module_name(name): + if not valid_fully_qualified_module_name(name): tty.die("Package name can only contain A-Z, a-z, 0-9, '_' and '-'") - tty.msg("This looks like a URL for %s version %s." % (name, version)) - tty.msg("Creating template for package %s" % name) + return name, version - # Create a directory for the new package. - pkg_path = spack.repo.filename_for_package_name(name, package_repo) - if os.path.exists(pkg_path) and not args.force: - tty.die("%s already exists." % pkg_path) + +def find_repository(spec, args): + # figure out namespace for spec + if spec.namespace and args.namespace and spec.namespace != args.namespace: + tty.die("Namespaces '%s' and '%s' do not match." % (spec.namespace, args.namespace)) + + if not spec.namespace and args.namespace: + spec.namespace = args.namespace + + # Figure out where the new package should live. + repo_path = args.repo + if repo_path is not None: + try: + repo = Repo(repo_path) + if spec.namespace and spec.namespace != repo.namespace: + tty.die("Can't create package with namespace %s in repo with namespace %s." + % (spec.namespace, repo.namespace)) + except RepoError as e: + tty.die(str(e)) else: - mkdirp(os.path.dirname(pkg_path)) + if spec.namespace: + repo = spack.repo.get_repo(spec.namespace, None) + if not repo: + tty.die("Unknown namespace: %s" % spec.namespace) + else: + repo = spack.repo.first_repo() + + # Set the namespace on the spec if it's not there already + if not spec.namespace: + spec.namespace = repo.namespace + + return repo + + +def fetch_tarballs(url, name, args): + """Try to find versions of the supplied archive by scraping the web. + + Prompts the user to select how many to download if many are found. + + """ versions = spack.util.web.find_versions_of_archive(url) rkeys = sorted(versions.keys(), reverse=True) versions = OrderedDict(zip(rkeys, (versions[v] for v in rkeys))) @@ -196,13 +233,35 @@ def create(parser, args): default=5, abort='q') if not archives_to_fetch: - tty.msg("Aborted.") - return + tty.die("Aborted.") + + sorted_versions = sorted(versions.keys(), reverse=True) + sorted_urls = [versions[v] for v in sorted_versions] + return sorted_versions[:archives_to_fetch], sorted_urls[:archives_to_fetch] + + +def create(parser, args): + url = args.url + if not url: + setup_parser.subparser.print_help() + return + + # Figure out a name and repo for the package. + name, version = guess_name_and_version(url, args) + spec = Spec(name) + name = spec.name # factors out namespace, if any + repo = find_repository(spec, args) + + tty.msg("This looks like a URL for %s version %s." % (name, version)) + tty.msg("Creating template for package %s" % name) + + # Fetch tarballs (prompting user if necessary) + versions, urls = fetch_tarballs(url, name, args) + # Try to guess what configure system is used. guesser = ConfigureGuesser() ver_hash_tuples = spack.cmd.checksum.get_checksums( - versions.keys()[:archives_to_fetch], - [versions[v] for v in versions.keys()[:archives_to_fetch]], + versions, urls, first_stage_function=guesser, keep_stage=args.keep_stage) @@ -214,7 +273,7 @@ def create(parser, args): name = 'py-%s' % name # Create a directory for the new package. - pkg_path = spack.repo.filename_for_package_name(name) + pkg_path = repo.filename_for_package_name(name) if os.path.exists(pkg_path) and not args.force: tty.die("%s already exists." % pkg_path) else: diff --git a/lib/spack/spack/cmd/diy.py b/lib/spack/spack/cmd/diy.py index 1acbebbc15..9df53312f8 100644 --- a/lib/spack/spack/cmd/diy.py +++ b/lib/spack/spack/cmd/diy.py @@ -69,7 +69,7 @@ def diy(self, args): sys.exit(1) else: tty.msg("Running 'spack edit -f %s'" % spec.name) - edit_package(spec.name, True) + edit_package(spec.name, spack.repo.first_repo(), None, True) return if not spec.version.concrete: diff --git a/lib/spack/spack/cmd/edit.py b/lib/spack/spack/cmd/edit.py index e0688dc96b..a20e40df9b 100644 --- a/lib/spack/spack/cmd/edit.py +++ b/lib/spack/spack/cmd/edit.py @@ -30,6 +30,8 @@ from llnl.util.filesystem import mkdirp, join_path import spack import spack.cmd +from spack.spec import Spec +from spack.repository import Repo from spack.util.naming import mod_to_class description = "Open package files in $EDITOR" @@ -53,9 +55,16 @@ class ${class_name}(Package): """) -def edit_package(name, force=False): - path = spack.repo.filename_for_package_name(name) +def edit_package(name, repo_path, namespace, force=False): + if repo_path: + repo = Repo(repo_path) + elif namespace: + repo = spack.repo.get_repo(namespace) + else: + repo = spack.repo + path = repo.filename_for_package_name(name) + spec = Spec(name) if os.path.exists(path): if not os.path.isfile(path): tty.die("Something's wrong. '%s' is not a file!" % path) @@ -63,13 +72,13 @@ def edit_package(name, force=False): tty.die("Insufficient permissions on '%s'!" % path) elif not force: tty.die("No package '%s'. Use spack create, or supply -f/--force " - "to edit a new file." % name) + "to edit a new file." % spec.name) else: mkdirp(os.path.dirname(path)) with open(path, "w") as pkg_file: pkg_file.write( package_template.substitute( - name=name, class_name=mod_to_class(name))) + name=spec.name, class_name=mod_to_class(spec.name))) spack.editor(path) @@ -79,17 +88,25 @@ def setup_parser(subparser): '-f', '--force', dest='force', action='store_true', help="Open a new file in $EDITOR even if package doesn't exist.") - filetypes = subparser.add_mutually_exclusive_group() - filetypes.add_argument( + excl_args = subparser.add_mutually_exclusive_group() + + # Various filetypes you can edit directly from the cmd line. + excl_args.add_argument( '-c', '--command', dest='path', action='store_const', const=spack.cmd.command_path, help="Edit the command with the supplied name.") - filetypes.add_argument( + excl_args.add_argument( '-t', '--test', dest='path', action='store_const', const=spack.test_path, help="Edit the test with the supplied name.") - filetypes.add_argument( + excl_args.add_argument( '-m', '--module', dest='path', action='store_const', const=spack.module_path, help="Edit the main spack module with the supplied name.") + # Options for editing packages + excl_args.add_argument( + '-r', '--repo', default=None, help="Path to repo to edit package in.") + excl_args.add_argument( + '-N', '--namespace', default=None, help="Namespace of package to edit.") + subparser.add_argument( 'name', nargs='?', default=None, help="name of package to edit") @@ -107,7 +124,7 @@ def edit(parser, args): spack.editor(path) elif name: - edit_package(name, args.force) + edit_package(name, args.repo, args.namespace, args.force) else: # By default open the directory where packages or commands live. spack.editor(path) diff --git a/lib/spack/spack/cmd/repo.py b/lib/spack/spack/cmd/repo.py index a792f04cfd..34c755fb67 100644 --- a/lib/spack/spack/cmd/repo.py +++ b/lib/spack/spack/cmd/repo.py @@ -44,9 +44,10 @@ def setup_parser(subparser): # Create create_parser = sp.add_parser('create', help=repo_create.__doc__) create_parser.add_argument( - 'namespace', help="Namespace to identify packages in the repository.") + 'directory', help="Directory to create the repo in.") create_parser.add_argument( - 'directory', help="Directory to create the repo in. Defaults to same as namespace.", nargs='?') + 'namespace', help="Namespace to identify packages in the repository. " + "Defaults to the directory name.", nargs='?') # List list_parser = sp.add_parser('list', help=repo_list.__doc__) @@ -72,14 +73,15 @@ def setup_parser(subparser): def repo_create(args): - """Create a new package repo for a particular namespace.""" + """Create a new package repository.""" + root = canonicalize_path(args.directory) namespace = args.namespace - if not re.match(r'\w[\.\w-]*', namespace): - tty.die("Invalid namespace: '%s'" % namespace) - root = args.directory - if not root: - root = namespace + if not args.namespace: + namespace = os.path.basename(root) + + if not re.match(r'\w[\.\w-]*', namespace): + tty.die("'%s' is not a valid namespace." % namespace) existed = False if os.path.exists(root): @@ -123,27 +125,22 @@ def repo_create(args): def repo_add(args): - """Add a package source to the Spack configuration""" + """Add a package source to Spack's configuration.""" path = args.path - # check if the path is relative to the spack directory. - real_path = path - if path.startswith('$spack'): - real_path = spack.repository.substitute_spack_prefix(path) - elif not os.path.isabs(real_path): - real_path = os.path.abspath(real_path) - path = real_path + # real_path is absolute and handles substitution. + canon_path = canonicalize_path(path) # check if the path exists - if not os.path.exists(real_path): + if not os.path.exists(canon_path): tty.die("No such file or directory: '%s'." % path) # Make sure the path is a directory. - if not os.path.isdir(real_path): + if not os.path.isdir(canon_path): tty.die("Not a Spack repository: '%s'." % path) # Make sure it's actually a spack repository by constructing it. - repo = Repo(real_path) + repo = Repo(canon_path) # If that succeeds, finally add it to the configuration. repos = spack.config.get_config('repos', args.scope) @@ -152,30 +149,32 @@ def repo_add(args): if repo.root in repos or path in repos: tty.die("Repository is already registered with Spack: '%s'" % path) - repos.insert(0, path) + repos.insert(0, canon_path) spack.config.update_config('repos', repos, args.scope) tty.msg("Created repo with namespace '%s'." % repo.namespace) def repo_remove(args): - """Remove a repository from the Spack configuration.""" + """Remove a repository from Spack's configuration.""" repos = spack.config.get_config('repos', args.scope) path_or_namespace = args.path_or_namespace # If the argument is a path, remove that repository from config. - path = os.path.abspath(path_or_namespace) - if path in repos: - repos.remove(path) - spack.config.update_config('repos', repos, args.scope) - tty.msg("Removed repository '%s'." % path) - return + canon_path = canonicalize_path(path_or_namespace) + for repo_path in repos: + repo_canon_path = canonicalize_path(repo_path) + if canon_path == repo_canon_path: + repos.remove(repo_path) + spack.config.update_config('repos', repos, args.scope) + tty.msg("Removed repository '%s'." % repo_path) + return # If it is a namespace, remove corresponding repo for path in repos: try: repo = Repo(path) if repo.namespace == path_or_namespace: - repos.remove(repo.root) + repos.remove(path) spack.config.update_config('repos', repos, args.scope) tty.msg("Removed repository '%s' with namespace %s." % (repo.root, repo.namespace)) @@ -188,7 +187,7 @@ def repo_remove(args): def repo_list(args): - """List package sources and their mnemoics""" + """Show registered repositories and their namespaces.""" roots = spack.config.get_config('repos', args.scope) repos = [] for r in roots: diff --git a/lib/spack/spack/repository.py b/lib/spack/spack/repository.py index 3367572ef5..31596cee7a 100644 --- a/lib/spack/spack/repository.py +++ b/lib/spack/spack/repository.py @@ -54,6 +54,9 @@ repo_config_name = 'repo.yaml' # Top-level filename for repo config. packages_dir_name = 'packages' # Top-level repo directory containing pkgs. package_file_name = 'package.py' # Filename for packages in a repository. +# Guaranteed unused default value for some functions. +NOT_PROVIDED = object() + def _autospec(function): """Decorator that automatically converts the argument of a single-arg @@ -75,7 +78,15 @@ def _make_namespace_module(ns): def substitute_spack_prefix(path): """Replaces instances of $spack with Spack's prefix.""" - return path.replace('$spack', spack.prefix) + return re.sub(r'^\$spack', spack.prefix, path) + + +def canonicalize_path(path): + """Substitute $spack, expand user home, take abspath.""" + path = substitute_spack_prefix(path) + path = os.path.expanduser(path) + path = os.path.abspath(path) + return path class RepoPath(object): @@ -109,7 +120,10 @@ class RepoPath(object): repo = Repo(root, self.super_namespace) self.put_last(repo) except RepoError as e: - tty.warn("Failed to initialize repository at '%s'." % root, e.message) + tty.warn("Failed to initialize repository at '%s'." % root, + e.message, + "To remove the bad repository, run this command:", + " spack repo rm %s" % root) def swap(self, other): @@ -173,6 +187,31 @@ class RepoPath(object): self.repos.remove(repo) + def get_repo(self, namespace, default=NOT_PROVIDED): + """Get a repository by namespace. + Arguments + namespace + Look up this namespace in the RepoPath, and return + it if found. + + Optional Arguments + default + If default is provided, return it when the namespace + isn't found. If not, raise an UnknownNamespaceError. + """ + fullspace = '%s.%s' % (self.super_namespace, namespace) + if fullspace not in self.by_namespace: + if default == NOT_PROVIDED: + raise UnknownNamespaceError(namespace) + return default + return self.by_namespace[fullspace] + + + def first_repo(self): + """Get the first repo in precedence order.""" + return self.repos[0] if self.repos else None + + def all_package_names(self): """Return all unique package names in all repositories.""" return self._all_package_names @@ -229,7 +268,6 @@ class RepoPath(object): if fullname in sys.modules: return sys.modules[fullname] - # partition fullname into prefix and module name. namespace, dot, module_name = fullname.rpartition('.') @@ -242,11 +280,23 @@ class RepoPath(object): return module - def repo_for_pkg(self, pkg_name): + @_autospec + def repo_for_pkg(self, spec): + """Given a spec, get the repository for its package.""" + # If the spec already has a namespace, then return the + # corresponding repo if we know about it. + if spec.namespace: + fullspace = '%s.%s' % (self.super_namespace, spec.namespace) + if fullspace not in self.by_namespace: + raise UnknownNamespaceError(spec.namespace) + return self.by_namespace[fullspace] + + # If there's no namespace, search in the RepoPath. for repo in self.repos: - if pkg_name in repo: + if spec.name in repo: return repo - raise UnknownPackageError(pkg_name) + else: + raise UnknownPackageError(spec.name) @_autospec @@ -255,16 +305,7 @@ class RepoPath(object): Raises UnknownPackageError if not found. """ - # if the spec has a fully qualified namespace, we grab it - # directly and ignore overlay precedence. - if spec.namespace: - fullspace = '%s.%s' % (self.super_namespace, spec.namespace) - if not fullspace in self.by_namespace: - raise UnknownPackageError( - "No configured repository contains package %s." % spec.fullname) - return self.by_namespace[fullspace].get(spec) - else: - return self.repo_for_pkg(spec.name).get(spec) + return self.repo_for_pkg(spec).get(spec) def dirname_for_package_name(self, pkg_name): @@ -310,7 +351,7 @@ class Repo(object): """ # Root directory, containing _repo.yaml and package dirs # Allow roots to by spack-relative by starting with '$spack' - self.root = substitute_spack_prefix(root) + self.root = canonicalize_path(root) # super-namespace for all packages in the Repo self.super_namespace = namespace @@ -330,7 +371,7 @@ class Repo(object): # Read configuration and validate namespace config = self._read_config() check('namespace' in config, '%s must define a namespace.' - % join_path(self.root, repo_config_name)) + % join_path(root, repo_config_name)) self.namespace = config['namespace'] check(re.match(r'[a-zA-Z][a-zA-Z0-9_.]+', self.namespace), @@ -524,13 +565,22 @@ class Repo(object): return [p for p in self.all_packages() if p.extends(extendee_spec)] - def dirname_for_package_name(self, pkg_name): + def _check_namespace(self, spec): + """Check that the spec's namespace is the same as this repository's.""" + if spec.namespace and spec.namespace != self.namespace: + raise UnknownNamespaceError(spec.namespace) + + + @_autospec + def dirname_for_package_name(self, spec): """Get the directory name for a particular package. This is the directory that contains its package.py file.""" - return join_path(self.packages_path, pkg_name) + self._check_namespace(spec) + return join_path(self.packages_path, spec.name) - def filename_for_package_name(self, pkg_name): + @_autospec + def filename_for_package_name(self, spec): """Get the filename for the module we should load for a particular package. Packages for a Repo live in ``$root//package.py`` @@ -539,8 +589,8 @@ class Repo(object): package doesn't exist yet, so callers will need to ensure the package exists before importing. """ - validate_module_name(pkg_name) - pkg_dir = self.dirname_for_package_name(pkg_name) + self._check_namespace(spec) + pkg_dir = self.dirname_for_package_name(spec.name) return join_path(pkg_dir, package_file_name) @@ -679,6 +729,13 @@ class UnknownPackageError(PackageLoadError): self.name = name +class UnknownNamespaceError(PackageLoadError): + """Raised when we encounter an unknown namespace""" + def __init__(self, namespace): + super(UnknownNamespaceError, self).__init__( + "Unknown namespace: %s" % namespace) + + class FailedConstructorError(PackageLoadError): """Raised when a package's class constructor fails.""" def __init__(self, name, exc_type, exc_obj, exc_tb): diff --git a/lib/spack/spack/util/naming.py b/lib/spack/spack/util/naming.py index 26ca86c77f..5025f15027 100644 --- a/lib/spack/spack/util/naming.py +++ b/lib/spack/spack/util/naming.py @@ -8,11 +8,15 @@ from StringIO import StringIO import spack __all__ = ['mod_to_class', 'spack_module_to_python_module', 'valid_module_name', + 'valid_fully_qualified_module_name', 'validate_fully_qualified_module_name', 'validate_module_name', 'possible_spack_module_names', 'NamespaceTrie'] # Valid module names can contain '-' but can't start with it. _valid_module_re = r'^\w[\w-]*$' +# Valid module names can contain '-' but can't start with it. +_valid_fully_qualified_module_re = r'^(\w[\w-]*)(\.\w[\w-]*)*$' + def mod_to_class(mod_name): """Convert a name from module style to class name style. Spack mostly @@ -75,16 +79,27 @@ def possible_spack_module_names(python_mod_name): def valid_module_name(mod_name): - """Return whether the mod_name is valid for use in Spack.""" + """Return whether mod_name is valid for use in Spack.""" return bool(re.match(_valid_module_re, mod_name)) +def valid_fully_qualified_module_name(mod_name): + """Return whether mod_name is a valid namespaced module name.""" + return bool(re.match(_valid_fully_qualified_module_re, mod_name)) + + def validate_module_name(mod_name): """Raise an exception if mod_name is not valid.""" if not valid_module_name(mod_name): raise InvalidModuleNameError(mod_name) +def validate_fully_qualified_module_name(mod_name): + """Raise an exception if mod_name is not a valid namespaced module name.""" + if not valid_fully_qualified_module_name(mod_name): + raise InvalidFullyQualifiedModuleNameError(mod_name) + + class InvalidModuleNameError(spack.error.SpackError): """Raised when we encounter a bad module name.""" def __init__(self, name): @@ -93,6 +108,14 @@ class InvalidModuleNameError(spack.error.SpackError): self.name = name +class InvalidFullyQualifiedModuleNameError(spack.error.SpackError): + """Raised when we encounter a bad full package name.""" + def __init__(self, name): + super(InvalidFullyQualifiedModuleNameError, self).__init__( + "Invalid fully qualified package name: " + name) + self.name = name + + class NamespaceTrie(object): class Element(object): def __init__(self, value): -- cgit v1.2.3-70-g09d2 From ad32f64ef6120ab2bc9da739f3032b2ba304a2f0 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 17 Jan 2016 19:55:20 -0800 Subject: Allow completely empty config files. - Previous version would give validation error for an empty file. Now this is properly ignored. - Also includes bugfix in ConfigFormatError --- lib/spack/spack/config.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 193b311434..6fecde9980 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -357,8 +357,8 @@ def _read_config_file(filename, schema): with open(filename) as f: data = syaml.load(f) - validate_section(data, schema) - + if data: + validate_section(data, schema) return data except MarkedYAMLError, e: @@ -514,7 +514,7 @@ class ConfigFormatError(ConfigError): # Try really hard to get the parent (which sometimes is not # set) This digs it out of the validated structure if it's not # on the validation_error. - if not parent_mark: + if path and not parent_mark: parent_path = list(path)[:-1] parent = get_path(parent_path, data) if path[-1] in parent: -- cgit v1.2.3-70-g09d2 From cee7bfa9f0e5dc0ec8f8641327d1dba56dc4510d Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 18 Jan 2016 00:01:30 -0800 Subject: Simplify output redirection in spack.util.executable - By default inherit parent's input/output descriptor - Only use pipes if we need to return output. - Allows subprocesses (like emacsclient) to detect terminal correctly --- lib/spack/spack/compiler.py | 2 +- lib/spack/spack/util/executable.py | 25 ++++++++++--------------- 2 files changed, 11 insertions(+), 16 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index b9abf943e8..a665f6062d 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -51,7 +51,7 @@ _version_cache = {} def get_compiler_version(compiler_path, version_arg, regex='(.*)'): if not compiler_path in _version_cache: compiler = Executable(compiler_path) - output = compiler(version_arg, return_output=True, error=None) + output = compiler(version_arg, return_output=True, error=os.devnull) match = re.search(regex, output) _version_cache[compiler_path] = match.group(1) if match else 'unknown' diff --git a/lib/spack/spack/util/executable.py b/lib/spack/spack/util/executable.py index 35bf086616..ba765eb662 100644 --- a/lib/spack/spack/util/executable.py +++ b/lib/spack/spack/util/executable.py @@ -60,16 +60,16 @@ class Executable(object): fail_on_error = kwargs.get("fail_on_error", True) ignore_errors = kwargs.get("ignore_errors", ()) - output = kwargs.get("output", sys.stdout) - error = kwargs.get("error", sys.stderr) + # Default values of None says to keep parent's file descriptors. + output = kwargs.get("output", None) + error = kwargs.get("error", None) input = kwargs.get("input", None) def streamify(arg, mode): if isinstance(arg, basestring): return open(arg, mode), True - elif arg is None and mode != 'r': - return open(os.devnull, mode), True - return arg, False + else: + return arg, False output, ostream = streamify(output, 'w') error, estream = streamify(error, 'w') input, istream = streamify(input, 'r') @@ -92,19 +92,14 @@ class Executable(object): tty.debug(cmd_line) try: + if return_output: + output = subprocess.PIPE + proc = subprocess.Popen( - cmd, - stdin=input, - stderr=subprocess.PIPE, - stdout=subprocess.PIPE) + cmd, stdin=input, stderr=error, stdout=output) out, err = proc.communicate() - self.returncode = proc.returncode - - if not return_output: - output.write(out) - error.write(err) - rc = proc.returncode + rc = self.returncode = proc.returncode if fail_on_error and rc != 0 and (rc not in ignore_errors): raise ProcessError("Command exited with status %d:" % proc.returncode, cmd_line) -- cgit v1.2.3-70-g09d2 From e6aa2610bc4314cab09de61b4f7b632409299fb2 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Tue, 19 Jan 2016 13:07:07 -0600 Subject: Add symlink for NAG Fortran Compiler --- lib/spack/env/nag/nagfor | 1 + 1 file changed, 1 insertion(+) create mode 120000 lib/spack/env/nag/nagfor (limited to 'lib') diff --git a/lib/spack/env/nag/nagfor b/lib/spack/env/nag/nagfor new file mode 120000 index 0000000000..82c2b8e90a --- /dev/null +++ b/lib/spack/env/nag/nagfor @@ -0,0 +1 @@ +../cc \ No newline at end of file -- cgit v1.2.3-70-g09d2 From 83de658ee42e08a27ce7d85d295955f4809c03dc Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Tue, 19 Jan 2016 13:28:35 -0600 Subject: Modify nag.py to match new compiler package structure --- lib/spack/spack/compilers/nag.py | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'lib') diff --git a/lib/spack/spack/compilers/nag.py b/lib/spack/spack/compilers/nag.py index 2dfc97af73..f1cc6be0d5 100644 --- a/lib/spack/spack/compilers/nag.py +++ b/lib/spack/spack/compilers/nag.py @@ -1,3 +1,5 @@ +from spack.compiler import * + class Nag(Compiler): # Subclasses use possible names of C compiler cc_names = [] @@ -11,6 +13,13 @@ class Nag(Compiler): # Subclasses use possible names of Fortran 90 compiler fc_names = ['nagfor'] + # Named wrapper links within spack.build_env_path + link_paths = { # Use default wrappers for C and C++, in case provided in compilers.yaml + 'cc' : 'cc', + 'cxx' : 'cxx', + 'f77' : 'nag/nagfor', + 'fc' : 'nag/nagfor' } + @classmethod def default_version(self, comp): """The '-V' option works for nag compilers. -- cgit v1.2.3-70-g09d2 From 6c9800bcc395ecbc319f4a3f6242efee4b972470 Mon Sep 17 00:00:00 2001 From: Ben Boeckel Date: Tue, 19 Jan 2016 15:22:16 -0500 Subject: typos: fix some letter transposes --- lib/spack/spack/cmd/deactivate.py | 2 +- lib/spack/spack/database.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/cmd/deactivate.py b/lib/spack/spack/cmd/deactivate.py index a0c78bf755..d6b23d6a08 100644 --- a/lib/spack/spack/cmd/deactivate.py +++ b/lib/spack/spack/cmd/deactivate.py @@ -37,7 +37,7 @@ def setup_parser(subparser): help="Run deactivation even if spec is NOT currently activated.") subparser.add_argument( '-a', '--all', action='store_true', - help="Deactivate all extensions of an extendable pacakge, or " + help="Deactivate all extensions of an extendable package, or " "deactivate an extension AND its dependencies.") subparser.add_argument( 'spec', nargs=argparse.REMAINDER, help="spec of package extension to deactivate.") diff --git a/lib/spack/spack/database.py b/lib/spack/spack/database.py index 0fa18db34b..9cbe7de44a 100644 --- a/lib/spack/spack/database.py +++ b/lib/spack/spack/database.py @@ -489,7 +489,7 @@ class Database(object): 1. Marks the spec as not installed. 2. Removes the spec if it has no more dependents. 3. If removed, recursively updates dependencies' ref counts - and remvoes them if they are no longer needed. + and removes them if they are no longer needed. """ # Take a lock around the entire removal. -- cgit v1.2.3-70-g09d2 From e25150296a4ef34a80a664565e3d26f00f7d999f Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Wed, 20 Jan 2016 14:44:43 -0600 Subject: Redirect STDERR to STDOUT for compiler version This is necessary for the NAG Fortran compiler, which prints its version message to STDERR instead of STDOUT. This message was previously being ignored, preventing spack from finding the compiler's version or automatically adding it to the compilers.yaml configuration file. --- lib/spack/spack/compiler.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index a665f6062d..887e416dc5 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -24,6 +24,7 @@ ############################################################################## import os import re +import subprocess import itertools from datetime import datetime @@ -51,7 +52,7 @@ _version_cache = {} def get_compiler_version(compiler_path, version_arg, regex='(.*)'): if not compiler_path in _version_cache: compiler = Executable(compiler_path) - output = compiler(version_arg, return_output=True, error=os.devnull) + output = compiler(version_arg, return_output=True, error=subprocess.STDOUT) match = re.search(regex, output) _version_cache[compiler_path] = match.group(1) if match else 'unknown' -- cgit v1.2.3-70-g09d2 From 088b6f0ea9953a3f007d207d06ca3150f83b3be4 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Wed, 20 Jan 2016 17:19:01 -0600 Subject: Fixed typo when running spack help --- lib/spack/spack/cmd/location.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/cmd/location.py b/lib/spack/spack/cmd/location.py index 39c225e9b2..307ee8982d 100644 --- a/lib/spack/spack/cmd/location.py +++ b/lib/spack/spack/cmd/location.py @@ -32,7 +32,7 @@ from llnl.util.filesystem import join_path import spack import spack.cmd -description="Print out locations of various diectories used by Spack" +description="Print out locations of various directories used by Spack" def setup_parser(subparser): global directories -- cgit v1.2.3-70-g09d2 From 10848c2e9a7cd3771c469dfd5ddb235f76b229ba Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Thu, 21 Jan 2016 02:40:38 -0800 Subject: Refactor args for Executable.__call__ - simplify output, error, and input redirection - `return_output=True` is now `output=str` - `return_output=True` will still work for the time being but is deprecated. - previously you could say `return_output=True` and `output=`, which wouldn't actually write to the stream. Now you actually can't specify erroneous cases since there is only one parameter with mutually exclusive options.. --- lib/spack/spack/cmd/bootstrap.py | 2 +- lib/spack/spack/cmd/create.py | 2 +- lib/spack/spack/cmd/pkg.py | 2 +- lib/spack/spack/compiler.py | 3 +- lib/spack/spack/fetch_strategy.py | 6 +- lib/spack/spack/test/cc.py | 6 +- lib/spack/spack/test/make_executable.py | 48 ++++++------- lib/spack/spack/test/mock_repo.py | 4 +- lib/spack/spack/test/svn_fetch.py | 2 +- lib/spack/spack/util/executable.py | 95 ++++++++++++++++++++----- var/spack/repos/builtin/packages/gcc/package.py | 6 +- 11 files changed, 116 insertions(+), 60 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/cmd/bootstrap.py b/lib/spack/spack/cmd/bootstrap.py index e4ec7da35d..bdbd623b39 100644 --- a/lib/spack/spack/cmd/bootstrap.py +++ b/lib/spack/spack/cmd/bootstrap.py @@ -42,7 +42,7 @@ def get_origin_url(): git = which('git', required=True) origin_url = git( '--git-dir=%s' % git_dir, 'config', '--get', 'remote.origin.url', - return_output=True) + output=str) return origin_url.strip() diff --git a/lib/spack/spack/cmd/create.py b/lib/spack/spack/cmd/create.py index 7cea39cb55..edcea0718c 100644 --- a/lib/spack/spack/cmd/create.py +++ b/lib/spack/spack/cmd/create.py @@ -132,7 +132,7 @@ class ConfigureGuesser(object): # Peek inside the tarball. tar = which('tar') output = tar( - "--exclude=*/*/*", "-tf", stage.archive_file, return_output=True) + "--exclude=*/*/*", "-tf", stage.archive_file, output=str) lines = output.split("\n") # Set the configure line to the one that matched. diff --git a/lib/spack/spack/cmd/pkg.py b/lib/spack/spack/cmd/pkg.py index 448f762841..cf478d3763 100644 --- a/lib/spack/spack/cmd/pkg.py +++ b/lib/spack/spack/cmd/pkg.py @@ -79,7 +79,7 @@ def list_packages(rev): git = get_git() relpath = spack.packages_path[len(spack.prefix + os.path.sep):] + os.path.sep output = git('ls-tree', '--full-tree', '--name-only', rev, relpath, - return_output=True) + output=str) return sorted(line[len(relpath):] for line in output.split('\n') if line) diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index 887e416dc5..12c02e0ea2 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -24,7 +24,6 @@ ############################################################################## import os import re -import subprocess import itertools from datetime import datetime @@ -52,7 +51,7 @@ _version_cache = {} def get_compiler_version(compiler_path, version_arg, regex='(.*)'): if not compiler_path in _version_cache: compiler = Executable(compiler_path) - output = compiler(version_arg, return_output=True, error=subprocess.STDOUT) + output = compiler(version_arg, output=str, error=str) match = re.search(regex, output) _version_cache[compiler_path] = match.group(1) if match else 'unknown' diff --git a/lib/spack/spack/fetch_strategy.py b/lib/spack/spack/fetch_strategy.py index 0657146bf6..337dd1e198 100644 --- a/lib/spack/spack/fetch_strategy.py +++ b/lib/spack/spack/fetch_strategy.py @@ -154,7 +154,7 @@ class URLFetchStrategy(FetchStrategy): # Run curl but grab the mime type from the http headers headers = spack.curl( - *curl_args, return_output=True, fail_on_error=False) + *curl_args, output=str, fail_on_error=False) if spack.curl.returncode != 0: # clean up archive on failure. @@ -375,7 +375,7 @@ class GitFetchStrategy(VCSFetchStrategy): @property def git_version(self): - vstring = self.git('--version', return_output=True).lstrip('git version ') + vstring = self.git('--version', output=str).lstrip('git version ') return Version(vstring) @@ -518,7 +518,7 @@ class SvnFetchStrategy(VCSFetchStrategy): def _remove_untracked_files(self): """Removes untracked files in an svn repository.""" - status = self.svn('status', '--no-ignore', return_output=True) + status = self.svn('status', '--no-ignore', output=str) self.svn('status', '--no-ignore') for line in status.split('\n'): if not re.match('^[I?]', line): diff --git a/lib/spack/spack/test/cc.py b/lib/spack/spack/test/cc.py index 4188b8d550..905af28a06 100644 --- a/lib/spack/spack/test/cc.py +++ b/lib/spack/spack/test/cc.py @@ -65,17 +65,17 @@ class CompilerTest(unittest.TestCase): def check_cc(self, command, args, expected): os.environ['SPACK_TEST_COMMAND'] = command - self.assertEqual(self.cc(*args, return_output=True).strip(), expected) + self.assertEqual(self.cc(*args, output=str).strip(), expected) def check_ld(self, command, args, expected): os.environ['SPACK_TEST_COMMAND'] = command - self.assertEqual(self.ld(*args, return_output=True).strip(), expected) + self.assertEqual(self.ld(*args, output=str).strip(), expected) def check_cpp(self, command, args, expected): os.environ['SPACK_TEST_COMMAND'] = command - self.assertEqual(self.cpp(*args, return_output=True).strip(), expected) + self.assertEqual(self.cpp(*args, output=str).strip(), expected) def test_vcheck_mode(self): diff --git a/lib/spack/spack/test/make_executable.py b/lib/spack/spack/test/make_executable.py index 09efec8580..d568a28d44 100644 --- a/lib/spack/spack/test/make_executable.py +++ b/lib/spack/spack/test/make_executable.py @@ -56,47 +56,47 @@ class MakeExecutableTest(unittest.TestCase): def test_make_normal(self): make = MakeExecutable('make', 8) - self.assertEqual(make(return_output=True).strip(), '-j8') - self.assertEqual(make('install', return_output=True).strip(), '-j8 install') + self.assertEqual(make(output=str).strip(), '-j8') + self.assertEqual(make('install', output=str).strip(), '-j8 install') def test_make_explicit(self): make = MakeExecutable('make', 8) - self.assertEqual(make(parallel=True, return_output=True).strip(), '-j8') - self.assertEqual(make('install', parallel=True, return_output=True).strip(), '-j8 install') + self.assertEqual(make(parallel=True, output=str).strip(), '-j8') + self.assertEqual(make('install', parallel=True, output=str).strip(), '-j8 install') def test_make_one_job(self): make = MakeExecutable('make', 1) - self.assertEqual(make(return_output=True).strip(), '') - self.assertEqual(make('install', return_output=True).strip(), 'install') + self.assertEqual(make(output=str).strip(), '') + self.assertEqual(make('install', output=str).strip(), 'install') def test_make_parallel_false(self): make = MakeExecutable('make', 8) - self.assertEqual(make(parallel=False, return_output=True).strip(), '') - self.assertEqual(make('install', parallel=False, return_output=True).strip(), 'install') + self.assertEqual(make(parallel=False, output=str).strip(), '') + self.assertEqual(make('install', parallel=False, output=str).strip(), 'install') def test_make_parallel_disabled(self): make = MakeExecutable('make', 8) os.environ['SPACK_NO_PARALLEL_MAKE'] = 'true' - self.assertEqual(make(return_output=True).strip(), '') - self.assertEqual(make('install', return_output=True).strip(), 'install') + self.assertEqual(make(output=str).strip(), '') + self.assertEqual(make('install', output=str).strip(), 'install') os.environ['SPACK_NO_PARALLEL_MAKE'] = '1' - self.assertEqual(make(return_output=True).strip(), '') - self.assertEqual(make('install', return_output=True).strip(), 'install') + self.assertEqual(make(output=str).strip(), '') + self.assertEqual(make('install', output=str).strip(), 'install') # These don't disable (false and random string) os.environ['SPACK_NO_PARALLEL_MAKE'] = 'false' - self.assertEqual(make(return_output=True).strip(), '-j8') - self.assertEqual(make('install', return_output=True).strip(), '-j8 install') + self.assertEqual(make(output=str).strip(), '-j8') + self.assertEqual(make('install', output=str).strip(), '-j8 install') os.environ['SPACK_NO_PARALLEL_MAKE'] = 'foobar' - self.assertEqual(make(return_output=True).strip(), '-j8') - self.assertEqual(make('install', return_output=True).strip(), '-j8 install') + self.assertEqual(make(output=str).strip(), '-j8') + self.assertEqual(make('install', output=str).strip(), '-j8 install') del os.environ['SPACK_NO_PARALLEL_MAKE'] @@ -106,20 +106,20 @@ class MakeExecutableTest(unittest.TestCase): # These should work os.environ['SPACK_NO_PARALLEL_MAKE'] = 'true' - self.assertEqual(make(parallel=True, return_output=True).strip(), '') - self.assertEqual(make('install', parallel=True, return_output=True).strip(), 'install') + self.assertEqual(make(parallel=True, output=str).strip(), '') + self.assertEqual(make('install', parallel=True, output=str).strip(), 'install') os.environ['SPACK_NO_PARALLEL_MAKE'] = '1' - self.assertEqual(make(parallel=True, return_output=True).strip(), '') - self.assertEqual(make('install', parallel=True, return_output=True).strip(), 'install') + self.assertEqual(make(parallel=True, output=str).strip(), '') + self.assertEqual(make('install', parallel=True, output=str).strip(), 'install') # These don't disable (false and random string) os.environ['SPACK_NO_PARALLEL_MAKE'] = 'false' - self.assertEqual(make(parallel=True, return_output=True).strip(), '-j8') - self.assertEqual(make('install', parallel=True, return_output=True).strip(), '-j8 install') + self.assertEqual(make(parallel=True, output=str).strip(), '-j8') + self.assertEqual(make('install', parallel=True, output=str).strip(), '-j8 install') os.environ['SPACK_NO_PARALLEL_MAKE'] = 'foobar' - self.assertEqual(make(parallel=True, return_output=True).strip(), '-j8') - self.assertEqual(make('install', parallel=True, return_output=True).strip(), '-j8 install') + self.assertEqual(make(parallel=True, output=str).strip(), '-j8') + self.assertEqual(make('install', parallel=True, output=str).strip(), '-j8 install') del os.environ['SPACK_NO_PARALLEL_MAKE'] diff --git a/lib/spack/spack/test/mock_repo.py b/lib/spack/spack/test/mock_repo.py index c454b1f106..9738ba4e72 100644 --- a/lib/spack/spack/test/mock_repo.py +++ b/lib/spack/spack/test/mock_repo.py @@ -141,7 +141,7 @@ class MockGitRepo(MockVCSRepo): self.url = self.path def rev_hash(self, rev): - return git('rev-parse', rev, return_output=True).strip() + return git('rev-parse', rev, output=str).strip() class MockSvnRepo(MockVCSRepo): @@ -193,4 +193,4 @@ class MockHgRepo(MockVCSRepo): def get_rev(self): """Get current mercurial revision.""" - return hg('id', '-i', return_output=True).strip() + return hg('id', '-i', output=str).strip() diff --git a/lib/spack/spack/test/svn_fetch.py b/lib/spack/spack/test/svn_fetch.py index 2ee4748fdb..7d150b42f4 100644 --- a/lib/spack/spack/test/svn_fetch.py +++ b/lib/spack/spack/test/svn_fetch.py @@ -65,7 +65,7 @@ class SvnFetchTest(MockPackagesTest): def assert_rev(self, rev): """Check that the current revision is equal to the supplied rev.""" def get_rev(): - output = svn('info', return_output=True) + output = svn('info', output=str) self.assertTrue("Revision" in output) for line in output.split('\n'): match = re.match(r'Revision: (\d+)', line) diff --git a/lib/spack/spack/util/executable.py b/lib/spack/spack/util/executable.py index ba765eb662..fc27b789d0 100644 --- a/lib/spack/spack/util/executable.py +++ b/lib/spack/spack/util/executable.py @@ -55,24 +55,80 @@ class Executable(object): def __call__(self, *args, **kwargs): - """Run the executable with subprocess.check_output, return output.""" - return_output = kwargs.get("return_output", False) - fail_on_error = kwargs.get("fail_on_error", True) - ignore_errors = kwargs.get("ignore_errors", ()) + """Run this executable in a subprocess. + + Arguments + args + command line arguments to the executable to run. + + Optional arguments + + fail_on_error + + Raise an exception if the subprocess returns an + error. Default is True. When not set, the return code is + avaiale as `exe.returncode`. + + ignore_errors + + An optional list/tuple of error codes that can be + *ignored*. i.e., if these codes are returned, this will + not raise an exception when `fail_on_error` is `True`. + + output, error + + These arguments allow you to specify new stdout and stderr + values. They default to `None`, which means the + subprocess will inherit the parent's file descriptors. + + You can set these to: + - python streams, e.g. open Python file objects, or os.devnull; + - filenames, which will be automatically opened for writing; or + - `str`, as in the Python string type. If you set these to `str`, + output and error will be written to pipes and returned as + a string. If both `output` and `error` are set to `str`, + then one string is returned containing output concatenated + with error. + + input + + Same as output, error, but `str` is not an allowed value. + + Deprecated arguments + + return_output[=False] + + Setting this to True is the same as setting output=str. + This argument may be removed in future Spack versions. + + """ + fail_on_error = kwargs.pop("fail_on_error", True) + ignore_errors = kwargs.pop("ignore_errors", ()) + + # TODO: This is deprecated. Remove in a future version. + return_output = kwargs.pop("return_output", False) # Default values of None says to keep parent's file descriptors. - output = kwargs.get("output", None) - error = kwargs.get("error", None) - input = kwargs.get("input", None) + if return_output: + output = str + else: + output = kwargs.pop("output", None) + + error = kwargs.pop("error", None) + input = kwargs.pop("input", None) + if input is str: + raise ValueError("Cannot use `str` as input stream.") def streamify(arg, mode): if isinstance(arg, basestring): return open(arg, mode), True + elif arg is str: + return subprocess.PIPE, False else: return arg, False - output, ostream = streamify(output, 'w') - error, estream = streamify(error, 'w') - input, istream = streamify(input, 'r') + ostream, close_ostream = streamify(output, 'w') + estream, close_estream = streamify(error, 'w') + istream, close_istream = streamify(input, 'r') # if they just want to ignore one error code, make it a tuple. if isinstance(ignore_errors, int): @@ -92,19 +148,20 @@ class Executable(object): tty.debug(cmd_line) try: - if return_output: - output = subprocess.PIPE - proc = subprocess.Popen( - cmd, stdin=input, stderr=error, stdout=output) + cmd, stdin=istream, stderr=estream, stdout=ostream) out, err = proc.communicate() rc = self.returncode = proc.returncode if fail_on_error and rc != 0 and (rc not in ignore_errors): raise ProcessError("Command exited with status %d:" % proc.returncode, cmd_line) - if return_output: - return out + + if output is str or error is str: + result = '' + if output is str: result += out + if error is str: result += err + return result except OSError, e: raise ProcessError( @@ -119,9 +176,9 @@ class Executable(object): % (proc.returncode, cmd_line)) finally: - if ostream: output.close() - if estream: error.close() - if istream: input.close() + if close_ostream: output.close() + if close_estream: error.close() + if close_istream: input.close() def __eq__(self, other): diff --git a/var/spack/repos/builtin/packages/gcc/package.py b/var/spack/repos/builtin/packages/gcc/package.py index 61b16f3fd8..3e5895cfb8 100644 --- a/var/spack/repos/builtin/packages/gcc/package.py +++ b/var/spack/repos/builtin/packages/gcc/package.py @@ -50,7 +50,7 @@ class Gcc(Package): version('4.5.4', '27e459c2566b8209ab064570e1b378f7') variant('gold', default=True, description="Build the gold linker plugin for ld-based LTO") - + depends_on("mpfr") depends_on("gmp") depends_on("mpc") # when @4.5: @@ -102,7 +102,7 @@ class Gcc(Package): configure(*options) make() make("install") - + self.write_rpath_specs() @@ -121,7 +121,7 @@ class Gcc(Package): return gcc = Executable(join_path(self.prefix.bin, 'gcc')) - lines = gcc('-dumpspecs', return_output=True).strip().split("\n") + lines = gcc('-dumpspecs', output=str).strip().split("\n") specs_file = join_path(self.spec_dir, 'specs') with closing(open(specs_file, 'w')) as out: for line in lines: -- cgit v1.2.3-70-g09d2