From 1fa20ec2ba404912bd69c628f75f7ae4387d2fad Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Thu, 1 May 2014 22:51:36 -0700 Subject: partial checkin --- lib/spack/spack/compiler.py | 57 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 lib/spack/spack/compiler.py diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py new file mode 100644 index 0000000000..f2df6a696d --- /dev/null +++ b/lib/spack/spack/compiler.py @@ -0,0 +1,57 @@ +import os + +import spack.error +from spack.version import Version +from spack.util import Executable + +from subprocess import check_output + + +def _verify_compiler(*paths): + for path in paths: + if not os.path.isfile(path) and os.access(path, os.X_OK): + raise InvalidCompilerPathError(path) + + +class Compiler(object): + """This class encapsulates a Spack "compiler", which includes C, + C++, Fortran, and F90 compilers. Subclasses should implement + support for specific compilers, their possible names, arguments, + and how to identify the particular type of compiler.""" + + # Subclasses use possible names of C compiler + cc_names = [] + + # Subclasses use possible names of C++ compiler + cxx_names = [] + + # Subclasses use possible names of Fortran 77 compiler + f77_names = [] + + # Subclasses use possible names of Fortran 90 compiler + f90_names = [] + + # Names of generic arguments used by this compiler + arg_version = '-dumpversion' + arg_rpath = '-Wl,-rpath,%s' + + + def __init__(self, cc, cxx, f77, f90): + _verify_compiler(cc, cxx, f77, f90) + self.cc = Executable(cc) + self.cxx = Executable(cxx) + self.f77 = Executable(f77) + self.f90 = Executable(f90) + + + @property + @memoized + def version(self): + v = self.cc(arg_version) + return Version(v) + + +class InvalidCompilerPathError(spack.error.SpackError): + def __init__(self, path): + super(InvalidCompilerPathError, self).__init__( + "'%s' is not a valid compiler." % path) -- cgit v1.2.3-60-g2f50 From 8d78e1142f66ad7e7a8c0735925c154bcc41c22b Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 5 May 2014 00:47:54 -0700 Subject: Make Compiler constructor behave like Spec constructor. --- lib/spack/spack/cmd/compilers.py | 13 +++++++++++-- lib/spack/spack/cmd/find.py | 17 ++++------------- lib/spack/spack/compiler.py | 6 +++--- lib/spack/spack/compilers/__init__.py | 7 +++++-- lib/spack/spack/globals.py | 7 +++++-- lib/spack/spack/spec.py | 24 +++++++++++++++++++----- 6 files changed, 47 insertions(+), 27 deletions(-) diff --git a/lib/spack/spack/cmd/compilers.py b/lib/spack/spack/cmd/compilers.py index c3f204e3d3..f2bf2ed0b1 100644 --- a/lib/spack/spack/cmd/compilers.py +++ b/lib/spack/spack/cmd/compilers.py @@ -24,11 +24,20 @@ ############################################################################## import llnl.util.tty as tty from llnl.util.tty.colify import colify +from llnl.util.lang import index_by import spack.compilers +import spack.spec description = "List available compilers" + def compilers(parser, args): - tty.msg("Supported compilers") - colify(spack.compilers.supported_compilers(), indent=4) + tty.msg("Available compilers") + + # Index compilers + index = index_by(spack.compilers.supported_compilers(), 'name') + + for name, compilers in index.items(): + tty.hline(name, char='=', color=spack.spec.compiler_color) + colify(compilers, indent=4) diff --git a/lib/spack/spack/cmd/find.py b/lib/spack/spack/cmd/find.py index b6ec123c92..eaa3632043 100644 --- a/lib/spack/spack/cmd/find.py +++ b/lib/spack/spack/cmd/find.py @@ -29,7 +29,7 @@ from StringIO import StringIO import llnl.util.tty as tty from llnl.util.tty.colify import colify from llnl.util.tty.color import * -from llnl.util.lang import partition_list +from llnl.util.lang import partition_list, index_by import spack import spack.spec @@ -49,9 +49,6 @@ def setup_parser(subparser): def find(parser, args): - def hasher(): - return collections.defaultdict(hasher) - # Filter out specs that don't exist. query_specs = spack.cmd.parse_specs(args.query_specs) query_specs, nonexisting = partition_list( @@ -64,15 +61,9 @@ def find(parser, args): return # Make a dict with specs keyed by architecture and compiler. - index = hasher() - for spec in spack.db.installed_package_specs(): - # Check whether this installed package matches any query. - if query_specs and not any(spec.satisfies(q) for q in query_specs): - continue - - if spec.compiler not in index[spec.architecture]: - index[spec.architecture][spec.compiler] = [] - index[spec.architecture][spec.compiler].append(spec) + specs = [s for s in spack.db.installed_package_specs() + if not query_specs or any(spec.satisfies(q) for q in query_specs)] + index = index_by(specs, 'architecture', 'compiler') # Traverse the index and print out each package for architecture in index: diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index f2df6a696d..38b7780264 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -7,7 +7,7 @@ from spack.util import Executable from subprocess import check_output -def _verify_compiler(*paths): +def _verify_executables(*paths): for path in paths: if not os.path.isfile(path) and os.access(path, os.X_OK): raise InvalidCompilerPathError(path) @@ -37,13 +37,13 @@ class Compiler(object): def __init__(self, cc, cxx, f77, f90): - _verify_compiler(cc, cxx, f77, f90) + _verify_executables(cc, cxx, f77, f90) + self.cc = Executable(cc) self.cxx = Executable(cxx) self.f77 = Executable(f77) self.f90 = Executable(f90) - @property @memoized def version(self): diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index 5448e709c4..c6398f736d 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -26,16 +26,19 @@ # This needs to be expanded for full compiler support. # from llnl.util.lang import memoized, list_modules + import spack import spack.compilers.gcc +import spack.spec @memoized def supported_compilers(): - return [c for c in list_modules(spack.compilers_path)] + return [spack.spec.Compiler(c) for c in list_modules(spack.compilers_path)] def supported(compiler): - return compiler in supported_compilers() + return True +# return compiler in supported_compilers() @memoized diff --git a/lib/spack/spack/globals.py b/lib/spack/spack/globals.py index 9fc40845b0..6c3c5c9645 100644 --- a/lib/spack/spack/globals.py +++ b/lib/spack/spack/globals.py @@ -43,14 +43,17 @@ spack_file = join_path(prefix, "bin", "spack") lib_path = join_path(prefix, "lib", "spack") build_env_path = join_path(lib_path, "env") module_path = join_path(lib_path, "spack") -compilers_path = join_path(module_path, "compilers") test_path = join_path(module_path, "test") - var_path = join_path(prefix, "var", "spack") stage_path = join_path(var_path, "stage") install_path = join_path(prefix, "opt") +# +# Place to look for usable compiler installations +# +compilers_path = join_path(var_path, "compilers") + # # Set up the packages database. # diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 6416ff9487..900e36a8fb 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -173,11 +173,16 @@ class Compiler(object): """The Compiler field represents the compiler or range of compiler versions that a package should be built with. Compilers have a name and a version list. """ - def __init__(self, name, version=None): + def __init__(self, compiler_spec_like): + c = SpecParser().parse_compiler(compiler_spec_like) + self.name = c.name + self.versions = c.versions + + + def __init__(self, name, version): self.name = name self.versions = VersionList() - if version: - self.versions.add(version) + self.versions.add(version) def _add_version(self, version): @@ -210,7 +215,8 @@ class Compiler(object): def copy(self): - clone = Compiler(self.name) + clone = Compiler.__new__(Compiler) + clone.name = self.name clone.versions = self.versions.copy() return clone @@ -1188,6 +1194,11 @@ class SpecParser(spack.parse.Parser): return specs + def parse_compiler(self, text): + self.setup(text) + return self.compiler() + + def spec(self): """Parse a spec out of the input. If a spec is supplied, then initialize and return it instead of creating a new one.""" @@ -1279,7 +1290,10 @@ class SpecParser(spack.parse.Parser): def compiler(self): self.expect(ID) self.check_identifier() - compiler = Compiler(self.token.value) + + compiler = Compiler.__new__(Compiler) + compiler.name = self.token.value + compiler.versions = VersionList() if self.accept(AT): vlist = self.version_list() for version in vlist: -- cgit v1.2.3-60-g2f50 From c6956bc8e7b9d1261b71d09adaad570bdf7d9bd6 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Wed, 7 May 2014 16:34:43 -0700 Subject: Sorted out spack.compilers vs var/spack/compilers --- lib/spack/spack/cmd/compilers.py | 5 +---- lib/spack/spack/compiler.py | 5 +++-- lib/spack/spack/compilers/__init__.py | 31 +++++++++++++++++++------- lib/spack/spack/compilers/clang.py | 41 +++++++++++++++++++++++++++++++++++ lib/spack/spack/compilers/gcc.py | 26 ++++++++++++---------- lib/spack/spack/compilers/intel.py | 26 ++++++++++++---------- lib/spack/spack/globals.py | 5 +++-- lib/spack/spack/spec.py | 32 ++++++++++++++++----------- 8 files changed, 118 insertions(+), 53 deletions(-) create mode 100644 lib/spack/spack/compilers/clang.py diff --git a/lib/spack/spack/cmd/compilers.py b/lib/spack/spack/cmd/compilers.py index f2bf2ed0b1..5658263c26 100644 --- a/lib/spack/spack/cmd/compilers.py +++ b/lib/spack/spack/cmd/compilers.py @@ -31,13 +31,10 @@ import spack.spec description = "List available compilers" - def compilers(parser, args): tty.msg("Available compilers") - # Index compilers index = index_by(spack.compilers.supported_compilers(), 'name') - for name, compilers in index.items(): - tty.hline(name, char='=', color=spack.spec.compiler_color) + tty.hline(name, char='-', color=spack.spec.compiler_color) colify(compilers, indent=4) diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index 38b7780264..27effa20c4 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -1,10 +1,11 @@ import os +from llnl.util.lang import memoized import spack.error from spack.version import Version -from spack.util import Executable +from spack.util.executable import Executable + -from subprocess import check_output def _verify_executables(*paths): diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index c6398f736d..4bfb855d94 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -27,21 +27,36 @@ # from llnl.util.lang import memoized, list_modules -import spack -import spack.compilers.gcc import spack.spec +from spack.util.executable import which + + @memoized def supported_compilers(): - return [spack.spec.Compiler(c) for c in list_modules(spack.compilers_path)] + """Return a list of compiler types supported by Spack.""" + return sorted(c for c in list_modules(spack.compilers_path)) + +def supported(compiler_spec): + """Test if a particular compiler is supported.""" + if isinstance(compiler_spec, spack.spec.Compiler): + return compiler_spec.name in supported_compilers() -def supported(compiler): - return True -# return compiler in supported_compilers() + elif isinstance(compiler_spec, basestring): + return compiler_spec in supported_compilers() + + else: + raise TypeError("compiler_spec must be string or spack.spec.Compiler") @memoized def default_compiler(): - from spack.spec import Compiler - return Compiler('gcc', gcc.get_version()) + """Get the spec for the default compiler supported by Spack. + Currently just returns the system's default gcc. + + TODO: provide a better way to specify/find this on startup. + """ + gcc = which('gcc', required=True) + version = gcc('-dumpversion', return_output=True) + return spack.spec.Compiler('gcc', version) diff --git a/lib/spack/spack/compilers/clang.py b/lib/spack/spack/compilers/clang.py new file mode 100644 index 0000000000..9ed7b57846 --- /dev/null +++ b/lib/spack/spack/compilers/clang.py @@ -0,0 +1,41 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack.compiler import Compiler + +class Clang(Compiler): + # Subclasses use possible names of C compiler + cc_names = ['clang'] + + # Subclasses use possible names of C++ compiler + cxx_names = ['clang++'] + + # Subclasses use possible names of Fortran 77 compiler + f77_names = [] + + # Subclasses use possible names of Fortran 90 compiler + f90_names = [] + + def __init__(self, cc, cxx, f77, f90): + super(Gcc, self).__init__(cc, cxx, f77, f90) diff --git a/lib/spack/spack/compilers/gcc.py b/lib/spack/spack/compilers/gcc.py index 073219dc20..638051008f 100644 --- a/lib/spack/spack/compilers/gcc.py +++ b/lib/spack/spack/compilers/gcc.py @@ -22,18 +22,20 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -# -# This is a stub module. It should be expanded when we implement full -# compiler support. -# +from spack.compiler import Compiler + +class Gcc(Compiler): + # Subclasses use possible names of C compiler + cc_names = ['gcc'] + + # Subclasses use possible names of C++ compiler + cxx_names = ['g++'] -import subprocess -from spack.version import Version + # Subclasses use possible names of Fortran 77 compiler + f77_names = ['gfortran'] -cc = 'gcc' -cxx = 'g++' -fortran = 'gfortran' + # Subclasses use possible names of Fortran 90 compiler + f90_names = ['gfortran'] -def get_version(): - v = subprocess.check_output([cc, '-dumpversion']) - return Version(v) + def __init__(self, cc, cxx, f77, f90): + super(Gcc, self).__init__(cc, cxx, f77, f90) diff --git a/lib/spack/spack/compilers/intel.py b/lib/spack/spack/compilers/intel.py index c27fee1719..ebbab57ed1 100644 --- a/lib/spack/spack/compilers/intel.py +++ b/lib/spack/spack/compilers/intel.py @@ -22,18 +22,20 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -# -# This is a stub module. It should be expanded when we implement full -# compiler support. -# +from spack.compiler import Compiler + +class Intel(Compiler): + # Subclasses use possible names of C compiler + cc_names = ['icc'] + + # Subclasses use possible names of C++ compiler + cxx_names = ['icpc'] -import subprocess -from spack.version import Version + # Subclasses use possible names of Fortran 77 compiler + f77_names = ['ifort'] -cc = 'icc' -cxx = 'icc' -fortran = 'ifort' + # Subclasses use possible names of Fortran 90 compiler + f90_names = ['ifort'] -def get_version(): - v = subprocess.check_output([cc, '-dumpversion']) - return Version(v) + def __init__(self, cc, cxx, f77, f90): + super(Gcc, self).__init__(cc, cxx, f77, f90) diff --git a/lib/spack/spack/globals.py b/lib/spack/spack/globals.py index 6c3c5c9645..c67cb02060 100644 --- a/lib/spack/spack/globals.py +++ b/lib/spack/spack/globals.py @@ -43,6 +43,7 @@ spack_file = join_path(prefix, "bin", "spack") lib_path = join_path(prefix, "lib", "spack") build_env_path = join_path(lib_path, "env") module_path = join_path(lib_path, "spack") +compilers_path = join_path(module_path, "compilers") test_path = join_path(module_path, "test") var_path = join_path(prefix, "var", "spack") stage_path = join_path(var_path, "stage") @@ -50,9 +51,9 @@ stage_path = join_path(var_path, "stage") install_path = join_path(prefix, "opt") # -# Place to look for usable compiler installations +# Place to look for usable compiler versions. # -compilers_path = join_path(var_path, "compilers") +compiler_version_path = join_path(var_path, "compilers") # # Set up the packages database. diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 900e36a8fb..bb8965ccc8 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -173,16 +173,23 @@ class Compiler(object): """The Compiler field represents the compiler or range of compiler versions that a package should be built with. Compilers have a name and a version list. """ - def __init__(self, compiler_spec_like): - c = SpecParser().parse_compiler(compiler_spec_like) - self.name = c.name - self.versions = c.versions + def __init__(self, *args): + nargs = len(args) + if nargs == 1: + # If there is one argument, it's a spec to parse + c = SpecParser().parse_compiler(args[0]) + self.name = c.name + self.versions = c.versions + + elif nargs == 2: + name, version = args + self.name = name + self.versions = VersionList() + self.versions.add(ver(version)) - - def __init__(self, name, version): - self.name = name - self.versions = VersionList() - self.versions.add(version) + else: + raise TypeError( + "__init__ takes 1 or 2 arguments. (%d given)" % nargs) def _add_version(self, version): @@ -776,11 +783,10 @@ class Spec(object): if not spec.virtual: spack.db.get(spec.name) - # validate compiler name in addition to the package name. + # validate compiler in addition to the package name. if spec.compiler: - compiler_name = spec.compiler.name - if not spack.compilers.supported(compiler_name): - raise UnknownCompilerError(compiler_name) + if not spack.compilers.supported(spec.compiler): + raise UnknownCompilerError(spec.compiler) def constrain(self, other, **kwargs): -- cgit v1.2.3-60-g2f50 From 45baf73c349465cce9c82db48777f2b8a853caa5 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Wed, 7 May 2014 20:01:12 -0700 Subject: Implemented compiler concretization policy. --- lib/spack/spack/cmd/compilers.py | 2 +- lib/spack/spack/compilers/__init__.py | 29 +++++++++++-------- lib/spack/spack/concretize.py | 34 +++++++++++++--------- lib/spack/spack/spec.py | 53 +++++++++++++++++++++++++---------- lib/spack/spack/test/concretize.py | 14 ++++++++- 5 files changed, 91 insertions(+), 41 deletions(-) diff --git a/lib/spack/spack/cmd/compilers.py b/lib/spack/spack/cmd/compilers.py index 5658263c26..8267ecbd6b 100644 --- a/lib/spack/spack/cmd/compilers.py +++ b/lib/spack/spack/cmd/compilers.py @@ -34,7 +34,7 @@ description = "List available compilers" def compilers(parser, args): tty.msg("Available compilers") - index = index_by(spack.compilers.supported_compilers(), 'name') + index = index_by(spack.compilers.available_compilers(), 'name') for name, compilers in index.items(): tty.hline(name, char='-', color=spack.spec.compiler_color) colify(compilers, indent=4) diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index 4bfb855d94..2b97cd1036 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -27,27 +27,34 @@ # from llnl.util.lang import memoized, list_modules +import spack import spack.spec from spack.util.executable import which - - @memoized def supported_compilers(): - """Return a list of compiler types supported by Spack.""" + """Return a list of names of compilers supported by Spack. + + See available_compilers() to get a list of all the available + versions of supported compilers. + """ return sorted(c for c in list_modules(spack.compilers_path)) -def supported(compiler_spec): - """Test if a particular compiler is supported.""" - if isinstance(compiler_spec, spack.spec.Compiler): - return compiler_spec.name in supported_compilers() +def available_compilers(): + """Return a list of specs for all the compiler versions currently + available to build with. These are instances of + spack.spec.Compiler. + """ + return [spack.spec.Compiler(c) + for c in list_modules(spack.compiler_version_path)] - elif isinstance(compiler_spec, basestring): - return compiler_spec in supported_compilers() - else: - raise TypeError("compiler_spec must be string or spack.spec.Compiler") +def supported(compiler_spec): + """Test if a particular compiler is supported.""" + if not isinstance(compiler_spec, spack.spec.Compiler): + compiler_spec = spack.spec.Compiler(compiler_spec) + return compiler_spec.name in supported_compilers() @memoized diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index fc360d59ba..8679d3e13e 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -89,15 +89,11 @@ class DefaultConcretizer(object): def concretize_compiler(self, spec): - """Currently just sets the compiler to gcc or throws an exception - if the compiler is set to something else. - - TODO: implement below description. - - If the spec already has a compiler, we're done. If not, then - take the compiler used for the nearest ancestor with a concrete - compiler, or use the system default if there is no ancestor - with a compiler. + """If the spec already has a compiler, we're done. If not, then take + the compiler used for the nearest ancestor with a compiler + spec and use that. If the ancestor's compiler is not + concrete, then give it a valid version. If there is no + ancestor with a compiler, use the system default compiler. Intuition: Use the system default if no package that depends on this one has a strict compiler requirement. Otherwise, try to @@ -105,10 +101,22 @@ class DefaultConcretizer(object): link to this one, to maximize compatibility. """ if spec.compiler and spec.compiler.concrete: - if spec.compiler != spack.compilers.default_compiler(): - raise spack.spec.UnknownCompilerError(str(spec.compiler)) - else: - spec.compiler = spack.compilers.default_compiler() + return + + try: + nearest = next(p for p in spec.preorder_traversal(direction='parents') + if p.compiler is not None).compiler + + if not nearest.concrete: + matches = [c for c in spack.compilers.available_compilers() + if c.name == nearest.name] + nearest.versions = sorted(matches)[-1].versions.copy() + assert(nearest.concrete) + + spec.compiler = nearest.copy() + + except StopIteration: + spec.compiler = spack.compilers.default_compiler().copy() def choose_provider(self, spec, providers): diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index bb8965ccc8..d0b08cca00 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -196,12 +196,20 @@ class Compiler(object): self.versions.add(version) + def _autospec(self, compiler_spec_like): + if not isinstance(compiler_spec_like, Compiler): + return Compiler(compiler_spec_like) + return compiler_spec_like + + def satisfies(self, other): + other = self._autospec(other) return (self.name == other.name and self.versions.overlaps(other.versions)) def constrain(self, other): + other = self._autospec(other) if not self.satisfies(other): raise UnsatisfiableCompilerSpecError(self, other) @@ -374,14 +382,14 @@ class Spec(object): """ if not self.dependents: return self - else: - # If the spec has multiple dependents, ensure that they all - # lead to the same place. Spack shouldn't deal with any DAGs - # with multiple roots, so something's wrong if we find one. - depiter = iter(self.dependents.values()) - first_root = next(depiter).root - assert(all(first_root is d.root for d in depiter)) - return first_root + + # If the spec has multiple dependents, ensure that they all + # lead to the same place. Spack shouldn't deal with any DAGs + # with multiple roots, so something's wrong if we find one. + depiter = iter(self.dependents.values()) + first_root = next(depiter).root + assert(all(first_root is d.root for d in depiter)) + return first_root @property @@ -441,17 +449,28 @@ class Spec(object): root [=True] If false, this won't yield the root node, just its descendents. + + direction [=children|parents] + If 'children', does a traversal of this spec's children. If + 'parents', traverses upwards in the DAG towards the root. + """ depth = kwargs.get('depth', False) key_fun = kwargs.get('key', id) yield_root = kwargs.get('root', True) cover = kwargs.get('cover', 'nodes') + direction = kwargs.get('direction', 'children') cover_values = ('nodes', 'edges', 'paths') if cover not in cover_values: raise ValueError("Invalid value for cover: %s. Choices are %s" % (cover, ",".join(cover_values))) + direction_values = ('children', 'parents') + if direction not in direction_values: + raise ValueError("Invalid value for direction: %s. Choices are %s" + % (direction, ",".join(direction_values))) + if visited is None: visited = set() @@ -465,9 +484,13 @@ class Spec(object): else: if yield_root or d > 0: yield result + successors = self.dependencies + if direction == 'parents': + successors = self.dependents + visited.add(key) - for name in sorted(self.dependencies): - child = self.dependencies[name] + for name in sorted(successors): + child = successors[name] for elt in child.preorder_traversal(visited, d+1, **kwargs): yield elt @@ -776,7 +799,7 @@ class Spec(object): def validate_names(self): """This checks that names of packages and compilers in this spec are real. If they're not, it will raise either UnknownPackageError or - UnknownCompilerError. + UnsupportedCompilerError. """ for spec in self.preorder_traversal(): # Don't get a package for a virtual name. @@ -786,7 +809,7 @@ class Spec(object): # validate compiler in addition to the package name. if spec.compiler: if not spack.compilers.supported(spec.compiler): - raise UnknownCompilerError(spec.compiler) + raise UnsupportedCompilerError(spec.compiler.name) def constrain(self, other, **kwargs): @@ -1385,11 +1408,11 @@ class DuplicateCompilerError(SpecError): super(DuplicateCompilerError, self).__init__(message) -class UnknownCompilerError(SpecError): +class UnsupportedCompilerError(SpecError): """Raised when the user asks for a compiler spack doesn't know about.""" def __init__(self, compiler_name): - super(UnknownCompilerError, self).__init__( - "Unknown compiler: %s" % compiler_name) + super(UnsupportedCompilerError, self).__init__( + "The '%s' compiler is not yet supported." % compiler_name) class DuplicateArchitectureError(SpecError): diff --git a/lib/spack/spack/test/concretize.py b/lib/spack/spack/test/concretize.py index 62e2732749..2a989f6766 100644 --- a/lib/spack/spack/test/concretize.py +++ b/lib/spack/spack/test/concretize.py @@ -25,7 +25,7 @@ import unittest import spack -from spack.spec import Spec +from spack.spec import Spec, Compiler from spack.test.mock_packages_test import * class ConcretizeTest(MockPackagesTest): @@ -163,3 +163,15 @@ class ConcretizeTest(MockPackagesTest): spec = Spec('indirect_mpich') spec.normalize() spec.concretize() + + + def test_compiler_inheritance(self): + spec = Spec('mpileaks') + spec.normalize() + + spec['dyninst'].compiler = Compiler('clang') + spec.concretize() + + # TODO: not exactly the syntax I would like. + self.assertTrue(spec['libdwarf'].compiler.satisfies('clang')) + self.assertTrue(spec['libelf'].compiler.satisfies('clang')) -- cgit v1.2.3-60-g2f50 From 285c5444ab0e1522b8e04b2052bfe5649f6838a3 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sat, 17 May 2014 13:01:00 -0700 Subject: Add CompilerSpec class and loading capability. - spack.spec.Compiler is now spack.spec.CompilerSpec - Can load a spack.compilers.* module for a particular spec - e.g. load Gcc module for gcc@4.7 spec. --- lib/spack/docs/packaging_guide.rst | 4 +-- lib/spack/spack/build_environment.py | 6 ++++ lib/spack/spack/compilers/__init__.py | 59 +++++++++++++++++++++++++++------ lib/spack/spack/concretize.py | 9 ++++++ lib/spack/spack/package.py | 1 + lib/spack/spack/packages.py | 51 ++--------------------------- lib/spack/spack/spec.py | 51 ++++++++++++++++++----------- lib/spack/spack/test/concretize.py | 4 +-- lib/spack/spack/test/packages.py | 11 ++++--- lib/spack/spack/test/spec_syntax.py | 12 +++---- lib/spack/spack/util/naming.py | 61 +++++++++++++++++++++++++++++++++++ 11 files changed, 177 insertions(+), 92 deletions(-) create mode 100644 lib/spack/spack/util/naming.py diff --git a/lib/spack/docs/packaging_guide.rst b/lib/spack/docs/packaging_guide.rst index 4e076aa991..da944b33c8 100644 --- a/lib/spack/docs/packaging_guide.rst +++ b/lib/spack/docs/packaging_guide.rst @@ -107,7 +107,7 @@ Package class names The **class name** (``Libelf`` in our example) is formed by converting words separated by `-` or ``_`` in the file name to camel case. If the name starts with a number, we prefix the class name with -``Num_``. Here are some examples: +``_``. Here are some examples: ================= ================= Module Name Class Name @@ -115,7 +115,7 @@ the name starts with a number, we prefix the class name with ``foo_bar`` ``FooBar`` ``docbook-xml`` ``DocbookXml`` ``FooBar`` ``Foobar`` - ``3proxy`` ``Num_3proxy`` + ``3proxy`` ``_3proxy`` ================= ================= The class name is needed by Spack to properly import a package, but diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index d6becb77db..8d3d0909db 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -34,6 +34,7 @@ import platform from llnl.util.filesystem import * import spack +from spack.compilers import compiler_for_spec from spack.util.executable import Executable, which from spack.util.environment import * @@ -79,6 +80,11 @@ class MakeExecutable(Executable): super(MakeExecutable, self).__call__(*args, **kwargs) +def set_compiler_environment_variables(pkg): + assert(pkg.spec.concrete) + compiler = compiler_for_spec(pkg.spec.compiler) + + def set_build_environment_variables(pkg): """This ensures a clean install environment when we build packages. """ diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index 2b97cd1036..095e26edb5 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -25,11 +25,21 @@ # # This needs to be expanded for full compiler support. # +import imp + from llnl.util.lang import memoized, list_modules +from llnl.util.filesystem import join_path import spack +import spack.error import spack.spec +from spack.compiler import Compiler from spack.util.executable import which +from spack.util.naming import mod_to_class + +_imported_compilers_module = 'spack.compiler.versions' +_imported_versions_module = 'spack.compilers' + @memoized def supported_compilers(): @@ -41,29 +51,58 @@ def supported_compilers(): return sorted(c for c in list_modules(spack.compilers_path)) +def supported(compiler_spec): + """Test if a particular compiler is supported.""" + if not isinstance(compiler_spec, spack.spec.CompilerSpec): + compiler_spec = spack.spec.CompilerSpec(compiler_spec) + return compiler_spec.name in supported_compilers() + + def available_compilers(): """Return a list of specs for all the compiler versions currently available to build with. These are instances of - spack.spec.Compiler. + CompilerSpec. """ - return [spack.spec.Compiler(c) + return [spack.spec.CompilerSpec(c) for c in list_modules(spack.compiler_version_path)] -def supported(compiler_spec): - """Test if a particular compiler is supported.""" - if not isinstance(compiler_spec, spack.spec.Compiler): - compiler_spec = spack.spec.Compiler(compiler_spec) - return compiler_spec.name in supported_compilers() +def compiler_for_spec(compiler_spec): + """This gets an instance of an actual spack.compiler.Compiler object + from a compiler spec. The spec needs to be concrete for this to + work; it will raise an error if passed an abstract compiler. + """ + matches = [c for c in available_compilers() if c.satisfies(compiler_spec)] + + # TODO: do something when there are zero matches. + assert(len(matches) >= 1) + + compiler = matches[0] + file_path = join_path(spack.compiler_version_path, "%s.py" % compiler) + + mod = imp.load_source(_imported_versions_module, file_path) + compiler_class = class_for_compiler_name(compiler.name) + + return compiler_class(mod.cc, mod.cxx, mod.f77, mod.f90) + + +def class_for_compiler_name(compiler_name): + assert(supported(compiler_name)) + + file_path = join_path(spack.compilers_path, compiler_name + ".py") + compiler_mod = imp.load_source(_imported_compilers_module, file_path) + return getattr(compiler_mod, mod_to_class(compiler_name)) @memoized def default_compiler(): - """Get the spec for the default compiler supported by Spack. + """Get the spec for the default compiler on the system. Currently just returns the system's default gcc. - TODO: provide a better way to specify/find this on startup. + TODO: provide a more sensible default. e.g. on Intel systems + we probably want icc. On Mac OS, clang. Probably need + to inspect the system and figure this out. """ gcc = which('gcc', required=True) version = gcc('-dumpversion', return_output=True) - return spack.spec.Compiler('gcc', version) + return spack.spec.CompilerSpec('gcc', version) diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 8679d3e13e..28efcaea64 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -99,6 +99,15 @@ class DefaultConcretizer(object): this one has a strict compiler requirement. Otherwise, try to build with the compiler that will be used by libraries that link to this one, to maximize compatibility. + + TODO: In many cases we probably want to look for installed + versions of each package and use *that* version if we + can link to it. The policy implemented here will + tend to rebuild a lot of stuff becasue it will prefer + a compiler in the spec to any compiler already- + installed things were built with. There is likely + some better policy that finds some middle ground + between these two extremes. """ if spec.compiler and spec.compiler.concrete: return diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 59d7ce7907..b40448df37 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -657,6 +657,7 @@ class Package(object): spack.install_layout.make_path_for_spec(self.spec) # Set up process's build environment before running install. + build_env.set_compiler_environment_variables(self) build_env.set_build_environment_variables(self) build_env.set_module_variables_for_package(self) diff --git a/lib/spack/spack/packages.py b/lib/spack/spack/packages.py index 08ded5cdbb..36f3d4286a 100644 --- a/lib/spack/spack/packages.py +++ b/lib/spack/spack/packages.py @@ -22,10 +22,8 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import re import os import sys -import string import inspect import glob import imp @@ -38,6 +36,7 @@ import spack import spack.error import spack.spec from spack.virtual import ProviderIndex +from spack.util.naming import mod_to_class, validate_module_name # Name of module under which packages are imported _imported_packages_module = 'spack.packages' @@ -45,42 +44,6 @@ _imported_packages_module = 'spack.packages' # Name of the package file inside a package directory _package_file_name = 'package.py' -# Valid package names can contain '-' but can't start with it. -valid_package_re = r'^\w[\w-]*$' - -# Don't allow consecutive [_-] in package names -invalid_package_re = r'[_-][_-]+' - - -def valid_package_name(pkg_name): - """Return whether the pkg_name is valid for use in Spack.""" - return (re.match(valid_package_re, pkg_name) and - not re.search(invalid_package_re, pkg_name)) - - -def validate_package_name(pkg_name): - """Raise an exception if pkg_name is not valid.""" - if not valid_package_name(pkg_name): - raise InvalidPackageNameError(pkg_name) - - -def class_name_for_package_name(pkg_name): - """Get a name for the class the package file should contain. Note that - conflicts don't matter because the classes are in different modules. - """ - validate_package_name(pkg_name) - - class_name = pkg_name.replace('_', '-') - class_name = string.capwords(class_name, '-') - class_name = class_name.replace('-', '') - - # If a class starts with a number, prefix it with Number_ to make it a valid - # Python class name. - if re.match(r'^[0-9]', class_name): - class_name = "Num_%s" % class_name - - return class_name - def _autospec(function): """Decorator that automatically converts the argument of a single-arg @@ -143,7 +106,7 @@ class PackageDB(object): package doesn't exist yet, so callers will need to ensure the package exists before importing. """ - validate_package_name(pkg_name) + validate_module_name(pkg_name) pkg_dir = self.dirname_for_package_name(pkg_name) return join_path(pkg_dir, _package_file_name) @@ -200,7 +163,7 @@ class PackageDB(object): else: raise UnknownPackageError(pkg_name) - class_name = class_name_for_package_name(pkg_name) + class_name = mod_to_class(pkg_name) try: module_name = _imported_packages_module + '.' + pkg_name module = imp.load_source(module_name, file_path) @@ -259,14 +222,6 @@ class PackageDB(object): out.write('}\n') -class InvalidPackageNameError(spack.error.SpackError): - """Raised when we encounter a bad package name.""" - def __init__(self, name): - super(InvalidPackageNameError, self).__init__( - "Invalid package name: " + name) - self.name = name - - class UnknownPackageError(spack.error.SpackError): """Raised when we encounter a package spack doesn't have.""" def __init__(self, name): diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index d0b08cca00..f0244695bc 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -102,8 +102,7 @@ from llnl.util.tty.color import * import spack import spack.parse import spack.error -import spack.compilers -import spack.compilers.gcc +from spack.compilers import supported as supported_compiler from spack.version import * from spack.util.string import * @@ -169,17 +168,29 @@ def colorize_spec(spec): @key_ordering -class Compiler(object): - """The Compiler field represents the compiler or range of compiler - versions that a package should be built with. Compilers have a +class CompilerSpec(object): + """The CompilerSpec field represents the compiler or range of compiler + versions that a package should be built with. CompilerSpecs have a name and a version list. """ def __init__(self, *args): nargs = len(args) if nargs == 1: - # If there is one argument, it's a spec to parse - c = SpecParser().parse_compiler(args[0]) - self.name = c.name - self.versions = c.versions + arg = args[0] + # If there is one argument, it's either another CompilerSpec + # to copy or a string to parse + if isinstance(arg, basestring): + c = SpecParser().parse_compiler(arg) + self.name = c.name + self.versions = c.versions + + elif isinstance(arg, CompilerSpec): + self.name = arg.name + self.versions = arg.versions.copy() + + else: + raise TypeError( + "Can only build CompilerSpec from string or CompilerSpec." + + " Found %s" % type(arg)) elif nargs == 2: name, version = args @@ -197,12 +208,14 @@ class Compiler(object): def _autospec(self, compiler_spec_like): - if not isinstance(compiler_spec_like, Compiler): - return Compiler(compiler_spec_like) - return compiler_spec_like + if isinstance(compiler_spec_like, CompilerSpec): + return compiler_spec_like + return CompilerSpec(compiler_spec_like) def satisfies(self, other): + # TODO: This should not just look for overlapping versions. + # TODO: e.g., 4.7.3 should satisfy a requirement for 4.7. other = self._autospec(other) return (self.name == other.name and self.versions.overlaps(other.versions)) @@ -218,7 +231,7 @@ class Compiler(object): @property def concrete(self): - """A Compiler spec is concrete if its versions are concrete.""" + """A CompilerSpec is concrete if its versions are concrete.""" return self.versions.concrete @@ -230,7 +243,7 @@ class Compiler(object): def copy(self): - clone = Compiler.__new__(Compiler) + clone = CompilerSpec.__new__(CompilerSpec) clone.name = self.name clone.versions = self.versions.copy() return clone @@ -353,7 +366,7 @@ class Spec(object): def _set_compiler(self, compiler): """Called by the parser to set the compiler.""" - if self.compiler: raise DuplicateCompilerError( + if self.compiler: raise DuplicateCompilerSpecError( "Spec for '%s' cannot have two compilers." % self.name) self.compiler = compiler @@ -808,7 +821,7 @@ class Spec(object): # validate compiler in addition to the package name. if spec.compiler: - if not spack.compilers.supported(spec.compiler): + if not supported_compiler(spec.compiler): raise UnsupportedCompilerError(spec.compiler.name) @@ -1320,7 +1333,7 @@ class SpecParser(spack.parse.Parser): self.expect(ID) self.check_identifier() - compiler = Compiler.__new__(Compiler) + compiler = CompilerSpec.__new__(CompilerSpec) compiler.name = self.token.value compiler.versions = VersionList() if self.accept(AT): @@ -1402,10 +1415,10 @@ class DuplicateVariantError(SpecError): super(DuplicateVariantError, self).__init__(message) -class DuplicateCompilerError(SpecError): +class DuplicateCompilerSpecError(SpecError): """Raised when the same compiler occurs in a spec twice.""" def __init__(self, message): - super(DuplicateCompilerError, self).__init__(message) + super(DuplicateCompilerSpecError, self).__init__(message) class UnsupportedCompilerError(SpecError): diff --git a/lib/spack/spack/test/concretize.py b/lib/spack/spack/test/concretize.py index 2a989f6766..6ad2ef29d8 100644 --- a/lib/spack/spack/test/concretize.py +++ b/lib/spack/spack/test/concretize.py @@ -25,7 +25,7 @@ import unittest import spack -from spack.spec import Spec, Compiler +from spack.spec import Spec, CompilerSpec from spack.test.mock_packages_test import * class ConcretizeTest(MockPackagesTest): @@ -169,7 +169,7 @@ class ConcretizeTest(MockPackagesTest): spec = Spec('mpileaks') spec.normalize() - spec['dyninst'].compiler = Compiler('clang') + spec['dyninst'].compiler = CompilerSpec('clang') spec.concretize() # TODO: not exactly the syntax I would like. diff --git a/lib/spack/spack/test/packages.py b/lib/spack/spack/test/packages.py index 1b2e0ab07a..a8183cf6a6 100644 --- a/lib/spack/spack/test/packages.py +++ b/lib/spack/spack/test/packages.py @@ -28,6 +28,7 @@ from llnl.util.filesystem import join_path import spack import spack.packages as packages +from spack.util.naming import mod_to_class from spack.test.mock_packages_test import * @@ -58,8 +59,8 @@ class PackagesTest(MockPackagesTest): def test_package_class_names(self): - self.assertEqual('Mpich', packages.class_name_for_package_name('mpich')) - self.assertEqual('PmgrCollective', packages.class_name_for_package_name('pmgr_collective')) - self.assertEqual('PmgrCollective', packages.class_name_for_package_name('pmgr-collective')) - self.assertEqual('Pmgrcollective', packages.class_name_for_package_name('PmgrCollective')) - self.assertEqual('Num_3db', packages.class_name_for_package_name('3db')) + self.assertEqual('Mpich', mod_to_class('mpich')) + self.assertEqual('PmgrCollective', mod_to_class('pmgr_collective')) + self.assertEqual('PmgrCollective', mod_to_class('pmgr-collective')) + self.assertEqual('Pmgrcollective', mod_to_class('PmgrCollective')) + self.assertEqual('_3db', mod_to_class('3db')) diff --git a/lib/spack/spack/test/spec_syntax.py b/lib/spack/spack/test/spec_syntax.py index 20468be1b1..404f38906e 100644 --- a/lib/spack/spack/test/spec_syntax.py +++ b/lib/spack/spack/test/spec_syntax.py @@ -133,12 +133,12 @@ class SpecSyntaxTest(unittest.TestCase): self.assertRaises(DuplicateDependencyError, self.check_parse, "x ^y ^y") def test_duplicate_compiler(self): - self.assertRaises(DuplicateCompilerError, self.check_parse, "x%intel%intel") - self.assertRaises(DuplicateCompilerError, self.check_parse, "x%intel%gcc") - self.assertRaises(DuplicateCompilerError, self.check_parse, "x%gcc%intel") - self.assertRaises(DuplicateCompilerError, self.check_parse, "x ^y%intel%intel") - self.assertRaises(DuplicateCompilerError, self.check_parse, "x ^y%intel%gcc") - self.assertRaises(DuplicateCompilerError, self.check_parse, "x ^y%gcc%intel") + self.assertRaises(DuplicateCompilerSpecError, self.check_parse, "x%intel%intel") + self.assertRaises(DuplicateCompilerSpecError, self.check_parse, "x%intel%gcc") + self.assertRaises(DuplicateCompilerSpecError, self.check_parse, "x%gcc%intel") + self.assertRaises(DuplicateCompilerSpecError, self.check_parse, "x ^y%intel%intel") + self.assertRaises(DuplicateCompilerSpecError, self.check_parse, "x ^y%intel%gcc") + self.assertRaises(DuplicateCompilerSpecError, self.check_parse, "x ^y%gcc%intel") # ================================================================================ diff --git a/lib/spack/spack/util/naming.py b/lib/spack/spack/util/naming.py new file mode 100644 index 0000000000..782afbd4bb --- /dev/null +++ b/lib/spack/spack/util/naming.py @@ -0,0 +1,61 @@ +# Need this because of spack.util.string +from __future__ import absolute_import +import string +import re + +import spack + +# Valid module names can contain '-' but can't start with it. +_valid_module_re = r'^\w[\w-]*$' + + +def mod_to_class(mod_name): + """Convert a name from module style to class name style. Spack mostly + follows `PEP-8 `_: + + * Module and package names use lowercase_with_underscores. + * Class names use the CapWords convention. + + Regular source code follows these convetions. Spack is a bit + more liberal with its Package names nad Compiler names: + + * They can contain '-' as well as '_', but cannot start with '-'. + * They can start with numbers, e.g. "3proxy". + + This function converts from the module convention to the class + convention by removing _ and - and converting surrounding + lowercase text to CapWords. If mod_name starts with a number, + the class name returned will be prepended with '_' to make a + valid Python identifier. + """ + validate_module_name(mod_name) + + class_name = re.sub(r'[-_]+', '-', mod_name) + class_name = string.capwords(class_name, '-') + class_name = class_name.replace('-', '') + + # If a class starts with a number, prefix it with Number_ to make it a valid + # Python class name. + if re.match(r'^[0-9]', class_name): + class_name = "_%s" % class_name + + return class_name + + +def valid_module_name(mod_name): + """Return whether the mod_name is valid for use in Spack.""" + return bool(re.match(_valid_module_re, mod_name)) + + +def validate_module_name(mod_name): + """Raise an exception if mod_name is not valid.""" + if not valid_module_name(mod_name): + raise InvalidModuleNameError(mod_name) + + +class InvalidModuleNameError(spack.error.SpackError): + """Raised when we encounter a bad module name.""" + def __init__(self, name): + super(InvalidModuleNameError, self).__init__( + "Invalid module name: " + name) + self.name = name -- cgit v1.2.3-60-g2f50 From ed6454fe78c8de2efb08d3c85e41fddd6fe704fb Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sat, 17 May 2014 15:17:40 -0700 Subject: Better satisfies: e.g., v4.7.3 now satisfies v4.7 - Changed how satisfies() is defined for the various version classes - Can't just use overlaps() with version lists -- need to account for more and less specific versions. If the version is more specific than the constriant (e.g., 4.7.3 is more specific than 4.7), then it should satisfy the constraint, because if a user asks for 4.7 they likely do not care about the minor version. If they do, they can specify it. New Version.satisfies() takes this into account. --- lib/spack/spack/multimethod.py | 2 +- lib/spack/spack/packages.py | 1 + lib/spack/spack/spec.py | 14 ++++---- lib/spack/spack/test/versions.py | 45 ++++++++++++++++++++++++++ lib/spack/spack/version.py | 69 +++++++++++++++++++++++++++++++++++++--- 5 files changed, 118 insertions(+), 13 deletions(-) diff --git a/lib/spack/spack/multimethod.py b/lib/spack/spack/multimethod.py index 8d91e4f86d..974401e1aa 100644 --- a/lib/spack/spack/multimethod.py +++ b/lib/spack/spack/multimethod.py @@ -117,7 +117,7 @@ class SpecMultiMethod(object): or if there is none, then raise a NoSuchMethodError. """ for spec, method in self.method_list: - if spec.satisfies(package_self.spec): + if package_self.spec.satisfies(spec): return method(package_self, *args, **kwargs) if self.default: diff --git a/lib/spack/spack/packages.py b/lib/spack/spack/packages.py index 36f3d4286a..5a31f1fbb9 100644 --- a/lib/spack/spack/packages.py +++ b/lib/spack/spack/packages.py @@ -77,6 +77,7 @@ class PackageDB(object): @_autospec def get_installed(self, spec): + """Get all the installed specs that satisfy the provided spec constraint.""" return [s for s in self.installed_package_specs() if s.satisfies(spec)] diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index f0244695bc..35a17621b6 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -214,17 +214,17 @@ class CompilerSpec(object): def satisfies(self, other): - # TODO: This should not just look for overlapping versions. - # TODO: e.g., 4.7.3 should satisfy a requirement for 4.7. other = self._autospec(other) return (self.name == other.name and - self.versions.overlaps(other.versions)) + self.versions.satisfies(other.versions)) def constrain(self, other): other = self._autospec(other) - if not self.satisfies(other): - raise UnsatisfiableCompilerSpecError(self, other) + + # ensure that other will actually constrain this spec. + if not other.satisfies(self): + raise UnsatisfiableCompilerSpecError(other, self) self.versions.intersect(other.versions) @@ -866,8 +866,8 @@ class Spec(object): # TODO: might want more detail than this, e.g. specific deps # in violation. if this becomes a priority get rid of this # check and be more specici about what's wrong. - if not self.satisfies_dependencies(other): - raise UnsatisfiableDependencySpecError(self, other) + if not other.satisfies_dependencies(self): + raise UnsatisfiableDependencySpecError(other, self) # Handle common first-order constraints directly for name in self.common_dependencies(other): diff --git a/lib/spack/spack/test/versions.py b/lib/spack/spack/test/versions.py index 37fd28a8e7..e272274a4f 100644 --- a/lib/spack/spack/test/versions.py +++ b/lib/spack/spack/test/versions.py @@ -83,6 +83,14 @@ class VersionsTest(unittest.TestCase): self.assertFalse(ver(v1).overlaps(ver(v2))) + def assert_satisfies(self, v1, v2): + self.assertTrue(ver(v1).satisfies(ver(v2))) + + + def assert_does_not_satisfy(self, v1, v2): + self.assertFalse(ver(v1).satisfies(ver(v2))) + + def check_intersection(self, expected, a, b): self.assertEqual(ver(expected), ver(a).intersection(ver(b))) @@ -301,3 +309,40 @@ class VersionsTest(unittest.TestCase): self.check_intersection(['2.5:2.7'], ['1.1:2.7'], ['2.5:3.0','1.0']) self.check_intersection(['0:1'], [':'], ['0:1']) + + + def test_satisfaction(self): + self.assert_satisfies('4.7.3', '4.7.3') + + self.assert_satisfies('4.7.3', '4.7') + self.assert_satisfies('4.7.3b2', '4.7') + self.assert_satisfies('4.7b6', '4.7') + + self.assert_satisfies('4.7.3', '4') + self.assert_satisfies('4.7.3b2', '4') + self.assert_satisfies('4.7b6', '4') + + self.assert_does_not_satisfy('4.8.0', '4.9') + self.assert_does_not_satisfy('4.8', '4.9') + self.assert_does_not_satisfy('4', '4.9') + + self.assert_satisfies('4.7b6', '4.3:4.7') + self.assert_satisfies('4.3.0', '4.3:4.7') + self.assert_satisfies('4.3.2', '4.3:4.7') + + self.assert_does_not_satisfy('4.8.0', '4.3:4.7') + self.assert_does_not_satisfy('4.3', '4.4:4.7') + + self.assert_satisfies('4.7b6', '4.3:4.7') + self.assert_does_not_satisfy('4.8.0', '4.3:4.7') + + self.assert_satisfies('4.7', '4.3, 4.6, 4.7') + self.assert_satisfies('4.7.3', '4.3, 4.6, 4.7') + self.assert_satisfies('4.6.5', '4.3, 4.6, 4.7') + self.assert_satisfies('4.6.5.2', '4.3, 4.6, 4.7') + + self.assert_does_not_satisfy('4', '4.3, 4.6, 4.7') + self.assert_does_not_satisfy('4.8.0', '4.2, 4.3:4.7') + + self.assert_satisfies('4.8.0', '4.2, 4.3:4.8') + self.assert_satisfies('4.8.2', '4.2, 4.3:4.8') diff --git a/lib/spack/spack/version.py b/lib/spack/spack/version.py index 0b5125fdf0..ce94303a9c 100644 --- a/lib/spack/spack/version.py +++ b/lib/spack/spack/version.py @@ -143,6 +143,18 @@ class Version(object): return self + @coerced + def satisfies(self, other): + """A Version 'satisfies' another if it is at least as specific and has a + common prefix. e.g., we want gcc@4.7.3 to satisfy a request for + gcc@4.7 so that when a user asks to build with gcc@4.7, we can find + a suitable compiler. + """ + nself = len(self.version) + nother = len(other.version) + return nother <= nself and self.version[:nother] == other.version + + def wildcard(self): """Create a regex that will match variants of this version string.""" def a_or_n(seg): @@ -326,6 +338,37 @@ class VersionRange(object): none_high.le(other.end, self.end)) + @coerced + def satisfies(self, other): + """A VersionRange satisfies another if some version in this range + would satisfy some version in the other range. To do this it must + either: + a) Overlap with the other range + b) The start of this range satisfies the end of the other range. + + This is essentially the same as overlaps(), but overlaps assumes + that its arguments are specific. That is, 4.7 is interpreted as + 4.7.0.0.0.0... . This funciton assumes that 4.7 woudl be satisfied + by 4.7.3.5, etc. + + Rationale: + If a user asks for gcc@4.5:4.7, and a package is only compatible with + gcc@4.7.3:4.8, then that package should be able to build under the + constraints. Just using overlaps() would not work here. + + Note that we don't need to check whether the end of this range + would satisfy the start of the other range, because overlaps() + already covers that case. + + Note further that overlaps() is a symmetric operation, while + satisfies() is not. + """ + return (self.overlaps(other) or + # if either self.start or other.end are None, then this can't + # satisfy, or overlaps() would've taken care of it. + self.start and other.end and self.start.satisfies(other.end)) + + @coerced def overlaps(self, other): return (other in self or self in other or @@ -444,11 +487,6 @@ class VersionList(object): return self[-1].highest() - def satisfies(self, other): - """Synonym for overlaps.""" - return self.overlaps(other) - - @coerced def overlaps(self, other): if not other or not self: @@ -465,6 +503,27 @@ class VersionList(object): return False + @coerced + def satisfies(self, other): + """A VersionList satisfies another if some version in the list would + would satisfy some version in the other list. This uses essentially + the same algorithm as overlaps() does for VersionList, but it calls + satisfies() on member Versions and VersionRanges. + """ + if not other or not self: + return False + + s = o = 0 + while s < len(self) and o < len(other): + if self[s].satisfies(other[o]): + return True + elif self[s] < other[o]: + s += 1 + else: + o += 1 + return False + + @coerced def update(self, other): for v in other.versions: -- cgit v1.2.3-60-g2f50 From f78475711343a98b1f0e756d6c39df87802c25c8 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 19 May 2014 16:07:42 -0700 Subject: Initial ability to swap compilers. Fixes SPACK-16 and forces compiler script to build using compiler wrappers. - works with gcc and clang on laptop. --- bin/spack | 3 +- lib/spack/env/cc | 47 ++++++++++++++++--------- lib/spack/spack/build_environment.py | 29 +++++++++++---- lib/spack/spack/cmd/compilers.py | 2 +- lib/spack/spack/cmd/find.py | 2 +- lib/spack/spack/cmd/install.py | 6 +++- lib/spack/spack/cmd/uninstall.py | 19 ++++++---- lib/spack/spack/compiler.py | 10 +++--- lib/spack/spack/compilers/__init__.py | 64 ++++++++++++++++++++++------------ lib/spack/spack/compilers/clang.py | 6 ++-- lib/spack/spack/compilers/gcc.py | 6 ++-- lib/spack/spack/compilers/intel.py | 6 ++-- lib/spack/spack/concretize.py | 24 ++++++++++--- lib/spack/spack/package.py | 14 ++++---- lib/spack/spack/spec.py | 41 ++++++++++++---------- lib/spack/spack/test/spec_semantics.py | 36 +++++++++---------- lib/spack/spack/test/versions.py | 30 +++++++++++++++- lib/spack/spack/util/executable.py | 2 +- var/spack/packages/libdwarf/package.py | 2 ++ 19 files changed, 231 insertions(+), 118 deletions(-) diff --git a/bin/spack b/bin/spack index fb9f045f35..fef793a6ce 100755 --- a/bin/spack +++ b/bin/spack @@ -41,7 +41,7 @@ sys.path.insert(0, SPACK_LIB_PATH) # If there is no working directory, use the spack prefix. try: - os.getcwd() + working_dir = os.getcwd() except OSError: os.chdir(SPACK_PREFIX) @@ -79,6 +79,7 @@ args = parser.parse_args() # Set up environment based on args. spack.verbose = args.verbose spack.debug = args.debug +spack.spack_working_dir = working_dir if args.mock: from spack.packages import PackageDB spack.db = PackageDB(spack.mock_packages_path) diff --git a/lib/spack/env/cc b/lib/spack/env/cc index e5dbf21beb..09abf3a31d 100755 --- a/lib/spack/env/cc +++ b/lib/spack/env/cc @@ -10,7 +10,7 @@ import argparse from contextlib import closing # Import spack parameters through the build environment. -spack_lib = os.environ.get("SPACK_LIB") +spack_lib = os.environ.get("SPACK_LIB") if not spack_lib: print "Spack compiler must be run from spack!" sys.exit(1) @@ -20,24 +20,23 @@ sys.path.append(spack_lib) from spack.compilation import * import llnl.util.tty as tty -spack_prefix = get_env_var("SPACK_PREFIX") -spack_build_root = get_env_var("SPACK_BUILD_ROOT") -spack_debug = get_env_flag("SPACK_DEBUG") -spack_deps = get_path("SPACK_DEPENDENCIES") -spack_env_path = get_path("SPACK_ENV_PATH") +spack_prefix = get_env_var("SPACK_PREFIX") +spack_debug = get_env_flag("SPACK_DEBUG") +spack_deps = get_path("SPACK_DEPENDENCIES") +spack_env_path = get_path("SPACK_ENV_PATH") +spack_debug_log_dir = get_env_var("SPACK_DEBUG_LOG_DIR") +spack_spec = get_env_var("SPACK_SPEC") + +spack_cc = get_env_var("SPACK_CC") +spack_cxx = get_env_var("SPACK_CXX") +spack_f77 = get_env_var("SPACK_F77") +spack_fc = get_env_var("SPACK_FC") # Figure out what type of operation we're doing command = os.path.basename(sys.argv[0]) cpp, cc, ccld, ld, version_check = range(5) -######################################################################## -# TODO: this can to be removed once JIRA issue SPACK-16 is resolved -# -if command == 'CC': - command = 'c++' -######################################################################## - if command == 'cpp': mode = cpp elif command == 'ld': @@ -49,7 +48,23 @@ elif '-c' in sys.argv: else: mode = ccld -if '-V' in sys.argv or '-v' in sys.argv or '--version' in sys.argv: + +if command in ('cc', 'gcc', 'c89', 'c99', 'clang'): + command = spack_cc +elif command in ('c++', 'CC', 'g++', 'clang++'): + command = spack_cxx +elif command in ('f77'): + command = spack_f77 +elif command in ('fc'): + command = spack_fc +elif command in ('ld', 'cpp'): + pass # leave it the same. TODO: what's the right thing? +else: + raise Exception("Unknown compiler: %s" % command) + + +version_args = ['-V', '-v', '--version', '-dumpversion'] +if any(arg in sys.argv for arg in version_args): mode = version_check # Parse out the includes, libs, etc. so we can adjust them if need be. @@ -104,8 +119,8 @@ os.environ["PATH"] = ":".join(path) full_command = [command] + arguments if spack_debug: - input_log = os.path.join(spack_build_root, 'spack_cc_in.log') - output_log = os.path.join(spack_build_root, 'spack_cc_out.log') + input_log = os.path.join(spack_debug_log_dir, 'spack-cc-%s.in.log' % spack_spec) + output_log = os.path.join(spack_debug_log_dir, 'spack-cc-%s.out.log' % spack_spec) with closing(open(input_log, 'a')) as log: args = [os.path.basename(sys.argv[0])] + sys.argv[1:] log.write("%s\n" % " ".join(arg.replace(' ', r'\ ') for arg in args)) diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 8d3d0909db..36dae74e84 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -34,7 +34,7 @@ import platform from llnl.util.filesystem import * import spack -from spack.compilers import compiler_for_spec +import spack.compilers as compilers from spack.util.executable import Executable, which from spack.util.environment import * @@ -52,7 +52,9 @@ SPACK_LIB = 'SPACK_LIB' SPACK_ENV_PATH = 'SPACK_ENV_PATH' SPACK_DEPENDENCIES = 'SPACK_DEPENDENCIES' SPACK_PREFIX = 'SPACK_PREFIX' -SPACK_BUILD_ROOT = 'SPACK_BUILD_ROOT' +SPACK_DEBUG = 'SPACK_DEBUG' +SPACK_SPEC = 'SPACK_SPEC' +SPACK_DEBUG_LOG_DIR = 'SPACK_DEBUG_LOG_DIR' class MakeExecutable(Executable): @@ -82,7 +84,19 @@ class MakeExecutable(Executable): def set_compiler_environment_variables(pkg): assert(pkg.spec.concrete) - compiler = compiler_for_spec(pkg.spec.compiler) + compiler = compilers.compiler_for_spec(pkg.spec.compiler) + + # Set compiler variables used by CMake and autotools + os.environ['CC'] = 'cc' + os.environ['CXX'] = 'c++' + os.environ['F77'] = 'f77' + os.environ['FC'] = 'fc' + + # Set SPACK compiler variables so that our wrapper knows what to call + os.environ['SPACK_CC'] = compiler.cc.command + os.environ['SPACK_CXX'] = compiler.cxx.command + os.environ['SPACK_F77'] = compiler.f77.command + os.environ['SPACK_FC'] = compiler.fc.command def set_build_environment_variables(pkg): @@ -108,9 +122,6 @@ def set_build_environment_variables(pkg): # Install prefix os.environ[SPACK_PREFIX] = pkg.prefix - # Build root for logging. - os.environ[SPACK_BUILD_ROOT] = pkg.stage.expanded_archive_path - # Remove these vars from the environment during build becaus they # can affect how some packages find libraries. We want to make # sure that builds never pull in unintended external dependencies. @@ -120,6 +131,12 @@ def set_build_environment_variables(pkg): bin_dirs = ['%s/bin' % prefix for prefix in dep_prefixes] path_put_first('PATH', [bin for bin in bin_dirs if os.path.isdir(bin)]) + # Working directory for the spack command itself, for debug logs. + if spack.debug: + os.environ[SPACK_DEBUG] = "TRUE" + os.environ[SPACK_SPEC] = str(pkg.spec) + os.environ[SPACK_DEBUG_LOG_DIR] = spack.spack_working_dir + def set_module_variables_for_package(pkg): """Populate the module scope of install() with some useful functions. diff --git a/lib/spack/spack/cmd/compilers.py b/lib/spack/spack/cmd/compilers.py index 8267ecbd6b..c34118f033 100644 --- a/lib/spack/spack/cmd/compilers.py +++ b/lib/spack/spack/cmd/compilers.py @@ -34,7 +34,7 @@ description = "List available compilers" def compilers(parser, args): tty.msg("Available compilers") - index = index_by(spack.compilers.available_compilers(), 'name') + index = index_by(spack.compilers.all_compilers(), 'name') for name, compilers in index.items(): tty.hline(name, char='-', color=spack.spec.compiler_color) colify(compilers, indent=4) diff --git a/lib/spack/spack/cmd/find.py b/lib/spack/spack/cmd/find.py index eaa3632043..08cfb5af96 100644 --- a/lib/spack/spack/cmd/find.py +++ b/lib/spack/spack/cmd/find.py @@ -62,7 +62,7 @@ def find(parser, args): # Make a dict with specs keyed by architecture and compiler. specs = [s for s in spack.db.installed_package_specs() - if not query_specs or any(spec.satisfies(q) for q in query_specs)] + if not query_specs or any(s.satisfies(q) for q in query_specs)] index = index_by(specs, 'architecture', 'compiler') # Traverse the index and print out each package diff --git a/lib/spack/spack/cmd/install.py b/lib/spack/spack/cmd/install.py index ea11cb89a9..4570d6c40f 100644 --- a/lib/spack/spack/cmd/install.py +++ b/lib/spack/spack/cmd/install.py @@ -36,7 +36,10 @@ def setup_parser(subparser): help="Do not try to install dependencies of requested packages.") subparser.add_argument( '--keep-prefix', action='store_true', dest='keep_prefix', - help="Don't clean up staging area when install completes.") + help="Don't remove the install prefix if installation fails.") + subparser.add_argument( + '--keep-stage', action='store_true', dest='keep_stage', + help="Don't remove the build stage if installation succeeds.") subparser.add_argument( '-n', '--no-checksum', action='store_true', dest='no_checksum', help="Do not check packages against checksum") @@ -55,4 +58,5 @@ def install(parser, args): for spec in specs: package = spack.db.get(spec) package.do_install(keep_prefix=args.keep_prefix, + keep_stage=args.keep_stage, ignore_deps=args.ignore_deps) diff --git a/lib/spack/spack/cmd/uninstall.py b/lib/spack/spack/cmd/uninstall.py index df208b3a6a..044a9b2960 100644 --- a/lib/spack/spack/cmd/uninstall.py +++ b/lib/spack/spack/cmd/uninstall.py @@ -35,6 +35,11 @@ def setup_parser(subparser): subparser.add_argument( '-f', '--force', action='store_true', dest='force', help="Remove regardless of whether other packages depend on this one.") + subparser.add_argument( + '-a', '--all', action='store_true', dest='all', + help="USE CAREFULLY. Remove ALL installed packages that match each supplied spec. " + + "i.e., if you say uninstall libelf, ALL versions of libelf are uninstalled. " + + "This is both useful and dangerous, like rm -r.") subparser.add_argument( 'packages', nargs=argparse.REMAINDER, help="specs of packages to uninstall") @@ -50,15 +55,17 @@ def uninstall(parser, args): pkgs = [] for spec in specs: matching_specs = spack.db.get_installed(spec) - if len(matching_specs) > 1: - tty.die("%s matches multiple packages. Which one did you mean?" - % spec, *matching_specs) + if not args.all and len(matching_specs) > 1: + tty.die("%s matches multiple packages." % spec, + "You can either:", + " a) Use spack uninstall -a to uninstall ALL matching specs, or", + " b) use a more specific spec.", + "Matching packages:", *(" " + str(s) for s in matching_specs)) - elif len(matching_specs) == 0: + if len(matching_specs) == 0: tty.die("%s does not match any installed packages." % spec) - installed_spec = matching_specs[0] - pkgs.append(spack.db.get(installed_spec)) + pkgs.extend(spack.db.get(s) for s in matching_specs) # Sort packages to be uninstalled by the number of installed dependents # This ensures we do things in the right order diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index 27effa20c4..3d097b6180 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -16,7 +16,7 @@ def _verify_executables(*paths): class Compiler(object): """This class encapsulates a Spack "compiler", which includes C, - C++, Fortran, and F90 compilers. Subclasses should implement + C++, and Fortran compilers. Subclasses should implement support for specific compilers, their possible names, arguments, and how to identify the particular type of compiler.""" @@ -30,20 +30,20 @@ class Compiler(object): f77_names = [] # Subclasses use possible names of Fortran 90 compiler - f90_names = [] + fc_names = [] # Names of generic arguments used by this compiler arg_version = '-dumpversion' arg_rpath = '-Wl,-rpath,%s' - def __init__(self, cc, cxx, f77, f90): - _verify_executables(cc, cxx, f77, f90) + def __init__(self, cc, cxx, f77, fc): + _verify_executables(cc, cxx, f77, fc) self.cc = Executable(cc) self.cxx = Executable(cxx) self.f77 = Executable(f77) - self.f90 = Executable(f90) + self.fc = Executable(fc) @property @memoized diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index 095e26edb5..a36ea618bc 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -41,49 +41,69 @@ _imported_compilers_module = 'spack.compiler.versions' _imported_versions_module = 'spack.compilers' +def _auto_compiler_spec(function): + def converter(cspec_like): + if not isinstance(cspec_like, spack.spec.CompilerSpec): + cspec_like = spack.spec.CompilerSpec(cspec_like) + return function(cspec_like) + return converter + + @memoized def supported_compilers(): - """Return a list of names of compilers supported by Spack. + """Return a set of names of compilers supported by Spack. See available_compilers() to get a list of all the available versions of supported compilers. """ - return sorted(c for c in list_modules(spack.compilers_path)) + return sorted(name for name in list_modules(spack.compilers_path)) +@_auto_compiler_spec def supported(compiler_spec): """Test if a particular compiler is supported.""" - if not isinstance(compiler_spec, spack.spec.CompilerSpec): - compiler_spec = spack.spec.CompilerSpec(compiler_spec) return compiler_spec.name in supported_compilers() -def available_compilers(): - """Return a list of specs for all the compiler versions currently - available to build with. These are instances of - CompilerSpec. +@memoized +def all_compilers(): + """Return a set of specs for all the compiler versions currently + available to build with. These are instances of CompilerSpec. """ - return [spack.spec.CompilerSpec(c) - for c in list_modules(spack.compiler_version_path)] + return set(spack.spec.CompilerSpec(c) + for c in list_modules(spack.compiler_version_path)) -def compiler_for_spec(compiler_spec): - """This gets an instance of an actual spack.compiler.Compiler object - from a compiler spec. The spec needs to be concrete for this to - work; it will raise an error if passed an abstract compiler. +@_auto_compiler_spec +def find(compiler_spec): + """Return specs of available compilers that match the supplied + compiler spec. Return an list if nothing found.""" + return [c for c in all_compilers() if c.satisfies(compiler_spec)] + + +@_auto_compiler_spec +def compilers_for_spec(compiler_spec): + """This gets all compilers that satisfy the supplied CompilerSpec. + Returns an empty list if none are found. """ - matches = [c for c in available_compilers() if c.satisfies(compiler_spec)] + matches = find(compiler_spec) - # TODO: do something when there are zero matches. - assert(len(matches) >= 1) + compilers = [] + for cspec in matches: + path = join_path(spack.compiler_version_path, "%s.py" % cspec) + mod = imp.load_source(_imported_versions_module, path) + cls = class_for_compiler_name(cspec.name) + compilers.append(cls(mod.cc, mod.cxx, mod.f77, mod.fc)) - compiler = matches[0] - file_path = join_path(spack.compiler_version_path, "%s.py" % compiler) + return compilers - mod = imp.load_source(_imported_versions_module, file_path) - compiler_class = class_for_compiler_name(compiler.name) - return compiler_class(mod.cc, mod.cxx, mod.f77, mod.f90) +@_auto_compiler_spec +def compiler_for_spec(compiler_spec): + assert(compiler_spec.concrete) + compilers = compilers_for_spec(compiler_spec) + assert(len(compilers) == 1) + return compilers[0] def class_for_compiler_name(compiler_name): diff --git a/lib/spack/spack/compilers/clang.py b/lib/spack/spack/compilers/clang.py index 9ed7b57846..1616fcaf08 100644 --- a/lib/spack/spack/compilers/clang.py +++ b/lib/spack/spack/compilers/clang.py @@ -35,7 +35,7 @@ class Clang(Compiler): f77_names = [] # Subclasses use possible names of Fortran 90 compiler - f90_names = [] + fc_names = [] - def __init__(self, cc, cxx, f77, f90): - super(Gcc, self).__init__(cc, cxx, f77, f90) + def __init__(self, cc, cxx, f77, fc): + super(Clang, self).__init__(cc, cxx, f77, fc) diff --git a/lib/spack/spack/compilers/gcc.py b/lib/spack/spack/compilers/gcc.py index 638051008f..f73cb08c63 100644 --- a/lib/spack/spack/compilers/gcc.py +++ b/lib/spack/spack/compilers/gcc.py @@ -35,7 +35,7 @@ class Gcc(Compiler): f77_names = ['gfortran'] # Subclasses use possible names of Fortran 90 compiler - f90_names = ['gfortran'] + fc_names = ['gfortran'] - def __init__(self, cc, cxx, f77, f90): - super(Gcc, self).__init__(cc, cxx, f77, f90) + def __init__(self, cc, cxx, f77, fc): + super(Gcc, self).__init__(cc, cxx, f77, fc) diff --git a/lib/spack/spack/compilers/intel.py b/lib/spack/spack/compilers/intel.py index ebbab57ed1..fe2aabd864 100644 --- a/lib/spack/spack/compilers/intel.py +++ b/lib/spack/spack/compilers/intel.py @@ -35,7 +35,7 @@ class Intel(Compiler): f77_names = ['ifort'] # Subclasses use possible names of Fortran 90 compiler - f90_names = ['ifort'] + fc_names = ['ifort'] - def __init__(self, cc, cxx, f77, f90): - super(Gcc, self).__init__(cc, cxx, f77, f90) + def __init__(self, cc, cxx, f77, fc): + super(Intel, self).__init__(cc, cxx, f77, fc) diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 28efcaea64..312b9ce1b1 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -33,9 +33,10 @@ or user preferences. TODO: make this customizable and allow users to configure concretization policies. """ -import spack.architecture -import spack.compilers import spack.spec +import spack.compilers +import spack.architecture +import spack.error from spack.version import * @@ -117,9 +118,13 @@ class DefaultConcretizer(object): if p.compiler is not None).compiler if not nearest.concrete: - matches = [c for c in spack.compilers.available_compilers() - if c.name == nearest.name] - nearest.versions = sorted(matches)[-1].versions.copy() + # Take the newest compiler that saisfies the spec + matches = sorted(spack.compilers.find(nearest)) + if not matches: + raise UnavailableCompilerVersionError(nearest) + + # copy concrete version into nearest spec + nearest.versions = matches[-1].versions.copy() assert(nearest.concrete) spec.compiler = nearest.copy() @@ -140,3 +145,12 @@ class DefaultConcretizer(object): first_key = sorted(index.keys())[0] latest_version = sorted(index[first_key])[-1] return latest_version + + +class UnavailableCompilerVersionError(spack.error.SpackError): + """Raised when there is no available compiler that satisfies a + compiler spec.""" + def __init__(self, compiler_spec): + super(UnavailableCompilerVersionError, self).__init__( + "No available compiler version matches '%s'" % compiler_spec, + "Run 'spack compilers' to see available compiler Options.") diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index b40448df37..069c66f4a7 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -626,6 +626,7 @@ class Package(object): """ # whether to keep the prefix on failure. Default is to destroy it. keep_prefix = kwargs.get('keep_prefix', False) + keep_stage = kwargs.get('keep_stage', False) ignore_deps = kwargs.get('ignore_deps', False) if not self.spec.concrete: @@ -671,9 +672,10 @@ class Package(object): "Install failed for %s. Nothing was installed!" % self.name) - # On successful install, remove the stage. - # Leave if there is an error - self.stage.destroy() + if not keep_stage: + # On successful install, remove the stage. + # Leave it if there is an error + self.stage.destroy() tty.msg("Successfully installed %s" % self.name) print_pkg(self.prefix) @@ -725,16 +727,16 @@ class Package(object): force = kwargs.get('force', False) if not self.installed: - raise InstallError(self.name + " is not installed.") + raise InstallError(self.spec + " is not installed.") if not force: deps = self.installed_dependents if deps: raise InstallError( "Cannot uninstall %s. The following installed packages depend on it: %s" - % (self.name, deps)) + % (self.spec, deps)) self.remove_prefix() - tty.msg("Successfully uninstalled %s." % self.name) + tty.msg("Successfully uninstalled %s." % self.spec) def do_clean(self): diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 35a17621b6..5848ac2000 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -102,7 +102,7 @@ from llnl.util.tty.color import * import spack import spack.parse import spack.error -from spack.compilers import supported as supported_compiler +import spack.compilers as compilers from spack.version import * from spack.util.string import * @@ -231,8 +231,9 @@ class CompilerSpec(object): @property def concrete(self): - """A CompilerSpec is concrete if its versions are concrete.""" - return self.versions.concrete + """A CompilerSpec is concrete if its versions are concrete and there + is an available compiler with the right version.""" + return self.versions.concrete and self in compilers.all_compilers() @property @@ -260,6 +261,9 @@ class CompilerSpec(object): out += "@%s" % vlist return out + def __repr__(self): + return str(self) + @key_ordering class Variant(object): @@ -821,12 +825,13 @@ class Spec(object): # validate compiler in addition to the package name. if spec.compiler: - if not supported_compiler(spec.compiler): + if not compilers.supported(spec.compiler): raise UnsupportedCompilerError(spec.compiler.name) def constrain(self, other, **kwargs): other = self._autospec(other) + constrain_deps = kwargs.get('deps', True) if not self.name == other.name: raise UnsatisfiableSpecNameError(self.name, other.name) @@ -854,7 +859,7 @@ class Spec(object): self.variants.update(other.variants) self.architecture = self.architecture or other.architecture - if kwargs.get('deps', True): + if constrain_deps: self._constrain_dependencies(other) @@ -911,28 +916,28 @@ class Spec(object): def satisfies(self, other, **kwargs): other = self._autospec(other) + satisfy_deps = kwargs.get('deps', True) # First thing we care about is whether the name matches if self.name != other.name: return False - # This function simplifies null checking below - def check(attribute, op): - s = getattr(self, attribute) - o = getattr(other, attribute) - return not s or not o or op(s,o) - - # All these attrs have satisfies criteria of their own - for attr in ('versions', 'variants', 'compiler'): - if not check(attr, lambda s, o: s.satisfies(o)): + # All these attrs have satisfies criteria of their own, + # but can be None to indicate no constraints. + for s, o in ((self.versions, other.versions), + (self.variants, other.variants), + (self.compiler, other.compiler)): + if s and o and not s.satisfies(o): return False - # Architecture is just a string - # TODO: inviestigate making an Architecture class for symmetry - if not check('architecture', lambda s,o: s == o): + # Architecture satisfaction is currently just string equality. + # Can be None for unconstrained, though. + if (self.architecture and other.architecture and + self.architecture != other.architecture): return False - if kwargs.get('deps', True): + # If we need to descend into dependencies, do it, otherwise we're done. + if satisfy_deps: return self.satisfies_dependencies(other) else: return True diff --git a/lib/spack/spack/test/spec_semantics.py b/lib/spack/spack/test/spec_semantics.py index dbd04f9449..5fb09e68af 100644 --- a/lib/spack/spack/test/spec_semantics.py +++ b/lib/spack/spack/test/spec_semantics.py @@ -37,16 +37,13 @@ class SpecSematicsTest(MockPackagesTest): left = Spec(spec) right = parse_anonymous_spec(anon_spec, left.name) + # Satisfies is one-directional. self.assertTrue(left.satisfies(right)) self.assertTrue(left.satisfies(anon_spec)) - self.assertTrue(right.satisfies(left)) - try: - left.copy().constrain(right) - left.copy().constrain(anon_spec) - right.copy().constrain(left) - except SpecError, e: - self.fail("Got a SpecError in constrain! " + e.message) + # if left satisfies right, then we should be able to consrain + # right by left. Reverse is not always true. + right.copy().constrain(left) def check_unsatisfiable(self, spec, anon_spec): @@ -56,25 +53,21 @@ class SpecSematicsTest(MockPackagesTest): self.assertFalse(left.satisfies(right)) self.assertFalse(left.satisfies(anon_spec)) - self.assertFalse(right.satisfies(left)) + self.assertRaises(UnsatisfiableSpecError, right.copy().constrain, left) - self.assertRaises(UnsatisfiableSpecError, left.constrain, right) - self.assertRaises(UnsatisfiableSpecError, left.constrain, anon_spec) - self.assertRaises(UnsatisfiableSpecError, right.constrain, left) - - def check_constrain(self, expected, constrained, constraint): + def check_constrain(self, expected, spec, constraint): exp = Spec(expected) - constrained = Spec(constrained) + spec = Spec(spec) constraint = Spec(constraint) - constrained.constrain(constraint) - self.assertEqual(exp, constrained) + spec.constrain(constraint) + self.assertEqual(exp, spec) - def check_invalid_constraint(self, constrained, constraint): - constrained = Spec(constrained) + def check_invalid_constraint(self, spec, constraint): + spec = Spec(spec) constraint = Spec(constraint) - self.assertRaises(UnsatisfiableSpecError, constrained.constrain, constraint) + self.assertRaises(UnsatisfiableSpecError, spec.constrain, constraint) # ================================================================================ @@ -177,3 +170,8 @@ class SpecSematicsTest(MockPackagesTest): self.check_invalid_constraint('libelf+debug~foo', 'libelf+debug+foo') self.check_invalid_constraint('libelf=bgqos_0', 'libelf=x86_54') + + + def test_compiler_satisfies(self): + self.check_satisfies('foo %gcc@4.7.3', '%gcc@4.7') + self.check_unsatisfiable('foo %gcc@4.7', '%gcc@4.7.3') diff --git a/lib/spack/spack/test/versions.py b/lib/spack/spack/test/versions.py index e272274a4f..454ab36b8a 100644 --- a/lib/spack/spack/test/versions.py +++ b/lib/spack/spack/test/versions.py @@ -311,7 +311,7 @@ class VersionsTest(unittest.TestCase): self.check_intersection(['0:1'], [':'], ['0:1']) - def test_satisfaction(self): + def test_basic_version_satisfaction(self): self.assert_satisfies('4.7.3', '4.7.3') self.assert_satisfies('4.7.3', '4.7') @@ -326,6 +326,22 @@ class VersionsTest(unittest.TestCase): self.assert_does_not_satisfy('4.8', '4.9') self.assert_does_not_satisfy('4', '4.9') + def test_basic_version_satisfaction_in_lists(self): + self.assert_satisfies(['4.7.3'], ['4.7.3']) + + self.assert_satisfies(['4.7.3'], ['4.7']) + self.assert_satisfies(['4.7.3b2'], ['4.7']) + self.assert_satisfies(['4.7b6'], ['4.7']) + + self.assert_satisfies(['4.7.3'], ['4']) + self.assert_satisfies(['4.7.3b2'], ['4']) + self.assert_satisfies(['4.7b6'], ['4']) + + self.assert_does_not_satisfy(['4.8.0'], ['4.9']) + self.assert_does_not_satisfy(['4.8'], ['4.9']) + self.assert_does_not_satisfy(['4'], ['4.9']) + + def test_version_range_satisfaction(self): self.assert_satisfies('4.7b6', '4.3:4.7') self.assert_satisfies('4.3.0', '4.3:4.7') self.assert_satisfies('4.3.2', '4.3:4.7') @@ -336,6 +352,18 @@ class VersionsTest(unittest.TestCase): self.assert_satisfies('4.7b6', '4.3:4.7') self.assert_does_not_satisfy('4.8.0', '4.3:4.7') + def test_version_range_satisfaction_in_lists(self): + self.assert_satisfies(['4.7b6'], ['4.3:4.7']) + self.assert_satisfies(['4.3.0'], ['4.3:4.7']) + self.assert_satisfies(['4.3.2'], ['4.3:4.7']) + + self.assert_does_not_satisfy(['4.8.0'], ['4.3:4.7']) + self.assert_does_not_satisfy(['4.3'], ['4.4:4.7']) + + self.assert_satisfies(['4.7b6'], ['4.3:4.7']) + self.assert_does_not_satisfy(['4.8.0'], ['4.3:4.7']) + + def test_satisfaction_with_lists(self): self.assert_satisfies('4.7', '4.3, 4.6, 4.7') self.assert_satisfies('4.7.3', '4.3, 4.6, 4.7') self.assert_satisfies('4.6.5', '4.3, 4.6, 4.7') diff --git a/lib/spack/spack/util/executable.py b/lib/spack/spack/util/executable.py index 845d1470e7..6dea46a5c8 100644 --- a/lib/spack/spack/util/executable.py +++ b/lib/spack/spack/util/executable.py @@ -43,7 +43,7 @@ class Executable(object): @property def command(self): - return self.exe[0] + return ' '.join(self.exe) def __call__(self, *args, **kwargs): diff --git a/var/spack/packages/libdwarf/package.py b/var/spack/packages/libdwarf/package.py index 657e84705a..000187736c 100644 --- a/var/spack/packages/libdwarf/package.py +++ b/var/spack/packages/libdwarf/package.py @@ -39,6 +39,8 @@ class Libdwarf(Package): depends_on("libelf") + parallel = False + def clean(self): for dir in dwarf_dirs: -- cgit v1.2.3-60-g2f50 From b6f10dcdf31731aea71856d1dbc533192d7ba29a Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 20 May 2014 14:28:44 -0700 Subject: Adding per-compiler python files --- var/spack/compilers/clang@3.3.py | 4 ++++ var/spack/compilers/gcc@4.5.4.py | 4 ++++ var/spack/compilers/gcc@4.7.3.py | 4 ++++ var/spack/compilers/gcc@4.8.1.py | 4 ++++ 4 files changed, 16 insertions(+) create mode 100644 var/spack/compilers/clang@3.3.py create mode 100644 var/spack/compilers/gcc@4.5.4.py create mode 100644 var/spack/compilers/gcc@4.7.3.py create mode 100644 var/spack/compilers/gcc@4.8.1.py diff --git a/var/spack/compilers/clang@3.3.py b/var/spack/compilers/clang@3.3.py new file mode 100644 index 0000000000..f07ca4d6cc --- /dev/null +++ b/var/spack/compilers/clang@3.3.py @@ -0,0 +1,4 @@ +cc = '/usr/bin/clang' +cxx = '/usr/bin/clang++' +f77 = 'gfortran' +fc = 'gfortran' diff --git a/var/spack/compilers/gcc@4.5.4.py b/var/spack/compilers/gcc@4.5.4.py new file mode 100644 index 0000000000..5ead666d70 --- /dev/null +++ b/var/spack/compilers/gcc@4.5.4.py @@ -0,0 +1,4 @@ +cc = '/Users/gamblin2/macports/bin/gcc-mp-4.5' +cxx = '/Users/gamblin2/macports/bin/g++-mp-4.5' +f77 = '/Users/gamblin2/macports/bin/gfortran-mp-4.5' +fc = '/Users/gamblin2/macports/bin/gfortran-mp-4.5' diff --git a/var/spack/compilers/gcc@4.7.3.py b/var/spack/compilers/gcc@4.7.3.py new file mode 100644 index 0000000000..cdc092c8fd --- /dev/null +++ b/var/spack/compilers/gcc@4.7.3.py @@ -0,0 +1,4 @@ +cc = '/Users/gamblin2/macports/bin/gcc-mp-4.7' +cxx = '/Users/gamblin2/macports/bin/g++-mp-4.7' +f77 = '/Users/gamblin2/macports/bin/gfortran-mp-4.7' +fc = '/Users/gamblin2/macports/bin/gfortran-mp-4.7' diff --git a/var/spack/compilers/gcc@4.8.1.py b/var/spack/compilers/gcc@4.8.1.py new file mode 100644 index 0000000000..f041b4d11c --- /dev/null +++ b/var/spack/compilers/gcc@4.8.1.py @@ -0,0 +1,4 @@ +cc = '/Users/gamblin2/macports/bin/gcc-mp-4.8' +cxx = '/Users/gamblin2/macports/bin/g++-mp-4.8' +f77 = '/Users/gamblin2/macports/bin/gfortran-mp-4.8' +fc = '/Users/gamblin2/macports/bin/gfortran-mp-4.8' -- cgit v1.2.3-60-g2f50 From e70ac872cb59414e0c0e1593162f73ec5945f8f3 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 15 Jun 2014 18:57:05 -0700 Subject: Move globals to spack's __init__.py --- lib/spack/spack/__init__.py | 141 +++++++++++++++++++++++++++++++++-- lib/spack/spack/globals.py | 139 ---------------------------------- lib/spack/spack/test/all_packages.py | 37 +++++++++ 3 files changed, 172 insertions(+), 145 deletions(-) delete mode 100644 lib/spack/spack/globals.py create mode 100644 lib/spack/spack/test/all_packages.py diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index 77aad98524..42769f0bdb 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -22,10 +22,139 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -from globals import * -from util import * -from error import * -from package import Package -from relations import depends_on, provides, patch -from multimethod import when +# +# When packages call 'from spack import *', this is what is brought in. +# +# Spack internal code calls 'import spack' and accesses other +# variables (spack.db, paths, etc.) directly. +# +__all__ = ['Package', 'when', 'provides', 'depends_on', 'patch'] + +import os +import tempfile +from llnl.util.filesystem import * + +# This lives in $prefix/lib/spac/spack/__file__ +prefix = ancestor(__file__, 4) + +# The spack script itself +spack_file = join_path(prefix, "bin", "spack") + +# spack directory hierarchy +etc_path = join_path(prefix, "etc") +lib_path = join_path(prefix, "lib", "spack") +build_env_path = join_path(lib_path, "env") +module_path = join_path(lib_path, "spack") +compilers_path = join_path(module_path, "compilers") +test_path = join_path(module_path, "test") +var_path = join_path(prefix, "var", "spack") +stage_path = join_path(var_path, "stage") +install_path = join_path(prefix, "opt") + +# +# Place to look for usable compiler versions. +# +compiler_version_path = join_path(var_path, "compilers") + +# +# Set up the packages database. +# +from spack.packages import PackageDB +packages_path = join_path(var_path, "packages") +db = PackageDB(packages_path) + +# +# This is the path to mock packages used by spack for testing. +# +mock_packages_path = join_path(var_path, "mock_packages") + +# +# This controls how spack lays out install prefixes and +# stage directories. +# +from spack.directory_layout import SpecHashDirectoryLayout +install_layout = SpecHashDirectoryLayout(install_path, prefix_size=6) + +# +# This controls how things are concretized in spack. +# Replace it with a subclass if you want different +# policies. +# +from spack.concretize import DefaultConcretizer +concretizer = DefaultConcretizer() + +# Version information +from spack.version import Version +spack_version = Version("1.0") + +# +# Executables used by Spack +# +from spack.util.executable import Executable, which + +# User's editor from the environment +editor = Executable(os.environ.get("EDITOR", "")) + +# Curl tool for fetching files. +curl = which("curl", required=True) + +# Whether to build in tmp space or directly in the stage_path. +# If this is true, then spack will make stage directories in +# a tmp filesystem, and it will symlink them into stage_path. +use_tmp_stage = True + +# Locations to use for staging and building, in order of preference +# Use a %u to add a username to the stage paths here, in case this +# is a shared filesystem. Spack will use the first of these paths +# that it can create. +tmp_dirs = [] +_default_tmp = tempfile.gettempdir() +if _default_tmp != os.getcwd(): + tmp_dirs.append(os.path.join(_default_tmp, 'spack-stage')) +tmp_dirs.append('/nfs/tmp2/%u/spack-stage') + +# Whether spack should allow installation of unsafe versions of +# software. "Unsafe" versions are ones it doesn't have a checksum +# for. +do_checksum = True + +# +# SYS_TYPE to use for the spack installation. +# Value of this determines what platform spack thinks it is by +# default. You can assign three types of values: +# 1. None +# Spack will try to determine the sys_type automatically. +# +# 2. A string +# Spack will assume that the sys_type is hardcoded to the value. +# +# 3. A function that returns a string: +# Spack will use this function to determine the sys_type. +# +sys_type = None + +# +# Places to download tarballs from. +# +# TODO: move to configuration. +# +# Examples: +# +# For a local directory: +# mirrors = ['file:///Users/gamblin2/spack-mirror'] +# +# For a website: +# mirrors = ['http://spackports.org/spack-mirror/'] +# +# For no mirrors: +# mirrors = [] +# +mirrors = [] + +# +# Extra imports that should be generally usable from package.py files. +# +from spack.package import Package +from spack.relations import depends_on, provides, patch +from spack.multimethod import when diff --git a/lib/spack/spack/globals.py b/lib/spack/spack/globals.py deleted file mode 100644 index c67cb02060..0000000000 --- a/lib/spack/spack/globals.py +++ /dev/null @@ -1,139 +0,0 @@ -############################################################################## -# Copyright (c) 2013, Lawrence Livermore National Security, LLC. -# Produced at the Lawrence Livermore National Laboratory. -# -# This file is part of Spack. -# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. -# LLNL-CODE-647188 -# -# For details, see https://scalability-llnl.github.io/spack -# Please also see the LICENSE file for our notice and the LGPL. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License (as published by -# the Free Software Foundation) version 2.1 dated February 1999. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and -# conditions of the GNU General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -############################################################################## -import os -import tempfile - -from llnl.util.filesystem import * - -from spack.version import Version -from spack.util.executable import * -from spack.directory_layout import SpecHashDirectoryLayout -from spack.concretize import DefaultConcretizer -from spack.packages import PackageDB - -# This lives in $prefix/lib/spac/spack/__file__ -prefix = ancestor(__file__, 4) - -# The spack script itself -spack_file = join_path(prefix, "bin", "spack") - -# spack directory hierarchy -lib_path = join_path(prefix, "lib", "spack") -build_env_path = join_path(lib_path, "env") -module_path = join_path(lib_path, "spack") -compilers_path = join_path(module_path, "compilers") -test_path = join_path(module_path, "test") -var_path = join_path(prefix, "var", "spack") -stage_path = join_path(var_path, "stage") - -install_path = join_path(prefix, "opt") - -# -# Place to look for usable compiler versions. -# -compiler_version_path = join_path(var_path, "compilers") - -# -# Set up the packages database. -# -packages_path = join_path(var_path, "packages") -db = PackageDB(packages_path) - -# -# This is the path to mock packages used by spack for testing. -# -mock_packages_path = join_path(var_path, "mock_packages") - -# -# This controls how spack lays out install prefixes and -# stage directories. -# -install_layout = SpecHashDirectoryLayout(install_path, prefix_size=6) - -# -# This controls how things are concretized in spack. -# Replace it with a subclass if you want different -# policies. -# -concretizer = DefaultConcretizer() - -# Version information -spack_version = Version("1.0") - -# User's editor from the environment -editor = Executable(os.environ.get("EDITOR", "")) - -# Curl tool for fetching files. -curl = which("curl", required=True) - -# Whether to build in tmp space or directly in the stage_path. -# If this is true, then spack will make stage directories in -# a tmp filesystem, and it will symlink them into stage_path. -use_tmp_stage = True - -# Locations to use for staging and building, in order of preference -# Use a %u to add a username to the stage paths here, in case this -# is a shared filesystem. Spack will use the first of these paths -# that it can create. -tmp_dirs = [] -_default_tmp = tempfile.gettempdir() -if _default_tmp != os.getcwd(): - tmp_dirs.append(os.path.join(_default_tmp, 'spack-stage')) -tmp_dirs.append('/nfs/tmp2/%u/spack-stage') - -# Whether spack should allow installation of unsafe versions of -# software. "Unsafe" versions are ones it doesn't have a checksum -# for. -do_checksum = True - -# -# SYS_TYPE to use for the spack installation. -# Value of this determines what platform spack thinks it is by -# default. You can assign three types of values: -# 1. None -# Spack will try to determine the sys_type automatically. -# -# 2. A string -# Spack will assume that the sys_type is hardcoded to the value. -# -# 3. A function that returns a string: -# Spack will use this function to determine the sys_type. -# -sys_type = None - -# -# Places to download tarballs from. Examples: -# -# For a local directory: -# mirrors = ['file:///Users/gamblin2/spack-mirror'] -# -# For a website: -# mirrors = ['http://spackports.org/spack-mirror/'] -# -# For no mirrors: -# mirrors = [] -# -mirrors = [] - diff --git a/lib/spack/spack/test/all_packages.py b/lib/spack/spack/test/all_packages.py new file mode 100644 index 0000000000..a525f3c012 --- /dev/null +++ b/lib/spack/spack/test/all_packages.py @@ -0,0 +1,37 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +import unittest +import spack + + +class AllPackagesTest(unittest.TestCase): + """This test makes sure that each package in Spack is possible to + import. If it's not then there's some basic error either + in the package or in some change that has been made in Spack. + """ + + def test_get_all_packages(self): + for name in spack.db.all_package_names(): + spack.db.get(name) -- cgit v1.2.3-60-g2f50 From 15f56ccee0b8002d7bdd7fec7d8f45eda49dcb2a Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 15 Jun 2014 18:57:05 -0700 Subject: Move globals to spack's __init__.py --- lib/spack/spack/test/all_packages.py | 37 ---------------------------------- lib/spack/spack/test/package_sanity.py | 7 +++++++ 2 files changed, 7 insertions(+), 37 deletions(-) delete mode 100644 lib/spack/spack/test/all_packages.py diff --git a/lib/spack/spack/test/all_packages.py b/lib/spack/spack/test/all_packages.py deleted file mode 100644 index a525f3c012..0000000000 --- a/lib/spack/spack/test/all_packages.py +++ /dev/null @@ -1,37 +0,0 @@ -############################################################################## -# Copyright (c) 2013, Lawrence Livermore National Security, LLC. -# Produced at the Lawrence Livermore National Laboratory. -# -# This file is part of Spack. -# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. -# LLNL-CODE-647188 -# -# For details, see https://scalability-llnl.github.io/spack -# Please also see the LICENSE file for our notice and the LGPL. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License (as published by -# the Free Software Foundation) version 2.1 dated February 1999. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and -# conditions of the GNU General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -############################################################################## -import unittest -import spack - - -class AllPackagesTest(unittest.TestCase): - """This test makes sure that each package in Spack is possible to - import. If it's not then there's some basic error either - in the package or in some change that has been made in Spack. - """ - - def test_get_all_packages(self): - for name in spack.db.all_package_names(): - spack.db.get(name) diff --git a/lib/spack/spack/test/package_sanity.py b/lib/spack/spack/test/package_sanity.py index a0bf4ceb4a..1a7bc5dc5e 100644 --- a/lib/spack/spack/test/package_sanity.py +++ b/lib/spack/spack/test/package_sanity.py @@ -31,6 +31,13 @@ import spack import spack.url as url class PackageSanityTest(unittest.TestCase): + + def test_get_all_packages(self): + """Get all packages once and make sure that works.""" + for name in spack.db.all_package_names(): + spack.db.get(name) + + def test_url_versions(self): """Ensure that url_for_version does the right thing for at least the default version of each package. -- cgit v1.2.3-60-g2f50 From 042a4730e345ac0ee58a43bfedd11a7d32efb490 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 15 Jun 2014 19:52:00 -0700 Subject: Proper exiting for forked process in do_install() --- lib/spack/spack/package.py | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 069c66f4a7..fff471599a 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -651,18 +651,18 @@ class Package(object): raise InstallError("Unable to fork build process: %s" % e) if pid == 0: - tty.msg("Building %s." % self.name) + try: + tty.msg("Building %s." % self.name) - # create the install directory (allow the layout to handle - # this in case it needs to add extra files) - spack.install_layout.make_path_for_spec(self.spec) + # create the install directory (allow the layout to handle + # this in case it needs to add extra files) + spack.install_layout.make_path_for_spec(self.spec) - # Set up process's build environment before running install. - build_env.set_compiler_environment_variables(self) - build_env.set_build_environment_variables(self) - build_env.set_module_variables_for_package(self) + # Set up process's build environment before running install. + build_env.set_compiler_environment_variables(self) + build_env.set_build_environment_variables(self) + build_env.set_module_variables_for_package(self) - try: # Subclasses implement install() to do the build & # install work. self.install(self.spec, self.prefix) @@ -693,7 +693,11 @@ class Package(object): "Spack will think this package is installed." + "Manually remove this directory to fix:", self.prefix) - raise + + # Child doesn't raise or return to main spack code. + # Just runs default exception handler and exits. + sys.excepthook(*sys.exc_info()) + os._exit(1) # Parent process just waits for the child to complete. If the # child exited badly, assume it already printed an appropriate @@ -727,7 +731,7 @@ class Package(object): force = kwargs.get('force', False) if not self.installed: - raise InstallError(self.spec + " is not installed.") + raise InstallError(str(self.spec) + " is not installed.") if not force: deps = self.installed_dependents -- cgit v1.2.3-60-g2f50 From c8414a8a40d73e7c93b3a0f3d84d5141246f518c Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 17 Jun 2014 19:23:14 -0500 Subject: Add support for configuration files. Fix SPACK-24. --- .gitignore | 1 + lib/spack/spack/cmd/config.py | 77 +++++++ lib/spack/spack/config.py | 449 +++++++++++++++++++++++++++++++++++++++ lib/spack/spack/test/__init__.py | 3 +- lib/spack/spack/test/config.py | 69 ++++++ 5 files changed, 598 insertions(+), 1 deletion(-) create mode 100644 lib/spack/spack/cmd/config.py create mode 100644 lib/spack/spack/config.py create mode 100644 lib/spack/spack/test/config.py diff --git a/.gitignore b/.gitignore index 7010bf7ede..ed2012d208 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ *~ .DS_Store .idea +/etc/spackconfig diff --git a/lib/spack/spack/cmd/config.py b/lib/spack/spack/cmd/config.py new file mode 100644 index 0000000000..25d302f94b --- /dev/null +++ b/lib/spack/spack/cmd/config.py @@ -0,0 +1,77 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +import sys +import argparse + +import llnl.util.tty as tty + +import spack.config + +description = "Get and set configuration options." + +def setup_parser(subparser): + scope_group = subparser.add_mutually_exclusive_group() + + # File scope + scope_group.add_argument( + '--user', action='store_const', const='user', dest='scope', + help="Use config file in user home directory (default).") + scope_group.add_argument( + '--site', action='store_const', const='site', dest='scope', + help="Use config file in spack prefix.") + + # Get (vs. default set) + subparser.add_argument( + '--get', action='store_true', dest='get', + help="Get the value associated with a key.") + + # positional arguments (value is only used on set) + subparser.add_argument( + 'key', help="Get the value associated with KEY") + subparser.add_argument( + 'value', nargs='?', default=None, + help="Value to associate with key") + + +def config(parser, args): + key, value = args.key, args.value + + # If we're writing need to do a few checks. + if not args.get: + # Default scope for writing is user scope. + if not args.scope: + args.scope = 'user' + + if args.value is None: + tty.die("No value for '%s'. " % args.key + + "Spack config requires a key and a value.") + + config = spack.config.get_config(args.scope) + + if args.get: + print config.get_value(key) + else: + config.set_value(key, value) + config.write() diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py new file mode 100644 index 0000000000..b36b83bfaa --- /dev/null +++ b/lib/spack/spack/config.py @@ -0,0 +1,449 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +"""This module implements Spack's configuration file handling. + +Configuration file scopes +=============================== + +When Spack runs, it pulls configuration data from several config +files, much like bash shells. In Spack, there are two configuration +scopes: + + 1. ``site``: Spack loads site-wide configuration options from + ``$(prefix)/etc/spackconfig``. + + 2. ``user``: Spack next loads per-user configuration options from + ~/.spackconfig. + +If user options have the same names as site options, the user options +take precedence. + + +Configuration file format +=============================== + +Configuration files are formatted using .gitconfig syntax, which is +much like Windows .INI format. This format is implemented by Python's +ConfigParser class, and it's easy to read and versatile. + +The file is divided into sections, like this ``compiler`` section:: + + [compiler] + cc = /usr/bin/gcc + +In each section there are options (cc), and each option has a value +(/usr/bin/gcc). + +Borrowing from git, we also allow named sections, e.g.: + + [compiler "gcc@4.7.3"] + cc = /usr/bin/gcc + +This is a compiler section, but it's for the specific compiler, +``gcc@4.7.3``. ``gcc@4.7.3`` is the name. + + +Keys +=============================== + +Together, the section, name, and option, separated by periods, are +called a ``key``. Keys can be used on the command line to set +configuration options explicitly (this is also borrowed from git). + +For example, to change the C compiler used by gcc@4.7.3, you could do +this: + + spack config compiler.gcc@4.7.3.cc /usr/local/bin/gcc + +That will create a named compiler section in the user's .spackconfig +like the one shown above. +""" +import os +import re +import inspect +from collections import OrderedDict +import ConfigParser as cp + +from llnl.util.lang import memoized + +import spack +import spack.error + +__all__ = [ + 'SpackConfigParser', 'get_config', 'SpackConfigurationError', + 'InvalidConfigurationScopeError', 'InvalidSectionNameError', + 'ReadOnlySpackConfigError', 'ConfigParserError', 'NoOptionError', + 'NoSectionError'] + +_named_section_re = r'([^ ]+) "([^"]+)"' + +"""Names of scopes and their corresponding configuration files.""" +_scopes = OrderedDict({ + 'site' : os.path.join(spack.etc_path, 'spackconfig'), + 'user' : os.path.expanduser('~/.spackconfig') +}) + +_field_regex = r'^([\w-]*)' \ + r'(?:\.(.*(?=.)))?' \ + r'(?:\.([\w-]+))?$' + +_section_regex = r'^([\w-]*)\s*' \ + r'\"([^"]*\)\"$' + + +def get_config(scope=None): + """Get a Spack configuration object, which can be used to set options. + + With no arguments, this returns a SpackConfigParser with config + options loaded from all config files. This is how client code + should read Spack configuration options. + + Optionally, a scope parameter can be provided. Valid scopes + are ``site`` and ``user``. If a scope is provided, only the + options from that scope's configuration file are loaded. The + caller can set or unset options, then call ``write()`` on the + config object to write it back out to the original config file. + """ + if scope is None: + return SpackConfigParser() + elif scope not in _scopes: + raise UnknownConfigurationScopeError(scope) + else: + return SpackConfigParser(_scopes[scope]) + + +def _parse_key(key): + """Return the section, name, and option the field describes. + Values are returned in a 3-tuple. + + e.g.: + The field name ``compiler.gcc@4.7.3.cc`` refers to the 'cc' key + in a section that looks like this: + + [compiler "gcc@4.7.3"] + cc = /usr/local/bin/gcc + + * The section is ``compiler`` + * The name is ``gcc@4.7.3`` + * The key is ``cc`` + """ + match = re.search(_field_regex, key) + if match: + return match.groups() + else: + raise InvalidSectionNameError(key) + + +def _make_section_name(section, name): + if not name: + return section + return '%s "%s"' % (section, name) + + +def _autokey(fun): + """Allow a function to be called with a string key like + 'compiler.gcc.cc', or with the section, name, and option + separated. Function should take at least three args, e.g.: + + fun(self, section, name, option, [...]) + + This will allow the function above to be called normally or + with a string key, e.g.: + + fun(self, key, [...]) + """ + argspec = inspect.getargspec(fun) + fun_nargs = len(argspec[0]) + + def string_key_func(*args): + nargs = len(args) + if nargs == fun_nargs - 2: + section, name, option = _parse_key(args[1]) + return fun(args[0], section, name, option, *args[2:]) + + elif nargs == fun_nargs: + return fun(*args) + + else: + raise TypeError( + "%s takes %d or %d args (found %d)." + % (fun.__name__, fun_nargs - 2, fun_nargs, len(args))) + return string_key_func + + + +class SpackConfigParser(cp.RawConfigParser): + """Slightly modified from Python's raw config file parser to accept + leading whitespace. + """ + # Slightly modified Python option expression. This one allows + # leading whitespace. + OPTCRE = re.compile( + r'\s*(?P