From 422d291b111464599618a538ee1ca3334c698ab8 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Thu, 4 Apr 2013 09:52:15 -0700 Subject: This adds support for multi-platform methods. You can now do this: class MyPackage(Package): def install(self): ...default install... @platform('bgqos_0') def install(self): ...specialized install for bgq... This works on functions other than install, as well (as long as they're in a Package) --- bin/spack | 6 +- lib/spack/spack/__init__.py | 3 +- lib/spack/spack/arch.py | 77 ++++++++++++------ lib/spack/spack/cmd/sys-type.py | 11 +++ lib/spack/spack/error.py | 13 ++++ lib/spack/spack/exception.py | 39 ---------- lib/spack/spack/globals.py | 18 ++++- lib/spack/spack/multi_function.py | 146 +++++++++++++++++++++++++++++++++++ lib/spack/spack/package.py | 32 +++++--- lib/spack/spack/packages/__init__.py | 14 +++- lib/spack/spack/packages/libdwarf.py | 8 ++ lib/spack/spack/stage.py | 10 ++- lib/spack/spack/utils.py | 2 +- lib/spack/spack/version.py | 37 ++++++++- 14 files changed, 334 insertions(+), 82 deletions(-) create mode 100644 lib/spack/spack/cmd/sys-type.py create mode 100644 lib/spack/spack/error.py delete mode 100644 lib/spack/spack/exception.py create mode 100644 lib/spack/spack/multi_function.py diff --git a/bin/spack b/bin/spack index 13312a69d6..294334ac43 100755 --- a/bin/spack +++ b/bin/spack @@ -15,6 +15,7 @@ sys.path.insert(0, SPACK_LIB_PATH) # clean up the scope and start using spack package instead. del SPACK_FILE, SPACK_PREFIX, SPACK_LIB_PATH import spack +import spack.tty as tty # Command parsing parser = argparse.ArgumentParser( @@ -43,4 +44,7 @@ spack.debug = args.debug # Try to load the particular command asked for and run it command = spack.cmd.get_command(args.command) -command(parser, args) +try: + command(parser, args) +except KeyboardInterrupt: + tty.die("Got a keyboard interrupt from the user.") diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index ef9e448413..02af7db4f3 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -1,5 +1,6 @@ from globals import * from utils import * -from exception import * +from error import * from package import Package, depends_on +from multi_function import platform diff --git a/lib/spack/spack/arch.py b/lib/spack/spack/arch.py index 88f04cb478..6610006804 100644 --- a/lib/spack/spack/arch.py +++ b/lib/spack/spack/arch.py @@ -1,34 +1,67 @@ import os -import platform +import platform as py_platform +import spack +import error as serr from version import Version from utils import memoized -instances = {} -macos_versions = [ - ('10.8', 'mountain_lion'), - ('10.7', 'lion'), - ('10.6', 'snow_leopard'), - ('10.5', 'leopard')] +class InvalidSysTypeError(serr.SpackError): + def __init__(self, sys_type): + super(InvalidSysTypeError, self).__init__( + "Invalid sys_type value for Spack: " + sys_type) -class SysType(object): - def __init__(self, arch_string): - self.arch_string = arch_string - def __repr__(self): - return self.arch_string +class NoSysTypeError(serr.SpackError): + def __init__(self): + super(NoSysTypeError, self).__init__( + "Could not determine sys_type for this machine.") + + +def get_sys_type_from_spack_globals(): + """Return the SYS_TYPE from spack globals, or None if it isn't set.""" + if not hasattr(spack, "sys_type"): + return None + elif hasattr(spack.sys_type, "__call__"): + return spack.sys_type() + else: + return spack.sys_type + + +def get_sys_type_from_environment(): + """Return $SYS_TYPE or None if it's not defined.""" + return os.environ.get('SYS_TYPE') + + +def get_mac_sys_type(): + """Return a Mac OS SYS_TYPE or None if this isn't a mac.""" + mac_ver = py_platform.mac_ver()[0] + if not mac_ver: + return None + + return "macosx_{}_{}".format( + Version(mac_ver).up_to(2), py_platform.machine()) - def __str__(self): - return self.__repr__() @memoized def sys_type(): - stype = os.environ.get('SYS_TYPE') - if stype: - return SysType(stype) - elif platform.mac_ver()[0]: - version = Version(platform.mac_ver()[0]) - for mac_ver, name in macos_versions: - if version >= Version(mac_ver): - return SysType(name) + """Returns a SysType for the current machine.""" + methods = [get_sys_type_from_spack_globals, + get_sys_type_from_environment, + get_mac_sys_type] + + # search for a method that doesn't return None + sys_type = None + for method in methods: + sys_type = method() + if sys_type: break + + # Couldn't determine the sys_type for this machine. + if sys_type == None: + raise NoSysTypeError() + + if not type(sys_type) == str: + raise InvalidSysTypeError(sys_type) + + return sys_type diff --git a/lib/spack/spack/cmd/sys-type.py b/lib/spack/spack/cmd/sys-type.py new file mode 100644 index 0000000000..7f57fb16ff --- /dev/null +++ b/lib/spack/spack/cmd/sys-type.py @@ -0,0 +1,11 @@ +import spack +import spack.arch as arch + +description = "Print the spack sys_type for this machine" + +def sys_type(parser, args): + configured_sys_type = arch.get_sys_type_from_spack_globals() + if not configured_sys_type: + configured_sys_type = "autodetect" + print "Configured sys_type: %s" % configured_sys_type + print "Autodetected default sys_type: %s" % arch.sys_type() diff --git a/lib/spack/spack/error.py b/lib/spack/spack/error.py new file mode 100644 index 0000000000..3c3c3dafc9 --- /dev/null +++ b/lib/spack/spack/error.py @@ -0,0 +1,13 @@ + +class SpackError(Exception): + """This is the superclass for all Spack errors. + Subclasses can be found in the modules they have to do with. + """ + def __init__(self, message): + super(SpackError, self).__init__(message) + + +class UnsupportedPlatformError(SpackError): + """Raised by packages when a platform is not supported""" + def __init__(self, message): + super(UnsupportedPlatformError, self).__init__(message) diff --git a/lib/spack/spack/exception.py b/lib/spack/spack/exception.py deleted file mode 100644 index 32167cf36a..0000000000 --- a/lib/spack/spack/exception.py +++ /dev/null @@ -1,39 +0,0 @@ - - -class SpackException(Exception): - def __init__(self, message): - self.message = message - - -class FailedDownloadException(SpackException): - def __init__(self, url): - super(FailedDownloadException, self).__init__("Failed to fetch file from URL: " + url) - self.url = url - - -class InvalidPackageNameException(SpackException): - def __init__(self, name): - super(InvalidPackageNameException, self).__init__("Invalid package name: " + name) - self.name = name - - -class CommandFailedException(SpackException): - def __init__(self, command): - super(CommandFailedException, self).__init__("Failed to execute command: " + command) - self.command = command - - -class VersionParseException(SpackException): - def __init__(self, msg, spec): - super(VersionParseException, self).__init__(msg) - self.spec = spec - - -class UndetectableVersionException(VersionParseException): - def __init__(self, spec): - super(UndetectableVersionException, self).__init__("Couldn't detect version in: " + spec, spec) - - -class UndetectableNameException(VersionParseException): - def __init__(self, spec): - super(UndetectableNameException, self).__init__("Couldn't parse package name in: " + spec) diff --git a/lib/spack/spack/globals.py b/lib/spack/spack/globals.py index ee67a461c3..b864c08533 100644 --- a/lib/spack/spack/globals.py +++ b/lib/spack/spack/globals.py @@ -1,6 +1,7 @@ import os from version import Version from utils import * +import arch # This lives in $prefix/lib/spac/spack/__file__ prefix = ancestor(__file__, 4) @@ -20,7 +21,7 @@ stage_path = new_path(var_path, "stage") install_path = new_path(prefix, "opt") # Version information -spack_version = Version("0.1") +spack_version = Version("0.2") # User's editor from the environment editor = Executable(os.environ.get("EDITOR", "")) @@ -39,6 +40,21 @@ use_tmp_stage = True # location per the python implementation of tempfile.mkdtemp(). tmp_dirs = ['/nfs/tmp2', '/var/tmp', '/tmp'] +# +# SYS_TYPE to use for the spack installation. +# Value of this determines what platform spack thinks it is by +# default. You can assign three types of values: +# 1. None +# Spack will try to determine the sys_type automatically. +# +# 2. A string +# Spack will assume that the sys_type is hardcoded to the value. +# +# 3. A function that returns a string: +# Spack will use this function to determine the sys_type. +# +sys_type = None + # Important environment variables SPACK_NO_PARALLEL_MAKE = 'SPACK_NO_PARALLEL_MAKE' SPACK_LIB = 'SPACK_LIB' diff --git a/lib/spack/spack/multi_function.py b/lib/spack/spack/multi_function.py new file mode 100644 index 0000000000..7d21b02f80 --- /dev/null +++ b/lib/spack/spack/multi_function.py @@ -0,0 +1,146 @@ +"""This module contains utilities for using multi-functions in spack. +You can think of multi-functions like overloaded functions -- they're +functions with the same name, and we need to select a version of the +function based on some criteria. e.g., for overloaded functions, you +would select a version of the function to call based on the types of +its arguments. + +For spack, we might want to select a version of the function based on +the platform we want to build a package for, or based on the versions +of the dependencies of the package. +""" +import sys +import functools + +import arch +import spack.error as serr + +class NoSuchVersionError(serr.SpackError): + """Raised when we can't find a version of a function for a platform.""" + def __init__(self, fun_name, sys_type): + super(NoSuchVersionError, self).__init__( + "No version of %s found for %s!" % (fun_name, sys_type)) + + +class PlatformMultiFunction(object): + """This is a callable type for storing a collection of versions + of an instance method. The platform decorator (see docs below) + creates PlatformMultiFunctions and registers function versions + with them. + + To register a function, you can do something like this: + pmf = PlatformMultiFunction() + pmf.regsiter("chaos_5_x86_64_ib", some_function) + + When the pmf is actually called, it selects a version of + the function to call based on the sys_type of the object + it is called on. + + See the docs for the platform decorator for more details. + """ + def __init__(self, default=None): + self.function_map = {} + self.default = default + if default: + self.__name__ = default.__name__ + + def register(self, platform, function): + """Register a version of a function for a particular sys_type.""" + self.function_map[platform] = function + if not hasattr(self, '__name__'): + self.__name__ = function.__name__ + else: + assert(self.__name__ == function.__name__) + + def __get__(self, obj, objtype): + """This makes __call__ support instance methods.""" + return functools.partial(self.__call__, obj) + + def __call__(self, package_self, *args, **kwargs): + """Try to find a function that matches package_self.sys_type. + If none is found, call the default function that this was + initialized with. If there is no default, raise an error. + """ + sys_type = package_self.sys_type + function = self.function_map.get(sys_type, self.default) + if function: + function(package_self, *args, **kwargs) + else: + raise NoSuchVersionError(self.__name__, sys_type) + + def __str__(self): + return "<%s, %s>" % (self.default, self.function_map) + + +class platform(object): + """This annotation lets packages declare platform-specific versions + of functions like install(). For example: + + class SomePackage(Package): + ... + + def install(self, prefix): + # Do default install + + @platform('chaos_5_x86_64_ib') + def install(self, prefix): + # This will be executed instead of the default install if + # the package's sys_type() is chaos_5_x86_64_ib. + + @platform('bgqos_0") + def install(self, prefix): + # This will be executed if the package's sys_type is bgqos_0 + + This allows each package to have a default version of install() AND + specialized versions for particular platforms. The version that is + called depends on the sys_type of SomePackage. + + Note that this works for functions other than install, as well. So, + if you only have part of the install that is platform specific, you + could do this: + + class SomePackage(Package): + ... + + def setup(self): + # do nothing in the default case + pass + + @platform('chaos_5_x86_64_ib') + def setup(self): + # do something for x86_64 + + def install(self, prefix): + # Do common install stuff + self.setup() + # Do more common install stuff + + If there is no specialized version for the package's sys_type, the + default (un-decorated) version will be called. If there is no default + version and no specialized version, the call raises a + NoSuchVersionError. + + Note that the default version of install() must *always* come first. + Otherwise it will override all of the platform-specific versions. + There's not much we can do to get around this because of the way + decorators work. + """ +class platform(object): + def __init__(self, sys_type): + self.sys_type = sys_type + + def __call__(self, fun): + # Record the sys_type as an attribute on this function + fun.sys_type = self.sys_type + + # Get the first definition of the function in the calling scope + calling_frame = sys._getframe(1).f_locals + original_fun = calling_frame.get(fun.__name__) + + # Create a multifunction out of the original function if it + # isn't one already. + if not type(original_fun) == PlatformMultiFunction: + original_fun = PlatformMultiFunction(original_fun) + + original_fun.register(self.sys_type, fun) + return original_fun diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index ae94690971..63bcf7a630 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -14,7 +14,7 @@ import inspect import os import re import subprocess -import platform +import platform as py_platform import shutil from spack import * @@ -24,6 +24,8 @@ import attr import validate import version import arch + +from multi_function import platform from stage import Stage @@ -226,7 +228,7 @@ class Package(object): clean() (some of them do this), and others to provide custom behavior. """ - def __init__(self, arch=arch.sys_type()): + def __init__(self, sys_type=arch.sys_type()): attr.required(self, 'homepage') attr.required(self, 'url') attr.required(self, 'md5') @@ -235,7 +237,7 @@ class Package(object): attr.setdefault(self, 'parallel', True) # Architecture for this package. - self.arch = arch + self.sys_type = sys_type # Name of package is the name of its module (the file that contains it) self.name = inspect.getmodulename(self.module.__file__) @@ -266,7 +268,7 @@ class Package(object): self.dirty = False # stage used to build this package. - Self.stage = Stage(self.stage_name, self.url) + self.stage = Stage(self.stage_name, self.url) def add_commands_to_module(self): @@ -289,7 +291,7 @@ class Package(object): # standard CMake arguments m.std_cmake_args = ['-DCMAKE_INSTALL_PREFIX=%s' % self.prefix, '-DCMAKE_BUILD_TYPE=None'] - if platform.mac_ver()[0]: + if py_platform.mac_ver()[0]: m.std_cmake_args.append('-DCMAKE_FIND_FRAMEWORK=LAST') # Emulate some shell commands for convenience @@ -361,11 +363,13 @@ class Package(object): def stage_name(self): return "%s-%s" % (self.name, self.version) - + # + # Below properties determine the path where this package is installed. + # @property def platform_path(self): """Directory for binaries for the current platform.""" - return new_path(install_path, self.arch) + return new_path(install_path, self.sys_type) @property @@ -388,6 +392,9 @@ class Package(object): def remove_prefix(self): """Removes the prefix for a package along with any empty parent directories.""" + if self.dirty: + return + if os.path.exists(self.prefix): shutil.rmtree(self.prefix, True) @@ -448,10 +455,15 @@ class Package(object): self.install(self.prefix) if not os.path.isdir(self.prefix): tty.die("Install failed for %s. No install dir created." % self.name) + except subprocess.CalledProcessError, e: - if not self.dirty: - self.remove_prefix() + self.remove_prefix() tty.die("Install failed for %s" % self.name, e.message) + + except KeyboardInterrupt, e: + self.remove_prefix() + raise + except Exception, e: if not self.dirty: self.remove_prefix() @@ -576,7 +588,7 @@ class Dependency(object): def depends_on(*args, **kwargs): - """Adds a depends_on local variable in the locals of + """Adds a dependencies local variable in the locals of the calling class, based on args. """ # This gets the calling frame so we can pop variables into it diff --git a/lib/spack/spack/packages/__init__.py b/lib/spack/spack/packages/__init__.py index e571134538..c32738bd6a 100644 --- a/lib/spack/spack/packages/__init__.py +++ b/lib/spack/spack/packages/__init__.py @@ -10,6 +10,7 @@ from spack.utils import * import spack.arch as arch import spack.version as version import spack.attr as attr +import spack.error as serr # Valid package names valid_package = r'^[a-zA-Z0-9_-]*$' @@ -19,6 +20,13 @@ invalid_package = r'[_-][_-]+' instances = {} +class InvalidPackageNameError(serr.SpackError): + """Raised when we encounter a bad package name.""" + def __init__(self, name): + super(InvalidPackageNameError, self).__init__( + "Invalid package name: " + name) + self.name = name + def valid_name(pkg): return re.match(valid_package, pkg) and not re.search(invalid_package, pkg) @@ -26,7 +34,7 @@ def valid_name(pkg): def validate_name(pkg): if not valid_name(pkg): - raise spack.InvalidPackageNameException(pkg) + raise spack.InvalidPackageNameError(pkg) def filename_for(pkg): @@ -36,7 +44,7 @@ def filename_for(pkg): def installed_packages(**kwargs): - """Returns a dict from SysType to lists of Package objects.""" + """Returns a dict from systype strings to lists of Package objects.""" list_installed = kwargs.get('installed', False) pkgs = {} @@ -44,7 +52,7 @@ def installed_packages(**kwargs): return pkgs for sys_type in os.listdir(spack.install_path): - sys_type = arch.SysType(sys_type) + sys_type = sys_type sys_path = new_path(spack.install_path, sys_type) pkgs[sys_type] = [get(pkg) for pkg in os.listdir(sys_path) if os.path.isdir(new_path(sys_path, pkg))] diff --git a/lib/spack/spack/packages/libdwarf.py b/lib/spack/spack/packages/libdwarf.py index 070fc8360f..b8a2f3e052 100644 --- a/lib/spack/spack/packages/libdwarf.py +++ b/lib/spack/spack/packages/libdwarf.py @@ -11,6 +11,7 @@ class Libdwarf(Package): depends_on("libelf") + def clean(self): for dir in dwarf_dirs: with working_dir(dir): @@ -19,6 +20,7 @@ class Libdwarf(Package): def install(self, prefix): + # dwarf build does not set arguments for ar properly make.add_default_arg('ARFLAGS=rcs') # Dwarf doesn't provide an install, so we have to do it. @@ -43,3 +45,9 @@ class Libdwarf(Package): install('dwarfdump', bin) install('dwarfdump.conf', lib) install('dwarfdump.1', man1) + + + @platform('macosx_10.8_x86_64') + def install(self, prefix): + raise UnsupportedPlatformError( + "libdwarf doesn't currently build on Mac OS X.") diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index ed48a48758..c8ffa915bc 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -5,8 +5,16 @@ import tempfile import getpass import spack +import spack.error as serr import tty +class FailedDownloadError(serr.SpackError): + """Raised wen a download fails.""" + def __init__(self, url): + super(FailedDownloadError, self).__init__( + "Failed to fetch file from URL: " + url) + self.url = url + class Stage(object): """A Stage object manaages a directory where an archive is downloaded, @@ -161,7 +169,7 @@ class Stage(object): "your internet gateway issue and install again.") if not self.archive_file: - raise FailedDownloadException(url) + raise FailedDownloadError(url) return self.archive_file diff --git a/lib/spack/spack/utils.py b/lib/spack/spack/utils.py index 0128d283cf..09e4985275 100644 --- a/lib/spack/spack/utils.py +++ b/lib/spack/spack/utils.py @@ -26,7 +26,7 @@ def memoized(obj): def memoizer(*args, **kwargs): if args not in cache: cache[args] = obj(*args, **kwargs) - return cache[args] + return cache[args] return memoizer diff --git a/lib/spack/spack/version.py b/lib/spack/spack/version.py index 75b09c2f5a..52b89883f7 100644 --- a/lib/spack/spack/version.py +++ b/lib/spack/spack/version.py @@ -2,7 +2,29 @@ import os import re import utils -from exception import * +import spack.error as serr + + +class VersionParseError(serr.SpackError): + """Raised when the version module can't parse something.""" + def __init__(self, msg, spec): + super(VersionParseError, self).__init__(msg) + self.spec = spec + + +class UndetectableVersionError(VersionParseError): + """Raised when we can't parse a version from a string.""" + def __init__(self, spec): + super(UndetectableVersionError, self).__init__( + "Couldn't detect version in: " + spec, spec) + + +class UndetectableNameError(VersionParseError): + """Raised when we can't parse a package name from a string.""" + def __init__(self, spec): + super(UndetectableNameError, self).__init__( + "Couldn't parse package name in: " + spec) + class Version(object): """Class to represent versions""" @@ -32,6 +54,15 @@ class Version(object): else: return None + def up_to(self, index): + """Return a version string up to the specified component, exclusive. + e.g., if this is 10.8.2, self.up_to(2) will return '10.8'. + """ + return '.'.join(str(x) for x in self[:index]) + + def __getitem__(self, idx): + return tuple(self.version[idx]) + def __repr__(self): return self.version_string @@ -123,7 +154,7 @@ def parse_version_string_with_indices(spec): if match and match.group(1) is not None: return match.group(1), match.start(1), match.end(1) - raise UndetectableVersionException(spec) + raise UndetectableVersionError(spec) def parse_version(spec): @@ -162,7 +193,7 @@ def parse_name(spec, ver=None): match = re.search(nt, spec) if match: return match.group(1) - raise UndetectableNameException(spec) + raise UndetectableNameError(spec) def parse(spec): ver = parse_version(spec) -- cgit v1.2.3-70-g09d2