diff options
Diffstat (limited to 'lib')
-rw-r--r-- | lib/spack/llnl/util/filesystem.py | 18 | ||||
-rw-r--r-- | lib/spack/spack/cmd/create.py | 2 | ||||
-rw-r--r-- | lib/spack/spack/concretize.py | 12 | ||||
-rw-r--r-- | lib/spack/spack/config.py | 13 | ||||
-rw-r--r-- | lib/spack/spack/environment.py | 10 | ||||
-rw-r--r-- | lib/spack/spack/modules.py | 25 | ||||
-rw-r--r-- | lib/spack/spack/operating_systems/cnl.py | 24 | ||||
-rw-r--r-- | lib/spack/spack/platforms/cray.py | 105 | ||||
-rw-r--r-- | lib/spack/spack/platforms/cray_xc.py | 72 | ||||
-rw-r--r-- | lib/spack/spack/preferred_packages.py | 4 | ||||
-rw-r--r-- | lib/spack/spack/stage.py | 113 | ||||
-rw-r--r-- | lib/spack/spack/test/__init__.py | 6 | ||||
-rw-r--r-- | lib/spack/spack/test/architecture.py | 4 | ||||
-rw-r--r-- | lib/spack/spack/test/concretize_preferences.py | 106 | ||||
-rw-r--r-- | lib/spack/spack/test/modules.py | 9 | ||||
-rw-r--r-- | lib/spack/spack/util/web.py | 28 |
16 files changed, 389 insertions, 162 deletions
diff --git a/lib/spack/llnl/util/filesystem.py b/lib/spack/llnl/util/filesystem.py index 6e4cd338fe..2478f5c159 100644 --- a/lib/spack/llnl/util/filesystem.py +++ b/lib/spack/llnl/util/filesystem.py @@ -29,8 +29,9 @@ import shutil import stat import errno import getpass -from contextlib import contextmanager, closing +from contextlib import contextmanager import subprocess +import fileinput import llnl.util.tty as tty @@ -85,13 +86,14 @@ def filter_file(regex, repl, *filenames, **kwargs): if ignore_absent and not os.path.exists(filename): continue - shutil.copy(filename, backup_filename) + # Create backup file. Don't overwrite an existing backup + # file in case this file is being filtered multiple times. + if not os.path.exists(backup_filename): + shutil.copy(filename, backup_filename) + try: - with closing(open(backup_filename)) as infile: - with closing(open(filename, 'w')) as outfile: - for line in infile: - foo = re.sub(regex, repl, line) - outfile.write(foo) + for line in fileinput.input(filename, inplace=True): + print(re.sub(regex, repl, line.rstrip())) except: # clean up the original file on failure. shutil.move(backup_filename, filename) @@ -189,7 +191,7 @@ def install(src, dest): def install_tree(src, dest, **kwargs): - """Manually install a file to a particular location.""" + """Manually install a directory tree to a particular location.""" tty.debug("Installing %s to %s" % (src, dest)) shutil.copytree(src, dest, **kwargs) diff --git a/lib/spack/spack/cmd/create.py b/lib/spack/spack/cmd/create.py index da74ceb2f6..51bf17a44b 100644 --- a/lib/spack/spack/cmd/create.py +++ b/lib/spack/spack/cmd/create.py @@ -165,7 +165,7 @@ install_dict = { 'python': """\ # FIXME: Add logic to build and install here. - python('setup.py', 'install', '--prefix={0}'.format(prefix))""", + setup_py('install', '--prefix={0}'.format(prefix))""", 'R': """\ # FIXME: Add logic to build and install here. diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 5180f3cf04..6f11c86ce8 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -336,14 +336,18 @@ class DefaultConcretizer(object): spack.pkgsort.compiler_compare, other_spec.name) matches = sorted(compiler_list, cmp=cmp_compilers) if not matches: - raise UnavailableCompilerVersionError(other_compiler) + arch = spec.architecture + raise UnavailableCompilerVersionError(other_compiler, + arch.platform_os) # copy concrete version into other_compiler index = 0 while not _proper_compiler_style(matches[index], spec.architecture): index += 1 if index == len(matches) - 1: - raise NoValidVersionError(spec) + arch = spec.architecture + raise UnavailableCompilerVersionError(spec.compiler, + arch.platform_os) spec.compiler = matches[index].copy() assert(spec.compiler.concrete) return True # things changed. @@ -489,9 +493,9 @@ class UnavailableCompilerVersionError(spack.error.SpackError): """Raised when there is no available compiler that satisfies a compiler spec.""" - def __init__(self, compiler_spec): + def __init__(self, compiler_spec, operating_system): super(UnavailableCompilerVersionError, self).__init__( - "No available compiler version matches '%s'" % compiler_spec, + "No available compiler version matches '%s' on operating_system %s" % (compiler_spec, operating_system), # NOQA: ignore=E501 "Run 'spack compilers' to see available compiler Options.") diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index e2e7dbc0ee..31f0eb3a56 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -266,6 +266,19 @@ section_schemas = { ], }, },},},},},}, + 'targets': { + '$schema': 'http://json-schema.org/schema#', + 'title': 'Spack target configuration file schema', + 'type': 'object', + 'additionalProperties': False, + 'patternProperties': { + r'targets:?': { + 'type': 'object', + 'default': {}, + 'additionalProperties': False, + 'patternProperties': { + r'\w[\w-]*': { # target name + 'type': 'string' ,},},},},}, 'modules': { '$schema': 'http://json-schema.org/schema#', 'title': 'Spack module file configuration file schema', diff --git a/lib/spack/spack/environment.py b/lib/spack/spack/environment.py index 30c6228ca4..41136ab2eb 100644 --- a/lib/spack/spack/environment.py +++ b/lib/spack/spack/environment.py @@ -37,6 +37,10 @@ class NameModifier(object): self.args = {'name': name} self.args.update(kwargs) + def update_args(self, **kwargs): + self.__dict__.update(kwargs) + self.args.update(kwargs) + class NameValueModifier(object): @@ -44,7 +48,11 @@ class NameValueModifier(object): self.name = name self.value = value self.separator = kwargs.get('separator', ':') - self.args = {'name': name, 'value': value, 'delim': self.separator} + self.args = {'name': name, 'value': value, 'separator': self.separator} + self.args.update(kwargs) + + def update_args(self, **kwargs): + self.__dict__.update(kwargs) self.args.update(kwargs) diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py index db8b20ae42..8701a31c49 100644 --- a/lib/spack/spack/modules.py +++ b/lib/spack/spack/modules.py @@ -272,14 +272,26 @@ class EnvModule(object): @property def tokens(self): + """Tokens that can be substituted in environment variable values + and naming schemes + """ tokens = { 'name': self.spec.name, 'version': self.spec.version, - 'compiler': self.spec.compiler + 'compiler': self.spec.compiler, + 'prefix': self.spec.package.prefix } return tokens @property + def upper_tokens(self): + """Tokens that can be substituted in environment variable names""" + upper_tokens = { + 'name': self.spec.name.replace('-', '_').upper() + } + return upper_tokens + + @property def use_name(self): """ Subclasses should implement this to return the name the module command @@ -438,6 +450,11 @@ class EnvModule(object): def process_environment_command(self, env): for command in env: + # Token expansion from configuration file + name = command.args.get('name', '').format(**self.upper_tokens) + value = str(command.args.get('value', '')).format(**self.tokens) + command.update_args(name=name, value=value) + # Format the line int the module file try: yield self.environment_modifications_formats[type( command)].format(**command.args) @@ -511,9 +528,9 @@ class TclModule(EnvModule): name = 'tcl' path = join_path(spack.share_path, "modules") environment_modifications_formats = { - PrependPath: 'prepend-path --delim "{delim}" {name} \"{value}\"\n', - AppendPath: 'append-path --delim "{delim}" {name} \"{value}\"\n', - RemovePath: 'remove-path --delim "{delim}" {name} \"{value}\"\n', + PrependPath: 'prepend-path --delim "{separator}" {name} \"{value}\"\n', + AppendPath: 'append-path --delim "{separator}" {name} \"{value}\"\n', + RemovePath: 'remove-path --delim "{separator}" {name} \"{value}\"\n', SetEnv: 'setenv {name} \"{value}\"\n', UnsetEnv: 'unsetenv {name}\n' } diff --git a/lib/spack/spack/operating_systems/cnl.py b/lib/spack/spack/operating_systems/cnl.py index c160a60be8..dbd2775861 100644 --- a/lib/spack/spack/operating_systems/cnl.py +++ b/lib/spack/spack/operating_systems/cnl.py @@ -7,11 +7,12 @@ import spack.spec from spack.util.multiproc import parmap import spack.compilers + class Cnl(OperatingSystem): """ Compute Node Linux (CNL) is the operating system used for the Cray XC series super computers. It is a very stripped down version of GNU/Linux. Any compilers found through this operating system will be used with - modules. If updated, user must make sure that version and name are + modules. If updated, user must make sure that version and name are updated to indicate that OS has been upgraded (or downgraded) """ def __init__(self): @@ -19,17 +20,19 @@ class Cnl(OperatingSystem): version = '10' super(Cnl, self).__init__(name, version) + def __str__(self): + return self.name def find_compilers(self, *paths): types = spack.compilers.all_compiler_types() - compiler_lists = parmap(lambda cmp_cls: self.find_compiler(cmp_cls, *paths), types) + compiler_lists = parmap( + lambda cmp_cls: self.find_compiler(cmp_cls, *paths), types) # ensure all the version calls we made are cached in the parent # process, as well. This speeds up Spack a lot. - clist = reduce(lambda x,y: x+y, compiler_lists) + clist = reduce(lambda x, y: x + y, compiler_lists) return clist - def find_compiler(self, cmp_cls, *paths): compilers = [] if cmp_cls.PrgEnv: @@ -45,13 +48,16 @@ class Cnl(OperatingSystem): if paths: module_paths = ':' + ':'.join(p for p in paths) os.environ['MODULEPATH'] = module_paths - - output = modulecmd('avail', cmp_cls.PrgEnv_compiler, output=str, error=str) - matches = re.findall(r'(%s)/([\d\.]+[\d])' % cmp_cls.PrgEnv_compiler, output) + + output = modulecmd( + 'avail', cmp_cls.PrgEnv_compiler, output=str, error=str) + matches = re.findall( + r'(%s)/([\d\.]+[\d])' % cmp_cls.PrgEnv_compiler, output) for name, version in matches: v = version - comp = cmp_cls(spack.spec.CompilerSpec(name + '@' + v), self, - ['cc', 'CC', 'ftn'], [cmp_cls.PrgEnv, name +'/' + v]) + comp = cmp_cls( + spack.spec.CompilerSpec(name + '@' + v), self, + ['cc', 'CC', 'ftn'], [cmp_cls.PrgEnv, name + '/' + v]) compilers.append(comp) diff --git a/lib/spack/spack/platforms/cray.py b/lib/spack/spack/platforms/cray.py new file mode 100644 index 0000000000..0059b49ff1 --- /dev/null +++ b/lib/spack/spack/platforms/cray.py @@ -0,0 +1,105 @@ +import os +import re +import spack.config +import llnl.util.tty as tty +from spack.util.executable import which +from spack.architecture import Platform, Target, NoPlatformError +from spack.operating_systems.linux_distro import LinuxDistro +from spack.operating_systems.cnl import Cnl +from llnl.util.filesystem import join_path + +# Craype- module prefixes that are not valid CPU targets. +NON_TARGETS = ('hugepages', 'network', 'target', 'accel', 'xtpe') + + +def _target_from_clean_env(name): + '''Return the default back_end target as loaded in a clean login session. + + A bash subshell is launched with a wiped environment and the list of loaded + modules is parsed for the first acceptable CrayPE target. + ''' + # Based on the incantation: + # echo "$(env - USER=$USER /bin/bash -l -c 'module list -lt')" + targets = [] + if name != 'front_end': + env = which('env') + env.add_default_arg('-') + # CAUTION - $USER is generally needed to initialize the environment. + # There may be other variables needed for general success. + output = env('USER=%s' % os.environ['USER'], + '/bin/bash', '--noprofile', '--norc', '-c', + '. /etc/profile; module list -lt', + output=str, error=str) + default_modules = [i for i in output.splitlines() + if len(i.split()) == 1] + tty.debug("Found default modules:", + *[" " + mod for mod in default_modules]) + pattern = 'craype-(?!{0})(\S*)'.format('|'.join(NON_TARGETS)) + for mod in default_modules: + if 'craype-' in mod: + targets.extend(re.findall(pattern, mod)) + return targets[0] if targets else None + + +class Cray(Platform): + priority = 10 + + def __init__(self): + ''' Create a Cray system platform. + + Target names should use craype target names but not include the + 'craype-' prefix. Uses first viable target from: + self + envars [SPACK_FRONT_END, SPACK_BACK_END] + configuration file "targets.yaml" with keys 'front_end', 'back_end' + scanning /etc/bash/bashrc.local for back_end only + ''' + super(Cray, self).__init__('cray') + + # Get targets from config or make best guess from environment: + conf = spack.config.get_config('targets') + for name in ('front_end', 'back_end'): + _target = getattr(self, name, None) + if _target is None: + _target = os.environ.get('SPACK_' + name.upper()) + if _target is None: + _target = conf.get(name) + if _target is None: + _target = _target_from_clean_env(name) + setattr(self, name, _target) + + if _target is not None: + self.add_target(name, Target(_target, 'craype-' + _target)) + self.add_target(_target, Target(_target, 'craype-' + _target)) + + if self.back_end is not None: + self.default = self.back_end + self.add_target( + 'default', Target(self.default, 'craype-' + self.default)) + else: + raise NoPlatformError() + + front_distro = LinuxDistro() + back_distro = Cnl() + + self.default_os = str(back_distro) + self.back_os = self.default_os + self.front_os = str(front_distro) + + self.add_operating_system(self.back_os, back_distro) + self.add_operating_system(self.front_os, front_distro) + + @classmethod + def setup_platform_environment(self, pkg, env): + """ Change the linker to default dynamic to be more + similar to linux/standard linker behavior + """ + env.set('CRAYPE_LINK_TYPE', 'dynamic') + cray_wrapper_names = join_path(spack.build_env_path, 'cray') + if os.path.isdir(cray_wrapper_names): + env.prepend_path('PATH', cray_wrapper_names) + env.prepend_path('SPACK_ENV_PATH', cray_wrapper_names) + + @classmethod + def detect(self): + return os.environ.get('CRAYPE_VERSION') is not None diff --git a/lib/spack/spack/platforms/cray_xc.py b/lib/spack/spack/platforms/cray_xc.py deleted file mode 100644 index 03d0383cc5..0000000000 --- a/lib/spack/spack/platforms/cray_xc.py +++ /dev/null @@ -1,72 +0,0 @@ -import os -import spack -from spack.architecture import Platform, Target -from spack.operating_systems.linux_distro import LinuxDistro -from spack.operating_systems.cnl import Cnl -from spack.util.executable import which -from llnl.util.filesystem import join_path - - -class CrayXc(Platform): - priority = 20 - front_end = 'sandybridge' - back_end = 'ivybridge' - default = 'ivybridge' - - back_os = "CNL10" - default_os = "CNL10" - - def __init__(self): - ''' Since cori doesn't have ivybridge as a front end it's better - if we use CRAY_CPU_TARGET as the default. This will ensure - that if we're on a XC-40 or XC-30 then we can detect the target - ''' - super(CrayXc, self).__init__('cray_xc') - - # Handle the default here so we can check for a key error - if 'CRAY_CPU_TARGET' in os.environ: - self.default = os.environ['CRAY_CPU_TARGET'] - - # Change the defaults to haswell if we're on an XC40 - if self.default == 'haswell': - self.front_end = self.default - self.back_end = self.default - - # Could switch to use modules and fe targets for front end - # Currently using compilers by path for front end. - self.add_target('sandybridge', Target('sandybridge')) - self.add_target('ivybridge', - Target('ivybridge', 'craype-ivybridge')) - self.add_target('haswell', - Target('haswell', 'craype-haswell')) - - # Front end of the cray platform is a linux distro. - linux_dist = LinuxDistro() - self.front_os = str(linux_dist) - self.add_operating_system(str(linux_dist), linux_dist) - self.add_operating_system('CNL10', Cnl()) - - @classmethod - def setup_platform_environment(self, pkg, env): - """ Change the linker to default dynamic to be more - similar to linux/standard linker behavior - """ - env.set('CRAYPE_LINK_TYPE', 'dynamic') - cray_wrapper_names = join_path(spack.build_env_path, 'cray') - if os.path.isdir(cray_wrapper_names): - env.prepend_path('PATH', cray_wrapper_names) - env.prepend_path('SPACK_ENV_PATHS', cray_wrapper_names) - - @classmethod - def detect(self): - try: - cc_verbose = which('ftn') - text = cc_verbose('-craype-verbose', - output=str, error=str, - ignore_errors=True).split() - if '-D__CRAYXC' in text: - return True - else: - return False - except: - return False diff --git a/lib/spack/spack/preferred_packages.py b/lib/spack/spack/preferred_packages.py index 5f18e212b6..f079c1ef8b 100644 --- a/lib/spack/spack/preferred_packages.py +++ b/lib/spack/spack/preferred_packages.py @@ -162,8 +162,8 @@ class PreferredPackages(object): """Return a VariantMap of preferred variants and their values""" variants = self.preferred.get(pkgname, {}).get('variants', '') if not isinstance(variants, basestring): - variants = "".join(variants) - return spack.spec.Spec(pkgname + variants).variants + variants = " ".join(variants) + return spack.spec.Spec("%s %s" % (pkgname, variants)).variants def version_compare(self, pkgname, a, b): """Return less-than-0, 0, or greater than 0 if version a of pkgname is diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index b08cce43b8..553c4ad05f 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -37,6 +37,7 @@ import spack import spack.config import spack.fetch_strategy as fs import spack.error +from spack.version import * STAGE_PREFIX = 'spack-stage-' @@ -51,10 +52,12 @@ class Stage(object): lifecycle looks like this: ``` - with Stage() as stage: # Context manager creates and destroys the stage directory + with Stage() as stage: # Context manager creates and destroys the + # stage directory stage.fetch() # Fetch a source archive into the stage. stage.expand_archive() # Expand the source archive. - <install> # Build and install the archive. (handled by user of Stage) + <install> # Build and install the archive. (handled by + # user of Stage) ``` When used as a context manager, the stage is automatically @@ -71,7 +74,8 @@ class Stage(object): stage.create() # Explicitly create the stage directory. stage.fetch() # Fetch a source archive into the stage. stage.expand_archive() # Expand the source archive. - <install> # Build and install the archive. (handled by user of Stage) + <install> # Build and install the archive. (handled by + # user of Stage) finally: stage.destroy() # Explicitly destroy the stage directory. ``` @@ -120,13 +124,17 @@ class Stage(object): elif isinstance(url_or_fetch_strategy, fs.FetchStrategy): self.fetcher = url_or_fetch_strategy else: - raise ValueError("Can't construct Stage without url or fetch strategy") + raise ValueError( + "Can't construct Stage without url or fetch strategy") self.fetcher.set_stage(self) - self.default_fetcher = self.fetcher # self.fetcher can change with mirrors. - self.skip_checksum_for_mirror = True # used for mirrored archives of repositories. + # self.fetcher can change with mirrors. + self.default_fetcher = self.fetcher + # used for mirrored archives of repositories. + self.skip_checksum_for_mirror = True - # TODO : this uses a protected member of tempfile, but seemed the only way to get a temporary name - # TODO : besides, the temporary link name won't be the same as the temporary stage area in tmp_root + # TODO : this uses a protected member of tempfile, but seemed the only + # TODO : way to get a temporary name besides, the temporary link name + # TODO : won't be the same as the temporary stage area in tmp_root self.name = name if name is None: self.name = STAGE_PREFIX + next(tempfile._get_candidate_names()) @@ -143,7 +151,6 @@ class Stage(object): # Flag to decide whether to delete the stage folder on exit or not self.keep = keep - def __enter__(self): """ Entering a stage context will create the stage directory @@ -154,7 +161,6 @@ class Stage(object): self.create() return self - def __exit__(self, exc_type, exc_val, exc_tb): """ Exiting from a stage context will delete the stage directory unless: @@ -173,12 +179,10 @@ class Stage(object): if exc_type is None and not self.keep: self.destroy() - def _need_to_create_path(self): """Makes sure nothing weird has happened since the last time we looked at path. Returns True if path already exists and is ok. - Returns False if path needs to be created. - """ + Returns False if path needs to be created.""" # Path doesn't exist yet. Will need to create it. if not os.path.exists(self.path): return True @@ -196,7 +200,8 @@ class Stage(object): if spack.use_tmp_stage: # If we're using a tmp dir, it's a link, and it points at the # right spot, then keep it. - if (real_path.startswith(real_tmp) and os.path.exists(real_path)): + if (real_path.startswith(real_tmp) and + os.path.exists(real_path)): return False else: # otherwise, just unlink it and start over. @@ -204,7 +209,8 @@ class Stage(object): return True else: - # If we're not tmp mode, then it's a link and we want a directory. + # If we're not tmp mode, then it's a link and we want a + # directory. os.unlink(self.path) return True @@ -215,10 +221,12 @@ class Stage(object): """Possible archive file paths.""" paths = [] if isinstance(self.fetcher, fs.URLFetchStrategy): - paths.append(os.path.join(self.path, os.path.basename(self.fetcher.url))) + paths.append(os.path.join( + self.path, os.path.basename(self.fetcher.url))) if self.mirror_path: - paths.append(os.path.join(self.path, os.path.basename(self.mirror_path))) + paths.append(os.path.join( + self.path, os.path.basename(self.mirror_path))) return paths @@ -227,10 +235,12 @@ class Stage(object): """Path to the source archive within this stage directory.""" paths = [] if isinstance(self.fetcher, fs.URLFetchStrategy): - paths.append(os.path.join(self.path, os.path.basename(self.fetcher.url))) + paths.append(os.path.join( + self.path, os.path.basename(self.fetcher.url))) if self.mirror_path: - paths.append(os.path.join(self.path, os.path.basename(self.mirror_path))) + paths.append(os.path.join( + self.path, os.path.basename(self.mirror_path))) for path in paths: if os.path.exists(path): @@ -262,7 +272,8 @@ class Stage(object): return None def chdir(self): - """Changes directory to the stage path. Or dies if it is not set up.""" + """Changes directory to the stage path. Or dies if it is not set + up.""" if os.path.isdir(self.path): os.chdir(self.path) else: @@ -306,6 +317,20 @@ class Stage(object): fetchers.insert(0, fs.URLFetchStrategy(url, digest)) fetchers.insert(0, spack.cache.fetcher(self.mirror_path, digest)) + # Look for the archive in list_url + package_name = os.path.dirname(self.mirror_path) + pkg = spack.repo.get(package_name) + if pkg.list_url is not None and pkg.url is not None: + archive_version = spack.url.parse_version( + self.default_fetcher.url) + versions = pkg.fetch_remote_versions() + try: + url_from_list = versions[Version(archive_version)] + fetchers.append(fs.URLFetchStrategy(url_from_list, digest)) + except KeyError: + tty.msg("Can not find version %s in url_list" % + archive_version) + for fetcher in fetchers: try: fetcher.set_stage(self) @@ -321,11 +346,11 @@ class Stage(object): self.fetcher = self.default_fetcher raise fs.FetchError(errMessage, None) - def check(self): """Check the downloaded archive against a checksum digest. No-op if this stage checks code out of a repository.""" - if self.fetcher is not self.default_fetcher and self.skip_checksum_for_mirror: + if self.fetcher is not self.default_fetcher and \ + self.skip_checksum_for_mirror: tty.warn("Fetching from mirror without a checksum!", "This package is normally checked out from a version " "control system, but it has been archived on a spack " @@ -335,16 +360,13 @@ class Stage(object): else: self.fetcher.check() - def cache_local(self): spack.cache.store(self.fetcher, self.mirror_path) - def expand_archive(self): """Changes to the stage directory and attempt to expand the downloaded - archive. Fail if the stage is not set up or if the archive is not yet - downloaded. - """ + archive. Fail if the stage is not set up or if the archive is not yet + downloaded.""" archive_dir = self.source_path if not archive_dir: self.fetcher.expand() @@ -386,8 +408,8 @@ class Stage(object): # Create the top-level stage directory mkdirp(spack.stage_path) remove_dead_links(spack.stage_path) - # If a tmp_root exists then create a directory there and then link it in the stage area, - # otherwise create the stage directory in self.path + # If a tmp_root exists then create a directory there and then link it + # in the stage area, otherwise create the stage directory in self.path if self._need_to_create_path(): if self.tmp_root: tmp_dir = tempfile.mkdtemp('', STAGE_PREFIX, self.tmp_root) @@ -409,6 +431,7 @@ class Stage(object): class ResourceStage(Stage): + def __init__(self, url_or_fetch_strategy, root, resource, **kwargs): super(ResourceStage, self).__init__(url_or_fetch_strategy, **kwargs) self.root_stage = root @@ -418,12 +441,15 @@ class ResourceStage(Stage): super(ResourceStage, self).expand_archive() root_stage = self.root_stage resource = self.resource - placement = os.path.basename(self.source_path) if resource.placement is None else resource.placement + placement = os.path.basename(self.source_path) \ + if resource.placement is None \ + else resource.placement if not isinstance(placement, dict): placement = {'': placement} # Make the paths in the dictionary absolute and link for key, value in placement.iteritems(): - target_path = join_path(root_stage.source_path, resource.destination) + target_path = join_path( + root_stage.source_path, resource.destination) destination_path = join_path(target_path, value) source_path = join_path(self.source_path, key) @@ -437,21 +463,23 @@ class ResourceStage(Stage): if not os.path.exists(destination_path): # Create a symlink - tty.info('Moving resource stage\n\tsource : {stage}\n\tdestination : {destination}'.format( - stage=source_path, destination=destination_path - )) + tty.info('Moving resource stage\n\tsource : ' + '{stage}\n\tdestination : {destination}'.format( + stage=source_path, destination=destination_path + )) shutil.move(source_path, destination_path) -@pattern.composite(method_list=['fetch', 'create', 'check', 'expand_archive', 'restage', 'destroy', 'cache_local']) +@pattern.composite(method_list=['fetch', 'create', 'check', 'expand_archive', + 'restage', 'destroy', 'cache_local']) class StageComposite: - """ - Composite for Stage type objects. The first item in this composite is considered to be the root package, and - operations that return a value are forwarded to it. - """ + """Composite for Stage type objects. The first item in this composite is + considered to be the root package, and operations that return a value are + forwarded to it.""" # # __enter__ and __exit__ delegate to all stages in the composite. # + def __enter__(self): for item in self: item.__enter__() @@ -496,8 +524,11 @@ class DIYStage(object): raise ChdirError("Setup failed: no such directory: " + self.path) # DIY stages do nothing as context managers. - def __enter__(self): pass - def __exit__(self, exc_type, exc_val, exc_tb): pass + def __enter__(self): + pass + + def __exit__(self, exc_type, exc_val, exc_tb): + pass def chdir_to_source(self): self.chdir() diff --git a/lib/spack/spack/test/__init__.py b/lib/spack/spack/test/__init__.py index a849d5f350..3439764ee6 100644 --- a/lib/spack/spack/test/__init__.py +++ b/lib/spack/spack/test/__init__.py @@ -39,9 +39,9 @@ test_names = [ 'pattern', 'python_version', 'git_fetch', 'svn_fetch', 'hg_fetch', 'mirror', 'modules', 'url_extrapolate', 'cc', 'link_tree', 'spec_yaml', 'optional_deps', 'make_executable', 'build_system_guess', 'lock', - 'database', 'namespace_trie', 'yaml', 'sbang', 'environment', 'cmd.find', - 'cmd.uninstall', 'cmd.test_install', 'cmd.test_compiler_cmd', - 'cmd.module' + 'database', 'namespace_trie', 'yaml', 'sbang', 'environment', + 'concretize_preferences', 'cmd.find', 'cmd.uninstall', 'cmd.test_install', + 'cmd.test_compiler_cmd', 'cmd.module' ] diff --git a/lib/spack/spack/test/architecture.py b/lib/spack/spack/test/architecture.py index 09bdb021af..42dd9f4c04 100644 --- a/lib/spack/spack/test/architecture.py +++ b/lib/spack/spack/test/architecture.py @@ -31,7 +31,7 @@ import platform as py_platform import spack import spack.architecture from spack.spec import * -from spack.platforms.cray_xc import CrayXc +from spack.platforms.cray import Cray from spack.platforms.linux import Linux from spack.platforms.bgq import Bgq from spack.platforms.darwin import Darwin @@ -76,7 +76,7 @@ class ArchitectureTest(MockPackagesTest): def test_platform(self): output_platform_class = spack.architecture.platform() if os.path.exists('/opt/cray/craype'): - my_platform_class = CrayXc() + my_platform_class = Cray() elif os.path.exists('/bgsys'): my_platform_class = Bgq() elif 'Linux' in py_platform.system(): diff --git a/lib/spack/spack/test/concretize_preferences.py b/lib/spack/spack/test/concretize_preferences.py new file mode 100644 index 0000000000..2c8bedc33f --- /dev/null +++ b/lib/spack/spack/test/concretize_preferences.py @@ -0,0 +1,106 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +import spack +import spack.architecture +from spack.test.mock_packages_test import * +from tempfile import mkdtemp + + +class ConcretizePreferencesTest(MockPackagesTest): + """Test concretization preferences are being applied correctly. + """ + + def setUp(self): + """Create config section to store concretization preferences + """ + super(ConcretizePreferencesTest, self).setUp() + self.tmp_dir = mkdtemp('.tmp', 'spack-config-test-') + spack.config.ConfigScope('concretize', + os.path.join(self.tmp_dir, 'concretize')) + + def tearDown(self): + super(ConcretizePreferencesTest, self).tearDown() + shutil.rmtree(self.tmp_dir, True) + spack.pkgsort = spack.PreferredPackages() + + def concretize(self, abstract_spec): + return Spec(abstract_spec).concretized() + + def update_packages(self, pkgname, section, value): + """Update config and reread package list""" + conf = {pkgname: {section: value}} + spack.config.update_config('packages', conf, 'concretize') + spack.pkgsort = spack.PreferredPackages() + + def assert_variant_values(self, spec, **variants): + concrete = self.concretize(spec) + for variant, value in variants.items(): + self.assertEqual(concrete.variants[variant].value, value) + + def test_preferred_variants(self): + """Test preferred variants are applied correctly + """ + self.update_packages('mpileaks', 'variants', + '~debug~opt+shared+static') + self.assert_variant_values('mpileaks', debug=False, opt=False, + shared=True, static=True) + + self.update_packages('mpileaks', 'variants', + ['+debug', '+opt', '~shared', '-static']) + self.assert_variant_values('mpileaks', debug=True, opt=True, + shared=False, static=False) + + def test_preferred_compilers(self): + """Test preferred compilers are applied correctly + """ + self.update_packages('mpileaks', 'compiler', ['clang@3.3']) + spec = self.concretize('mpileaks') + self.assertEqual(spec.compiler, spack.spec.CompilerSpec('clang@3.3')) + + self.update_packages('mpileaks', 'compiler', ['gcc@4.5.0']) + spec = self.concretize('mpileaks') + self.assertEqual(spec.compiler, spack.spec.CompilerSpec('gcc@4.5.0')) + + def test_preferred_versions(self): + """Test preferred package versions are applied correctly + """ + self.update_packages('mpileaks', 'version', ['2.3']) + spec = self.concretize('mpileaks') + self.assertEqual(spec.version, spack.spec.Version('2.3')) + + self.update_packages('mpileaks', 'version', ['2.2']) + spec = self.concretize('mpileaks') + self.assertEqual(spec.version, spack.spec.Version('2.2')) + + def test_preferred_providers(self): + """Test preferred providers of virtual packages are applied correctly + """ + self.update_packages('all', 'providers', {'mpi': ['mpich']}) + spec = self.concretize('mpileaks') + self.assertTrue('mpich' in spec) + + self.update_packages('all', 'providers', {'mpi': ['zmpi']}) + spec = self.concretize('mpileaks') + self.assertTrue('zmpi', spec) diff --git a/lib/spack/spack/test/modules.py b/lib/spack/spack/test/modules.py index 135cd028e3..5e280d8e43 100644 --- a/lib/spack/spack/test/modules.py +++ b/lib/spack/spack/test/modules.py @@ -89,7 +89,10 @@ configuration_alter_environment = { 'enable': ['tcl'], 'tcl': { 'all': { - 'filter': {'environment_blacklist': ['CMAKE_PREFIX_PATH']} + 'filter': {'environment_blacklist': ['CMAKE_PREFIX_PATH']}, + 'environment': { + 'set': {'{name}_ROOT': '{prefix}'} + } }, 'platform=test target=x86_64': { 'environment': { @@ -248,6 +251,8 @@ class TclTests(MockPackagesTest): self.assertEqual( len([x for x in content if 'setenv FOO "foo"' in x]), 1) self.assertEqual(len([x for x in content if 'unsetenv BAR' in x]), 1) + self.assertEqual( + len([x for x in content if 'setenv MPILEAKS_ROOT' in x]), 1) spec = spack.spec.Spec('libdwarf %clang platform=test target=x86_32') content = self.get_modulefile_content(spec) @@ -262,6 +267,8 @@ class TclTests(MockPackagesTest): len([x for x in content if 'is-loaded foo/bar' in x]), 1) self.assertEqual( len([x for x in content if 'module load foo/bar' in x]), 1) + self.assertEqual( + len([x for x in content if 'setenv LIBDWARF_ROOT' in x]), 1) def test_blacklist(self): spack.modules.CONFIGURATION = configuration_blacklist diff --git a/lib/spack/spack/util/web.py b/lib/spack/spack/util/web.py index 47abc507e0..cac783a368 100644 --- a/lib/spack/spack/util/web.py +++ b/lib/spack/spack/util/web.py @@ -25,8 +25,7 @@ import re import os import sys -import subprocess -import urllib2, cookielib +import urllib2 import urlparse from multiprocessing import Pool from HTMLParser import HTMLParser, HTMLParseError @@ -84,7 +83,7 @@ def _spider(args): req.get_method = lambda: "HEAD" resp = urllib2.urlopen(req, timeout=TIMEOUT) - if not "Content-type" in resp.headers: + if "Content-type" not in resp.headers: tty.debug("ignoring page " + url) return pages, links @@ -125,11 +124,11 @@ def _spider(args): if abs_link in visited: continue - # If we're not at max depth, follow links. - if depth < max_depth: - subcalls.append((abs_link, visited, root, None, - depth+1, max_depth, raise_on_error)) - visited.add(abs_link) + # If we're not at max depth, follow links. + if depth < max_depth: + subcalls.append((abs_link, visited, root, None, + depth + 1, max_depth, raise_on_error)) + visited.add(abs_link) if subcalls: try: @@ -142,22 +141,22 @@ def _spider(args): pool.terminate() pool.join() - except urllib2.URLError, e: + except urllib2.URLError as e: tty.debug(e) if raise_on_error: raise spack.error.NoNetworkConnectionError(str(e), url) - except HTMLParseError, e: + except HTMLParseError as e: # This error indicates that Python's HTML parser sucks. msg = "Got an error parsing HTML." # Pre-2.7.3 Pythons in particular have rather prickly HTML parsing. - if sys.version_info[:3] < (2,7,3): + if sys.version_info[:3] < (2, 7, 3): msg += " Use Python 2.7.3 or newer for better HTML parsing." tty.warn(msg, url, "HTMLParseError: " + str(e)) - except Exception, e: + except Exception as e: # Other types of errors are completely ignored, except in debug mode. tty.debug("Error in _spider: %s" % e) @@ -173,7 +172,8 @@ def spider(root_url, **kwargs): performance over a sequential fetch. """ max_depth = kwargs.setdefault('depth', 1) - pages, links = _spider((root_url, set(), root_url, None, 1, max_depth, False)) + pages, links = _spider((root_url, set(), root_url, None, + 1, max_depth, False)) return pages, links @@ -235,7 +235,7 @@ def find_versions_of_archive(*archive_urls, **kwargs): try: ver = spack.url.parse_version(url) versions[ver] = url - except spack.url.UndetectableVersionError as e: + except spack.url.UndetectableVersionError: continue return versions |