diff options
Diffstat (limited to 'lib')
| -rw-r--r-- | lib/spack/llnl/util/filesystem.py | 18 | ||||
| -rw-r--r-- | lib/spack/spack/concretize.py | 12 | ||||
| -rw-r--r-- | lib/spack/spack/config.py | 13 | ||||
| -rw-r--r-- | lib/spack/spack/operating_systems/cnl.py | 24 | ||||
| -rw-r--r-- | lib/spack/spack/platforms/cray.py | 105 | ||||
| -rw-r--r-- | lib/spack/spack/platforms/cray_xc.py | 72 | ||||
| -rw-r--r-- | lib/spack/spack/stage.py | 112 | ||||
| -rw-r--r-- | lib/spack/spack/test/architecture.py | 4 |
8 files changed, 224 insertions, 136 deletions
diff --git a/lib/spack/llnl/util/filesystem.py b/lib/spack/llnl/util/filesystem.py index 6e4cd338fe..2478f5c159 100644 --- a/lib/spack/llnl/util/filesystem.py +++ b/lib/spack/llnl/util/filesystem.py @@ -29,8 +29,9 @@ import shutil import stat import errno import getpass -from contextlib import contextmanager, closing +from contextlib import contextmanager import subprocess +import fileinput import llnl.util.tty as tty @@ -85,13 +86,14 @@ def filter_file(regex, repl, *filenames, **kwargs): if ignore_absent and not os.path.exists(filename): continue - shutil.copy(filename, backup_filename) + # Create backup file. Don't overwrite an existing backup + # file in case this file is being filtered multiple times. + if not os.path.exists(backup_filename): + shutil.copy(filename, backup_filename) + try: - with closing(open(backup_filename)) as infile: - with closing(open(filename, 'w')) as outfile: - for line in infile: - foo = re.sub(regex, repl, line) - outfile.write(foo) + for line in fileinput.input(filename, inplace=True): + print(re.sub(regex, repl, line.rstrip())) except: # clean up the original file on failure. shutil.move(backup_filename, filename) @@ -189,7 +191,7 @@ def install(src, dest): def install_tree(src, dest, **kwargs): - """Manually install a file to a particular location.""" + """Manually install a directory tree to a particular location.""" tty.debug("Installing %s to %s" % (src, dest)) shutil.copytree(src, dest, **kwargs) diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 5180f3cf04..eced9917c9 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -336,14 +336,18 @@ class DefaultConcretizer(object): spack.pkgsort.compiler_compare, other_spec.name) matches = sorted(compiler_list, cmp=cmp_compilers) if not matches: - raise UnavailableCompilerVersionError(other_compiler) + arch = spec.architecture + raise UnavailableCompilerVersionError(other_compiler, + arch.platform_os) # copy concrete version into other_compiler index = 0 while not _proper_compiler_style(matches[index], spec.architecture): index += 1 if index == len(matches) - 1: - raise NoValidVersionError(spec) + arch = spec.architecture + raise UnavailableCompilerVersionError(spec.compiler, + arch.platform_os) spec.compiler = matches[index].copy() assert(spec.compiler.concrete) return True # things changed. @@ -489,9 +493,9 @@ class UnavailableCompilerVersionError(spack.error.SpackError): """Raised when there is no available compiler that satisfies a compiler spec.""" - def __init__(self, compiler_spec): + def __init__(self, compiler_spec, operating_system): super(UnavailableCompilerVersionError, self).__init__( - "No available compiler version matches '%s'" % compiler_spec, + "No available compiler version matches '%s' on operating_system %s" % compiler_spec, operating_system, # NOQA: ignore=E501 "Run 'spack compilers' to see available compiler Options.") diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index e2e7dbc0ee..31f0eb3a56 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -266,6 +266,19 @@ section_schemas = { ], }, },},},},},}, + 'targets': { + '$schema': 'http://json-schema.org/schema#', + 'title': 'Spack target configuration file schema', + 'type': 'object', + 'additionalProperties': False, + 'patternProperties': { + r'targets:?': { + 'type': 'object', + 'default': {}, + 'additionalProperties': False, + 'patternProperties': { + r'\w[\w-]*': { # target name + 'type': 'string' ,},},},},}, 'modules': { '$schema': 'http://json-schema.org/schema#', 'title': 'Spack module file configuration file schema', diff --git a/lib/spack/spack/operating_systems/cnl.py b/lib/spack/spack/operating_systems/cnl.py index c160a60be8..dbd2775861 100644 --- a/lib/spack/spack/operating_systems/cnl.py +++ b/lib/spack/spack/operating_systems/cnl.py @@ -7,11 +7,12 @@ import spack.spec from spack.util.multiproc import parmap import spack.compilers + class Cnl(OperatingSystem): """ Compute Node Linux (CNL) is the operating system used for the Cray XC series super computers. It is a very stripped down version of GNU/Linux. Any compilers found through this operating system will be used with - modules. If updated, user must make sure that version and name are + modules. If updated, user must make sure that version and name are updated to indicate that OS has been upgraded (or downgraded) """ def __init__(self): @@ -19,17 +20,19 @@ class Cnl(OperatingSystem): version = '10' super(Cnl, self).__init__(name, version) + def __str__(self): + return self.name def find_compilers(self, *paths): types = spack.compilers.all_compiler_types() - compiler_lists = parmap(lambda cmp_cls: self.find_compiler(cmp_cls, *paths), types) + compiler_lists = parmap( + lambda cmp_cls: self.find_compiler(cmp_cls, *paths), types) # ensure all the version calls we made are cached in the parent # process, as well. This speeds up Spack a lot. - clist = reduce(lambda x,y: x+y, compiler_lists) + clist = reduce(lambda x, y: x + y, compiler_lists) return clist - def find_compiler(self, cmp_cls, *paths): compilers = [] if cmp_cls.PrgEnv: @@ -45,13 +48,16 @@ class Cnl(OperatingSystem): if paths: module_paths = ':' + ':'.join(p for p in paths) os.environ['MODULEPATH'] = module_paths - - output = modulecmd('avail', cmp_cls.PrgEnv_compiler, output=str, error=str) - matches = re.findall(r'(%s)/([\d\.]+[\d])' % cmp_cls.PrgEnv_compiler, output) + + output = modulecmd( + 'avail', cmp_cls.PrgEnv_compiler, output=str, error=str) + matches = re.findall( + r'(%s)/([\d\.]+[\d])' % cmp_cls.PrgEnv_compiler, output) for name, version in matches: v = version - comp = cmp_cls(spack.spec.CompilerSpec(name + '@' + v), self, - ['cc', 'CC', 'ftn'], [cmp_cls.PrgEnv, name +'/' + v]) + comp = cmp_cls( + spack.spec.CompilerSpec(name + '@' + v), self, + ['cc', 'CC', 'ftn'], [cmp_cls.PrgEnv, name + '/' + v]) compilers.append(comp) diff --git a/lib/spack/spack/platforms/cray.py b/lib/spack/spack/platforms/cray.py new file mode 100644 index 0000000000..2a3b81cf9c --- /dev/null +++ b/lib/spack/spack/platforms/cray.py @@ -0,0 +1,105 @@ +import os +import re +import spack.config +import llnl.util.tty as tty +from spack.util.executable import which +from spack.architecture import Platform, Target, NoPlatformError +from spack.operating_systems.linux_distro import LinuxDistro +from spack.operating_systems.cnl import Cnl +from llnl.util.filesystem import join_path + +# Craype- module prefixes that are not valid CPU targets. +NON_TARGETS = ('hugepages', 'network', 'target', 'accel', 'xtpe') + + +def _target_from_clean_env(name): + '''Return the default back_end target as loaded in a clean login session. + + A bash subshell is launched with a wiped environment and the list of loaded + modules is parsed for the first acceptable CrayPE target. + ''' + # Based on the incantation: + # echo "$(env - USER=$USER /bin/bash -l -c 'module list -lt')" + targets = [] + if name != 'front_end': + env = which('env') + env.add_default_arg('-') + # CAUTION - $USER is generally needed to initialize the environment. + # There may be other variables needed for general success. + output = env('USER=%s' % os.environ['USER'], + '/bin/bash', '--noprofile', '--norc', '-c', + '. /etc/profile; module list -lt', + output=str, error=str) + default_modules = [i for i in output.splitlines() + if len(i.split()) == 1] + tty.debug("Found default modules:", + *[" " + mod for mod in default_modules]) + pattern = 'craype-(?!{0})(\S*)'.format('|'.join(NON_TARGETS)) + for mod in default_modules: + if 'craype-' in mod: + targets.extend(re.findall(pattern, mod)) + return targets[0] if targets else None + + +class Cray(Platform): + priority = 10 + + def __init__(self): + ''' Create a Cray system platform. + + Target names should use craype target names but not include the + 'craype-' prefix. Uses first viable target from: + self + envars [SPACK_FRONT_END, SPACK_BACK_END] + configuration file "targets.yaml" with keys 'front_end', 'back_end' + scanning /etc/bash/bashrc.local for back_end only + ''' + super(Cray, self).__init__('cray') + + # Get targets from config or make best guess from environment: + conf = spack.config.get_config('targets') + for name in ('front_end', 'back_end'): + _target = getattr(self, name, None) + if _target is None: + _target = os.environ.get('SPACK_' + name.upper()) + if _target is None: + _target = conf.get(name) + if _target is None: + _target = _target_from_clean_env(name) + setattr(self, name, _target) + + if _target is not None: + self.add_target(name, Target(_target, 'craype-' + _target)) + self.add_target(_target, Target(_target, 'craype-' + _target)) + + if self.back_end is not None: + self.default = self.back_end + self.add_target( + 'default', Target(self.default, 'craype-' + self.default)) + else: + raise NoPlatformError() + + front_distro = LinuxDistro() + back_distro = Cnl() + + self.default_os = str(back_distro) + self.back_os = self.default_os + self.front_os = str(front_distro) + + self.add_operating_system(self.back_os, back_distro) + self.add_operating_system(self.front_os, front_distro) + + @classmethod + def setup_platform_environment(self, pkg, env): + """ Change the linker to default dynamic to be more + similar to linux/standard linker behavior + """ + env.set('CRAYPE_LINK_TYPE', 'dynamic') + cray_wrapper_names = join_path(spack.build_env_path, 'cray') + if os.path.isdir(cray_wrapper_names): + env.prepend_path('PATH', cray_wrapper_names) + env.prepend_path('SPACK_ENV_PATHS', cray_wrapper_names) + + @classmethod + def detect(self): + return os.environ.get('CRAYPE_VERSION') is not None diff --git a/lib/spack/spack/platforms/cray_xc.py b/lib/spack/spack/platforms/cray_xc.py deleted file mode 100644 index 03d0383cc5..0000000000 --- a/lib/spack/spack/platforms/cray_xc.py +++ /dev/null @@ -1,72 +0,0 @@ -import os -import spack -from spack.architecture import Platform, Target -from spack.operating_systems.linux_distro import LinuxDistro -from spack.operating_systems.cnl import Cnl -from spack.util.executable import which -from llnl.util.filesystem import join_path - - -class CrayXc(Platform): - priority = 20 - front_end = 'sandybridge' - back_end = 'ivybridge' - default = 'ivybridge' - - back_os = "CNL10" - default_os = "CNL10" - - def __init__(self): - ''' Since cori doesn't have ivybridge as a front end it's better - if we use CRAY_CPU_TARGET as the default. This will ensure - that if we're on a XC-40 or XC-30 then we can detect the target - ''' - super(CrayXc, self).__init__('cray_xc') - - # Handle the default here so we can check for a key error - if 'CRAY_CPU_TARGET' in os.environ: - self.default = os.environ['CRAY_CPU_TARGET'] - - # Change the defaults to haswell if we're on an XC40 - if self.default == 'haswell': - self.front_end = self.default - self.back_end = self.default - - # Could switch to use modules and fe targets for front end - # Currently using compilers by path for front end. - self.add_target('sandybridge', Target('sandybridge')) - self.add_target('ivybridge', - Target('ivybridge', 'craype-ivybridge')) - self.add_target('haswell', - Target('haswell', 'craype-haswell')) - - # Front end of the cray platform is a linux distro. - linux_dist = LinuxDistro() - self.front_os = str(linux_dist) - self.add_operating_system(str(linux_dist), linux_dist) - self.add_operating_system('CNL10', Cnl()) - - @classmethod - def setup_platform_environment(self, pkg, env): - """ Change the linker to default dynamic to be more - similar to linux/standard linker behavior - """ - env.set('CRAYPE_LINK_TYPE', 'dynamic') - cray_wrapper_names = join_path(spack.build_env_path, 'cray') - if os.path.isdir(cray_wrapper_names): - env.prepend_path('PATH', cray_wrapper_names) - env.prepend_path('SPACK_ENV_PATHS', cray_wrapper_names) - - @classmethod - def detect(self): - try: - cc_verbose = which('ftn') - text = cc_verbose('-craype-verbose', - output=str, error=str, - ignore_errors=True).split() - if '-D__CRAYXC' in text: - return True - else: - return False - except: - return False diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index b08cce43b8..1a8b1a169a 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -37,6 +37,7 @@ import spack import spack.config import spack.fetch_strategy as fs import spack.error +from spack.version import * STAGE_PREFIX = 'spack-stage-' @@ -51,10 +52,12 @@ class Stage(object): lifecycle looks like this: ``` - with Stage() as stage: # Context manager creates and destroys the stage directory + with Stage() as stage: # Context manager creates and destroys the + # stage directory stage.fetch() # Fetch a source archive into the stage. stage.expand_archive() # Expand the source archive. - <install> # Build and install the archive. (handled by user of Stage) + <install> # Build and install the archive. (handled by + # user of Stage) ``` When used as a context manager, the stage is automatically @@ -71,7 +74,8 @@ class Stage(object): stage.create() # Explicitly create the stage directory. stage.fetch() # Fetch a source archive into the stage. stage.expand_archive() # Expand the source archive. - <install> # Build and install the archive. (handled by user of Stage) + <install> # Build and install the archive. (handled by + # user of Stage) finally: stage.destroy() # Explicitly destroy the stage directory. ``` @@ -120,13 +124,17 @@ class Stage(object): elif isinstance(url_or_fetch_strategy, fs.FetchStrategy): self.fetcher = url_or_fetch_strategy else: - raise ValueError("Can't construct Stage without url or fetch strategy") + raise ValueError( + "Can't construct Stage without url or fetch strategy") self.fetcher.set_stage(self) - self.default_fetcher = self.fetcher # self.fetcher can change with mirrors. - self.skip_checksum_for_mirror = True # used for mirrored archives of repositories. + # self.fetcher can change with mirrors. + self.default_fetcher = self.fetcher + # used for mirrored archives of repositories. + self.skip_checksum_for_mirror = True - # TODO : this uses a protected member of tempfile, but seemed the only way to get a temporary name - # TODO : besides, the temporary link name won't be the same as the temporary stage area in tmp_root + # TODO : this uses a protected member of tempfile, but seemed the only + # TODO : way to get a temporary name besides, the temporary link name + # TODO : won't be the same as the temporary stage area in tmp_root self.name = name if name is None: self.name = STAGE_PREFIX + next(tempfile._get_candidate_names()) @@ -143,7 +151,6 @@ class Stage(object): # Flag to decide whether to delete the stage folder on exit or not self.keep = keep - def __enter__(self): """ Entering a stage context will create the stage directory @@ -154,7 +161,6 @@ class Stage(object): self.create() return self - def __exit__(self, exc_type, exc_val, exc_tb): """ Exiting from a stage context will delete the stage directory unless: @@ -173,12 +179,10 @@ class Stage(object): if exc_type is None and not self.keep: self.destroy() - def _need_to_create_path(self): """Makes sure nothing weird has happened since the last time we looked at path. Returns True if path already exists and is ok. - Returns False if path needs to be created. - """ + Returns False if path needs to be created.""" # Path doesn't exist yet. Will need to create it. if not os.path.exists(self.path): return True @@ -196,7 +200,8 @@ class Stage(object): if spack.use_tmp_stage: # If we're using a tmp dir, it's a link, and it points at the # right spot, then keep it. - if (real_path.startswith(real_tmp) and os.path.exists(real_path)): + if (real_path.startswith(real_tmp) and + os.path.exists(real_path)): return False else: # otherwise, just unlink it and start over. @@ -204,7 +209,8 @@ class Stage(object): return True else: - # If we're not tmp mode, then it's a link and we want a directory. + # If we're not tmp mode, then it's a link and we want a + # directory. os.unlink(self.path) return True @@ -215,10 +221,12 @@ class Stage(object): """Possible archive file paths.""" paths = [] if isinstance(self.fetcher, fs.URLFetchStrategy): - paths.append(os.path.join(self.path, os.path.basename(self.fetcher.url))) + paths.append(os.path.join( + self.path, os.path.basename(self.fetcher.url))) if self.mirror_path: - paths.append(os.path.join(self.path, os.path.basename(self.mirror_path))) + paths.append(os.path.join( + self.path, os.path.basename(self.mirror_path))) return paths @@ -227,10 +235,12 @@ class Stage(object): """Path to the source archive within this stage directory.""" paths = [] if isinstance(self.fetcher, fs.URLFetchStrategy): - paths.append(os.path.join(self.path, os.path.basename(self.fetcher.url))) + paths.append(os.path.join( + self.path, os.path.basename(self.fetcher.url))) if self.mirror_path: - paths.append(os.path.join(self.path, os.path.basename(self.mirror_path))) + paths.append(os.path.join( + self.path, os.path.basename(self.mirror_path))) for path in paths: if os.path.exists(path): @@ -262,7 +272,8 @@ class Stage(object): return None def chdir(self): - """Changes directory to the stage path. Or dies if it is not set up.""" + """Changes directory to the stage path. Or dies if it is not set + up.""" if os.path.isdir(self.path): os.chdir(self.path) else: @@ -306,6 +317,19 @@ class Stage(object): fetchers.insert(0, fs.URLFetchStrategy(url, digest)) fetchers.insert(0, spack.cache.fetcher(self.mirror_path, digest)) + # Look for the archive in list_url + archive_version = spack.url.parse_version(self.default_fetcher.url) + package_name = os.path.dirname(self.mirror_path) + pkg = spack.repo.get(package_name) + if pkg.list_url is not None and pkg.url is not None: + versions = pkg.fetch_remote_versions() + try: + url_from_list = versions[Version(archive_version)] + fetchers.append(fs.URLFetchStrategy(url_from_list, digest)) + except KeyError: + tty.msg("Can not find version %s in url_list" % + archive_version) + for fetcher in fetchers: try: fetcher.set_stage(self) @@ -321,11 +345,11 @@ class Stage(object): self.fetcher = self.default_fetcher raise fs.FetchError(errMessage, None) - def check(self): """Check the downloaded archive against a checksum digest. No-op if this stage checks code out of a repository.""" - if self.fetcher is not self.default_fetcher and self.skip_checksum_for_mirror: + if self.fetcher is not self.default_fetcher and \ + self.skip_checksum_for_mirror: tty.warn("Fetching from mirror without a checksum!", "This package is normally checked out from a version " "control system, but it has been archived on a spack " @@ -335,16 +359,13 @@ class Stage(object): else: self.fetcher.check() - def cache_local(self): spack.cache.store(self.fetcher, self.mirror_path) - def expand_archive(self): """Changes to the stage directory and attempt to expand the downloaded - archive. Fail if the stage is not set up or if the archive is not yet - downloaded. - """ + archive. Fail if the stage is not set up or if the archive is not yet + downloaded.""" archive_dir = self.source_path if not archive_dir: self.fetcher.expand() @@ -386,8 +407,8 @@ class Stage(object): # Create the top-level stage directory mkdirp(spack.stage_path) remove_dead_links(spack.stage_path) - # If a tmp_root exists then create a directory there and then link it in the stage area, - # otherwise create the stage directory in self.path + # If a tmp_root exists then create a directory there and then link it + # in the stage area, otherwise create the stage directory in self.path if self._need_to_create_path(): if self.tmp_root: tmp_dir = tempfile.mkdtemp('', STAGE_PREFIX, self.tmp_root) @@ -409,6 +430,7 @@ class Stage(object): class ResourceStage(Stage): + def __init__(self, url_or_fetch_strategy, root, resource, **kwargs): super(ResourceStage, self).__init__(url_or_fetch_strategy, **kwargs) self.root_stage = root @@ -418,12 +440,15 @@ class ResourceStage(Stage): super(ResourceStage, self).expand_archive() root_stage = self.root_stage resource = self.resource - placement = os.path.basename(self.source_path) if resource.placement is None else resource.placement + placement = os.path.basename(self.source_path) \ + if resource.placement is None \ + else resource.placement if not isinstance(placement, dict): placement = {'': placement} # Make the paths in the dictionary absolute and link for key, value in placement.iteritems(): - target_path = join_path(root_stage.source_path, resource.destination) + target_path = join_path( + root_stage.source_path, resource.destination) destination_path = join_path(target_path, value) source_path = join_path(self.source_path, key) @@ -437,21 +462,23 @@ class ResourceStage(Stage): if not os.path.exists(destination_path): # Create a symlink - tty.info('Moving resource stage\n\tsource : {stage}\n\tdestination : {destination}'.format( - stage=source_path, destination=destination_path - )) + tty.info('Moving resource stage\n\tsource : ' + '{stage}\n\tdestination : {destination}'.format( + stage=source_path, destination=destination_path + )) shutil.move(source_path, destination_path) -@pattern.composite(method_list=['fetch', 'create', 'check', 'expand_archive', 'restage', 'destroy', 'cache_local']) +@pattern.composite(method_list=['fetch', 'create', 'check', 'expand_archive', + 'restage', 'destroy', 'cache_local']) class StageComposite: - """ - Composite for Stage type objects. The first item in this composite is considered to be the root package, and - operations that return a value are forwarded to it. - """ + """Composite for Stage type objects. The first item in this composite is + considered to be the root package, and operations that return a value are + forwarded to it.""" # # __enter__ and __exit__ delegate to all stages in the composite. # + def __enter__(self): for item in self: item.__enter__() @@ -496,8 +523,11 @@ class DIYStage(object): raise ChdirError("Setup failed: no such directory: " + self.path) # DIY stages do nothing as context managers. - def __enter__(self): pass - def __exit__(self, exc_type, exc_val, exc_tb): pass + def __enter__(self): + pass + + def __exit__(self, exc_type, exc_val, exc_tb): + pass def chdir_to_source(self): self.chdir() diff --git a/lib/spack/spack/test/architecture.py b/lib/spack/spack/test/architecture.py index 09bdb021af..42dd9f4c04 100644 --- a/lib/spack/spack/test/architecture.py +++ b/lib/spack/spack/test/architecture.py @@ -31,7 +31,7 @@ import platform as py_platform import spack import spack.architecture from spack.spec import * -from spack.platforms.cray_xc import CrayXc +from spack.platforms.cray import Cray from spack.platforms.linux import Linux from spack.platforms.bgq import Bgq from spack.platforms.darwin import Darwin @@ -76,7 +76,7 @@ class ArchitectureTest(MockPackagesTest): def test_platform(self): output_platform_class = spack.architecture.platform() if os.path.exists('/opt/cray/craype'): - my_platform_class = CrayXc() + my_platform_class = Cray() elif os.path.exists('/bgsys'): my_platform_class = Bgq() elif 'Linux' in py_platform.system(): |