summaryrefslogtreecommitdiff
path: root/lib
diff options
context:
space:
mode:
Diffstat (limited to 'lib')
-rw-r--r--lib/spack/llnl/util/filesystem.py18
-rw-r--r--lib/spack/spack/cmd/create.py2
-rw-r--r--lib/spack/spack/concretize.py12
-rw-r--r--lib/spack/spack/config.py13
-rw-r--r--lib/spack/spack/environment.py10
-rw-r--r--lib/spack/spack/modules.py25
-rw-r--r--lib/spack/spack/operating_systems/cnl.py24
-rw-r--r--lib/spack/spack/platforms/cray.py105
-rw-r--r--lib/spack/spack/platforms/cray_xc.py72
-rw-r--r--lib/spack/spack/preferred_packages.py4
-rw-r--r--lib/spack/spack/stage.py113
-rw-r--r--lib/spack/spack/test/__init__.py6
-rw-r--r--lib/spack/spack/test/architecture.py4
-rw-r--r--lib/spack/spack/test/concretize_preferences.py106
-rw-r--r--lib/spack/spack/test/modules.py9
-rw-r--r--lib/spack/spack/util/web.py28
16 files changed, 389 insertions, 162 deletions
diff --git a/lib/spack/llnl/util/filesystem.py b/lib/spack/llnl/util/filesystem.py
index 6e4cd338fe..2478f5c159 100644
--- a/lib/spack/llnl/util/filesystem.py
+++ b/lib/spack/llnl/util/filesystem.py
@@ -29,8 +29,9 @@ import shutil
import stat
import errno
import getpass
-from contextlib import contextmanager, closing
+from contextlib import contextmanager
import subprocess
+import fileinput
import llnl.util.tty as tty
@@ -85,13 +86,14 @@ def filter_file(regex, repl, *filenames, **kwargs):
if ignore_absent and not os.path.exists(filename):
continue
- shutil.copy(filename, backup_filename)
+ # Create backup file. Don't overwrite an existing backup
+ # file in case this file is being filtered multiple times.
+ if not os.path.exists(backup_filename):
+ shutil.copy(filename, backup_filename)
+
try:
- with closing(open(backup_filename)) as infile:
- with closing(open(filename, 'w')) as outfile:
- for line in infile:
- foo = re.sub(regex, repl, line)
- outfile.write(foo)
+ for line in fileinput.input(filename, inplace=True):
+ print(re.sub(regex, repl, line.rstrip()))
except:
# clean up the original file on failure.
shutil.move(backup_filename, filename)
@@ -189,7 +191,7 @@ def install(src, dest):
def install_tree(src, dest, **kwargs):
- """Manually install a file to a particular location."""
+ """Manually install a directory tree to a particular location."""
tty.debug("Installing %s to %s" % (src, dest))
shutil.copytree(src, dest, **kwargs)
diff --git a/lib/spack/spack/cmd/create.py b/lib/spack/spack/cmd/create.py
index da74ceb2f6..51bf17a44b 100644
--- a/lib/spack/spack/cmd/create.py
+++ b/lib/spack/spack/cmd/create.py
@@ -165,7 +165,7 @@ install_dict = {
'python': """\
# FIXME: Add logic to build and install here.
- python('setup.py', 'install', '--prefix={0}'.format(prefix))""",
+ setup_py('install', '--prefix={0}'.format(prefix))""",
'R': """\
# FIXME: Add logic to build and install here.
diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py
index 5180f3cf04..6f11c86ce8 100644
--- a/lib/spack/spack/concretize.py
+++ b/lib/spack/spack/concretize.py
@@ -336,14 +336,18 @@ class DefaultConcretizer(object):
spack.pkgsort.compiler_compare, other_spec.name)
matches = sorted(compiler_list, cmp=cmp_compilers)
if not matches:
- raise UnavailableCompilerVersionError(other_compiler)
+ arch = spec.architecture
+ raise UnavailableCompilerVersionError(other_compiler,
+ arch.platform_os)
# copy concrete version into other_compiler
index = 0
while not _proper_compiler_style(matches[index], spec.architecture):
index += 1
if index == len(matches) - 1:
- raise NoValidVersionError(spec)
+ arch = spec.architecture
+ raise UnavailableCompilerVersionError(spec.compiler,
+ arch.platform_os)
spec.compiler = matches[index].copy()
assert(spec.compiler.concrete)
return True # things changed.
@@ -489,9 +493,9 @@ class UnavailableCompilerVersionError(spack.error.SpackError):
"""Raised when there is no available compiler that satisfies a
compiler spec."""
- def __init__(self, compiler_spec):
+ def __init__(self, compiler_spec, operating_system):
super(UnavailableCompilerVersionError, self).__init__(
- "No available compiler version matches '%s'" % compiler_spec,
+ "No available compiler version matches '%s' on operating_system %s" % (compiler_spec, operating_system), # NOQA: ignore=E501
"Run 'spack compilers' to see available compiler Options.")
diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py
index e2e7dbc0ee..31f0eb3a56 100644
--- a/lib/spack/spack/config.py
+++ b/lib/spack/spack/config.py
@@ -266,6 +266,19 @@ section_schemas = {
], },
},},},},},},
+ 'targets': {
+ '$schema': 'http://json-schema.org/schema#',
+ 'title': 'Spack target configuration file schema',
+ 'type': 'object',
+ 'additionalProperties': False,
+ 'patternProperties': {
+ r'targets:?': {
+ 'type': 'object',
+ 'default': {},
+ 'additionalProperties': False,
+ 'patternProperties': {
+ r'\w[\w-]*': { # target name
+ 'type': 'string' ,},},},},},
'modules': {
'$schema': 'http://json-schema.org/schema#',
'title': 'Spack module file configuration file schema',
diff --git a/lib/spack/spack/environment.py b/lib/spack/spack/environment.py
index 30c6228ca4..41136ab2eb 100644
--- a/lib/spack/spack/environment.py
+++ b/lib/spack/spack/environment.py
@@ -37,6 +37,10 @@ class NameModifier(object):
self.args = {'name': name}
self.args.update(kwargs)
+ def update_args(self, **kwargs):
+ self.__dict__.update(kwargs)
+ self.args.update(kwargs)
+
class NameValueModifier(object):
@@ -44,7 +48,11 @@ class NameValueModifier(object):
self.name = name
self.value = value
self.separator = kwargs.get('separator', ':')
- self.args = {'name': name, 'value': value, 'delim': self.separator}
+ self.args = {'name': name, 'value': value, 'separator': self.separator}
+ self.args.update(kwargs)
+
+ def update_args(self, **kwargs):
+ self.__dict__.update(kwargs)
self.args.update(kwargs)
diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py
index db8b20ae42..8701a31c49 100644
--- a/lib/spack/spack/modules.py
+++ b/lib/spack/spack/modules.py
@@ -272,14 +272,26 @@ class EnvModule(object):
@property
def tokens(self):
+ """Tokens that can be substituted in environment variable values
+ and naming schemes
+ """
tokens = {
'name': self.spec.name,
'version': self.spec.version,
- 'compiler': self.spec.compiler
+ 'compiler': self.spec.compiler,
+ 'prefix': self.spec.package.prefix
}
return tokens
@property
+ def upper_tokens(self):
+ """Tokens that can be substituted in environment variable names"""
+ upper_tokens = {
+ 'name': self.spec.name.replace('-', '_').upper()
+ }
+ return upper_tokens
+
+ @property
def use_name(self):
"""
Subclasses should implement this to return the name the module command
@@ -438,6 +450,11 @@ class EnvModule(object):
def process_environment_command(self, env):
for command in env:
+ # Token expansion from configuration file
+ name = command.args.get('name', '').format(**self.upper_tokens)
+ value = str(command.args.get('value', '')).format(**self.tokens)
+ command.update_args(name=name, value=value)
+ # Format the line int the module file
try:
yield self.environment_modifications_formats[type(
command)].format(**command.args)
@@ -511,9 +528,9 @@ class TclModule(EnvModule):
name = 'tcl'
path = join_path(spack.share_path, "modules")
environment_modifications_formats = {
- PrependPath: 'prepend-path --delim "{delim}" {name} \"{value}\"\n',
- AppendPath: 'append-path --delim "{delim}" {name} \"{value}\"\n',
- RemovePath: 'remove-path --delim "{delim}" {name} \"{value}\"\n',
+ PrependPath: 'prepend-path --delim "{separator}" {name} \"{value}\"\n',
+ AppendPath: 'append-path --delim "{separator}" {name} \"{value}\"\n',
+ RemovePath: 'remove-path --delim "{separator}" {name} \"{value}\"\n',
SetEnv: 'setenv {name} \"{value}\"\n',
UnsetEnv: 'unsetenv {name}\n'
}
diff --git a/lib/spack/spack/operating_systems/cnl.py b/lib/spack/spack/operating_systems/cnl.py
index c160a60be8..dbd2775861 100644
--- a/lib/spack/spack/operating_systems/cnl.py
+++ b/lib/spack/spack/operating_systems/cnl.py
@@ -7,11 +7,12 @@ import spack.spec
from spack.util.multiproc import parmap
import spack.compilers
+
class Cnl(OperatingSystem):
""" Compute Node Linux (CNL) is the operating system used for the Cray XC
series super computers. It is a very stripped down version of GNU/Linux.
Any compilers found through this operating system will be used with
- modules. If updated, user must make sure that version and name are
+ modules. If updated, user must make sure that version and name are
updated to indicate that OS has been upgraded (or downgraded)
"""
def __init__(self):
@@ -19,17 +20,19 @@ class Cnl(OperatingSystem):
version = '10'
super(Cnl, self).__init__(name, version)
+ def __str__(self):
+ return self.name
def find_compilers(self, *paths):
types = spack.compilers.all_compiler_types()
- compiler_lists = parmap(lambda cmp_cls: self.find_compiler(cmp_cls, *paths), types)
+ compiler_lists = parmap(
+ lambda cmp_cls: self.find_compiler(cmp_cls, *paths), types)
# ensure all the version calls we made are cached in the parent
# process, as well. This speeds up Spack a lot.
- clist = reduce(lambda x,y: x+y, compiler_lists)
+ clist = reduce(lambda x, y: x + y, compiler_lists)
return clist
-
def find_compiler(self, cmp_cls, *paths):
compilers = []
if cmp_cls.PrgEnv:
@@ -45,13 +48,16 @@ class Cnl(OperatingSystem):
if paths:
module_paths = ':' + ':'.join(p for p in paths)
os.environ['MODULEPATH'] = module_paths
-
- output = modulecmd('avail', cmp_cls.PrgEnv_compiler, output=str, error=str)
- matches = re.findall(r'(%s)/([\d\.]+[\d])' % cmp_cls.PrgEnv_compiler, output)
+
+ output = modulecmd(
+ 'avail', cmp_cls.PrgEnv_compiler, output=str, error=str)
+ matches = re.findall(
+ r'(%s)/([\d\.]+[\d])' % cmp_cls.PrgEnv_compiler, output)
for name, version in matches:
v = version
- comp = cmp_cls(spack.spec.CompilerSpec(name + '@' + v), self,
- ['cc', 'CC', 'ftn'], [cmp_cls.PrgEnv, name +'/' + v])
+ comp = cmp_cls(
+ spack.spec.CompilerSpec(name + '@' + v), self,
+ ['cc', 'CC', 'ftn'], [cmp_cls.PrgEnv, name + '/' + v])
compilers.append(comp)
diff --git a/lib/spack/spack/platforms/cray.py b/lib/spack/spack/platforms/cray.py
new file mode 100644
index 0000000000..0059b49ff1
--- /dev/null
+++ b/lib/spack/spack/platforms/cray.py
@@ -0,0 +1,105 @@
+import os
+import re
+import spack.config
+import llnl.util.tty as tty
+from spack.util.executable import which
+from spack.architecture import Platform, Target, NoPlatformError
+from spack.operating_systems.linux_distro import LinuxDistro
+from spack.operating_systems.cnl import Cnl
+from llnl.util.filesystem import join_path
+
+# Craype- module prefixes that are not valid CPU targets.
+NON_TARGETS = ('hugepages', 'network', 'target', 'accel', 'xtpe')
+
+
+def _target_from_clean_env(name):
+ '''Return the default back_end target as loaded in a clean login session.
+
+ A bash subshell is launched with a wiped environment and the list of loaded
+ modules is parsed for the first acceptable CrayPE target.
+ '''
+ # Based on the incantation:
+ # echo "$(env - USER=$USER /bin/bash -l -c 'module list -lt')"
+ targets = []
+ if name != 'front_end':
+ env = which('env')
+ env.add_default_arg('-')
+ # CAUTION - $USER is generally needed to initialize the environment.
+ # There may be other variables needed for general success.
+ output = env('USER=%s' % os.environ['USER'],
+ '/bin/bash', '--noprofile', '--norc', '-c',
+ '. /etc/profile; module list -lt',
+ output=str, error=str)
+ default_modules = [i for i in output.splitlines()
+ if len(i.split()) == 1]
+ tty.debug("Found default modules:",
+ *[" " + mod for mod in default_modules])
+ pattern = 'craype-(?!{0})(\S*)'.format('|'.join(NON_TARGETS))
+ for mod in default_modules:
+ if 'craype-' in mod:
+ targets.extend(re.findall(pattern, mod))
+ return targets[0] if targets else None
+
+
+class Cray(Platform):
+ priority = 10
+
+ def __init__(self):
+ ''' Create a Cray system platform.
+
+ Target names should use craype target names but not include the
+ 'craype-' prefix. Uses first viable target from:
+ self
+ envars [SPACK_FRONT_END, SPACK_BACK_END]
+ configuration file "targets.yaml" with keys 'front_end', 'back_end'
+ scanning /etc/bash/bashrc.local for back_end only
+ '''
+ super(Cray, self).__init__('cray')
+
+ # Get targets from config or make best guess from environment:
+ conf = spack.config.get_config('targets')
+ for name in ('front_end', 'back_end'):
+ _target = getattr(self, name, None)
+ if _target is None:
+ _target = os.environ.get('SPACK_' + name.upper())
+ if _target is None:
+ _target = conf.get(name)
+ if _target is None:
+ _target = _target_from_clean_env(name)
+ setattr(self, name, _target)
+
+ if _target is not None:
+ self.add_target(name, Target(_target, 'craype-' + _target))
+ self.add_target(_target, Target(_target, 'craype-' + _target))
+
+ if self.back_end is not None:
+ self.default = self.back_end
+ self.add_target(
+ 'default', Target(self.default, 'craype-' + self.default))
+ else:
+ raise NoPlatformError()
+
+ front_distro = LinuxDistro()
+ back_distro = Cnl()
+
+ self.default_os = str(back_distro)
+ self.back_os = self.default_os
+ self.front_os = str(front_distro)
+
+ self.add_operating_system(self.back_os, back_distro)
+ self.add_operating_system(self.front_os, front_distro)
+
+ @classmethod
+ def setup_platform_environment(self, pkg, env):
+ """ Change the linker to default dynamic to be more
+ similar to linux/standard linker behavior
+ """
+ env.set('CRAYPE_LINK_TYPE', 'dynamic')
+ cray_wrapper_names = join_path(spack.build_env_path, 'cray')
+ if os.path.isdir(cray_wrapper_names):
+ env.prepend_path('PATH', cray_wrapper_names)
+ env.prepend_path('SPACK_ENV_PATH', cray_wrapper_names)
+
+ @classmethod
+ def detect(self):
+ return os.environ.get('CRAYPE_VERSION') is not None
diff --git a/lib/spack/spack/platforms/cray_xc.py b/lib/spack/spack/platforms/cray_xc.py
deleted file mode 100644
index 03d0383cc5..0000000000
--- a/lib/spack/spack/platforms/cray_xc.py
+++ /dev/null
@@ -1,72 +0,0 @@
-import os
-import spack
-from spack.architecture import Platform, Target
-from spack.operating_systems.linux_distro import LinuxDistro
-from spack.operating_systems.cnl import Cnl
-from spack.util.executable import which
-from llnl.util.filesystem import join_path
-
-
-class CrayXc(Platform):
- priority = 20
- front_end = 'sandybridge'
- back_end = 'ivybridge'
- default = 'ivybridge'
-
- back_os = "CNL10"
- default_os = "CNL10"
-
- def __init__(self):
- ''' Since cori doesn't have ivybridge as a front end it's better
- if we use CRAY_CPU_TARGET as the default. This will ensure
- that if we're on a XC-40 or XC-30 then we can detect the target
- '''
- super(CrayXc, self).__init__('cray_xc')
-
- # Handle the default here so we can check for a key error
- if 'CRAY_CPU_TARGET' in os.environ:
- self.default = os.environ['CRAY_CPU_TARGET']
-
- # Change the defaults to haswell if we're on an XC40
- if self.default == 'haswell':
- self.front_end = self.default
- self.back_end = self.default
-
- # Could switch to use modules and fe targets for front end
- # Currently using compilers by path for front end.
- self.add_target('sandybridge', Target('sandybridge'))
- self.add_target('ivybridge',
- Target('ivybridge', 'craype-ivybridge'))
- self.add_target('haswell',
- Target('haswell', 'craype-haswell'))
-
- # Front end of the cray platform is a linux distro.
- linux_dist = LinuxDistro()
- self.front_os = str(linux_dist)
- self.add_operating_system(str(linux_dist), linux_dist)
- self.add_operating_system('CNL10', Cnl())
-
- @classmethod
- def setup_platform_environment(self, pkg, env):
- """ Change the linker to default dynamic to be more
- similar to linux/standard linker behavior
- """
- env.set('CRAYPE_LINK_TYPE', 'dynamic')
- cray_wrapper_names = join_path(spack.build_env_path, 'cray')
- if os.path.isdir(cray_wrapper_names):
- env.prepend_path('PATH', cray_wrapper_names)
- env.prepend_path('SPACK_ENV_PATHS', cray_wrapper_names)
-
- @classmethod
- def detect(self):
- try:
- cc_verbose = which('ftn')
- text = cc_verbose('-craype-verbose',
- output=str, error=str,
- ignore_errors=True).split()
- if '-D__CRAYXC' in text:
- return True
- else:
- return False
- except:
- return False
diff --git a/lib/spack/spack/preferred_packages.py b/lib/spack/spack/preferred_packages.py
index 5f18e212b6..f079c1ef8b 100644
--- a/lib/spack/spack/preferred_packages.py
+++ b/lib/spack/spack/preferred_packages.py
@@ -162,8 +162,8 @@ class PreferredPackages(object):
"""Return a VariantMap of preferred variants and their values"""
variants = self.preferred.get(pkgname, {}).get('variants', '')
if not isinstance(variants, basestring):
- variants = "".join(variants)
- return spack.spec.Spec(pkgname + variants).variants
+ variants = " ".join(variants)
+ return spack.spec.Spec("%s %s" % (pkgname, variants)).variants
def version_compare(self, pkgname, a, b):
"""Return less-than-0, 0, or greater than 0 if version a of pkgname is
diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py
index b08cce43b8..553c4ad05f 100644
--- a/lib/spack/spack/stage.py
+++ b/lib/spack/spack/stage.py
@@ -37,6 +37,7 @@ import spack
import spack.config
import spack.fetch_strategy as fs
import spack.error
+from spack.version import *
STAGE_PREFIX = 'spack-stage-'
@@ -51,10 +52,12 @@ class Stage(object):
lifecycle looks like this:
```
- with Stage() as stage: # Context manager creates and destroys the stage directory
+ with Stage() as stage: # Context manager creates and destroys the
+ # stage directory
stage.fetch() # Fetch a source archive into the stage.
stage.expand_archive() # Expand the source archive.
- <install> # Build and install the archive. (handled by user of Stage)
+ <install> # Build and install the archive. (handled by
+ # user of Stage)
```
When used as a context manager, the stage is automatically
@@ -71,7 +74,8 @@ class Stage(object):
stage.create() # Explicitly create the stage directory.
stage.fetch() # Fetch a source archive into the stage.
stage.expand_archive() # Expand the source archive.
- <install> # Build and install the archive. (handled by user of Stage)
+ <install> # Build and install the archive. (handled by
+ # user of Stage)
finally:
stage.destroy() # Explicitly destroy the stage directory.
```
@@ -120,13 +124,17 @@ class Stage(object):
elif isinstance(url_or_fetch_strategy, fs.FetchStrategy):
self.fetcher = url_or_fetch_strategy
else:
- raise ValueError("Can't construct Stage without url or fetch strategy")
+ raise ValueError(
+ "Can't construct Stage without url or fetch strategy")
self.fetcher.set_stage(self)
- self.default_fetcher = self.fetcher # self.fetcher can change with mirrors.
- self.skip_checksum_for_mirror = True # used for mirrored archives of repositories.
+ # self.fetcher can change with mirrors.
+ self.default_fetcher = self.fetcher
+ # used for mirrored archives of repositories.
+ self.skip_checksum_for_mirror = True
- # TODO : this uses a protected member of tempfile, but seemed the only way to get a temporary name
- # TODO : besides, the temporary link name won't be the same as the temporary stage area in tmp_root
+ # TODO : this uses a protected member of tempfile, but seemed the only
+ # TODO : way to get a temporary name besides, the temporary link name
+ # TODO : won't be the same as the temporary stage area in tmp_root
self.name = name
if name is None:
self.name = STAGE_PREFIX + next(tempfile._get_candidate_names())
@@ -143,7 +151,6 @@ class Stage(object):
# Flag to decide whether to delete the stage folder on exit or not
self.keep = keep
-
def __enter__(self):
"""
Entering a stage context will create the stage directory
@@ -154,7 +161,6 @@ class Stage(object):
self.create()
return self
-
def __exit__(self, exc_type, exc_val, exc_tb):
"""
Exiting from a stage context will delete the stage directory unless:
@@ -173,12 +179,10 @@ class Stage(object):
if exc_type is None and not self.keep:
self.destroy()
-
def _need_to_create_path(self):
"""Makes sure nothing weird has happened since the last time we
looked at path. Returns True if path already exists and is ok.
- Returns False if path needs to be created.
- """
+ Returns False if path needs to be created."""
# Path doesn't exist yet. Will need to create it.
if not os.path.exists(self.path):
return True
@@ -196,7 +200,8 @@ class Stage(object):
if spack.use_tmp_stage:
# If we're using a tmp dir, it's a link, and it points at the
# right spot, then keep it.
- if (real_path.startswith(real_tmp) and os.path.exists(real_path)):
+ if (real_path.startswith(real_tmp) and
+ os.path.exists(real_path)):
return False
else:
# otherwise, just unlink it and start over.
@@ -204,7 +209,8 @@ class Stage(object):
return True
else:
- # If we're not tmp mode, then it's a link and we want a directory.
+ # If we're not tmp mode, then it's a link and we want a
+ # directory.
os.unlink(self.path)
return True
@@ -215,10 +221,12 @@ class Stage(object):
"""Possible archive file paths."""
paths = []
if isinstance(self.fetcher, fs.URLFetchStrategy):
- paths.append(os.path.join(self.path, os.path.basename(self.fetcher.url)))
+ paths.append(os.path.join(
+ self.path, os.path.basename(self.fetcher.url)))
if self.mirror_path:
- paths.append(os.path.join(self.path, os.path.basename(self.mirror_path)))
+ paths.append(os.path.join(
+ self.path, os.path.basename(self.mirror_path)))
return paths
@@ -227,10 +235,12 @@ class Stage(object):
"""Path to the source archive within this stage directory."""
paths = []
if isinstance(self.fetcher, fs.URLFetchStrategy):
- paths.append(os.path.join(self.path, os.path.basename(self.fetcher.url)))
+ paths.append(os.path.join(
+ self.path, os.path.basename(self.fetcher.url)))
if self.mirror_path:
- paths.append(os.path.join(self.path, os.path.basename(self.mirror_path)))
+ paths.append(os.path.join(
+ self.path, os.path.basename(self.mirror_path)))
for path in paths:
if os.path.exists(path):
@@ -262,7 +272,8 @@ class Stage(object):
return None
def chdir(self):
- """Changes directory to the stage path. Or dies if it is not set up."""
+ """Changes directory to the stage path. Or dies if it is not set
+ up."""
if os.path.isdir(self.path):
os.chdir(self.path)
else:
@@ -306,6 +317,20 @@ class Stage(object):
fetchers.insert(0, fs.URLFetchStrategy(url, digest))
fetchers.insert(0, spack.cache.fetcher(self.mirror_path, digest))
+ # Look for the archive in list_url
+ package_name = os.path.dirname(self.mirror_path)
+ pkg = spack.repo.get(package_name)
+ if pkg.list_url is not None and pkg.url is not None:
+ archive_version = spack.url.parse_version(
+ self.default_fetcher.url)
+ versions = pkg.fetch_remote_versions()
+ try:
+ url_from_list = versions[Version(archive_version)]
+ fetchers.append(fs.URLFetchStrategy(url_from_list, digest))
+ except KeyError:
+ tty.msg("Can not find version %s in url_list" %
+ archive_version)
+
for fetcher in fetchers:
try:
fetcher.set_stage(self)
@@ -321,11 +346,11 @@ class Stage(object):
self.fetcher = self.default_fetcher
raise fs.FetchError(errMessage, None)
-
def check(self):
"""Check the downloaded archive against a checksum digest.
No-op if this stage checks code out of a repository."""
- if self.fetcher is not self.default_fetcher and self.skip_checksum_for_mirror:
+ if self.fetcher is not self.default_fetcher and \
+ self.skip_checksum_for_mirror:
tty.warn("Fetching from mirror without a checksum!",
"This package is normally checked out from a version "
"control system, but it has been archived on a spack "
@@ -335,16 +360,13 @@ class Stage(object):
else:
self.fetcher.check()
-
def cache_local(self):
spack.cache.store(self.fetcher, self.mirror_path)
-
def expand_archive(self):
"""Changes to the stage directory and attempt to expand the downloaded
- archive. Fail if the stage is not set up or if the archive is not yet
- downloaded.
- """
+ archive. Fail if the stage is not set up or if the archive is not yet
+ downloaded."""
archive_dir = self.source_path
if not archive_dir:
self.fetcher.expand()
@@ -386,8 +408,8 @@ class Stage(object):
# Create the top-level stage directory
mkdirp(spack.stage_path)
remove_dead_links(spack.stage_path)
- # If a tmp_root exists then create a directory there and then link it in the stage area,
- # otherwise create the stage directory in self.path
+ # If a tmp_root exists then create a directory there and then link it
+ # in the stage area, otherwise create the stage directory in self.path
if self._need_to_create_path():
if self.tmp_root:
tmp_dir = tempfile.mkdtemp('', STAGE_PREFIX, self.tmp_root)
@@ -409,6 +431,7 @@ class Stage(object):
class ResourceStage(Stage):
+
def __init__(self, url_or_fetch_strategy, root, resource, **kwargs):
super(ResourceStage, self).__init__(url_or_fetch_strategy, **kwargs)
self.root_stage = root
@@ -418,12 +441,15 @@ class ResourceStage(Stage):
super(ResourceStage, self).expand_archive()
root_stage = self.root_stage
resource = self.resource
- placement = os.path.basename(self.source_path) if resource.placement is None else resource.placement
+ placement = os.path.basename(self.source_path) \
+ if resource.placement is None \
+ else resource.placement
if not isinstance(placement, dict):
placement = {'': placement}
# Make the paths in the dictionary absolute and link
for key, value in placement.iteritems():
- target_path = join_path(root_stage.source_path, resource.destination)
+ target_path = join_path(
+ root_stage.source_path, resource.destination)
destination_path = join_path(target_path, value)
source_path = join_path(self.source_path, key)
@@ -437,21 +463,23 @@ class ResourceStage(Stage):
if not os.path.exists(destination_path):
# Create a symlink
- tty.info('Moving resource stage\n\tsource : {stage}\n\tdestination : {destination}'.format(
- stage=source_path, destination=destination_path
- ))
+ tty.info('Moving resource stage\n\tsource : '
+ '{stage}\n\tdestination : {destination}'.format(
+ stage=source_path, destination=destination_path
+ ))
shutil.move(source_path, destination_path)
-@pattern.composite(method_list=['fetch', 'create', 'check', 'expand_archive', 'restage', 'destroy', 'cache_local'])
+@pattern.composite(method_list=['fetch', 'create', 'check', 'expand_archive',
+ 'restage', 'destroy', 'cache_local'])
class StageComposite:
- """
- Composite for Stage type objects. The first item in this composite is considered to be the root package, and
- operations that return a value are forwarded to it.
- """
+ """Composite for Stage type objects. The first item in this composite is
+ considered to be the root package, and operations that return a value are
+ forwarded to it."""
#
# __enter__ and __exit__ delegate to all stages in the composite.
#
+
def __enter__(self):
for item in self:
item.__enter__()
@@ -496,8 +524,11 @@ class DIYStage(object):
raise ChdirError("Setup failed: no such directory: " + self.path)
# DIY stages do nothing as context managers.
- def __enter__(self): pass
- def __exit__(self, exc_type, exc_val, exc_tb): pass
+ def __enter__(self):
+ pass
+
+ def __exit__(self, exc_type, exc_val, exc_tb):
+ pass
def chdir_to_source(self):
self.chdir()
diff --git a/lib/spack/spack/test/__init__.py b/lib/spack/spack/test/__init__.py
index a849d5f350..3439764ee6 100644
--- a/lib/spack/spack/test/__init__.py
+++ b/lib/spack/spack/test/__init__.py
@@ -39,9 +39,9 @@ test_names = [
'pattern', 'python_version', 'git_fetch', 'svn_fetch', 'hg_fetch',
'mirror', 'modules', 'url_extrapolate', 'cc', 'link_tree', 'spec_yaml',
'optional_deps', 'make_executable', 'build_system_guess', 'lock',
- 'database', 'namespace_trie', 'yaml', 'sbang', 'environment', 'cmd.find',
- 'cmd.uninstall', 'cmd.test_install', 'cmd.test_compiler_cmd',
- 'cmd.module'
+ 'database', 'namespace_trie', 'yaml', 'sbang', 'environment',
+ 'concretize_preferences', 'cmd.find', 'cmd.uninstall', 'cmd.test_install',
+ 'cmd.test_compiler_cmd', 'cmd.module'
]
diff --git a/lib/spack/spack/test/architecture.py b/lib/spack/spack/test/architecture.py
index 09bdb021af..42dd9f4c04 100644
--- a/lib/spack/spack/test/architecture.py
+++ b/lib/spack/spack/test/architecture.py
@@ -31,7 +31,7 @@ import platform as py_platform
import spack
import spack.architecture
from spack.spec import *
-from spack.platforms.cray_xc import CrayXc
+from spack.platforms.cray import Cray
from spack.platforms.linux import Linux
from spack.platforms.bgq import Bgq
from spack.platforms.darwin import Darwin
@@ -76,7 +76,7 @@ class ArchitectureTest(MockPackagesTest):
def test_platform(self):
output_platform_class = spack.architecture.platform()
if os.path.exists('/opt/cray/craype'):
- my_platform_class = CrayXc()
+ my_platform_class = Cray()
elif os.path.exists('/bgsys'):
my_platform_class = Bgq()
elif 'Linux' in py_platform.system():
diff --git a/lib/spack/spack/test/concretize_preferences.py b/lib/spack/spack/test/concretize_preferences.py
new file mode 100644
index 0000000000..2c8bedc33f
--- /dev/null
+++ b/lib/spack/spack/test/concretize_preferences.py
@@ -0,0 +1,106 @@
+##############################################################################
+# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC.
+# Produced at the Lawrence Livermore National Laboratory.
+#
+# This file is part of Spack.
+# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
+# LLNL-CODE-647188
+#
+# For details, see https://github.com/llnl/spack
+# Please also see the LICENSE file for our notice and the LGPL.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License (as
+# published by the Free Software Foundation) version 2.1, February 1999.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
+# conditions of the GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+##############################################################################
+import spack
+import spack.architecture
+from spack.test.mock_packages_test import *
+from tempfile import mkdtemp
+
+
+class ConcretizePreferencesTest(MockPackagesTest):
+ """Test concretization preferences are being applied correctly.
+ """
+
+ def setUp(self):
+ """Create config section to store concretization preferences
+ """
+ super(ConcretizePreferencesTest, self).setUp()
+ self.tmp_dir = mkdtemp('.tmp', 'spack-config-test-')
+ spack.config.ConfigScope('concretize',
+ os.path.join(self.tmp_dir, 'concretize'))
+
+ def tearDown(self):
+ super(ConcretizePreferencesTest, self).tearDown()
+ shutil.rmtree(self.tmp_dir, True)
+ spack.pkgsort = spack.PreferredPackages()
+
+ def concretize(self, abstract_spec):
+ return Spec(abstract_spec).concretized()
+
+ def update_packages(self, pkgname, section, value):
+ """Update config and reread package list"""
+ conf = {pkgname: {section: value}}
+ spack.config.update_config('packages', conf, 'concretize')
+ spack.pkgsort = spack.PreferredPackages()
+
+ def assert_variant_values(self, spec, **variants):
+ concrete = self.concretize(spec)
+ for variant, value in variants.items():
+ self.assertEqual(concrete.variants[variant].value, value)
+
+ def test_preferred_variants(self):
+ """Test preferred variants are applied correctly
+ """
+ self.update_packages('mpileaks', 'variants',
+ '~debug~opt+shared+static')
+ self.assert_variant_values('mpileaks', debug=False, opt=False,
+ shared=True, static=True)
+
+ self.update_packages('mpileaks', 'variants',
+ ['+debug', '+opt', '~shared', '-static'])
+ self.assert_variant_values('mpileaks', debug=True, opt=True,
+ shared=False, static=False)
+
+ def test_preferred_compilers(self):
+ """Test preferred compilers are applied correctly
+ """
+ self.update_packages('mpileaks', 'compiler', ['clang@3.3'])
+ spec = self.concretize('mpileaks')
+ self.assertEqual(spec.compiler, spack.spec.CompilerSpec('clang@3.3'))
+
+ self.update_packages('mpileaks', 'compiler', ['gcc@4.5.0'])
+ spec = self.concretize('mpileaks')
+ self.assertEqual(spec.compiler, spack.spec.CompilerSpec('gcc@4.5.0'))
+
+ def test_preferred_versions(self):
+ """Test preferred package versions are applied correctly
+ """
+ self.update_packages('mpileaks', 'version', ['2.3'])
+ spec = self.concretize('mpileaks')
+ self.assertEqual(spec.version, spack.spec.Version('2.3'))
+
+ self.update_packages('mpileaks', 'version', ['2.2'])
+ spec = self.concretize('mpileaks')
+ self.assertEqual(spec.version, spack.spec.Version('2.2'))
+
+ def test_preferred_providers(self):
+ """Test preferred providers of virtual packages are applied correctly
+ """
+ self.update_packages('all', 'providers', {'mpi': ['mpich']})
+ spec = self.concretize('mpileaks')
+ self.assertTrue('mpich' in spec)
+
+ self.update_packages('all', 'providers', {'mpi': ['zmpi']})
+ spec = self.concretize('mpileaks')
+ self.assertTrue('zmpi', spec)
diff --git a/lib/spack/spack/test/modules.py b/lib/spack/spack/test/modules.py
index 135cd028e3..5e280d8e43 100644
--- a/lib/spack/spack/test/modules.py
+++ b/lib/spack/spack/test/modules.py
@@ -89,7 +89,10 @@ configuration_alter_environment = {
'enable': ['tcl'],
'tcl': {
'all': {
- 'filter': {'environment_blacklist': ['CMAKE_PREFIX_PATH']}
+ 'filter': {'environment_blacklist': ['CMAKE_PREFIX_PATH']},
+ 'environment': {
+ 'set': {'{name}_ROOT': '{prefix}'}
+ }
},
'platform=test target=x86_64': {
'environment': {
@@ -248,6 +251,8 @@ class TclTests(MockPackagesTest):
self.assertEqual(
len([x for x in content if 'setenv FOO "foo"' in x]), 1)
self.assertEqual(len([x for x in content if 'unsetenv BAR' in x]), 1)
+ self.assertEqual(
+ len([x for x in content if 'setenv MPILEAKS_ROOT' in x]), 1)
spec = spack.spec.Spec('libdwarf %clang platform=test target=x86_32')
content = self.get_modulefile_content(spec)
@@ -262,6 +267,8 @@ class TclTests(MockPackagesTest):
len([x for x in content if 'is-loaded foo/bar' in x]), 1)
self.assertEqual(
len([x for x in content if 'module load foo/bar' in x]), 1)
+ self.assertEqual(
+ len([x for x in content if 'setenv LIBDWARF_ROOT' in x]), 1)
def test_blacklist(self):
spack.modules.CONFIGURATION = configuration_blacklist
diff --git a/lib/spack/spack/util/web.py b/lib/spack/spack/util/web.py
index 47abc507e0..cac783a368 100644
--- a/lib/spack/spack/util/web.py
+++ b/lib/spack/spack/util/web.py
@@ -25,8 +25,7 @@
import re
import os
import sys
-import subprocess
-import urllib2, cookielib
+import urllib2
import urlparse
from multiprocessing import Pool
from HTMLParser import HTMLParser, HTMLParseError
@@ -84,7 +83,7 @@ def _spider(args):
req.get_method = lambda: "HEAD"
resp = urllib2.urlopen(req, timeout=TIMEOUT)
- if not "Content-type" in resp.headers:
+ if "Content-type" not in resp.headers:
tty.debug("ignoring page " + url)
return pages, links
@@ -125,11 +124,11 @@ def _spider(args):
if abs_link in visited:
continue
- # If we're not at max depth, follow links.
- if depth < max_depth:
- subcalls.append((abs_link, visited, root, None,
- depth+1, max_depth, raise_on_error))
- visited.add(abs_link)
+ # If we're not at max depth, follow links.
+ if depth < max_depth:
+ subcalls.append((abs_link, visited, root, None,
+ depth + 1, max_depth, raise_on_error))
+ visited.add(abs_link)
if subcalls:
try:
@@ -142,22 +141,22 @@ def _spider(args):
pool.terminate()
pool.join()
- except urllib2.URLError, e:
+ except urllib2.URLError as e:
tty.debug(e)
if raise_on_error:
raise spack.error.NoNetworkConnectionError(str(e), url)
- except HTMLParseError, e:
+ except HTMLParseError as e:
# This error indicates that Python's HTML parser sucks.
msg = "Got an error parsing HTML."
# Pre-2.7.3 Pythons in particular have rather prickly HTML parsing.
- if sys.version_info[:3] < (2,7,3):
+ if sys.version_info[:3] < (2, 7, 3):
msg += " Use Python 2.7.3 or newer for better HTML parsing."
tty.warn(msg, url, "HTMLParseError: " + str(e))
- except Exception, e:
+ except Exception as e:
# Other types of errors are completely ignored, except in debug mode.
tty.debug("Error in _spider: %s" % e)
@@ -173,7 +172,8 @@ def spider(root_url, **kwargs):
performance over a sequential fetch.
"""
max_depth = kwargs.setdefault('depth', 1)
- pages, links = _spider((root_url, set(), root_url, None, 1, max_depth, False))
+ pages, links = _spider((root_url, set(), root_url, None,
+ 1, max_depth, False))
return pages, links
@@ -235,7 +235,7 @@ def find_versions_of_archive(*archive_urls, **kwargs):
try:
ver = spack.url.parse_version(url)
versions[ver] = url
- except spack.url.UndetectableVersionError as e:
+ except spack.url.UndetectableVersionError:
continue
return versions
generated by cgit v1.2.3-60-g2f50 (git 2.42.0) at 2024-07-17 06:06:41 +0000