diff options
Diffstat (limited to 'lib')
27 files changed, 425 insertions, 113 deletions
diff --git a/lib/spack/docs/configuration.rst b/lib/spack/docs/configuration.rst index c613071c65..a6f876b2aa 100644 --- a/lib/spack/docs/configuration.rst +++ b/lib/spack/docs/configuration.rst @@ -140,7 +140,6 @@ Here's an example packages.yaml file that sets preferred packages: packages: dyninst: compiler: [gcc@4.9] - variants: +debug gperftools: version: [2.2, 2.4, 2.3] all: @@ -150,8 +149,8 @@ Here's an example packages.yaml file that sets preferred packages: At a high level, this example is specifying how packages should be -concretized. The dyninst package should prefer using gcc 4.9 and -be built with debug options. The gperftools package should prefer version +concretized. The dyninst package should prefer using gcc 4.9. +The gperftools package should prefer version 2.2 over 2.4. Every package on the system should prefer mvapich for its MPI and gcc 4.4.7 (except for Dyninst, which overrides this by preferring gcc 4.9). These options are used to fill in implicit defaults. Any of them can be overwritten @@ -160,7 +159,7 @@ on the command line if explicitly requested. Each packages.yaml file begins with the string ``packages:`` and package names are specified on the next level. The special string ``all`` applies settings to each package. Underneath each package name is -one or more components: ``compiler``, ``variants``, ``version``, +one or more components: ``compiler``, ``version``, or ``providers``. Each component has an ordered list of spec ``constraints``, with earlier entries in the list being preferred over later entries. diff --git a/lib/spack/llnl/util/filesystem.py b/lib/spack/llnl/util/filesystem.py index d72e8bae92..e800c6717a 100644 --- a/lib/spack/llnl/util/filesystem.py +++ b/lib/spack/llnl/util/filesystem.py @@ -48,7 +48,7 @@ __all__ = ['set_install_permissions', 'install', 'install_tree', def filter_file(regex, repl, *filenames, **kwargs): """Like sed, but uses python regular expressions. - Filters every line of file through regex and replaces the file + Filters every line of each file through regex and replaces the file with a filtered version. Preserves mode of filtered files. As with re.sub, ``repl`` can be either a string or a callable. @@ -59,7 +59,7 @@ def filter_file(regex, repl, *filenames, **kwargs): Keyword Options: string[=False] If True, treat regex as a plain string. - backup[=True] Make a backup files suffixed with ~ + backup[=True] Make backup file(s) suffixed with ~ ignore_absent[=False] Ignore any files that don't exist. """ string = kwargs.get('string', False) @@ -80,26 +80,26 @@ def filter_file(regex, repl, *filenames, **kwargs): regex = re.escape(regex) for filename in filenames: - backup = filename + "~" + backup_filename = filename + "~" if ignore_absent and not os.path.exists(filename): continue - shutil.copy(filename, backup) + shutil.copy(filename, backup_filename) try: - with closing(open(backup)) as infile: + with closing(open(backup_filename)) as infile: with closing(open(filename, 'w')) as outfile: for line in infile: foo = re.sub(regex, repl, line) outfile.write(foo) except: # clean up the original file on failure. - shutil.move(backup, filename) + shutil.move(backup_filename, filename) raise finally: if not backup: - shutil.rmtree(backup, ignore_errors=True) + os.remove(backup_filename) class FileFilter(object): @@ -114,7 +114,7 @@ class FileFilter(object): def change_sed_delimiter(old_delim, new_delim, *filenames): """Find all sed search/replace commands and change the delimiter. e.g., if the file contains seds that look like 's///', you can - call change_sed_delimeter('/', '@', file) to change the + call change_sed_delimiter('/', '@', file) to change the delimiter to '@'. NOTE that this routine will fail if the delimiter is ' or ". @@ -179,7 +179,7 @@ def install(src, dest): """Manually install a file to a particular location.""" tty.debug("Installing %s to %s" % (src, dest)) - # Expand dsst to its eventual full path if it is a directory. + # Expand dest to its eventual full path if it is a directory. if os.path.isdir(dest): dest = join_path(dest, os.path.basename(src)) @@ -219,7 +219,7 @@ def mkdirp(*paths): if not os.path.exists(path): os.makedirs(path) elif not os.path.isdir(path): - raise OSError(errno.EEXIST, "File alredy exists", path) + raise OSError(errno.EEXIST, "File already exists", path) def force_remove(*paths): @@ -309,7 +309,7 @@ def traverse_tree(source_root, dest_root, rel_path='', **kwargs): Optional args: - order=[pre|post] -- Whether to do pre- or post-order traveral. + order=[pre|post] -- Whether to do pre- or post-order traversal. ignore=<predicate> -- Predicate indicating which files to ignore. @@ -414,7 +414,7 @@ def fix_darwin_install_name(path): currently won't follow subfolders. Args: - path: directory in which .dylib files are alocated + path: directory in which .dylib files are located """ libs = glob.glob(join_path(path, "*.dylib")) @@ -438,7 +438,7 @@ def to_link_flags(library): A string of linking flags. """ dir = os.path.dirname(library) - # Asume libXYZ.suffix + # Assume libXYZ.suffix name = os.path.basename(library)[3:].split(".")[0] res = '-L%s -l%s' % (dir, name) return res diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index 75ddca1abc..965e3a7f78 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -48,6 +48,10 @@ var_path = join_path(spack_root, "var", "spack") stage_path = join_path(var_path, "stage") repos_path = join_path(var_path, "repos") share_path = join_path(spack_root, "share", "spack") +cache_path = join_path(var_path, "cache") + +import spack.fetch_strategy +cache = spack.fetch_strategy.FsCache(cache_path) prefix = spack_root opt_path = join_path(prefix, "opt") diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index cbac7b41d6..a7cda2bf68 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -91,16 +91,10 @@ from spack.util.multiproc import parmap import spack.error as serr -class InvalidSysTypeError(serr.SpackError): - def __init__(self, sys_type): - super(InvalidSysTypeError, self).__init__( - "Invalid sys_type value for Spack: " + sys_type) - - -class NoSysTypeError(serr.SpackError): +class NoPlatformError(serr.SpackError): def __init__(self): - super(NoSysTypeError, self).__init__( - "Could not determine sys_type for this machine.") + super(NoPlatformError, self).__init__( + "Could not determine a platform for this machine.") @key_ordering @@ -345,14 +339,17 @@ class OperatingSystem(object): @key_ordering class Arch(object): - "Architecture is now a class to help with setting attributes" - - def __init__(self, platform=None, platform_os=None, target=None): - self.platform = platform - if platform and platform_os: - platform_os = self.platform.operating_system(platform_os) - self.platform_os = platform_os - if platform and target: + """Architecture is now a class to help with setting attributes. + + TODO: refactor so that we don't need this class. + """ + + def __init__(self, plat=None, os=None, target=None): + self.platform = plat + if plat and os: + os = self.platform.operating_system(os) + self.platform_os = os + if plat and target: target = self.platform.target(target) self.target = target @@ -409,27 +406,27 @@ class Arch(object): return d -def _target_from_dict(target_name, platform=None): +def _target_from_dict(target_name, plat=None): """ Creates new instance of target and assigns all the attributes of that target from the dictionary """ - if not platform: - platform = sys_type() - return platform.target(target_name) + if not plat: + plat = platform() + return plat.target(target_name) -def _operating_system_from_dict(os_name, platform=None): +def _operating_system_from_dict(os_name, plat=None): """ uses platform's operating system method to grab the constructed operating systems that are valid on the platform. """ - if not platform: - platform = sys_type() + if not plat: + plat = platform() if isinstance(os_name, dict): name = os_name['name'] version = os_name['version'] - return platform.operating_system(name+version) + return plat.operating_system(name+version) else: - return platform.operating_system(os_name) + return plat.operating_system(os_name) def _platform_from_dict(platform_name): @@ -504,16 +501,35 @@ def all_platforms(): @memoized -def sys_type(): - """ Gather a list of all available subclasses of platforms. - Sorts the list according to their priority looking. Priority is - an arbitrarily set number. Detects platform either using uname or - a file path (/opt/cray...) +def platform(): + """Detects the platform for this machine. + + Gather a list of all available subclasses of platforms. + Sorts the list according to their priority looking. Priority is + an arbitrarily set number. Detects platform either using uname or + a file path (/opt/cray...) """ # Try to create a Platform object using the config file FIRST platform_list = all_platforms() platform_list.sort(key=lambda a: a.priority) - for platform in platform_list: - if platform.detect(): - return platform() + for platform_cls in platform_list: + if platform_cls.detect(): + return platform_cls() + + +@memoized +def sys_type(): + """Print out the "default" platform-os-target tuple for this machine. + + On machines with only one target OS/target, prints out the + platform-os-target for the frontend. For machines with a frontend + and a backend, prints the default backend. + + TODO: replace with use of more explicit methods to get *all* the + backends, as client code should really be aware of cross-compiled + architectures. + + """ + arch = Arch(platform(), 'default_os', 'default_target') + return str(arch) diff --git a/lib/spack/spack/cmd/purge.py b/lib/spack/spack/cmd/purge.py index 7b33ef7f69..f4e27a3969 100644 --- a/lib/spack/spack/cmd/purge.py +++ b/lib/spack/spack/cmd/purge.py @@ -22,9 +22,31 @@ # License along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## +import spack import spack.stage as stage -description = "Remove all temporary build files and downloaded archives" +description = "Remove temporary build files and/or downloaded archives" + + +def setup_parser(subparser): + subparser.add_argument( + '-s', '--stage', action='store_true', default=True, + help="Remove all temporary build stages (default).") + subparser.add_argument( + '-c', '--cache', action='store_true', help="Remove cached downloads.") + subparser.add_argument( + '-a', '--all', action='store_true', + help="Remove all of the above.") + def purge(parser, args): - stage.purge() + # Special case: no flags. + if not any((args.stage, args.cache, args.all)): + stage.purge() + return + + # handle other flags with fall through. + if args.stage or args.all: + stage.purge() + if args.cache or args.all: + spack.cache.destroy() diff --git a/lib/spack/spack/cmd/repo.py b/lib/spack/spack/cmd/repo.py index 399237b169..cbd8f4784e 100644 --- a/lib/spack/spack/cmd/repo.py +++ b/lib/spack/spack/cmd/repo.py @@ -26,7 +26,7 @@ import os import re import shutil -from external import argparse +import argparse import llnl.util.tty as tty from llnl.util.filesystem import join_path, mkdirp diff --git a/lib/spack/spack/cmd/test.py b/lib/spack/spack/cmd/test.py index cb9dd26c71..36810321ef 100644 --- a/lib/spack/spack/cmd/test.py +++ b/lib/spack/spack/cmd/test.py @@ -31,6 +31,7 @@ from llnl.util.lang import list_modules import spack import spack.test +from spack.fetch_strategy import FetchError description ="Run unit tests" @@ -50,6 +51,24 @@ def setup_parser(subparser): help="verbose output") +class MockCache(object): + def store(self, copyCmd, relativeDst): + pass + + def fetcher(self, targetPath, digest): + return MockCacheFetcher() + + +class MockCacheFetcher(object): + def set_stage(self, stage): + pass + + def fetch(self): + raise FetchError("Mock cache always fails for tests") + + def __str__(self): + return "[mock fetcher]" + def test(parser, args): if args.list: print "Available tests:" @@ -66,4 +85,5 @@ def test(parser, args): if not os.path.exists(outputDir): mkdirp(outputDir) + spack.cache = MockCache() spack.test.run(args.names, outputDir, args.verbose) diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index a70d42982f..0ba94741da 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -93,7 +93,7 @@ def get_compiler_config(scope=None, init_config=True): for compiler in compilers: compilers_dict.append(_to_dict(compiler)) spack.config.update_config('compilers', compilers_dict, scope=scope) - + config = spack.config.get_config('compilers', scope=scope) # Update the configuration if there are currently no compilers # configured. Avoid updating automatically if there ARE site @@ -142,8 +142,8 @@ def remove_compiler_from_config(compiler_spec, scope=None): """ compiler_config = get_compiler_config(scope) config_length = len(compiler_config) - - filtered_compiler_config = [comp for comp in compiler_config + + filtered_compiler_config = [comp for comp in compiler_config if spack.spec.CompilerSpec(comp['compiler']['spec']) != compiler_spec] # Need a better way for this global _cache_config_file @@ -315,7 +315,7 @@ def all_os_classes(): """ classes = [] - platform = spack.architecture.sys_type() + platform = spack.architecture.platform() for os_class in platform.operating_sys.values(): classes.append(os_class) diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 1f37455c77..f792008c49 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -221,7 +221,7 @@ class DefaultConcretizer(object): if isinstance(spec.root.architecture.platform,spack.architecture.Platform): spec.architecture.platform = spec.root.architecture.platform else: - spec.architecture.platform = spack.architecture.sys_type() + spec.architecture.platform = spack.architecture.platform() return True #changed? def concretize_architecture(self, spec): @@ -334,7 +334,7 @@ class DefaultConcretizer(object): Default specs set at the compiler level will still be added later. """ - + if not spec.architecture.platform_os: #Although this usually means changed, this means awaiting other changes return True @@ -347,7 +347,7 @@ class DefaultConcretizer(object): and flag in p.compiler_flags)) if not flag in spec.compiler_flags or \ not (sorted(spec.compiler_flags[flag]) >= sorted(nearest.compiler_flags[flag])): - if flag in spec.compiler_flags: + if flag in spec.compiler_flags: spec.compiler_flags[flag] = list(set(spec.compiler_flags[flag]) | set(nearest.compiler_flags[flag])) else: diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index db0787edc6..84179e1469 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -307,6 +307,8 @@ section_schemas = { 'autoload': {'$ref': '#/definitions/dependency_selection'}, 'prerequisites': {'$ref': '#/definitions/dependency_selection'}, 'conflict': {'$ref': '#/definitions/array_of_strings'}, + 'load': {'$ref': '#/definitions/array_of_strings'}, + 'suffixes': {'$ref': '#/definitions/dictionary_of_strings'}, 'environment': { 'type': 'object', 'default': {}, diff --git a/lib/spack/spack/database.py b/lib/spack/spack/database.py index f941346bb1..a4bbff3d5a 100644 --- a/lib/spack/spack/database.py +++ b/lib/spack/spack/database.py @@ -631,11 +631,13 @@ class WriteTransaction(_Transaction): class CorruptDatabaseError(SpackError): def __init__(self, path, msg=''): super(CorruptDatabaseError, self).__init__( - "Spack database is corrupt: %s. %s" % (path, msg)) + "Spack database is corrupt: %s. %s." + \ + "Try running `spack reindex` to fix." % (path, msg)) class InvalidDatabaseVersionError(SpackError): def __init__(self, expected, found): super(InvalidDatabaseVersionError, self).__init__( - "Expected database version %s but found version %s" % + "Expected database version %s but found version %s." + \ + "Try running `spack reindex` to fix." % (expected, found)) diff --git a/lib/spack/spack/directory_layout.py b/lib/spack/spack/directory_layout.py index 7e20365b0f..ee13e2dcbc 100644 --- a/lib/spack/spack/directory_layout.py +++ b/lib/spack/spack/directory_layout.py @@ -165,7 +165,7 @@ class DirectoryLayout(object): class YamlDirectoryLayout(DirectoryLayout): """Lays out installation directories like this:: <install root>/ - <target>/ + <platform-os-target>/ <compiler>-<compiler version>/ <name>-<version>-<variants>-<hash> diff --git a/lib/spack/spack/fetch_strategy.py b/lib/spack/spack/fetch_strategy.py index 1953d7c1b3..69c04f7920 100644 --- a/lib/spack/spack/fetch_strategy.py +++ b/lib/spack/spack/fetch_strategy.py @@ -310,7 +310,7 @@ class URLFetchStrategy(FetchStrategy): if not extension(destination) == extension(self.archive_file): raise ValueError("Cannot archive without matching extensions.") - shutil.move(self.archive_file, destination) + shutil.copy(self.archive_file, destination) @_needs_stage def check(self): @@ -348,7 +348,7 @@ class URLFetchStrategy(FetchStrategy): def __repr__(self): url = self.url if self.url else "no url" - return "URLFetchStrategy<%s>" % url + return "%s<%s>" % (self.__class__.__name__, url) def __str__(self): if self.url: @@ -357,6 +357,24 @@ class URLFetchStrategy(FetchStrategy): return "[no url]" +class CacheURLFetchStrategy(URLFetchStrategy): + """The resource associated with a cache URL may be out of date.""" + def __init__(self, *args, **kwargs): + super(CacheURLFetchStrategy, self).__init__(*args, **kwargs) + + @_needs_stage + def fetch(self): + super(CacheURLFetchStrategy, self).fetch() + if self.digest: + try: + self.check() + except ChecksumError: + # Future fetchers will assume they don't need to download if the + # file remains + os.remove(self.archive_file) + raise + + class VCSFetchStrategy(FetchStrategy): def __init__(self, name, *rev_types, **kwargs): @@ -815,6 +833,35 @@ def for_package_version(pkg, version): raise InvalidArgsError(pkg, version) +class FsCache(object): + def __init__(self, root): + self.root = os.path.abspath(root) + + def store(self, fetcher, relativeDst): + unique = False + uidGroups = [['tag', 'commit'], ['digest'], ['revision']] + for grp in uidGroups: + try: + unique |= any(getattr(fetcher, x) for x in grp) + except AttributeError: + pass + if unique: + break + if not unique: + return + + dst = join_path(self.root, relativeDst) + mkdirp(os.path.dirname(dst)) + fetcher.archive(dst) + + def fetcher(self, targetPath, digest): + url = "file://" + join_path(self.root, targetPath) + return CacheURLFetchStrategy(url, digest) + + def destroy(self): + shutil.rmtree(self.root, ignore_errors=True) + + class FetchError(spack.error.SpackError): def __init__(self, msg, long_msg=None): diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py index d2b819e80a..ce46047fa3 100644 --- a/lib/spack/spack/modules.py +++ b/lib/spack/spack/modules.py @@ -285,11 +285,18 @@ class EnvModule(object): naming_tokens = self.tokens naming_scheme = self.naming_scheme name = naming_scheme.format(**naming_tokens) - name += '-' + self.spec.dag_hash( - ) # Always append the hash to make the module file unique # Not everybody is working on linux... parts = name.split('/') name = join_path(*parts) + # Add optional suffixes based on constraints + configuration, _ = parse_config_options(self) + suffixes = [name] + for constraint, suffix in configuration.get('suffixes', {}).items(): + if constraint in self.spec: + suffixes.append(suffix) + # Always append the hash to make the module file unique + suffixes.append(self.spec.dag_hash()) + name = '-'.join(suffixes) return name @property @@ -381,6 +388,8 @@ class EnvModule(object): for x in filter_blacklisted( module_configuration.pop('autoload', []), self.name): module_file_content += self.autoload(x) + for x in module_configuration.pop('load', []): + module_file_content += self.autoload(x) for x in filter_blacklisted( module_configuration.pop('prerequisites', []), self.name): module_file_content += self.prerequisite(x) @@ -402,8 +411,12 @@ class EnvModule(object): return tuple() def autoload(self, spec): - m = type(self)(spec) - return self.autoload_format.format(module_file=m.use_name) + if not isinstance(spec, str): + m = type(self)(spec) + module_file = m.use_name + else: + module_file = spec + return self.autoload_format.format(module_file=module_file) def prerequisite(self, spec): m = type(self)(spec) @@ -441,7 +454,6 @@ class EnvModule(object): class Dotkit(EnvModule): name = 'dotkit' - path = join_path(spack.share_path, "dotkit") environment_modifications_formats = { PrependPath: 'dk_alter {name} {value}\n', @@ -454,7 +466,7 @@ class Dotkit(EnvModule): @property def file_name(self): - return join_path(Dotkit.path, self.spec.architecture, + return join_path(spack.share_path, "dotkit", self.spec.architecture, '%s.dk' % self.use_name) @property @@ -482,7 +494,6 @@ class Dotkit(EnvModule): class TclModule(EnvModule): name = 'tcl' - path = join_path(spack.share_path, "modules") environment_modifications_formats = { PrependPath: 'prepend-path --delim "{delim}" {name} \"{value}\"\n', @@ -503,7 +514,7 @@ class TclModule(EnvModule): @property def file_name(self): - return join_path(TclModule.path, self.spec.architecture, self.use_name) + return join_path(spack.share_path, "modules", self.spec.architecture, self.use_name) @property def header(self): diff --git a/lib/spack/spack/multimethod.py b/lib/spack/spack/multimethod.py index 170ef3cea2..0818f9092f 100644 --- a/lib/spack/spack/multimethod.py +++ b/lib/spack/spack/multimethod.py @@ -149,7 +149,7 @@ class when(object): @when('arch=chaos_5_x86_64_ib') def install(self, prefix): # This will be executed instead of the default install if - # the package's sys_type() is chaos_5_x86_64_ib. + # the package's platform() is chaos_5_x86_64_ib. @when('arch=bgqos_0") def install(self, prefix): diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 6a61b3d52b..b73056fd30 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -748,6 +748,9 @@ class Package(object): if spack.do_checksum and self.version in self.versions: self.stage.check() + self.stage.cache_local() + + def do_stage(self, mirror_only=False): """Unpacks the fetched tarball, then changes into the expanded tarball directory.""" diff --git a/lib/spack/spack/repository.py b/lib/spack/spack/repository.py index 70134964ad..2c160a5f45 100644 --- a/lib/spack/spack/repository.py +++ b/lib/spack/spack/repository.py @@ -30,7 +30,7 @@ import imp import re import traceback from bisect import bisect_left -from external import yaml +import yaml import llnl.util.tty as tty from llnl.util.filesystem import * diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 54219ec1b4..77bc57147d 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -2216,7 +2216,7 @@ class SpecParser(spack.parse.Parser): for spec in specs: for s in spec.traverse(): if s.architecture.os_string or s.architecture.target_string: - s._set_platform(spack.architecture.sys_type()) + s._set_platform(spack.architecture.platform()) return specs def parse_compiler(self, text): diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index a76ec168ad..b08cce43b8 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -304,6 +304,7 @@ class Stage(object): # Add URL strategies for all the mirrors with the digest for url in urls: fetchers.insert(0, fs.URLFetchStrategy(url, digest)) + fetchers.insert(0, spack.cache.fetcher(self.mirror_path, digest)) for fetcher in fetchers: try: @@ -320,6 +321,7 @@ class Stage(object): self.fetcher = self.default_fetcher raise fs.FetchError(errMessage, None) + def check(self): """Check the downloaded archive against a checksum digest. No-op if this stage checks code out of a repository.""" @@ -333,6 +335,11 @@ class Stage(object): else: self.fetcher.check() + + def cache_local(self): + spack.cache.store(self.fetcher, self.mirror_path) + + def expand_archive(self): """Changes to the stage directory and attempt to expand the downloaded archive. Fail if the stage is not set up or if the archive is not yet @@ -436,7 +443,7 @@ class ResourceStage(Stage): shutil.move(source_path, destination_path) -@pattern.composite(method_list=['fetch', 'create', 'check', 'expand_archive', 'restage', 'destroy']) +@pattern.composite(method_list=['fetch', 'create', 'check', 'expand_archive', 'restage', 'destroy', 'cache_local']) class StageComposite: """ Composite for Stage type objects. The first item in this composite is considered to be the root package, and @@ -511,6 +518,9 @@ class DIYStage(object): # No need to destroy DIY stage. pass + def cache_local(self): + tty.msg("Sources for DIY stages are not cached") + def _get_mirrors(): """Get mirrors from spack configuration.""" diff --git a/lib/spack/spack/test/architecture.py b/lib/spack/spack/test/architecture.py index a6847c5744..ae3f08deed 100644 --- a/lib/spack/spack/test/architecture.py +++ b/lib/spack/spack/test/architecture.py @@ -5,7 +5,7 @@ import unittest import os import platform as py_platform import spack -from spack.architecture import * +import spack.architecture from spack.spec import * from spack.platforms.cray_xc import CrayXc from spack.platforms.linux import Linux @@ -19,14 +19,14 @@ class ArchitectureTest(MockPackagesTest): def setUp(self): super(ArchitectureTest, self).setUp() - self.platform = sys_type() + self.platform = spack.architecture.platform() def tearDown(self): super(ArchitectureTest, self).tearDown() def test_dict_functions_for_architecture(self): - arch = Arch() - arch.platform = spack.architecture.sys_type() + arch = spack.architecture.Arch() + arch.platform = spack.architecture.platform() arch.platform_os = arch.platform.operating_system('default_os') arch.target = arch.platform.target('default_target') @@ -36,18 +36,23 @@ class ArchitectureTest(MockPackagesTest): self.assertEqual(arch, new_arch) - self.assertTrue( isinstance(arch, Arch) ) - self.assertTrue( isinstance(arch.platform, Platform) ) - self.assertTrue( isinstance(arch.platform_os, OperatingSystem) ) - self.assertTrue( isinstance(arch.target, Target) ) - self.assertTrue( isinstance(new_arch, Arch) ) - self.assertTrue( isinstance(new_arch.platform, Platform) ) - self.assertTrue( isinstance(new_arch.platform_os, OperatingSystem) ) - self.assertTrue( isinstance(new_arch.target, Target) ) - - - def test_sys_type(self): - output_platform_class = sys_type() + self.assertTrue( isinstance(arch, spack.architecture.Arch) ) + self.assertTrue( isinstance(arch.platform, spack.architecture.Platform) ) + self.assertTrue( isinstance(arch.platform_os, + spack.architecture.OperatingSystem) ) + self.assertTrue( isinstance(arch.target, + spack.architecture.Target) ) + self.assertTrue( isinstance(new_arch, spack.architecture.Arch) ) + self.assertTrue( isinstance(new_arch.platform, + spack.architecture.Platform) ) + self.assertTrue( isinstance(new_arch.platform_os, + spack.architecture.OperatingSystem) ) + self.assertTrue( isinstance(new_arch.target, + spack.architecture.Target) ) + + + def test_platform(self): + output_platform_class = spack.architecture.platform() my_arch_class = None if os.path.exists('/opt/cray/craype'): my_platform_class = CrayXc() diff --git a/lib/spack/spack/test/concretize.py b/lib/spack/spack/test/concretize.py index ab201f406a..ce02b08bc3 100644 --- a/lib/spack/spack/test/concretize.py +++ b/lib/spack/spack/test/concretize.py @@ -257,8 +257,8 @@ class ConcretizeTest(MockPackagesTest): def test_external_package_module(self): # No tcl modules on darwin/linux machines # TODO: improved way to check for this. - if (spack.architecture.sys_type().name == 'darwin' or - spack.architecture.sys_type().name == 'linux'): + if (spack.architecture.platform().name == 'darwin' or + spack.architecture.platform().name == 'linux'): return spec = Spec('externalmodule') diff --git a/lib/spack/spack/test/install.py b/lib/spack/spack/test/install.py index cfe6ea9b27..390ec096a9 100644 --- a/lib/spack/spack/test/install.py +++ b/lib/spack/spack/test/install.py @@ -28,6 +28,7 @@ import tempfile import spack from llnl.util.filesystem import * from spack.directory_layout import YamlDirectoryLayout +from spack.database import Database from spack.fetch_strategy import URLFetchStrategy, FetchStrategyComposite from spack.test.mock_packages_test import * from spack.test.mock_repo import MockArchive @@ -49,7 +50,10 @@ class InstallTest(MockPackagesTest): # installed pkgs and mock packages. self.tmpdir = tempfile.mkdtemp() self.orig_layout = spack.install_layout + self.orig_db = spack.installed_db + spack.install_layout = YamlDirectoryLayout(self.tmpdir) + spack.installed_db = Database(self.tmpdir) def tearDown(self): @@ -61,6 +65,7 @@ class InstallTest(MockPackagesTest): # restore spack's layout. spack.install_layout = self.orig_layout + spack.installed_db = self.orig_db shutil.rmtree(self.tmpdir, ignore_errors=True) @@ -71,7 +76,7 @@ class InstallTest(MockPackagesTest): pkg.fetcher = fetcher - def ztest_install_and_uninstall(self): + def test_install_and_uninstall(self): # Get a basic concrete spec for the trivial install package. spec = Spec('trivial_install_test_package') spec.concretize() diff --git a/lib/spack/spack/test/mock_packages_test.py b/lib/spack/spack/test/mock_packages_test.py index a56bd8ebdc..e1acc32cb7 100644 --- a/lib/spack/spack/test/mock_packages_test.py +++ b/lib/spack/spack/test/mock_packages_test.py @@ -34,7 +34,7 @@ from ordereddict_backport import OrderedDict from spack.repository import RepoPath from spack.spec import Spec -platform = spack.architecture.sys_type() +platform = spack.architecture.platform() linux_os_name = 'debian' linux_os_version = '6' @@ -56,7 +56,7 @@ compilers: fc: None modules: 'None' - compiler: - spec: gcc@4.5.0 + spec: gcc@4.5.0 operating_system: {0}{1} paths: cc: /path/to/gcc @@ -144,7 +144,7 @@ compilers: fc: /path/to/gfortran operating_system: elcapitan spec: gcc@4.5.0 - modules: 'None' + modules: 'None' - compiler: spec: clang@3.3 operating_system: elcapitan @@ -201,6 +201,10 @@ class MockPackagesTest(unittest.TestCase): spack.config.ConfigScope('site', self.mock_site_config) spack.config.ConfigScope('user', self.mock_user_config) + # Keep tests from interfering with the actual module path. + self.real_share_path = spack.share_path + spack.share_path = tempfile.mkdtemp() + # Store changes to the package's dependencies so we can # restore later. self.saved_deps = {} @@ -235,6 +239,9 @@ class MockPackagesTest(unittest.TestCase): pkg.dependencies.clear() pkg.dependencies.update(deps) + shutil.rmtree(spack.share_path, ignore_errors=True) + spack.share_path = self.real_share_path + def setUp(self): self.initmock() diff --git a/lib/spack/spack/test/modules.py b/lib/spack/spack/test/modules.py index 582e067860..6d2e3705bd 100644 --- a/lib/spack/spack/test/modules.py +++ b/lib/spack/spack/test/modules.py @@ -27,6 +27,7 @@ from contextlib import contextmanager import StringIO import spack.modules +import unittest from spack.test.mock_packages_test import MockPackagesTest FILE_REGISTRY = collections.defaultdict(StringIO.StringIO) @@ -67,6 +68,24 @@ configuration_autoload_all = { } } +configuration_prerequisites_direct = { + 'enable': ['tcl'], + 'tcl': { + 'all': { + 'prerequisites': 'direct' + } + } +} + +configuration_prerequisites_all = { + 'enable': ['tcl'], + 'tcl': { + 'all': { + 'prerequisites': 'all' + } + } +} + configuration_alter_environment = { 'enable': ['tcl'], 'tcl': { @@ -74,8 +93,13 @@ configuration_alter_environment = { 'filter': {'environment_blacklist': ['CMAKE_PREFIX_PATH']} }, 'platform=test target=x86_64': { - 'environment': {'set': {'FOO': 'foo'}, - 'unset': ['BAR']} + 'environment': { + 'set': {'FOO': 'foo'}, + 'unset': ['BAR'] + } + }, + 'platform=test target=x86_32': { + 'load': ['foo/bar'] } } } @@ -83,7 +107,8 @@ configuration_alter_environment = { configuration_blacklist = { 'enable': ['tcl'], 'tcl': { - 'blacklist': ['callpath'], + 'whitelist': ['zmpi'], + 'blacklist': ['callpath', 'mpi'], 'all': { 'autoload': 'direct' } @@ -100,8 +125,68 @@ configuration_conflicts = { } } +configuration_wrong_conflicts = { + 'enable': ['tcl'], + 'tcl': { + 'naming_scheme': '{name}/{version}-{compiler.name}', + 'all': { + 'conflict': ['{name}/{compiler.name}'] + } + } +} + +configuration_suffix = { + 'enable': ['tcl'], + 'tcl': { + 'mpileaks': { + 'suffixes': { + '+debug': 'foo', + '~debug': 'bar' + } + } + } +} + + +class HelperFunctionsTests(MockPackagesTest): + + def test_update_dictionary_extending_list(self): + target = { + 'foo': { + 'a': 1, + 'b': 2, + 'd': 4 + }, + 'bar': [1, 2, 4], + 'baz': 'foobar' + } + update = { + 'foo': { + 'c': 3, + }, + 'bar': [3], + 'baz': 'foobaz', + 'newkey': { + 'd': 4 + } + } + spack.modules.update_dictionary_extending_lists(target, update) + self.assertTrue(len(target) == 4) + self.assertTrue(len(target['foo']) == 4) + self.assertTrue(len(target['bar']) == 4) + self.assertEqual(target['baz'], 'foobaz') + + def test_inspect_path(self): + env = spack.modules.inspect_path('/usr') + names = [item.name for item in env] + self.assertTrue('PATH' in names) + self.assertTrue('LIBRARY_PATH' in names) + self.assertTrue('LD_LIBRARY_PATH' in names) + self.assertTrue('CPATH' in names) + class TclTests(MockPackagesTest): + def setUp(self): super(TclTests, self).setUp() self.configuration_obj = spack.modules.CONFIGURATION @@ -116,7 +201,6 @@ class TclTests(MockPackagesTest): def get_modulefile_content(self, spec): spec.concretize() - print spec, '&&&&&' generator = spack.modules.TclModule(spec) generator.write() content = FILE_REGISTRY[generator.file_name].split('\n') @@ -127,6 +211,8 @@ class TclTests(MockPackagesTest): spec = spack.spec.Spec('mpich@3.0.4') content = self.get_modulefile_content(spec) self.assertTrue('module-whatis "mpich @3.0.4"' in content) + self.assertRaises(TypeError, spack.modules.dependencies, + spec, 'non-existing-tag') def test_autoload(self): spack.modules.CONFIGURATION = configuration_autoload_direct @@ -141,11 +227,21 @@ class TclTests(MockPackagesTest): self.assertEqual(len([x for x in content if 'is-loaded' in x]), 5) self.assertEqual(len([x for x in content if 'module load ' in x]), 5) + def test_prerequisites(self): + spack.modules.CONFIGURATION = configuration_prerequisites_direct + spec = spack.spec.Spec('mpileaks arch=x86-linux') + content = self.get_modulefile_content(spec) + self.assertEqual(len([x for x in content if 'prereq' in x]), 2) + + spack.modules.CONFIGURATION = configuration_prerequisites_all + spec = spack.spec.Spec('mpileaks arch=x86-linux') + content = self.get_modulefile_content(spec) + self.assertEqual(len([x for x in content if 'prereq' in x]), 5) + def test_alter_environment(self): spack.modules.CONFIGURATION = configuration_alter_environment spec = spack.spec.Spec('mpileaks platform=test target=x86_64') content = self.get_modulefile_content(spec) - print content self.assertEqual( len([x for x in content @@ -156,7 +252,6 @@ class TclTests(MockPackagesTest): spec = spack.spec.Spec('libdwarf %clang platform=test target=x86_32') content = self.get_modulefile_content(spec) - print content self.assertEqual( len([x for x in content @@ -164,6 +259,10 @@ class TclTests(MockPackagesTest): self.assertEqual( len([x for x in content if 'setenv FOO "foo"' in x]), 0) self.assertEqual(len([x for x in content if 'unsetenv BAR' in x]), 0) + self.assertEqual( + len([x for x in content if 'is-loaded foo/bar' in x]), 1) + self.assertEqual( + len([x for x in content if 'module load foo/bar' in x]), 1) def test_blacklist(self): spack.modules.CONFIGURATION = configuration_blacklist @@ -171,6 +270,13 @@ class TclTests(MockPackagesTest): content = self.get_modulefile_content(spec) self.assertEqual(len([x for x in content if 'is-loaded' in x]), 1) self.assertEqual(len([x for x in content if 'module load ' in x]), 1) + spec = spack.spec.Spec('callpath arch=x86-linux') + # Returns a StringIO instead of a string as no module file was written + self.assertRaises(AttributeError, self.get_modulefile_content, spec) + spec = spack.spec.Spec('zmpi arch=x86-linux') + content = self.get_modulefile_content(spec) + self.assertEqual(len([x for x in content if 'is-loaded' in x]), 1) + self.assertEqual(len([x for x in content if 'module load ' in x]), 1) def test_conflicts(self): spack.modules.CONFIGURATION = configuration_conflicts @@ -182,3 +288,57 @@ class TclTests(MockPackagesTest): len([x for x in content if x == 'conflict mpileaks']), 1) self.assertEqual( len([x for x in content if x == 'conflict intel/14.0.1']), 1) + + spack.modules.CONFIGURATION = configuration_wrong_conflicts + self.assertRaises(SystemExit, self.get_modulefile_content, spec) + + def test_suffixes(self): + spack.modules.CONFIGURATION = configuration_suffix + spec = spack.spec.Spec('mpileaks+debug arch=x86-linux') + spec.concretize() + generator = spack.modules.TclModule(spec) + self.assertTrue('foo' in generator.use_name) + + spec = spack.spec.Spec('mpileaks~debug arch=x86-linux') + spec.concretize() + generator = spack.modules.TclModule(spec) + self.assertTrue('bar' in generator.use_name) + + +configuration_dotkit = { + 'enable': ['dotkit'], + 'dotkit': { + 'all': { + 'prerequisites': 'direct' + } + } +} + + +class DotkitTests(MockPackagesTest): + + def setUp(self): + super(DotkitTests, self).setUp() + self.configuration_obj = spack.modules.CONFIGURATION + spack.modules.open = mock_open + # Make sure that a non-mocked configuration will trigger an error + spack.modules.CONFIGURATION = None + + def tearDown(self): + del spack.modules.open + spack.modules.CONFIGURATION = self.configuration_obj + super(DotkitTests, self).tearDown() + + def get_modulefile_content(self, spec): + spec.concretize() + generator = spack.modules.Dotkit(spec) + generator.write() + content = FILE_REGISTRY[generator.file_name].split('\n') + return content + + def test_dotkit(self): + spack.modules.CONFIGURATION = configuration_dotkit + spec = spack.spec.Spec('mpileaks arch=x86-linux') + content = self.get_modulefile_content(spec) + self.assertTrue('#c spack' in content) + self.assertTrue('#d mpileaks @2.3' in content) diff --git a/lib/spack/spack/test/multimethod.py b/lib/spack/spack/test/multimethod.py index 034e6b3923..c233ea4fd6 100644 --- a/lib/spack/spack/test/multimethod.py +++ b/lib/spack/spack/test/multimethod.py @@ -93,7 +93,7 @@ class MultiMethodTest(MockPackagesTest): def test_target_match(self): - platform = spack.architecture.sys_type() + platform = spack.architecture.platform() targets = platform.targets.values() for target in targets[:-1]: pkg = spack.repo.get('multimethod target='+target.name) diff --git a/lib/spack/spack/test/spec_dag.py b/lib/spack/spack/test/spec_dag.py index 712f07ac4d..c56c70b1fe 100644 --- a/lib/spack/spack/test/spec_dag.py +++ b/lib/spack/spack/test/spec_dag.py @@ -242,7 +242,7 @@ class SpecDagTest(MockPackagesTest): def test_unsatisfiable_architecture(self): - platform = spack.architecture.sys_type() + platform = spack.architecture.platform() self.set_pkg_dep('mpileaks', 'mpich platform=test target=be') spec = Spec('mpileaks ^mpich platform=test target=fe ^callpath ^dyninst ^libelf ^libdwarf') diff --git a/lib/spack/spack/test/spec_semantics.py b/lib/spack/spack/test/spec_semantics.py index 9876bfd5a8..b174e5305c 100644 --- a/lib/spack/spack/test/spec_semantics.py +++ b/lib/spack/spack/test/spec_semantics.py @@ -141,7 +141,6 @@ class SpecSematicsTest(MockPackagesTest): def test_satisfies_architecture(self): - platform = spack.architecture.sys_type() self.check_satisfies( 'foo platform=test target=frontend os=frontend', 'platform=test target=frontend os=frontend') @@ -396,7 +395,7 @@ class SpecSematicsTest(MockPackagesTest): self.check_constrain_changed('libelf', 'debug=2') self.check_constrain_changed('libelf', 'cppflags="-O3"') - platform = spack.architecture.sys_type() + platform = spack.architecture.platform() self.check_constrain_changed('libelf', 'target='+platform.target('default_target').name) self.check_constrain_changed('libelf', 'os='+platform.operating_system('default_os').name) @@ -412,7 +411,7 @@ class SpecSematicsTest(MockPackagesTest): self.check_constrain_not_changed('libelf debug=2', 'debug=2') self.check_constrain_not_changed('libelf cppflags="-O3"', 'cppflags="-O3"') - platform = spack.architecture.sys_type() + platform = spack.architecture.platform() default_target = platform.target('default_target').name self.check_constrain_not_changed('libelf target='+default_target, 'target='+default_target) @@ -425,7 +424,7 @@ class SpecSematicsTest(MockPackagesTest): self.check_constrain_changed('libelf^foo', 'libelf^foo+debug') self.check_constrain_changed('libelf^foo', 'libelf^foo~debug') - platform = spack.architecture.sys_type() + platform = spack.architecture.platform() default_target = platform.target('default_target').name self.check_constrain_changed('libelf^foo', 'libelf^foo target='+default_target) @@ -439,6 +438,6 @@ class SpecSematicsTest(MockPackagesTest): self.check_constrain_not_changed('libelf^foo~debug', 'libelf^foo~debug') self.check_constrain_not_changed('libelf^foo cppflags="-O3"', 'libelf^foo cppflags="-O3"') - platform = spack.architecture.sys_type() + platform = spack.architecture.platform() default_target = platform.target('default_target').name self.check_constrain_not_changed('libelf^foo target='+default_target, 'libelf^foo target='+default_target) |