diff options
Diffstat (limited to 'lib')
| -rw-r--r-- | lib/spack/spack/cmd/external.py | 91 | ||||
| -rw-r--r-- | lib/spack/spack/cmd/uninstall.py | 14 | ||||
| -rw-r--r-- | lib/spack/spack/compilers/__init__.py | 10 | ||||
| -rw-r--r-- | lib/spack/spack/cray_manifest.py | 165 | ||||
| -rw-r--r-- | lib/spack/spack/database.py | 13 | ||||
| -rw-r--r-- | lib/spack/spack/rewiring.py | 8 | ||||
| -rw-r--r-- | lib/spack/spack/schema/cray_manifest.py | 131 | ||||
| -rw-r--r-- | lib/spack/spack/solver/asp.py | 9 | ||||
| -rw-r--r-- | lib/spack/spack/spec.py | 56 | ||||
| -rw-r--r-- | lib/spack/spack/test/cmd/external.py | 42 | ||||
| -rw-r--r-- | lib/spack/spack/test/conftest.py | 27 | ||||
| -rw-r--r-- | lib/spack/spack/test/cray_manifest.py | 247 | ||||
| -rw-r--r-- | lib/spack/spack/test/rewiring.py | 2 | ||||
| -rw-r--r-- | lib/spack/spack/test/spec_dag.py | 2 |
14 files changed, 764 insertions, 53 deletions
diff --git a/lib/spack/spack/cmd/external.py b/lib/spack/spack/cmd/external.py index 42f033d979..50d77890fd 100644 --- a/lib/spack/spack/cmd/external.py +++ b/lib/spack/spack/cmd/external.py @@ -5,6 +5,7 @@ from __future__ import print_function import argparse +import os import sys import llnl.util.tty as tty @@ -13,6 +14,7 @@ import llnl.util.tty.colify as colify import spack import spack.cmd import spack.cmd.common.arguments +import spack.cray_manifest as cray_manifest import spack.detection import spack.error import spack.util.environment @@ -55,8 +57,40 @@ def setup_parser(subparser): 'list', help='list detectable packages, by repository and name' ) + read_cray_manifest = sp.add_parser( + 'read-cray-manifest', help=( + "consume a Spack-compatible description of externally-installed " + "packages, including dependency relationships" + ) + ) + read_cray_manifest.add_argument( + '--file', default=None, + help="specify a location other than the default") + read_cray_manifest.add_argument( + '--directory', default=None, + help="specify a directory storing a group of manifest files") + read_cray_manifest.add_argument( + '--dry-run', action='store_true', default=False, + help="don't modify DB with files that are read") + read_cray_manifest.add_argument( + '--fail-on-error', action='store_true', + help=("if a manifest file cannot be parsed, fail and report the " + "full stack trace") + ) + def external_find(args): + if args.all or not (args.tags or args.packages): + # If the user calls 'spack external find' with no arguments, and + # this system has a description of installed packages, then we should + # consume it automatically. + try: + _collect_and_consume_cray_manifest_files() + except NoManifestFileError: + # It's fine to not find any manifest file if we are doing the + # search implicitly (i.e. as part of 'spack external find') + pass + # If the user didn't specify anything, search for build tools by default if not args.tags and not args.all and not args.packages: args.tags = ['core-packages', 'build-tools'] @@ -106,6 +140,56 @@ def external_find(args): tty.msg('No new external packages detected') +def external_read_cray_manifest(args): + _collect_and_consume_cray_manifest_files( + manifest_file=args.file, + manifest_directory=args.directory, + dry_run=args.dry_run, + fail_on_error=args.fail_on_error + ) + + +def _collect_and_consume_cray_manifest_files( + manifest_file=None, manifest_directory=None, dry_run=False, + fail_on_error=False): + + manifest_files = [] + if manifest_file: + manifest_files.append(manifest_file) + + manifest_dirs = [] + if manifest_directory: + manifest_dirs.append(manifest_directory) + + if os.path.isdir(cray_manifest.default_path): + tty.debug( + "Cray manifest path {0} exists: collecting all files to read." + .format(cray_manifest.default_path)) + manifest_dirs.append(cray_manifest.default_path) + else: + tty.debug("Default Cray manifest directory {0} does not exist." + .format(cray_manifest.default_path)) + + for directory in manifest_dirs: + for fname in os.listdir(directory): + manifest_files.append(os.path.join(directory, fname)) + + if not manifest_files: + raise NoManifestFileError( + "--file/--directory not specified, and no manifest found at {0}" + .format(cray_manifest.default_path)) + + for path in manifest_files: + try: + cray_manifest.read(path, not dry_run) + except (spack.compilers.UnknownCompilerError, spack.error.SpackError) as e: + if fail_on_error: + raise + else: + tty.warn("Failure reading manifest file: {0}" + "\n\t{1}".format(path, str(e))) + + def external_list(args): # Trigger a read of all packages, might take a long time. list(spack.repo.path.all_packages()) @@ -117,5 +201,10 @@ def external_list(args): def external(parser, args): - action = {'find': external_find, 'list': external_list} + action = {'find': external_find, 'list': external_list, + 'read-cray-manifest': external_read_cray_manifest} action[args.external_command](args) + + +class NoManifestFileError(spack.error.SpackError): + pass diff --git a/lib/spack/spack/cmd/uninstall.py b/lib/spack/spack/cmd/uninstall.py index d82cb7eb3f..ce3d6f7dab 100644 --- a/lib/spack/spack/cmd/uninstall.py +++ b/lib/spack/spack/cmd/uninstall.py @@ -62,9 +62,14 @@ def setup_parser(subparser): '-a', '--all', action='store_true', dest='all', help="remove ALL installed packages that match each supplied spec" ) + subparser.add_argument( + '--origin', dest='origin', + help="only remove DB records with the specified origin" + ) -def find_matching_specs(env, specs, allow_multiple_matches=False, force=False): +def find_matching_specs(env, specs, allow_multiple_matches=False, force=False, + origin=None): """Returns a list of specs matching the not necessarily concretized specs given from cli @@ -85,8 +90,8 @@ def find_matching_specs(env, specs, allow_multiple_matches=False, force=False): has_errors = False for spec in specs: install_query = [InstallStatuses.INSTALLED, InstallStatuses.DEPRECATED] - matching = spack.store.db.query_local(spec, hashes=hashes, - installed=install_query) + matching = spack.store.db.query_local( + spec, hashes=hashes, installed=install_query, origin=origin) # For each spec provided, make sure it refers to only one package. # Fail and ask user to be unambiguous if it doesn't if not allow_multiple_matches and len(matching) > 1: @@ -240,7 +245,8 @@ def do_uninstall(env, specs, force): def get_uninstall_list(args, specs, env): # Gets the list of installed specs that match the ones give via cli # args.all takes care of the case where '-a' is given in the cli - uninstall_list = find_matching_specs(env, specs, args.all, args.force) + uninstall_list = find_matching_specs(env, specs, args.all, args.force, + args.origin) # Takes care of '-R' active_dpts, inactive_dpts = installed_dependents(uninstall_list, env) diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index 5aad6b2b98..73ba07237d 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -495,7 +495,8 @@ def get_compiler_duplicates(compiler_spec, arch_spec): @llnl.util.lang.memoized def class_for_compiler_name(compiler_name): """Given a compiler module name, get the corresponding Compiler class.""" - assert supported(compiler_name) + if not supported(compiler_name): + raise UnknownCompilerError(compiler_name) # Hack to be able to call the compiler `apple-clang` while still # using a valid python name for the module @@ -788,6 +789,13 @@ class NoCompilersError(spack.error.SpackError): "Spack could not find any compilers!") +class UnknownCompilerError(spack.error.SpackError): + def __init__(self, compiler_name): + super(UnknownCompilerError, self).__init__( + "Spack doesn't support the requested compiler: {0}" + .format(compiler_name)) + + class NoCompilerForSpecError(spack.error.SpackError): def __init__(self, compiler_spec, target): super(NoCompilerForSpecError, self).__init__( diff --git a/lib/spack/spack/cray_manifest.py b/lib/spack/spack/cray_manifest.py new file mode 100644 index 0000000000..6b9af13a90 --- /dev/null +++ b/lib/spack/spack/cray_manifest.py @@ -0,0 +1,165 @@ +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import json + +import jsonschema +import six + +import llnl.util.tty as tty + +import spack.cmd +import spack.hash_types as hash_types +from spack.schema.cray_manifest import schema as manifest_schema + +#: Cray systems can store a Spack-compatible description of system +#: packages here. +default_path = '/opt/cray/pe/cpe-descriptive-manifest/' + + +def compiler_from_entry(entry): + compiler_name = entry['name'] + paths = entry['executables'] + version = entry['version'] + arch = entry['arch'] + operating_system = arch['os'] + target = arch['target'] + + compiler_cls = spack.compilers.class_for_compiler_name(compiler_name) + spec = spack.spec.CompilerSpec(compiler_cls.name, version) + paths = [paths.get(x, None) for x in ('cc', 'cxx', 'f77', 'fc')] + return compiler_cls( + spec, operating_system, target, paths + ) + + +def spec_from_entry(entry): + arch_str = "" + if 'arch' in entry: + arch_format = "arch={platform}-{os}-{target}" + arch_str = arch_format.format( + platform=entry['arch']['platform'], + os=entry['arch']['platform_os'], + target=entry['arch']['target']['name'] + ) + + compiler_str = "" + if 'compiler' in entry: + compiler_format = "%{name}@{version}" + compiler_str = compiler_format.format( + name=entry['compiler']['name'], + version=entry['compiler']['version'] + ) + + spec_format = "{name}@{version} {compiler} {arch}" + spec_str = spec_format.format( + name=entry['name'], + version=entry['version'], + compiler=compiler_str, + arch=arch_str + ) + + package = spack.repo.get(entry['name']) + + if 'parameters' in entry: + variant_strs = list() + for name, value in entry['parameters'].items(): + # TODO: also ensure that the variant value is valid? + if not (name in package.variants): + tty.debug("Omitting variant {0} for entry {1}/{2}" + .format(name, entry['name'], entry['hash'][:7])) + continue + + # Value could be a list (of strings), boolean, or string + if isinstance(value, six.string_types): + variant_strs.append('{0}={1}'.format(name, value)) + else: + try: + iter(value) + variant_strs.append( + '{0}={1}'.format(name, ','.join(value))) + continue + except TypeError: + # Not an iterable + pass + # At this point not a string or collection, check for boolean + if value in [True, False]: + bool_symbol = '+' if value else '~' + variant_strs.append('{0}{1}'.format(bool_symbol, name)) + else: + raise ValueError( + "Unexpected value for {0} ({1}): {2}".format( + name, str(type(value)), str(value) + ) + ) + spec_str += ' ' + ' '.join(variant_strs) + + spec, = spack.cmd.parse_specs(spec_str.split()) + + for ht in [hash_types.dag_hash, hash_types.build_hash, + hash_types.full_hash]: + setattr(spec, ht.attr, entry['hash']) + + spec._concrete = True + spec._hashes_final = True + spec.external_path = entry['prefix'] + spec.origin = 'external-db' + spack.spec.Spec.ensure_valid_variants(spec) + + return spec + + +def entries_to_specs(entries): + spec_dict = {} + for entry in entries: + try: + spec = spec_from_entry(entry) + spec_dict[spec._hash] = spec + except spack.repo.UnknownPackageError: + tty.debug("Omitting package {0}: no corresponding repo package" + .format(entry['name'])) + except spack.error.SpackError: + raise + except Exception: + tty.warn("Could not parse entry: " + str(entry)) + + for entry in filter(lambda x: 'dependencies' in x, entries): + dependencies = entry['dependencies'] + for name, properties in dependencies.items(): + dep_hash = properties['hash'] + deptypes = properties['type'] + if dep_hash in spec_dict: + if entry['hash'] not in spec_dict: + continue + parent_spec = spec_dict[entry['hash']] + dep_spec = spec_dict[dep_hash] + parent_spec._add_dependency(dep_spec, deptypes) + + return spec_dict + + +def read(path, apply_updates): + with open(path, 'r') as json_file: + json_data = json.load(json_file) + + jsonschema.validate(json_data, manifest_schema) + + specs = entries_to_specs(json_data['specs']) + tty.debug("{0}: {1} specs read from manifest".format( + path, + str(len(specs)))) + compilers = list() + if 'compilers' in json_data: + compilers.extend(compiler_from_entry(x) + for x in json_data['compilers']) + tty.debug("{0}: {1} compilers read from manifest".format( + path, + str(len(compilers)))) + if apply_updates and compilers: + spack.compilers.add_compilers_to_config( + compilers, init_config=False) + if apply_updates: + for spec in specs.values(): + spack.store.db.add(spec, directory_layout=None) diff --git a/lib/spack/spack/database.py b/lib/spack/spack/database.py index 3c7258728f..6a16a83e1c 100644 --- a/lib/spack/spack/database.py +++ b/lib/spack/spack/database.py @@ -187,6 +187,7 @@ class InstallRecord(object): installation_time=None, deprecated_for=None, in_buildcache=False, + origin=None ): self.spec = spec self.path = str(path) if path else None @@ -196,6 +197,7 @@ class InstallRecord(object): self.installation_time = installation_time or _now() self.deprecated_for = deprecated_for self.in_buildcache = in_buildcache + self.origin = origin def install_type_matches(self, installed): installed = InstallStatuses.canonicalize(installed) @@ -217,6 +219,9 @@ class InstallRecord(object): else: rec_dict.update({field_name: getattr(self, field_name)}) + if self.origin: + rec_dict['origin'] = self.origin + return rec_dict @classmethod @@ -1131,6 +1136,10 @@ class Database(object): 'explicit': explicit, 'installation_time': installation_time } + # Commands other than 'spack install' may add specs to the DB, + # we can record the source of an installed Spec with 'origin' + if hasattr(spec, 'origin'): + extra_args['origin'] = spec.origin self._data[key] = InstallRecord( new_spec, path, installed, ref_count=0, **extra_args ) @@ -1462,6 +1471,7 @@ class Database(object): end_date=None, hashes=None, in_buildcache=any, + origin=None ): """Run a query on the database.""" @@ -1490,6 +1500,9 @@ class Database(object): if hashes is not None and rec.spec.dag_hash() not in hashes: continue + if origin and not (origin == rec.origin): + continue + if not rec.install_type_matches(installed): continue diff --git a/lib/spack/spack/rewiring.py b/lib/spack/spack/rewiring.py index feeb8d1a14..c91485219b 100644 --- a/lib/spack/spack/rewiring.py +++ b/lib/spack/spack/rewiring.py @@ -95,7 +95,8 @@ def rewire_node(spec, explicit): spec.prefix) relocate.relocate_text_bin(binaries=bins_to_relocate, prefixes=prefix_to_prefix) - # copy package into place (shutil.copytree) + # Copy package into place, except for spec.json (because spec.json + # describes the old spec and not the new spliced spec). shutil.copytree(os.path.join(tempdir, spec.dag_hash()), spec.prefix, ignore=shutil.ignore_patterns('spec.json', 'install_manifest.json')) @@ -104,7 +105,10 @@ def rewire_node(spec, explicit): spec.build_spec.prefix, spec.prefix) shutil.rmtree(tempdir) - # handle all metadata changes; don't copy over spec.json file in .spack/ + # Above, we did not copy spec.json: instead, here we write the new + # (spliced) spec into spec.json, without this, Database.add would fail on + # the next line (because it checks the spec.json in the prefix against the + # spec being added to look for mismatches) spack.store.layout.write_spec(spec, spack.store.layout.spec_file_path(spec)) # add to database, not sure about explicit spack.store.db.add(spec, spack.store.layout, explicit=explicit) diff --git a/lib/spack/spack/schema/cray_manifest.py b/lib/spack/spack/schema/cray_manifest.py new file mode 100644 index 0000000000..9613bbf1dc --- /dev/null +++ b/lib/spack/spack/schema/cray_manifest.py @@ -0,0 +1,131 @@ +# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +"""Schema for Cray descriptive manifest: this describes a set of + installed packages on the system and also specifies dependency + relationships between them (so this provides more information than + external entries in packages configuration). + + This does not specify a configuration - it is an input format + that is consumed and transformed into Spack DB records. +""" + +schema = { + "$schema": "http://json-schema.org/schema#", + "title": "CPE manifest schema", + "type": "object", + "additionalProperties": False, + "properties": { + "_meta": { + "type": "object", + "additionalProperties": False, + "properties": { + "file-type": {"type": "string", "minLength": 1}, + "cpe-version": {"type": "string", "minLength": 1}, + "system-type": {"type": "string", "minLength": 1}, + "schema-version": {"type": "string", "minLength": 1}, + } + }, + "compilers": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": False, + "properties": { + "name": {"type": "string", "minLength": 1}, + "version": {"type": "string", "minLength": 1}, + "prefix": {"type": "string", "minLength": 1}, + "executables": { + "type": "object", + "additionalProperties": False, + "properties": { + "cc": {"type": "string", "minLength": 1}, + "cxx": {"type": "string", "minLength": 1}, + "fc": {"type": "string", "minLength": 1} + } + }, + "arch": { + "type": "object", + "required": ["os", "target"], + "additionalProperties": False, + "properties": { + "os": {"type": "string", "minLength": 1}, + "target": {"type": "string", "minLength": 1} + } + } + } + } + }, + "specs": { + "type": "array", + "items": { + "type": "object", + "required": [ + "name", + "version", + "arch", + "compiler", + "prefix", + "hash"], + "additionalProperties": False, + "properties": { + "name": {"type": "string", "minLength": 1}, + "version": {"type": "string", "minLength": 1}, + "arch": { + "type": "object", + "required": ["platform", "platform_os", "target"], + "additioanlProperties": False, + "properties": { + "platform": {"type": "string", "minLength": 1}, + "platform_os": {"type": "string", "minLength": 1}, + "target": { + "type": "object", + "additionalProperties": False, + "required": ["name"], + "properties": { + "name": {"type": "string", "minLength": 1} + } + } + } + }, + "compiler": { + "type": "object", + "required": ["name", "version"], + "additionalProperties": False, + "properties": { + "name": {"type": "string", "minLength": 1}, + "version": {"type": "string", "minLength": 1} + } + }, + "dependencies": { + "type": "object", + "patternProperties": { + "\\w[\\w-]*": { + "type": "object", + "required": ["hash"], + "additionalProperties": False, + "properties": { + "hash": {"type": "string", "minLength": 1}, + "type": { + "type": "array", + "items": { + "type": "string", "minLength": 1} + } + } + } + } + }, + "prefix": { + "type": "string", "minLength": 1}, + "rpm": {"type": "string", "minLength": 1}, + "hash": {"type": "string", "minLength": 1}, + "parameters": { + "type": "object", + } + } + } + } + } +} diff --git a/lib/spack/spack/solver/asp.py b/lib/spack/spack/solver/asp.py index d3a29fe250..52f1b8f67a 100644 --- a/lib/spack/spack/solver/asp.py +++ b/lib/spack/spack/solver/asp.py @@ -1526,9 +1526,12 @@ class SpackSolverSetup(object): continue if strict and s.compiler not in cspecs: - raise spack.concretize.UnavailableCompilerVersionError( - s.compiler - ) + if not s.concrete: + raise spack.concretize.UnavailableCompilerVersionError( + s.compiler + ) + # Allow unknown compilers to exist if the associated spec + # is already built else: cspecs.add(s.compiler) self.gen.fact(fn.allow_compiler( diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index c37d3e754e..5189844b63 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -2943,7 +2943,7 @@ class Spec(object): if a list of names activate them for the packages in the list, if True activate 'test' dependencies for all packages. """ - clone = self.copy(caches=True) + clone = self.copy() clone.concretize(tests=tests) return clone @@ -3242,8 +3242,8 @@ class Spec(object): "Attempting to normalize anonymous spec") # Set _normal and _concrete to False when forced - if force: - self._mark_concrete(False) + if force and not self._concrete: + self._normal = False if self._normal: return False @@ -3712,7 +3712,7 @@ class Spec(object): return patches - def _dup(self, other, deps=True, cleardeps=True, caches=None): + def _dup(self, other, deps=True, cleardeps=True): """Copy the spec other into self. This is an overwriting copy. It does not copy any dependents (parents), but by default copies dependencies. @@ -3727,10 +3727,6 @@ class Spec(object): cleardeps (bool): if True clears the dependencies of ``self``, before possibly copying the dependencies of ``other`` onto ``self`` - caches (bool or None): preserve cached fields such as - ``_normal``, ``_hash``, and ``_dunder_hash``. By - default this is ``False`` if DAG structure would be - changed by the copy, ``True`` if it's an exact copy. Returns: True if ``self`` changed because of the copy operation, @@ -3781,12 +3777,6 @@ class Spec(object): self.extra_attributes = other.extra_attributes self.namespace = other.namespace - # Cached fields are results of expensive operations. - # If we preserved the original structure, we can copy them - # safely. If not, they need to be recomputed. - if caches is None: - caches = (deps is True or deps == dp.all_deptypes) - # If we copy dependencies, preserve DAG structure in the new spec if deps: # If caller restricted deptypes to be copied, adjust that here. @@ -3794,29 +3784,31 @@ class Spec(object): deptypes = dp.all_deptypes if isinstance(deps, (tuple, list)): deptypes = deps - self._dup_deps(other, deptypes, caches) + self._dup_deps(other, deptypes) self._concrete = other._concrete self._hashes_final = other._hashes_final - if caches: + if self._concrete: self._hash = other._hash self._build_hash = other._build_hash self._dunder_hash = other._dunder_hash - self._normal = other._normal + self._normal = True self._full_hash = other._full_hash self._package_hash = other._package_hash else: self._hash = None self._build_hash = None self._dunder_hash = None + # Note, we could use other._normal if we are copying all deps, but + # always set it False here to avoid the complexity of checking self._normal = False self._full_hash = None self._package_hash = None return changed - def _dup_deps(self, other, deptypes, caches): + def _dup_deps(self, other, deptypes): def spid(spec): return id(spec) @@ -3827,11 +3819,11 @@ class Spec(object): if spid(edge.parent) not in new_specs: new_specs[spid(edge.parent)] = edge.parent.copy( - deps=False, caches=caches + deps=False ) if spid(edge.spec) not in new_specs: - new_specs[spid(edge.spec)] = edge.spec.copy(deps=False, caches=caches) + new_specs[spid(edge.spec)] = edge.spec.copy(deps=False) new_specs[spid(edge.parent)].add_dependency_edge( new_specs[spid(edge.spec)], edge.deptypes @@ -4709,16 +4701,16 @@ class Spec(object): return False return True - self_nodes = dict((s.name, s.copy(deps=False, caches=True)) + self_nodes = dict((s.name, s.copy(deps=False)) for s in self.traverse(root=True) if from_self(s.name, transitive)) if transitive: - other_nodes = dict((s.name, s.copy(deps=False, caches=True)) + other_nodes = dict((s.name, s.copy(deps=False)) for s in other.traverse(root=True)) else: # NOTE: Does not fully validate providers; loader races possible - other_nodes = dict((s.name, s.copy(deps=False, caches=True)) + other_nodes = dict((s.name, s.copy(deps=False)) for s in other.traverse(root=True) if s is other or s.name not in self) @@ -4751,23 +4743,12 @@ class Spec(object): for dep in ret.traverse(root=True, order='post'): opposite = other_nodes if dep.name in self_nodes else self_nodes if any(name in dep for name in opposite.keys()): - # Record whether hashes are already cached - # So we don't try to compute a hash from insufficient - # provenance later - has_build_hash = getattr(dep, ht.build_hash.name, None) - has_full_hash = getattr(dep, ht.full_hash.name, None) - # package hash cannot be affected by splice dep.clear_cached_hashes(ignore=['package_hash']) - # Since this is a concrete spec, we want to make sure hashes - # are cached writing specs only writes cached hashes in case - # the spec is too old to have full provenance for these hashes, - # so we can't rely on doing it at write time. - if has_build_hash: - _ = dep.build_hash() - if has_full_hash: - _ = dep.full_hash() + dep.build_hash() + dep.full_hash() + dep.dag_hash() return nodes[self.name] @@ -4779,6 +4760,7 @@ class Spec(object): if h.attr not in ignore: if hasattr(self, h.attr): setattr(self, h.attr, None) + self._dunder_hash = None def __hash__(self): # If the spec is concrete, we leverage the DAG hash and just use diff --git a/lib/spack/spack/test/cmd/external.py b/lib/spack/spack/test/cmd/external.py index 3a3c4face3..bfdcb0b4eb 100644 --- a/lib/spack/spack/test/cmd/external.py +++ b/lib/spack/spack/test/cmd/external.py @@ -167,6 +167,48 @@ def test_find_external_cmd_full_repo( assert {'spec': 'find-externals1@1.foo', 'prefix': prefix} in pkg_externals +def test_find_external_no_manifest( + mutable_config, working_env, mock_executable, mutable_mock_repo, + _platform_executables, monkeypatch): + """The user runs 'spack external find'; the default path for storing + manifest files does not exist. Ensure that the command does not + fail. + """ + monkeypatch.setenv('PATH', '') + monkeypatch.setattr(spack.cray_manifest, 'default_path', + os.path.join('a', 'path', 'that', 'doesnt', 'exist')) + external('find') + + +def test_find_external_empty_default_manifest_dir( + mutable_config, working_env, mock_executable, mutable_mock_repo, + _platform_executables, tmpdir, monkeypatch): + """The user runs 'spack external find'; the default path for storing + manifest files exists but is empty. Ensure that the command does not + fail. + """ + empty_manifest_dir = str(tmpdir.mkdir('manifest_dir')) + monkeypatch.setenv('PATH', '') + monkeypatch.setattr(spack.cray_manifest, 'default_path', + empty_manifest_dir) + external('find') + + +def test_find_external_nonempty_default_manifest_dir( + mutable_database, mutable_mock_repo, + _platform_executables, tmpdir, monkeypatch, + directory_with_manifest): + """The user runs 'spack external find'; the default manifest directory + contains a manifest file. Ensure that the specs are read. + """ + monkeypatch.setenv('PATH', '') + monkeypatch.setattr(spack.cray_manifest, 'default_path', + str(directory_with_manifest)) + external('find') + specs = spack.store.db.query('hwloc') + assert any(x.dag_hash() == 'hwlocfakehashaaa' for x in specs) + + def test_find_external_merge(mutable_config, mutable_mock_repo): """Check that 'spack find external' doesn't overwrite an existing spec entry in packages.yaml. diff --git a/lib/spack/spack/test/conftest.py b/lib/spack/spack/test/conftest.py index 1a1b15f469..11a24ba044 100644 --- a/lib/spack/spack/test/conftest.py +++ b/lib/spack/spack/test/conftest.py @@ -43,6 +43,7 @@ import spack.repo import spack.stage import spack.store import spack.subprocess_context +import spack.test.cray_manifest import spack.util.executable import spack.util.gpg import spack.util.spack_yaml as syaml @@ -809,7 +810,16 @@ def database(mock_store, mock_packages, config): @pytest.fixture(scope='function') -def mutable_database(database, _store_dir_and_cache): +def database_mutable_config(mock_store, mock_packages, mutable_config, + monkeypatch): + """This activates the mock store, packages, AND config.""" + with spack.store.use_store(str(mock_store)) as store: + yield store.db + store.db.last_seen_verifier = '' + + +@pytest.fixture(scope='function') +def mutable_database(database_mutable_config, _store_dir_and_cache): """Writeable version of the fixture, restored to its initial state after each test. """ @@ -817,7 +827,7 @@ def mutable_database(database, _store_dir_and_cache): store_path, store_cache = _store_dir_and_cache store_path.join('.spack-db').chmod(mode=0o755, rec=1) - yield database + yield database_mutable_config # Restore the initial state by copying the content of the cache back into # the store and making the database read-only @@ -1635,6 +1645,19 @@ def brand_new_binary_cache(): spack.binary_distribution._binary_index) +@pytest.fixture +def directory_with_manifest(tmpdir): + """Create a manifest file in a directory. Used by 'spack external'. + """ + with tmpdir.as_cwd(): + test_db_fname = 'external-db.json' + with open(test_db_fname, 'w') as db_file: + json.dump(spack.test.cray_manifest.create_manifest_content(), + db_file) + + yield str(tmpdir) + + @pytest.fixture() def noncyclical_dir_structure(tmpdir): """ diff --git a/lib/spack/spack/test/cray_manifest.py b/lib/spack/spack/test/cray_manifest.py new file mode 100644 index 0000000000..d93990c677 --- /dev/null +++ b/lib/spack/spack/test/cray_manifest.py @@ -0,0 +1,247 @@ +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import json + +import pytest + +import spack +import spack.cray_manifest as cray_manifest +from spack.cray_manifest import compiler_from_entry, entries_to_specs + +example_x_json_str = """\ +{ + "name": "packagex", + "hash": "hash-of-x", + "prefix": "/path/to/packagex-install/", + "version": "1.0", + "arch": { + "platform": "linux", + "platform_os": "centos8", + "target": { + "name": "haswell" + } + }, + "compiler": { + "name": "gcc", + "version": "10.2.0" + }, + "dependencies": { + "packagey": { + "hash": "hash-of-y", + "type": ["link"] + } + }, + "parameters": { + "precision": ["double", "float"] + } +} +""" + + +example_compiler_entry = """\ +{ + "name": "gcc", + "prefix": "/path/to/compiler/", + "version": "7.5.0", + "arch": { + "os": "centos8", + "target": "x86_64" + }, + "executables": { + "cc": "/path/to/compiler/cc", + "cxx": "/path/to/compiler/cxx", + "fc": "/path/to/compiler/fc" + } +} +""" + + +class JsonSpecEntry(object): + def __init__(self, name, hash, prefix, version, arch, compiler, + dependencies, parameters): + self.name = name + self.hash = hash + self.prefix = prefix + self.version = version + self.arch = arch + self.compiler = compiler + self.dependencies = dependencies + self.parameters = parameters + + def to_dict(self): + return { + 'name': self.name, + 'hash': self.hash, + 'prefix': self.prefix, + 'version': self.version, + 'arch': self.arch, + 'compiler': self.compiler, + 'dependencies': self.dependencies, + 'parameters': self.parameters + } + + def as_dependency(self, deptypes): + return (self.name, + {'hash': self.hash, + 'type': list(deptypes)}) + + +class JsonArchEntry(object): + def __init__(self, platform, os, target): + self.platform = platform + self.os = os + self.target = target + + def to_dict(self): + return { + 'platform': self.platform, + 'platform_os': self.os, + 'target': { + 'name': self.target + } + } + + +class JsonCompilerEntry(object): + def __init__(self, name, version): + self.name = name + self.version = version + + def to_dict(self): + return { + 'name': self.name, + 'version': self.version + } + + +_common_arch = JsonArchEntry( + platform='linux', + os='centos8', + target='haswell' +).to_dict() + + +_common_compiler = JsonCompilerEntry( + name='gcc', + version='10.2.0' +).to_dict() + + +def test_compatibility(): + """Make sure that JsonSpecEntry outputs the expected JSON structure + by comparing it with JSON parsed from an example string. This + ensures that the testing objects like JsonSpecEntry produce the + same JSON structure as the expected file format. + """ + y = JsonSpecEntry( + name='packagey', + hash='hash-of-y', + prefix='/path/to/packagey-install/', + version='1.0', + arch=_common_arch, + compiler=_common_compiler, + dependencies={}, + parameters={} + ) + + x = JsonSpecEntry( + name='packagex', + hash='hash-of-x', + prefix='/path/to/packagex-install/', + version='1.0', + arch=_common_arch, + compiler=_common_compiler, + dependencies=dict([y.as_dependency(deptypes=['link'])]), + parameters={'precision': ['double', 'float']} + ) + + x_from_entry = x.to_dict() + x_from_str = json.loads(example_x_json_str) + assert x_from_entry == x_from_str + + +def test_compiler_from_entry(): + compiler_data = json.loads(example_compiler_entry) + compiler_from_entry(compiler_data) + + +def generate_openmpi_entries(): + """Generate two example JSON entries that refer to an OpenMPI + installation and a hwloc dependency. + """ + # The hashes need to be padded with 'a' at the end to align with 8-byte + # boundaries (for base-32 decoding) + hwloc = JsonSpecEntry( + name='hwloc', + hash='hwlocfakehashaaa', + prefix='/path/to/hwloc-install/', + version='2.0.3', + arch=_common_arch, + compiler=_common_compiler, + dependencies={}, + parameters={} + ) + + # This includes a variant which is guaranteed not to appear in the + # OpenMPI package: we need to make sure we can use such package + # descriptions. + openmpi = JsonSpecEntry( + name='openmpi', + hash='openmpifakehasha', + prefix='/path/to/openmpi-install/', + version='4.1.0', + arch=_common_arch, + compiler=_common_compiler, + dependencies=dict([hwloc.as_dependency(deptypes=['link'])]), + parameters={ + 'internal-hwloc': False, + 'fabrics': ['psm'], + 'missing_variant': True + } + ) + + return [openmpi, hwloc] + + +def test_spec_conversion(): + """Given JSON entries, check that we can form a set of Specs + including dependency references. + """ + entries = list(x.to_dict() for x in generate_openmpi_entries()) + specs = entries_to_specs(entries) + openmpi_spec, = list(x for x in specs.values() if x.name == 'openmpi') + assert openmpi_spec['hwloc'] + + +def create_manifest_content(): + return { + 'specs': list(x.to_dict() for x in generate_openmpi_entries()), + 'compilers': [] + } + + +def test_read_cray_manifest( + tmpdir, mutable_config, mock_packages, mutable_database): + """Check that (a) we can read the cray manifest and add it to the Spack + Database and (b) we can concretize specs based on that. + """ + if spack.config.get('config:concretizer') == 'clingo': + pytest.skip("The ASP-based concretizer is currently picky about " + " OS matching and will fail.") + + with tmpdir.as_cwd(): + test_db_fname = 'external-db.json' + with open(test_db_fname, 'w') as db_file: + json.dump(create_manifest_content(), db_file) + cray_manifest.read(test_db_fname, True) + query_specs = spack.store.db.query('openmpi') + assert any(x.dag_hash() == 'openmpifakehasha' for x in query_specs) + + concretized_specs = spack.cmd.parse_specs( + 'depends-on-openmpi %gcc@4.5.0 arch=test-redhat6-x86_64' + ' ^/openmpifakehasha'.split(), + concretize=True) + assert concretized_specs[0]['hwloc'].dag_hash() == 'hwlocfakehashaaa' diff --git a/lib/spack/spack/test/rewiring.py b/lib/spack/spack/test/rewiring.py index e15be34fdc..f883c222a5 100644 --- a/lib/spack/spack/test/rewiring.py +++ b/lib/spack/spack/test/rewiring.py @@ -23,7 +23,7 @@ else: @pytest.mark.requires_executables(*args) @pytest.mark.parametrize('transitive', [True, False]) -def test_rewire(mock_fetch, install_mockery, transitive): +def test_rewire_db(mock_fetch, install_mockery, transitive): """Tests basic rewiring without binary executables.""" spec = Spec('splice-t^splice-h~foo').concretized() dep = Spec('splice-h+foo').concretized() diff --git a/lib/spack/spack/test/spec_dag.py b/lib/spack/spack/test/spec_dag.py index dcf830fd78..78555340a7 100644 --- a/lib/spack/spack/test/spec_dag.py +++ b/lib/spack/spack/test/spec_dag.py @@ -602,8 +602,6 @@ class TestSpecDag(object): assert orig == copy assert orig.eq_dag(copy) - assert orig._normal == copy._normal - assert orig._concrete == copy._concrete # ensure no shared nodes bt/w orig and copy. orig_ids = set(id(s) for s in orig.traverse()) |