From d450a2fce289ed48f7fcff767bad8a802253ddee Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Mon, 10 Dec 2018 14:01:16 -0800 Subject: stacks: initial implementation of stacks on environments - stack syntax in env schema - switch environment specs over to SpecList object - add stack functionality to environments - handle definition extensions through stack.yaml and SpecList - implement conditional definitions - tests --- lib/spack/spack/cmd/add.py | 5 +- lib/spack/spack/cmd/find.py | 1 + lib/spack/spack/cmd/remove.py | 5 +- lib/spack/spack/environment.py | 259 +++++++++++++++++++++++++++++------- lib/spack/spack/schema/env.py | 83 ++++++++++-- lib/spack/spack/spec.py | 2 +- lib/spack/spack/spec_list.py | 168 ++++++++++++++++++++++++ lib/spack/spack/test/cmd/env.py | 269 ++++++++++++++++++++++++++++++++++++++ lib/spack/spack/test/spec_list.py | 144 ++++++++++++++++++++ 9 files changed, 875 insertions(+), 61 deletions(-) create mode 100644 lib/spack/spack/spec_list.py create mode 100644 lib/spack/spack/test/spec_list.py diff --git a/lib/spack/spack/cmd/add.py b/lib/spack/spack/cmd/add.py index e7a7e05b98..894161fe00 100644 --- a/lib/spack/spack/cmd/add.py +++ b/lib/spack/spack/cmd/add.py @@ -17,6 +17,9 @@ level = "long" def setup_parser(subparser): + subparser.add_argument('-l', '--list-name', + dest='list_name', default='specs', + help="name of the list to add specs to") subparser.add_argument( 'specs', nargs=argparse.REMAINDER, help="specs of packages to add") @@ -25,7 +28,7 @@ def add(parser, args): env = ev.get_env(args, 'add', required=True) for spec in spack.cmd.parse_specs(args.specs): - if not env.add(spec): + if not env.add(spec, args.list_name): tty.msg("Package {0} was already added to {1}" .format(spec.name, env.name)) else: diff --git a/lib/spack/spack/cmd/find.py b/lib/spack/spack/cmd/find.py index c86a7f9cd1..68d7a0f1d5 100644 --- a/lib/spack/spack/cmd/find.py +++ b/lib/spack/spack/cmd/find.py @@ -175,6 +175,7 @@ def find(parser, args): tty.msg('No root specs') else: tty.msg('Root specs') + # TODO: Change this to not print extraneous deps and variants display_specs( env.user_specs, args, decorator=lambda s, f: color.colorize('@*{%s}' % f)) diff --git a/lib/spack/spack/cmd/remove.py b/lib/spack/spack/cmd/remove.py index 87942660e8..1d07830f03 100644 --- a/lib/spack/spack/cmd/remove.py +++ b/lib/spack/spack/cmd/remove.py @@ -20,6 +20,9 @@ def setup_parser(subparser): subparser.add_argument( '-a', '--all', action='store_true', help="remove all specs from (clear) the environment") + subparser.add_argument('-l', '--list-name', + dest='list_name', default='specs', + help="name of the list to remove specs from") subparser.add_argument( '-f', '--force', action='store_true', help="remove concretized spec (if any) immediately") @@ -35,5 +38,5 @@ def remove(parser, args): else: for spec in spack.cmd.parse_specs(args.specs): tty.msg('Removing %s from environment %s' % (spec, env.name)) - env.remove(spec, force=args.force) + env.remove(spec, args.list_name, force=args.force) env.write() diff --git a/lib/spack/spack/environment.py b/lib/spack/spack/environment.py index 02b0a6d93c..abc4c63e6b 100644 --- a/lib/spack/spack/environment.py +++ b/lib/spack/spack/environment.py @@ -7,10 +7,13 @@ import os import re import sys import shutil +import copy import ruamel.yaml import six +from ordereddict_backport import OrderedDict + import llnl.util.filesystem as fs import llnl.util.tty as tty from llnl.util.tty.color import colorize @@ -21,10 +24,13 @@ import spack.schema.env import spack.spec import spack.util.spack_json as sjson import spack.config -from spack.spec import Spec from spack.filesystem_view import YamlFilesystemView - from spack.util.environment import EnvironmentModifications +import spack.architecture as architecture +from spack.spec import Spec +from spack.spec_list import SpecList, InvalidSpecConstraintError +from spack.variant import UnknownVariantError +from spack.util.executable import which #: environment variable used to indicate the active environment spack_env_var = 'SPACK_ENV' @@ -385,6 +391,29 @@ def _write_yaml(data, str_or_file): default_flow_style=False) +def _eval_conditional(string): + """Evaluate conditional definitions using restricted variable scope.""" + arch = architecture.Arch( + architecture.platform(), 'default_os', 'default_target') + valid_variables = { + 'target': str(arch.target), + 'os': str(arch.platform_os), + 'platform': str(arch.platform), + 'arch': str(arch), + 'architecture': str(arch), + 're': re, + 'env': os.environ, + } + hostname_bin = which('hostname') + if hostname_bin: + hostname = str(hostname_bin(output=str, error=str)).strip() + valid_variables['hostname'] = hostname + else: + tty.warn("Spack was unable to find the executable `hostname`" + " hostname will be unavailable in conditionals") + return eval(string, valid_variables) + + class Environment(object): def __init__(self, path, init_file=None, with_view=None): """Create a new environment. @@ -425,6 +454,18 @@ class Environment(object): if default_manifest: self._set_user_specs_from_lockfile() + if os.path.exists(self.manifest_path): + # read the spack.yaml file, if exists + with open(self.manifest_path) as f: + self._read_manifest(f) + elif self.concretized_user_specs: + # if not, take user specs from the lockfile + self._set_user_specs_from_lockfile() + self.yaml = _read_yaml(default_manifest_yaml) + else: + # if there's no manifest or lockfile, use the default + self._read_manifest(default_manifest_yaml) + if with_view is False: self._view_path = None elif isinstance(with_view, six.string_types): @@ -435,9 +476,38 @@ class Environment(object): def _read_manifest(self, f): """Read manifest file and set up user specs.""" self.yaml = _read_yaml(f) + + self.read_specs = OrderedDict() + + for item in list(self.yaml.values())[0].get('definitions', []): + entry = copy.deepcopy(item) + when = _eval_conditional(entry.pop('when', 'True')) + assert len(entry) == 1 + if when: + name, spec_list = list(entry.items())[0] + user_specs = SpecList(name, spec_list, self.read_specs.copy()) + if name in self.read_specs: + self.read_specs[name].extend(user_specs) + else: + self.read_specs[name] = user_specs + spec_list = config_dict(self.yaml).get('specs') - if spec_list: - self.user_specs = [Spec(s) for s in spec_list if s] + user_specs = SpecList('specs', [s for s in spec_list if s], + self.read_specs.copy()) + self.read_specs['specs'] = user_specs + + enable_view = config_dict(self.yaml).get('view') + # enable_view can be boolean, string, or None + if enable_view is True or enable_view is None: + self._view_path = self.default_view_path + elif isinstance(enable_view, six.string_types): + self._view_path = enable_view + else: + self._view_path = None + + @property + def user_specs(self): + return self.read_specs['specs'] enable_view = config_dict(self.yaml).get('view') # enable_view can be true/false, a string, or None (if the manifest did @@ -452,10 +522,14 @@ class Environment(object): def _set_user_specs_from_lockfile(self): """Copy user_specs from a read-in lockfile.""" - self.user_specs = [Spec(s) for s in self.concretized_user_specs] + self.read_specs = { + 'specs': SpecList( + 'specs', [Spec(s) for s in self.concretized_user_specs] + ) + } def clear(self): - self.user_specs = [] # current user specs + self.read_specs = {'specs': SpecList()} # specs read from yaml self.concretized_user_specs = [] # user specs from last concretize self.concretized_order = [] # roots of last concretize, in order self.specs_by_hash = {} # concretized specs by hash @@ -578,7 +652,7 @@ class Environment(object): """Remove this environment from Spack entirely.""" shutil.rmtree(self.path) - def add(self, user_spec): + def add(self, user_spec, list_name='specs'): """Add a single user_spec (non-concretized) to the Environment Returns: @@ -587,47 +661,86 @@ class Environment(object): """ spec = Spec(user_spec) - if not spec.name: - raise SpackEnvironmentError( - 'cannot add anonymous specs to an environment!') - elif not spack.repo.path.exists(spec.name): - raise SpackEnvironmentError('no such package: %s' % spec.name) - existing = set(s for s in self.user_specs if s.name == spec.name) - if not existing: - self.user_specs.append(spec) + if list_name not in self.read_specs: + raise SpackEnvironmentError( + 'No list %s exists in environment %s' % (list_name, self.name) + ) + + if list_name == 'specs': + if not spec.name: + raise SpackEnvironmentError( + 'cannot add anonymous specs to an environment!') + elif not spack.repo.path.exists(spec.name): + raise SpackEnvironmentError('no such package: %s' % spec.name) + + added = False + existing = False + for i, (name, speclist) in enumerate(self.read_specs.items()): + # Iterate over all named lists from an OrderedDict() + if name == list_name: + # We need to modify this list + # TODO: Add conditional which reimplements name-level checking + existing = str(spec) in speclist.yaml_list + if not existing: + speclist.add(str(spec)) + added = True + elif added: + # We've already modified a list, so all later lists need to + # have their references updated. + new_reference = dict((n, self.read_specs[n]) + for n in list(self.read_specs.keys())[:i]) + speclist.update_reference(new_reference) return bool(not existing) - def remove(self, query_spec, force=False): + def remove(self, query_spec, list_name='specs', force=False): """Remove specs from an environment that match a query_spec""" query_spec = Spec(query_spec) - # try abstract specs first - matches = [] - if not query_spec.concrete: - matches = [s for s in self.user_specs if s.satisfies(query_spec)] - - if not matches: - # concrete specs match against concrete specs in the env - specs_hashes = zip( - self.concretized_user_specs, self.concretized_order) - matches = [ - s for s, h in specs_hashes if query_spec.dag_hash() == h] - - if not matches: - raise SpackEnvironmentError("Not found: {0}".format(query_spec)) - - for spec in matches: - if spec in self.user_specs: - self.user_specs.remove(spec) - - if force and spec in self.concretized_user_specs: - i = self.concretized_user_specs.index(spec) - del self.concretized_user_specs[i] - - dag_hash = self.concretized_order[i] - del self.concretized_order[i] - del self.specs_by_hash[dag_hash] + removed = False + for i, (name, speclist) in enumerate(self.read_specs.items()): + # Iterate over all named lists from an OrderedDict() + if name == list_name: + # We need to modify this list + # try abstract specs first + matches = [] + + if not query_spec.concrete: + matches = [s for s in speclist if s.satisfies(query_spec)] + + if not matches: + # concrete specs match against concrete specs in the env + specs_hashes = zip( + self.concretized_user_specs, self.concretized_order) + matches = [ + s for s, h in specs_hashes + if query_spec.dag_hash() == h + ] + + if not matches: + raise SpackEnvironmentError( + "Not found: {0}".format(query_spec)) + + for spec in matches: + if spec in speclist: + speclist.remove(spec) + removed = True + + if force and spec in self.concretized_user_specs: + i = self.concretized_user_specs.index(spec) + del self.concretized_user_specs[i] + + dag_hash = self.concretized_order[i] + del self.concretized_order[i] + del self.specs_by_hash[dag_hash] + removed = True + + elif removed: + # We've already modified one list, so all later lists need + # their references updated. + new_reference = dict((n, self.read_specs[n]) + for n in list(self.read_specs.keys())[:i]) + speclist.update_reference(new_reference) def concretize(self, force=False): """Concretize user_specs in this environment. @@ -663,10 +776,11 @@ class Environment(object): self._add_concrete_spec(s, concrete, new=False) # concretize any new user specs that we haven't concretized yet - for uspec in self.user_specs: + for uspec, uspec_constraints in zip( + self.user_specs, self.user_specs.specs_as_constraints): if uspec not in old_concretized_user_specs: tty.msg('Concretizing %s' % uspec) - concrete = uspec.concretized() + concrete = _concretize_from_constraints(uspec_constraints) self._add_concrete_spec(uspec, concrete) # Display concretized spec to the user @@ -689,7 +803,8 @@ class Environment(object): self._add_concrete_spec(spec, concrete) else: # spec might be in the user_specs, but not installed. - spec = next(s for s in self.user_specs if s.name == spec.name) + # TODO: Redo name-based comparison for old style envs + spec = next(s for s in self.user_specs if s.satisfies(user_spec)) concrete = self.specs_by_hash.get(spec.dag_hash()) if not concrete: concrete = spec.concretized() @@ -1018,10 +1133,16 @@ class Environment(object): # invalidate _repo cache self._repo = None + # put any changes in the definitions in the YAML + named_speclists = list(self.read_specs.items()) + for i, (name, speclist) in enumerate(named_speclists[:-1]): + conf = config_dict(self.yaml) + yaml_list = conf.get('definitions', [])[i].setdefault(name, []) + yaml_list[:] = speclist.yaml_list + # put the new user specs in the YAML - yaml_dict = config_dict(self.yaml) - yaml_spec_list = yaml_dict.setdefault('specs', []) - yaml_spec_list[:] = [str(s) for s in self.user_specs] + yaml_spec_list = config_dict(self.yaml).setdefault('specs', []) + yaml_spec_list[:] = self.user_specs.yaml_list if self._view_path == self.default_view_path: view = True @@ -1057,6 +1178,48 @@ class Environment(object): activate(self._previous_active) +def _concretize_from_constraints(spec_constraints): + # Accept only valid constraints from list and concretize spec + # Get the named spec even if out of order + root_spec = [s for s in spec_constraints if s.name] + if len(root_spec) != 1: + m = 'The constraints %s are not a valid spec ' % spec_constraints + m += 'concretization target. all specs must have a single name ' + m += 'constraint for concretization.' + raise InvalidSpecConstraintError(m) + spec_constraints.remove(root_spec[0]) + + invalid_constraints = [] + while True: + # Attach all anonymous constraints to one named spec + s = root_spec[0].copy() + for c in spec_constraints: + if c not in invalid_constraints: + s.constrain(c) + try: + return s.concretized() + except spack.spec.InvalidDependencyError as e: + dep_index = e.message.index('depend on ') + len('depend on ') + invalid_msg = e.message[dep_index:] + invalid_deps_string = ['^' + d.strip(',') + for d in invalid_msg.split() + if d != 'or'] + invalid_deps = [c for c in spec_constraints + if any(c.satisfies(invd) + for invd in invalid_deps_string)] + if len(invalid_deps) != len(invalid_deps_string): + raise e + invalid_constraints.extend(invalid_deps) + except UnknownVariantError as e: + invalid_variants = re.findall(r"'(\w+)'", e.message) + invalid_deps = [c for c in spec_constraints + if any(name in c.variants + for name in invalid_variants)] + if len(invalid_deps) != len(invalid_variants): + raise e + invalid_constraints.extend(invalid_deps) + + def make_repo_path(root): """Make a RepoPath from the repo subdirectories in an environment.""" path = spack.repo.RepoPath() diff --git a/lib/spack/spack/schema/env.py b/lib/spack/spack/schema/env.py index 91931ec90c..3c5265bacb 100644 --- a/lib/spack/spack/schema/env.py +++ b/lib/spack/spack/schema/env.py @@ -11,8 +11,41 @@ from llnl.util.lang import union_dicts import spack.schema.merged +import spack.schema.projections +spec_list_schema = { + 'type': 'array', + 'default': [], + 'items': { + 'anyOf': [ + {'type': 'object', + 'additionalProperties': False, + 'properties': { + 'matrix': { + 'type': 'array', + 'items': { + 'type': 'array', + 'items': { + 'type': 'string', + } + } + }, + 'exclude': { + 'type': 'array', + 'items': { + 'type': 'string' + } + } + }}, + {'type': 'string'}, + {'type': 'null'} + ] + } +} + +projections_scheme = spack.schema.projections.properties['projections'] + schema = { '$schema': 'http://json-schema.org/schema#', 'title': 'Spack environment file schema', @@ -34,22 +67,52 @@ schema = { 'type': 'string' }, }, - 'specs': { - # Specs is a list of specs, which can have - # optional additional properties in a sub-dict + 'view': { + 'type': ['boolean', 'string'] + }, + 'definitions': { 'type': 'array', 'default': [], - 'additionalProperties': False, 'items': { - 'anyOf': [ - {'type': 'string'}, - {'type': 'null'}, - {'type': 'object'}, - ] + 'type': 'object', + 'properties': { + 'when': { + 'type': 'string' + } + }, + 'patternProperties': { + '^(?!when$)\w*': spec_list_schema + } } }, + 'specs': spec_list_schema, 'view': { - 'type': ['boolean', 'string'] + 'anyOf': [ + {'type': 'boolean'}, + {'type': 'string'}, + {'type': 'object', + 'required': ['root'], + 'additionalProperties': False, + 'properties': { + 'root': { + 'type': 'string' + }, + 'select': { + 'type': 'array', + 'items': { + 'type': 'string' + } + }, + 'exclude': { + 'type': 'array', + 'items': { + 'type': 'string' + } + }, + 'projections': projections_scheme + } + } + ] } } ) diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index d50c7e7fd7..8db4c765ca 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -2494,7 +2494,7 @@ class Spec(object): """Apply constraints of other spec's dependencies to this spec.""" other = self._autospec(other) - if not self._dependencies or not other._dependencies: + if not other._dependencies: return False # TODO: might want more detail than this, e.g. specific deps diff --git a/lib/spack/spack/spec_list.py b/lib/spack/spack/spec_list.py new file mode 100644 index 0000000000..d7a254195e --- /dev/null +++ b/lib/spack/spack/spec_list.py @@ -0,0 +1,168 @@ +# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +import itertools +from six import string_types + +from spack.spec import Spec +from spack.error import SpackError + + +def spec_ordering_key(s): + if s.startswith('^'): + return 5 + elif s.startswith('/'): + return 4 + elif s.startswith('%'): + return 3 + elif any(s.startswith(c) for c in '~-+@') or '=' in s: + return 2 + else: + return 1 + + +class SpecList(object): + + def __init__(self, name='specs', yaml_list=[], reference={}): + self.name = name + self._reference = reference # TODO: Do we need defensive copy here? + + self.yaml_list = yaml_list[:] + + # Expansions can be expensive to compute and difficult to keep updated + # We cache results and invalidate when self.yaml_list changes + self._expanded_list = None + self._constraints = None + self._specs = None + + @property + def specs_as_yaml_list(self): + if self._expanded_list is None: + self._expanded_list = self._expand_references(self.yaml_list) + return self._expanded_list + + @property + def specs_as_constraints(self): + if self._constraints is None: + constraints = [] + for item in self.specs_as_yaml_list: + if isinstance(item, dict): # matrix of specs + excludes = item.get('exclude', []) + for combo in itertools.product(*(item['matrix'])): + # Test against the excludes using a single spec + ordered_combo = sorted(combo, key=spec_ordering_key) + test_spec = Spec(' '.join(ordered_combo)) + if any(test_spec.satisfies(x) for x in excludes): + continue + + # Add as list of constraints + constraints.append([Spec(x) for x in ordered_combo]) + else: # individual spec + constraints.append([Spec(item)]) + self._constraints = constraints + + return self._constraints + + @property + def specs(self): + if self._specs is None: + specs = [] + # This could be slightly faster done directly from yaml_list, + # but this way is easier to maintain. + for constraint_list in self.specs_as_constraints: + spec = constraint_list[0].copy() + for const in constraint_list[1:]: + spec.constrain(const) + specs.append(spec) + self._specs = specs + + return self._specs + + def add(self, spec): + self.yaml_list.append(str(spec)) + + # expanded list can be updated without invalidation + if self._expanded_list is not None: + self._expanded_list.append(str(spec)) + + # Invalidate cache variables when we change the list + self._constraints = None + self._specs = None + + def remove(self, spec): + # Get spec to remove from list + remove = [s for s in self.yaml_list + if (isinstance(s, string_types) and not s.startswith('$')) + and Spec(s) == Spec(spec)] + if not remove: + msg = 'Cannot remove %s from SpecList %s\n' % (spec, self.name) + msg += 'Either %s is not in %s or %s is ' % (spec, self.name, spec) + msg += 'expanded from a matrix and cannot be removed directly.' + raise SpecListError(msg) + assert len(remove) == 1 + self.yaml_list.remove(remove[0]) + + # invalidate cache variables when we change the list + self._expanded_list = None + self._constraints = None + self._specs = None + + def extend(self, other, copy_reference=True): + self.yaml_list.extend(other.yaml_list) + self._expanded_list = None + self._constraints = None + self._specs = None + + if copy_reference: + self._reference = other._reference + + def update_reference(self, reference): + self._reference = reference + self._expanded_list = None + self._constraints = None + self._specs = None + + def _expand_references(self, yaml): + if isinstance(yaml, list): + for idx, item in enumerate(yaml): + if isinstance(item, string_types) and item.startswith('$'): + name = item[1:] + if name in self._reference: + ret = [self._expand_references(i) for i in yaml[:idx]] + ret += self._reference[name].specs_as_yaml_list + ret += [self._expand_references(i) + for i in yaml[idx + 1:]] + return ret + else: + msg = 'SpecList %s refers to ' % self.name + msg = 'named list %s ' % name + msg += 'which does not appear in its reference dict' + raise UndefinedReferenceError(msg) + # No references in this + return [self._expand_references(item) for item in yaml] + elif isinstance(yaml, dict): + # There can't be expansions in dicts + return dict((name, self._expand_references(val)) + for (name, val) in yaml.items()) + else: + # Strings are just returned + return yaml + + def __len__(self): + return len(self.specs) + + def __getitem__(self, key): + return self.specs[key] + + +class SpecListError(SpackError): + """Error class for all errors related to SpecList objects.""" + + +class UndefinedReferenceError(SpecListError): + """Error class for undefined references in Spack stacks.""" + + +class InvalidSpecConstraintError(SpecListError): + """Error class for invalid spec constraints at concretize time.""" diff --git a/lib/spack/spack/test/cmd/env.py b/lib/spack/spack/test/cmd/env.py index e8aa38375e..28bda0482c 100644 --- a/lib/spack/spack/test/cmd/env.py +++ b/lib/spack/spack/test/cmd/env.py @@ -15,6 +15,7 @@ import spack.environment as ev from spack.cmd.env import _env_create from spack.spec import Spec from spack.main import SpackCommand +from spack.spec_list import SpecListError # everything here uses the mock_env_path @@ -763,3 +764,271 @@ def test_env_activate_view_fails( """Sanity check on env activate to make sure it requires shell support""" out = env('activate', 'test') assert "To initialize spack's shell commands:" in out + + +def test_stack_yaml_definitions(tmpdir): + filename = str(tmpdir.join('spack.yaml')) + with open(filename, 'w') as f: + f.write("""\ +env: + definitions: + - packages: [mpileaks, callpath] + specs: + - $packages +""") + with tmpdir.as_cwd(): + env('create', 'test', './spack.yaml') + test = ev.read('test') + + assert Spec('mpileaks') in test.user_specs + assert Spec('callpath') in test.user_specs + + +def test_stack_yaml_add_to_list(tmpdir): + filename = str(tmpdir.join('spack.yaml')) + with open(filename, 'w') as f: + f.write("""\ +env: + definitions: + - packages: [mpileaks, callpath] + specs: + - $packages +""") + with tmpdir.as_cwd(): + env('create', 'test', './spack.yaml') + with ev.read('test'): + add('-l', 'packages', 'libelf') + + test = ev.read('test') + + assert Spec('libelf') in test.user_specs + assert Spec('mpileaks') in test.user_specs + assert Spec('callpath') in test.user_specs + + +def test_stack_yaml_remove_from_list(tmpdir): + filename = str(tmpdir.join('spack.yaml')) + with open(filename, 'w') as f: + f.write("""\ +env: + definitions: + - packages: [mpileaks, callpath] + specs: + - $packages +""") + with tmpdir.as_cwd(): + env('create', 'test', './spack.yaml') + with ev.read('test'): + remove('-l', 'packages', 'mpileaks') + + test = ev.read('test') + + assert Spec('mpileaks') not in test.user_specs + assert Spec('callpath') in test.user_specs + + +def test_stack_yaml_attempt_remove_from_matrix(tmpdir): + filename = str(tmpdir.join('spack.yaml')) + with open(filename, 'w') as f: + f.write("""\ +env: + definitions: + - packages: + - matrix: + - [mpileaks, callpath] + - [target=be] + specs: + - $packages +""") + with tmpdir.as_cwd(): + env('create', 'test', './spack.yaml') + with pytest.raises(SpecListError): + with ev.read('test'): + remove('-l', 'packages', 'mpileaks') + + +def test_stack_concretize_extraneous_deps(tmpdir, config, mock_packages): + filename = str(tmpdir.join('spack.yaml')) + with open(filename, 'w') as f: + f.write("""\ +env: + definitions: + - packages: [libelf, mpileaks] + - install: + - matrix: + - [$packages] + - ['^zmpi', '^mpich'] + specs: + - $install +""") + with tmpdir.as_cwd(): + env('create', 'test', './spack.yaml') + with ev.read('test'): + concretize() + + test = ev.read('test') + + for user, concrete in test.concretized_specs(): + assert concrete.concrete + assert not user.concrete + if user.name == 'libelf': + assert not concrete.satisfies('^mpi', strict=True) + elif user.name == 'mpileaks': + assert concrete.satisfies('^mpi', strict=True) + + +def test_stack_concretize_extraneous_variants(tmpdir, config, mock_packages): + filename = str(tmpdir.join('spack.yaml')) + with open(filename, 'w') as f: + f.write("""\ +env: + definitions: + - packages: [libelf, mpileaks] + - install: + - matrix: + - [$packages] + - ['~shared', '+shared'] + specs: + - $install +""") + with tmpdir.as_cwd(): + env('create', 'test', './spack.yaml') + with ev.read('test'): + concretize() + + test = ev.read('test') + + for user, concrete in test.concretized_specs(): + assert concrete.concrete + assert not user.concrete + if user.name == 'libelf': + assert 'shared' not in concrete.variants + if user.name == 'mpileaks': + assert (concrete.variants['shared'].value == + user.variants['shared'].value) + + +def test_stack_definition_extension(tmpdir): + filename = str(tmpdir.join('spack.yaml')) + with open(filename, 'w') as f: + f.write("""\ +env: + definitions: + - packages: [libelf, mpileaks] + - packages: [callpath] + specs: + - $packages +""") + with tmpdir.as_cwd(): + env('create', 'test', './spack.yaml') + + test = ev.read('test') + + assert Spec('libelf') in test.user_specs + assert Spec('mpileaks') in test.user_specs + assert Spec('callpath') in test.user_specs + + +def test_stack_definition_conditional_false(tmpdir): + filename = str(tmpdir.join('spack.yaml')) + with open(filename, 'w') as f: + f.write("""\ +env: + definitions: + - packages: [libelf, mpileaks] + - packages: [callpath] + when: 'False' + specs: + - $packages +""") + with tmpdir.as_cwd(): + env('create', 'test', './spack.yaml') + + test = ev.read('test') + + assert Spec('libelf') in test.user_specs + assert Spec('mpileaks') in test.user_specs + assert Spec('callpath') not in test.user_specs + + +def test_stack_definition_conditional_true(tmpdir): + filename = str(tmpdir.join('spack.yaml')) + with open(filename, 'w') as f: + f.write("""\ +env: + definitions: + - packages: [libelf, mpileaks] + - packages: [callpath] + when: 'True' + specs: + - $packages +""") + with tmpdir.as_cwd(): + env('create', 'test', './spack.yaml') + + test = ev.read('test') + + assert Spec('libelf') in test.user_specs + assert Spec('mpileaks') in test.user_specs + assert Spec('callpath') in test.user_specs + + +def test_stack_definition_conditional_with_variable(tmpdir): + filename = str(tmpdir.join('spack.yaml')) + with open(filename, 'w') as f: + f.write("""\ +env: + definitions: + - packages: [libelf, mpileaks] + - packages: [callpath] + when: platform == 'test' + specs: + - $packages +""") + with tmpdir.as_cwd(): + env('create', 'test', './spack.yaml') + + test = ev.read('test') + + assert Spec('libelf') in test.user_specs + assert Spec('mpileaks') in test.user_specs + assert Spec('callpath') in test.user_specs + + +def test_stack_definition_complex_conditional(tmpdir): + filename = str(tmpdir.join('spack.yaml')) + with open(filename, 'w') as f: + f.write("""\ +env: + definitions: + - packages: [libelf, mpileaks] + - packages: [callpath] + when: re.search(r'foo', hostname) and env['test'] == 'THISSHOULDBEFALSE' + specs: + - $packages +""") + with tmpdir.as_cwd(): + env('create', 'test', './spack.yaml') + + test = ev.read('test') + + assert Spec('libelf') in test.user_specs + assert Spec('mpileaks') in test.user_specs + assert Spec('callpath') not in test.user_specs + + +def test_stack_definition_conditional_invalid_variable(tmpdir): + filename = str(tmpdir.join('spack.yaml')) + with open(filename, 'w') as f: + f.write("""\ +env: + definitions: + - packages: [libelf, mpileaks] + - packages: [callpath] + when: bad_variable == 'test' + specs: + - $packages +""") + with tmpdir.as_cwd(): + with pytest.raises(NameError): + env('create', 'test', './spack.yaml') diff --git a/lib/spack/spack/test/spec_list.py b/lib/spack/spack/test/spec_list.py new file mode 100644 index 0000000000..9dbfc80a28 --- /dev/null +++ b/lib/spack/spack/test/spec_list.py @@ -0,0 +1,144 @@ +# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +from spack.spec_list import SpecList +from spack.spec import Spec + + +class TestSpecList(object): + default_input = ['mpileaks', '$mpis', + {'matrix': [['hypre'], ['$gccs', '%clang@3.3']]}, + 'libelf'] + + default_reference = {'gccs': SpecList('gccs', ['%gcc@4.5.0']), + 'mpis': SpecList('mpis', ['zmpi@1.0', 'mpich@3.0'])} + + default_expansion = ['mpileaks', 'zmpi@1.0', 'mpich@3.0', + {'matrix': [ + ['hypre'], + ['%gcc@4.5.0', '%clang@3.3'], + ]}, + 'libelf'] + + default_constraints = [[Spec('mpileaks')], + [Spec('zmpi@1.0')], + [Spec('mpich@3.0')], + [Spec('hypre'), Spec('%gcc@4.5.0')], + [Spec('hypre'), Spec('%clang@3.3')], + [Spec('libelf')]] + + default_specs = [Spec('mpileaks'), Spec('zmpi@1.0'), + Spec('mpich@3.0'), Spec('hypre%gcc@4.5.0'), + Spec('hypre%clang@3.3'), Spec('libelf')] + + def test_spec_list_expansions(self): + speclist = SpecList('specs', self.default_input, + self.default_reference) + + assert speclist.specs_as_yaml_list == self.default_expansion + assert speclist.specs_as_constraints == self.default_constraints + assert speclist.specs == self.default_specs + + def test_spec_list_constraint_ordering(self): + specs = [{'matrix': [ + ['^zmpi'], + ['%gcc@4.5.0'], + ['hypre', 'libelf'], + ['~shared'], + ['cflags=-O3', 'cflags="-g -O0"'], + ['^foo'] + ]}] + + speclist = SpecList('specs', specs) + + expected_specs = [ + Spec('hypre cflags=-O3 ~shared %gcc@4.5.0 ^foo ^zmpi'), + Spec('hypre cflags="-g -O0" ~shared %gcc@4.5.0 ^foo ^zmpi'), + Spec('libelf cflags=-O3 ~shared %gcc@4.5.0 ^foo ^zmpi'), + Spec('libelf cflags="-g -O0" ~shared %gcc@4.5.0 ^foo ^zmpi'), + ] + assert speclist.specs == expected_specs + + def test_spec_list_add(self): + speclist = SpecList('specs', self.default_input, + self.default_reference) + + assert speclist.specs_as_yaml_list == self.default_expansion + assert speclist.specs_as_constraints == self.default_constraints + assert speclist.specs == self.default_specs + + speclist.add('libdwarf') + + assert speclist.specs_as_yaml_list == self.default_expansion + [ + 'libdwarf'] + assert speclist.specs_as_constraints == self.default_constraints + [ + [Spec('libdwarf')]] + assert speclist.specs == self.default_specs + [Spec('libdwarf')] + + def test_spec_list_remove(self): + speclist = SpecList('specs', self.default_input, + self.default_reference) + + assert speclist.specs_as_yaml_list == self.default_expansion + assert speclist.specs_as_constraints == self.default_constraints + assert speclist.specs == self.default_specs + + speclist.remove('libelf') + + assert speclist.specs_as_yaml_list + [ + 'libelf' + ] == self.default_expansion + + assert speclist.specs_as_constraints + [ + [Spec('libelf')] + ] == self.default_constraints + + assert speclist.specs + [Spec('libelf')] == self.default_specs + + def test_spec_list_update_reference(self): + speclist = SpecList('specs', self.default_input, + self.default_reference) + + assert speclist.specs_as_yaml_list == self.default_expansion + assert speclist.specs_as_constraints == self.default_constraints + assert speclist.specs == self.default_specs + + new_mpis = SpecList('mpis', self.default_reference['mpis'].yaml_list) + new_mpis.add('mpich@3.3') + new_reference = self.default_reference.copy() + new_reference['mpis'] = new_mpis + + speclist.update_reference(new_reference) + + expansion = list(self.default_expansion) + expansion.insert(3, 'mpich@3.3') + constraints = list(self.default_constraints) + constraints.insert(3, [Spec('mpich@3.3')]) + specs = list(self.default_specs) + specs.insert(3, Spec('mpich@3.3')) + + assert speclist.specs_as_yaml_list == expansion + assert speclist.specs_as_constraints == constraints + assert speclist.specs == specs + + def test_spec_list_extension(self): + speclist = SpecList('specs', self.default_input, + self.default_reference) + + assert speclist.specs_as_yaml_list == self.default_expansion + assert speclist.specs_as_constraints == self.default_constraints + assert speclist.specs == self.default_specs + + new_ref = self.default_reference.copy() + otherlist = SpecList('specs', + ['zlib', {'matrix': [['callpath'], + ['%intel@18']]}], + new_ref) + + speclist.extend(otherlist) + + assert speclist.specs_as_yaml_list == (self.default_expansion + + otherlist.specs_as_yaml_list) + assert speclist.specs == self.default_specs + otherlist.specs + assert speclist._reference is new_ref -- cgit v1.2.3-60-g2f50