stacks: initial implementation of stacks on environments

- stack syntax in env schema
- switch environment specs over to SpecList object
- add stack functionality to environments
- handle definition extensions through stack.yaml and SpecList
- implement conditional definitions
- tests

9 files changed, 875 insertions, 61 deletions

diff --git a/lib/spack/spack/cmd/add.py b/lib/spack/spack/cmd/add.py
index e7a7e05b98..894161fe00 100644
--- a/lib/spack/spack/cmd/add.py
+++ b/lib/spack/spack/cmd/add.py
@@ -17,6 +17,9 @@ level = "long"
 
 
 def setup_parser(subparser):
+    subparser.add_argument('-l', '--list-name',
+                           dest='list_name', default='specs',
+                           help="name of the list to add specs to")
     subparser.add_argument(
         'specs', nargs=argparse.REMAINDER, help="specs of packages to add")
 
@@ -25,7 +28,7 @@ def add(parser, args):
     env = ev.get_env(args, 'add', required=True)
 
     for spec in spack.cmd.parse_specs(args.specs):
-        if not env.add(spec):
+        if not env.add(spec, args.list_name):
             tty.msg("Package {0} was already added to {1}"
                     .format(spec.name, env.name))
         else:
diff --git a/lib/spack/spack/cmd/find.py b/lib/spack/spack/cmd/find.py
index c86a7f9cd1..68d7a0f1d5 100644
--- a/lib/spack/spack/cmd/find.py
+++ b/lib/spack/spack/cmd/find.py
@@ -175,6 +175,7 @@ def find(parser, args):
             tty.msg('No root specs')
         else:
             tty.msg('Root specs')
+            # TODO: Change this to not print extraneous deps and variants
             display_specs(
                 env.user_specs, args,
                 decorator=lambda s, f: color.colorize('@*{%s}' % f))
diff --git a/lib/spack/spack/cmd/remove.py b/lib/spack/spack/cmd/remove.py
index 87942660e8..1d07830f03 100644
--- a/lib/spack/spack/cmd/remove.py
+++ b/lib/spack/spack/cmd/remove.py
@@ -20,6 +20,9 @@ def setup_parser(subparser):
     subparser.add_argument(
         '-a', '--all', action='store_true',
         help="remove all specs from (clear) the environment")
+    subparser.add_argument('-l', '--list-name',
+                           dest='list_name', default='specs',
+                           help="name of the list to remove specs from")
     subparser.add_argument(
         '-f', '--force', action='store_true',
         help="remove concretized spec (if any) immediately")
@@ -35,5 +38,5 @@ def remove(parser, args):
     else:
         for spec in spack.cmd.parse_specs(args.specs):
             tty.msg('Removing %s from environment %s' % (spec, env.name))
-            env.remove(spec, force=args.force)
+            env.remove(spec, args.list_name, force=args.force)
     env.write()
diff --git a/lib/spack/spack/environment.py b/lib/spack/spack/environment.py
index 02b0a6d93c..abc4c63e6b 100644
--- a/lib/spack/spack/environment.py
+++ b/lib/spack/spack/environment.py
@@ -7,10 +7,13 @@ import os
 import re
 import sys
 import shutil
+import copy
 
 import ruamel.yaml
 import six
 
+from ordereddict_backport import OrderedDict
+
 import llnl.util.filesystem as fs
 import llnl.util.tty as tty
 from llnl.util.tty.color import colorize
@@ -21,10 +24,13 @@ import spack.schema.env
 import spack.spec
 import spack.util.spack_json as sjson
 import spack.config
-from spack.spec import Spec
 from spack.filesystem_view import YamlFilesystemView
-
 from spack.util.environment import EnvironmentModifications
+import spack.architecture as architecture
+from spack.spec import Spec
+from spack.spec_list import SpecList, InvalidSpecConstraintError
+from spack.variant import UnknownVariantError
+from spack.util.executable import which
 
 #: environment variable used to indicate the active environment
 spack_env_var = 'SPACK_ENV'
@@ -385,6 +391,29 @@ def _write_yaml(data, str_or_file):
                      default_flow_style=False)
 
 
+def _eval_conditional(string):
+    """Evaluate conditional definitions using restricted variable scope."""
+    arch = architecture.Arch(
+        architecture.platform(), 'default_os', 'default_target')
+    valid_variables = {
+        'target': str(arch.target),
+        'os': str(arch.platform_os),
+        'platform': str(arch.platform),
+        'arch': str(arch),
+        'architecture': str(arch),
+        're': re,
+        'env': os.environ,
+    }
+    hostname_bin = which('hostname')
+    if hostname_bin:
+        hostname = str(hostname_bin(output=str, error=str)).strip()
+        valid_variables['hostname'] = hostname
+    else:
+        tty.warn("Spack was unable to find the executable `hostname`"
+                 " hostname will be unavailable in conditionals")
+    return eval(string, valid_variables)
+
+
 class Environment(object):
     def __init__(self, path, init_file=None, with_view=None):
         """Create a new environment.
@@ -425,6 +454,18 @@ class Environment(object):
                 if default_manifest:
                     self._set_user_specs_from_lockfile()
 
+            if os.path.exists(self.manifest_path):
+                # read the spack.yaml file, if exists
+                with open(self.manifest_path) as f:
+                    self._read_manifest(f)
+            elif self.concretized_user_specs:
+                # if not, take user specs from the lockfile
+                self._set_user_specs_from_lockfile()
+                self.yaml = _read_yaml(default_manifest_yaml)
+            else:
+                # if there's no manifest or lockfile, use the default
+                self._read_manifest(default_manifest_yaml)
+
         if with_view is False:
             self._view_path = None
         elif isinstance(with_view, six.string_types):
@@ -435,9 +476,38 @@ class Environment(object):
     def _read_manifest(self, f):
         """Read manifest file and set up user specs."""
         self.yaml = _read_yaml(f)
+
+        self.read_specs = OrderedDict()
+
+        for item in list(self.yaml.values())[0].get('definitions', []):
+            entry = copy.deepcopy(item)
+            when = _eval_conditional(entry.pop('when', 'True'))
+            assert len(entry) == 1
+            if when:
+                name, spec_list = list(entry.items())[0]
+                user_specs = SpecList(name, spec_list, self.read_specs.copy())
+                if name in self.read_specs:
+                    self.read_specs[name].extend(user_specs)
+                else:
+                    self.read_specs[name] = user_specs
+
         spec_list = config_dict(self.yaml).get('specs')
-        if spec_list:
-            self.user_specs = [Spec(s) for s in spec_list if s]
+        user_specs = SpecList('specs', [s for s in spec_list if s],
+                              self.read_specs.copy())
+        self.read_specs['specs'] = user_specs
+
+        enable_view = config_dict(self.yaml).get('view')
+        # enable_view can be boolean, string, or None
+        if enable_view is True or enable_view is None:
+            self._view_path = self.default_view_path
+        elif isinstance(enable_view, six.string_types):
+            self._view_path = enable_view
+        else:
+            self._view_path = None
+
+    @property
+    def user_specs(self):
+        return self.read_specs['specs']
 
         enable_view = config_dict(self.yaml).get('view')
         # enable_view can be true/false, a string, or None (if the manifest did
@@ -452,10 +522,14 @@ class Environment(object):
 
     def _set_user_specs_from_lockfile(self):
         """Copy user_specs from a read-in lockfile."""
-        self.user_specs = [Spec(s) for s in self.concretized_user_specs]
+        self.read_specs = {
+            'specs': SpecList(
+                'specs', [Spec(s) for s in self.concretized_user_specs]
+            )
+        }
 
     def clear(self):
-        self.user_specs = []              # current user specs
+        self.read_specs = {'specs': SpecList()}      # specs read from yaml
         self.concretized_user_specs = []  # user specs from last concretize
         self.concretized_order = []       # roots of last concretize, in order
         self.specs_by_hash = {}           # concretized specs by hash
@@ -578,7 +652,7 @@ class Environment(object):
         """Remove this environment from Spack entirely."""
         shutil.rmtree(self.path)
 
-    def add(self, user_spec):
+    def add(self, user_spec, list_name='specs'):
         """Add a single user_spec (non-concretized) to the Environment
 
         Returns:
@@ -587,47 +661,86 @@ class Environment(object):
 
         """
         spec = Spec(user_spec)
-        if not spec.name:
-            raise SpackEnvironmentError(
-                'cannot add anonymous specs to an environment!')
-        elif not spack.repo.path.exists(spec.name):
-            raise SpackEnvironmentError('no such package: %s' % spec.name)
 
-        existing = set(s for s in self.user_specs if s.name == spec.name)
-        if not existing:
-            self.user_specs.append(spec)
+        if list_name not in self.read_specs:
+            raise SpackEnvironmentError(
+                'No list %s exists in environment %s' % (list_name, self.name)
+            )
+
+        if list_name == 'specs':
+            if not spec.name:
+                raise SpackEnvironmentError(
+                    'cannot add anonymous specs to an environment!')
+            elif not spack.repo.path.exists(spec.name):
+                raise SpackEnvironmentError('no such package: %s' % spec.name)
+
+        added = False
+        existing = False
+        for i, (name, speclist) in enumerate(self.read_specs.items()):
+            # Iterate over all named lists from an OrderedDict()
+            if name == list_name:
+                # We need to modify this list
+                # TODO: Add conditional which reimplements name-level checking
+                existing = str(spec) in speclist.yaml_list
+                if not existing:
+                    speclist.add(str(spec))
+                    added = True
+            elif added:
+                # We've already modified a list, so all later lists need to
+                # have their references updated.
+                new_reference = dict((n, self.read_specs[n])
+                                     for n in list(self.read_specs.keys())[:i])
+                speclist.update_reference(new_reference)
         return bool(not existing)
 
-    def remove(self, query_spec, force=False):
+    def remove(self, query_spec, list_name='specs', force=False):
         """Remove specs from an environment that match a query_spec"""
         query_spec = Spec(query_spec)
 
-        # try abstract specs first
-        matches = []
-        if not query_spec.concrete:
-            matches = [s for s in self.user_specs if s.satisfies(query_spec)]
-
-        if not matches:
-            # concrete specs match against concrete specs in the env
-            specs_hashes = zip(
-                self.concretized_user_specs, self.concretized_order)
-            matches = [
-                s for s, h in specs_hashes if query_spec.dag_hash() == h]
-
-        if not matches:
-            raise SpackEnvironmentError("Not found: {0}".format(query_spec))
-
-        for spec in matches:
-            if spec in self.user_specs:
-                self.user_specs.remove(spec)
-
-            if force and spec in self.concretized_user_specs:
-                i = self.concretized_user_specs.index(spec)
-                del self.concretized_user_specs[i]
-
-                dag_hash = self.concretized_order[i]
-                del self.concretized_order[i]
-                del self.specs_by_hash[dag_hash]
+        removed = False
+        for i, (name, speclist) in enumerate(self.read_specs.items()):
+            # Iterate over all named lists from an OrderedDict()
+            if name == list_name:
+                # We need to modify this list
+                # try abstract specs first
+                matches = []
+
+                if not query_spec.concrete:
+                    matches = [s for s in speclist if s.satisfies(query_spec)]
+
+                if not matches:
+                    # concrete specs match against concrete specs in the env
+                    specs_hashes = zip(
+                        self.concretized_user_specs, self.concretized_order)
+                    matches = [
+                        s for s, h in specs_hashes
+                        if query_spec.dag_hash() == h
+                    ]
+
+                if not matches:
+                    raise SpackEnvironmentError(
+                        "Not found: {0}".format(query_spec))
+
+                for spec in matches:
+                    if spec in speclist:
+                        speclist.remove(spec)
+                        removed = True
+
+                    if force and spec in self.concretized_user_specs:
+                        i = self.concretized_user_specs.index(spec)
+                        del self.concretized_user_specs[i]
+
+                        dag_hash = self.concretized_order[i]
+                        del self.concretized_order[i]
+                        del self.specs_by_hash[dag_hash]
+                        removed = True
+
+            elif removed:
+                # We've already modified one list, so all later lists need
+                # their references updated.
+                new_reference = dict((n, self.read_specs[n])
+                                     for n in list(self.read_specs.keys())[:i])
+                speclist.update_reference(new_reference)
 
     def concretize(self, force=False):
         """Concretize user_specs in this environment.
@@ -663,10 +776,11 @@ class Environment(object):
                 self._add_concrete_spec(s, concrete, new=False)
 
         # concretize any new user specs that we haven't concretized yet
-        for uspec in self.user_specs:
+        for uspec, uspec_constraints in zip(
+                self.user_specs, self.user_specs.specs_as_constraints):
             if uspec not in old_concretized_user_specs:
                 tty.msg('Concretizing %s' % uspec)
-                concrete = uspec.concretized()
+                concrete = _concretize_from_constraints(uspec_constraints)
                 self._add_concrete_spec(uspec, concrete)
 
                 # Display concretized spec to the user
@@ -689,7 +803,8 @@ class Environment(object):
             self._add_concrete_spec(spec, concrete)
         else:
             # spec might be in the user_specs, but not installed.
-            spec = next(s for s in self.user_specs if s.name == spec.name)
+            # TODO: Redo name-based comparison for old style envs
+            spec = next(s for s in self.user_specs if s.satisfies(user_spec))
             concrete = self.specs_by_hash.get(spec.dag_hash())
             if not concrete:
                 concrete = spec.concretized()
@@ -1018,10 +1133,16 @@ class Environment(object):
         # invalidate _repo cache
         self._repo = None
 
+        # put any changes in the definitions in the YAML
+        named_speclists = list(self.read_specs.items())
+        for i, (name, speclist) in enumerate(named_speclists[:-1]):
+            conf = config_dict(self.yaml)
+            yaml_list = conf.get('definitions', [])[i].setdefault(name, [])
+            yaml_list[:] = speclist.yaml_list
+
         # put the new user specs in the YAML
-        yaml_dict = config_dict(self.yaml)
-        yaml_spec_list = yaml_dict.setdefault('specs', [])
-        yaml_spec_list[:] = [str(s) for s in self.user_specs]
+        yaml_spec_list = config_dict(self.yaml).setdefault('specs', [])
+        yaml_spec_list[:] = self.user_specs.yaml_list
 
         if self._view_path == self.default_view_path:
             view = True
@@ -1057,6 +1178,48 @@ class Environment(object):
             activate(self._previous_active)
 
 
+def _concretize_from_constraints(spec_constraints):
+    # Accept only valid constraints from list and concretize spec
+    # Get the named spec even if out of order
+    root_spec = [s for s in spec_constraints if s.name]
+    if len(root_spec) != 1:
+        m = 'The constraints %s are not a valid spec ' % spec_constraints
+        m += 'concretization target. all specs must have a single name '
+        m += 'constraint for concretization.'
+        raise InvalidSpecConstraintError(m)
+    spec_constraints.remove(root_spec[0])
+
+    invalid_constraints = []
+    while True:
+        # Attach all anonymous constraints to one named spec
+        s = root_spec[0].copy()
+        for c in spec_constraints:
+            if c not in invalid_constraints:
+                s.constrain(c)
+        try:
+            return s.concretized()
+        except spack.spec.InvalidDependencyError as e:
+            dep_index = e.message.index('depend on ') + len('depend on ')
+            invalid_msg = e.message[dep_index:]
+            invalid_deps_string = ['^' + d.strip(',')
+                                   for d in invalid_msg.split()
+                                   if d != 'or']
+            invalid_deps = [c for c in spec_constraints
+                            if any(c.satisfies(invd)
+                                   for invd in invalid_deps_string)]
+            if len(invalid_deps) != len(invalid_deps_string):
+                raise e
+            invalid_constraints.extend(invalid_deps)
+        except UnknownVariantError as e:
+            invalid_variants = re.findall(r"'(\w+)'", e.message)
+            invalid_deps = [c for c in spec_constraints
+                            if any(name in c.variants
+                                   for name in invalid_variants)]
+            if len(invalid_deps) != len(invalid_variants):
+                raise e
+            invalid_constraints.extend(invalid_deps)
+
+
 def make_repo_path(root):
     """Make a RepoPath from the repo subdirectories in an environment."""
     path = spack.repo.RepoPath()
diff --git a/lib/spack/spack/schema/env.py b/lib/spack/spack/schema/env.py
index 91931ec90c..3c5265bacb 100644
--- a/lib/spack/spack/schema/env.py
+++ b/lib/spack/spack/schema/env.py
@@ -11,8 +11,41 @@
 from llnl.util.lang import union_dicts
 
 import spack.schema.merged
+import spack.schema.projections
 
 
+spec_list_schema = {
+    'type': 'array',
+    'default': [],
+    'items': {
+        'anyOf': [
+            {'type': 'object',
+             'additionalProperties': False,
+             'properties': {
+                 'matrix': {
+                     'type': 'array',
+                     'items': {
+                         'type': 'array',
+                         'items': {
+                             'type': 'string',
+                         }
+                     }
+                 },
+                 'exclude': {
+                     'type': 'array',
+                     'items': {
+                         'type': 'string'
+                     }
+                 }
+             }},
+            {'type': 'string'},
+            {'type': 'null'}
+        ]
+    }
+}
+
+projections_scheme = spack.schema.projections.properties['projections']
+
 schema = {
     '$schema': 'http://json-schema.org/schema#',
     'title': 'Spack environment file schema',
@@ -34,22 +67,52 @@ schema = {
                             'type': 'string'
                         },
                     },
-                    'specs': {
-                        # Specs is a list of specs, which can have
-                        # optional additional properties in a sub-dict
+                    'view': {
+                        'type': ['boolean', 'string']
+                    },
+                    'definitions': {
                         'type': 'array',
                         'default': [],
-                        'additionalProperties': False,
                         'items': {
-                            'anyOf': [
-                                {'type': 'string'},
-                                {'type': 'null'},
-                                {'type': 'object'},
-                            ]
+                            'type': 'object',
+                            'properties': {
+                                'when': {
+                                    'type': 'string'
+                                }
+                            },
+                            'patternProperties': {
+                                '^(?!when$)\w*': spec_list_schema
+                            }
                         }
                     },
+                    'specs': spec_list_schema,
                     'view': {
-                        'type': ['boolean', 'string']
+                        'anyOf': [
+                            {'type': 'boolean'},
+                            {'type': 'string'},
+                            {'type': 'object',
+                             'required': ['root'],
+                             'additionalProperties': False,
+                             'properties': {
+                                 'root': {
+                                     'type': 'string'
+                                 },
+                                 'select': {
+                                     'type': 'array',
+                                     'items': {
+                                         'type': 'string'
+                                     }
+                                 },
+                                 'exclude': {
+                                     'type': 'array',
+                                     'items': {
+                                         'type': 'string'
+                                     }
+                                 },
+                                 'projections': projections_scheme
+                             }
+                            }
+                        ]
                     }
                 }
             )
diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py
index d50c7e7fd7..8db4c765ca 100644
--- a/lib/spack/spack/spec.py
+++ b/lib/spack/spack/spec.py
@@ -2494,7 +2494,7 @@ class Spec(object):
         """Apply constraints of other spec's dependencies to this spec."""
         other = self._autospec(other)
 
-        if not self._dependencies or not other._dependencies:
+        if not other._dependencies:
             return False
 
         # TODO: might want more detail than this, e.g. specific deps
diff --git a/lib/spack/spack/spec_list.py b/lib/spack/spack/spec_list.py
new file mode 100644
index 0000000000..d7a254195e
--- /dev/null
+++ b/lib/spack/spack/spec_list.py
@@ -0,0 +1,168 @@
+# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+import itertools
+from six import string_types
+
+from spack.spec import Spec
+from spack.error import SpackError
+
+
+def spec_ordering_key(s):
+    if s.startswith('^'):
+        return 5
+    elif s.startswith('/'):
+        return 4
+    elif s.startswith('%'):
+        return 3
+    elif any(s.startswith(c) for c in '~-+@') or '=' in s:
+        return 2
+    else:
+        return 1
+
+
+class SpecList(object):
+
+    def __init__(self, name='specs', yaml_list=[], reference={}):
+        self.name = name
+        self._reference = reference  # TODO: Do we need defensive copy here?
+
+        self.yaml_list = yaml_list[:]
+
+        # Expansions can be expensive to compute and difficult to keep updated
+        # We cache results and invalidate when self.yaml_list changes
+        self._expanded_list = None
+        self._constraints = None
+        self._specs = None
+
+    @property
+    def specs_as_yaml_list(self):
+        if self._expanded_list is None:
+            self._expanded_list = self._expand_references(self.yaml_list)
+        return self._expanded_list
+
+    @property
+    def specs_as_constraints(self):
+        if self._constraints is None:
+            constraints = []
+            for item in self.specs_as_yaml_list:
+                if isinstance(item, dict):  # matrix of specs
+                    excludes = item.get('exclude', [])
+                    for combo in itertools.product(*(item['matrix'])):
+                        # Test against the excludes using a single spec
+                        ordered_combo = sorted(combo, key=spec_ordering_key)
+                        test_spec = Spec(' '.join(ordered_combo))
+                        if any(test_spec.satisfies(x) for x in excludes):
+                            continue
+
+                        # Add as list of constraints
+                        constraints.append([Spec(x) for x in ordered_combo])
+                else:  # individual spec
+                    constraints.append([Spec(item)])
+            self._constraints = constraints
+
+        return self._constraints
+
+    @property
+    def specs(self):
+        if self._specs is None:
+            specs = []
+            # This could be slightly faster done directly from yaml_list,
+            # but this way is easier to maintain.
+            for constraint_list in self.specs_as_constraints:
+                spec = constraint_list[0].copy()
+                for const in constraint_list[1:]:
+                    spec.constrain(const)
+                specs.append(spec)
+            self._specs = specs
+
+        return self._specs
+
+    def add(self, spec):
+        self.yaml_list.append(str(spec))
+
+        # expanded list can be updated without invalidation
+        if self._expanded_list is not None:
+            self._expanded_list.append(str(spec))
+
+        # Invalidate cache variables when we change the list
+        self._constraints = None
+        self._specs = None
+
+    def remove(self, spec):
+        # Get spec to remove from list
+        remove = [s for s in self.yaml_list
+                  if (isinstance(s, string_types) and not s.startswith('$'))
+                  and Spec(s) == Spec(spec)]
+        if not remove:
+            msg = 'Cannot remove %s from SpecList %s\n' % (spec, self.name)
+            msg += 'Either %s is not in %s or %s is ' % (spec, self.name, spec)
+            msg += 'expanded from a matrix and cannot be removed directly.'
+            raise SpecListError(msg)
+        assert len(remove) == 1
+        self.yaml_list.remove(remove[0])
+
+        # invalidate cache variables when we change the list
+        self._expanded_list = None
+        self._constraints = None
+        self._specs = None
+
+    def extend(self, other, copy_reference=True):
+        self.yaml_list.extend(other.yaml_list)
+        self._expanded_list = None
+        self._constraints = None
+        self._specs = None
+
+        if copy_reference:
+            self._reference = other._reference
+
+    def update_reference(self, reference):
+        self._reference = reference
+        self._expanded_list = None
+        self._constraints = None
+        self._specs = None
+
+    def _expand_references(self, yaml):
+        if isinstance(yaml, list):
+            for idx, item in enumerate(yaml):
+                if isinstance(item, string_types) and item.startswith('$'):
+                    name = item[1:]
+                    if name in self._reference:
+                        ret = [self._expand_references(i) for i in yaml[:idx]]
+                        ret += self._reference[name].specs_as_yaml_list
+                        ret += [self._expand_references(i)
+                                for i in yaml[idx + 1:]]
+                        return ret
+                    else:
+                        msg = 'SpecList %s refers to ' % self.name
+                        msg = 'named list %s ' % name
+                        msg += 'which does not appear in its reference dict'
+                        raise UndefinedReferenceError(msg)
+            # No references in this
+            return [self._expand_references(item) for item in yaml]
+        elif isinstance(yaml, dict):
+            # There can't be expansions in dicts
+            return dict((name, self._expand_references(val))
+                        for (name, val) in yaml.items())
+        else:
+            # Strings are just returned
+            return yaml
+
+    def __len__(self):
+        return len(self.specs)
+
+    def __getitem__(self, key):
+        return self.specs[key]
+
+
+class SpecListError(SpackError):
+    """Error class for all errors related to SpecList objects."""
+
+
+class UndefinedReferenceError(SpecListError):
+    """Error class for undefined references in Spack stacks."""
+
+
+class InvalidSpecConstraintError(SpecListError):
+    """Error class for invalid spec constraints at concretize time."""
diff --git a/lib/spack/spack/test/cmd/env.py b/lib/spack/spack/test/cmd/env.py
index e8aa38375e..28bda0482c 100644
--- a/lib/spack/spack/test/cmd/env.py
+++ b/lib/spack/spack/test/cmd/env.py
@@ -15,6 +15,7 @@ import spack.environment as ev
 from spack.cmd.env import _env_create
 from spack.spec import Spec
 from spack.main import SpackCommand
+from spack.spec_list import SpecListError
 
 
 # everything here uses the mock_env_path
@@ -763,3 +764,271 @@ def test_env_activate_view_fails(
     """Sanity check on env activate to make sure it requires shell support"""
     out = env('activate', 'test')
     assert "To initialize spack's shell commands:" in out
+
+
+def test_stack_yaml_definitions(tmpdir):
+    filename = str(tmpdir.join('spack.yaml'))
+    with open(filename, 'w') as f:
+        f.write("""\
+env:
+  definitions:
+    - packages: [mpileaks, callpath]
+  specs:
+    - $packages
+""")
+    with tmpdir.as_cwd():
+        env('create', 'test', './spack.yaml')
+        test = ev.read('test')
+
+        assert Spec('mpileaks') in test.user_specs
+        assert Spec('callpath') in test.user_specs
+
+
+def test_stack_yaml_add_to_list(tmpdir):
+    filename = str(tmpdir.join('spack.yaml'))
+    with open(filename, 'w') as f:
+        f.write("""\
+env:
+  definitions:
+    - packages: [mpileaks, callpath]
+  specs:
+    - $packages
+""")
+    with tmpdir.as_cwd():
+        env('create', 'test', './spack.yaml')
+        with ev.read('test'):
+            add('-l', 'packages', 'libelf')
+
+        test = ev.read('test')
+
+        assert Spec('libelf') in test.user_specs
+        assert Spec('mpileaks') in test.user_specs
+        assert Spec('callpath') in test.user_specs
+
+
+def test_stack_yaml_remove_from_list(tmpdir):
+    filename = str(tmpdir.join('spack.yaml'))
+    with open(filename, 'w') as f:
+        f.write("""\
+env:
+  definitions:
+    - packages: [mpileaks, callpath]
+  specs:
+    - $packages
+""")
+    with tmpdir.as_cwd():
+        env('create', 'test', './spack.yaml')
+        with ev.read('test'):
+            remove('-l', 'packages', 'mpileaks')
+
+        test = ev.read('test')
+
+        assert Spec('mpileaks') not in test.user_specs
+        assert Spec('callpath') in test.user_specs
+
+
+def test_stack_yaml_attempt_remove_from_matrix(tmpdir):
+    filename = str(tmpdir.join('spack.yaml'))
+    with open(filename, 'w') as f:
+        f.write("""\
+env:
+  definitions:
+    - packages:
+        - matrix:
+            - [mpileaks, callpath]
+            - [target=be]
+  specs:
+    - $packages
+""")
+    with tmpdir.as_cwd():
+        env('create', 'test', './spack.yaml')
+        with pytest.raises(SpecListError):
+            with ev.read('test'):
+                remove('-l', 'packages', 'mpileaks')
+
+
+def test_stack_concretize_extraneous_deps(tmpdir, config, mock_packages):
+    filename = str(tmpdir.join('spack.yaml'))
+    with open(filename, 'w') as f:
+        f.write("""\
+env:
+  definitions:
+    - packages: [libelf, mpileaks]
+    - install:
+        - matrix:
+            - [$packages]
+            - ['^zmpi', '^mpich']
+  specs:
+    - $install
+""")
+    with tmpdir.as_cwd():
+        env('create', 'test', './spack.yaml')
+        with ev.read('test'):
+            concretize()
+
+        test = ev.read('test')
+
+        for user, concrete in test.concretized_specs():
+            assert concrete.concrete
+            assert not user.concrete
+            if user.name == 'libelf':
+                assert not concrete.satisfies('^mpi', strict=True)
+            elif user.name == 'mpileaks':
+                assert concrete.satisfies('^mpi', strict=True)
+
+
+def test_stack_concretize_extraneous_variants(tmpdir, config, mock_packages):
+    filename = str(tmpdir.join('spack.yaml'))
+    with open(filename, 'w') as f:
+        f.write("""\
+env:
+  definitions:
+    - packages: [libelf, mpileaks]
+    - install:
+        - matrix:
+            - [$packages]
+            - ['~shared', '+shared']
+  specs:
+    - $install
+""")
+    with tmpdir.as_cwd():
+        env('create', 'test', './spack.yaml')
+        with ev.read('test'):
+            concretize()
+
+        test = ev.read('test')
+
+        for user, concrete in test.concretized_specs():
+            assert concrete.concrete
+            assert not user.concrete
+            if user.name == 'libelf':
+                assert 'shared' not in concrete.variants
+            if user.name  == 'mpileaks':
+                assert (concrete.variants['shared'].value ==
+                        user.variants['shared'].value)
+
+
+def test_stack_definition_extension(tmpdir):
+    filename = str(tmpdir.join('spack.yaml'))
+    with open(filename, 'w') as f:
+        f.write("""\
+env:
+  definitions:
+    - packages: [libelf, mpileaks]
+    - packages: [callpath]
+  specs:
+    - $packages
+""")
+    with tmpdir.as_cwd():
+        env('create', 'test', './spack.yaml')
+
+        test = ev.read('test')
+
+        assert Spec('libelf') in test.user_specs
+        assert Spec('mpileaks') in test.user_specs
+        assert Spec('callpath') in test.user_specs
+
+
+def test_stack_definition_conditional_false(tmpdir):
+    filename = str(tmpdir.join('spack.yaml'))
+    with open(filename, 'w') as f:
+        f.write("""\
+env:
+  definitions:
+    - packages: [libelf, mpileaks]
+    - packages: [callpath]
+      when: 'False'
+  specs:
+    - $packages
+""")
+    with tmpdir.as_cwd():
+        env('create', 'test', './spack.yaml')
+
+        test = ev.read('test')
+
+        assert Spec('libelf') in test.user_specs
+        assert Spec('mpileaks') in test.user_specs
+        assert Spec('callpath') not in test.user_specs
+
+
+def test_stack_definition_conditional_true(tmpdir):
+    filename = str(tmpdir.join('spack.yaml'))
+    with open(filename, 'w') as f:
+        f.write("""\
+env:
+  definitions:
+    - packages: [libelf, mpileaks]
+    - packages: [callpath]
+      when: 'True'
+  specs:
+    - $packages
+""")
+    with tmpdir.as_cwd():
+        env('create', 'test', './spack.yaml')
+
+        test = ev.read('test')
+
+        assert Spec('libelf') in test.user_specs
+        assert Spec('mpileaks') in test.user_specs
+        assert Spec('callpath') in test.user_specs
+
+
+def test_stack_definition_conditional_with_variable(tmpdir):
+    filename = str(tmpdir.join('spack.yaml'))
+    with open(filename, 'w') as f:
+        f.write("""\
+env:
+  definitions:
+    - packages: [libelf, mpileaks]
+    - packages: [callpath]
+      when: platform == 'test'
+  specs:
+    - $packages
+""")
+    with tmpdir.as_cwd():
+        env('create', 'test', './spack.yaml')
+
+        test = ev.read('test')
+
+        assert Spec('libelf') in test.user_specs
+        assert Spec('mpileaks') in test.user_specs
+        assert Spec('callpath') in test.user_specs
+
+
+def test_stack_definition_complex_conditional(tmpdir):
+    filename = str(tmpdir.join('spack.yaml'))
+    with open(filename, 'w') as f:
+        f.write("""\
+env:
+  definitions:
+    - packages: [libelf, mpileaks]
+    - packages: [callpath]
+      when: re.search(r'foo', hostname) and env['test'] == 'THISSHOULDBEFALSE'
+  specs:
+    - $packages
+""")
+    with tmpdir.as_cwd():
+        env('create', 'test', './spack.yaml')
+
+        test = ev.read('test')
+
+        assert Spec('libelf') in test.user_specs
+        assert Spec('mpileaks') in test.user_specs
+        assert Spec('callpath') not in test.user_specs
+
+
+def test_stack_definition_conditional_invalid_variable(tmpdir):
+    filename = str(tmpdir.join('spack.yaml'))
+    with open(filename, 'w') as f:
+        f.write("""\
+env:
+  definitions:
+    - packages: [libelf, mpileaks]
+    - packages: [callpath]
+      when: bad_variable == 'test'
+  specs:
+    - $packages
+""")
+    with tmpdir.as_cwd():
+        with pytest.raises(NameError):
+            env('create', 'test', './spack.yaml')
diff --git a/lib/spack/spack/test/spec_list.py b/lib/spack/spack/test/spec_list.py
new file mode 100644
index 0000000000..9dbfc80a28
--- /dev/null
+++ b/lib/spack/spack/test/spec_list.py
@@ -0,0 +1,144 @@
+# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+from spack.spec_list import SpecList
+from spack.spec import Spec
+
+
+class TestSpecList(object):
+    default_input = ['mpileaks', '$mpis',
+                     {'matrix': [['hypre'], ['$gccs', '%clang@3.3']]},
+                     'libelf']
+
+    default_reference = {'gccs': SpecList('gccs', ['%gcc@4.5.0']),
+                         'mpis': SpecList('mpis', ['zmpi@1.0', 'mpich@3.0'])}
+
+    default_expansion = ['mpileaks', 'zmpi@1.0', 'mpich@3.0',
+                         {'matrix': [
+                             ['hypre'],
+                             ['%gcc@4.5.0', '%clang@3.3'],
+                         ]},
+                         'libelf']
+
+    default_constraints = [[Spec('mpileaks')],
+                           [Spec('zmpi@1.0')],
+                           [Spec('mpich@3.0')],
+                           [Spec('hypre'), Spec('%gcc@4.5.0')],
+                           [Spec('hypre'), Spec('%clang@3.3')],
+                           [Spec('libelf')]]
+
+    default_specs = [Spec('mpileaks'), Spec('zmpi@1.0'),
+                     Spec('mpich@3.0'), Spec('hypre%gcc@4.5.0'),
+                     Spec('hypre%clang@3.3'), Spec('libelf')]
+
+    def test_spec_list_expansions(self):
+        speclist = SpecList('specs', self.default_input,
+                            self.default_reference)
+
+        assert speclist.specs_as_yaml_list == self.default_expansion
+        assert speclist.specs_as_constraints == self.default_constraints
+        assert speclist.specs == self.default_specs
+
+    def test_spec_list_constraint_ordering(self):
+        specs = [{'matrix': [
+            ['^zmpi'],
+            ['%gcc@4.5.0'],
+            ['hypre', 'libelf'],
+            ['~shared'],
+            ['cflags=-O3', 'cflags="-g -O0"'],
+            ['^foo']
+        ]}]
+
+        speclist = SpecList('specs', specs)
+
+        expected_specs = [
+            Spec('hypre cflags=-O3 ~shared %gcc@4.5.0 ^foo ^zmpi'),
+            Spec('hypre cflags="-g -O0" ~shared %gcc@4.5.0 ^foo ^zmpi'),
+            Spec('libelf cflags=-O3 ~shared %gcc@4.5.0 ^foo ^zmpi'),
+            Spec('libelf cflags="-g -O0" ~shared %gcc@4.5.0 ^foo ^zmpi'),
+        ]
+        assert speclist.specs == expected_specs
+
+    def test_spec_list_add(self):
+        speclist = SpecList('specs', self.default_input,
+                            self.default_reference)
+
+        assert speclist.specs_as_yaml_list == self.default_expansion
+        assert speclist.specs_as_constraints == self.default_constraints
+        assert speclist.specs == self.default_specs
+
+        speclist.add('libdwarf')
+
+        assert speclist.specs_as_yaml_list == self.default_expansion + [
+            'libdwarf']
+        assert speclist.specs_as_constraints == self.default_constraints + [
+            [Spec('libdwarf')]]
+        assert speclist.specs == self.default_specs + [Spec('libdwarf')]
+
+    def test_spec_list_remove(self):
+        speclist = SpecList('specs', self.default_input,
+                            self.default_reference)
+
+        assert speclist.specs_as_yaml_list == self.default_expansion
+        assert speclist.specs_as_constraints == self.default_constraints
+        assert speclist.specs == self.default_specs
+
+        speclist.remove('libelf')
+
+        assert speclist.specs_as_yaml_list + [
+            'libelf'
+        ] == self.default_expansion
+
+        assert speclist.specs_as_constraints + [
+            [Spec('libelf')]
+        ] == self.default_constraints
+
+        assert speclist.specs + [Spec('libelf')] == self.default_specs
+
+    def test_spec_list_update_reference(self):
+        speclist = SpecList('specs', self.default_input,
+                            self.default_reference)
+
+        assert speclist.specs_as_yaml_list == self.default_expansion
+        assert speclist.specs_as_constraints == self.default_constraints
+        assert speclist.specs == self.default_specs
+
+        new_mpis = SpecList('mpis', self.default_reference['mpis'].yaml_list)
+        new_mpis.add('mpich@3.3')
+        new_reference = self.default_reference.copy()
+        new_reference['mpis'] = new_mpis
+
+        speclist.update_reference(new_reference)
+
+        expansion = list(self.default_expansion)
+        expansion.insert(3, 'mpich@3.3')
+        constraints = list(self.default_constraints)
+        constraints.insert(3, [Spec('mpich@3.3')])
+        specs = list(self.default_specs)
+        specs.insert(3, Spec('mpich@3.3'))
+
+        assert speclist.specs_as_yaml_list == expansion
+        assert speclist.specs_as_constraints == constraints
+        assert speclist.specs == specs
+
+    def test_spec_list_extension(self):
+        speclist = SpecList('specs', self.default_input,
+                            self.default_reference)
+
+        assert speclist.specs_as_yaml_list == self.default_expansion
+        assert speclist.specs_as_constraints == self.default_constraints
+        assert speclist.specs == self.default_specs
+
+        new_ref = self.default_reference.copy()
+        otherlist = SpecList('specs',
+                             ['zlib', {'matrix': [['callpath'],
+                                                  ['%intel@18']]}],
+                             new_ref)
+
+        speclist.extend(otherlist)
+
+        assert speclist.specs_as_yaml_list == (self.default_expansion +
+                                               otherlist.specs_as_yaml_list)
+        assert speclist.specs == self.default_specs + otherlist.specs
+        assert speclist._reference is new_ref