add CombinatorialSpecSet class for taking cross-products of Specs.

- add CombinatorialSpecSet in spack.util.spec_set module.
  - class is iterable and encaspulated YAML parsing and validation.

- Adjust YAML format to be more generic
  - YAML spec-set format now has a `matrix` section, which can contain
    multiple lists of specs, generated different ways. Including:
    - specs: a raw list of specs.
    - packages: a list of package names and versions
    - compilers: a list of compiler names and versions

  - All of the elements of `matrix` are dimensions for the build matrix;
    we take the cartesian product of these lists of specs to generate a
    build matrix.  This means we can add things like [^mpich, ^openmpi]
    to get builds with different MPI versions.  It also means we can
    multiply the build matrix out with lots of different parameters.

- Add a schema format for spec-sets

5 files changed, 623 insertions, 0 deletions

diff --git a/lib/spack/docs/example_files/spec_set.yaml b/lib/spack/docs/example_files/spec_set.yaml
new file mode 100644
index 0000000000..cf5c5e7d1d
--- /dev/null
+++ b/lib/spack/docs/example_files/spec_set.yaml
@@ -0,0 +1,21 @@
+spec-set:
+    include: [ ape, atompaw, transset]
+    exclude: [binutils,tk]
+    packages:
+        ape:
+            versions: [2.2.1]
+        atompaw:
+            versions: [3.1.0.3, 4.0.0.13]
+        binutils:
+            versions: [2.20.1, 2.25, 2.23.2, 2.24, 2.27, 2.26]
+        tk:
+            versions: [8.6.5, 8.6.3]
+        transset:
+            versions: [1.0.1]
+    compilers:
+        gcc:
+            versions: [4.9, 4.8, 4.7]
+        clang:
+            versions: [3.5, 3.6]
+
+    dashboard: ["https://spack.io/cdash/submit.php?project=spack"]
diff --git a/lib/spack/docs/index.rst b/lib/spack/docs/index.rst
index 3beea06c50..5e4ed764a2 100644
--- a/lib/spack/docs/index.rst
+++ b/lib/spack/docs/index.rst
@@ -81,6 +81,11 @@ or refer to the full manual below.
    build_systems
    developer_guide
    docker_for_developers
+
+.. toctree::
+   :maxdepth: 2
+   :caption: API Docs
+
    Spack API Docs <spack>
    LLNL API Docs <llnl>
 
diff --git a/lib/spack/spack/schema/spec_set.py b/lib/spack/spack/schema/spec_set.py
new file mode 100644
index 0000000000..6f15a86376
--- /dev/null
+++ b/lib/spack/spack/schema/spec_set.py
@@ -0,0 +1,110 @@
+# Copyright 2013-2018 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+"""Schema for Spack spec-set configuration file.
+
+.. literalinclude:: ../spack/schema/spec_set.py
+   :lines: 32-
+"""
+
+
+schema = {
+    '$schema': 'http://json-schema.org/schema#',
+    'title': 'Spack test configuration file schema',
+    'definitions': {
+        # used for include/exclude
+        'list_of_specs': {
+            'type': 'array',
+            'items': {'type': 'string'}
+        },
+        # used for compilers and for packages
+        'objects_with_version_list': {
+            'type': 'object',
+            'additionalProperties': False,
+            'patternProperties': {
+                r'\w[\w-]*': {
+                    'type': 'object',
+                    'additionalProperties': False,
+                    'required': ['versions'],
+                    'properties': {
+                        'versions': {
+                            'type': 'array',
+                            'items': {
+                                'oneOf': [
+                                    {'type': 'string'},
+                                    {'type': 'number'},
+                                ],
+                            },
+                        },
+                    },
+                },
+            },
+        },
+        'packages': {
+            'type': 'object',
+            'additionalProperties': False,
+            'properties': {
+                'packages': {
+                    '$ref': '#/definitions/objects_with_version_list'
+                },
+            }
+        },
+        'compilers': {
+            'type': 'object',
+            'additionalProperties': False,
+            'properties': {
+                'compilers': {
+                    '$ref': '#/definitions/objects_with_version_list'
+                },
+            }
+        },
+        'specs': {
+            'type': 'object',
+            'additionalProperties': False,
+            'properties': {
+                'specs': {'$ref': '#/definitions/list_of_specs'},
+            }
+        },
+    },
+    # this is the actual top level object
+    'type': 'object',
+    'additionalProperties': False,
+    'properties': {
+        'spec-set': {
+            'type': 'object',
+            'additionalProperties': False,
+            'required': ['matrix'],
+            'properties': {
+                # top-level settings are keys and need to be unique
+                'include': {'$ref': '#/definitions/list_of_specs'},
+                'exclude': {'$ref': '#/definitions/list_of_specs'},
+                'cdash': {
+                    'oneOf': [
+                        {'type': 'string'},
+                        {'type': 'array',
+                         'items': {'type': 'string'}
+                        },
+                    ],
+                },
+                'project': {
+                    'type': 'string',
+                },
+                # things under matrix (packages, compilers, etc.)  are a
+                # list so that we can potentiall have multiple of them.
+                'matrix': {
+                    'type': 'array',
+                    'items': {
+                        'type': 'object',
+                        'oneOf': [
+                            {'$ref': '#/definitions/specs'},
+                            {'$ref': '#/definitions/packages'},
+                            {'$ref': '#/definitions/compilers'},
+                        ],
+                    },
+                },
+            },
+        },
+    },
+}
diff --git a/lib/spack/spack/spec_set.py b/lib/spack/spack/spec_set.py
new file mode 100644
index 0000000000..bc38fc04ac
--- /dev/null
+++ b/lib/spack/spack/spec_set.py
@@ -0,0 +1,188 @@
+# Copyright 2013-2018 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+import itertools
+from jsonschema import validate
+
+import llnl.util.tty as tty
+from llnl.util.tty.colify import colify
+
+import spack
+import spack.compilers
+import spack.architecture as sarch
+import spack.schema.spec_set as spec_set_schema
+import spack.util.spack_yaml as syaml
+
+from spack.error import SpackError
+from spack.spec import Spec, ArchSpec
+
+
+class CombinatorialSpecSet:
+    """Set of combinatorial Specs constructed from YAML file."""
+
+    def __init__(self, yaml_like, ignore_invalid=True):
+        """Construct a combinatorial Spec set.
+
+        Args:
+            yaml_like: either raw YAML data as a dict, a file-like object
+                to read the YAML from, or a string containing YAML.  In the
+                first case, we assume already-parsed YAML data.  In the second
+                two cases, we just run yaml.load() on the data.
+            ignore_invalid (bool): whether to ignore invalid specs when
+                expanding the values of this spec set.
+        """
+        self.ignore_invalid = ignore_invalid
+
+        if isinstance(yaml_like, dict):
+            # if it's raw data, just assign it to self.data
+            self.data = yaml_like
+        else:
+            # otherwise try to load it.
+            self.data = syaml.load(yaml_like)
+
+        # validate against the spec set schema
+        validate(self.data, spec_set_schema.schema)
+
+        # chop off the initial spec-set label after valiation.
+        self.data = self.data['spec-set']
+
+        # initialize these from data.
+        self.cdash = self.data.get('cdash', None)
+        if isinstance(self.cdash, str):
+            self.cdash = [self.cdash]
+        self.project = self.data.get('project', None)
+
+        # _spec_lists is a list of lists of specs, to be combined as a
+        # cartesian product when we iterate over all specs in the set.
+        # it's initialized lazily.
+        self._spec_lists = None
+        self._include = []
+        self._exclude = []
+
+    @staticmethod
+    def from_file(path):
+        try:
+            with open(path, 'r') as fin:
+                specs_yaml = syaml.load(fin.read())
+
+                # For now, turn off ignoring invalid specs, as it prevents
+                # iteration if the specified compilers can't be found.
+                return CombinatorialSpecSet(specs_yaml, ignore_invalid=False)
+        except Exception as e:
+            emsg = e.message
+            if not emsg:
+                emsg = e.problem
+            msg = ('Unable to create CombinatorialSpecSet from file ({0})'
+                   ' due to {1}'.format(path, emsg))
+            raise SpackError(msg)
+
+    def all_package_versions(self):
+        """Get package/version combinations for all spack packages."""
+        for name in spack.repo.all_package_names():
+            pkg = spack.repo.get(name)
+            for v in pkg.versions:
+                yield Spec('{0}@{1}'.format(name, v))
+
+    def _specs(self, data):
+        """Read a list of specs from YAML data"""
+        return [Spec(s) for s in data]
+
+    def _compiler_specs(self, data):
+        """Read compiler specs from YAML data.
+        Example YAML:
+            gcc:
+                versions: [4.4.8, 4.9.3]
+            clang:
+                versions: [3.6.1, 3.7.2, 3.8]
+
+        Optionally, data can be 'all', in which case all compilers for
+        the current platform are returned.
+        """
+        # get usable compilers for current platform.
+        arch = ArchSpec(str(sarch.platform()), 'default_os', 'default_target')
+        available_compilers = [
+            c.spec for c in spack.compilers.compilers_for_arch(arch)]
+
+        # return compilers for this platform if asked for everything.
+        if data == 'all':
+            return [cspec.copy() for cspec in available_compilers]
+
+        # otherwise create specs from the YAML file.
+        cspecs = set([
+            Spec('%{0}@{1}'.format(compiler, version))
+            for compiler in data for version in data[compiler]['versions']])
+
+        # filter out invalid specs if caller said to ignore them.
+        if self.ignore_invalid:
+            missing = [c for c in cspecs if not any(
+                c.compiler.satisfies(comp) for comp in available_compilers)]
+            tty.warn("The following compilers were unavailable:")
+            colify(sorted(m.compiler for m in missing))
+            cspecs -= set(missing)
+
+        return cspecs
+
+    def _package_specs(self, data):
+        """Read package/version specs from YAML data.
+        Example YAML:
+            gmake:
+                versions: [4.0, 4.1, 4.2]
+            qt:
+                versions: [4.8.6, 5.2.1, 5.7.1]
+
+        Optionally, data can be 'all', in which case all packages and
+        versions from the package repository are returned.
+        """
+        if data == 'all':
+            return set(self.all_package_versions())
+
+        return set([
+            Spec('{0}@{1}'.format(name, version))
+            for name in data for version in data[name]['versions']])
+
+    def _get_specs(self, matrix_dict):
+        """Parse specs out of an element in the build matrix."""
+        readers = {
+            'packages': self._package_specs,
+            'compilers': self._compiler_specs,
+            'specs': self._specs
+        }
+
+        key = next(iter(matrix_dict), None)
+        assert key in readers
+        return readers[key](matrix_dict[key])
+
+    def __iter__(self):
+        # read in data from YAML file lazily.
+        if self._spec_lists is None:
+            self._spec_lists = [self._get_specs(spec_list)
+                                for spec_list in self.data['matrix']]
+
+            if 'include' in self.data:
+                self._include = [Spec(s) for s in self.data['include']]
+            if 'exclude' in self.data:
+                self._exclude = [Spec(s) for s in self.data['exclude']]
+
+        for spec_list in itertools.product(*self._spec_lists):
+            # if there is an empty array in spec_lists, we'll get this.
+            if not spec_list:
+                yield spec_list
+                continue
+
+            # merge all the constraints in spec_list with each other
+            spec = spec_list[0].copy()
+            for s in spec_list[1:]:
+                spec.constrain(s)
+
+            # test each spec for include/exclude
+            if (self._include and
+                not any(spec.satisfies(s) for s in self._include)):
+                continue
+
+            if any(spec.satisfies(s) for s in self._exclude):
+                continue
+
+            # we now know we can include this spec in the set
+            yield spec
diff --git a/lib/spack/spack/test/spec_set.py b/lib/spack/spack/test/spec_set.py
new file mode 100644
index 0000000000..6bb9e98277
--- /dev/null
+++ b/lib/spack/spack/test/spec_set.py
@@ -0,0 +1,299 @@
+# Copyright 2013-2018 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+import pytest
+
+from spack.spec import Spec
+from jsonschema import ValidationError
+from spack.spec_set import CombinatorialSpecSet
+
+
+pytestmark = pytest.mark.usefixtures('config')
+
+
+basic_yaml_file = {
+    'spec-set': {
+        'cdash': 'http://example.com/cdash',
+        'project': 'testproj',
+        'include': ['gmake'],
+        'matrix': [
+            {'packages': {
+                'gmake': {
+                    'versions': ['4.0']
+                }
+            }},
+            {'compilers': {
+                'gcc': {
+                    'versions': ['4.2.1', '6.3.0']
+                }, 'clang': {
+                    'versions': ['8.0', '3.8']
+                }
+            }},
+        ]
+    }
+}
+
+
+def test_spec_set_basic():
+    """The "include" isn't required, but if it is present, we should only
+    see specs mentioned there.  Also, if we include cdash and project
+    properties, those should be captured and stored on the resulting
+    CombinatorialSpecSet as attributes."""
+    spec_set = CombinatorialSpecSet(basic_yaml_file, False)
+    specs = list(spec for spec in spec_set)
+    assert len(specs) == 4
+    assert spec_set.cdash == ['http://example.com/cdash']
+    assert spec_set.project == 'testproj'
+
+
+def test_spec_set_no_include():
+    """Make sure that without any exclude or include, we get the full cross-
+    product of specs/versions."""
+    yaml_file = {
+        'spec-set': {
+            'matrix': [
+                {'packages': {
+                    'gmake': {
+                        'versions': ['4.0']
+                    }
+                }},
+                {'compilers': {
+                    'gcc': {
+                        'versions': ['4.2.1', '6.3.0']
+                    }, 'clang': {
+                        'versions': ['8.0', '3.8']
+                    }
+                }},
+            ]
+        }
+    }
+    spec_set = CombinatorialSpecSet(yaml_file, False)
+    specs = list(spec for spec in spec_set)
+    assert len(specs) == 4
+
+
+def test_spec_set_include_exclude_conflict():
+    """Exclude should override include"""
+    yaml_file = {
+        'spec-set': {
+            'include': ['gmake'],
+            'exclude': ['gmake'],
+            'matrix': [
+                {'packages': {
+                    'gmake': {
+                        'versions': ['4.0']
+                    }
+                }},
+                {'compilers': {
+                    'gcc': {
+                        'versions': ['4.2.1', '6.3.0']
+                    }, 'clang': {
+                        'versions': ['8.0', '3.8']
+                    }
+                }},
+            ]
+        }
+    }
+    spec_set = CombinatorialSpecSet(yaml_file, False)
+    specs = list(spec for spec in spec_set)
+    assert len(specs) == 0
+
+
+def test_spec_set_exclude():
+    """The exclude property isn't required, but if it appears, any specs
+    mentioned there should not appear in the output specs"""
+    yaml_file = {
+        'spec-set': {
+            'exclude': ['gmake'],
+            'matrix': [
+                {'packages': {
+                    'gmake': {
+                        'versions': ['4.0']
+                    },
+                    'appres': {
+                        'versions': ['1.0.4']
+                    },
+                    'allinea-reports': {
+                        'versions': ['6.0.4']
+                    }
+                }},
+                {'compilers': {
+                    'gcc': {
+                        'versions': ['4.2.1', '6.3.0']
+                    }, 'clang': {
+                        'versions': ['8.0', '3.8']
+                    }
+                }},
+            ]
+        }
+    }
+    spec_set = CombinatorialSpecSet(yaml_file, False)
+    specs = list(spec for spec in spec_set)
+    assert len(specs) == 8
+
+
+def test_spec_set_include_limited_packages():
+    """If we see the include key, it is a filter and only the specs mentioned
+    there should actually be included."""
+    yaml_file = {
+        'spec-set': {
+            'include': ['gmake'],
+            'matrix': [
+                {'packages': {
+                    'gmake': {
+                        'versions': ['4.0']
+                    },
+                    'appres': {
+                        'versions': ['1.0.4']
+                    },
+                    'allinea-reports': {
+                        'versions': ['6.0.4']
+                    }
+                }},
+                {'compilers': {
+                    'gcc': {
+                        'versions': ['4.2.1', '6.3.0']
+                    }, 'clang': {
+                        'versions': ['8.0', '3.8']
+                    }
+                }},
+            ]
+        }
+    }
+    spec_set = CombinatorialSpecSet(yaml_file, False)
+    specs = list(spec for spec in spec_set)
+    assert len(specs) == 4
+
+
+def test_spec_set_simple_spec_list():
+    """Make sure we can handle the slightly more concise syntax where we
+    include the package name/version together and skip the extra keys in
+    the dictionary."""
+    yaml_file = {
+        'spec-set': {
+            'matrix': [
+                {'specs': [
+                    'gmake@4.0',
+                    'appres@1.0.4',
+                    'allinea-reports@6.0.4'
+                ]},
+            ]
+        }
+    }
+    spec_set = CombinatorialSpecSet(yaml_file, False)
+    specs = list(spec for spec in spec_set)
+    assert len(specs) == 3
+
+
+def test_spec_set_with_specs():
+    """Make sure we only see the specs mentioned in the include"""
+    yaml_file = {
+        'spec-set': {
+            'include': ['gmake', 'appres'],
+            'matrix': [
+                {'specs': [
+                    'gmake@4.0',
+                    'appres@1.0.4',
+                    'allinea-reports@6.0.4'
+                ]},
+                {'compilers': {
+                    'gcc': {
+                        'versions': ['4.2.1', '6.3.0']
+                    }, 'clang': {
+                        'versions': ['8.0', '3.8']
+                    }
+                }},
+            ]
+        }
+    }
+    spec_set = CombinatorialSpecSet(yaml_file, False)
+    specs = list(spec for spec in spec_set)
+    assert len(specs) == 8
+
+
+def test_spec_set_packages_no_matrix():
+    """The matrix property is required, make sure we error out if it is
+    missing"""
+    yaml_file = {
+        'spec-set': {
+            'include': ['gmake'],
+            'packages': {
+                'gmake': {
+                    'versions': ['4.0']
+                },
+                'appres': {
+                    'versions': ['1.0.4']
+                },
+                'allinea-reports': {
+                    'versions': ['6.0.4']
+                }
+            },
+        }
+    }
+    with pytest.raises(ValidationError):
+        CombinatorialSpecSet(yaml_file)
+
+
+def test_spec_set_get_cdash_array():
+    """Make sure we can handle multiple cdash sites in a list"""
+    yaml_file = {
+        'spec-set': {
+            'cdash': ['http://example.com/cdash', 'http://example.com/cdash2'],
+            'project': 'testproj',
+            'matrix': [
+                {'packages': {
+                    'gmake': {'versions': ['4.0']},
+                }},
+                {'compilers': {
+                    'gcc': {'versions': ['4.2.1', '6.3.0']},
+                    'clang': {'versions': ['8.0', '3.8']},
+                }},
+            ]
+        }
+    }
+
+    spec_set = CombinatorialSpecSet(yaml_file)
+    assert spec_set.cdash == [
+        'http://example.com/cdash', 'http://example.com/cdash2']
+    assert spec_set.project == 'testproj'
+
+
+def test_compiler_specs():
+    spec_set = CombinatorialSpecSet(basic_yaml_file, False)
+    compilers = spec_set._compiler_specs({
+        'gcc': {
+            'versions': ['4.2.1', '6.3.0']
+        }, 'clang': {
+            'versions': ['8.0', '3.8']
+        }})
+
+    assert len(list(compilers)) == 4
+    assert Spec('%gcc@4.2.1') in compilers
+    assert Spec('%gcc@6.3.0') in compilers
+    assert Spec('%clang@8.0') in compilers
+    assert Spec('%clang@3.8') in compilers
+
+
+def test_package_specs():
+    spec_set = CombinatorialSpecSet(basic_yaml_file, False)
+
+    packages = spec_set._package_specs({
+        'gmake': {
+            'versions': ['4.0', '5.0']
+        },
+        'appres': {
+            'versions': ['1.0.4']
+        },
+        'allinea-reports': {
+            'versions': ['6.0.1', '6.0.3', '6.0.4']
+        }
+    })
+
+    assert Spec('gmake@4.0') in packages
+    assert Spec('gmake@5.0') in packages
+    assert Spec('appres@1.0.4') in packages
+    assert Spec('allinea-reports@6.0.1') in packages
+    assert Spec('allinea-reports@6.0.3') in packages
+    assert Spec('allinea-reports@6.0.4') in packages