summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--lib/spack/docs/example_files/spec_set.yaml21
-rw-r--r--lib/spack/docs/index.rst5
-rw-r--r--lib/spack/spack/schema/spec_set.py110
-rw-r--r--lib/spack/spack/spec_set.py188
-rw-r--r--lib/spack/spack/test/spec_set.py299
5 files changed, 623 insertions, 0 deletions
diff --git a/lib/spack/docs/example_files/spec_set.yaml b/lib/spack/docs/example_files/spec_set.yaml
new file mode 100644
index 0000000000..cf5c5e7d1d
--- /dev/null
+++ b/lib/spack/docs/example_files/spec_set.yaml
@@ -0,0 +1,21 @@
+spec-set:
+ include: [ ape, atompaw, transset]
+ exclude: [binutils,tk]
+ packages:
+ ape:
+ versions: [2.2.1]
+ atompaw:
+ versions: [3.1.0.3, 4.0.0.13]
+ binutils:
+ versions: [2.20.1, 2.25, 2.23.2, 2.24, 2.27, 2.26]
+ tk:
+ versions: [8.6.5, 8.6.3]
+ transset:
+ versions: [1.0.1]
+ compilers:
+ gcc:
+ versions: [4.9, 4.8, 4.7]
+ clang:
+ versions: [3.5, 3.6]
+
+ dashboard: ["https://spack.io/cdash/submit.php?project=spack"]
diff --git a/lib/spack/docs/index.rst b/lib/spack/docs/index.rst
index 3beea06c50..5e4ed764a2 100644
--- a/lib/spack/docs/index.rst
+++ b/lib/spack/docs/index.rst
@@ -81,6 +81,11 @@ or refer to the full manual below.
build_systems
developer_guide
docker_for_developers
+
+.. toctree::
+ :maxdepth: 2
+ :caption: API Docs
+
Spack API Docs <spack>
LLNL API Docs <llnl>
diff --git a/lib/spack/spack/schema/spec_set.py b/lib/spack/spack/schema/spec_set.py
new file mode 100644
index 0000000000..6f15a86376
--- /dev/null
+++ b/lib/spack/spack/schema/spec_set.py
@@ -0,0 +1,110 @@
+# Copyright 2013-2018 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+"""Schema for Spack spec-set configuration file.
+
+.. literalinclude:: ../spack/schema/spec_set.py
+ :lines: 32-
+"""
+
+
+schema = {
+ '$schema': 'http://json-schema.org/schema#',
+ 'title': 'Spack test configuration file schema',
+ 'definitions': {
+ # used for include/exclude
+ 'list_of_specs': {
+ 'type': 'array',
+ 'items': {'type': 'string'}
+ },
+ # used for compilers and for packages
+ 'objects_with_version_list': {
+ 'type': 'object',
+ 'additionalProperties': False,
+ 'patternProperties': {
+ r'\w[\w-]*': {
+ 'type': 'object',
+ 'additionalProperties': False,
+ 'required': ['versions'],
+ 'properties': {
+ 'versions': {
+ 'type': 'array',
+ 'items': {
+ 'oneOf': [
+ {'type': 'string'},
+ {'type': 'number'},
+ ],
+ },
+ },
+ },
+ },
+ },
+ },
+ 'packages': {
+ 'type': 'object',
+ 'additionalProperties': False,
+ 'properties': {
+ 'packages': {
+ '$ref': '#/definitions/objects_with_version_list'
+ },
+ }
+ },
+ 'compilers': {
+ 'type': 'object',
+ 'additionalProperties': False,
+ 'properties': {
+ 'compilers': {
+ '$ref': '#/definitions/objects_with_version_list'
+ },
+ }
+ },
+ 'specs': {
+ 'type': 'object',
+ 'additionalProperties': False,
+ 'properties': {
+ 'specs': {'$ref': '#/definitions/list_of_specs'},
+ }
+ },
+ },
+ # this is the actual top level object
+ 'type': 'object',
+ 'additionalProperties': False,
+ 'properties': {
+ 'spec-set': {
+ 'type': 'object',
+ 'additionalProperties': False,
+ 'required': ['matrix'],
+ 'properties': {
+ # top-level settings are keys and need to be unique
+ 'include': {'$ref': '#/definitions/list_of_specs'},
+ 'exclude': {'$ref': '#/definitions/list_of_specs'},
+ 'cdash': {
+ 'oneOf': [
+ {'type': 'string'},
+ {'type': 'array',
+ 'items': {'type': 'string'}
+ },
+ ],
+ },
+ 'project': {
+ 'type': 'string',
+ },
+ # things under matrix (packages, compilers, etc.) are a
+ # list so that we can potentiall have multiple of them.
+ 'matrix': {
+ 'type': 'array',
+ 'items': {
+ 'type': 'object',
+ 'oneOf': [
+ {'$ref': '#/definitions/specs'},
+ {'$ref': '#/definitions/packages'},
+ {'$ref': '#/definitions/compilers'},
+ ],
+ },
+ },
+ },
+ },
+ },
+}
diff --git a/lib/spack/spack/spec_set.py b/lib/spack/spack/spec_set.py
new file mode 100644
index 0000000000..bc38fc04ac
--- /dev/null
+++ b/lib/spack/spack/spec_set.py
@@ -0,0 +1,188 @@
+# Copyright 2013-2018 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+import itertools
+from jsonschema import validate
+
+import llnl.util.tty as tty
+from llnl.util.tty.colify import colify
+
+import spack
+import spack.compilers
+import spack.architecture as sarch
+import spack.schema.spec_set as spec_set_schema
+import spack.util.spack_yaml as syaml
+
+from spack.error import SpackError
+from spack.spec import Spec, ArchSpec
+
+
+class CombinatorialSpecSet:
+ """Set of combinatorial Specs constructed from YAML file."""
+
+ def __init__(self, yaml_like, ignore_invalid=True):
+ """Construct a combinatorial Spec set.
+
+ Args:
+ yaml_like: either raw YAML data as a dict, a file-like object
+ to read the YAML from, or a string containing YAML. In the
+ first case, we assume already-parsed YAML data. In the second
+ two cases, we just run yaml.load() on the data.
+ ignore_invalid (bool): whether to ignore invalid specs when
+ expanding the values of this spec set.
+ """
+ self.ignore_invalid = ignore_invalid
+
+ if isinstance(yaml_like, dict):
+ # if it's raw data, just assign it to self.data
+ self.data = yaml_like
+ else:
+ # otherwise try to load it.
+ self.data = syaml.load(yaml_like)
+
+ # validate against the spec set schema
+ validate(self.data, spec_set_schema.schema)
+
+ # chop off the initial spec-set label after valiation.
+ self.data = self.data['spec-set']
+
+ # initialize these from data.
+ self.cdash = self.data.get('cdash', None)
+ if isinstance(self.cdash, str):
+ self.cdash = [self.cdash]
+ self.project = self.data.get('project', None)
+
+ # _spec_lists is a list of lists of specs, to be combined as a
+ # cartesian product when we iterate over all specs in the set.
+ # it's initialized lazily.
+ self._spec_lists = None
+ self._include = []
+ self._exclude = []
+
+ @staticmethod
+ def from_file(path):
+ try:
+ with open(path, 'r') as fin:
+ specs_yaml = syaml.load(fin.read())
+
+ # For now, turn off ignoring invalid specs, as it prevents
+ # iteration if the specified compilers can't be found.
+ return CombinatorialSpecSet(specs_yaml, ignore_invalid=False)
+ except Exception as e:
+ emsg = e.message
+ if not emsg:
+ emsg = e.problem
+ msg = ('Unable to create CombinatorialSpecSet from file ({0})'
+ ' due to {1}'.format(path, emsg))
+ raise SpackError(msg)
+
+ def all_package_versions(self):
+ """Get package/version combinations for all spack packages."""
+ for name in spack.repo.all_package_names():
+ pkg = spack.repo.get(name)
+ for v in pkg.versions:
+ yield Spec('{0}@{1}'.format(name, v))
+
+ def _specs(self, data):
+ """Read a list of specs from YAML data"""
+ return [Spec(s) for s in data]
+
+ def _compiler_specs(self, data):
+ """Read compiler specs from YAML data.
+ Example YAML:
+ gcc:
+ versions: [4.4.8, 4.9.3]
+ clang:
+ versions: [3.6.1, 3.7.2, 3.8]
+
+ Optionally, data can be 'all', in which case all compilers for
+ the current platform are returned.
+ """
+ # get usable compilers for current platform.
+ arch = ArchSpec(str(sarch.platform()), 'default_os', 'default_target')
+ available_compilers = [
+ c.spec for c in spack.compilers.compilers_for_arch(arch)]
+
+ # return compilers for this platform if asked for everything.
+ if data == 'all':
+ return [cspec.copy() for cspec in available_compilers]
+
+ # otherwise create specs from the YAML file.
+ cspecs = set([
+ Spec('%{0}@{1}'.format(compiler, version))
+ for compiler in data for version in data[compiler]['versions']])
+
+ # filter out invalid specs if caller said to ignore them.
+ if self.ignore_invalid:
+ missing = [c for c in cspecs if not any(
+ c.compiler.satisfies(comp) for comp in available_compilers)]
+ tty.warn("The following compilers were unavailable:")
+ colify(sorted(m.compiler for m in missing))
+ cspecs -= set(missing)
+
+ return cspecs
+
+ def _package_specs(self, data):
+ """Read package/version specs from YAML data.
+ Example YAML:
+ gmake:
+ versions: [4.0, 4.1, 4.2]
+ qt:
+ versions: [4.8.6, 5.2.1, 5.7.1]
+
+ Optionally, data can be 'all', in which case all packages and
+ versions from the package repository are returned.
+ """
+ if data == 'all':
+ return set(self.all_package_versions())
+
+ return set([
+ Spec('{0}@{1}'.format(name, version))
+ for name in data for version in data[name]['versions']])
+
+ def _get_specs(self, matrix_dict):
+ """Parse specs out of an element in the build matrix."""
+ readers = {
+ 'packages': self._package_specs,
+ 'compilers': self._compiler_specs,
+ 'specs': self._specs
+ }
+
+ key = next(iter(matrix_dict), None)
+ assert key in readers
+ return readers[key](matrix_dict[key])
+
+ def __iter__(self):
+ # read in data from YAML file lazily.
+ if self._spec_lists is None:
+ self._spec_lists = [self._get_specs(spec_list)
+ for spec_list in self.data['matrix']]
+
+ if 'include' in self.data:
+ self._include = [Spec(s) for s in self.data['include']]
+ if 'exclude' in self.data:
+ self._exclude = [Spec(s) for s in self.data['exclude']]
+
+ for spec_list in itertools.product(*self._spec_lists):
+ # if there is an empty array in spec_lists, we'll get this.
+ if not spec_list:
+ yield spec_list
+ continue
+
+ # merge all the constraints in spec_list with each other
+ spec = spec_list[0].copy()
+ for s in spec_list[1:]:
+ spec.constrain(s)
+
+ # test each spec for include/exclude
+ if (self._include and
+ not any(spec.satisfies(s) for s in self._include)):
+ continue
+
+ if any(spec.satisfies(s) for s in self._exclude):
+ continue
+
+ # we now know we can include this spec in the set
+ yield spec
diff --git a/lib/spack/spack/test/spec_set.py b/lib/spack/spack/test/spec_set.py
new file mode 100644
index 0000000000..6bb9e98277
--- /dev/null
+++ b/lib/spack/spack/test/spec_set.py
@@ -0,0 +1,299 @@
+# Copyright 2013-2018 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+import pytest
+
+from spack.spec import Spec
+from jsonschema import ValidationError
+from spack.spec_set import CombinatorialSpecSet
+
+
+pytestmark = pytest.mark.usefixtures('config')
+
+
+basic_yaml_file = {
+ 'spec-set': {
+ 'cdash': 'http://example.com/cdash',
+ 'project': 'testproj',
+ 'include': ['gmake'],
+ 'matrix': [
+ {'packages': {
+ 'gmake': {
+ 'versions': ['4.0']
+ }
+ }},
+ {'compilers': {
+ 'gcc': {
+ 'versions': ['4.2.1', '6.3.0']
+ }, 'clang': {
+ 'versions': ['8.0', '3.8']
+ }
+ }},
+ ]
+ }
+}
+
+
+def test_spec_set_basic():
+ """The "include" isn't required, but if it is present, we should only
+ see specs mentioned there. Also, if we include cdash and project
+ properties, those should be captured and stored on the resulting
+ CombinatorialSpecSet as attributes."""
+ spec_set = CombinatorialSpecSet(basic_yaml_file, False)
+ specs = list(spec for spec in spec_set)
+ assert len(specs) == 4
+ assert spec_set.cdash == ['http://example.com/cdash']
+ assert spec_set.project == 'testproj'
+
+
+def test_spec_set_no_include():
+ """Make sure that without any exclude or include, we get the full cross-
+ product of specs/versions."""
+ yaml_file = {
+ 'spec-set': {
+ 'matrix': [
+ {'packages': {
+ 'gmake': {
+ 'versions': ['4.0']
+ }
+ }},
+ {'compilers': {
+ 'gcc': {
+ 'versions': ['4.2.1', '6.3.0']
+ }, 'clang': {
+ 'versions': ['8.0', '3.8']
+ }
+ }},
+ ]
+ }
+ }
+ spec_set = CombinatorialSpecSet(yaml_file, False)
+ specs = list(spec for spec in spec_set)
+ assert len(specs) == 4
+
+
+def test_spec_set_include_exclude_conflict():
+ """Exclude should override include"""
+ yaml_file = {
+ 'spec-set': {
+ 'include': ['gmake'],
+ 'exclude': ['gmake'],
+ 'matrix': [
+ {'packages': {
+ 'gmake': {
+ 'versions': ['4.0']
+ }
+ }},
+ {'compilers': {
+ 'gcc': {
+ 'versions': ['4.2.1', '6.3.0']
+ }, 'clang': {
+ 'versions': ['8.0', '3.8']
+ }
+ }},
+ ]
+ }
+ }
+ spec_set = CombinatorialSpecSet(yaml_file, False)
+ specs = list(spec for spec in spec_set)
+ assert len(specs) == 0
+
+
+def test_spec_set_exclude():
+ """The exclude property isn't required, but if it appears, any specs
+ mentioned there should not appear in the output specs"""
+ yaml_file = {
+ 'spec-set': {
+ 'exclude': ['gmake'],
+ 'matrix': [
+ {'packages': {
+ 'gmake': {
+ 'versions': ['4.0']
+ },
+ 'appres': {
+ 'versions': ['1.0.4']
+ },
+ 'allinea-reports': {
+ 'versions': ['6.0.4']
+ }
+ }},
+ {'compilers': {
+ 'gcc': {
+ 'versions': ['4.2.1', '6.3.0']
+ }, 'clang': {
+ 'versions': ['8.0', '3.8']
+ }
+ }},
+ ]
+ }
+ }
+ spec_set = CombinatorialSpecSet(yaml_file, False)
+ specs = list(spec for spec in spec_set)
+ assert len(specs) == 8
+
+
+def test_spec_set_include_limited_packages():
+ """If we see the include key, it is a filter and only the specs mentioned
+ there should actually be included."""
+ yaml_file = {
+ 'spec-set': {
+ 'include': ['gmake'],
+ 'matrix': [
+ {'packages': {
+ 'gmake': {
+ 'versions': ['4.0']
+ },
+ 'appres': {
+ 'versions': ['1.0.4']
+ },
+ 'allinea-reports': {
+ 'versions': ['6.0.4']
+ }
+ }},
+ {'compilers': {
+ 'gcc': {
+ 'versions': ['4.2.1', '6.3.0']
+ }, 'clang': {
+ 'versions': ['8.0', '3.8']
+ }
+ }},
+ ]
+ }
+ }
+ spec_set = CombinatorialSpecSet(yaml_file, False)
+ specs = list(spec for spec in spec_set)
+ assert len(specs) == 4
+
+
+def test_spec_set_simple_spec_list():
+ """Make sure we can handle the slightly more concise syntax where we
+ include the package name/version together and skip the extra keys in
+ the dictionary."""
+ yaml_file = {
+ 'spec-set': {
+ 'matrix': [
+ {'specs': [
+ 'gmake@4.0',
+ 'appres@1.0.4',
+ 'allinea-reports@6.0.4'
+ ]},
+ ]
+ }
+ }
+ spec_set = CombinatorialSpecSet(yaml_file, False)
+ specs = list(spec for spec in spec_set)
+ assert len(specs) == 3
+
+
+def test_spec_set_with_specs():
+ """Make sure we only see the specs mentioned in the include"""
+ yaml_file = {
+ 'spec-set': {
+ 'include': ['gmake', 'appres'],
+ 'matrix': [
+ {'specs': [
+ 'gmake@4.0',
+ 'appres@1.0.4',
+ 'allinea-reports@6.0.4'
+ ]},
+ {'compilers': {
+ 'gcc': {
+ 'versions': ['4.2.1', '6.3.0']
+ }, 'clang': {
+ 'versions': ['8.0', '3.8']
+ }
+ }},
+ ]
+ }
+ }
+ spec_set = CombinatorialSpecSet(yaml_file, False)
+ specs = list(spec for spec in spec_set)
+ assert len(specs) == 8
+
+
+def test_spec_set_packages_no_matrix():
+ """The matrix property is required, make sure we error out if it is
+ missing"""
+ yaml_file = {
+ 'spec-set': {
+ 'include': ['gmake'],
+ 'packages': {
+ 'gmake': {
+ 'versions': ['4.0']
+ },
+ 'appres': {
+ 'versions': ['1.0.4']
+ },
+ 'allinea-reports': {
+ 'versions': ['6.0.4']
+ }
+ },
+ }
+ }
+ with pytest.raises(ValidationError):
+ CombinatorialSpecSet(yaml_file)
+
+
+def test_spec_set_get_cdash_array():
+ """Make sure we can handle multiple cdash sites in a list"""
+ yaml_file = {
+ 'spec-set': {
+ 'cdash': ['http://example.com/cdash', 'http://example.com/cdash2'],
+ 'project': 'testproj',
+ 'matrix': [
+ {'packages': {
+ 'gmake': {'versions': ['4.0']},
+ }},
+ {'compilers': {
+ 'gcc': {'versions': ['4.2.1', '6.3.0']},
+ 'clang': {'versions': ['8.0', '3.8']},
+ }},
+ ]
+ }
+ }
+
+ spec_set = CombinatorialSpecSet(yaml_file)
+ assert spec_set.cdash == [
+ 'http://example.com/cdash', 'http://example.com/cdash2']
+ assert spec_set.project == 'testproj'
+
+
+def test_compiler_specs():
+ spec_set = CombinatorialSpecSet(basic_yaml_file, False)
+ compilers = spec_set._compiler_specs({
+ 'gcc': {
+ 'versions': ['4.2.1', '6.3.0']
+ }, 'clang': {
+ 'versions': ['8.0', '3.8']
+ }})
+
+ assert len(list(compilers)) == 4
+ assert Spec('%gcc@4.2.1') in compilers
+ assert Spec('%gcc@6.3.0') in compilers
+ assert Spec('%clang@8.0') in compilers
+ assert Spec('%clang@3.8') in compilers
+
+
+def test_package_specs():
+ spec_set = CombinatorialSpecSet(basic_yaml_file, False)
+
+ packages = spec_set._package_specs({
+ 'gmake': {
+ 'versions': ['4.0', '5.0']
+ },
+ 'appres': {
+ 'versions': ['1.0.4']
+ },
+ 'allinea-reports': {
+ 'versions': ['6.0.1', '6.0.3', '6.0.4']
+ }
+ })
+
+ assert Spec('gmake@4.0') in packages
+ assert Spec('gmake@5.0') in packages
+ assert Spec('appres@1.0.4') in packages
+ assert Spec('allinea-reports@6.0.1') in packages
+ assert Spec('allinea-reports@6.0.3') in packages
+ assert Spec('allinea-reports@6.0.4') in packages