diff options
| -rw-r--r-- | lib/spack/docs/example_files/spec_set.yaml | 21 | ||||
| -rw-r--r-- | lib/spack/docs/index.rst | 5 | ||||
| -rw-r--r-- | lib/spack/spack/schema/spec_set.py | 110 | ||||
| -rw-r--r-- | lib/spack/spack/spec_set.py | 188 | ||||
| -rw-r--r-- | lib/spack/spack/test/spec_set.py | 299 |
5 files changed, 623 insertions, 0 deletions
diff --git a/lib/spack/docs/example_files/spec_set.yaml b/lib/spack/docs/example_files/spec_set.yaml new file mode 100644 index 0000000000..cf5c5e7d1d --- /dev/null +++ b/lib/spack/docs/example_files/spec_set.yaml @@ -0,0 +1,21 @@ +spec-set: + include: [ ape, atompaw, transset] + exclude: [binutils,tk] + packages: + ape: + versions: [2.2.1] + atompaw: + versions: [3.1.0.3, 4.0.0.13] + binutils: + versions: [2.20.1, 2.25, 2.23.2, 2.24, 2.27, 2.26] + tk: + versions: [8.6.5, 8.6.3] + transset: + versions: [1.0.1] + compilers: + gcc: + versions: [4.9, 4.8, 4.7] + clang: + versions: [3.5, 3.6] + + dashboard: ["https://spack.io/cdash/submit.php?project=spack"] diff --git a/lib/spack/docs/index.rst b/lib/spack/docs/index.rst index 3beea06c50..5e4ed764a2 100644 --- a/lib/spack/docs/index.rst +++ b/lib/spack/docs/index.rst @@ -81,6 +81,11 @@ or refer to the full manual below. build_systems developer_guide docker_for_developers + +.. toctree:: + :maxdepth: 2 + :caption: API Docs + Spack API Docs <spack> LLNL API Docs <llnl> diff --git a/lib/spack/spack/schema/spec_set.py b/lib/spack/spack/schema/spec_set.py new file mode 100644 index 0000000000..6f15a86376 --- /dev/null +++ b/lib/spack/spack/schema/spec_set.py @@ -0,0 +1,110 @@ +# Copyright 2013-2018 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +"""Schema for Spack spec-set configuration file. + +.. literalinclude:: ../spack/schema/spec_set.py + :lines: 32- +""" + + +schema = { + '$schema': 'http://json-schema.org/schema#', + 'title': 'Spack test configuration file schema', + 'definitions': { + # used for include/exclude + 'list_of_specs': { + 'type': 'array', + 'items': {'type': 'string'} + }, + # used for compilers and for packages + 'objects_with_version_list': { + 'type': 'object', + 'additionalProperties': False, + 'patternProperties': { + r'\w[\w-]*': { + 'type': 'object', + 'additionalProperties': False, + 'required': ['versions'], + 'properties': { + 'versions': { + 'type': 'array', + 'items': { + 'oneOf': [ + {'type': 'string'}, + {'type': 'number'}, + ], + }, + }, + }, + }, + }, + }, + 'packages': { + 'type': 'object', + 'additionalProperties': False, + 'properties': { + 'packages': { + '$ref': '#/definitions/objects_with_version_list' + }, + } + }, + 'compilers': { + 'type': 'object', + 'additionalProperties': False, + 'properties': { + 'compilers': { + '$ref': '#/definitions/objects_with_version_list' + }, + } + }, + 'specs': { + 'type': 'object', + 'additionalProperties': False, + 'properties': { + 'specs': {'$ref': '#/definitions/list_of_specs'}, + } + }, + }, + # this is the actual top level object + 'type': 'object', + 'additionalProperties': False, + 'properties': { + 'spec-set': { + 'type': 'object', + 'additionalProperties': False, + 'required': ['matrix'], + 'properties': { + # top-level settings are keys and need to be unique + 'include': {'$ref': '#/definitions/list_of_specs'}, + 'exclude': {'$ref': '#/definitions/list_of_specs'}, + 'cdash': { + 'oneOf': [ + {'type': 'string'}, + {'type': 'array', + 'items': {'type': 'string'} + }, + ], + }, + 'project': { + 'type': 'string', + }, + # things under matrix (packages, compilers, etc.) are a + # list so that we can potentiall have multiple of them. + 'matrix': { + 'type': 'array', + 'items': { + 'type': 'object', + 'oneOf': [ + {'$ref': '#/definitions/specs'}, + {'$ref': '#/definitions/packages'}, + {'$ref': '#/definitions/compilers'}, + ], + }, + }, + }, + }, + }, +} diff --git a/lib/spack/spack/spec_set.py b/lib/spack/spack/spec_set.py new file mode 100644 index 0000000000..bc38fc04ac --- /dev/null +++ b/lib/spack/spack/spec_set.py @@ -0,0 +1,188 @@ +# Copyright 2013-2018 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import itertools +from jsonschema import validate + +import llnl.util.tty as tty +from llnl.util.tty.colify import colify + +import spack +import spack.compilers +import spack.architecture as sarch +import spack.schema.spec_set as spec_set_schema +import spack.util.spack_yaml as syaml + +from spack.error import SpackError +from spack.spec import Spec, ArchSpec + + +class CombinatorialSpecSet: + """Set of combinatorial Specs constructed from YAML file.""" + + def __init__(self, yaml_like, ignore_invalid=True): + """Construct a combinatorial Spec set. + + Args: + yaml_like: either raw YAML data as a dict, a file-like object + to read the YAML from, or a string containing YAML. In the + first case, we assume already-parsed YAML data. In the second + two cases, we just run yaml.load() on the data. + ignore_invalid (bool): whether to ignore invalid specs when + expanding the values of this spec set. + """ + self.ignore_invalid = ignore_invalid + + if isinstance(yaml_like, dict): + # if it's raw data, just assign it to self.data + self.data = yaml_like + else: + # otherwise try to load it. + self.data = syaml.load(yaml_like) + + # validate against the spec set schema + validate(self.data, spec_set_schema.schema) + + # chop off the initial spec-set label after valiation. + self.data = self.data['spec-set'] + + # initialize these from data. + self.cdash = self.data.get('cdash', None) + if isinstance(self.cdash, str): + self.cdash = [self.cdash] + self.project = self.data.get('project', None) + + # _spec_lists is a list of lists of specs, to be combined as a + # cartesian product when we iterate over all specs in the set. + # it's initialized lazily. + self._spec_lists = None + self._include = [] + self._exclude = [] + + @staticmethod + def from_file(path): + try: + with open(path, 'r') as fin: + specs_yaml = syaml.load(fin.read()) + + # For now, turn off ignoring invalid specs, as it prevents + # iteration if the specified compilers can't be found. + return CombinatorialSpecSet(specs_yaml, ignore_invalid=False) + except Exception as e: + emsg = e.message + if not emsg: + emsg = e.problem + msg = ('Unable to create CombinatorialSpecSet from file ({0})' + ' due to {1}'.format(path, emsg)) + raise SpackError(msg) + + def all_package_versions(self): + """Get package/version combinations for all spack packages.""" + for name in spack.repo.all_package_names(): + pkg = spack.repo.get(name) + for v in pkg.versions: + yield Spec('{0}@{1}'.format(name, v)) + + def _specs(self, data): + """Read a list of specs from YAML data""" + return [Spec(s) for s in data] + + def _compiler_specs(self, data): + """Read compiler specs from YAML data. + Example YAML: + gcc: + versions: [4.4.8, 4.9.3] + clang: + versions: [3.6.1, 3.7.2, 3.8] + + Optionally, data can be 'all', in which case all compilers for + the current platform are returned. + """ + # get usable compilers for current platform. + arch = ArchSpec(str(sarch.platform()), 'default_os', 'default_target') + available_compilers = [ + c.spec for c in spack.compilers.compilers_for_arch(arch)] + + # return compilers for this platform if asked for everything. + if data == 'all': + return [cspec.copy() for cspec in available_compilers] + + # otherwise create specs from the YAML file. + cspecs = set([ + Spec('%{0}@{1}'.format(compiler, version)) + for compiler in data for version in data[compiler]['versions']]) + + # filter out invalid specs if caller said to ignore them. + if self.ignore_invalid: + missing = [c for c in cspecs if not any( + c.compiler.satisfies(comp) for comp in available_compilers)] + tty.warn("The following compilers were unavailable:") + colify(sorted(m.compiler for m in missing)) + cspecs -= set(missing) + + return cspecs + + def _package_specs(self, data): + """Read package/version specs from YAML data. + Example YAML: + gmake: + versions: [4.0, 4.1, 4.2] + qt: + versions: [4.8.6, 5.2.1, 5.7.1] + + Optionally, data can be 'all', in which case all packages and + versions from the package repository are returned. + """ + if data == 'all': + return set(self.all_package_versions()) + + return set([ + Spec('{0}@{1}'.format(name, version)) + for name in data for version in data[name]['versions']]) + + def _get_specs(self, matrix_dict): + """Parse specs out of an element in the build matrix.""" + readers = { + 'packages': self._package_specs, + 'compilers': self._compiler_specs, + 'specs': self._specs + } + + key = next(iter(matrix_dict), None) + assert key in readers + return readers[key](matrix_dict[key]) + + def __iter__(self): + # read in data from YAML file lazily. + if self._spec_lists is None: + self._spec_lists = [self._get_specs(spec_list) + for spec_list in self.data['matrix']] + + if 'include' in self.data: + self._include = [Spec(s) for s in self.data['include']] + if 'exclude' in self.data: + self._exclude = [Spec(s) for s in self.data['exclude']] + + for spec_list in itertools.product(*self._spec_lists): + # if there is an empty array in spec_lists, we'll get this. + if not spec_list: + yield spec_list + continue + + # merge all the constraints in spec_list with each other + spec = spec_list[0].copy() + for s in spec_list[1:]: + spec.constrain(s) + + # test each spec for include/exclude + if (self._include and + not any(spec.satisfies(s) for s in self._include)): + continue + + if any(spec.satisfies(s) for s in self._exclude): + continue + + # we now know we can include this spec in the set + yield spec diff --git a/lib/spack/spack/test/spec_set.py b/lib/spack/spack/test/spec_set.py new file mode 100644 index 0000000000..6bb9e98277 --- /dev/null +++ b/lib/spack/spack/test/spec_set.py @@ -0,0 +1,299 @@ +# Copyright 2013-2018 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import pytest + +from spack.spec import Spec +from jsonschema import ValidationError +from spack.spec_set import CombinatorialSpecSet + + +pytestmark = pytest.mark.usefixtures('config') + + +basic_yaml_file = { + 'spec-set': { + 'cdash': 'http://example.com/cdash', + 'project': 'testproj', + 'include': ['gmake'], + 'matrix': [ + {'packages': { + 'gmake': { + 'versions': ['4.0'] + } + }}, + {'compilers': { + 'gcc': { + 'versions': ['4.2.1', '6.3.0'] + }, 'clang': { + 'versions': ['8.0', '3.8'] + } + }}, + ] + } +} + + +def test_spec_set_basic(): + """The "include" isn't required, but if it is present, we should only + see specs mentioned there. Also, if we include cdash and project + properties, those should be captured and stored on the resulting + CombinatorialSpecSet as attributes.""" + spec_set = CombinatorialSpecSet(basic_yaml_file, False) + specs = list(spec for spec in spec_set) + assert len(specs) == 4 + assert spec_set.cdash == ['http://example.com/cdash'] + assert spec_set.project == 'testproj' + + +def test_spec_set_no_include(): + """Make sure that without any exclude or include, we get the full cross- + product of specs/versions.""" + yaml_file = { + 'spec-set': { + 'matrix': [ + {'packages': { + 'gmake': { + 'versions': ['4.0'] + } + }}, + {'compilers': { + 'gcc': { + 'versions': ['4.2.1', '6.3.0'] + }, 'clang': { + 'versions': ['8.0', '3.8'] + } + }}, + ] + } + } + spec_set = CombinatorialSpecSet(yaml_file, False) + specs = list(spec for spec in spec_set) + assert len(specs) == 4 + + +def test_spec_set_include_exclude_conflict(): + """Exclude should override include""" + yaml_file = { + 'spec-set': { + 'include': ['gmake'], + 'exclude': ['gmake'], + 'matrix': [ + {'packages': { + 'gmake': { + 'versions': ['4.0'] + } + }}, + {'compilers': { + 'gcc': { + 'versions': ['4.2.1', '6.3.0'] + }, 'clang': { + 'versions': ['8.0', '3.8'] + } + }}, + ] + } + } + spec_set = CombinatorialSpecSet(yaml_file, False) + specs = list(spec for spec in spec_set) + assert len(specs) == 0 + + +def test_spec_set_exclude(): + """The exclude property isn't required, but if it appears, any specs + mentioned there should not appear in the output specs""" + yaml_file = { + 'spec-set': { + 'exclude': ['gmake'], + 'matrix': [ + {'packages': { + 'gmake': { + 'versions': ['4.0'] + }, + 'appres': { + 'versions': ['1.0.4'] + }, + 'allinea-reports': { + 'versions': ['6.0.4'] + } + }}, + {'compilers': { + 'gcc': { + 'versions': ['4.2.1', '6.3.0'] + }, 'clang': { + 'versions': ['8.0', '3.8'] + } + }}, + ] + } + } + spec_set = CombinatorialSpecSet(yaml_file, False) + specs = list(spec for spec in spec_set) + assert len(specs) == 8 + + +def test_spec_set_include_limited_packages(): + """If we see the include key, it is a filter and only the specs mentioned + there should actually be included.""" + yaml_file = { + 'spec-set': { + 'include': ['gmake'], + 'matrix': [ + {'packages': { + 'gmake': { + 'versions': ['4.0'] + }, + 'appres': { + 'versions': ['1.0.4'] + }, + 'allinea-reports': { + 'versions': ['6.0.4'] + } + }}, + {'compilers': { + 'gcc': { + 'versions': ['4.2.1', '6.3.0'] + }, 'clang': { + 'versions': ['8.0', '3.8'] + } + }}, + ] + } + } + spec_set = CombinatorialSpecSet(yaml_file, False) + specs = list(spec for spec in spec_set) + assert len(specs) == 4 + + +def test_spec_set_simple_spec_list(): + """Make sure we can handle the slightly more concise syntax where we + include the package name/version together and skip the extra keys in + the dictionary.""" + yaml_file = { + 'spec-set': { + 'matrix': [ + {'specs': [ + 'gmake@4.0', + 'appres@1.0.4', + 'allinea-reports@6.0.4' + ]}, + ] + } + } + spec_set = CombinatorialSpecSet(yaml_file, False) + specs = list(spec for spec in spec_set) + assert len(specs) == 3 + + +def test_spec_set_with_specs(): + """Make sure we only see the specs mentioned in the include""" + yaml_file = { + 'spec-set': { + 'include': ['gmake', 'appres'], + 'matrix': [ + {'specs': [ + 'gmake@4.0', + 'appres@1.0.4', + 'allinea-reports@6.0.4' + ]}, + {'compilers': { + 'gcc': { + 'versions': ['4.2.1', '6.3.0'] + }, 'clang': { + 'versions': ['8.0', '3.8'] + } + }}, + ] + } + } + spec_set = CombinatorialSpecSet(yaml_file, False) + specs = list(spec for spec in spec_set) + assert len(specs) == 8 + + +def test_spec_set_packages_no_matrix(): + """The matrix property is required, make sure we error out if it is + missing""" + yaml_file = { + 'spec-set': { + 'include': ['gmake'], + 'packages': { + 'gmake': { + 'versions': ['4.0'] + }, + 'appres': { + 'versions': ['1.0.4'] + }, + 'allinea-reports': { + 'versions': ['6.0.4'] + } + }, + } + } + with pytest.raises(ValidationError): + CombinatorialSpecSet(yaml_file) + + +def test_spec_set_get_cdash_array(): + """Make sure we can handle multiple cdash sites in a list""" + yaml_file = { + 'spec-set': { + 'cdash': ['http://example.com/cdash', 'http://example.com/cdash2'], + 'project': 'testproj', + 'matrix': [ + {'packages': { + 'gmake': {'versions': ['4.0']}, + }}, + {'compilers': { + 'gcc': {'versions': ['4.2.1', '6.3.0']}, + 'clang': {'versions': ['8.0', '3.8']}, + }}, + ] + } + } + + spec_set = CombinatorialSpecSet(yaml_file) + assert spec_set.cdash == [ + 'http://example.com/cdash', 'http://example.com/cdash2'] + assert spec_set.project == 'testproj' + + +def test_compiler_specs(): + spec_set = CombinatorialSpecSet(basic_yaml_file, False) + compilers = spec_set._compiler_specs({ + 'gcc': { + 'versions': ['4.2.1', '6.3.0'] + }, 'clang': { + 'versions': ['8.0', '3.8'] + }}) + + assert len(list(compilers)) == 4 + assert Spec('%gcc@4.2.1') in compilers + assert Spec('%gcc@6.3.0') in compilers + assert Spec('%clang@8.0') in compilers + assert Spec('%clang@3.8') in compilers + + +def test_package_specs(): + spec_set = CombinatorialSpecSet(basic_yaml_file, False) + + packages = spec_set._package_specs({ + 'gmake': { + 'versions': ['4.0', '5.0'] + }, + 'appres': { + 'versions': ['1.0.4'] + }, + 'allinea-reports': { + 'versions': ['6.0.1', '6.0.3', '6.0.4'] + } + }) + + assert Spec('gmake@4.0') in packages + assert Spec('gmake@5.0') in packages + assert Spec('appres@1.0.4') in packages + assert Spec('allinea-reports@6.0.1') in packages + assert Spec('allinea-reports@6.0.3') in packages + assert Spec('allinea-reports@6.0.4') in packages |