From ce6ac93abed2a7193745f7082670e1cbe2ffede2 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 5 Jun 2016 00:52:52 -0700 Subject: rename `virtual` module to `provider_index` --- lib/spack/spack/provider_index.py | 255 +++++++++++++++++++++++++++++++++ lib/spack/spack/repository.py | 2 +- lib/spack/spack/spec.py | 15 +- lib/spack/spack/test/__init__.py | 5 +- lib/spack/spack/test/provider_index.py | 49 +++++++ lib/spack/spack/test/virtual.py | 49 ------- lib/spack/spack/virtual.py | 255 --------------------------------- 7 files changed, 317 insertions(+), 313 deletions(-) create mode 100644 lib/spack/spack/provider_index.py create mode 100644 lib/spack/spack/test/provider_index.py delete mode 100644 lib/spack/spack/test/virtual.py delete mode 100644 lib/spack/spack/virtual.py diff --git a/lib/spack/spack/provider_index.py b/lib/spack/spack/provider_index.py new file mode 100644 index 0000000000..785ab98918 --- /dev/null +++ b/lib/spack/spack/provider_index.py @@ -0,0 +1,255 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +""" +The ``virtual`` module contains utility classes for virtual dependencies. +""" +import itertools +import yaml +from yaml.error import MarkedYAMLError + +import spack + + +class ProviderIndex(object): + """This is a dict of dicts used for finding providers of particular + virtual dependencies. The dict of dicts looks like: + + { vpkg name : + { full vpkg spec : set(packages providing spec) } } + + Callers can use this to first find which packages provide a vpkg, + then find a matching full spec. e.g., in this scenario: + + { 'mpi' : + { mpi@:1.1 : set([mpich]), + mpi@:2.3 : set([mpich2@1.9:]) } } + + Calling providers_for(spec) will find specs that provide a + matching implementation of MPI. + """ + def __init__(self, specs=None, **kwargs): + # TODO: come up with another name for this. This "restricts" values to + # the verbatim impu specs (i.e., it doesn't pre-apply package's constraints, and + # keeps things as broad as possible, so it's really the wrong name) + if specs is None: specs = [] + self.restrict = kwargs.setdefault('restrict', False) + + self.providers = {} + + for spec in specs: + if not isinstance(spec, spack.spec.Spec): + spec = spack.spec.Spec(spec) + + if spec.virtual: + continue + + self.update(spec) + + + def update(self, spec): + if not isinstance(spec, spack.spec.Spec): + spec = spack.spec.Spec(spec) + + if not spec.name: + # Empty specs do not have a package + return + + assert(not spec.virtual) + + pkg = spec.package + for provided_spec, provider_spec in pkg.provided.iteritems(): + # We want satisfaction other than flags + provider_spec.compiler_flags = spec.compiler_flags.copy() + if provider_spec.satisfies(spec, deps=False): + provided_name = provided_spec.name + + provider_map = self.providers.setdefault(provided_name, {}) + if not provided_spec in provider_map: + provider_map[provided_spec] = set() + + if self.restrict: + provider_set = provider_map[provided_spec] + + # If this package existed in the index before, + # need to take the old versions out, as they're + # now more constrained. + old = set([s for s in provider_set if s.name == spec.name]) + provider_set.difference_update(old) + + # Now add the new version. + provider_set.add(spec) + + else: + # Before putting the spec in the map, constrain it so that + # it provides what was asked for. + constrained = spec.copy() + constrained.constrain(provider_spec) + provider_map[provided_spec].add(constrained) + + + def providers_for(self, *vpkg_specs): + """Gives specs of all packages that provide virtual packages + with the supplied specs.""" + providers = set() + for vspec in vpkg_specs: + # Allow string names to be passed as input, as well as specs + if type(vspec) == str: + vspec = spack.spec.Spec(vspec) + + # Add all the providers that satisfy the vpkg spec. + if vspec.name in self.providers: + for provider_spec, spec_set in self.providers[vspec.name].items(): + if provider_spec.satisfies(vspec, deps=False): + providers.update(spec_set) + + # Return providers in order + return sorted(providers) + + + # TODO: this is pretty darned nasty, and inefficient, but there + # are not that many vdeps in most specs. + def _cross_provider_maps(self, lmap, rmap): + result = {} + for lspec, rspec in itertools.product(lmap, rmap): + try: + constrained = lspec.constrained(rspec) + except spack.spec.UnsatisfiableSpecError: + continue + + # lp and rp are left and right provider specs. + for lp_spec, rp_spec in itertools.product(lmap[lspec], rmap[rspec]): + if lp_spec.name == rp_spec.name: + try: + const = lp_spec.constrained(rp_spec, deps=False) + result.setdefault(constrained, set()).add(const) + except spack.spec.UnsatisfiableSpecError: + continue + return result + + + def __contains__(self, name): + """Whether a particular vpkg name is in the index.""" + return name in self.providers + + + def satisfies(self, other): + """Check that providers of virtual specs are compatible.""" + common = set(self.providers) & set(other.providers) + if not common: + return True + + # This ensures that some provider in other COULD satisfy the + # vpkg constraints on self. + result = {} + for name in common: + crossed = self._cross_provider_maps(self.providers[name], + other.providers[name]) + if crossed: + result[name] = crossed + + return all(c in result for c in common) + + + def to_yaml(self, stream=None): + provider_list = dict( + (name, [[vpkg.to_node_dict(), [p.to_node_dict() for p in pset]] + for vpkg, pset in pdict.items()]) + for name, pdict in self.providers.items()) + + yaml.dump({'provider_index': {'providers': provider_list}}, + stream=stream) + + + @staticmethod + def from_yaml(stream): + try: + yfile = yaml.load(stream) + except MarkedYAMLError, e: + raise spack.spec.SpackYAMLError( + "error parsing YAML ProviderIndex cache:", str(e)) + + if not isinstance(yfile, dict): + raise spack.spec.SpackYAMLError( + "YAML ProviderIndex was not a dict.") + + if not 'provider_index' in yfile: + raise spack.spec.SpackYAMLError( + "YAML ProviderIndex does not start with 'provider_index'") + + index = ProviderIndex() + providers = yfile['provider_index']['providers'] + index.providers = dict( + (name, dict((spack.spec.Spec.from_node_dict(vpkg), + set(spack.spec.Spec.from_node_dict(p) for p in plist)) + for vpkg, plist in pdict_list)) + for name, pdict_list in providers.items()) + + return index + + + def merge(self, other): + """Merge `other` ProviderIndex into this one.""" + other = other.copy() # defensive copy. + + for pkg in other.providers: + if pkg not in self.providers: + self.providers[pkg] = other.providers[pkg] + continue + + spdict, opdict = self.providers[pkg], other.providers[pkg] + for provided_spec in opdict: + if provided_spec not in spdict: + spdict[provided_spec] = opdict[provided_spec] + continue + + spdict[provided_spec] += opdict[provided_spec] + + + def remove_provider(self, pkg_name): + """Remove a provider from the ProviderIndex.""" + for pkg in self.providers: + pkg_dict = self.providers[pkg] + for provided, pset in pkg_dict.items(): + for provider in pset: + if provider.fullname == pkg_name: + pset.remove(provider) + if not pset: + del pkg_dict[provided] + if not pkg_dict: + del self.providers[pkg] + + + def copy(self): + """Deep copy of this ProviderIndex.""" + clone = ProviderIndex() + clone.providers = dict( + (name, dict((vpkg, set((p.copy() for p in pset))) + for vpkg, pset in pdict.items())) + for name, pdict in self.providers.items()) + return clone + + + def __eq__(self, other): + return self.providers == other.providers diff --git a/lib/spack/spack/repository.py b/lib/spack/spack/repository.py index 6aa9b8dd2f..3b0d3167b3 100644 --- a/lib/spack/spack/repository.py +++ b/lib/spack/spack/repository.py @@ -42,7 +42,7 @@ from llnl.util.filesystem import * import spack.error import spack.config import spack.spec -from spack.virtual import ProviderIndex +from spack.provider_index import ProviderIndex from spack.util.naming import * # diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 24459fd3b3..8a47ec95ad 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -102,23 +102,26 @@ import sys from StringIO import StringIO from operator import attrgetter +import yaml +from yaml.error import MarkedYAMLError + import llnl.util.tty as tty +from llnl.util.filesystem import join_path +from llnl.util.lang import * +from llnl.util.tty.color import * + import spack import spack.architecture import spack.compilers as compilers import spack.error import spack.parse -import yaml -from llnl.util.filesystem import join_path -from llnl.util.lang import * -from llnl.util.tty.color import * from spack.build_environment import get_path_from_module, load_module from spack.util.naming import mod_to_class from spack.util.prefix import Prefix from spack.util.string import * from spack.version import * -from spack.virtual import ProviderIndex -from yaml.error import MarkedYAMLError +from spack.provider_index import ProviderIndex + # Valid pattern for an identifier in Spack identifier_re = r'\w[\w-]*' diff --git a/lib/spack/spack/test/__init__.py b/lib/spack/spack/test/__init__.py index 11f7298c04..e092a50913 100644 --- a/lib/spack/spack/test/__init__.py +++ b/lib/spack/spack/test/__init__.py @@ -32,6 +32,8 @@ from llnl.util.tty.colify import colify from spack.test.tally_plugin import Tally """Names of tests to be included in Spack's test suite""" +# All the tests Spack knows about. +# Keep these one per line so that it's easy to see changes in diffs. test_names = [ 'architecture', 'build_system_guess', @@ -74,11 +76,10 @@ test_names = [ 'url_parse', 'url_substitution', 'versions', - 'virtual', + 'provider_index', 'yaml', ] - def list_tests(): """Return names of all tests that can be run for Spack.""" return test_names diff --git a/lib/spack/spack/test/provider_index.py b/lib/spack/spack/test/provider_index.py new file mode 100644 index 0000000000..15fb9acff2 --- /dev/null +++ b/lib/spack/spack/test/provider_index.py @@ -0,0 +1,49 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from StringIO import StringIO +import unittest + +import spack +from spack.provider_index import ProviderIndex + + +class ProviderIndexTest(unittest.TestCase): + + def test_write_and_read(self): + p = ProviderIndex(spack.repo.all_package_names()) + + ostream = StringIO() + p.to_yaml(ostream) + + istream = StringIO(ostream.getvalue()) + q = ProviderIndex.from_yaml(istream) + + self.assertTrue(p == q) + + + def test_copy(self): + p = ProviderIndex(spack.repo.all_package_names()) + q = p.copy() + self.assertTrue(p == q) diff --git a/lib/spack/spack/test/virtual.py b/lib/spack/spack/test/virtual.py deleted file mode 100644 index 1923e7006f..0000000000 --- a/lib/spack/spack/test/virtual.py +++ /dev/null @@ -1,49 +0,0 @@ -############################################################################## -# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. -# Produced at the Lawrence Livermore National Laboratory. -# -# This file is part of Spack. -# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. -# LLNL-CODE-647188 -# -# For details, see https://github.com/llnl/spack -# Please also see the LICENSE file for our notice and the LGPL. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU Lesser General Public License (as -# published by the Free Software Foundation) version 2.1, February 1999. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and -# conditions of the GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -############################################################################## -from StringIO import StringIO -import unittest - -import spack -from spack.virtual import ProviderIndex - - -class VirtualTest(unittest.TestCase): - - def test_write_and_read(self): - p = ProviderIndex(spack.repo.all_package_names()) - - ostream = StringIO () - p.to_yaml(ostream) - - istream = StringIO(ostream.getvalue()) - q = ProviderIndex.from_yaml(istream) - - self.assertTrue(p == q) - - - def test_copy(self): - p = ProviderIndex(spack.repo.all_package_names()) - q = p.copy() - self.assertTrue(p == q) diff --git a/lib/spack/spack/virtual.py b/lib/spack/spack/virtual.py deleted file mode 100644 index 785ab98918..0000000000 --- a/lib/spack/spack/virtual.py +++ /dev/null @@ -1,255 +0,0 @@ -############################################################################## -# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. -# Produced at the Lawrence Livermore National Laboratory. -# -# This file is part of Spack. -# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. -# LLNL-CODE-647188 -# -# For details, see https://github.com/llnl/spack -# Please also see the LICENSE file for our notice and the LGPL. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU Lesser General Public License (as -# published by the Free Software Foundation) version 2.1, February 1999. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and -# conditions of the GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -############################################################################## -""" -The ``virtual`` module contains utility classes for virtual dependencies. -""" -import itertools -import yaml -from yaml.error import MarkedYAMLError - -import spack - - -class ProviderIndex(object): - """This is a dict of dicts used for finding providers of particular - virtual dependencies. The dict of dicts looks like: - - { vpkg name : - { full vpkg spec : set(packages providing spec) } } - - Callers can use this to first find which packages provide a vpkg, - then find a matching full spec. e.g., in this scenario: - - { 'mpi' : - { mpi@:1.1 : set([mpich]), - mpi@:2.3 : set([mpich2@1.9:]) } } - - Calling providers_for(spec) will find specs that provide a - matching implementation of MPI. - """ - def __init__(self, specs=None, **kwargs): - # TODO: come up with another name for this. This "restricts" values to - # the verbatim impu specs (i.e., it doesn't pre-apply package's constraints, and - # keeps things as broad as possible, so it's really the wrong name) - if specs is None: specs = [] - self.restrict = kwargs.setdefault('restrict', False) - - self.providers = {} - - for spec in specs: - if not isinstance(spec, spack.spec.Spec): - spec = spack.spec.Spec(spec) - - if spec.virtual: - continue - - self.update(spec) - - - def update(self, spec): - if not isinstance(spec, spack.spec.Spec): - spec = spack.spec.Spec(spec) - - if not spec.name: - # Empty specs do not have a package - return - - assert(not spec.virtual) - - pkg = spec.package - for provided_spec, provider_spec in pkg.provided.iteritems(): - # We want satisfaction other than flags - provider_spec.compiler_flags = spec.compiler_flags.copy() - if provider_spec.satisfies(spec, deps=False): - provided_name = provided_spec.name - - provider_map = self.providers.setdefault(provided_name, {}) - if not provided_spec in provider_map: - provider_map[provided_spec] = set() - - if self.restrict: - provider_set = provider_map[provided_spec] - - # If this package existed in the index before, - # need to take the old versions out, as they're - # now more constrained. - old = set([s for s in provider_set if s.name == spec.name]) - provider_set.difference_update(old) - - # Now add the new version. - provider_set.add(spec) - - else: - # Before putting the spec in the map, constrain it so that - # it provides what was asked for. - constrained = spec.copy() - constrained.constrain(provider_spec) - provider_map[provided_spec].add(constrained) - - - def providers_for(self, *vpkg_specs): - """Gives specs of all packages that provide virtual packages - with the supplied specs.""" - providers = set() - for vspec in vpkg_specs: - # Allow string names to be passed as input, as well as specs - if type(vspec) == str: - vspec = spack.spec.Spec(vspec) - - # Add all the providers that satisfy the vpkg spec. - if vspec.name in self.providers: - for provider_spec, spec_set in self.providers[vspec.name].items(): - if provider_spec.satisfies(vspec, deps=False): - providers.update(spec_set) - - # Return providers in order - return sorted(providers) - - - # TODO: this is pretty darned nasty, and inefficient, but there - # are not that many vdeps in most specs. - def _cross_provider_maps(self, lmap, rmap): - result = {} - for lspec, rspec in itertools.product(lmap, rmap): - try: - constrained = lspec.constrained(rspec) - except spack.spec.UnsatisfiableSpecError: - continue - - # lp and rp are left and right provider specs. - for lp_spec, rp_spec in itertools.product(lmap[lspec], rmap[rspec]): - if lp_spec.name == rp_spec.name: - try: - const = lp_spec.constrained(rp_spec, deps=False) - result.setdefault(constrained, set()).add(const) - except spack.spec.UnsatisfiableSpecError: - continue - return result - - - def __contains__(self, name): - """Whether a particular vpkg name is in the index.""" - return name in self.providers - - - def satisfies(self, other): - """Check that providers of virtual specs are compatible.""" - common = set(self.providers) & set(other.providers) - if not common: - return True - - # This ensures that some provider in other COULD satisfy the - # vpkg constraints on self. - result = {} - for name in common: - crossed = self._cross_provider_maps(self.providers[name], - other.providers[name]) - if crossed: - result[name] = crossed - - return all(c in result for c in common) - - - def to_yaml(self, stream=None): - provider_list = dict( - (name, [[vpkg.to_node_dict(), [p.to_node_dict() for p in pset]] - for vpkg, pset in pdict.items()]) - for name, pdict in self.providers.items()) - - yaml.dump({'provider_index': {'providers': provider_list}}, - stream=stream) - - - @staticmethod - def from_yaml(stream): - try: - yfile = yaml.load(stream) - except MarkedYAMLError, e: - raise spack.spec.SpackYAMLError( - "error parsing YAML ProviderIndex cache:", str(e)) - - if not isinstance(yfile, dict): - raise spack.spec.SpackYAMLError( - "YAML ProviderIndex was not a dict.") - - if not 'provider_index' in yfile: - raise spack.spec.SpackYAMLError( - "YAML ProviderIndex does not start with 'provider_index'") - - index = ProviderIndex() - providers = yfile['provider_index']['providers'] - index.providers = dict( - (name, dict((spack.spec.Spec.from_node_dict(vpkg), - set(spack.spec.Spec.from_node_dict(p) for p in plist)) - for vpkg, plist in pdict_list)) - for name, pdict_list in providers.items()) - - return index - - - def merge(self, other): - """Merge `other` ProviderIndex into this one.""" - other = other.copy() # defensive copy. - - for pkg in other.providers: - if pkg not in self.providers: - self.providers[pkg] = other.providers[pkg] - continue - - spdict, opdict = self.providers[pkg], other.providers[pkg] - for provided_spec in opdict: - if provided_spec not in spdict: - spdict[provided_spec] = opdict[provided_spec] - continue - - spdict[provided_spec] += opdict[provided_spec] - - - def remove_provider(self, pkg_name): - """Remove a provider from the ProviderIndex.""" - for pkg in self.providers: - pkg_dict = self.providers[pkg] - for provided, pset in pkg_dict.items(): - for provider in pset: - if provider.fullname == pkg_name: - pset.remove(provider) - if not pset: - del pkg_dict[provided] - if not pkg_dict: - del self.providers[pkg] - - - def copy(self): - """Deep copy of this ProviderIndex.""" - clone = ProviderIndex() - clone.providers = dict( - (name, dict((vpkg, set((p.copy() for p in pset))) - for vpkg, pset in pdict.items())) - for name, pdict in self.providers.items()) - return clone - - - def __eq__(self, other): - return self.providers == other.providers -- cgit v1.2.3-60-g2f50