From bd91dd9d6d4607fd603cc1f3baaf753a5f4915bf Mon Sep 17 00:00:00 2001 From: Paul Hopkins Date: Mon, 11 Jul 2016 11:44:15 +0100 Subject: Allow users to supply preferred variants via packages.yaml --- lib/spack/spack/concretize.py | 9 +++++++-- lib/spack/spack/config.py | 8 +++++++- lib/spack/spack/preferred_packages.py | 7 +++++++ 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 386df08b2e..622a7efde5 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -254,13 +254,18 @@ class DefaultConcretizer(object): def concretize_variants(self, spec): """If the spec already has variants filled in, return. Otherwise, add - the default variants from the package specification. + the user preferences from packages.yaml or the default variants from + the package specification. """ changed = False + preferred_variants = spack.pkgsort.spec_preferred_variants(spec.package_class.name) for name, variant in spec.package_class.variants.items(): if name not in spec.variants: - spec.variants[name] = spack.spec.VariantSpec(name, variant.default) changed = True + if name in preferred_variants: + spec.variants[name] = preferred_variants.get(name) + else: + spec.variants[name] = spack.spec.VariantSpec(name, variant.default) return changed diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 8b5e96f97d..e2e7dbc0ee 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -257,7 +257,13 @@ section_schemas = { 'paths': { 'type' : 'object', 'default' : {}, - } + }, + 'variants': { + 'oneOf' : [ + { 'type' : 'string' }, + { 'type' : 'array', + 'items' : { 'type' : 'string' } }, + ], }, },},},},},}, 'modules': { diff --git a/lib/spack/spack/preferred_packages.py b/lib/spack/spack/preferred_packages.py index 1b94f03de7..5f18e212b6 100644 --- a/lib/spack/spack/preferred_packages.py +++ b/lib/spack/spack/preferred_packages.py @@ -158,6 +158,13 @@ class PreferredPackages(object): return bool(self._order_for_package(pkgname, 'providers', provider_str, False)) + def spec_preferred_variants(self, pkgname): + """Return a VariantMap of preferred variants and their values""" + variants = self.preferred.get(pkgname, {}).get('variants', '') + if not isinstance(variants, basestring): + variants = "".join(variants) + return spack.spec.Spec(pkgname + variants).variants + def version_compare(self, pkgname, a, b): """Return less-than-0, 0, or greater than 0 if version a of pkgname is respectively less-than, equal-to, or greater-than version b of -- cgit v1.2.3-70-g09d2 From 480fe9cb9ac2e0c6e2da4557abe9fd99fff6c64d Mon Sep 17 00:00:00 2001 From: Paul Hopkins Date: Mon, 11 Jul 2016 12:03:28 +0100 Subject: Re-add documentation for variant preferences --- lib/spack/docs/configuration.rst | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/lib/spack/docs/configuration.rst b/lib/spack/docs/configuration.rst index f2ffa07264..d39c932021 100644 --- a/lib/spack/docs/configuration.rst +++ b/lib/spack/docs/configuration.rst @@ -142,8 +142,9 @@ Here's an example packages.yaml file that sets preferred packages: .. code-block:: sh packages: - dyninst: + opencv: compiler: [gcc@4.9] + variants: +debug gperftools: version: [2.2, 2.4, 2.3] all: @@ -153,17 +154,17 @@ Here's an example packages.yaml file that sets preferred packages: At a high level, this example is specifying how packages should be -concretized. The dyninst package should prefer using gcc 4.9. -The gperftools package should prefer version +concretized. The opencv package should prefer using gcc 4.9 and +be built with debug options. The gperftools package should prefer version 2.2 over 2.4. Every package on the system should prefer mvapich for -its MPI and gcc 4.4.7 (except for Dyninst, which overrides this by preferring gcc 4.9). +its MPI and gcc 4.4.7 (except for opencv, which overrides this by preferring gcc 4.9). These options are used to fill in implicit defaults. Any of them can be overwritten on the command line if explicitly requested. Each packages.yaml file begins with the string ``packages:`` and package names are specified on the next level. The special string ``all`` applies settings to each package. Underneath each package name is -one or more components: ``compiler``, ``version``, +one or more components: ``compiler``, ``variants``, ``version``, or ``providers``. Each component has an ordered list of spec ``constraints``, with earlier entries in the list being preferred over later entries. -- cgit v1.2.3-70-g09d2 From 00280936da8993b2403af9ce6474bc08e494620b Mon Sep 17 00:00:00 2001 From: Paul Hopkins Date: Mon, 11 Jul 2016 16:14:09 +0100 Subject: Flake8 fixes to ensure qa tests pass --- lib/spack/spack/concretize.py | 174 +++++++++++++++++++++++++----------------- 1 file changed, 103 insertions(+), 71 deletions(-) diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 622a7efde5..5180f3cf04 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -40,12 +40,12 @@ import spack.architecture import spack.error from spack.version import * from functools import partial -from spec import DependencyMap from itertools import chain from spack.config import * class DefaultConcretizer(object): + """This class doesn't have any state, it just provides some methods for concretization. You can subclass it to override just some of the default concretization strategies, or you can override all of them. @@ -61,14 +61,17 @@ class DefaultConcretizer(object): if not providers: raise UnsatisfiableProviderSpecError(providers[0], spec) spec_w_preferred_providers = find_spec( - spec, lambda(x): spack.pkgsort.spec_has_preferred_provider(x.name, spec.name)) + spec, lambda x: spack.pkgsort.spec_has_preferred_provider(x.name, spec.name)) # NOQA: ignore=E501 if not spec_w_preferred_providers: spec_w_preferred_providers = spec - provider_cmp = partial(spack.pkgsort.provider_compare, spec_w_preferred_providers.name, spec.name) + provider_cmp = partial(spack.pkgsort.provider_compare, + spec_w_preferred_providers.name, + spec.name) candidates = sorted(providers, cmp=provider_cmp) - # For each candidate package, if it has externals, add those to the usable list. - # if it's not buildable, then *only* add the externals. + # For each candidate package, if it has externals, add those + # to the usable list. if it's not buildable, then *only* add + # the externals. usable = [] for cspec in candidates: if is_spec_buildable(cspec): @@ -85,7 +88,7 @@ class DefaultConcretizer(object): def cmp_externals(a, b): if a.name != b.name and (not a.external or a.external_module and - not b.external and b.external_module): + not b.external and b.external_module): # We're choosing between different providers, so # maintain order from provider sort return candidates.index(a) - candidates.index(b) @@ -114,26 +117,26 @@ class DefaultConcretizer(object): # Find the nearest spec in the dag that has a compiler. We'll # use that spec to calibrate compiler compatibility. - abi_exemplar = find_spec(spec, lambda(x): x.compiler) + abi_exemplar = find_spec(spec, lambda x: x.compiler) if not abi_exemplar: abi_exemplar = spec.root # Make a list including ABI compatibility of specs with the exemplar. strict = [spack.abi.compatible(c, abi_exemplar) for c in candidates] - loose = [spack.abi.compatible(c, abi_exemplar, loose=True) for c in candidates] + loose = [spack.abi.compatible(c, abi_exemplar, loose=True) + for c in candidates] keys = zip(strict, loose, candidates) # Sort candidates from most to least compatibility. # Note: # 1. We reverse because True > False. # 2. Sort is stable, so c's keep their order. - keys.sort(key=lambda k:k[:2], reverse=True) + keys.sort(key=lambda k: k[:2], reverse=True) # Pull the candidates back out and return them in order - candidates = [c for s,l,c in keys] + candidates = [c for s, l, c in keys] return candidates - def concretize_version(self, spec): """If the spec is already concrete, return. Otherwise take the preferred version from spackconfig, and default to the package's @@ -167,7 +170,8 @@ class DefaultConcretizer(object): if valid_versions: # Disregard @develop and take the next valid version - if ver(valid_versions[0]) == ver('develop') and len(valid_versions) > 1: + if ver(valid_versions[0]) == ver('develop') and \ + len(valid_versions) > 1: spec.versions = ver([valid_versions[1]]) else: spec.versions = ver([valid_versions[0]]) @@ -193,40 +197,45 @@ class DefaultConcretizer(object): return True # Things changed def _concretize_operating_system(self, spec): - platform = spec.architecture.platform if spec.architecture.platform_os is not None and isinstance( - spec.architecture.platform_os,spack.architecture.OperatingSystem): + spec.architecture.platform_os, + spack.architecture.OperatingSystem): return False if spec.root.architecture and spec.root.architecture.platform_os: - if isinstance(spec.root.architecture.platform_os,spack.architecture.OperatingSystem): - spec.architecture.platform_os = spec.root.architecture.platform_os + if isinstance(spec.root.architecture.platform_os, + spack.architecture.OperatingSystem): + spec.architecture.platform_os = \ + spec.root.architecture.platform_os else: - spec.architecture.platform_os = spec.architecture.platform.operating_system('default_os') - return True #changed + spec.architecture.platform_os = \ + spec.architecture.platform.operating_system('default_os') + return True # changed def _concretize_target(self, spec): - platform = spec.architecture.platform if spec.architecture.target is not None and isinstance( spec.architecture.target, spack.architecture.Target): return False if spec.root.architecture and spec.root.architecture.target: - if isinstance(spec.root.architecture.target,spack.architecture.Target): + if isinstance(spec.root.architecture.target, + spack.architecture.Target): spec.architecture.target = spec.root.architecture.target else: - spec.architecture.target = spec.architecture.platform.target('default_target') - return True #changed + spec.architecture.target = spec.architecture.platform.target( + 'default_target') + return True # changed def _concretize_platform(self, spec): if spec.architecture.platform is not None and isinstance( spec.architecture.platform, spack.architecture.Platform): return False if spec.root.architecture and spec.root.architecture.platform: - if isinstance(spec.root.architecture.platform,spack.architecture.Platform): + if isinstance(spec.root.architecture.platform, + spack.architecture.Platform): spec.architecture.platform = spec.root.architecture.platform else: spec.architecture.platform = spack.architecture.platform() - return True #changed? + return True # changed? def concretize_architecture(self, spec): """If the spec is empty provide the defaults of the platform. If the @@ -245,30 +254,29 @@ class DefaultConcretizer(object): return True # Concretize the operating_system and target based of the spec - ret = any((self._concretize_platform(spec), - self._concretize_operating_system(spec), - self._concretize_target(spec))) + ret = any((self._concretize_platform(spec), + self._concretize_operating_system(spec), + self._concretize_target(spec))) return ret - - def concretize_variants(self, spec): """If the spec already has variants filled in, return. Otherwise, add the user preferences from packages.yaml or the default variants from the package specification. """ changed = False - preferred_variants = spack.pkgsort.spec_preferred_variants(spec.package_class.name) + preferred_variants = spack.pkgsort.spec_preferred_variants( + spec.package_class.name) for name, variant in spec.package_class.variants.items(): if name not in spec.variants: changed = True if name in preferred_variants: spec.variants[name] = preferred_variants.get(name) else: - spec.variants[name] = spack.spec.VariantSpec(name, variant.default) + spec.variants[name] = \ + spack.spec.VariantSpec(name, variant.default) return changed - def concretize_compiler(self, spec): """If the spec already has a compiler, we're done. If not, then take the compiler used for the nearest ancestor with a compiler @@ -283,30 +291,32 @@ class DefaultConcretizer(object): """ # Pass on concretizing the compiler if the target is not yet determined if not spec.architecture.platform_os: - #Although this usually means changed, this means awaiting other changes + # Although this usually means changed, this means awaiting other + # changes return True # Only use a matching compiler if it is of the proper style - # Takes advantage of the proper logic already existing in compiler_for_spec - # Should think whether this can be more efficient + # Takes advantage of the proper logic already existing in + # compiler_for_spec Should think whether this can be more + # efficient def _proper_compiler_style(cspec, arch): platform = arch.platform compilers = spack.compilers.compilers_for_spec(cspec, platform=platform) return filter(lambda c: c.operating_system == - arch.platform_os, compilers) - #return compilers - + arch.platform_os, compilers) + # return compilers all_compilers = spack.compilers.all_compilers() if (spec.compiler and spec.compiler.concrete and - spec.compiler in all_compilers): + spec.compiler in all_compilers): return False - #Find the another spec that has a compiler, or the root if none do - other_spec = spec if spec.compiler else find_spec(spec, lambda(x) : x.compiler) + # Find the another spec that has a compiler, or the root if none do + other_spec = spec if spec.compiler else find_spec( + spec, lambda x: x.compiler) if not other_spec: other_spec = spec.root @@ -318,9 +328,12 @@ class DefaultConcretizer(object): spec.compiler = other_compiler.copy() return True - # Filter the compilers into a sorted list based on the compiler_order from spackconfig - compiler_list = all_compilers if not other_compiler else spack.compilers.find(other_compiler) - cmp_compilers = partial(spack.pkgsort.compiler_compare, other_spec.name) + # Filter the compilers into a sorted list based on the compiler_order + # from spackconfig + compiler_list = all_compilers if not other_compiler else \ + spack.compilers.find(other_compiler) + cmp_compilers = partial( + spack.pkgsort.compiler_compare, other_spec.name) matches = sorted(compiler_list, cmp=cmp_compilers) if not matches: raise UnavailableCompilerVersionError(other_compiler) @@ -335,7 +348,6 @@ class DefaultConcretizer(object): assert(spec.compiler.concrete) return True # things changed. - def concretize_compiler_flags(self, spec): """ The compiler flags are updated to match those of the spec whose @@ -343,54 +355,66 @@ class DefaultConcretizer(object): Default specs set at the compiler level will still be added later. """ - if not spec.architecture.platform_os: - #Although this usually means changed, this means awaiting other changes + # Although this usually means changed, this means awaiting other + # changes return True ret = False for flag in spack.spec.FlagMap.valid_compiler_flags(): try: nearest = next(p for p in spec.traverse(direction='parents') - if ((p.compiler == spec.compiler and p is not spec) - and flag in p.compiler_flags)) - if not flag in spec.compiler_flags or \ - not (sorted(spec.compiler_flags[flag]) >= sorted(nearest.compiler_flags[flag])): + if ((p.compiler == spec.compiler and + p is not spec) and + flag in p.compiler_flags)) + if flag not in spec.compiler_flags or \ + not (sorted(spec.compiler_flags[flag]) >= + sorted(nearest.compiler_flags[flag])): if flag in spec.compiler_flags: - spec.compiler_flags[flag] = list(set(spec.compiler_flags[flag]) | - set(nearest.compiler_flags[flag])) + spec.compiler_flags[flag] = list( + set(spec.compiler_flags[flag]) | + set(nearest.compiler_flags[flag])) else: - spec.compiler_flags[flag] = nearest.compiler_flags[flag] + spec.compiler_flags[ + flag] = nearest.compiler_flags[flag] ret = True except StopIteration: - if (flag in spec.root.compiler_flags and ((not flag in spec.compiler_flags) or - sorted(spec.compiler_flags[flag]) != sorted(spec.root.compiler_flags[flag]))): + if (flag in spec.root.compiler_flags and + ((flag not in spec.compiler_flags) or + sorted(spec.compiler_flags[flag]) != + sorted(spec.root.compiler_flags[flag]))): if flag in spec.compiler_flags: - spec.compiler_flags[flag] = list(set(spec.compiler_flags[flag]) | - set(spec.root.compiler_flags[flag])) + spec.compiler_flags[flag] = list( + set(spec.compiler_flags[flag]) | + set(spec.root.compiler_flags[flag])) else: - spec.compiler_flags[flag] = spec.root.compiler_flags[flag] + spec.compiler_flags[ + flag] = spec.root.compiler_flags[flag] ret = True else: - if not flag in spec.compiler_flags: + if flag not in spec.compiler_flags: spec.compiler_flags[flag] = [] # Include the compiler flag defaults from the config files # This ensures that spack will detect conflicts that stem from a change # in default compiler flags. - compiler = spack.compilers.compiler_for_spec(spec.compiler, spec.architecture) + compiler = spack.compilers.compiler_for_spec( + spec.compiler, spec.architecture) for flag in compiler.flags: if flag not in spec.compiler_flags: spec.compiler_flags[flag] = compiler.flags[flag] if compiler.flags[flag] != []: ret = True else: - if ((sorted(spec.compiler_flags[flag]) != sorted(compiler.flags[flag])) and - (not set(spec.compiler_flags[flag]) >= set(compiler.flags[flag]))): + if ((sorted(spec.compiler_flags[flag]) != + sorted(compiler.flags[flag])) and + (not set(spec.compiler_flags[flag]) >= + set(compiler.flags[flag]))): ret = True - spec.compiler_flags[flag] = list(set(spec.compiler_flags[flag]) | - set(compiler.flags[flag])) + spec.compiler_flags[flag] = list( + set(spec.compiler_flags[flag]) | + set(compiler.flags[flag])) return ret @@ -401,8 +425,8 @@ def find_spec(spec, condition): # First search parents, then search children deptype = ('build', 'link') dagiter = chain( - spec.traverse(direction='parents', deptype=deptype, root=False), - spec.traverse(direction='children', deptype=deptype, root=False)) + spec.traverse(direction='parents', deptype=deptype, root=False), + spec.traverse(direction='children', deptype=deptype, root=False)) visited = set() for relative in dagiter: if condition(relative): @@ -411,8 +435,10 @@ def find_spec(spec, condition): # Then search all other relatives in the DAG *except* spec for relative in spec.root.traverse(deptypes=spack.alldeps): - if relative is spec: continue - if id(relative) in visited: continue + if relative is spec: + continue + if id(relative) in visited: + continue if condition(relative): return relative @@ -459,8 +485,10 @@ def cmp_specs(lhs, rhs): class UnavailableCompilerVersionError(spack.error.SpackError): + """Raised when there is no available compiler that satisfies a compiler spec.""" + def __init__(self, compiler_spec): super(UnavailableCompilerVersionError, self).__init__( "No available compiler version matches '%s'" % compiler_spec, @@ -468,16 +496,20 @@ class UnavailableCompilerVersionError(spack.error.SpackError): class NoValidVersionError(spack.error.SpackError): + """Raised when there is no way to have a concrete version for a particular spec.""" + def __init__(self, spec): super(NoValidVersionError, self).__init__( - "There are no valid versions for %s that match '%s'" % (spec.name, spec.versions)) + "There are no valid versions for %s that match '%s'" % (spec.name, spec.versions)) # NOQA: ignore=E501 class NoBuildError(spack.error.SpackError): + """Raised when a package is configured with the buildable option False, but no satisfactory external versions can be found""" + def __init__(self, spec): super(NoBuildError, self).__init__( - "The spec '%s' is configured as not buildable, and no matching external installs were found" % spec.name) + "The spec '%s' is configured as not buildable,and no matching external installs were found" % spec.name) # NOQA: ignore=E501 -- cgit v1.2.3-70-g09d2