concretizer: virtual entry in packages.yaml, external modules

This commit adds support for specifying rules in
packages.yaml that refer to virtual packages.

The approach is to normalize in memory each
configuration and turn it into an equivalent
configuration without rules on virtual. This
is possible if the set of packages to be handled
is considered fixed.

2 files changed, 54 insertions, 11 deletions

diff --git a/lib/spack/spack/solver/asp.py b/lib/spack/spack/solver/asp.py
index 3575d58c03..935c5da9cd 100644
--- a/lib/spack/spack/solver/asp.py
+++ b/lib/spack/spack/solver/asp.py
@@ -6,6 +6,8 @@
 from __future__ import print_function
 
 import collections
+import copy
+import itertools
 import os
 import pkgutil
 import pprint
@@ -412,6 +414,30 @@ def _normalize(body):
     return args
 
 
+def _normalize_packages_yaml(packages_yaml):
+    normalized_yaml = copy.copy(packages_yaml)
+    for pkg_name in packages_yaml:
+        is_virtual = spack.repo.path.is_virtual(pkg_name)
+        if pkg_name == 'all' or not is_virtual:
+            continue
+
+        # Remove the virtual entry from the normalized configuration
+        data = normalized_yaml.pop(pkg_name)
+        is_buildable = data.get('buildable', True)
+        if not is_buildable:
+            for provider in spack.repo.path.providers_for(pkg_name):
+                entry = normalized_yaml.setdefault(provider.name, {})
+                entry['buildable'] = False
+
+        externals = data.get('externals', [])
+        keyfn = lambda x: spack.spec.Spec(x['spec']).name
+        for provider, specs in itertools.groupby(externals, key=keyfn):
+            entry = normalized_yaml.setdefault(provider, {})
+            entry.setdefault('externals', []).extend(specs)
+
+    return normalized_yaml
+
+
 class PyclingoDriver(object):
     def __init__(self, cores=True, asp=None):
         """Driver for the Python clingo interface.
@@ -934,7 +960,12 @@ class SpackSolverSetup(object):
 
     def external_packages(self):
         """Facts on external packages, as read from packages.yaml"""
+        # Read packages.yaml and normalize it, so that it
+        # will not contain entries referring to virtual
+        # packages.
         packages_yaml = spack.config.get("packages")
+        packages_yaml = _normalize_packages_yaml(packages_yaml)
+
         self.gen.h1('External packages')
         for pkg_name, data in packages_yaml.items():
             if pkg_name == 'all':
@@ -1526,6 +1557,7 @@ class SpecBuilder(object):
         has been selected for this package.
         """
         packages_yaml = spack.config.get('packages')
+        packages_yaml = _normalize_packages_yaml(packages_yaml)
         spec_info = packages_yaml[pkg]['externals'][int(idx)]
         self._specs[pkg].external_path = spec_info.get('prefix', None)
         self._specs[pkg].external_modules = spec_info.get('modules', [])
@@ -1623,6 +1655,10 @@ class SpecBuilder(object):
         # fix flags after all specs are constructed
         self.reorder_flags()
 
+        # Add external paths to specs with just external modules
+        for s in self._specs.values():
+            spack.spec.Spec.ensure_external_path_if_external(s)
+
         return self._specs
 
 
diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py
index 235d4ad147..5a9bb73ee1 100644
--- a/lib/spack/spack/spec.py
+++ b/lib/spack/spack/spec.py
@@ -2362,17 +2362,9 @@ class Spec(object):
                 t[-1] for t in ordered_hashes)
 
         for s in self.traverse():
-            if s.external_modules and not s.external_path:
-                compiler = spack.compilers.compiler_for_spec(
-                    s.compiler, s.architecture)
-                for mod in compiler.modules:
-                    md.load_module(mod)
-
-                # get the path from the module
-                # the package can override the default
-                s.external_path = getattr(s.package, 'external_prefix',
-                                          md.path_from_modules(
-                                              s.external_modules))
+            # TODO: Refactor this into a common method to build external specs
+            # TODO: or turn external_path into a lazy property
+            self.ensure_external_path_if_external(s)
 
         # Mark everything in the spec as concrete, as well.
         self._mark_concrete()
@@ -2419,6 +2411,21 @@ class Spec(object):
         # there are declared inconsistencies)
         self.architecture.target.optimization_flags(self.compiler)
 
+    @staticmethod
+    def ensure_external_path_if_external(external_spec):
+        if external_spec.external_modules and not external_spec.external_path:
+            compiler = spack.compilers.compiler_for_spec(
+                external_spec.compiler, external_spec.architecture)
+            for mod in compiler.modules:
+                md.load_module(mod)
+
+            # get the path from the module
+            # the package can override the default
+            external_spec.external_path = getattr(
+                external_spec.package, 'external_prefix',
+                md.path_from_modules(external_spec.external_modules)
+            )
+
     def _new_concretize(self, tests=False):
         import spack.solver.asp