concretizer: optimize microarchitectures, constrained by compiler support

Weight microarchitectures and prefers more rercent ones.  Also disallow
nodes where the compiler does not support the selected target.

We should revisit this at some point as it seems like if I play around
with the compiler support for different architectures, the solver runs
very slowly.  See notes in comments -- the bad case was gcc supporting
broadwell and skylake with clang maxing out at haswell.

3 files changed, 145 insertions, 66 deletions

diff --git a/lib/spack/spack/solver/asp.py b/lib/spack/spack/solver/asp.py
index cc0616f24d..d85eae81bf 100644
--- a/lib/spack/spack/solver/asp.py
+++ b/lib/spack/spack/solver/asp.py
@@ -265,6 +265,8 @@ class AspGenerator(object):
 
     def available_compilers(self):
         """Facts about available compilers."""
+
+        self.h2("Available compilers")
         compilers = spack.compilers.all_compiler_specs()
 
         compiler_versions = collections.defaultdict(lambda: set())
@@ -438,18 +440,18 @@ class AspGenerator(object):
         # TODO: do this with consistent suffixes.
         class Head(object):
             node = fn.node
-            arch_platform = fn.arch_platform_set
-            arch_os = fn.arch_os_set
-            arch_target = fn.arch_target_set
+            node_platform = fn.node_platform_set
+            node_os = fn.node_os_set
+            node_target = fn.node_target_set
             variant = fn.variant_set
             node_compiler = fn.node_compiler
             node_compiler_version = fn.node_compiler_version
 
         class Body(object):
             node = fn.node
-            arch_platform = fn.arch_platform
-            arch_os = fn.arch_os
-            arch_target = fn.arch_target
+            node_platform = fn.node_platform
+            node_os = fn.node_os
+            node_target = fn.node_target
             variant = fn.variant_value
             node_compiler = fn.node_compiler
             node_compiler_version = fn.node_compiler_version
@@ -466,11 +468,11 @@ class AspGenerator(object):
         arch = spec.architecture
         if arch:
             if arch.platform:
-                clauses.append(f.arch_platform(spec.name, arch.platform))
+                clauses.append(f.node_platform(spec.name, arch.platform))
             if arch.os:
-                clauses.append(f.arch_os(spec.name, arch.os))
+                clauses.append(f.node_os(spec.name, arch.os))
             if arch.target:
-                clauses.append(f.arch_target(spec.name, arch.target))
+                clauses.append(f.node_target(spec.name, arch.target))
 
         # variants
         for vname, variant in sorted(spec.variants.items()):
@@ -528,13 +530,83 @@ class AspGenerator(object):
                 if dep.versions.concrete:
                     self.possible_versions[dep.name].add(dep.version)
 
+    def _supported_targets(self, compiler, targets):
+        """Get a list of which targets are supported by the compiler.
+
+        Results are ordered most to least recent.
+        """
+        supported = []
+
+        for target in targets:
+            compiler_info = target.compilers.get(compiler.name)
+            if not compiler_info:
+                # if we don't know, we assume it's supported and leave it
+                # to the user to debug
+                supported.append(target)
+                continue
+
+            if not isinstance(compiler_info, list):
+                compiler_info = [compiler_info]
+
+            for info in compiler_info:
+                version = ver(info['versions'])
+                if compiler.version.satisfies(version):
+                    supported.append(target)
+
+        return sorted(supported, reverse=True)
+
     def arch_defaults(self):
         """Add facts about the default architecture for a package."""
         self.h2('Default architecture')
-        default_arch = spack.spec.ArchSpec(spack.architecture.sys_type())
-        self.fact(fn.arch_platform_default(default_arch.platform))
-        self.fact(fn.arch_os_default(default_arch.os))
-        self.fact(fn.arch_target_default(default_arch.target))
+        default = spack.spec.ArchSpec(spack.architecture.sys_type())
+        self.fact(fn.node_platform_default(default.platform))
+        self.fact(fn.node_os_default(default.os))
+        self.fact(fn.node_target_default(default.target))
+
+        uarch = default.target.microarchitecture
+
+        self.h2('Target compatibility')
+
+        # listing too many targets can be slow, at least with our current
+        # encoding. To reduce the number of options to consider, only
+        # consider the *best* target that each compiler supports, along
+        # with the family.
+        compatible_targets = [uarch] + uarch.ancestors
+        compilers = self.compilers_for_default_arch()
+
+        # this loop can be used to limit the number of targets
+        # considered. Right now we consider them all, but it seems that
+        # many targets can make things slow.
+        # TODO: investigate this.
+        best_targets = set([uarch.family.name])
+        for compiler in compilers:
+            supported = self._supported_targets(compiler, compatible_targets)
+            if not supported:
+                continue
+
+            for target in supported:
+                best_targets.add(target.name)
+                self.fact(fn.compiler_supports_target(
+                    compiler.name, compiler.version, target.name))
+                self.fact(fn.compiler_supports_target(
+                    compiler.name, compiler.version, uarch.family.name))
+
+        i = 0
+        for target in compatible_targets:
+            self.fact(fn.target(target.name))
+            self.fact(fn.target_family(target.name, target.family.name))
+            for parent in sorted(target.parents):
+                self.fact(fn.target_parent(target.name, parent.name))
+
+            # prefer best possible targets; weight others poorly so
+            # they're not used unless set explicitly
+            if target.name in best_targets:
+                self.fact(fn.target_weight(target.name, i))
+                i += 1
+            else:
+                self.fact(fn.target_weight(target.name, 100))
+
+            self.out.write("\n")
 
     def virtual_providers(self):
         self.h2("Virtual providers")
@@ -556,20 +628,13 @@ class AspGenerator(object):
             specs (list): list of Specs to solve
         """
         # get list of all possible dependencies
-        pkg_names = set(spec.fullname for spec in specs)
-
-        possible = set()
         self.possible_virtuals = set()
-        for name in pkg_names:
-            pkg = spack.repo.path.get_pkg_class(name)
-            possible.update(
-                pkg.possible_dependencies(
-                    virtuals=self.possible_virtuals,
-                    deptype=("build", "link", "run")
-                )
-            )
-
-        pkgs = set(possible) | set(pkg_names)
+        possible = spack.package.possible_dependencies(
+            *specs,
+            virtuals=self.possible_virtuals,
+            deptype=("build", "link", "run")
+        )
+        pkgs = set(possible)
 
         # get possible compilers
         self.possible_compilers = self.compilers_for_default_arch()
@@ -623,13 +688,13 @@ class ResultParser(object):
             self._specs[pkg].architecture = arch
         return arch
 
-    def arch_platform(self, pkg, platform):
+    def node_platform(self, pkg, platform):
         self._arch(pkg).platform = platform
 
-    def arch_os(self, pkg, os):
+    def node_os(self, pkg, os):
         self._arch(pkg).os = os
 
-    def arch_target(self, pkg, target):
+    def node_target(self, pkg, target):
         self._arch(pkg).target = target
 
     def variant_value(self, pkg, name, value):
diff --git a/lib/spack/spack/solver/concretize.lp b/lib/spack/spack/solver/concretize.lp
index 6f5b584dfa..9f0c09c701 100644
--- a/lib/spack/spack/solver/concretize.lp
+++ b/lib/spack/spack/solver/concretize.lp
@@ -107,40 +107,47 @@ variant_not_default(P, V, X, 0)
 #defined variant_single_value/2.
 
 %-----------------------------------------------------------------------------
-% Architecture semantics
+% Platform/OS semantics
 %-----------------------------------------------------------------------------
-
-% one platform, os, target per node.
-1 { arch_platform(P, A) : arch_platform(P, A) } 1 :- node(P).
-1 { arch_os(P, A) : arch_os(P, A) } 1             :- node(P).
-1 { arch_target(P, T) : arch_target(P, T) } 1     :- node(P).
+% one platform, os per node
+% TODO: convert these to use optimization, like targets.
+1 { node_platform(P, A) : node_platform(P, A) } 1 :- node(P).
+1 { node_os(P, A) : node_os(P, A) } 1 :- node(P).
 
 % arch fields for pkg P are set if set to anything
-arch_platform_set(P) :- arch_platform_set(P, _).
-arch_os_set(P) :- arch_os_set(P, _).
-arch_target_set(P) :- arch_target_set(P, _).
+node_platform_set(P) :- node_platform_set(P, _).
+node_os_set(P) :- node_os_set(P, _).
+
+% if no platform/os is set, fall back to the defaults
+node_platform(P, A)
+    :- node(P), not node_platform_set(P), node_platform_default(A).
+node_os(P, A) :- node(P), not node_os_set(P), node_os_default(A).
+
+% setting os/platform on a node is a hard constraint
+node_platform(P, A) :- node(P), node_platform_set(P, A).
+node_os(P, A) :- node(P), node_os_set(P, A).
 
 % avoid info warnings (see variants)
-#defined arch_platform_set/2.
-#defined arch_os_set/2.
-#defined arch_target_set/2.
-
-% if architecture value is set, it's the value
-arch_platform(P, A) :- node(P), arch_platform_set(P, A).
-arch_os(P, A) :- node(P), arch_os_set(P, A).
-arch_target(P, A) :- node(P), arch_target_set(P, A).
-
-% if no architecture is set, fall back to the default architecture value.
-arch_platform(P, A) :- node(P), not arch_platform_set(P),
-                       arch_platform_default(A).
-arch_os(P, A)       :- node(P), not arch_os_set(P), arch_os_default(A).
-arch_target(P, A)   :- node(P), not arch_target_set(P), arch_target_default(A).
-
-% propagate platform, os, target downwards
-% TODO: handle multiple dependents and arch compatibility
-arch_platform_set(D, A) :- node(D), depends_on(P, D), arch_platform_set(P, A).
-arch_os_set(D, A) :- node(D), depends_on(P, D), arch_os_set(P, A).
-arch_target_set(D, A) :- node(D), depends_on(P, D), arch_target_set(P, A).
+#defined node_platform_set/2.
+#defined node_os_set/2.
+
+%-----------------------------------------------------------------------------
+% Target semantics
+%-----------------------------------------------------------------------------
+% one target per node -- optimization will pick the "best" one
+1 { node_target(P, T) : target(T) } 1 :- node(P).
+
+% can't use targets on node if the compiler for the node doesn't support them
+:- node_target(P, T), not compiler_supports_target(C, V, T),
+   node_compiler(P, C), node_compiler_version(P, C, V).
+
+% if a target is set explicitly, respect it
+node_target(P, T) :- node(P), node_target_set(P, T).
+
+% each node has the weight of its assigned target
+node_target_weight(P, N) :- node(P), node_target(P, T), target_weight(T, N).
+
+#defined node_target_set/2.
 
 %-----------------------------------------------------------------------------
 % Compiler semantics
@@ -212,14 +219,21 @@ root(D, 2) :- root(D), node(D).
 root(D, 1) :- not root(D), node(D).
 
 % prefer default variants
-#minimize { N*R@5,P,V,X : variant_not_default(P, V, X, N), root(P, R) }.
+#minimize { N*R@10,P,V,X : variant_not_default(P, V, X, N), root(P, R) }.
 
 % pick most preferred virtual providers
-#minimize{ N*R@4,D : provider_weight(D, N), root(P, R) }.
+#minimize{ N*R@9,D : provider_weight(D, N), root(P, R) }.
 
 % prefer more recent versions.
-#minimize{ N@3,P,V : version_weight(P, V, N) }.
+#minimize{ N@8,P,V : version_weight(P, V, N) }.
 
 % compiler preferences
-#maximize{ N@2,P : compiler_match(P, N) }.
-#minimize{ N@1,P : compiler_weight(P, N) }.
+#maximize{ N@7,P : compiler_match(P, N) }.
+#minimize{ N@6,P : compiler_weight(P, N) }.
+
+% fastest target for node
+
+% TODO: if these are slightly different by compiler (e.g., skylake is
+% best, gcc supports skylake and broadweell, clang's best is haswell)
+% things seem to get really slow.
+#minimize{ N@5,P : node_target_weight(P, N) }.
diff --git a/lib/spack/spack/solver/display.lp b/lib/spack/spack/solver/display.lp
index e5be23b55c..e2eae00f79 100644
--- a/lib/spack/spack/solver/display.lp
+++ b/lib/spack/spack/solver/display.lp
@@ -7,8 +7,8 @@
 #show depends_on/3.
 #show version/2.
 #show variant_value/3.
-#show arch_platform/2.
-#show arch_os/2.
-#show arch_target/2.
+#show node_platform/2.
+#show node_os/2.
+#show node_target/2.
 #show node_compiler/2.
 #show node_compiler_version/3.