summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorMassimiliano Culpo <massimiliano.culpo@gmail.com>2019-05-03 20:04:38 +0200
committerGreg Becker <becker33@llnl.gov>2019-05-04 03:04:38 +0900
commit5ffb27071405c77f825444577fb3066ed32b200c (patch)
tree3c9c8e6d91dce5610bafcb26a5f1a3dbdae44730
parent0425670942d44948ab49cda55be43dfb1adbeccf (diff)
downloadspack-5ffb27071405c77f825444577fb3066ed32b200c.tar.gz
spack-5ffb27071405c77f825444577fb3066ed32b200c.tar.bz2
spack-5ffb27071405c77f825444577fb3066ed32b200c.tar.xz
spack-5ffb27071405c77f825444577fb3066ed32b200c.zip
Added a function that concretizes specs together (#11158)
* Added a function that concretizes specs together * Specs concretized together are copied instead of being referenced This makes the specs different objects and removes any reference to the fake root package that is needed currently for concretization. * Factored creating a repository for concretization into its own function * Added a test on overlapping dependencies
-rw-r--r--lib/spack/spack/concretize.py63
-rw-r--r--lib/spack/spack/repo.py12
-rw-r--r--lib/spack/spack/test/concretize.py39
-rw-r--r--share/spack/templates/misc/coconcretization.pyt15
4 files changed, 127 insertions, 2 deletions
diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py
index a4d01af996..551251162e 100644
--- a/lib/spack/spack/concretize.py
+++ b/lib/spack/spack/concretize.py
@@ -15,6 +15,12 @@ TODO: make this customizable and allow users to configure
concretization policies.
"""
from __future__ import print_function
+
+import os.path
+import tempfile
+import llnl.util.filesystem as fs
+import llnl.util.tty as tty
+
from itertools import chain
from functools_backport import reverse_order
from contextlib import contextmanager
@@ -28,6 +34,7 @@ import spack.spec
import spack.compilers
import spack.architecture
import spack.error
+import spack.tengine
from spack.config import config
from spack.version import ver, Version, VersionList, VersionRange
from spack.package_prefs import PackagePrefs, spec_externals, is_spec_buildable
@@ -465,6 +472,57 @@ def _compiler_concretization_failure(compiler_spec, arch):
raise UnavailableCompilerVersionError(compiler_spec, arch)
+def concretize_specs_together(*abstract_specs):
+ """Given a number of specs as input, tries to concretize them together.
+
+ Args:
+ *abstract_specs: abstract specs to be concretized, given either
+ as Specs or strings
+
+ Returns:
+ List of concretized specs
+ """
+ def make_concretization_repository(abstract_specs):
+ """Returns the path to a temporary repository created to contain
+ a fake package that depends on all of the abstract specs.
+ """
+ tmpdir = tempfile.mkdtemp()
+ repo_path, _ = spack.repo.create_repo(tmpdir)
+
+ debug_msg = '[CONCRETIZATION]: Creating helper repository in {0}'
+ tty.debug(debug_msg.format(repo_path))
+
+ pkg_dir = os.path.join(repo_path, 'packages', 'concretizationroot')
+ fs.mkdirp(pkg_dir)
+ environment = spack.tengine.make_environment()
+ template = environment.get_template('misc/coconcretization.pyt')
+
+ # Split recursive specs, as it seems the concretizer has issue
+ # respecting conditions on dependents expressed like
+ # depends_on('foo ^bar@1.0'), see issue #11160
+ split_specs = [dep for spec in abstract_specs
+ for dep in spec.traverse(root=True)]
+
+ with open(os.path.join(pkg_dir, 'package.py'), 'w') as f:
+ f.write(template.render(specs=[str(s) for s in split_specs]))
+
+ return spack.repo.Repo(repo_path)
+
+ abstract_specs = [spack.spec.Spec(s) for s in abstract_specs]
+ concretization_repository = make_concretization_repository(abstract_specs)
+
+ with spack.repo.additional_repository(concretization_repository):
+ # Spec from a helper package that depends on all the abstract_specs
+ concretization_root = spack.spec.Spec('concretizationroot')
+ concretization_root.concretize()
+ # Retrieve the direct dependencies
+ concrete_specs = [
+ concretization_root[spec.name].copy() for spec in abstract_specs
+ ]
+
+ return concrete_specs
+
+
class NoCompilersForArchError(spack.error.SpackError):
def __init__(self, arch, available_os_targets):
err_msg = ("No compilers found"
@@ -474,8 +532,9 @@ class NoCompilersForArchError(spack.error.SpackError):
(arch.os, arch.target))
available_os_target_strs = list()
- for os, t in available_os_targets:
- os_target_str = "%s-%s" % (os, t) if t else os
+ for operating_system, t in available_os_targets:
+ os_target_str = "%s-%s" % (operating_system, t) if t \
+ else operating_system
available_os_target_strs.append(os_target_str)
err_msg += (
"\nCompilers are defined for the following"
diff --git a/lib/spack/spack/repo.py b/lib/spack/spack/repo.py
index 26780e1b8a..c90f0f9ad4 100644
--- a/lib/spack/spack/repo.py
+++ b/lib/spack/spack/repo.py
@@ -1223,6 +1223,18 @@ def swap(repo_path):
path = saved
+@contextlib.contextmanager
+def additional_repository(repository):
+ """Adds temporarily a repository to the default one.
+
+ Args:
+ repository: repository to be added
+ """
+ path.put_first(repository)
+ yield
+ path.remove(repository)
+
+
class RepoError(spack.error.SpackError):
"""Superclass for repository-related errors."""
diff --git a/lib/spack/spack/test/concretize.py b/lib/spack/spack/test/concretize.py
index 364eb36b3d..d8b0783297 100644
--- a/lib/spack/spack/test/concretize.py
+++ b/lib/spack/spack/test/concretize.py
@@ -7,6 +7,7 @@ import pytest
import llnl.util.lang
import spack.architecture
+import spack.concretize
import spack.repo
from spack.concretize import find_spec
@@ -525,3 +526,41 @@ class TestConcretize(object):
t.concretize()
assert s.dag_hash() == t.dag_hash()
+
+ @pytest.mark.parametrize('abstract_specs', [
+ # Establish a baseline - concretize a single spec
+ ('mpileaks',),
+ # When concretized together with older version of callpath
+ # and dyninst it uses those older versions
+ ('mpileaks', 'callpath@0.9', 'dyninst@8.1.1'),
+ # Handle recursive syntax within specs
+ ('mpileaks', 'callpath@0.9 ^dyninst@8.1.1', 'dyninst'),
+ # Test specs that have overlapping dependencies but are not
+ # one a dependency of the other
+ ('mpileaks', 'direct-mpich')
+ ])
+ def test_simultaneous_concretization_of_specs(self, abstract_specs):
+
+ abstract_specs = [Spec(x) for x in abstract_specs]
+ concrete_specs = spack.concretize.concretize_specs_together(
+ *abstract_specs
+ )
+
+ # Check there's only one configuration of each package in the DAG
+ names = set(
+ dep.name for spec in concrete_specs for dep in spec.traverse()
+ )
+ for name in names:
+ name_specs = set(
+ spec[name] for spec in concrete_specs if name in spec
+ )
+ assert len(name_specs) == 1
+
+ # Check that there's at least one Spec that satisfies the
+ # initial abstract request
+ for aspec in abstract_specs:
+ assert any(cspec.satisfies(aspec) for cspec in concrete_specs)
+
+ # Make sure the concrete spec are top-level specs with no dependents
+ for spec in concrete_specs:
+ assert not spec.dependents()
diff --git a/share/spack/templates/misc/coconcretization.pyt b/share/spack/templates/misc/coconcretization.pyt
new file mode 100644
index 0000000000..eddab6a70c
--- /dev/null
+++ b/share/spack/templates/misc/coconcretization.pyt
@@ -0,0 +1,15 @@
+# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+
+class Concretizationroot(Package):
+ url = 'fake_url'
+
+ version('1.0')
+
+{% for dep in specs %}
+ depends_on('{{ dep }}')
+{% endfor %}
+