summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorGreg Becker <becker33@llnl.gov>2019-03-07 17:30:48 -0800
committerGitHub <noreply@github.com>2019-03-07 17:30:48 -0800
commitf4d4322a41acb41c43fd318d3dac67ef296b41b4 (patch)
tree98bcdce4eb8dacec0e77893c3d46cdca77fac225
parent0d07dea2eb1e32157c3ae58a5514bcd46e2d32ac (diff)
downloadspack-f4d4322a41acb41c43fd318d3dac67ef296b41b4.tar.gz
spack-f4d4322a41acb41c43fd318d3dac67ef296b41b4.tar.bz2
spack-f4d4322a41acb41c43fd318d3dac67ef296b41b4.tar.xz
spack-f4d4322a41acb41c43fd318d3dac67ef296b41b4.zip
Create option to build compilers as needed (#10761)
* Create option to build missing compilers and add them to config before installing packages that use them * Clean up kwarg passing for do_install, put compiler bootstrapping in separate method
-rw-r--r--etc/spack/defaults/config.yaml6
-rw-r--r--lib/spack/spack/compilers/__init__.py22
-rw-r--r--lib/spack/spack/concretize.py66
-rw-r--r--lib/spack/spack/package.py82
-rw-r--r--lib/spack/spack/package_prefs.py6
-rw-r--r--lib/spack/spack/schema/config.py1
-rw-r--r--lib/spack/spack/test/concretize.py19
7 files changed, 137 insertions, 65 deletions
diff --git a/etc/spack/defaults/config.yaml b/etc/spack/defaults/config.yaml
index 8ccf7db535..6e8c1905e8 100644
--- a/etc/spack/defaults/config.yaml
+++ b/etc/spack/defaults/config.yaml
@@ -66,6 +66,12 @@ config:
verify_ssl: true
+ # If set to true, Spack will attempt to build any compiler on the spec
+ # that is not already available. If set to False, Spack will only use
+ # compilers already configured in compilers.yaml
+ install_missing_compilers: False
+
+
# If set to true, Spack will always check checksums after downloading
# archives. If false, Spack skips the checksum step.
checksum: true
diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py
index decb98970a..b2be6d84fd 100644
--- a/lib/spack/spack/compilers/__init__.py
+++ b/lib/spack/spack/compilers/__init__.py
@@ -28,6 +28,17 @@ _cache_config_file = []
#: cache of compilers constructed from config data, keyed by config entry id.
_compiler_cache = {}
+_compiler_to_pkg = {
+ 'clang': 'llvm+clang'
+}
+
+
+def pkg_spec_for_compiler(cspec):
+ """Return the spec of the package that provides the compiler."""
+ spec_str = '%s@%s' % (_compiler_to_pkg.get(cspec.name, cspec.name),
+ cspec.versions)
+ return spack.spec.Spec(spec_str)
+
def _auto_compiler_spec(function):
def converter(cspec_like, *args, **kwargs):
@@ -203,6 +214,17 @@ def find(compiler_spec, scope=None, init_config=True):
if c.satisfies(compiler_spec)]
+@_auto_compiler_spec
+def find_specs_by_arch(compiler_spec, arch_spec, scope=None, init_config=True):
+ """Return specs of available compilers that match the supplied
+ compiler spec. Return an empty list if nothing found."""
+ return [c.spec for c in compilers_for_spec(compiler_spec,
+ arch_spec,
+ scope,
+ True,
+ init_config)]
+
+
def all_compilers(scope=None):
config = get_compiler_config(scope)
compilers = list()
diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py
index 633c3040d8..01f32b8c95 100644
--- a/lib/spack/spack/concretize.py
+++ b/lib/spack/spack/concretize.py
@@ -28,6 +28,7 @@ import spack.spec
import spack.compilers
import spack.architecture
import spack.error
+from spack.config import config
from spack.version import ver, Version, VersionList, VersionRange
from spack.package_prefs import PackagePrefs, spec_externals, is_spec_buildable
@@ -47,7 +48,8 @@ class Concretizer(object):
def __init__(self):
# controls whether we check that compiler versions actually exist
# during concretization. Used for testing and for mirror creation
- self.check_for_compiler_existence = True
+ self.check_for_compiler_existence = not config.get(
+ 'config:install_missing_compilers', False)
@contextmanager
def disable_compiler_existence_check(self):
@@ -56,6 +58,13 @@ class Concretizer(object):
yield
self.check_for_compiler_existence = saved
+ @contextmanager
+ def enable_compiler_existence_check(self):
+ saved = self.check_for_compiler_existence
+ self.check_for_compiler_existence = True
+ yield
+ self.check_for_compiler_existence = saved
+
def _valid_virtuals_and_externals(self, spec):
"""Returns a list of candidate virtual dep providers and external
packages that coiuld be used to concretize a spec.
@@ -303,36 +312,37 @@ class Concretizer(object):
assert(other_spec)
# Check if the compiler is already fully specified
- if (other_compiler and other_compiler.concrete and
- not self.check_for_compiler_existence):
- spec.compiler = other_compiler.copy()
- return True
-
- all_compiler_specs = spack.compilers.all_compiler_specs()
- if not all_compiler_specs:
- # If compiler existence checking is disabled, then we would have
- # exited by now if there were sufficient hints to form a full
- # compiler spec. Therefore even if compiler existence checking is
- # disabled, compilers must be available at this point because the
- # available compilers are used to choose a compiler. If compiler
- # existence checking is enabled then some compiler must exist in
- # order to complete the spec.
- raise spack.compilers.NoCompilersError()
-
- if other_compiler in all_compiler_specs:
- spec.compiler = other_compiler.copy()
- if not _proper_compiler_style(spec.compiler, spec.architecture):
+ if other_compiler and other_compiler.concrete:
+ if (self.check_for_compiler_existence and not
+ _proper_compiler_style(other_compiler, spec.architecture)):
_compiler_concretization_failure(
- spec.compiler, spec.architecture)
+ other_compiler, spec.architecture)
+ spec.compiler = other_compiler
return True
- # Filter the compilers into a sorted list based on the compiler_order
- # from spackconfig
- compiler_list = all_compiler_specs if not other_compiler else \
- spack.compilers.find(other_compiler)
- if not compiler_list:
- # No compiler with a satisfactory spec was found
- raise UnavailableCompilerVersionError(other_compiler)
+ if other_compiler: # Another node has abstract compiler information
+ compiler_list = spack.compilers.find_specs_by_arch(
+ other_compiler, spec.architecture
+ )
+ if not compiler_list:
+ # We don't have a matching compiler installed
+ if not self.check_for_compiler_existence:
+ # Concretize compiler spec versions as a package to build
+ cpkg_spec = spack.compilers.pkg_spec_for_compiler(
+ other_compiler
+ )
+ self.concretize_version(cpkg_spec)
+ spec.compiler.versions = cpkg_spec.versions
+ return True
+ else:
+ # No compiler with a satisfactory spec was found
+ raise UnavailableCompilerVersionError(other_compiler)
+ else:
+ # We have no hints to go by, grab any compiler
+ compiler_list = spack.compilers.all_compiler_specs()
+ if not compiler_list:
+ # Spack has no compilers.
+ raise spack.compilers.NoCompilersError()
# By default, prefer later versions of compilers
compiler_list = sorted(
diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py
index 9c1ba04207..76e1595daf 100644
--- a/lib/spack/spack/package.py
+++ b/lib/spack/spack/package.py
@@ -61,6 +61,7 @@ from spack.util.package_hash import package_hash
from spack.version import Version
from spack.package_prefs import get_package_dir_permissions, get_package_group
+
"""Allowed URL schemes for spack packages."""
_ALLOWED_URL_SCHEMES = ["http", "https", "ftp", "file", "git"]
@@ -1324,19 +1325,29 @@ class PackageBase(with_metaclass(PackageMeta, PackageViewMixin, object)):
self.spec, spack.store.layout, explicit=explicit)
return True
- def do_install(self,
- keep_prefix=False,
- keep_stage=False,
- install_source=False,
- install_deps=True,
- skip_patch=False,
- verbose=False,
- make_jobs=None,
- fake=False,
- explicit=False,
- tests=False,
- dirty=None,
- **kwargs):
+ def bootstrap_compiler(self, **kwargs):
+ """Called by do_install to setup ensure Spack has the right compiler.
+
+ Checks Spack's compiler configuration for a compiler that
+ matches the package spec. If none are configured, installs and
+ adds to the compiler configuration the compiler matching the
+ CompilerSpec object."""
+ compilers = spack.compilers.compilers_for_spec(
+ self.spec.compiler,
+ arch_spec=self.spec.architecture
+ )
+ if not compilers:
+ dep = spack.compilers.pkg_spec_for_compiler(self.spec.compiler)
+ # concrete CompilerSpec has less info than concrete Spec
+ # concretize as Spec to add that information
+ dep.concretize()
+ dep.package.do_install(**kwargs)
+ spack.compilers.add_compilers_to_config(
+ spack.compilers.find_compilers(dep.prefix)
+ )
+
+ def do_install(self, **kwargs):
+
"""Called by commands to install a package and its dependencies.
Package implementations should override install() to describe
@@ -1363,18 +1374,34 @@ class PackageBase(with_metaclass(PackageMeta, PackageViewMixin, object)):
tests (bool or list or set): False to run no tests, True to test
all packages, or a list of package names to run tests for some
dirty (bool): Don't clean the build environment before installing.
+ restage (bool): Force spack to restage the package source.
force (bool): Install again, even if already installed.
+ use_cache (bool): Install from binary package, if available.
+ stop_at (InstallPhase): last installation phase to be executed
+ (or None)
"""
if not self.spec.concrete:
raise ValueError("Can only install concrete packages: %s."
% self.spec.name)
+ keep_prefix = kwargs.get('keep_prefix', False)
+ keep_stage = kwargs.get('keep_stage', False)
+ install_source = kwargs.get('install_source', False)
+ install_deps = kwargs.get('install_deps', True)
+ skip_patch = kwargs.get('skip_patch', False)
+ verbose = kwargs.get('verbose', False)
+ make_jobs = kwargs.get('make_jobs', None)
+ fake = kwargs.get('fake', False)
+ explicit = kwargs.get('explicit', False)
+ tests = kwargs.get('tests', False)
+ dirty = kwargs.get('dirty', False)
+ restage = kwargs.get('restage', False)
+
# For external packages the workflow is simplified, and basically
# consists in module file generation and registration in the DB
if self.spec.external:
return self._process_external_package(explicit)
- restage = kwargs.get('restage', False)
partial = self.check_for_unfinished_installation(keep_prefix, restage)
# Ensure package is not already installed
@@ -1399,21 +1426,22 @@ class PackageBase(with_metaclass(PackageMeta, PackageViewMixin, object)):
# First, install dependencies recursively.
if install_deps:
tty.debug('Installing {0} dependencies'.format(self.name))
+ dep_kwargs = kwargs.copy()
+ dep_kwargs['explicit'] = False
+ dep_kwargs['install_deps'] = False
for dep in self.spec.traverse(order='post', root=False):
- dep.package.do_install(
- install_deps=False,
- explicit=False,
- keep_prefix=keep_prefix,
- keep_stage=keep_stage,
- install_source=install_source,
- fake=fake,
- skip_patch=skip_patch,
- verbose=verbose,
- make_jobs=make_jobs,
- tests=tests,
- dirty=dirty,
- **kwargs)
+ dep.package.do_install(**dep_kwargs)
+ # Then, install the compiler if it is not already installed.
+ if install_deps:
+ tty.debug('Boostrapping {0} compiler for {1}'.format(
+ self.spec.compiler, self.name
+ ))
+ comp_kwargs = kwargs.copy()
+ comp_kwargs['explicit'] = False
+ self.bootstrap_compiler(**comp_kwargs)
+
+ # Then, install the package proper
tty.msg(colorize('@*{Installing} @*g{%s}' % self.name))
if kwargs.get('use_cache', True):
diff --git a/lib/spack/spack/package_prefs.py b/lib/spack/spack/package_prefs.py
index 6882892a23..f9945a7ea0 100644
--- a/lib/spack/spack/package_prefs.py
+++ b/lib/spack/spack/package_prefs.py
@@ -115,7 +115,7 @@ class PackagePrefs(object):
return cls._packages_config_cache
@classmethod
- def _order_for_package(cls, pkgname, component, vpkg=None, all=True):
+ def order_for_package(cls, pkgname, component, vpkg=None, all=True):
"""Given a package name, sort component (e.g, version, compiler, ...),
and an optional vpkg, return the list from the packages config.
"""
@@ -151,7 +151,7 @@ class PackagePrefs(object):
specs = cls._spec_cache.get(key)
if specs is None:
- pkglist = cls._order_for_package(pkgname, component, vpkg)
+ pkglist = cls.order_for_package(pkgname, component, vpkg)
spec_type = _spec_type(component)
specs = [spec_type(s) for s in pkglist]
cls._spec_cache[key] = specs
@@ -166,7 +166,7 @@ class PackagePrefs(object):
@classmethod
def has_preferred_providers(cls, pkgname, vpkg):
"""Whether specific package has a preferred vpkg providers."""
- return bool(cls._order_for_package(pkgname, 'providers', vpkg, False))
+ return bool(cls.order_for_package(pkgname, 'providers', vpkg, False))
@classmethod
def preferred_variants(cls, pkg_name):
diff --git a/lib/spack/spack/schema/config.py b/lib/spack/spack/schema/config.py
index 01c4452c80..1aa54bf48e 100644
--- a/lib/spack/spack/schema/config.py
+++ b/lib/spack/spack/schema/config.py
@@ -41,6 +41,7 @@ properties = {
'source_cache': {'type': 'string'},
'misc_cache': {'type': 'string'},
'verify_ssl': {'type': 'boolean'},
+ 'install_missing_compilers': {'type': 'boolean'},
'debug': {'type': 'boolean'},
'checksum': {'type': 'boolean'},
'locks': {'type': 'boolean'},
diff --git a/lib/spack/spack/test/concretize.py b/lib/spack/spack/test/concretize.py
index 92a389992c..364eb36b3d 100644
--- a/lib/spack/spack/test/concretize.py
+++ b/lib/spack/spack/test/concretize.py
@@ -130,9 +130,11 @@ class TestConcretize(object):
concrete = check_concretize('mpileaks ^mpich2@1.3.1:1.4')
assert concrete['mpich2'].satisfies('mpich2@1.3.1:1.4')
- def test_concretize_disable_compiler_existence_check(self):
- with pytest.raises(spack.concretize.UnavailableCompilerVersionError):
- check_concretize('dttop %gcc@100.100')
+ def test_concretize_enable_disable_compiler_existence_check(self):
+ with spack.concretize.concretizer.enable_compiler_existence_check():
+ with pytest.raises(
+ spack.concretize.UnavailableCompilerVersionError):
+ check_concretize('dttop %gcc@100.100')
with spack.concretize.concretizer.disable_compiler_existence_check():
spec = check_concretize('dttop %gcc@100.100')
@@ -267,10 +269,13 @@ class TestConcretize(object):
with pytest.raises(spack.spec.MultipleProviderError):
s.concretize()
- def test_no_matching_compiler_specs(self):
- s = Spec('a %gcc@0.0.0')
- with pytest.raises(spack.concretize.UnavailableCompilerVersionError):
- s.concretize()
+ def test_no_matching_compiler_specs(self, mock_config):
+ # only relevant when not building compilers as needed
+ with spack.concretize.concretizer.enable_compiler_existence_check():
+ s = Spec('a %gcc@0.0.0')
+ with pytest.raises(
+ spack.concretize.UnavailableCompilerVersionError):
+ s.concretize()
def test_no_compilers_for_arch(self):
s = Spec('a arch=linux-rhel0-x86_64')