From f4d4322a41acb41c43fd318d3dac67ef296b41b4 Mon Sep 17 00:00:00 2001 From: Greg Becker Date: Thu, 7 Mar 2019 17:30:48 -0800 Subject: Create option to build compilers as needed (#10761) * Create option to build missing compilers and add them to config before installing packages that use them * Clean up kwarg passing for do_install, put compiler bootstrapping in separate method --- lib/spack/spack/compilers/__init__.py | 22 ++++++++++ lib/spack/spack/concretize.py | 66 ++++++++++++++++------------ lib/spack/spack/package.py | 82 +++++++++++++++++++++++------------ lib/spack/spack/package_prefs.py | 6 +-- lib/spack/spack/schema/config.py | 1 + lib/spack/spack/test/concretize.py | 19 +++++--- 6 files changed, 131 insertions(+), 65 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index decb98970a..b2be6d84fd 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -28,6 +28,17 @@ _cache_config_file = [] #: cache of compilers constructed from config data, keyed by config entry id. _compiler_cache = {} +_compiler_to_pkg = { + 'clang': 'llvm+clang' +} + + +def pkg_spec_for_compiler(cspec): + """Return the spec of the package that provides the compiler.""" + spec_str = '%s@%s' % (_compiler_to_pkg.get(cspec.name, cspec.name), + cspec.versions) + return spack.spec.Spec(spec_str) + def _auto_compiler_spec(function): def converter(cspec_like, *args, **kwargs): @@ -203,6 +214,17 @@ def find(compiler_spec, scope=None, init_config=True): if c.satisfies(compiler_spec)] +@_auto_compiler_spec +def find_specs_by_arch(compiler_spec, arch_spec, scope=None, init_config=True): + """Return specs of available compilers that match the supplied + compiler spec. Return an empty list if nothing found.""" + return [c.spec for c in compilers_for_spec(compiler_spec, + arch_spec, + scope, + True, + init_config)] + + def all_compilers(scope=None): config = get_compiler_config(scope) compilers = list() diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 633c3040d8..01f32b8c95 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -28,6 +28,7 @@ import spack.spec import spack.compilers import spack.architecture import spack.error +from spack.config import config from spack.version import ver, Version, VersionList, VersionRange from spack.package_prefs import PackagePrefs, spec_externals, is_spec_buildable @@ -47,7 +48,8 @@ class Concretizer(object): def __init__(self): # controls whether we check that compiler versions actually exist # during concretization. Used for testing and for mirror creation - self.check_for_compiler_existence = True + self.check_for_compiler_existence = not config.get( + 'config:install_missing_compilers', False) @contextmanager def disable_compiler_existence_check(self): @@ -56,6 +58,13 @@ class Concretizer(object): yield self.check_for_compiler_existence = saved + @contextmanager + def enable_compiler_existence_check(self): + saved = self.check_for_compiler_existence + self.check_for_compiler_existence = True + yield + self.check_for_compiler_existence = saved + def _valid_virtuals_and_externals(self, spec): """Returns a list of candidate virtual dep providers and external packages that coiuld be used to concretize a spec. @@ -303,36 +312,37 @@ class Concretizer(object): assert(other_spec) # Check if the compiler is already fully specified - if (other_compiler and other_compiler.concrete and - not self.check_for_compiler_existence): - spec.compiler = other_compiler.copy() - return True - - all_compiler_specs = spack.compilers.all_compiler_specs() - if not all_compiler_specs: - # If compiler existence checking is disabled, then we would have - # exited by now if there were sufficient hints to form a full - # compiler spec. Therefore even if compiler existence checking is - # disabled, compilers must be available at this point because the - # available compilers are used to choose a compiler. If compiler - # existence checking is enabled then some compiler must exist in - # order to complete the spec. - raise spack.compilers.NoCompilersError() - - if other_compiler in all_compiler_specs: - spec.compiler = other_compiler.copy() - if not _proper_compiler_style(spec.compiler, spec.architecture): + if other_compiler and other_compiler.concrete: + if (self.check_for_compiler_existence and not + _proper_compiler_style(other_compiler, spec.architecture)): _compiler_concretization_failure( - spec.compiler, spec.architecture) + other_compiler, spec.architecture) + spec.compiler = other_compiler return True - # Filter the compilers into a sorted list based on the compiler_order - # from spackconfig - compiler_list = all_compiler_specs if not other_compiler else \ - spack.compilers.find(other_compiler) - if not compiler_list: - # No compiler with a satisfactory spec was found - raise UnavailableCompilerVersionError(other_compiler) + if other_compiler: # Another node has abstract compiler information + compiler_list = spack.compilers.find_specs_by_arch( + other_compiler, spec.architecture + ) + if not compiler_list: + # We don't have a matching compiler installed + if not self.check_for_compiler_existence: + # Concretize compiler spec versions as a package to build + cpkg_spec = spack.compilers.pkg_spec_for_compiler( + other_compiler + ) + self.concretize_version(cpkg_spec) + spec.compiler.versions = cpkg_spec.versions + return True + else: + # No compiler with a satisfactory spec was found + raise UnavailableCompilerVersionError(other_compiler) + else: + # We have no hints to go by, grab any compiler + compiler_list = spack.compilers.all_compiler_specs() + if not compiler_list: + # Spack has no compilers. + raise spack.compilers.NoCompilersError() # By default, prefer later versions of compilers compiler_list = sorted( diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 9c1ba04207..76e1595daf 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -61,6 +61,7 @@ from spack.util.package_hash import package_hash from spack.version import Version from spack.package_prefs import get_package_dir_permissions, get_package_group + """Allowed URL schemes for spack packages.""" _ALLOWED_URL_SCHEMES = ["http", "https", "ftp", "file", "git"] @@ -1324,19 +1325,29 @@ class PackageBase(with_metaclass(PackageMeta, PackageViewMixin, object)): self.spec, spack.store.layout, explicit=explicit) return True - def do_install(self, - keep_prefix=False, - keep_stage=False, - install_source=False, - install_deps=True, - skip_patch=False, - verbose=False, - make_jobs=None, - fake=False, - explicit=False, - tests=False, - dirty=None, - **kwargs): + def bootstrap_compiler(self, **kwargs): + """Called by do_install to setup ensure Spack has the right compiler. + + Checks Spack's compiler configuration for a compiler that + matches the package spec. If none are configured, installs and + adds to the compiler configuration the compiler matching the + CompilerSpec object.""" + compilers = spack.compilers.compilers_for_spec( + self.spec.compiler, + arch_spec=self.spec.architecture + ) + if not compilers: + dep = spack.compilers.pkg_spec_for_compiler(self.spec.compiler) + # concrete CompilerSpec has less info than concrete Spec + # concretize as Spec to add that information + dep.concretize() + dep.package.do_install(**kwargs) + spack.compilers.add_compilers_to_config( + spack.compilers.find_compilers(dep.prefix) + ) + + def do_install(self, **kwargs): + """Called by commands to install a package and its dependencies. Package implementations should override install() to describe @@ -1363,18 +1374,34 @@ class PackageBase(with_metaclass(PackageMeta, PackageViewMixin, object)): tests (bool or list or set): False to run no tests, True to test all packages, or a list of package names to run tests for some dirty (bool): Don't clean the build environment before installing. + restage (bool): Force spack to restage the package source. force (bool): Install again, even if already installed. + use_cache (bool): Install from binary package, if available. + stop_at (InstallPhase): last installation phase to be executed + (or None) """ if not self.spec.concrete: raise ValueError("Can only install concrete packages: %s." % self.spec.name) + keep_prefix = kwargs.get('keep_prefix', False) + keep_stage = kwargs.get('keep_stage', False) + install_source = kwargs.get('install_source', False) + install_deps = kwargs.get('install_deps', True) + skip_patch = kwargs.get('skip_patch', False) + verbose = kwargs.get('verbose', False) + make_jobs = kwargs.get('make_jobs', None) + fake = kwargs.get('fake', False) + explicit = kwargs.get('explicit', False) + tests = kwargs.get('tests', False) + dirty = kwargs.get('dirty', False) + restage = kwargs.get('restage', False) + # For external packages the workflow is simplified, and basically # consists in module file generation and registration in the DB if self.spec.external: return self._process_external_package(explicit) - restage = kwargs.get('restage', False) partial = self.check_for_unfinished_installation(keep_prefix, restage) # Ensure package is not already installed @@ -1399,21 +1426,22 @@ class PackageBase(with_metaclass(PackageMeta, PackageViewMixin, object)): # First, install dependencies recursively. if install_deps: tty.debug('Installing {0} dependencies'.format(self.name)) + dep_kwargs = kwargs.copy() + dep_kwargs['explicit'] = False + dep_kwargs['install_deps'] = False for dep in self.spec.traverse(order='post', root=False): - dep.package.do_install( - install_deps=False, - explicit=False, - keep_prefix=keep_prefix, - keep_stage=keep_stage, - install_source=install_source, - fake=fake, - skip_patch=skip_patch, - verbose=verbose, - make_jobs=make_jobs, - tests=tests, - dirty=dirty, - **kwargs) + dep.package.do_install(**dep_kwargs) + # Then, install the compiler if it is not already installed. + if install_deps: + tty.debug('Boostrapping {0} compiler for {1}'.format( + self.spec.compiler, self.name + )) + comp_kwargs = kwargs.copy() + comp_kwargs['explicit'] = False + self.bootstrap_compiler(**comp_kwargs) + + # Then, install the package proper tty.msg(colorize('@*{Installing} @*g{%s}' % self.name)) if kwargs.get('use_cache', True): diff --git a/lib/spack/spack/package_prefs.py b/lib/spack/spack/package_prefs.py index 6882892a23..f9945a7ea0 100644 --- a/lib/spack/spack/package_prefs.py +++ b/lib/spack/spack/package_prefs.py @@ -115,7 +115,7 @@ class PackagePrefs(object): return cls._packages_config_cache @classmethod - def _order_for_package(cls, pkgname, component, vpkg=None, all=True): + def order_for_package(cls, pkgname, component, vpkg=None, all=True): """Given a package name, sort component (e.g, version, compiler, ...), and an optional vpkg, return the list from the packages config. """ @@ -151,7 +151,7 @@ class PackagePrefs(object): specs = cls._spec_cache.get(key) if specs is None: - pkglist = cls._order_for_package(pkgname, component, vpkg) + pkglist = cls.order_for_package(pkgname, component, vpkg) spec_type = _spec_type(component) specs = [spec_type(s) for s in pkglist] cls._spec_cache[key] = specs @@ -166,7 +166,7 @@ class PackagePrefs(object): @classmethod def has_preferred_providers(cls, pkgname, vpkg): """Whether specific package has a preferred vpkg providers.""" - return bool(cls._order_for_package(pkgname, 'providers', vpkg, False)) + return bool(cls.order_for_package(pkgname, 'providers', vpkg, False)) @classmethod def preferred_variants(cls, pkg_name): diff --git a/lib/spack/spack/schema/config.py b/lib/spack/spack/schema/config.py index 01c4452c80..1aa54bf48e 100644 --- a/lib/spack/spack/schema/config.py +++ b/lib/spack/spack/schema/config.py @@ -41,6 +41,7 @@ properties = { 'source_cache': {'type': 'string'}, 'misc_cache': {'type': 'string'}, 'verify_ssl': {'type': 'boolean'}, + 'install_missing_compilers': {'type': 'boolean'}, 'debug': {'type': 'boolean'}, 'checksum': {'type': 'boolean'}, 'locks': {'type': 'boolean'}, diff --git a/lib/spack/spack/test/concretize.py b/lib/spack/spack/test/concretize.py index 92a389992c..364eb36b3d 100644 --- a/lib/spack/spack/test/concretize.py +++ b/lib/spack/spack/test/concretize.py @@ -130,9 +130,11 @@ class TestConcretize(object): concrete = check_concretize('mpileaks ^mpich2@1.3.1:1.4') assert concrete['mpich2'].satisfies('mpich2@1.3.1:1.4') - def test_concretize_disable_compiler_existence_check(self): - with pytest.raises(spack.concretize.UnavailableCompilerVersionError): - check_concretize('dttop %gcc@100.100') + def test_concretize_enable_disable_compiler_existence_check(self): + with spack.concretize.concretizer.enable_compiler_existence_check(): + with pytest.raises( + spack.concretize.UnavailableCompilerVersionError): + check_concretize('dttop %gcc@100.100') with spack.concretize.concretizer.disable_compiler_existence_check(): spec = check_concretize('dttop %gcc@100.100') @@ -267,10 +269,13 @@ class TestConcretize(object): with pytest.raises(spack.spec.MultipleProviderError): s.concretize() - def test_no_matching_compiler_specs(self): - s = Spec('a %gcc@0.0.0') - with pytest.raises(spack.concretize.UnavailableCompilerVersionError): - s.concretize() + def test_no_matching_compiler_specs(self, mock_config): + # only relevant when not building compilers as needed + with spack.concretize.concretizer.enable_compiler_existence_check(): + s = Spec('a %gcc@0.0.0') + with pytest.raises( + spack.concretize.UnavailableCompilerVersionError): + s.concretize() def test_no_compilers_for_arch(self): s = Spec('a arch=linux-rhel0-x86_64') -- cgit v1.2.3-60-g2f50