diff options
Diffstat (limited to 'lib')
| -rw-r--r-- | lib/spack/spack/bootstrap.py | 293 | ||||
| -rw-r--r-- | lib/spack/spack/cmd/buildcache.py | 14 | ||||
| -rw-r--r-- | lib/spack/spack/modules/common.py | 6 | ||||
| -rw-r--r-- | lib/spack/spack/test/cmd/gpg.py | 7 | ||||
| -rw-r--r-- | lib/spack/spack/util/gpg.py | 6 |
5 files changed, 251 insertions, 75 deletions
diff --git a/lib/spack/spack/bootstrap.py b/lib/spack/spack/bootstrap.py index 78ed54429f..97a38d20a0 100644 --- a/lib/spack/spack/bootstrap.py +++ b/lib/spack/spack/bootstrap.py @@ -6,6 +6,7 @@ from __future__ import print_function import contextlib import fnmatch +import functools import json import os import os.path @@ -34,11 +35,14 @@ import spack.platforms import spack.repo import spack.spec import spack.store -import spack.user_environment as uenv +import spack.user_environment import spack.util.executable import spack.util.path from spack.util.environment import EnvironmentModifications +#: "spack buildcache" command, initialized lazily +_buildcache_cmd = None + #: Map a bootstrapper type to the corresponding class _bootstrap_methods = {} @@ -171,6 +175,34 @@ def _fix_ext_suffix(candidate_spec): os.symlink(abs_path, link_name) +def _executables_in_store(executables, abstract_spec_str): + """Return True if at least one of the executables can be retrieved from + a spec in store, False otherwise. + + The different executables must provide the same functionality and are + "alternate" to each other, i.e. the function will exit True on the first + executable found. + + Args: + executables: list of executables to be searched + abstract_spec_str: abstract spec that may provide the executable + """ + executables_str = ', '.join(executables) + msg = "[BOOTSTRAP EXECUTABLES {0}] Try installed specs with query '{1}'" + tty.debug(msg.format(executables_str, abstract_spec_str)) + installed_specs = spack.store.db.query(abstract_spec_str, installed=True) + if installed_specs: + for concrete_spec in installed_specs: + bin_dir = concrete_spec.prefix.bin + # IF we have a "bin" directory and it contains + # the executables we are looking for + if (os.path.exists(bin_dir) and os.path.isdir(bin_dir) and + spack.util.executable.which_string(*executables, path=bin_dir)): + spack.util.environment.path_put_first('PATH', [bin_dir]) + return True + return False + + @_bootstrapper(type='buildcache') class _BuildcacheBootstrapper(object): """Install the software needed during bootstrapping from a buildcache.""" @@ -178,93 +210,140 @@ class _BuildcacheBootstrapper(object): self.name = conf['name'] self.url = conf['info']['url'] - def try_import(self, module, abstract_spec_str): - if _try_import_from_store(module, abstract_spec_str): - return True + @staticmethod + def _spec_and_platform(abstract_spec_str): + """Return the spec object and platform we need to use when + querying the buildcache. - tty.info("Bootstrapping {0} from pre-built binaries".format(module)) + Args: + abstract_spec_str: abstract spec string we are looking for + """ + # This import is local since it is needed only on Cray + import spack.platforms.linux # Try to install from an unsigned binary cache - abstract_spec = spack.spec.Spec( - abstract_spec_str + ' ^' + spec_for_current_python() - ) - + abstract_spec = spack.spec.Spec(abstract_spec_str) # On Cray we want to use Linux binaries if available from mirrors bincache_platform = spack.platforms.real_host() if str(bincache_platform) == 'cray': bincache_platform = spack.platforms.Linux() with spack.platforms.use_platform(bincache_platform): - abstract_spec = spack.spec.Spec( - abstract_spec_str + ' ^' + spec_for_current_python() - ) + abstract_spec = spack.spec.Spec(abstract_spec_str) + return abstract_spec, bincache_platform - # Read information on verified clingo binaries - json_filename = '{0}.json'.format(module) + def _read_metadata(self, package_name): + """Return metadata about the given package.""" + json_filename = '{0}.json'.format(package_name) json_path = os.path.join( spack.paths.share_path, 'bootstrap', self.name, json_filename ) with open(json_path) as f: data = json.load(f) - - buildcache = spack.main.SpackCommand('buildcache') + return data + + def _install_by_hash(self, pkg_hash, pkg_sha256, index, bincache_platform): + global _buildcache_cmd + + if _buildcache_cmd is None: + _buildcache_cmd = spack.main.SpackCommand('buildcache') + + index_spec = next(x for x in index if x.dag_hash() == pkg_hash) + # Reconstruct the compiler that we need to use for bootstrapping + compiler_entry = { + "modules": [], + "operating_system": str(index_spec.os), + "paths": { + "cc": "/dev/null", + "cxx": "/dev/null", + "f77": "/dev/null", + "fc": "/dev/null" + }, + "spec": str(index_spec.compiler), + "target": str(index_spec.target.family) + } + with spack.platforms.use_platform(bincache_platform): + with spack.config.override( + 'compilers', [{'compiler': compiler_entry}] + ): + spec_str = '/' + pkg_hash + install_args = [ + 'install', + '--sha256', pkg_sha256, + '--only-root', + '-a', '-u', '-o', '-f', spec_str + ] + _buildcache_cmd(*install_args, fail_on_error=False) + + def _install_and_test( + self, abstract_spec, bincache_platform, bincache_data, test_fn + ): # Ensure we see only the buildcache being used to bootstrap - mirror_scope = spack.config.InternalConfigScope( - 'bootstrap_buildcache', {'mirrors:': {self.name: self.url}} - ) - with spack.config.override(mirror_scope): + with spack.config.override(self.mirror_scope): # This index is currently needed to get the compiler used to build some - # specs that wwe know by dag hash. + # specs that we know by dag hash. spack.binary_distribution.binary_index.regenerate_spec_cache() index = spack.binary_distribution.update_cache_and_get_specs() if not index: raise RuntimeError("The binary index is empty") - for item in data['verified']: + for item in bincache_data['verified']: candidate_spec = item['spec'] - python_spec = item['python'] + # This will be None for things that don't depend on python + python_spec = item.get('python', None) # Skip specs which are not compatible if not abstract_spec.satisfies(candidate_spec): continue - if python_spec not in abstract_spec: + if python_spec is not None and python_spec not in abstract_spec: continue for pkg_name, pkg_hash, pkg_sha256 in item['binaries']: - msg = ('[BOOTSTRAP MODULE {0}] Try installing "{1}" from binary ' - 'cache at "{2}"') - tty.debug(msg.format(module, pkg_name, self.url)) - index_spec = next(x for x in index if x.dag_hash() == pkg_hash) - # Reconstruct the compiler that we need to use for bootstrapping - compiler_entry = { - "modules": [], - "operating_system": str(index_spec.os), - "paths": { - "cc": "/dev/null", - "cxx": "/dev/null", - "f77": "/dev/null", - "fc": "/dev/null" - }, - "spec": str(index_spec.compiler), - "target": str(index_spec.target.family) - } - with spack.platforms.use_platform(bincache_platform): - with spack.config.override( - 'compilers', [{'compiler': compiler_entry}] - ): - spec_str = '/' + pkg_hash - install_args = [ - 'install', - '--sha256', pkg_sha256, - '-a', '-u', '-o', '-f', spec_str - ] - buildcache(*install_args, fail_on_error=False) - # TODO: undo installations that didn't complete? - - if _try_import_from_store(module, abstract_spec_str): + # TODO: undo installations that didn't complete? + self._install_by_hash( + pkg_hash, pkg_sha256, index, bincache_platform + ) + + if test_fn(): return True return False + @property + def mirror_scope(self): + return spack.config.InternalConfigScope( + 'bootstrap', {'mirrors:': {self.name: self.url}} + ) + + def try_import(self, module, abstract_spec_str): + test_fn = functools.partial(_try_import_from_store, module, abstract_spec_str) + if test_fn(): + return True + + tty.info("Bootstrapping {0} from pre-built binaries".format(module)) + abstract_spec, bincache_platform = self._spec_and_platform( + abstract_spec_str + ' ^' + spec_for_current_python() + ) + data = self._read_metadata(module) + return self._install_and_test( + abstract_spec, bincache_platform, data, test_fn + ) + + def try_search_path(self, executables, abstract_spec_str): + test_fn = functools.partial( + _executables_in_store, executables, abstract_spec_str + ) + if test_fn(): + return True + + abstract_spec, bincache_platform = self._spec_and_platform( + abstract_spec_str + ) + tty.info("Bootstrapping {0} from pre-built binaries".format(abstract_spec.name)) + data = self._read_metadata(abstract_spec.name) + return self._install_and_test( + abstract_spec, bincache_platform, data, test_fn + ) + @_bootstrapper(type='install') class _SourceBootstrapper(object): @@ -307,6 +386,26 @@ class _SourceBootstrapper(object): return _try_import_from_store(module, abstract_spec_str=abstract_spec_str) + def try_search_path(self, executables, abstract_spec_str): + if _executables_in_store(executables, abstract_spec_str): + return True + + # If we compile code from sources detecting a few build tools + # might reduce compilation time by a fair amount + _add_externals_if_missing() + + # Add hint to use frontend operating system on Cray + if str(spack.platforms.host()) == 'cray': + abstract_spec_str += ' os=fe' + + concrete_spec = spack.spec.Spec(abstract_spec_str) + concrete_spec.concretize() + + msg = "[BOOTSTRAP GnuPG] Try installing '{0}' from sources" + tty.debug(msg.format(abstract_spec_str)) + concrete_spec.package.do_install() + return _executables_in_store(executables, abstract_spec_str) + def _make_bootstrapper(conf): """Return a bootstrap object built according to the @@ -418,6 +517,44 @@ def ensure_module_importable_or_raise(module, abstract_spec=None): raise ImportError(msg) +def ensure_executables_in_path_or_raise(executables, abstract_spec): + """Ensure that some executables are in path or raise. + + Args: + executables (list): list of executables to be searched in the PATH, + in order. The function exits on the first one found. + abstract_spec (str): abstract spec that provides the executables + + Raises: + RuntimeError: if the executables cannot be ensured to be in PATH + """ + if spack.util.executable.which_string(*executables): + return + + executables_str = ', '.join(executables) + source_configs = spack.config.get('bootstrap:sources', []) + for current_config in source_configs: + if not _source_is_trusted(current_config): + msg = ('[BOOTSTRAP EXECUTABLES {0}] Skipping source "{1}" since it is ' + 'not trusted').format(executables_str, current_config['name']) + tty.debug(msg) + continue + + b = _make_bootstrapper(current_config) + try: + if b.try_search_path(executables, abstract_spec): + return + except Exception as e: + msg = '[BOOTSTRAP EXECUTABLES {0}] Unexpected error "{1}"' + tty.debug(msg.format(executables_str, str(e))) + + # We couldn't import in any way, so raise an import error + msg = 'cannot bootstrap any of the {0} executables'.format(executables_str) + if abstract_spec: + msg += ' from spec "{0}"'.format(abstract_spec) + raise RuntimeError(msg) + + def _python_import(module): try: __import__(module) @@ -455,7 +592,9 @@ def get_executable(exe, spec=None, install=False): ret = spack.util.executable.Executable(exe_path[0]) envmod = EnvironmentModifications() for dep in ispec.traverse(root=True, order='post'): - envmod.extend(uenv.environment_modifications_for_spec(dep)) + envmod.extend( + spack.user_environment.environment_modifications_for_spec(dep) + ) ret.add_default_envmod(envmod) return ret else: @@ -484,7 +623,9 @@ def get_executable(exe, spec=None, install=False): ret = spack.util.executable.Executable(exe_path[0]) envmod = EnvironmentModifications() for dep in spec.traverse(root=True, order='post'): - envmod.extend(uenv.environment_modifications_for_spec(dep)) + envmod.extend( + spack.user_environment.environment_modifications_for_spec(dep) + ) ret.add_default_envmod(envmod) return ret @@ -523,8 +664,11 @@ def _add_compilers_if_missing(): def _add_externals_if_missing(): search_list = [ + # clingo spack.repo.path.get('cmake'), - spack.repo.path.get('bison') + spack.repo.path.get('bison'), + # GnuPG + spack.repo.path.get('gawk') ] detected_packages = spack.detection.by_executable(search_list) spack.detection.update_configuration(detected_packages, scope='bootstrap') @@ -600,10 +744,12 @@ def _config_path(): ) -def clingo_root_spec(): - # Construct the root spec that will be used to bootstrap clingo - spec_str = 'clingo-bootstrap@spack+python' +def _root_spec(spec_str): + """Add a proper compiler and target to a spec used during bootstrapping. + Args: + spec_str (str): spec to be bootstrapped. Must be without compiler and target. + """ # Add a proper compiler hint to the root spec. We use GCC for # everything but MacOS. if str(spack.platforms.host()) == 'darwin': @@ -611,17 +757,32 @@ def clingo_root_spec(): else: spec_str += ' %gcc' - # Add the generic target - generic_target = archspec.cpu.host().family - spec_str += ' target={0}'.format(str(generic_target)) - - tty.debug('[BOOTSTRAP ROOT SPEC] clingo: {0}'.format(spec_str)) + target = archspec.cpu.host().family + spec_str += ' target={0}'.format(target) + tty.debug('[BOOTSTRAP ROOT SPEC] {0}'.format(spec_str)) return spec_str +def clingo_root_spec(): + """Return the root spec used to bootstrap clingo""" + return _root_spec('clingo-bootstrap@spack+python') + + def ensure_clingo_importable_or_raise(): """Ensure that the clingo module is available for import.""" ensure_module_importable_or_raise( module='clingo', abstract_spec=clingo_root_spec() ) + + +def gnupg_root_spec(): + """Return the root spec used to bootstrap GnuPG""" + return _root_spec('gnupg@2.3:') + + +def ensure_gpg_in_path_or_raise(): + """Ensure gpg or gpg2 are in the PATH or raise.""" + ensure_executables_in_path_or_raise( + executables=['gpg2', 'gpg'], abstract_spec=gnupg_root_spec(), + ) diff --git a/lib/spack/spack/cmd/buildcache.py b/lib/spack/spack/cmd/buildcache.py index 13b0158ff4..1fda884e9a 100644 --- a/lib/spack/spack/cmd/buildcache.py +++ b/lib/spack/spack/cmd/buildcache.py @@ -104,6 +104,9 @@ def setup_parser(subparser): " instead of default platform and OS") # This argument is needed by the bootstrapping logic to verify checksums install.add_argument('--sha256', help=argparse.SUPPRESS) + install.add_argument( + '--only-root', action='store_true', help=argparse.SUPPRESS + ) arguments.add_common_arguments(install, ['specs']) install.set_defaults(func=installtarball) @@ -534,9 +537,14 @@ def install_tarball(spec, args): if s.external or s.virtual: tty.warn("Skipping external or virtual package %s" % spec.format()) return - for d in s.dependencies(deptype=('link', 'run')): - tty.msg("Installing buildcache for dependency spec %s" % d) - install_tarball(d, args) + + # This argument is used only for bootstrapping specs without signatures, + # since we need to check the sha256 of each tarball + if not args.only_root: + for d in s.dependencies(deptype=('link', 'run')): + tty.msg("Installing buildcache for dependency spec %s" % d) + install_tarball(d, args) + package = spack.repo.get(spec) if s.concrete and package.installed and not args.force: tty.warn("Package for spec %s already installed." % spec.format()) diff --git a/lib/spack/spack/modules/common.py b/lib/spack/spack/modules/common.py index ce1b9115cd..8855e57e64 100644 --- a/lib/spack/spack/modules/common.py +++ b/lib/spack/spack/modules/common.py @@ -40,7 +40,7 @@ import llnl.util.filesystem import llnl.util.tty as tty from llnl.util.lang import dedupe -import spack.build_environment as build_environment +import spack.build_environment import spack.config import spack.environment import spack.error @@ -732,12 +732,12 @@ class BaseContext(tengine.Context): # Let the extendee/dependency modify their extensions/dependencies # before asking for package-specific modifications env.extend( - build_environment.modifications_from_dependencies( + spack.build_environment.modifications_from_dependencies( spec, context='run' ) ) # Package specific modifications - build_environment.set_module_variables_for_package(spec.package) + spack.build_environment.set_module_variables_for_package(spec.package) spec.package.setup_run_environment(env) # Modifications required from modules.yaml diff --git a/lib/spack/spack/test/cmd/gpg.py b/lib/spack/spack/test/cmd/gpg.py index 6c9728d872..182773b6ce 100644 --- a/lib/spack/spack/test/cmd/gpg.py +++ b/lib/spack/spack/test/cmd/gpg.py @@ -9,6 +9,7 @@ import pytest import llnl.util.filesystem as fs +import spack.bootstrap import spack.util.executable import spack.util.gpg from spack.main import SpackCommand @@ -17,6 +18,7 @@ from spack.util.executable import ProcessError #: spack command used by tests below gpg = SpackCommand('gpg') +bootstrap = SpackCommand('bootstrap') # test gpg command detection @@ -46,9 +48,10 @@ def test_find_gpg(cmd_name, version, tmpdir, mock_gnupghome, monkeypatch): assert spack.util.gpg.GPGCONF is not None -def test_no_gpg_in_path(tmpdir, mock_gnupghome, monkeypatch): +def test_no_gpg_in_path(tmpdir, mock_gnupghome, monkeypatch, mutable_config): monkeypatch.setitem(os.environ, "PATH", str(tmpdir)) - with pytest.raises(spack.util.gpg.SpackGPGError): + bootstrap('disable') + with pytest.raises(RuntimeError): spack.util.gpg.init(force=True) diff --git a/lib/spack/spack/util/gpg.py b/lib/spack/spack/util/gpg.py index 87059d7d34..b0a4b4e430 100644 --- a/lib/spack/spack/util/gpg.py +++ b/lib/spack/spack/util/gpg.py @@ -8,6 +8,7 @@ import functools import os import re +import spack.bootstrap import spack.error import spack.paths import spack.util.executable @@ -59,7 +60,10 @@ def init(gnupghome=None, force=False): spack.paths.gpg_path) # Set the executable objects for "gpg" and "gpgconf" - GPG, GPGCONF = _gpg(), _gpgconf() + with spack.bootstrap.ensure_bootstrap_configuration(): + spack.bootstrap.ensure_gpg_in_path_or_raise() + GPG, GPGCONF = _gpg(), _gpgconf() + GPG.add_default_env('GNUPGHOME', GNUPGHOME) if GPGCONF: GPGCONF.add_default_env('GNUPGHOME', GNUPGHOME) |