From 78c08fccd56d073a336eeee3dd4548d81101c920 Mon Sep 17 00:00:00 2001
From: Massimiliano Culpo <massimiliano.culpo@gmail.com>
Date: Wed, 3 Nov 2021 07:15:24 +0100
Subject: Bootstrap GnuPG (#24003)
* GnuPG: allow bootstrapping from buildcache and sources
* Add a test to bootstrap GnuPG from binaries
* Disable bootstrapping in tests
* Add e2e test to bootstrap GnuPG from sources on Ubuntu
* Add e2e test to bootstrap GnuPG on macOS
---
lib/spack/spack/bootstrap.py | 293 +++++++++++++++++++++++++++++---------
lib/spack/spack/cmd/buildcache.py | 14 +-
lib/spack/spack/modules/common.py | 6 +-
lib/spack/spack/test/cmd/gpg.py | 7 +-
lib/spack/spack/util/gpg.py | 6 +-
5 files changed, 251 insertions(+), 75 deletions(-)
(limited to 'lib')
diff --git a/lib/spack/spack/bootstrap.py b/lib/spack/spack/bootstrap.py
index 78ed54429f..97a38d20a0 100644
--- a/lib/spack/spack/bootstrap.py
+++ b/lib/spack/spack/bootstrap.py
@@ -6,6 +6,7 @@ from __future__ import print_function
import contextlib
import fnmatch
+import functools
import json
import os
import os.path
@@ -34,11 +35,14 @@ import spack.platforms
import spack.repo
import spack.spec
import spack.store
-import spack.user_environment as uenv
+import spack.user_environment
import spack.util.executable
import spack.util.path
from spack.util.environment import EnvironmentModifications
+#: "spack buildcache" command, initialized lazily
+_buildcache_cmd = None
+
#: Map a bootstrapper type to the corresponding class
_bootstrap_methods = {}
@@ -171,6 +175,34 @@ def _fix_ext_suffix(candidate_spec):
os.symlink(abs_path, link_name)
+def _executables_in_store(executables, abstract_spec_str):
+ """Return True if at least one of the executables can be retrieved from
+ a spec in store, False otherwise.
+
+ The different executables must provide the same functionality and are
+ "alternate" to each other, i.e. the function will exit True on the first
+ executable found.
+
+ Args:
+ executables: list of executables to be searched
+ abstract_spec_str: abstract spec that may provide the executable
+ """
+ executables_str = ', '.join(executables)
+ msg = "[BOOTSTRAP EXECUTABLES {0}] Try installed specs with query '{1}'"
+ tty.debug(msg.format(executables_str, abstract_spec_str))
+ installed_specs = spack.store.db.query(abstract_spec_str, installed=True)
+ if installed_specs:
+ for concrete_spec in installed_specs:
+ bin_dir = concrete_spec.prefix.bin
+ # IF we have a "bin" directory and it contains
+ # the executables we are looking for
+ if (os.path.exists(bin_dir) and os.path.isdir(bin_dir) and
+ spack.util.executable.which_string(*executables, path=bin_dir)):
+ spack.util.environment.path_put_first('PATH', [bin_dir])
+ return True
+ return False
+
+
@_bootstrapper(type='buildcache')
class _BuildcacheBootstrapper(object):
"""Install the software needed during bootstrapping from a buildcache."""
@@ -178,93 +210,140 @@ class _BuildcacheBootstrapper(object):
self.name = conf['name']
self.url = conf['info']['url']
- def try_import(self, module, abstract_spec_str):
- if _try_import_from_store(module, abstract_spec_str):
- return True
+ @staticmethod
+ def _spec_and_platform(abstract_spec_str):
+ """Return the spec object and platform we need to use when
+ querying the buildcache.
- tty.info("Bootstrapping {0} from pre-built binaries".format(module))
+ Args:
+ abstract_spec_str: abstract spec string we are looking for
+ """
+ # This import is local since it is needed only on Cray
+ import spack.platforms.linux
# Try to install from an unsigned binary cache
- abstract_spec = spack.spec.Spec(
- abstract_spec_str + ' ^' + spec_for_current_python()
- )
-
+ abstract_spec = spack.spec.Spec(abstract_spec_str)
# On Cray we want to use Linux binaries if available from mirrors
bincache_platform = spack.platforms.real_host()
if str(bincache_platform) == 'cray':
bincache_platform = spack.platforms.Linux()
with spack.platforms.use_platform(bincache_platform):
- abstract_spec = spack.spec.Spec(
- abstract_spec_str + ' ^' + spec_for_current_python()
- )
+ abstract_spec = spack.spec.Spec(abstract_spec_str)
+ return abstract_spec, bincache_platform
- # Read information on verified clingo binaries
- json_filename = '{0}.json'.format(module)
+ def _read_metadata(self, package_name):
+ """Return metadata about the given package."""
+ json_filename = '{0}.json'.format(package_name)
json_path = os.path.join(
spack.paths.share_path, 'bootstrap', self.name, json_filename
)
with open(json_path) as f:
data = json.load(f)
-
- buildcache = spack.main.SpackCommand('buildcache')
+ return data
+
+ def _install_by_hash(self, pkg_hash, pkg_sha256, index, bincache_platform):
+ global _buildcache_cmd
+
+ if _buildcache_cmd is None:
+ _buildcache_cmd = spack.main.SpackCommand('buildcache')
+
+ index_spec = next(x for x in index if x.dag_hash() == pkg_hash)
+ # Reconstruct the compiler that we need to use for bootstrapping
+ compiler_entry = {
+ "modules": [],
+ "operating_system": str(index_spec.os),
+ "paths": {
+ "cc": "/dev/null",
+ "cxx": "/dev/null",
+ "f77": "/dev/null",
+ "fc": "/dev/null"
+ },
+ "spec": str(index_spec.compiler),
+ "target": str(index_spec.target.family)
+ }
+ with spack.platforms.use_platform(bincache_platform):
+ with spack.config.override(
+ 'compilers', [{'compiler': compiler_entry}]
+ ):
+ spec_str = '/' + pkg_hash
+ install_args = [
+ 'install',
+ '--sha256', pkg_sha256,
+ '--only-root',
+ '-a', '-u', '-o', '-f', spec_str
+ ]
+ _buildcache_cmd(*install_args, fail_on_error=False)
+
+ def _install_and_test(
+ self, abstract_spec, bincache_platform, bincache_data, test_fn
+ ):
# Ensure we see only the buildcache being used to bootstrap
- mirror_scope = spack.config.InternalConfigScope(
- 'bootstrap_buildcache', {'mirrors:': {self.name: self.url}}
- )
- with spack.config.override(mirror_scope):
+ with spack.config.override(self.mirror_scope):
# This index is currently needed to get the compiler used to build some
- # specs that wwe know by dag hash.
+ # specs that we know by dag hash.
spack.binary_distribution.binary_index.regenerate_spec_cache()
index = spack.binary_distribution.update_cache_and_get_specs()
if not index:
raise RuntimeError("The binary index is empty")
- for item in data['verified']:
+ for item in bincache_data['verified']:
candidate_spec = item['spec']
- python_spec = item['python']
+ # This will be None for things that don't depend on python
+ python_spec = item.get('python', None)
# Skip specs which are not compatible
if not abstract_spec.satisfies(candidate_spec):
continue
- if python_spec not in abstract_spec:
+ if python_spec is not None and python_spec not in abstract_spec:
continue
for pkg_name, pkg_hash, pkg_sha256 in item['binaries']:
- msg = ('[BOOTSTRAP MODULE {0}] Try installing "{1}" from binary '
- 'cache at "{2}"')
- tty.debug(msg.format(module, pkg_name, self.url))
- index_spec = next(x for x in index if x.dag_hash() == pkg_hash)
- # Reconstruct the compiler that we need to use for bootstrapping
- compiler_entry = {
- "modules": [],
- "operating_system": str(index_spec.os),
- "paths": {
- "cc": "/dev/null",
- "cxx": "/dev/null",
- "f77": "/dev/null",
- "fc": "/dev/null"
- },
- "spec": str(index_spec.compiler),
- "target": str(index_spec.target.family)
- }
- with spack.platforms.use_platform(bincache_platform):
- with spack.config.override(
- 'compilers', [{'compiler': compiler_entry}]
- ):
- spec_str = '/' + pkg_hash
- install_args = [
- 'install',
- '--sha256', pkg_sha256,
- '-a', '-u', '-o', '-f', spec_str
- ]
- buildcache(*install_args, fail_on_error=False)
- # TODO: undo installations that didn't complete?
-
- if _try_import_from_store(module, abstract_spec_str):
+ # TODO: undo installations that didn't complete?
+ self._install_by_hash(
+ pkg_hash, pkg_sha256, index, bincache_platform
+ )
+
+ if test_fn():
return True
return False
+ @property
+ def mirror_scope(self):
+ return spack.config.InternalConfigScope(
+ 'bootstrap', {'mirrors:': {self.name: self.url}}
+ )
+
+ def try_import(self, module, abstract_spec_str):
+ test_fn = functools.partial(_try_import_from_store, module, abstract_spec_str)
+ if test_fn():
+ return True
+
+ tty.info("Bootstrapping {0} from pre-built binaries".format(module))
+ abstract_spec, bincache_platform = self._spec_and_platform(
+ abstract_spec_str + ' ^' + spec_for_current_python()
+ )
+ data = self._read_metadata(module)
+ return self._install_and_test(
+ abstract_spec, bincache_platform, data, test_fn
+ )
+
+ def try_search_path(self, executables, abstract_spec_str):
+ test_fn = functools.partial(
+ _executables_in_store, executables, abstract_spec_str
+ )
+ if test_fn():
+ return True
+
+ abstract_spec, bincache_platform = self._spec_and_platform(
+ abstract_spec_str
+ )
+ tty.info("Bootstrapping {0} from pre-built binaries".format(abstract_spec.name))
+ data = self._read_metadata(abstract_spec.name)
+ return self._install_and_test(
+ abstract_spec, bincache_platform, data, test_fn
+ )
+
@_bootstrapper(type='install')
class _SourceBootstrapper(object):
@@ -307,6 +386,26 @@ class _SourceBootstrapper(object):
return _try_import_from_store(module, abstract_spec_str=abstract_spec_str)
+ def try_search_path(self, executables, abstract_spec_str):
+ if _executables_in_store(executables, abstract_spec_str):
+ return True
+
+ # If we compile code from sources detecting a few build tools
+ # might reduce compilation time by a fair amount
+ _add_externals_if_missing()
+
+ # Add hint to use frontend operating system on Cray
+ if str(spack.platforms.host()) == 'cray':
+ abstract_spec_str += ' os=fe'
+
+ concrete_spec = spack.spec.Spec(abstract_spec_str)
+ concrete_spec.concretize()
+
+ msg = "[BOOTSTRAP GnuPG] Try installing '{0}' from sources"
+ tty.debug(msg.format(abstract_spec_str))
+ concrete_spec.package.do_install()
+ return _executables_in_store(executables, abstract_spec_str)
+
def _make_bootstrapper(conf):
"""Return a bootstrap object built according to the
@@ -418,6 +517,44 @@ def ensure_module_importable_or_raise(module, abstract_spec=None):
raise ImportError(msg)
+def ensure_executables_in_path_or_raise(executables, abstract_spec):
+ """Ensure that some executables are in path or raise.
+
+ Args:
+ executables (list): list of executables to be searched in the PATH,
+ in order. The function exits on the first one found.
+ abstract_spec (str): abstract spec that provides the executables
+
+ Raises:
+ RuntimeError: if the executables cannot be ensured to be in PATH
+ """
+ if spack.util.executable.which_string(*executables):
+ return
+
+ executables_str = ', '.join(executables)
+ source_configs = spack.config.get('bootstrap:sources', [])
+ for current_config in source_configs:
+ if not _source_is_trusted(current_config):
+ msg = ('[BOOTSTRAP EXECUTABLES {0}] Skipping source "{1}" since it is '
+ 'not trusted').format(executables_str, current_config['name'])
+ tty.debug(msg)
+ continue
+
+ b = _make_bootstrapper(current_config)
+ try:
+ if b.try_search_path(executables, abstract_spec):
+ return
+ except Exception as e:
+ msg = '[BOOTSTRAP EXECUTABLES {0}] Unexpected error "{1}"'
+ tty.debug(msg.format(executables_str, str(e)))
+
+ # We couldn't import in any way, so raise an import error
+ msg = 'cannot bootstrap any of the {0} executables'.format(executables_str)
+ if abstract_spec:
+ msg += ' from spec "{0}"'.format(abstract_spec)
+ raise RuntimeError(msg)
+
+
def _python_import(module):
try:
__import__(module)
@@ -455,7 +592,9 @@ def get_executable(exe, spec=None, install=False):
ret = spack.util.executable.Executable(exe_path[0])
envmod = EnvironmentModifications()
for dep in ispec.traverse(root=True, order='post'):
- envmod.extend(uenv.environment_modifications_for_spec(dep))
+ envmod.extend(
+ spack.user_environment.environment_modifications_for_spec(dep)
+ )
ret.add_default_envmod(envmod)
return ret
else:
@@ -484,7 +623,9 @@ def get_executable(exe, spec=None, install=False):
ret = spack.util.executable.Executable(exe_path[0])
envmod = EnvironmentModifications()
for dep in spec.traverse(root=True, order='post'):
- envmod.extend(uenv.environment_modifications_for_spec(dep))
+ envmod.extend(
+ spack.user_environment.environment_modifications_for_spec(dep)
+ )
ret.add_default_envmod(envmod)
return ret
@@ -523,8 +664,11 @@ def _add_compilers_if_missing():
def _add_externals_if_missing():
search_list = [
+ # clingo
spack.repo.path.get('cmake'),
- spack.repo.path.get('bison')
+ spack.repo.path.get('bison'),
+ # GnuPG
+ spack.repo.path.get('gawk')
]
detected_packages = spack.detection.by_executable(search_list)
spack.detection.update_configuration(detected_packages, scope='bootstrap')
@@ -600,10 +744,12 @@ def _config_path():
)
-def clingo_root_spec():
- # Construct the root spec that will be used to bootstrap clingo
- spec_str = 'clingo-bootstrap@spack+python'
+def _root_spec(spec_str):
+ """Add a proper compiler and target to a spec used during bootstrapping.
+ Args:
+ spec_str (str): spec to be bootstrapped. Must be without compiler and target.
+ """
# Add a proper compiler hint to the root spec. We use GCC for
# everything but MacOS.
if str(spack.platforms.host()) == 'darwin':
@@ -611,17 +757,32 @@ def clingo_root_spec():
else:
spec_str += ' %gcc'
- # Add the generic target
- generic_target = archspec.cpu.host().family
- spec_str += ' target={0}'.format(str(generic_target))
-
- tty.debug('[BOOTSTRAP ROOT SPEC] clingo: {0}'.format(spec_str))
+ target = archspec.cpu.host().family
+ spec_str += ' target={0}'.format(target)
+ tty.debug('[BOOTSTRAP ROOT SPEC] {0}'.format(spec_str))
return spec_str
+def clingo_root_spec():
+ """Return the root spec used to bootstrap clingo"""
+ return _root_spec('clingo-bootstrap@spack+python')
+
+
def ensure_clingo_importable_or_raise():
"""Ensure that the clingo module is available for import."""
ensure_module_importable_or_raise(
module='clingo', abstract_spec=clingo_root_spec()
)
+
+
+def gnupg_root_spec():
+ """Return the root spec used to bootstrap GnuPG"""
+ return _root_spec('gnupg@2.3:')
+
+
+def ensure_gpg_in_path_or_raise():
+ """Ensure gpg or gpg2 are in the PATH or raise."""
+ ensure_executables_in_path_or_raise(
+ executables=['gpg2', 'gpg'], abstract_spec=gnupg_root_spec(),
+ )
diff --git a/lib/spack/spack/cmd/buildcache.py b/lib/spack/spack/cmd/buildcache.py
index 13b0158ff4..1fda884e9a 100644
--- a/lib/spack/spack/cmd/buildcache.py
+++ b/lib/spack/spack/cmd/buildcache.py
@@ -104,6 +104,9 @@ def setup_parser(subparser):
" instead of default platform and OS")
# This argument is needed by the bootstrapping logic to verify checksums
install.add_argument('--sha256', help=argparse.SUPPRESS)
+ install.add_argument(
+ '--only-root', action='store_true', help=argparse.SUPPRESS
+ )
arguments.add_common_arguments(install, ['specs'])
install.set_defaults(func=installtarball)
@@ -534,9 +537,14 @@ def install_tarball(spec, args):
if s.external or s.virtual:
tty.warn("Skipping external or virtual package %s" % spec.format())
return
- for d in s.dependencies(deptype=('link', 'run')):
- tty.msg("Installing buildcache for dependency spec %s" % d)
- install_tarball(d, args)
+
+ # This argument is used only for bootstrapping specs without signatures,
+ # since we need to check the sha256 of each tarball
+ if not args.only_root:
+ for d in s.dependencies(deptype=('link', 'run')):
+ tty.msg("Installing buildcache for dependency spec %s" % d)
+ install_tarball(d, args)
+
package = spack.repo.get(spec)
if s.concrete and package.installed and not args.force:
tty.warn("Package for spec %s already installed." % spec.format())
diff --git a/lib/spack/spack/modules/common.py b/lib/spack/spack/modules/common.py
index ce1b9115cd..8855e57e64 100644
--- a/lib/spack/spack/modules/common.py
+++ b/lib/spack/spack/modules/common.py
@@ -40,7 +40,7 @@ import llnl.util.filesystem
import llnl.util.tty as tty
from llnl.util.lang import dedupe
-import spack.build_environment as build_environment
+import spack.build_environment
import spack.config
import spack.environment
import spack.error
@@ -732,12 +732,12 @@ class BaseContext(tengine.Context):
# Let the extendee/dependency modify their extensions/dependencies
# before asking for package-specific modifications
env.extend(
- build_environment.modifications_from_dependencies(
+ spack.build_environment.modifications_from_dependencies(
spec, context='run'
)
)
# Package specific modifications
- build_environment.set_module_variables_for_package(spec.package)
+ spack.build_environment.set_module_variables_for_package(spec.package)
spec.package.setup_run_environment(env)
# Modifications required from modules.yaml
diff --git a/lib/spack/spack/test/cmd/gpg.py b/lib/spack/spack/test/cmd/gpg.py
index 6c9728d872..182773b6ce 100644
--- a/lib/spack/spack/test/cmd/gpg.py
+++ b/lib/spack/spack/test/cmd/gpg.py
@@ -9,6 +9,7 @@ import pytest
import llnl.util.filesystem as fs
+import spack.bootstrap
import spack.util.executable
import spack.util.gpg
from spack.main import SpackCommand
@@ -17,6 +18,7 @@ from spack.util.executable import ProcessError
#: spack command used by tests below
gpg = SpackCommand('gpg')
+bootstrap = SpackCommand('bootstrap')
# test gpg command detection
@@ -46,9 +48,10 @@ def test_find_gpg(cmd_name, version, tmpdir, mock_gnupghome, monkeypatch):
assert spack.util.gpg.GPGCONF is not None
-def test_no_gpg_in_path(tmpdir, mock_gnupghome, monkeypatch):
+def test_no_gpg_in_path(tmpdir, mock_gnupghome, monkeypatch, mutable_config):
monkeypatch.setitem(os.environ, "PATH", str(tmpdir))
- with pytest.raises(spack.util.gpg.SpackGPGError):
+ bootstrap('disable')
+ with pytest.raises(RuntimeError):
spack.util.gpg.init(force=True)
diff --git a/lib/spack/spack/util/gpg.py b/lib/spack/spack/util/gpg.py
index 87059d7d34..b0a4b4e430 100644
--- a/lib/spack/spack/util/gpg.py
+++ b/lib/spack/spack/util/gpg.py
@@ -8,6 +8,7 @@ import functools
import os
import re
+import spack.bootstrap
import spack.error
import spack.paths
import spack.util.executable
@@ -59,7 +60,10 @@ def init(gnupghome=None, force=False):
spack.paths.gpg_path)
# Set the executable objects for "gpg" and "gpgconf"
- GPG, GPGCONF = _gpg(), _gpgconf()
+ with spack.bootstrap.ensure_bootstrap_configuration():
+ spack.bootstrap.ensure_gpg_in_path_or_raise()
+ GPG, GPGCONF = _gpg(), _gpgconf()
+
GPG.add_default_env('GNUPGHOME', GNUPGHOME)
if GPGCONF:
GPGCONF.add_default_env('GNUPGHOME', GNUPGHOME)
--
cgit v1.2.3-70-g09d2