Bootstrap GnuPG (#24003)

* GnuPG: allow bootstrapping from buildcache and sources

* Add a test to bootstrap GnuPG from binaries

* Disable bootstrapping in tests

* Add e2e test to bootstrap GnuPG from sources on Ubuntu

* Add e2e test to bootstrap GnuPG on macOS

5 files changed, 251 insertions, 75 deletions

diff --git a/lib/spack/spack/bootstrap.py b/lib/spack/spack/bootstrap.py
index 78ed54429f..97a38d20a0 100644
--- a/lib/spack/spack/bootstrap.py
+++ b/lib/spack/spack/bootstrap.py
@@ -6,6 +6,7 @@ from __future__ import print_function
 
 import contextlib
 import fnmatch
+import functools
 import json
 import os
 import os.path
@@ -34,11 +35,14 @@ import spack.platforms
 import spack.repo
 import spack.spec
 import spack.store
-import spack.user_environment as uenv
+import spack.user_environment
 import spack.util.executable
 import spack.util.path
 from spack.util.environment import EnvironmentModifications
 
+#: "spack buildcache" command, initialized lazily
+_buildcache_cmd = None
+
 #: Map a bootstrapper type to the corresponding class
 _bootstrap_methods = {}
 
@@ -171,6 +175,34 @@ def _fix_ext_suffix(candidate_spec):
         os.symlink(abs_path, link_name)
 
 
+def _executables_in_store(executables, abstract_spec_str):
+    """Return True if at least one of the executables can be retrieved from
+    a spec in store, False otherwise.
+
+    The different executables must provide the same functionality and are
+    "alternate" to each other, i.e. the function will exit True on the first
+    executable found.
+
+    Args:
+        executables: list of executables to be searched
+        abstract_spec_str: abstract spec that may provide the executable
+    """
+    executables_str = ', '.join(executables)
+    msg = "[BOOTSTRAP EXECUTABLES {0}] Try installed specs with query '{1}'"
+    tty.debug(msg.format(executables_str, abstract_spec_str))
+    installed_specs = spack.store.db.query(abstract_spec_str, installed=True)
+    if installed_specs:
+        for concrete_spec in installed_specs:
+            bin_dir = concrete_spec.prefix.bin
+            # IF we have a "bin" directory and it contains
+            # the executables we are looking for
+            if (os.path.exists(bin_dir) and os.path.isdir(bin_dir) and
+                    spack.util.executable.which_string(*executables, path=bin_dir)):
+                spack.util.environment.path_put_first('PATH', [bin_dir])
+                return True
+    return False
+
+
 @_bootstrapper(type='buildcache')
 class _BuildcacheBootstrapper(object):
     """Install the software needed during bootstrapping from a buildcache."""
@@ -178,93 +210,140 @@ class _BuildcacheBootstrapper(object):
         self.name = conf['name']
         self.url = conf['info']['url']
 
-    def try_import(self, module, abstract_spec_str):
-        if _try_import_from_store(module, abstract_spec_str):
-            return True
+    @staticmethod
+    def _spec_and_platform(abstract_spec_str):
+        """Return the spec object and platform we need to use when
+        querying the buildcache.
 
-        tty.info("Bootstrapping {0} from pre-built binaries".format(module))
+        Args:
+            abstract_spec_str: abstract spec string we are looking for
+        """
+        # This import is local since it is needed only on Cray
+        import spack.platforms.linux
 
         # Try to install from an unsigned binary cache
-        abstract_spec = spack.spec.Spec(
-            abstract_spec_str + ' ^' + spec_for_current_python()
-        )
-
+        abstract_spec = spack.spec.Spec(abstract_spec_str)
         # On Cray we want to use Linux binaries if available from mirrors
         bincache_platform = spack.platforms.real_host()
         if str(bincache_platform) == 'cray':
             bincache_platform = spack.platforms.Linux()
             with spack.platforms.use_platform(bincache_platform):
-                abstract_spec = spack.spec.Spec(
-                    abstract_spec_str + ' ^' + spec_for_current_python()
-                )
+                abstract_spec = spack.spec.Spec(abstract_spec_str)
+        return abstract_spec, bincache_platform
 
-        # Read information on verified clingo binaries
-        json_filename = '{0}.json'.format(module)
+    def _read_metadata(self, package_name):
+        """Return metadata about the given package."""
+        json_filename = '{0}.json'.format(package_name)
         json_path = os.path.join(
             spack.paths.share_path, 'bootstrap', self.name, json_filename
         )
         with open(json_path) as f:
             data = json.load(f)
-
-        buildcache = spack.main.SpackCommand('buildcache')
+        return data
+
+    def _install_by_hash(self, pkg_hash, pkg_sha256, index, bincache_platform):
+        global _buildcache_cmd
+
+        if _buildcache_cmd is None:
+            _buildcache_cmd = spack.main.SpackCommand('buildcache')
+
+        index_spec = next(x for x in index if x.dag_hash() == pkg_hash)
+        # Reconstruct the compiler that we need to use for bootstrapping
+        compiler_entry = {
+            "modules": [],
+            "operating_system": str(index_spec.os),
+            "paths": {
+                "cc": "/dev/null",
+                "cxx": "/dev/null",
+                "f77": "/dev/null",
+                "fc": "/dev/null"
+            },
+            "spec": str(index_spec.compiler),
+            "target": str(index_spec.target.family)
+        }
+        with spack.platforms.use_platform(bincache_platform):
+            with spack.config.override(
+                    'compilers', [{'compiler': compiler_entry}]
+            ):
+                spec_str = '/' + pkg_hash
+                install_args = [
+                    'install',
+                    '--sha256', pkg_sha256,
+                    '--only-root',
+                    '-a', '-u', '-o', '-f', spec_str
+                ]
+                _buildcache_cmd(*install_args, fail_on_error=False)
+
+    def _install_and_test(
+            self, abstract_spec, bincache_platform, bincache_data, test_fn
+    ):
         # Ensure we see only the buildcache being used to bootstrap
-        mirror_scope = spack.config.InternalConfigScope(
-            'bootstrap_buildcache', {'mirrors:': {self.name: self.url}}
-        )
-        with spack.config.override(mirror_scope):
+        with spack.config.override(self.mirror_scope):
             # This index is currently needed to get the compiler used to build some
-            # specs that wwe know by dag hash.
+            # specs that we know by dag hash.
             spack.binary_distribution.binary_index.regenerate_spec_cache()
             index = spack.binary_distribution.update_cache_and_get_specs()
 
             if not index:
                 raise RuntimeError("The binary index is empty")
 
-            for item in data['verified']:
+            for item in bincache_data['verified']:
                 candidate_spec = item['spec']
-                python_spec = item['python']
+                # This will be None for things that don't depend on python
+                python_spec = item.get('python', None)
                 # Skip specs which are not compatible
                 if not abstract_spec.satisfies(candidate_spec):
                     continue
 
-                if python_spec not in abstract_spec:
+                if python_spec is not None and python_spec not in abstract_spec:
                     continue
 
                 for pkg_name, pkg_hash, pkg_sha256 in item['binaries']:
-                    msg = ('[BOOTSTRAP MODULE {0}] Try installing "{1}" from binary '
-                           'cache at "{2}"')
-                    tty.debug(msg.format(module, pkg_name, self.url))
-                    index_spec = next(x for x in index if x.dag_hash() == pkg_hash)
-                    # Reconstruct the compiler that we need to use for bootstrapping
-                    compiler_entry = {
-                        "modules": [],
-                        "operating_system": str(index_spec.os),
-                        "paths": {
-                            "cc": "/dev/null",
-                            "cxx": "/dev/null",
-                            "f77": "/dev/null",
-                            "fc": "/dev/null"
-                        },
-                        "spec": str(index_spec.compiler),
-                        "target": str(index_spec.target.family)
-                    }
-                    with spack.platforms.use_platform(bincache_platform):
-                        with spack.config.override(
-                                'compilers', [{'compiler': compiler_entry}]
-                        ):
-                            spec_str = '/' + pkg_hash
-                            install_args = [
-                                'install',
-                                '--sha256', pkg_sha256,
-                                '-a', '-u', '-o', '-f', spec_str
-                            ]
-                            buildcache(*install_args, fail_on_error=False)
-                # TODO: undo installations that didn't complete?
-
-                if _try_import_from_store(module, abstract_spec_str):
+                    # TODO: undo installations that didn't complete?
+                    self._install_by_hash(
+                        pkg_hash, pkg_sha256, index, bincache_platform
+                    )
+
+                if test_fn():
                     return True
         return False
 
+    @property
+    def mirror_scope(self):
+        return spack.config.InternalConfigScope(
+            'bootstrap', {'mirrors:': {self.name: self.url}}
+        )
+
+    def try_import(self, module, abstract_spec_str):
+        test_fn = functools.partial(_try_import_from_store, module, abstract_spec_str)
+        if test_fn():
+            return True
+
+        tty.info("Bootstrapping {0} from pre-built binaries".format(module))
+        abstract_spec, bincache_platform = self._spec_and_platform(
+            abstract_spec_str + ' ^' + spec_for_current_python()
+        )
+        data = self._read_metadata(module)
+        return self._install_and_test(
+            abstract_spec, bincache_platform, data, test_fn
+        )
+
+    def try_search_path(self, executables, abstract_spec_str):
+        test_fn = functools.partial(
+            _executables_in_store, executables, abstract_spec_str
+        )
+        if test_fn():
+            return True
+
+        abstract_spec, bincache_platform = self._spec_and_platform(
+            abstract_spec_str
+        )
+        tty.info("Bootstrapping {0} from pre-built binaries".format(abstract_spec.name))
+        data = self._read_metadata(abstract_spec.name)
+        return self._install_and_test(
+            abstract_spec, bincache_platform, data, test_fn
+        )
+
 
 @_bootstrapper(type='install')
 class _SourceBootstrapper(object):
@@ -307,6 +386,26 @@ class _SourceBootstrapper(object):
 
         return _try_import_from_store(module, abstract_spec_str=abstract_spec_str)
 
+    def try_search_path(self, executables, abstract_spec_str):
+        if _executables_in_store(executables, abstract_spec_str):
+            return True
+
+        # If we compile code from sources detecting a few build tools
+        # might reduce compilation time by a fair amount
+        _add_externals_if_missing()
+
+        # Add hint to use frontend operating system on Cray
+        if str(spack.platforms.host()) == 'cray':
+            abstract_spec_str += ' os=fe'
+
+        concrete_spec = spack.spec.Spec(abstract_spec_str)
+        concrete_spec.concretize()
+
+        msg = "[BOOTSTRAP GnuPG] Try installing '{0}' from sources"
+        tty.debug(msg.format(abstract_spec_str))
+        concrete_spec.package.do_install()
+        return _executables_in_store(executables, abstract_spec_str)
+
 
 def _make_bootstrapper(conf):
     """Return a bootstrap object built according to the
@@ -418,6 +517,44 @@ def ensure_module_importable_or_raise(module, abstract_spec=None):
     raise ImportError(msg)
 
 
+def ensure_executables_in_path_or_raise(executables, abstract_spec):
+    """Ensure that some executables are in path or raise.
+
+    Args:
+        executables (list): list of executables to be searched in the PATH,
+            in order. The function exits on the first one found.
+        abstract_spec (str): abstract spec that provides the executables
+
+    Raises:
+        RuntimeError: if the executables cannot be ensured to be in PATH
+    """
+    if spack.util.executable.which_string(*executables):
+        return
+
+    executables_str = ', '.join(executables)
+    source_configs = spack.config.get('bootstrap:sources', [])
+    for current_config in source_configs:
+        if not _source_is_trusted(current_config):
+            msg = ('[BOOTSTRAP EXECUTABLES {0}] Skipping source "{1}" since it is '
+                   'not trusted').format(executables_str, current_config['name'])
+            tty.debug(msg)
+            continue
+
+        b = _make_bootstrapper(current_config)
+        try:
+            if b.try_search_path(executables, abstract_spec):
+                return
+        except Exception as e:
+            msg = '[BOOTSTRAP EXECUTABLES {0}] Unexpected error "{1}"'
+            tty.debug(msg.format(executables_str, str(e)))
+
+    # We couldn't import in any way, so raise an import error
+    msg = 'cannot bootstrap any of the {0} executables'.format(executables_str)
+    if abstract_spec:
+        msg += ' from spec "{0}"'.format(abstract_spec)
+    raise RuntimeError(msg)
+
+
 def _python_import(module):
     try:
         __import__(module)
@@ -455,7 +592,9 @@ def get_executable(exe, spec=None, install=False):
             ret = spack.util.executable.Executable(exe_path[0])
             envmod = EnvironmentModifications()
             for dep in ispec.traverse(root=True, order='post'):
-                envmod.extend(uenv.environment_modifications_for_spec(dep))
+                envmod.extend(
+                    spack.user_environment.environment_modifications_for_spec(dep)
+                )
             ret.add_default_envmod(envmod)
             return ret
         else:
@@ -484,7 +623,9 @@ def get_executable(exe, spec=None, install=False):
         ret = spack.util.executable.Executable(exe_path[0])
         envmod = EnvironmentModifications()
         for dep in spec.traverse(root=True, order='post'):
-            envmod.extend(uenv.environment_modifications_for_spec(dep))
+            envmod.extend(
+                spack.user_environment.environment_modifications_for_spec(dep)
+            )
         ret.add_default_envmod(envmod)
         return ret
 
@@ -523,8 +664,11 @@ def _add_compilers_if_missing():
 
 def _add_externals_if_missing():
     search_list = [
+        # clingo
         spack.repo.path.get('cmake'),
-        spack.repo.path.get('bison')
+        spack.repo.path.get('bison'),
+        # GnuPG
+        spack.repo.path.get('gawk')
     ]
     detected_packages = spack.detection.by_executable(search_list)
     spack.detection.update_configuration(detected_packages, scope='bootstrap')
@@ -600,10 +744,12 @@ def _config_path():
     )
 
 
-def clingo_root_spec():
-    # Construct the root spec that will be used to bootstrap clingo
-    spec_str = 'clingo-bootstrap@spack+python'
+def _root_spec(spec_str):
+    """Add a proper compiler and target to a spec used during bootstrapping.
 
+    Args:
+        spec_str (str): spec to be bootstrapped. Must be without compiler and target.
+    """
     # Add a proper compiler hint to the root spec. We use GCC for
     # everything but MacOS.
     if str(spack.platforms.host()) == 'darwin':
@@ -611,17 +757,32 @@ def clingo_root_spec():
     else:
         spec_str += ' %gcc'
 
-    # Add the generic target
-    generic_target = archspec.cpu.host().family
-    spec_str += ' target={0}'.format(str(generic_target))
-
-    tty.debug('[BOOTSTRAP ROOT SPEC] clingo: {0}'.format(spec_str))
+    target = archspec.cpu.host().family
+    spec_str += ' target={0}'.format(target)
 
+    tty.debug('[BOOTSTRAP ROOT SPEC] {0}'.format(spec_str))
     return spec_str
 
 
+def clingo_root_spec():
+    """Return the root spec used to bootstrap clingo"""
+    return _root_spec('clingo-bootstrap@spack+python')
+
+
 def ensure_clingo_importable_or_raise():
     """Ensure that the clingo module is available for import."""
     ensure_module_importable_or_raise(
         module='clingo', abstract_spec=clingo_root_spec()
     )
+
+
+def gnupg_root_spec():
+    """Return the root spec used to bootstrap GnuPG"""
+    return _root_spec('gnupg@2.3:')
+
+
+def ensure_gpg_in_path_or_raise():
+    """Ensure gpg or gpg2 are in the PATH or raise."""
+    ensure_executables_in_path_or_raise(
+        executables=['gpg2', 'gpg'], abstract_spec=gnupg_root_spec(),
+    )
diff --git a/lib/spack/spack/cmd/buildcache.py b/lib/spack/spack/cmd/buildcache.py
index 13b0158ff4..1fda884e9a 100644
--- a/lib/spack/spack/cmd/buildcache.py
+++ b/lib/spack/spack/cmd/buildcache.py
@@ -104,6 +104,9 @@ def setup_parser(subparser):
                               " instead of default platform and OS")
     # This argument is needed by the bootstrapping logic to verify checksums
     install.add_argument('--sha256', help=argparse.SUPPRESS)
+    install.add_argument(
+        '--only-root', action='store_true', help=argparse.SUPPRESS
+    )
 
     arguments.add_common_arguments(install, ['specs'])
     install.set_defaults(func=installtarball)
@@ -534,9 +537,14 @@ def install_tarball(spec, args):
     if s.external or s.virtual:
         tty.warn("Skipping external or virtual package %s" % spec.format())
         return
-    for d in s.dependencies(deptype=('link', 'run')):
-        tty.msg("Installing buildcache for dependency spec %s" % d)
-        install_tarball(d, args)
+
+    # This argument is used only for bootstrapping specs without signatures,
+    # since we need to check the sha256 of each tarball
+    if not args.only_root:
+        for d in s.dependencies(deptype=('link', 'run')):
+            tty.msg("Installing buildcache for dependency spec %s" % d)
+            install_tarball(d, args)
+
     package = spack.repo.get(spec)
     if s.concrete and package.installed and not args.force:
         tty.warn("Package for spec %s already installed." % spec.format())
diff --git a/lib/spack/spack/modules/common.py b/lib/spack/spack/modules/common.py
index ce1b9115cd..8855e57e64 100644
--- a/lib/spack/spack/modules/common.py
+++ b/lib/spack/spack/modules/common.py
@@ -40,7 +40,7 @@ import llnl.util.filesystem
 import llnl.util.tty as tty
 from llnl.util.lang import dedupe
 
-import spack.build_environment as build_environment
+import spack.build_environment
 import spack.config
 import spack.environment
 import spack.error
@@ -732,12 +732,12 @@ class BaseContext(tengine.Context):
         # Let the extendee/dependency modify their extensions/dependencies
         # before asking for package-specific modifications
         env.extend(
-            build_environment.modifications_from_dependencies(
+            spack.build_environment.modifications_from_dependencies(
                 spec, context='run'
             )
         )
         # Package specific modifications
-        build_environment.set_module_variables_for_package(spec.package)
+        spack.build_environment.set_module_variables_for_package(spec.package)
         spec.package.setup_run_environment(env)
 
         # Modifications required from modules.yaml
diff --git a/lib/spack/spack/test/cmd/gpg.py b/lib/spack/spack/test/cmd/gpg.py
index 6c9728d872..182773b6ce 100644
--- a/lib/spack/spack/test/cmd/gpg.py
+++ b/lib/spack/spack/test/cmd/gpg.py
@@ -9,6 +9,7 @@ import pytest
 
 import llnl.util.filesystem as fs
 
+import spack.bootstrap
 import spack.util.executable
 import spack.util.gpg
 from spack.main import SpackCommand
@@ -17,6 +18,7 @@ from spack.util.executable import ProcessError
 
 #: spack command used by tests below
 gpg = SpackCommand('gpg')
+bootstrap = SpackCommand('bootstrap')
 
 
 # test gpg command detection
@@ -46,9 +48,10 @@ def test_find_gpg(cmd_name, version, tmpdir, mock_gnupghome, monkeypatch):
         assert spack.util.gpg.GPGCONF is not None
 
 
-def test_no_gpg_in_path(tmpdir, mock_gnupghome, monkeypatch):
+def test_no_gpg_in_path(tmpdir, mock_gnupghome, monkeypatch, mutable_config):
     monkeypatch.setitem(os.environ, "PATH", str(tmpdir))
-    with pytest.raises(spack.util.gpg.SpackGPGError):
+    bootstrap('disable')
+    with pytest.raises(RuntimeError):
         spack.util.gpg.init(force=True)
 
 
diff --git a/lib/spack/spack/util/gpg.py b/lib/spack/spack/util/gpg.py
index 87059d7d34..b0a4b4e430 100644
--- a/lib/spack/spack/util/gpg.py
+++ b/lib/spack/spack/util/gpg.py
@@ -8,6 +8,7 @@ import functools
 import os
 import re
 
+import spack.bootstrap
 import spack.error
 import spack.paths
 import spack.util.executable
@@ -59,7 +60,10 @@ def init(gnupghome=None, force=False):
                  spack.paths.gpg_path)
 
     # Set the executable objects for "gpg" and "gpgconf"
-    GPG, GPGCONF = _gpg(), _gpgconf()
+    with spack.bootstrap.ensure_bootstrap_configuration():
+        spack.bootstrap.ensure_gpg_in_path_or_raise()
+        GPG, GPGCONF = _gpg(), _gpgconf()
+
     GPG.add_default_env('GNUPGHOME', GNUPGHOME)
     if GPGCONF:
         GPGCONF.add_default_env('GNUPGHOME', GNUPGHOME)