Bootstrap clingo from sources (#21446)

* Allow the bootstrapping of clingo from sources

Allow python builds with system python as external
for MacOS

* Ensure consistent configuration when bootstrapping clingo

This commit uses context managers to ensure we can
bootstrap clingo using a consistent configuration
regardless of the use case being managed.

* Github actions: test clingo with bootstrapping from sources

* Add command to inspect and clean the bootstrap store

 Prevent users to set the install tree root to the bootstrap store

* clingo: documented how to bootstrap from sources

Co-authored-by: Gregory Becker <becker33@llnl.gov>

10 files changed, 348 insertions, 19 deletions

diff --git a/lib/spack/docs/getting_started.rst b/lib/spack/docs/getting_started.rst
index 6f3340b18b..260dc25c51 100644
--- a/lib/spack/docs/getting_started.rst
+++ b/lib/spack/docs/getting_started.rst
@@ -111,6 +111,53 @@ environment*, especially for ``PATH``.  Only software that comes with
 the system, or that you know you wish to use with Spack, should be
 included.  This procedure will avoid many strange build errors.
 
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Optional: Bootstrapping clingo
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Spack supports using clingo as an external solver to compute which software
+needs to be installed. If you have a default compiler supporting C++14 Spack
+can automatically bootstrap this tool from sources the first time it is
+needed:
+
+.. code-block:: console
+
+   $ spack solve zlib
+   [+] /usr (external bison-3.0.4-wu5pgjchxzemk5ya2l3ddqug2d7jv6eb)
+   [+] /usr (external cmake-3.19.4-a4kmcfzxxy45mzku4ipmj5kdiiz5a57b)
+   [+] /usr (external python-3.6.9-x4fou4iqqlh5ydwddx3pvfcwznfrqztv)
+   ==> Installing re2c-1.2.1-e3x6nxtk3ahgd63ykgy44mpuva6jhtdt
+   [ ... ]
+   ==> Optimization: [0, 0, 0, 0, 0, 1, 0, 0, 0]
+   zlib@1.2.11%gcc@10.1.0+optimize+pic+shared arch=linux-ubuntu18.04-broadwell
+
+If you want to speed-up bootstrapping, you may try to search for ``cmake`` and ``bison``
+on your system:
+
+.. code-block:: console
+
+   $ spack external find cmake bison
+   ==> The following specs have been detected on this system and added to /home/spack/.spack/packages.yaml
+   bison@3.0.4  cmake@3.19.4
+
+All the tools Spack needs for its own functioning are installed in a separate store, which lives
+under the ``${HOME}/.spack`` directory. The software installed there can be queried with:
+
+.. code-block:: console
+
+   $ spack find --bootstrap
+   ==> Showing internal bootstrap store at "/home/spack/.spack/bootstrap/store"
+   ==> 3 installed packages
+   -- linux-ubuntu18.04-x86_64 / gcc@10.1.0 ------------------------
+   clingo-bootstrap@spack  python@3.6.9  re2c@1.2.1
+
+In case it's needed the bootstrap store can also be cleaned with:
+
+.. code-block:: console
+
+   $ spack clean -b
+   ==> Removing software in "/home/spack/.spack/bootstrap/store"
+
 ^^^^^^^^^^^^^^^^^^^^^^^^^^
 Optional: Alternate Prefix
 ^^^^^^^^^^^^^^^^^^^^^^^^^^
diff --git a/lib/spack/spack/bootstrap.py b/lib/spack/spack/bootstrap.py
new file mode 100644
index 0000000000..93e79fbe71
--- /dev/null
+++ b/lib/spack/spack/bootstrap.py
@@ -0,0 +1,188 @@
+# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+import contextlib
+import os
+import sys
+
+import llnl.util.filesystem as fs
+import llnl.util.tty as tty
+
+import spack.architecture
+import spack.config
+import spack.paths
+import spack.repo
+import spack.spec
+import spack.store
+import spack.user_environment as uenv
+import spack.util.executable
+from spack.util.environment import EnvironmentModifications
+
+
+@contextlib.contextmanager
+def spack_python_interpreter():
+    """Override the current configuration to set the interpreter under
+    which Spack is currently running as the only Python external spec
+    available.
+    """
+    python_cls = type(spack.spec.Spec('python').package)
+    python_prefix = os.path.dirname(os.path.dirname(sys.executable))
+    externals = python_cls.determine_spec_details(
+        python_prefix, [os.path.basename(sys.executable)])
+    external_python = externals[0]
+
+    entry = {
+        'buildable': False,
+        'externals': [
+            {'prefix': python_prefix, 'spec': str(external_python)}
+        ]
+    }
+
+    with spack.config.override('packages:python::', entry):
+        yield
+
+
+def make_module_available(module, spec=None, install=False):
+    """Ensure module is importable"""
+    # If we already can import it, that's great
+    try:
+        __import__(module)
+        return
+    except ImportError:
+        pass
+
+    # If it's already installed, use it
+    # Search by spec
+    spec = spack.spec.Spec(spec or module)
+
+    # We have to run as part of this python
+    # We can constrain by a shortened version in place of a version range
+    # because this spec is only used for querying or as a placeholder to be
+    # replaced by an external that already has a concrete version. This syntax
+    # is not suffucient when concretizing without an external, as it will
+    # concretize to python@X.Y instead of python@X.Y.Z
+    spec.constrain('^python@%d.%d' % sys.version_info[:2])
+    installed_specs = spack.store.db.query(spec, installed=True)
+
+    for ispec in installed_specs:
+        # TODO: make sure run-environment is appropriate
+        module_path = os.path.join(ispec.prefix,
+                                   ispec['python'].package.site_packages_dir)
+        module_path_64 = module_path.replace('/lib/', '/lib64/')
+        try:
+            sys.path.append(module_path)
+            sys.path.append(module_path_64)
+            __import__(module)
+            return
+        except ImportError:
+            tty.warn("Spec %s did not provide module %s" % (ispec, module))
+            sys.path = sys.path[:-2]
+
+    def _raise_error(module_name, module_spec):
+        error_msg = 'cannot import module "{0}"'.format(module_name)
+        if module_spec:
+            error_msg += ' from spec "{0}'.format(module_spec)
+        raise ImportError(error_msg)
+
+    if not install:
+        _raise_error(module, spec)
+
+    with spack_python_interpreter():
+        # We will install for ourselves, using this python if needed
+        # Concretize the spec
+        spec.concretize()
+    spec.package.do_install()
+
+    module_path = os.path.join(spec.prefix,
+                               spec['python'].package.site_packages_dir)
+    module_path_64 = module_path.replace('/lib/', '/lib64/')
+    try:
+        sys.path.append(module_path)
+        sys.path.append(module_path_64)
+        __import__(module)
+        return
+    except ImportError:
+        sys.path = sys.path[:-2]
+        _raise_error(module, spec)
+
+
+def get_executable(exe, spec=None, install=False):
+    """Find an executable named exe, either in PATH or in Spack
+
+    Args:
+        exe (str): needed executable name
+        spec (Spec or str): spec to search for exe in (default exe)
+        install (bool): install spec if not available
+
+    When ``install`` is True, Spack will use the python used to run Spack as an
+    external. The ``install`` option should only be used with packages that
+    install quickly (when using external python) or are guaranteed by Spack
+    organization to be in a binary mirror (clingo)."""
+    # Search the system first
+    runner = spack.util.executable.which(exe)
+    if runner:
+        return runner
+
+    # Check whether it's already installed
+    spec = spack.spec.Spec(spec or exe)
+    installed_specs = spack.store.db.query(spec, installed=True)
+    for ispec in installed_specs:
+        # filter out directories of the same name as the executable
+        exe_path = [exe_p for exe_p in fs.find(ispec.prefix, exe)
+                    if fs.is_exe(exe_p)]
+        if exe_path:
+            ret = spack.util.executable.Executable(exe_path[0])
+            envmod = EnvironmentModifications()
+            for dep in ispec.traverse(root=True, order='post'):
+                envmod.extend(uenv.environment_modifications_for_spec(dep))
+            ret.add_default_envmod(envmod)
+            return ret
+        else:
+            tty.warn('Exe %s not found in prefix %s' % (exe, ispec.prefix))
+
+    def _raise_error(executable, exe_spec):
+        error_msg = 'cannot find the executable "{0}"'.format(executable)
+        if exe_spec:
+            error_msg += ' from spec "{0}'.format(exe_spec)
+        raise RuntimeError(error_msg)
+
+    # If we're not allowed to install this for ourselves, we can't find it
+    if not install:
+        _raise_error(exe, spec)
+
+    with spack_python_interpreter():
+        # We will install for ourselves, using this python if needed
+        # Concretize the spec
+        spec.concretize()
+
+    spec.package.do_install()
+    # filter out directories of the same name as the executable
+    exe_path = [exe_p for exe_p in fs.find(spec.prefix, exe)
+                if fs.is_exe(exe_p)]
+    if exe_path:
+        ret = spack.util.executable.Executable(exe_path[0])
+        envmod = EnvironmentModifications()
+        for dep in spec.traverse(root=True, order='post'):
+            envmod.extend(uenv.environment_modifications_for_spec(dep))
+        ret.add_default_envmod(envmod)
+        return ret
+
+    _raise_error(exe, spec)
+
+
+@contextlib.contextmanager
+def ensure_bootstrap_configuration():
+    # Default configuration scopes excluding command
+    # line and builtin
+    config_scopes = [
+        spack.config.ConfigScope(name, path)
+        for name, path in spack.config.configuration_paths
+    ]
+
+    with spack.architecture.use_platform(spack.architecture.real_platform()):
+        with spack.config.use_configuration(*config_scopes):
+            with spack.repo.use_repositories(spack.paths.packages_path):
+                with spack.store.use_store(spack.paths.user_bootstrap_store):
+                    with spack_python_interpreter():
+                        yield
diff --git a/lib/spack/spack/build_systems/python.py b/lib/spack/spack/build_systems/python.py
index c3ad02bb2f..7ffceeb0d6 100644
--- a/lib/spack/spack/build_systems/python.py
+++ b/lib/spack/spack/build_systems/python.py
@@ -243,7 +243,28 @@ class PythonPackage(PackageBase):
         if ('py-setuptools' == spec.name or          # this is setuptools, or
             'py-setuptools' in spec._dependencies and  # it's an immediate dep
             'build' in spec._dependencies['py-setuptools'].deptypes):
-            args += ['--single-version-externally-managed', '--root=/']
+            args += ['--single-version-externally-managed']
+
+        # Get all relative paths since we set the root to `prefix`
+        # We query the python with which these will be used for the lib and inc
+        # directories. This ensures we use `lib`/`lib64` as expected by python.
+        python = spec['python'].package.command
+        command_start = 'print(distutils.sysconfig.'
+        commands = ';'.join([
+            'import distutils.sysconfig',
+            command_start + 'get_python_lib(plat_specific=False, prefix=""))',
+            command_start + 'get_python_lib(plat_specific=True, prefix=""))',
+            command_start + 'get_python_inc(plat_specific=True, prefix=""))'])
+        pure_site_packages_dir, plat_site_packages_dir, inc_dir = python(
+            '-c', commands, output=str, error=str).strip().split('\n')
+
+        args += ['--root=%s' % prefix,
+                 '--install-purelib=%s' % pure_site_packages_dir,
+                 '--install-platlib=%s' % plat_site_packages_dir,
+                 '--install-scripts=bin',
+                 '--install-data=""',
+                 '--install-headers=%s' % inc_dir
+                 ]
 
         return args
 
diff --git a/lib/spack/spack/cmd/clean.py b/lib/spack/spack/cmd/clean.py
index 5c7e02c685..f82198abbe 100644
--- a/lib/spack/spack/cmd/clean.py
+++ b/lib/spack/spack/cmd/clean.py
@@ -10,11 +10,12 @@ import shutil
 import llnl.util.tty as tty
 
 import spack.caches
+import spack.config
 import spack.cmd.test
 import spack.cmd.common.arguments as arguments
+import spack.main
 import spack.repo
 import spack.stage
-import spack.config
 from spack.paths import lib_path, var_path
 
 
@@ -26,7 +27,7 @@ level = "long"
 class AllClean(argparse.Action):
     """Activates flags -s -d -f -m and -p simultaneously"""
     def __call__(self, parser, namespace, values, option_string=None):
-        parser.parse_args(['-sdfmp'], namespace=namespace)
+        parser.parse_args(['-sdfmpb'], namespace=namespace)
 
 
 def setup_parser(subparser):
@@ -46,7 +47,10 @@ def setup_parser(subparser):
         '-p', '--python-cache', action='store_true',
         help="remove .pyc, .pyo files and __pycache__ folders")
     subparser.add_argument(
-        '-a', '--all', action=AllClean, help="equivalent to -sdfmp", nargs=0
+        '-b', '--bootstrap', action='store_true',
+        help="remove software needed to bootstrap Spack")
+    subparser.add_argument(
+        '-a', '--all', action=AllClean, help="equivalent to -sdfmpb", nargs=0
     )
     arguments.add_common_arguments(subparser, ['specs'])
 
@@ -54,7 +58,7 @@ def setup_parser(subparser):
 def clean(parser, args):
     # If nothing was set, activate the default
     if not any([args.specs, args.stage, args.downloads, args.failures,
-                args.misc_cache, args.python_cache]):
+                args.misc_cache, args.python_cache, args.bootstrap]):
         args.stage = True
 
     # Then do the cleaning falling through the cases
@@ -96,3 +100,10 @@ def clean(parser, args):
                         dname = os.path.join(root, d)
                         tty.debug('Removing {0}'.format(dname))
                         shutil.rmtree(dname)
+
+    if args.bootstrap:
+        msg = 'Removing software in "{0}"'
+        tty.msg(msg.format(spack.paths.user_bootstrap_store))
+        with spack.store.use_store(spack.paths.user_bootstrap_store):
+            uninstall = spack.main.SpackCommand('uninstall')
+            uninstall('-a', '-y')
diff --git a/lib/spack/spack/cmd/find.py b/lib/spack/spack/cmd/find.py
index 9ed889f928..21d4d01ce8 100644
--- a/lib/spack/spack/cmd/find.py
+++ b/lib/spack/spack/cmd/find.py
@@ -109,6 +109,10 @@ def setup_parser(subparser):
     subparser.add_argument(
         '--end-date', help='latest date of installation [YYYY-MM-DD]'
     )
+    subparser.add_argument(
+        '-b', '--bootstrap', action='store_true',
+        help='show software in the internal bootstrap store'
+    )
 
     arguments.add_common_arguments(subparser, ['constraint'])
 
@@ -201,7 +205,14 @@ def display_env(env, args, decorator):
 
 def find(parser, args):
     q_args = query_arguments(args)
-    results = args.specs(**q_args)
+    # Query the current store or the internal bootstrap store if required
+    if args.bootstrap:
+        msg = 'Showing internal bootstrap store at "{0}"'
+        tty.msg(msg.format(spack.paths.user_bootstrap_store))
+        with spack.store.use_store(spack.paths.user_bootstrap_store):
+            results = args.specs(**q_args)
+    else:
+        results = args.specs(**q_args)
 
     decorator = lambda s, f: f
     added = set()
diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py
index 47574196eb..20c014371a 100644
--- a/lib/spack/spack/paths.py
+++ b/lib/spack/spack/paths.py
@@ -50,7 +50,8 @@ mock_packages_path = os.path.join(repos_path, "builtin.mock")
 
 #: User configuration location
 user_config_path = os.path.expanduser('~/.spack')
-
+user_bootstrap_path = os.path.join(user_config_path, 'bootstrap')
+user_bootstrap_store = os.path.join(user_bootstrap_path, 'store')
 
 opt_path        = os.path.join(prefix, "opt")
 etc_path        = os.path.join(prefix, "etc")
diff --git a/lib/spack/spack/solver/asp.py b/lib/spack/spack/solver/asp.py
index 661a4582ae..975dbfcc91 100644
--- a/lib/spack/spack/solver/asp.py
+++ b/lib/spack/spack/solver/asp.py
@@ -22,6 +22,7 @@ try:
     clingo_cffi = hasattr(clingo.Symbol, '_rep')
 except ImportError:
     clingo = None  # type: ignore
+    clingo_cffi = False
 
 import llnl.util.lang
 import llnl.util.tty as tty
@@ -38,6 +39,7 @@ import spack.spec
 import spack.package
 import spack.package_prefs
 import spack.repo
+import spack.bootstrap
 import spack.variant
 import spack.version
 
@@ -246,7 +248,20 @@ class PyclingoDriver(object):
             asp (file-like): optional stream to write a text-based ASP program
                 for debugging or verification.
         """
-        assert clingo, "PyclingoDriver requires clingo with Python support"
+        global clingo
+        if not clingo:
+            # TODO: Find a way to vendor the concrete spec
+            # in a cross-platform way
+            with spack.bootstrap.ensure_bootstrap_configuration():
+                generic_target = archspec.cpu.host().family
+                spec_str = 'clingo-bootstrap@spack+python target={0}'.format(
+                    str(generic_target)
+                )
+                clingo_spec = spack.spec.Spec(spec_str)
+                clingo_spec._old_concretize()
+                spack.bootstrap.make_module_available(
+                    'clingo', spec=clingo_spec, install=True)
+                import clingo
         self.out = asp or llnl.util.lang.Devnull()
         self.cores = cores
 
diff --git a/lib/spack/spack/store.py b/lib/spack/spack/store.py
index 80a7ae0831..2378ff98d8 100644
--- a/lib/spack/spack/store.py
+++ b/lib/spack/spack/store.py
@@ -196,6 +196,16 @@ def _store():
     config_dict = spack.config.get('config')
     root, unpadded_root, projections = parse_install_tree(config_dict)
     hash_length = spack.config.get('config:install_hash_length')
+
+    # Check that the user is not trying to install software into the store
+    # reserved by Spack to bootstrap its own dependencies, since this would
+    # lead to bizarre behaviors (e.g. cleaning the bootstrap area would wipe
+    # user installed software)
+    if spack.paths.user_bootstrap_store == root:
+        msg = ('please change the install tree root "{0}" in your '
+               'configuration [path reserved for Spack internal use]')
+        raise ValueError(msg.format(root))
+
     return Store(root=root,
                  unpadded_root=unpadded_root,
                  projections=projections,
diff --git a/lib/spack/spack/util/environment.py b/lib/spack/spack/util/environment.py
index a9b3529869..9b5785c9a7 100644
--- a/lib/spack/spack/util/environment.py
+++ b/lib/spack/spack/util/environment.py
@@ -528,13 +528,18 @@ class EnvironmentModifications(object):
 
         return rev
 
-    def apply_modifications(self):
+    def apply_modifications(self, env=None):
         """Applies the modifications and clears the list."""
+        # Use os.environ if not specified
+        # Do not copy, we want to modify it in place
+        if env is None:
+            env = os.environ
+
         modifications = self.group_by_name()
         # Apply modifications one variable at a time
         for name, actions in sorted(modifications.items()):
             for x in actions:
-                x.execute(os.environ)
+                x.execute(env)
 
     def shell_modifications(self, shell='sh'):
         """Return shell code to apply the modifications and clears the list."""
diff --git a/lib/spack/spack/util/executable.py b/lib/spack/spack/util/executable.py
index d536f91612..3822382937 100644
--- a/lib/spack/spack/util/executable.py
+++ b/lib/spack/spack/util/executable.py
@@ -22,6 +22,8 @@ class Executable(object):
     def __init__(self, name):
         self.exe = shlex.split(str(name))
         self.default_env = {}
+        from spack.util.environment import EnvironmentModifications  # no cycle
+        self.default_envmod = EnvironmentModifications()
         self.returncode = None
 
         if not self.exe:
@@ -40,6 +42,10 @@ class Executable(object):
         """
         self.default_env[key] = value
 
+    def add_default_envmod(self, envmod):
+        """Set an EnvironmentModifications to use when the command is run."""
+        self.default_envmod.extend(envmod)
+
     @property
     def command(self):
         """The command-line string.
@@ -76,9 +82,10 @@ class Executable(object):
         Keyword Arguments:
             _dump_env (dict): Dict to be set to the environment actually
                 used (envisaged for testing purposes only)
-            env (dict): The environment to run the executable with
-            extra_env (dict): Extra items to add to the environment
-                (neither requires nor precludes env)
+            env (dict or EnvironmentModifications): The environment with which
+                to run the executable
+            extra_env (dict or EnvironmentModifications): Extra items to add to
+                the environment (neither requires nor precludes env)
             fail_on_error (bool): Raise an exception if the subprocess returns
                 an error. Default is True. The return code is available as
                 ``exe.returncode``
@@ -107,13 +114,26 @@ class Executable(object):
         """
         # Environment
         env_arg = kwargs.get('env', None)
-        if env_arg is None:
-            env = os.environ.copy()
-            env.update(self.default_env)
-        else:
-            env = self.default_env.copy()
+
+        # Setup default environment
+        env = os.environ.copy() if env_arg is None else {}
+        self.default_envmod.apply_modifications(env)
+        env.update(self.default_env)
+
+        from spack.util.environment import EnvironmentModifications  # no cycle
+        # Apply env argument
+        if isinstance(env_arg, EnvironmentModifications):
+            env_arg.apply_modifications(env)
+        elif env_arg:
             env.update(env_arg)
-        env.update(kwargs.get('extra_env', {}))
+
+        # Apply extra env
+        extra_env = kwargs.get('extra_env', {})
+        if isinstance(extra_env, EnvironmentModifications):
+            extra_env.apply_modifications(env)
+        else:
+            env.update(extra_env)
+
         if '_dump_env' in kwargs:
             kwargs['_dump_env'].clear()
             kwargs['_dump_env'].update(env)