Bootstrap clingo from sources (#21446)

* Allow the bootstrapping of clingo from sources

Allow python builds with system python as external
for MacOS

* Ensure consistent configuration when bootstrapping clingo

This commit uses context managers to ensure we can
bootstrap clingo using a consistent configuration
regardless of the use case being managed.

* Github actions: test clingo with bootstrapping from sources

* Add command to inspect and clean the bootstrap store

 Prevent users to set the install tree root to the bootstrap store

* clingo: documented how to bootstrap from sources

Co-authored-by: Gregory Becker <becker33@llnl.gov>
(cherry picked from commit 10e9e142b75c6ca8bc61f688260c002201cc1b22)

12 files changed, 363 insertions, 46 deletions

diff --git a/.github/workflows/linux_unit_tests.yaml b/.github/workflows/linux_unit_tests.yaml
index 5df0957659..f21f9cd0ab 100644
--- a/.github/workflows/linux_unit_tests.yaml
+++ b/.github/workflows/linux_unit_tests.yaml
@@ -15,6 +15,7 @@ jobs:
     strategy:
       matrix:
         python-version: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9]
+        concretizer: ['original', 'clingo']
 
     steps:
     - uses: actions/checkout@v2
@@ -50,16 +51,23 @@ jobs:
           mkdir -p ${KCOV_ROOT}/build
           cd ${KCOV_ROOT}/build && cmake -Wno-dev ${KCOV_ROOT}/kcov-${KCOV_VERSION} && cd -
           make -C ${KCOV_ROOT}/build && sudo  make -C ${KCOV_ROOT}/build install
+    - name: Bootstrap clingo from sources
+      if: ${{ matrix.concretizer == 'clingo' }}
+      run: |
+          . share/spack/setup-env.sh
+          spack external find --not-buildable cmake bison
+          spack -v solve zlib
     - name: Run unit tests
       env:
           COVERAGE: true
+          SPACK_TEST_SOLVER: ${{ matrix.concretizer }}
       run: |
           share/spack/qa/run-unit-tests
           coverage combine
           coverage xml
     - uses: codecov/codecov-action@v1
       with:
-        flags: unittests,linux
+        flags: unittests,linux,${{ matrix.concretizer }}
   shell:
     runs-on: ubuntu-latest
     steps:
@@ -103,6 +111,7 @@ jobs:
     - uses: codecov/codecov-action@v1
       with:
         flags: shelltests,linux
+
   centos6:
     # Test for Python2.6 run on Centos 6
     runs-on: ubuntu-latest
@@ -117,27 +126,7 @@ jobs:
           git fetch origin ${{ github.ref }}:test-branch
           git checkout test-branch
           share/spack/qa/run-unit-tests
-  clingo:
-    # Test for the clingo based solver
-    runs-on: ubuntu-latest
-    container: spack/github-actions:clingo
-    steps:
-      - name: Run unit tests
-        run: |
-          whoami && echo PWD=$PWD && echo HOME=$HOME && echo SPACK_TEST_SOLVER=$SPACK_TEST_SOLVER
-          which clingo && clingo --version
-          git clone https://github.com/spack/spack.git && cd spack
-          git fetch origin ${{ github.ref }}:test-branch
-          git checkout test-branch
-          . share/spack/setup-env.sh
-          spack compiler find
-          spack solve mpileaks%gcc
-          coverage run $(which spack) unit-test -v
-          coverage combine
-          coverage xml
-      - uses: codecov/codecov-action@v1
-        with:
-          flags: unittests,linux,clingo
+
   clingo-cffi:
     # Test for the clingo based solver (using clingo-cffi)
     runs-on: ubuntu-latest
diff --git a/lib/spack/docs/getting_started.rst b/lib/spack/docs/getting_started.rst
index f1df0343bd..09cc2c488a 100644
--- a/lib/spack/docs/getting_started.rst
+++ b/lib/spack/docs/getting_started.rst
@@ -103,6 +103,53 @@ environment*, especially for ``PATH``.  Only software that comes with
 the system, or that you know you wish to use with Spack, should be
 included.  This procedure will avoid many strange build errors.
 
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Optional: Bootstrapping clingo
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Spack supports using clingo as an external solver to compute which software
+needs to be installed. If you have a default compiler supporting C++14 Spack
+can automatically bootstrap this tool from sources the first time it is
+needed:
+
+.. code-block:: console
+
+   $ spack solve zlib
+   [+] /usr (external bison-3.0.4-wu5pgjchxzemk5ya2l3ddqug2d7jv6eb)
+   [+] /usr (external cmake-3.19.4-a4kmcfzxxy45mzku4ipmj5kdiiz5a57b)
+   [+] /usr (external python-3.6.9-x4fou4iqqlh5ydwddx3pvfcwznfrqztv)
+   ==> Installing re2c-1.2.1-e3x6nxtk3ahgd63ykgy44mpuva6jhtdt
+   [ ... ]
+   ==> Optimization: [0, 0, 0, 0, 0, 1, 0, 0, 0]
+   zlib@1.2.11%gcc@10.1.0+optimize+pic+shared arch=linux-ubuntu18.04-broadwell
+
+If you want to speed-up bootstrapping, you may try to search for ``cmake`` and ``bison``
+on your system:
+
+.. code-block:: console
+
+   $ spack external find cmake bison
+   ==> The following specs have been detected on this system and added to /home/spack/.spack/packages.yaml
+   bison@3.0.4  cmake@3.19.4
+
+All the tools Spack needs for its own functioning are installed in a separate store, which lives
+under the ``${HOME}/.spack`` directory. The software installed there can be queried with:
+
+.. code-block:: console
+
+   $ spack find --bootstrap
+   ==> Showing internal bootstrap store at "/home/spack/.spack/bootstrap/store"
+   ==> 3 installed packages
+   -- linux-ubuntu18.04-x86_64 / gcc@10.1.0 ------------------------
+   clingo-bootstrap@spack  python@3.6.9  re2c@1.2.1
+
+In case it's needed the bootstrap store can also be cleaned with:
+
+.. code-block:: console
+
+   $ spack clean -b
+   ==> Removing software in "/home/spack/.spack/bootstrap/store"
+
 ^^^^^^^^^^^^^^^^^^^^^^^^^^
 Optional: Alternate Prefix
 ^^^^^^^^^^^^^^^^^^^^^^^^^^
diff --git a/lib/spack/spack/bootstrap.py b/lib/spack/spack/bootstrap.py
new file mode 100644
index 0000000000..93e79fbe71
--- /dev/null
+++ b/lib/spack/spack/bootstrap.py
@@ -0,0 +1,188 @@
+# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+import contextlib
+import os
+import sys
+
+import llnl.util.filesystem as fs
+import llnl.util.tty as tty
+
+import spack.architecture
+import spack.config
+import spack.paths
+import spack.repo
+import spack.spec
+import spack.store
+import spack.user_environment as uenv
+import spack.util.executable
+from spack.util.environment import EnvironmentModifications
+
+
+@contextlib.contextmanager
+def spack_python_interpreter():
+    """Override the current configuration to set the interpreter under
+    which Spack is currently running as the only Python external spec
+    available.
+    """
+    python_cls = type(spack.spec.Spec('python').package)
+    python_prefix = os.path.dirname(os.path.dirname(sys.executable))
+    externals = python_cls.determine_spec_details(
+        python_prefix, [os.path.basename(sys.executable)])
+    external_python = externals[0]
+
+    entry = {
+        'buildable': False,
+        'externals': [
+            {'prefix': python_prefix, 'spec': str(external_python)}
+        ]
+    }
+
+    with spack.config.override('packages:python::', entry):
+        yield
+
+
+def make_module_available(module, spec=None, install=False):
+    """Ensure module is importable"""
+    # If we already can import it, that's great
+    try:
+        __import__(module)
+        return
+    except ImportError:
+        pass
+
+    # If it's already installed, use it
+    # Search by spec
+    spec = spack.spec.Spec(spec or module)
+
+    # We have to run as part of this python
+    # We can constrain by a shortened version in place of a version range
+    # because this spec is only used for querying or as a placeholder to be
+    # replaced by an external that already has a concrete version. This syntax
+    # is not suffucient when concretizing without an external, as it will
+    # concretize to python@X.Y instead of python@X.Y.Z
+    spec.constrain('^python@%d.%d' % sys.version_info[:2])
+    installed_specs = spack.store.db.query(spec, installed=True)
+
+    for ispec in installed_specs:
+        # TODO: make sure run-environment is appropriate
+        module_path = os.path.join(ispec.prefix,
+                                   ispec['python'].package.site_packages_dir)
+        module_path_64 = module_path.replace('/lib/', '/lib64/')
+        try:
+            sys.path.append(module_path)
+            sys.path.append(module_path_64)
+            __import__(module)
+            return
+        except ImportError:
+            tty.warn("Spec %s did not provide module %s" % (ispec, module))
+            sys.path = sys.path[:-2]
+
+    def _raise_error(module_name, module_spec):
+        error_msg = 'cannot import module "{0}"'.format(module_name)
+        if module_spec:
+            error_msg += ' from spec "{0}'.format(module_spec)
+        raise ImportError(error_msg)
+
+    if not install:
+        _raise_error(module, spec)
+
+    with spack_python_interpreter():
+        # We will install for ourselves, using this python if needed
+        # Concretize the spec
+        spec.concretize()
+    spec.package.do_install()
+
+    module_path = os.path.join(spec.prefix,
+                               spec['python'].package.site_packages_dir)
+    module_path_64 = module_path.replace('/lib/', '/lib64/')
+    try:
+        sys.path.append(module_path)
+        sys.path.append(module_path_64)
+        __import__(module)
+        return
+    except ImportError:
+        sys.path = sys.path[:-2]
+        _raise_error(module, spec)
+
+
+def get_executable(exe, spec=None, install=False):
+    """Find an executable named exe, either in PATH or in Spack
+
+    Args:
+        exe (str): needed executable name
+        spec (Spec or str): spec to search for exe in (default exe)
+        install (bool): install spec if not available
+
+    When ``install`` is True, Spack will use the python used to run Spack as an
+    external. The ``install`` option should only be used with packages that
+    install quickly (when using external python) or are guaranteed by Spack
+    organization to be in a binary mirror (clingo)."""
+    # Search the system first
+    runner = spack.util.executable.which(exe)
+    if runner:
+        return runner
+
+    # Check whether it's already installed
+    spec = spack.spec.Spec(spec or exe)
+    installed_specs = spack.store.db.query(spec, installed=True)
+    for ispec in installed_specs:
+        # filter out directories of the same name as the executable
+        exe_path = [exe_p for exe_p in fs.find(ispec.prefix, exe)
+                    if fs.is_exe(exe_p)]
+        if exe_path:
+            ret = spack.util.executable.Executable(exe_path[0])
+            envmod = EnvironmentModifications()
+            for dep in ispec.traverse(root=True, order='post'):
+                envmod.extend(uenv.environment_modifications_for_spec(dep))
+            ret.add_default_envmod(envmod)
+            return ret
+        else:
+            tty.warn('Exe %s not found in prefix %s' % (exe, ispec.prefix))
+
+    def _raise_error(executable, exe_spec):
+        error_msg = 'cannot find the executable "{0}"'.format(executable)
+        if exe_spec:
+            error_msg += ' from spec "{0}'.format(exe_spec)
+        raise RuntimeError(error_msg)
+
+    # If we're not allowed to install this for ourselves, we can't find it
+    if not install:
+        _raise_error(exe, spec)
+
+    with spack_python_interpreter():
+        # We will install for ourselves, using this python if needed
+        # Concretize the spec
+        spec.concretize()
+
+    spec.package.do_install()
+    # filter out directories of the same name as the executable
+    exe_path = [exe_p for exe_p in fs.find(spec.prefix, exe)
+                if fs.is_exe(exe_p)]
+    if exe_path:
+        ret = spack.util.executable.Executable(exe_path[0])
+        envmod = EnvironmentModifications()
+        for dep in spec.traverse(root=True, order='post'):
+            envmod.extend(uenv.environment_modifications_for_spec(dep))
+        ret.add_default_envmod(envmod)
+        return ret
+
+    _raise_error(exe, spec)
+
+
+@contextlib.contextmanager
+def ensure_bootstrap_configuration():
+    # Default configuration scopes excluding command
+    # line and builtin
+    config_scopes = [
+        spack.config.ConfigScope(name, path)
+        for name, path in spack.config.configuration_paths
+    ]
+
+    with spack.architecture.use_platform(spack.architecture.real_platform()):
+        with spack.config.use_configuration(*config_scopes):
+            with spack.repo.use_repositories(spack.paths.packages_path):
+                with spack.store.use_store(spack.paths.user_bootstrap_store):
+                    with spack_python_interpreter():
+                        yield
diff --git a/lib/spack/spack/build_systems/python.py b/lib/spack/spack/build_systems/python.py
index 76159d88a1..2616642732 100644
--- a/lib/spack/spack/build_systems/python.py
+++ b/lib/spack/spack/build_systems/python.py
@@ -233,7 +233,28 @@ class PythonPackage(PackageBase):
         if ('py-setuptools' == spec.name or          # this is setuptools, or
             'py-setuptools' in spec._dependencies and  # it's an immediate dep
             'build' in spec._dependencies['py-setuptools'].deptypes):
-            args += ['--single-version-externally-managed', '--root=/']
+            args += ['--single-version-externally-managed']
+
+        # Get all relative paths since we set the root to `prefix`
+        # We query the python with which these will be used for the lib and inc
+        # directories. This ensures we use `lib`/`lib64` as expected by python.
+        python = spec['python'].package.command
+        command_start = 'print(distutils.sysconfig.'
+        commands = ';'.join([
+            'import distutils.sysconfig',
+            command_start + 'get_python_lib(plat_specific=False, prefix=""))',
+            command_start + 'get_python_lib(plat_specific=True, prefix=""))',
+            command_start + 'get_python_inc(plat_specific=True, prefix=""))'])
+        pure_site_packages_dir, plat_site_packages_dir, inc_dir = python(
+            '-c', commands, output=str, error=str).strip().split('\n')
+
+        args += ['--root=%s' % prefix,
+                 '--install-purelib=%s' % pure_site_packages_dir,
+                 '--install-platlib=%s' % plat_site_packages_dir,
+                 '--install-scripts=bin',
+                 '--install-data=""',
+                 '--install-headers=%s' % inc_dir
+                 ]
 
         return args
 
diff --git a/lib/spack/spack/cmd/clean.py b/lib/spack/spack/cmd/clean.py
index f69b959293..2e1ec1255f 100644
--- a/lib/spack/spack/cmd/clean.py
+++ b/lib/spack/spack/cmd/clean.py
@@ -10,11 +10,12 @@ import shutil
 import llnl.util.tty as tty
 
 import spack.caches
+import spack.config
 import spack.cmd.test
 import spack.cmd.common.arguments as arguments
+import spack.main
 import spack.repo
 import spack.stage
-import spack.config
 from spack.paths import lib_path, var_path
 
 
@@ -26,7 +27,7 @@ level = "long"
 class AllClean(argparse.Action):
     """Activates flags -s -d -f -m and -p simultaneously"""
     def __call__(self, parser, namespace, values, option_string=None):
-        parser.parse_args(['-sdfmp'], namespace=namespace)
+        parser.parse_args(['-sdfmpb'], namespace=namespace)
 
 
 def setup_parser(subparser):
@@ -46,7 +47,10 @@ def setup_parser(subparser):
         '-p', '--python-cache', action='store_true',
         help="remove .pyc, .pyo files and __pycache__ folders")
     subparser.add_argument(
-        '-a', '--all', action=AllClean, help="equivalent to -sdfmp", nargs=0
+        '-b', '--bootstrap', action='store_true',
+        help="remove software needed to bootstrap Spack")
+    subparser.add_argument(
+        '-a', '--all', action=AllClean, help="equivalent to -sdfmpb", nargs=0
     )
     arguments.add_common_arguments(subparser, ['specs'])
 
@@ -54,7 +58,7 @@ def setup_parser(subparser):
 def clean(parser, args):
     # If nothing was set, activate the default
     if not any([args.specs, args.stage, args.downloads, args.failures,
-                args.misc_cache, args.python_cache]):
+                args.misc_cache, args.python_cache, args.bootstrap]):
         args.stage = True
 
     # Then do the cleaning falling through the cases
@@ -96,3 +100,10 @@ def clean(parser, args):
                         dname = os.path.join(root, d)
                         tty.debug('Removing {0}'.format(dname))
                         shutil.rmtree(dname)
+
+    if args.bootstrap:
+        msg = 'Removing software in "{0}"'
+        tty.msg(msg.format(spack.paths.user_bootstrap_store))
+        with spack.store.use_store(spack.paths.user_bootstrap_store):
+            uninstall = spack.main.SpackCommand('uninstall')
+            uninstall('-a', '-y')
diff --git a/lib/spack/spack/cmd/find.py b/lib/spack/spack/cmd/find.py
index d9d51b853c..f770a02b25 100644
--- a/lib/spack/spack/cmd/find.py
+++ b/lib/spack/spack/cmd/find.py
@@ -109,6 +109,10 @@ def setup_parser(subparser):
     subparser.add_argument(
         '--end-date', help='latest date of installation [YYYY-MM-DD]'
     )
+    subparser.add_argument(
+        '-b', '--bootstrap', action='store_true',
+        help='show software in the internal bootstrap store'
+    )
 
     arguments.add_common_arguments(subparser, ['constraint'])
 
@@ -201,7 +205,14 @@ def display_env(env, args, decorator):
 
 def find(parser, args):
     q_args = query_arguments(args)
-    results = args.specs(**q_args)
+    # Query the current store or the internal bootstrap store if required
+    if args.bootstrap:
+        msg = 'Showing internal bootstrap store at "{0}"'
+        tty.msg(msg.format(spack.paths.user_bootstrap_store))
+        with spack.store.use_store(spack.paths.user_bootstrap_store):
+            results = args.specs(**q_args)
+    else:
+        results = args.specs(**q_args)
 
     decorator = lambda s, f: f
     added = set()
diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py
index 9c803cba7e..068a7a6fdb 100644
--- a/lib/spack/spack/paths.py
+++ b/lib/spack/spack/paths.py
@@ -50,7 +50,8 @@ mock_packages_path = os.path.join(repos_path, "builtin.mock")
 
 #: User configuration location
 user_config_path = os.path.expanduser('~/.spack')
-
+user_bootstrap_path = os.path.join(user_config_path, 'bootstrap')
+user_bootstrap_store = os.path.join(user_bootstrap_path, 'store')
 
 opt_path        = os.path.join(prefix, "opt")
 etc_path        = os.path.join(prefix, "etc")
diff --git a/lib/spack/spack/solver/asp.py b/lib/spack/spack/solver/asp.py
index ac2983cecc..8d2b2b8aa5 100644
--- a/lib/spack/spack/solver/asp.py
+++ b/lib/spack/spack/solver/asp.py
@@ -1,4 +1,4 @@
-# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
@@ -22,6 +22,7 @@ try:
     clingo_cffi = hasattr(clingo.Symbol, '_rep')
 except ImportError:
     clingo = None
+    clingo_cffi = False
 
 import llnl.util.lang
 import llnl.util.tty as tty
@@ -38,6 +39,7 @@ import spack.spec
 import spack.package
 import spack.package_prefs
 import spack.repo
+import spack.bootstrap
 import spack.variant
 import spack.version
 
@@ -246,7 +248,20 @@ class PyclingoDriver(object):
             asp (file-like): optional stream to write a text-based ASP program
                 for debugging or verification.
         """
-        assert clingo, "PyclingoDriver requires clingo with Python support"
+        global clingo
+        if not clingo:
+            # TODO: Find a way to vendor the concrete spec
+            # in a cross-platform way
+            with spack.bootstrap.ensure_bootstrap_configuration():
+                generic_target = archspec.cpu.host().family
+                spec_str = 'clingo-bootstrap@spack+python target={0}'.format(
+                    str(generic_target)
+                )
+                clingo_spec = spack.spec.Spec(spec_str)
+                clingo_spec._old_concretize()
+                spack.bootstrap.make_module_available(
+                    'clingo', spec=clingo_spec, install=True)
+                import clingo
         self.out = asp or llnl.util.lang.Devnull()
         self.cores = cores
 
@@ -1112,7 +1127,6 @@ class SpackSolverSetup(object):
             if not spec.architecture or not spec.architecture.target:
                 continue
 
-            print("TTYPE:", type(platform.target(spec.target.name)))
             target = archspec.cpu.TARGETS.get(spec.target.name)
             if not target:
                 self.target_ranges(spec, None)
diff --git a/lib/spack/spack/store.py b/lib/spack/spack/store.py
index 6040145b64..7b77768a37 100644
--- a/lib/spack/spack/store.py
+++ b/lib/spack/spack/store.py
@@ -173,6 +173,16 @@ def _store():
     config_dict = spack.config.get('config')
     root, unpadded_root, projections = parse_install_tree(config_dict)
     hash_length = spack.config.get('config:install_hash_length')
+
+    # Check that the user is not trying to install software into the store
+    # reserved by Spack to bootstrap its own dependencies, since this would
+    # lead to bizarre behaviors (e.g. cleaning the bootstrap area would wipe
+    # user installed software)
+    if spack.paths.user_bootstrap_store == root:
+        msg = ('please change the install tree root "{0}" in your '
+               'configuration [path reserved for Spack internal use]')
+        raise ValueError(msg.format(root))
+
     return Store(root=root,
                  unpadded_root=unpadded_root,
                  projections=projections,
diff --git a/lib/spack/spack/util/environment.py b/lib/spack/spack/util/environment.py
index b370053c3c..53e0da32d7 100644
--- a/lib/spack/spack/util/environment.py
+++ b/lib/spack/spack/util/environment.py
@@ -528,13 +528,18 @@ class EnvironmentModifications(object):
 
         return rev
 
-    def apply_modifications(self):
+    def apply_modifications(self, env=None):
         """Applies the modifications and clears the list."""
+        # Use os.environ if not specified
+        # Do not copy, we want to modify it in place
+        if env is None:
+            env = os.environ
+
         modifications = self.group_by_name()
         # Apply modifications one variable at a time
         for name, actions in sorted(modifications.items()):
             for x in actions:
-                x.execute(os.environ)
+                x.execute(env)
 
     def shell_modifications(self, shell='sh'):
         """Return shell code to apply the modifications and clears the list."""
diff --git a/lib/spack/spack/util/executable.py b/lib/spack/spack/util/executable.py
index 614bc1725a..fb192d6885 100644
--- a/lib/spack/spack/util/executable.py
+++ b/lib/spack/spack/util/executable.py
@@ -22,6 +22,8 @@ class Executable(object):
     def __init__(self, name):
         self.exe = shlex.split(str(name))
         self.default_env = {}
+        from spack.util.environment import EnvironmentModifications  # no cycle
+        self.default_envmod = EnvironmentModifications()
         self.returncode = None
 
         if not self.exe:
@@ -40,6 +42,10 @@ class Executable(object):
         """
         self.default_env[key] = value
 
+    def add_default_envmod(self, envmod):
+        """Set an EnvironmentModifications to use when the command is run."""
+        self.default_envmod.extend(envmod)
+
     @property
     def command(self):
         """The command-line string.
@@ -76,9 +82,10 @@ class Executable(object):
         Keyword Arguments:
             _dump_env (dict): Dict to be set to the environment actually
                 used (envisaged for testing purposes only)
-            env (dict): The environment to run the executable with
-            extra_env (dict): Extra items to add to the environment
-                (neither requires nor precludes env)
+            env (dict or EnvironmentModifications): The environment with which
+                to run the executable
+            extra_env (dict or EnvironmentModifications): Extra items to add to
+                the environment (neither requires nor precludes env)
             fail_on_error (bool): Raise an exception if the subprocess returns
                 an error. Default is True. The return code is available as
                 ``exe.returncode``
@@ -107,13 +114,26 @@ class Executable(object):
         """
         # Environment
         env_arg = kwargs.get('env', None)
-        if env_arg is None:
-            env = os.environ.copy()
-            env.update(self.default_env)
-        else:
-            env = self.default_env.copy()
+
+        # Setup default environment
+        env = os.environ.copy() if env_arg is None else {}
+        self.default_envmod.apply_modifications(env)
+        env.update(self.default_env)
+
+        from spack.util.environment import EnvironmentModifications  # no cycle
+        # Apply env argument
+        if isinstance(env_arg, EnvironmentModifications):
+            env_arg.apply_modifications(env)
+        elif env_arg:
             env.update(env_arg)
-        env.update(kwargs.get('extra_env', {}))
+
+        # Apply extra env
+        extra_env = kwargs.get('extra_env', {})
+        if isinstance(extra_env, EnvironmentModifications):
+            extra_env.apply_modifications(env)
+        else:
+            env.update(extra_env)
+
         if '_dump_env' in kwargs:
             kwargs['_dump_env'].clear()
             kwargs['_dump_env'].update(env)
diff --git a/share/spack/spack-completion.bash b/share/spack/spack-completion.bash
index bf44039ad9..46e92a0bd5 100755
--- a/share/spack/spack-completion.bash
+++ b/share/spack/spack-completion.bash
@@ -1,4 +1,4 @@
-# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
@@ -475,7 +475,7 @@ _spack_ci_rebuild() {
 _spack_clean() {
     if $list_options
     then
-        SPACK_COMPREPLY="-h --help -s --stage -d --downloads -f --failures -m --misc-cache -p --python-cache -a --all"
+        SPACK_COMPREPLY="-h --help -s --stage -d --downloads -f --failures -m --misc-cache -p --python-cache -b --bootstrap -a --all"
     else
         _all_packages
     fi
@@ -891,7 +891,7 @@ _spack_fetch() {
 _spack_find() {
     if $list_options
     then
-        SPACK_COMPREPLY="-h --help --format --json -d --deps -p --paths --groups --no-groups -l --long -L --very-long -t --tags -c --show-concretized -f --show-flags --show-full-compiler -x --explicit -X --implicit -u --unknown -m --missing -v --variants --loaded -M --only-missing --deprecated --only-deprecated -N --namespace --start-date --end-date"
+        SPACK_COMPREPLY="-h --help --format --json -d --deps -p --paths --groups --no-groups -l --long -L --very-long -t --tags -c --show-concretized -f --show-flags --show-full-compiler -x --explicit -X --implicit -u --unknown -m --missing -v --variants --loaded -M --only-missing --deprecated --only-deprecated -N --namespace --start-date --end-date -b --bootstrap"
     else
         _installed_packages
     fi