summaryrefslogtreecommitdiff
path: root/lib
diff options
context:
space:
mode:
authorMassimiliano Culpo <massimiliano.culpo@gmail.com>2021-03-03 18:37:46 +0100
committerTodd Gamblin <tgamblin@llnl.gov>2021-05-22 11:50:59 -0700
commit16f7a02654095801dfd67cd647136c6466a9f73c (patch)
treebc2be1067691edc30d1e6dc67df5436a381a76dc /lib
parent6c1b348d912175ce49141386cee8d79d8035269b (diff)
downloadspack-16f7a02654095801dfd67cd647136c6466a9f73c.tar.gz
spack-16f7a02654095801dfd67cd647136c6466a9f73c.tar.bz2
spack-16f7a02654095801dfd67cd647136c6466a9f73c.tar.xz
spack-16f7a02654095801dfd67cd647136c6466a9f73c.zip
Bootstrap clingo from sources (#21446)
* Allow the bootstrapping of clingo from sources Allow python builds with system python as external for MacOS * Ensure consistent configuration when bootstrapping clingo This commit uses context managers to ensure we can bootstrap clingo using a consistent configuration regardless of the use case being managed. * Github actions: test clingo with bootstrapping from sources * Add command to inspect and clean the bootstrap store Prevent users to set the install tree root to the bootstrap store * clingo: documented how to bootstrap from sources Co-authored-by: Gregory Becker <becker33@llnl.gov> (cherry picked from commit 10e9e142b75c6ca8bc61f688260c002201cc1b22)
Diffstat (limited to 'lib')
-rw-r--r--lib/spack/docs/getting_started.rst47
-rw-r--r--lib/spack/spack/bootstrap.py188
-rw-r--r--lib/spack/spack/build_systems/python.py23
-rw-r--r--lib/spack/spack/cmd/clean.py19
-rw-r--r--lib/spack/spack/cmd/find.py13
-rw-r--r--lib/spack/spack/paths.py3
-rw-r--r--lib/spack/spack/solver/asp.py20
-rw-r--r--lib/spack/spack/store.py10
-rw-r--r--lib/spack/spack/util/environment.py9
-rw-r--r--lib/spack/spack/util/executable.py38
10 files changed, 349 insertions, 21 deletions
diff --git a/lib/spack/docs/getting_started.rst b/lib/spack/docs/getting_started.rst
index f1df0343bd..09cc2c488a 100644
--- a/lib/spack/docs/getting_started.rst
+++ b/lib/spack/docs/getting_started.rst
@@ -103,6 +103,53 @@ environment*, especially for ``PATH``. Only software that comes with
the system, or that you know you wish to use with Spack, should be
included. This procedure will avoid many strange build errors.
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Optional: Bootstrapping clingo
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Spack supports using clingo as an external solver to compute which software
+needs to be installed. If you have a default compiler supporting C++14 Spack
+can automatically bootstrap this tool from sources the first time it is
+needed:
+
+.. code-block:: console
+
+ $ spack solve zlib
+ [+] /usr (external bison-3.0.4-wu5pgjchxzemk5ya2l3ddqug2d7jv6eb)
+ [+] /usr (external cmake-3.19.4-a4kmcfzxxy45mzku4ipmj5kdiiz5a57b)
+ [+] /usr (external python-3.6.9-x4fou4iqqlh5ydwddx3pvfcwznfrqztv)
+ ==> Installing re2c-1.2.1-e3x6nxtk3ahgd63ykgy44mpuva6jhtdt
+ [ ... ]
+ ==> Optimization: [0, 0, 0, 0, 0, 1, 0, 0, 0]
+ zlib@1.2.11%gcc@10.1.0+optimize+pic+shared arch=linux-ubuntu18.04-broadwell
+
+If you want to speed-up bootstrapping, you may try to search for ``cmake`` and ``bison``
+on your system:
+
+.. code-block:: console
+
+ $ spack external find cmake bison
+ ==> The following specs have been detected on this system and added to /home/spack/.spack/packages.yaml
+ bison@3.0.4 cmake@3.19.4
+
+All the tools Spack needs for its own functioning are installed in a separate store, which lives
+under the ``${HOME}/.spack`` directory. The software installed there can be queried with:
+
+.. code-block:: console
+
+ $ spack find --bootstrap
+ ==> Showing internal bootstrap store at "/home/spack/.spack/bootstrap/store"
+ ==> 3 installed packages
+ -- linux-ubuntu18.04-x86_64 / gcc@10.1.0 ------------------------
+ clingo-bootstrap@spack python@3.6.9 re2c@1.2.1
+
+In case it's needed the bootstrap store can also be cleaned with:
+
+.. code-block:: console
+
+ $ spack clean -b
+ ==> Removing software in "/home/spack/.spack/bootstrap/store"
+
^^^^^^^^^^^^^^^^^^^^^^^^^^
Optional: Alternate Prefix
^^^^^^^^^^^^^^^^^^^^^^^^^^
diff --git a/lib/spack/spack/bootstrap.py b/lib/spack/spack/bootstrap.py
new file mode 100644
index 0000000000..93e79fbe71
--- /dev/null
+++ b/lib/spack/spack/bootstrap.py
@@ -0,0 +1,188 @@
+# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+import contextlib
+import os
+import sys
+
+import llnl.util.filesystem as fs
+import llnl.util.tty as tty
+
+import spack.architecture
+import spack.config
+import spack.paths
+import spack.repo
+import spack.spec
+import spack.store
+import spack.user_environment as uenv
+import spack.util.executable
+from spack.util.environment import EnvironmentModifications
+
+
+@contextlib.contextmanager
+def spack_python_interpreter():
+ """Override the current configuration to set the interpreter under
+ which Spack is currently running as the only Python external spec
+ available.
+ """
+ python_cls = type(spack.spec.Spec('python').package)
+ python_prefix = os.path.dirname(os.path.dirname(sys.executable))
+ externals = python_cls.determine_spec_details(
+ python_prefix, [os.path.basename(sys.executable)])
+ external_python = externals[0]
+
+ entry = {
+ 'buildable': False,
+ 'externals': [
+ {'prefix': python_prefix, 'spec': str(external_python)}
+ ]
+ }
+
+ with spack.config.override('packages:python::', entry):
+ yield
+
+
+def make_module_available(module, spec=None, install=False):
+ """Ensure module is importable"""
+ # If we already can import it, that's great
+ try:
+ __import__(module)
+ return
+ except ImportError:
+ pass
+
+ # If it's already installed, use it
+ # Search by spec
+ spec = spack.spec.Spec(spec or module)
+
+ # We have to run as part of this python
+ # We can constrain by a shortened version in place of a version range
+ # because this spec is only used for querying or as a placeholder to be
+ # replaced by an external that already has a concrete version. This syntax
+ # is not suffucient when concretizing without an external, as it will
+ # concretize to python@X.Y instead of python@X.Y.Z
+ spec.constrain('^python@%d.%d' % sys.version_info[:2])
+ installed_specs = spack.store.db.query(spec, installed=True)
+
+ for ispec in installed_specs:
+ # TODO: make sure run-environment is appropriate
+ module_path = os.path.join(ispec.prefix,
+ ispec['python'].package.site_packages_dir)
+ module_path_64 = module_path.replace('/lib/', '/lib64/')
+ try:
+ sys.path.append(module_path)
+ sys.path.append(module_path_64)
+ __import__(module)
+ return
+ except ImportError:
+ tty.warn("Spec %s did not provide module %s" % (ispec, module))
+ sys.path = sys.path[:-2]
+
+ def _raise_error(module_name, module_spec):
+ error_msg = 'cannot import module "{0}"'.format(module_name)
+ if module_spec:
+ error_msg += ' from spec "{0}'.format(module_spec)
+ raise ImportError(error_msg)
+
+ if not install:
+ _raise_error(module, spec)
+
+ with spack_python_interpreter():
+ # We will install for ourselves, using this python if needed
+ # Concretize the spec
+ spec.concretize()
+ spec.package.do_install()
+
+ module_path = os.path.join(spec.prefix,
+ spec['python'].package.site_packages_dir)
+ module_path_64 = module_path.replace('/lib/', '/lib64/')
+ try:
+ sys.path.append(module_path)
+ sys.path.append(module_path_64)
+ __import__(module)
+ return
+ except ImportError:
+ sys.path = sys.path[:-2]
+ _raise_error(module, spec)
+
+
+def get_executable(exe, spec=None, install=False):
+ """Find an executable named exe, either in PATH or in Spack
+
+ Args:
+ exe (str): needed executable name
+ spec (Spec or str): spec to search for exe in (default exe)
+ install (bool): install spec if not available
+
+ When ``install`` is True, Spack will use the python used to run Spack as an
+ external. The ``install`` option should only be used with packages that
+ install quickly (when using external python) or are guaranteed by Spack
+ organization to be in a binary mirror (clingo)."""
+ # Search the system first
+ runner = spack.util.executable.which(exe)
+ if runner:
+ return runner
+
+ # Check whether it's already installed
+ spec = spack.spec.Spec(spec or exe)
+ installed_specs = spack.store.db.query(spec, installed=True)
+ for ispec in installed_specs:
+ # filter out directories of the same name as the executable
+ exe_path = [exe_p for exe_p in fs.find(ispec.prefix, exe)
+ if fs.is_exe(exe_p)]
+ if exe_path:
+ ret = spack.util.executable.Executable(exe_path[0])
+ envmod = EnvironmentModifications()
+ for dep in ispec.traverse(root=True, order='post'):
+ envmod.extend(uenv.environment_modifications_for_spec(dep))
+ ret.add_default_envmod(envmod)
+ return ret
+ else:
+ tty.warn('Exe %s not found in prefix %s' % (exe, ispec.prefix))
+
+ def _raise_error(executable, exe_spec):
+ error_msg = 'cannot find the executable "{0}"'.format(executable)
+ if exe_spec:
+ error_msg += ' from spec "{0}'.format(exe_spec)
+ raise RuntimeError(error_msg)
+
+ # If we're not allowed to install this for ourselves, we can't find it
+ if not install:
+ _raise_error(exe, spec)
+
+ with spack_python_interpreter():
+ # We will install for ourselves, using this python if needed
+ # Concretize the spec
+ spec.concretize()
+
+ spec.package.do_install()
+ # filter out directories of the same name as the executable
+ exe_path = [exe_p for exe_p in fs.find(spec.prefix, exe)
+ if fs.is_exe(exe_p)]
+ if exe_path:
+ ret = spack.util.executable.Executable(exe_path[0])
+ envmod = EnvironmentModifications()
+ for dep in spec.traverse(root=True, order='post'):
+ envmod.extend(uenv.environment_modifications_for_spec(dep))
+ ret.add_default_envmod(envmod)
+ return ret
+
+ _raise_error(exe, spec)
+
+
+@contextlib.contextmanager
+def ensure_bootstrap_configuration():
+ # Default configuration scopes excluding command
+ # line and builtin
+ config_scopes = [
+ spack.config.ConfigScope(name, path)
+ for name, path in spack.config.configuration_paths
+ ]
+
+ with spack.architecture.use_platform(spack.architecture.real_platform()):
+ with spack.config.use_configuration(*config_scopes):
+ with spack.repo.use_repositories(spack.paths.packages_path):
+ with spack.store.use_store(spack.paths.user_bootstrap_store):
+ with spack_python_interpreter():
+ yield
diff --git a/lib/spack/spack/build_systems/python.py b/lib/spack/spack/build_systems/python.py
index 76159d88a1..2616642732 100644
--- a/lib/spack/spack/build_systems/python.py
+++ b/lib/spack/spack/build_systems/python.py
@@ -233,7 +233,28 @@ class PythonPackage(PackageBase):
if ('py-setuptools' == spec.name or # this is setuptools, or
'py-setuptools' in spec._dependencies and # it's an immediate dep
'build' in spec._dependencies['py-setuptools'].deptypes):
- args += ['--single-version-externally-managed', '--root=/']
+ args += ['--single-version-externally-managed']
+
+ # Get all relative paths since we set the root to `prefix`
+ # We query the python with which these will be used for the lib and inc
+ # directories. This ensures we use `lib`/`lib64` as expected by python.
+ python = spec['python'].package.command
+ command_start = 'print(distutils.sysconfig.'
+ commands = ';'.join([
+ 'import distutils.sysconfig',
+ command_start + 'get_python_lib(plat_specific=False, prefix=""))',
+ command_start + 'get_python_lib(plat_specific=True, prefix=""))',
+ command_start + 'get_python_inc(plat_specific=True, prefix=""))'])
+ pure_site_packages_dir, plat_site_packages_dir, inc_dir = python(
+ '-c', commands, output=str, error=str).strip().split('\n')
+
+ args += ['--root=%s' % prefix,
+ '--install-purelib=%s' % pure_site_packages_dir,
+ '--install-platlib=%s' % plat_site_packages_dir,
+ '--install-scripts=bin',
+ '--install-data=""',
+ '--install-headers=%s' % inc_dir
+ ]
return args
diff --git a/lib/spack/spack/cmd/clean.py b/lib/spack/spack/cmd/clean.py
index f69b959293..2e1ec1255f 100644
--- a/lib/spack/spack/cmd/clean.py
+++ b/lib/spack/spack/cmd/clean.py
@@ -10,11 +10,12 @@ import shutil
import llnl.util.tty as tty
import spack.caches
+import spack.config
import spack.cmd.test
import spack.cmd.common.arguments as arguments
+import spack.main
import spack.repo
import spack.stage
-import spack.config
from spack.paths import lib_path, var_path
@@ -26,7 +27,7 @@ level = "long"
class AllClean(argparse.Action):
"""Activates flags -s -d -f -m and -p simultaneously"""
def __call__(self, parser, namespace, values, option_string=None):
- parser.parse_args(['-sdfmp'], namespace=namespace)
+ parser.parse_args(['-sdfmpb'], namespace=namespace)
def setup_parser(subparser):
@@ -46,7 +47,10 @@ def setup_parser(subparser):
'-p', '--python-cache', action='store_true',
help="remove .pyc, .pyo files and __pycache__ folders")
subparser.add_argument(
- '-a', '--all', action=AllClean, help="equivalent to -sdfmp", nargs=0
+ '-b', '--bootstrap', action='store_true',
+ help="remove software needed to bootstrap Spack")
+ subparser.add_argument(
+ '-a', '--all', action=AllClean, help="equivalent to -sdfmpb", nargs=0
)
arguments.add_common_arguments(subparser, ['specs'])
@@ -54,7 +58,7 @@ def setup_parser(subparser):
def clean(parser, args):
# If nothing was set, activate the default
if not any([args.specs, args.stage, args.downloads, args.failures,
- args.misc_cache, args.python_cache]):
+ args.misc_cache, args.python_cache, args.bootstrap]):
args.stage = True
# Then do the cleaning falling through the cases
@@ -96,3 +100,10 @@ def clean(parser, args):
dname = os.path.join(root, d)
tty.debug('Removing {0}'.format(dname))
shutil.rmtree(dname)
+
+ if args.bootstrap:
+ msg = 'Removing software in "{0}"'
+ tty.msg(msg.format(spack.paths.user_bootstrap_store))
+ with spack.store.use_store(spack.paths.user_bootstrap_store):
+ uninstall = spack.main.SpackCommand('uninstall')
+ uninstall('-a', '-y')
diff --git a/lib/spack/spack/cmd/find.py b/lib/spack/spack/cmd/find.py
index d9d51b853c..f770a02b25 100644
--- a/lib/spack/spack/cmd/find.py
+++ b/lib/spack/spack/cmd/find.py
@@ -109,6 +109,10 @@ def setup_parser(subparser):
subparser.add_argument(
'--end-date', help='latest date of installation [YYYY-MM-DD]'
)
+ subparser.add_argument(
+ '-b', '--bootstrap', action='store_true',
+ help='show software in the internal bootstrap store'
+ )
arguments.add_common_arguments(subparser, ['constraint'])
@@ -201,7 +205,14 @@ def display_env(env, args, decorator):
def find(parser, args):
q_args = query_arguments(args)
- results = args.specs(**q_args)
+ # Query the current store or the internal bootstrap store if required
+ if args.bootstrap:
+ msg = 'Showing internal bootstrap store at "{0}"'
+ tty.msg(msg.format(spack.paths.user_bootstrap_store))
+ with spack.store.use_store(spack.paths.user_bootstrap_store):
+ results = args.specs(**q_args)
+ else:
+ results = args.specs(**q_args)
decorator = lambda s, f: f
added = set()
diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py
index 9c803cba7e..068a7a6fdb 100644
--- a/lib/spack/spack/paths.py
+++ b/lib/spack/spack/paths.py
@@ -50,7 +50,8 @@ mock_packages_path = os.path.join(repos_path, "builtin.mock")
#: User configuration location
user_config_path = os.path.expanduser('~/.spack')
-
+user_bootstrap_path = os.path.join(user_config_path, 'bootstrap')
+user_bootstrap_store = os.path.join(user_bootstrap_path, 'store')
opt_path = os.path.join(prefix, "opt")
etc_path = os.path.join(prefix, "etc")
diff --git a/lib/spack/spack/solver/asp.py b/lib/spack/spack/solver/asp.py
index ac2983cecc..8d2b2b8aa5 100644
--- a/lib/spack/spack/solver/asp.py
+++ b/lib/spack/spack/solver/asp.py
@@ -1,4 +1,4 @@
-# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
@@ -22,6 +22,7 @@ try:
clingo_cffi = hasattr(clingo.Symbol, '_rep')
except ImportError:
clingo = None
+ clingo_cffi = False
import llnl.util.lang
import llnl.util.tty as tty
@@ -38,6 +39,7 @@ import spack.spec
import spack.package
import spack.package_prefs
import spack.repo
+import spack.bootstrap
import spack.variant
import spack.version
@@ -246,7 +248,20 @@ class PyclingoDriver(object):
asp (file-like): optional stream to write a text-based ASP program
for debugging or verification.
"""
- assert clingo, "PyclingoDriver requires clingo with Python support"
+ global clingo
+ if not clingo:
+ # TODO: Find a way to vendor the concrete spec
+ # in a cross-platform way
+ with spack.bootstrap.ensure_bootstrap_configuration():
+ generic_target = archspec.cpu.host().family
+ spec_str = 'clingo-bootstrap@spack+python target={0}'.format(
+ str(generic_target)
+ )
+ clingo_spec = spack.spec.Spec(spec_str)
+ clingo_spec._old_concretize()
+ spack.bootstrap.make_module_available(
+ 'clingo', spec=clingo_spec, install=True)
+ import clingo
self.out = asp or llnl.util.lang.Devnull()
self.cores = cores
@@ -1112,7 +1127,6 @@ class SpackSolverSetup(object):
if not spec.architecture or not spec.architecture.target:
continue
- print("TTYPE:", type(platform.target(spec.target.name)))
target = archspec.cpu.TARGETS.get(spec.target.name)
if not target:
self.target_ranges(spec, None)
diff --git a/lib/spack/spack/store.py b/lib/spack/spack/store.py
index 6040145b64..7b77768a37 100644
--- a/lib/spack/spack/store.py
+++ b/lib/spack/spack/store.py
@@ -173,6 +173,16 @@ def _store():
config_dict = spack.config.get('config')
root, unpadded_root, projections = parse_install_tree(config_dict)
hash_length = spack.config.get('config:install_hash_length')
+
+ # Check that the user is not trying to install software into the store
+ # reserved by Spack to bootstrap its own dependencies, since this would
+ # lead to bizarre behaviors (e.g. cleaning the bootstrap area would wipe
+ # user installed software)
+ if spack.paths.user_bootstrap_store == root:
+ msg = ('please change the install tree root "{0}" in your '
+ 'configuration [path reserved for Spack internal use]')
+ raise ValueError(msg.format(root))
+
return Store(root=root,
unpadded_root=unpadded_root,
projections=projections,
diff --git a/lib/spack/spack/util/environment.py b/lib/spack/spack/util/environment.py
index b370053c3c..53e0da32d7 100644
--- a/lib/spack/spack/util/environment.py
+++ b/lib/spack/spack/util/environment.py
@@ -528,13 +528,18 @@ class EnvironmentModifications(object):
return rev
- def apply_modifications(self):
+ def apply_modifications(self, env=None):
"""Applies the modifications and clears the list."""
+ # Use os.environ if not specified
+ # Do not copy, we want to modify it in place
+ if env is None:
+ env = os.environ
+
modifications = self.group_by_name()
# Apply modifications one variable at a time
for name, actions in sorted(modifications.items()):
for x in actions:
- x.execute(os.environ)
+ x.execute(env)
def shell_modifications(self, shell='sh'):
"""Return shell code to apply the modifications and clears the list."""
diff --git a/lib/spack/spack/util/executable.py b/lib/spack/spack/util/executable.py
index 614bc1725a..fb192d6885 100644
--- a/lib/spack/spack/util/executable.py
+++ b/lib/spack/spack/util/executable.py
@@ -22,6 +22,8 @@ class Executable(object):
def __init__(self, name):
self.exe = shlex.split(str(name))
self.default_env = {}
+ from spack.util.environment import EnvironmentModifications # no cycle
+ self.default_envmod = EnvironmentModifications()
self.returncode = None
if not self.exe:
@@ -40,6 +42,10 @@ class Executable(object):
"""
self.default_env[key] = value
+ def add_default_envmod(self, envmod):
+ """Set an EnvironmentModifications to use when the command is run."""
+ self.default_envmod.extend(envmod)
+
@property
def command(self):
"""The command-line string.
@@ -76,9 +82,10 @@ class Executable(object):
Keyword Arguments:
_dump_env (dict): Dict to be set to the environment actually
used (envisaged for testing purposes only)
- env (dict): The environment to run the executable with
- extra_env (dict): Extra items to add to the environment
- (neither requires nor precludes env)
+ env (dict or EnvironmentModifications): The environment with which
+ to run the executable
+ extra_env (dict or EnvironmentModifications): Extra items to add to
+ the environment (neither requires nor precludes env)
fail_on_error (bool): Raise an exception if the subprocess returns
an error. Default is True. The return code is available as
``exe.returncode``
@@ -107,13 +114,26 @@ class Executable(object):
"""
# Environment
env_arg = kwargs.get('env', None)
- if env_arg is None:
- env = os.environ.copy()
- env.update(self.default_env)
- else:
- env = self.default_env.copy()
+
+ # Setup default environment
+ env = os.environ.copy() if env_arg is None else {}
+ self.default_envmod.apply_modifications(env)
+ env.update(self.default_env)
+
+ from spack.util.environment import EnvironmentModifications # no cycle
+ # Apply env argument
+ if isinstance(env_arg, EnvironmentModifications):
+ env_arg.apply_modifications(env)
+ elif env_arg:
env.update(env_arg)
- env.update(kwargs.get('extra_env', {}))
+
+ # Apply extra env
+ extra_env = kwargs.get('extra_env', {})
+ if isinstance(extra_env, EnvironmentModifications):
+ extra_env.apply_modifications(env)
+ else:
+ env.update(extra_env)
+
if '_dump_env' in kwargs:
kwargs['_dump_env'].clear()
kwargs['_dump_env'].update(env)