diff options
Diffstat (limited to 'lib')
| -rw-r--r-- | lib/spack/docs/getting_started.rst | 47 | ||||
| -rw-r--r-- | lib/spack/spack/bootstrap.py | 188 | ||||
| -rw-r--r-- | lib/spack/spack/build_systems/python.py | 23 | ||||
| -rw-r--r-- | lib/spack/spack/cmd/clean.py | 19 | ||||
| -rw-r--r-- | lib/spack/spack/cmd/find.py | 13 | ||||
| -rw-r--r-- | lib/spack/spack/paths.py | 3 | ||||
| -rw-r--r-- | lib/spack/spack/solver/asp.py | 20 | ||||
| -rw-r--r-- | lib/spack/spack/store.py | 10 | ||||
| -rw-r--r-- | lib/spack/spack/util/environment.py | 9 | ||||
| -rw-r--r-- | lib/spack/spack/util/executable.py | 38 |
10 files changed, 349 insertions, 21 deletions
diff --git a/lib/spack/docs/getting_started.rst b/lib/spack/docs/getting_started.rst index f1df0343bd..09cc2c488a 100644 --- a/lib/spack/docs/getting_started.rst +++ b/lib/spack/docs/getting_started.rst @@ -103,6 +103,53 @@ environment*, especially for ``PATH``. Only software that comes with the system, or that you know you wish to use with Spack, should be included. This procedure will avoid many strange build errors. +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Optional: Bootstrapping clingo +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Spack supports using clingo as an external solver to compute which software +needs to be installed. If you have a default compiler supporting C++14 Spack +can automatically bootstrap this tool from sources the first time it is +needed: + +.. code-block:: console + + $ spack solve zlib + [+] /usr (external bison-3.0.4-wu5pgjchxzemk5ya2l3ddqug2d7jv6eb) + [+] /usr (external cmake-3.19.4-a4kmcfzxxy45mzku4ipmj5kdiiz5a57b) + [+] /usr (external python-3.6.9-x4fou4iqqlh5ydwddx3pvfcwznfrqztv) + ==> Installing re2c-1.2.1-e3x6nxtk3ahgd63ykgy44mpuva6jhtdt + [ ... ] + ==> Optimization: [0, 0, 0, 0, 0, 1, 0, 0, 0] + zlib@1.2.11%gcc@10.1.0+optimize+pic+shared arch=linux-ubuntu18.04-broadwell + +If you want to speed-up bootstrapping, you may try to search for ``cmake`` and ``bison`` +on your system: + +.. code-block:: console + + $ spack external find cmake bison + ==> The following specs have been detected on this system and added to /home/spack/.spack/packages.yaml + bison@3.0.4 cmake@3.19.4 + +All the tools Spack needs for its own functioning are installed in a separate store, which lives +under the ``${HOME}/.spack`` directory. The software installed there can be queried with: + +.. code-block:: console + + $ spack find --bootstrap + ==> Showing internal bootstrap store at "/home/spack/.spack/bootstrap/store" + ==> 3 installed packages + -- linux-ubuntu18.04-x86_64 / gcc@10.1.0 ------------------------ + clingo-bootstrap@spack python@3.6.9 re2c@1.2.1 + +In case it's needed the bootstrap store can also be cleaned with: + +.. code-block:: console + + $ spack clean -b + ==> Removing software in "/home/spack/.spack/bootstrap/store" + ^^^^^^^^^^^^^^^^^^^^^^^^^^ Optional: Alternate Prefix ^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/lib/spack/spack/bootstrap.py b/lib/spack/spack/bootstrap.py new file mode 100644 index 0000000000..93e79fbe71 --- /dev/null +++ b/lib/spack/spack/bootstrap.py @@ -0,0 +1,188 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +import contextlib +import os +import sys + +import llnl.util.filesystem as fs +import llnl.util.tty as tty + +import spack.architecture +import spack.config +import spack.paths +import spack.repo +import spack.spec +import spack.store +import spack.user_environment as uenv +import spack.util.executable +from spack.util.environment import EnvironmentModifications + + +@contextlib.contextmanager +def spack_python_interpreter(): + """Override the current configuration to set the interpreter under + which Spack is currently running as the only Python external spec + available. + """ + python_cls = type(spack.spec.Spec('python').package) + python_prefix = os.path.dirname(os.path.dirname(sys.executable)) + externals = python_cls.determine_spec_details( + python_prefix, [os.path.basename(sys.executable)]) + external_python = externals[0] + + entry = { + 'buildable': False, + 'externals': [ + {'prefix': python_prefix, 'spec': str(external_python)} + ] + } + + with spack.config.override('packages:python::', entry): + yield + + +def make_module_available(module, spec=None, install=False): + """Ensure module is importable""" + # If we already can import it, that's great + try: + __import__(module) + return + except ImportError: + pass + + # If it's already installed, use it + # Search by spec + spec = spack.spec.Spec(spec or module) + + # We have to run as part of this python + # We can constrain by a shortened version in place of a version range + # because this spec is only used for querying or as a placeholder to be + # replaced by an external that already has a concrete version. This syntax + # is not suffucient when concretizing without an external, as it will + # concretize to python@X.Y instead of python@X.Y.Z + spec.constrain('^python@%d.%d' % sys.version_info[:2]) + installed_specs = spack.store.db.query(spec, installed=True) + + for ispec in installed_specs: + # TODO: make sure run-environment is appropriate + module_path = os.path.join(ispec.prefix, + ispec['python'].package.site_packages_dir) + module_path_64 = module_path.replace('/lib/', '/lib64/') + try: + sys.path.append(module_path) + sys.path.append(module_path_64) + __import__(module) + return + except ImportError: + tty.warn("Spec %s did not provide module %s" % (ispec, module)) + sys.path = sys.path[:-2] + + def _raise_error(module_name, module_spec): + error_msg = 'cannot import module "{0}"'.format(module_name) + if module_spec: + error_msg += ' from spec "{0}'.format(module_spec) + raise ImportError(error_msg) + + if not install: + _raise_error(module, spec) + + with spack_python_interpreter(): + # We will install for ourselves, using this python if needed + # Concretize the spec + spec.concretize() + spec.package.do_install() + + module_path = os.path.join(spec.prefix, + spec['python'].package.site_packages_dir) + module_path_64 = module_path.replace('/lib/', '/lib64/') + try: + sys.path.append(module_path) + sys.path.append(module_path_64) + __import__(module) + return + except ImportError: + sys.path = sys.path[:-2] + _raise_error(module, spec) + + +def get_executable(exe, spec=None, install=False): + """Find an executable named exe, either in PATH or in Spack + + Args: + exe (str): needed executable name + spec (Spec or str): spec to search for exe in (default exe) + install (bool): install spec if not available + + When ``install`` is True, Spack will use the python used to run Spack as an + external. The ``install`` option should only be used with packages that + install quickly (when using external python) or are guaranteed by Spack + organization to be in a binary mirror (clingo).""" + # Search the system first + runner = spack.util.executable.which(exe) + if runner: + return runner + + # Check whether it's already installed + spec = spack.spec.Spec(spec or exe) + installed_specs = spack.store.db.query(spec, installed=True) + for ispec in installed_specs: + # filter out directories of the same name as the executable + exe_path = [exe_p for exe_p in fs.find(ispec.prefix, exe) + if fs.is_exe(exe_p)] + if exe_path: + ret = spack.util.executable.Executable(exe_path[0]) + envmod = EnvironmentModifications() + for dep in ispec.traverse(root=True, order='post'): + envmod.extend(uenv.environment_modifications_for_spec(dep)) + ret.add_default_envmod(envmod) + return ret + else: + tty.warn('Exe %s not found in prefix %s' % (exe, ispec.prefix)) + + def _raise_error(executable, exe_spec): + error_msg = 'cannot find the executable "{0}"'.format(executable) + if exe_spec: + error_msg += ' from spec "{0}'.format(exe_spec) + raise RuntimeError(error_msg) + + # If we're not allowed to install this for ourselves, we can't find it + if not install: + _raise_error(exe, spec) + + with spack_python_interpreter(): + # We will install for ourselves, using this python if needed + # Concretize the spec + spec.concretize() + + spec.package.do_install() + # filter out directories of the same name as the executable + exe_path = [exe_p for exe_p in fs.find(spec.prefix, exe) + if fs.is_exe(exe_p)] + if exe_path: + ret = spack.util.executable.Executable(exe_path[0]) + envmod = EnvironmentModifications() + for dep in spec.traverse(root=True, order='post'): + envmod.extend(uenv.environment_modifications_for_spec(dep)) + ret.add_default_envmod(envmod) + return ret + + _raise_error(exe, spec) + + +@contextlib.contextmanager +def ensure_bootstrap_configuration(): + # Default configuration scopes excluding command + # line and builtin + config_scopes = [ + spack.config.ConfigScope(name, path) + for name, path in spack.config.configuration_paths + ] + + with spack.architecture.use_platform(spack.architecture.real_platform()): + with spack.config.use_configuration(*config_scopes): + with spack.repo.use_repositories(spack.paths.packages_path): + with spack.store.use_store(spack.paths.user_bootstrap_store): + with spack_python_interpreter(): + yield diff --git a/lib/spack/spack/build_systems/python.py b/lib/spack/spack/build_systems/python.py index 76159d88a1..2616642732 100644 --- a/lib/spack/spack/build_systems/python.py +++ b/lib/spack/spack/build_systems/python.py @@ -233,7 +233,28 @@ class PythonPackage(PackageBase): if ('py-setuptools' == spec.name or # this is setuptools, or 'py-setuptools' in spec._dependencies and # it's an immediate dep 'build' in spec._dependencies['py-setuptools'].deptypes): - args += ['--single-version-externally-managed', '--root=/'] + args += ['--single-version-externally-managed'] + + # Get all relative paths since we set the root to `prefix` + # We query the python with which these will be used for the lib and inc + # directories. This ensures we use `lib`/`lib64` as expected by python. + python = spec['python'].package.command + command_start = 'print(distutils.sysconfig.' + commands = ';'.join([ + 'import distutils.sysconfig', + command_start + 'get_python_lib(plat_specific=False, prefix=""))', + command_start + 'get_python_lib(plat_specific=True, prefix=""))', + command_start + 'get_python_inc(plat_specific=True, prefix=""))']) + pure_site_packages_dir, plat_site_packages_dir, inc_dir = python( + '-c', commands, output=str, error=str).strip().split('\n') + + args += ['--root=%s' % prefix, + '--install-purelib=%s' % pure_site_packages_dir, + '--install-platlib=%s' % plat_site_packages_dir, + '--install-scripts=bin', + '--install-data=""', + '--install-headers=%s' % inc_dir + ] return args diff --git a/lib/spack/spack/cmd/clean.py b/lib/spack/spack/cmd/clean.py index f69b959293..2e1ec1255f 100644 --- a/lib/spack/spack/cmd/clean.py +++ b/lib/spack/spack/cmd/clean.py @@ -10,11 +10,12 @@ import shutil import llnl.util.tty as tty import spack.caches +import spack.config import spack.cmd.test import spack.cmd.common.arguments as arguments +import spack.main import spack.repo import spack.stage -import spack.config from spack.paths import lib_path, var_path @@ -26,7 +27,7 @@ level = "long" class AllClean(argparse.Action): """Activates flags -s -d -f -m and -p simultaneously""" def __call__(self, parser, namespace, values, option_string=None): - parser.parse_args(['-sdfmp'], namespace=namespace) + parser.parse_args(['-sdfmpb'], namespace=namespace) def setup_parser(subparser): @@ -46,7 +47,10 @@ def setup_parser(subparser): '-p', '--python-cache', action='store_true', help="remove .pyc, .pyo files and __pycache__ folders") subparser.add_argument( - '-a', '--all', action=AllClean, help="equivalent to -sdfmp", nargs=0 + '-b', '--bootstrap', action='store_true', + help="remove software needed to bootstrap Spack") + subparser.add_argument( + '-a', '--all', action=AllClean, help="equivalent to -sdfmpb", nargs=0 ) arguments.add_common_arguments(subparser, ['specs']) @@ -54,7 +58,7 @@ def setup_parser(subparser): def clean(parser, args): # If nothing was set, activate the default if not any([args.specs, args.stage, args.downloads, args.failures, - args.misc_cache, args.python_cache]): + args.misc_cache, args.python_cache, args.bootstrap]): args.stage = True # Then do the cleaning falling through the cases @@ -96,3 +100,10 @@ def clean(parser, args): dname = os.path.join(root, d) tty.debug('Removing {0}'.format(dname)) shutil.rmtree(dname) + + if args.bootstrap: + msg = 'Removing software in "{0}"' + tty.msg(msg.format(spack.paths.user_bootstrap_store)) + with spack.store.use_store(spack.paths.user_bootstrap_store): + uninstall = spack.main.SpackCommand('uninstall') + uninstall('-a', '-y') diff --git a/lib/spack/spack/cmd/find.py b/lib/spack/spack/cmd/find.py index d9d51b853c..f770a02b25 100644 --- a/lib/spack/spack/cmd/find.py +++ b/lib/spack/spack/cmd/find.py @@ -109,6 +109,10 @@ def setup_parser(subparser): subparser.add_argument( '--end-date', help='latest date of installation [YYYY-MM-DD]' ) + subparser.add_argument( + '-b', '--bootstrap', action='store_true', + help='show software in the internal bootstrap store' + ) arguments.add_common_arguments(subparser, ['constraint']) @@ -201,7 +205,14 @@ def display_env(env, args, decorator): def find(parser, args): q_args = query_arguments(args) - results = args.specs(**q_args) + # Query the current store or the internal bootstrap store if required + if args.bootstrap: + msg = 'Showing internal bootstrap store at "{0}"' + tty.msg(msg.format(spack.paths.user_bootstrap_store)) + with spack.store.use_store(spack.paths.user_bootstrap_store): + results = args.specs(**q_args) + else: + results = args.specs(**q_args) decorator = lambda s, f: f added = set() diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index 9c803cba7e..068a7a6fdb 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -50,7 +50,8 @@ mock_packages_path = os.path.join(repos_path, "builtin.mock") #: User configuration location user_config_path = os.path.expanduser('~/.spack') - +user_bootstrap_path = os.path.join(user_config_path, 'bootstrap') +user_bootstrap_store = os.path.join(user_bootstrap_path, 'store') opt_path = os.path.join(prefix, "opt") etc_path = os.path.join(prefix, "etc") diff --git a/lib/spack/spack/solver/asp.py b/lib/spack/spack/solver/asp.py index ac2983cecc..8d2b2b8aa5 100644 --- a/lib/spack/spack/solver/asp.py +++ b/lib/spack/spack/solver/asp.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -22,6 +22,7 @@ try: clingo_cffi = hasattr(clingo.Symbol, '_rep') except ImportError: clingo = None + clingo_cffi = False import llnl.util.lang import llnl.util.tty as tty @@ -38,6 +39,7 @@ import spack.spec import spack.package import spack.package_prefs import spack.repo +import spack.bootstrap import spack.variant import spack.version @@ -246,7 +248,20 @@ class PyclingoDriver(object): asp (file-like): optional stream to write a text-based ASP program for debugging or verification. """ - assert clingo, "PyclingoDriver requires clingo with Python support" + global clingo + if not clingo: + # TODO: Find a way to vendor the concrete spec + # in a cross-platform way + with spack.bootstrap.ensure_bootstrap_configuration(): + generic_target = archspec.cpu.host().family + spec_str = 'clingo-bootstrap@spack+python target={0}'.format( + str(generic_target) + ) + clingo_spec = spack.spec.Spec(spec_str) + clingo_spec._old_concretize() + spack.bootstrap.make_module_available( + 'clingo', spec=clingo_spec, install=True) + import clingo self.out = asp or llnl.util.lang.Devnull() self.cores = cores @@ -1112,7 +1127,6 @@ class SpackSolverSetup(object): if not spec.architecture or not spec.architecture.target: continue - print("TTYPE:", type(platform.target(spec.target.name))) target = archspec.cpu.TARGETS.get(spec.target.name) if not target: self.target_ranges(spec, None) diff --git a/lib/spack/spack/store.py b/lib/spack/spack/store.py index 6040145b64..7b77768a37 100644 --- a/lib/spack/spack/store.py +++ b/lib/spack/spack/store.py @@ -173,6 +173,16 @@ def _store(): config_dict = spack.config.get('config') root, unpadded_root, projections = parse_install_tree(config_dict) hash_length = spack.config.get('config:install_hash_length') + + # Check that the user is not trying to install software into the store + # reserved by Spack to bootstrap its own dependencies, since this would + # lead to bizarre behaviors (e.g. cleaning the bootstrap area would wipe + # user installed software) + if spack.paths.user_bootstrap_store == root: + msg = ('please change the install tree root "{0}" in your ' + 'configuration [path reserved for Spack internal use]') + raise ValueError(msg.format(root)) + return Store(root=root, unpadded_root=unpadded_root, projections=projections, diff --git a/lib/spack/spack/util/environment.py b/lib/spack/spack/util/environment.py index b370053c3c..53e0da32d7 100644 --- a/lib/spack/spack/util/environment.py +++ b/lib/spack/spack/util/environment.py @@ -528,13 +528,18 @@ class EnvironmentModifications(object): return rev - def apply_modifications(self): + def apply_modifications(self, env=None): """Applies the modifications and clears the list.""" + # Use os.environ if not specified + # Do not copy, we want to modify it in place + if env is None: + env = os.environ + modifications = self.group_by_name() # Apply modifications one variable at a time for name, actions in sorted(modifications.items()): for x in actions: - x.execute(os.environ) + x.execute(env) def shell_modifications(self, shell='sh'): """Return shell code to apply the modifications and clears the list.""" diff --git a/lib/spack/spack/util/executable.py b/lib/spack/spack/util/executable.py index 614bc1725a..fb192d6885 100644 --- a/lib/spack/spack/util/executable.py +++ b/lib/spack/spack/util/executable.py @@ -22,6 +22,8 @@ class Executable(object): def __init__(self, name): self.exe = shlex.split(str(name)) self.default_env = {} + from spack.util.environment import EnvironmentModifications # no cycle + self.default_envmod = EnvironmentModifications() self.returncode = None if not self.exe: @@ -40,6 +42,10 @@ class Executable(object): """ self.default_env[key] = value + def add_default_envmod(self, envmod): + """Set an EnvironmentModifications to use when the command is run.""" + self.default_envmod.extend(envmod) + @property def command(self): """The command-line string. @@ -76,9 +82,10 @@ class Executable(object): Keyword Arguments: _dump_env (dict): Dict to be set to the environment actually used (envisaged for testing purposes only) - env (dict): The environment to run the executable with - extra_env (dict): Extra items to add to the environment - (neither requires nor precludes env) + env (dict or EnvironmentModifications): The environment with which + to run the executable + extra_env (dict or EnvironmentModifications): Extra items to add to + the environment (neither requires nor precludes env) fail_on_error (bool): Raise an exception if the subprocess returns an error. Default is True. The return code is available as ``exe.returncode`` @@ -107,13 +114,26 @@ class Executable(object): """ # Environment env_arg = kwargs.get('env', None) - if env_arg is None: - env = os.environ.copy() - env.update(self.default_env) - else: - env = self.default_env.copy() + + # Setup default environment + env = os.environ.copy() if env_arg is None else {} + self.default_envmod.apply_modifications(env) + env.update(self.default_env) + + from spack.util.environment import EnvironmentModifications # no cycle + # Apply env argument + if isinstance(env_arg, EnvironmentModifications): + env_arg.apply_modifications(env) + elif env_arg: env.update(env_arg) - env.update(kwargs.get('extra_env', {})) + + # Apply extra env + extra_env = kwargs.get('extra_env', {}) + if isinstance(extra_env, EnvironmentModifications): + extra_env.apply_modifications(env) + else: + env.update(extra_env) + if '_dump_env' in kwargs: kwargs['_dump_env'].clear() kwargs['_dump_env'].update(env) |