diff options
| -rw-r--r-- | etc/spack/defaults/bootstrap.yaml | 29 | ||||
| -rw-r--r-- | lib/spack/docs/bootstrapping.rst | 160 | ||||
| -rw-r--r-- | lib/spack/docs/index.rst | 1 | ||||
| -rw-r--r-- | lib/spack/spack/bootstrap.py | 107 | ||||
| -rw-r--r-- | lib/spack/spack/cmd/bootstrap.py | 207 | ||||
| -rw-r--r-- | lib/spack/spack/schema/bootstrap.py | 6 | ||||
| -rw-r--r-- | lib/spack/spack/test/cmd/bootstrap.py | 78 | ||||
| -rw-r--r-- | lib/spack/spack/test/data/config/bootstrap.yaml | 9 | ||||
| -rw-r--r-- | share/spack/bootstrap/github-actions-v0.1/metadata.yaml | 8 | ||||
| -rw-r--r-- | share/spack/bootstrap/github-actions-v0.2/metadata.yaml | 8 | ||||
| -rw-r--r-- | share/spack/bootstrap/spack-install/metadata.yaml | 8 | ||||
| -rwxr-xr-x | share/spack/spack-completion.bash | 29 |
12 files changed, 580 insertions, 70 deletions
diff --git a/etc/spack/defaults/bootstrap.yaml b/etc/spack/defaults/bootstrap.yaml index 8c1592055e..b3ab1c99df 100644 --- a/etc/spack/defaults/bootstrap.yaml +++ b/etc/spack/defaults/bootstrap.yaml @@ -6,34 +6,15 @@ bootstrap: # by Spack is installed in a "store" subfolder of this root directory root: $user_cache_path/bootstrap # Methods that can be used to bootstrap software. Each method may or - # may not be able to bootstrap all of the software that Spack needs, + # may not be able to bootstrap all the software that Spack needs, # depending on its type. sources: - name: 'github-actions-v0.2' - type: buildcache - description: | - Buildcache generated from a public workflow using Github Actions. - The sha256 checksum of binaries is checked before installation. - info: - url: https://mirror.spack.io/bootstrap/github-actions/v0.2 - homepage: https://github.com/spack/spack-bootstrap-mirrors - releases: https://github.com/spack/spack-bootstrap-mirrors/releases + metadata: $spack/share/spack/bootstrap/github-actions-v0.2 - name: 'github-actions-v0.1' - type: buildcache - description: | - Buildcache generated from a public workflow using Github Actions. - The sha256 checksum of binaries is checked before installation. - info: - url: https://mirror.spack.io/bootstrap/github-actions/v0.1 - homepage: https://github.com/spack/spack-bootstrap-mirrors - releases: https://github.com/spack/spack-bootstrap-mirrors/releases - # This method is just Spack bootstrapping the software it needs from sources. - # It has been added here so that users can selectively disable bootstrapping - # from sources by "untrusting" it. - - name: spack-install - type: install - description: | - Specs built from sources by Spack. May take a long time. + metadata: $spack/share/spack/bootstrap/github-actions-v0.1 + - name: 'spack-install' + metadata: $spack/share/spack/bootstrap/spack-install trusted: # By default we trust bootstrapping from sources and from binaries # produced on Github via the workflow diff --git a/lib/spack/docs/bootstrapping.rst b/lib/spack/docs/bootstrapping.rst new file mode 100644 index 0000000000..a38e96ac2f --- /dev/null +++ b/lib/spack/docs/bootstrapping.rst @@ -0,0 +1,160 @@ +.. Copyright 2013-2021 Lawrence Livermore National Security, LLC and other + Spack Project Developers. See the top-level COPYRIGHT file for details. + + SPDX-License-Identifier: (Apache-2.0 OR MIT) + +.. _bootstrapping: + +============= +Bootstrapping +============= + +In the :ref:`Getting started <getting_started>` Section we already mentioned that +Spack can bootstrap some of its dependencies, including ``clingo``. In fact, there +is an entire command dedicated to the management of every aspect of bootstrapping: + +.. command-output:: spack bootstrap --help + +The first thing to know to understand bootstrapping in Spack is that each of +Spack's dependencies is bootstrapped lazily; i.e. the first time it is needed and +can't be found. You can readily check if any prerequisite for using Spack +is missing by running: + +.. code-block:: console + + % spack bootstrap status + Spack v0.17.1 - python@3.8 + + [FAIL] Core Functionalities + [B] MISSING "clingo": required to concretize specs + + [FAIL] Binary packages + [B] MISSING "gpg2": required to sign/verify buildcaches + + + Spack will take care of bootstrapping any missing dependency marked as [B]. Dependencies marked as [-] are instead required to be found on the system. + +In the case of the output shown above Spack detected that both ``clingo`` and ``gnupg`` +are missing and it's giving detailed information on why they are needed and whether +they can be bootstrapped. Running a command that concretize a spec, like: + +.. code-block:: console + + % spack solve zlib + ==> Bootstrapping clingo from pre-built binaries + ==> Fetching https://mirror.spack.io/bootstrap/github-actions/v0.1/build_cache/darwin-catalina-x86_64/apple-clang-12.0.0/clingo-bootstrap-spack/darwin-catalina-x86_64-apple-clang-12.0.0-clingo-bootstrap-spack-p5on7i4hejl775ezndzfdkhvwra3hatn.spack + ==> Installing "clingo-bootstrap@spack%apple-clang@12.0.0~docs~ipo+python build_type=Release arch=darwin-catalina-x86_64" from a buildcache + [ ... ] + +triggers the bootstrapping of clingo from pre-built binaries as expected. + +----------------------- +The Bootstrapping store +----------------------- + +The software installed for bootstrapping purposes is deployed in a separate store. +Its location can be checked with the following command: + +.. code-block:: console + + % spack bootstrap root + +It can also be changed with the same command by just specifying the newly desired path: + +.. code-block:: console + + % spack bootstrap root /opt/spack/bootstrap + +You can check what is installed in the bootstrapping store at any time using: + +.. code-block:: console + + % spack find -b + ==> Showing internal bootstrap store at "/Users/spack/.spack/bootstrap/store" + ==> 11 installed packages + -- darwin-catalina-x86_64 / apple-clang@12.0.0 ------------------ + clingo-bootstrap@spack libassuan@2.5.5 libgpg-error@1.42 libksba@1.5.1 pinentry@1.1.1 zlib@1.2.11 + gnupg@2.3.1 libgcrypt@1.9.3 libiconv@1.16 npth@1.6 python@3.8 + +In case it is needed you can remove all the software in the current bootstrapping store with: + +.. code-block:: console + + % spack clean -b + ==> Removing bootstrapped software and configuration in "/Users/spack/.spack/bootstrap" + + % spack find -b + ==> Showing internal bootstrap store at "/Users/spack/.spack/bootstrap/store" + ==> 0 installed packages + +-------------------------------------------- +Enabling and disabling bootstrapping methods +-------------------------------------------- + +Bootstrapping is always performed by trying the methods listed by: + +.. command-output:: spack bootstrap list + +in the order they appear, from top to bottom. By default Spack is +configured to try first bootstrapping from pre-built binaries and to +fall-back to bootstrapping from sources if that failed. + +If need be, you can disable bootstrapping altogether by running: + +.. code-block:: console + + % spack bootstrap disable + +in which case it's your responsibility to ensure Spack runs in an +environment where all its prerequisites are installed. You can +also configure Spack to skip certain bootstrapping methods by *untrusting* +them. For instance: + +.. code-block:: console + + % spack bootstrap untrust github-actions + ==> "github-actions" is now untrusted and will not be used for bootstrapping + +tells Spack to skip trying to bootstrap from binaries. To add the "github-actions" method back you can: + +.. code-block:: console + + % spack bootstrap trust github-actions + +There is also an option to reset the bootstrapping configuration to Spack's defaults: + +.. code-block:: console + + % spack bootstrap reset + ==> Bootstrapping configuration is being reset to Spack's defaults. Current configuration will be lost. + Do you want to continue? [Y/n] + % + +---------------------------------------- +Creating a mirror for air-gapped systems +---------------------------------------- + +Spack's default configuration for bootstrapping relies on the user having +access to the internet, either to fetch pre-compiled binaries or source tarballs. +Sometimes though Spack is deployed on air-gapped systems where such access is denied. + +To help with similar situations Spack has a command that recreates, in a local folder +of choice, a mirror containing the source tarballs and/or binary packages needed for +bootstrapping. + +.. code-block:: console + + % spack bootstrap mirror --binary-packages /opt/bootstrap + ==> Adding "clingo-bootstrap@spack+python %apple-clang target=x86_64" and dependencies to the mirror at /opt/bootstrap/local-mirror + ==> Adding "gnupg@2.3: %apple-clang target=x86_64" and dependencies to the mirror at /opt/bootstrap/local-mirror + ==> Adding "patchelf@0.13.1:0.13.99 %apple-clang target=x86_64" and dependencies to the mirror at /opt/bootstrap/local-mirror + ==> Adding binary packages from "https://github.com/alalazo/spack-bootstrap-mirrors/releases/download/v0.1-rc.2/bootstrap-buildcache.tar.gz" to the mirror at /opt/bootstrap/local-mirror + + To register the mirror on the platform where it's supposed to be used run the following command(s): + % spack bootstrap add --trust local-sources /opt/bootstrap/metadata/sources + % spack bootstrap add --trust local-binaries /opt/bootstrap/metadata/binaries + + +This command needs to be run on a machine with internet access and the resulting folder +has to be moved over to the air-gapped system. Once the local sources are added using the +commands suggested at the prompt, they can be used to bootstrap Spack.
\ No newline at end of file diff --git a/lib/spack/docs/index.rst b/lib/spack/docs/index.rst index 6f17eb9bf4..b755f2f376 100644 --- a/lib/spack/docs/index.rst +++ b/lib/spack/docs/index.rst @@ -63,6 +63,7 @@ or refer to the full manual below. configuration config_yaml + bootstrapping build_settings environments containers diff --git a/lib/spack/spack/bootstrap.py b/lib/spack/spack/bootstrap.py index aae85239fd..20668cf077 100644 --- a/lib/spack/spack/bootstrap.py +++ b/lib/spack/spack/bootstrap.py @@ -5,6 +5,7 @@ from __future__ import print_function import contextlib +import copy import fnmatch import functools import json @@ -37,6 +38,11 @@ import spack.user_environment import spack.util.environment import spack.util.executable import spack.util.path +import spack.util.spack_yaml +import spack.util.url + +#: Name of the file containing metadata about the bootstrapping source +METADATA_YAML_FILENAME = 'metadata.yaml' #: Map a bootstrapper type to the corresponding class _bootstrap_methods = {} @@ -204,12 +210,43 @@ def _executables_in_store(executables, query_spec, query_info=None): return False -@_bootstrapper(type='buildcache') -class _BuildcacheBootstrapper(object): - """Install the software needed during bootstrapping from a buildcache.""" +class _BootstrapperBase(object): + """Base class to derive types that can bootstrap software for Spack""" + config_scope_name = '' + def __init__(self, conf): self.name = conf['name'] self.url = conf['info']['url'] + + @property + def mirror_url(self): + # Absolute paths + if os.path.isabs(self.url): + return spack.util.url.format(self.url) + + # Check for :// and assume it's an url if we find it + if '://' in self.url: + return self.url + + # Otherwise, it's a relative path + return spack.util.url.format(os.path.join(self.metadata_dir, self.url)) + + @property + def mirror_scope(self): + return spack.config.InternalConfigScope( + self.config_scope_name, {'mirrors:': {self.name: self.mirror_url}} + ) + + +@_bootstrapper(type='buildcache') +class _BuildcacheBootstrapper(_BootstrapperBase): + """Install the software needed during bootstrapping from a buildcache.""" + + config_scope_name = 'bootstrap_buildcache' + + def __init__(self, conf): + super(_BuildcacheBootstrapper, self).__init__(conf) + self.metadata_dir = spack.util.path.canonicalize_path(conf['metadata']) self.last_search = None @staticmethod @@ -232,9 +269,8 @@ class _BuildcacheBootstrapper(object): def _read_metadata(self, package_name): """Return metadata about the given package.""" json_filename = '{0}.json'.format(package_name) - json_path = os.path.join( - spack.paths.share_path, 'bootstrap', self.name, json_filename - ) + json_dir = self.metadata_dir + json_path = os.path.join(json_dir, json_filename) with open(json_path) as f: data = json.load(f) return data @@ -308,12 +344,6 @@ class _BuildcacheBootstrapper(object): return True return False - @property - def mirror_scope(self): - return spack.config.InternalConfigScope( - 'bootstrap_buildcache', {'mirrors:': {self.name: self.url}} - ) - def try_import(self, module, abstract_spec_str): test_fn, info = functools.partial(_try_import_from_store, module), {} if test_fn(query_spec=abstract_spec_str, query_info=info): @@ -343,9 +373,13 @@ class _BuildcacheBootstrapper(object): @_bootstrapper(type='install') -class _SourceBootstrapper(object): +class _SourceBootstrapper(_BootstrapperBase): """Install the software needed during bootstrapping from sources.""" + config_scope_name = 'bootstrap_source' + def __init__(self, conf): + super(_SourceBootstrapper, self).__init__(conf) + self.metadata_dir = spack.util.path.canonicalize_path(conf['metadata']) self.conf = conf self.last_search = None @@ -378,7 +412,8 @@ class _SourceBootstrapper(object): tty.debug(msg.format(module, abstract_spec_str)) # Install the spec that should make the module importable - concrete_spec.package.do_install(fail_fast=True) + with spack.config.override(self.mirror_scope): + concrete_spec.package.do_install(fail_fast=True) if _try_import_from_store(module, query_spec=concrete_spec, query_info=info): self.last_search = info @@ -391,6 +426,8 @@ class _SourceBootstrapper(object): self.last_search = info return True + tty.info("Bootstrapping {0} from sources".format(abstract_spec_str)) + # If we compile code from sources detecting a few build tools # might reduce compilation time by a fair amount _add_externals_if_missing() @@ -403,7 +440,8 @@ class _SourceBootstrapper(object): msg = "[BOOTSTRAP] Try installing '{0}' from sources" tty.debug(msg.format(abstract_spec_str)) - concrete_spec.package.do_install() + with spack.config.override(self.mirror_scope): + concrete_spec.package.do_install() if _executables_in_store(executables, concrete_spec, query_info=info): self.last_search = info return True @@ -486,11 +524,10 @@ def ensure_module_importable_or_raise(module, abstract_spec=None): return abstract_spec = abstract_spec or module - source_configs = spack.config.get('bootstrap:sources', []) h = GroupedExceptionHandler() - for current_config in source_configs: + for current_config in bootstrapping_sources(): with h.forward(current_config['name']): _validate_source_is_trusted(current_config) @@ -529,11 +566,10 @@ def ensure_executables_in_path_or_raise(executables, abstract_spec): return cmd executables_str = ', '.join(executables) - source_configs = spack.config.get('bootstrap:sources', []) h = GroupedExceptionHandler() - for current_config in source_configs: + for current_config in bootstrapping_sources(): with h.forward(current_config['name']): _validate_source_is_trusted(current_config) @@ -818,6 +854,19 @@ def ensure_flake8_in_path_or_raise(): return ensure_executables_in_path_or_raise([executable], abstract_spec=root_spec) +def all_root_specs(development=False): + """Return a list of all the root specs that may be used to bootstrap Spack. + + Args: + development (bool): if True include dev dependencies + """ + specs = [clingo_root_spec(), gnupg_root_spec(), patchelf_root_spec()] + if development: + specs += [isort_root_spec(), mypy_root_spec(), + black_root_spec(), flake8_root_spec()] + return specs + + def _missing(name, purpose, system_only=True): """Message to be printed if an executable is not found""" msg = '[{2}] MISSING "{0}": {1}' @@ -955,3 +1004,23 @@ def status_message(section): msg += '\n' msg = msg.format(pass_token if not missing_software else fail_token) return msg, missing_software + + +def bootstrapping_sources(scope=None): + """Return the list of configured sources of software for bootstrapping Spack + + Args: + scope (str or None): if a valid configuration scope is given, return the + list only from that scope + """ + source_configs = spack.config.get('bootstrap:sources', default=None, scope=scope) + source_configs = source_configs or [] + list_of_sources = [] + for entry in source_configs: + current = copy.copy(entry) + metadata_dir = spack.util.path.canonicalize_path(entry['metadata']) + metadata_yaml = os.path.join(metadata_dir, METADATA_YAML_FILENAME) + with open(metadata_yaml) as f: + current.update(spack.util.spack_yaml.load(f)) + list_of_sources.append(current) + return list_of_sources diff --git a/lib/spack/spack/cmd/bootstrap.py b/lib/spack/spack/cmd/bootstrap.py index 070192a814..e1bb4b034f 100644 --- a/lib/spack/spack/cmd/bootstrap.py +++ b/lib/spack/spack/cmd/bootstrap.py @@ -6,7 +6,9 @@ from __future__ import print_function import os.path import shutil +import tempfile +import llnl.util.filesystem import llnl.util.tty import llnl.util.tty.color @@ -15,6 +17,9 @@ import spack.bootstrap import spack.cmd.common.arguments import spack.config import spack.main +import spack.mirror +import spack.spec +import spack.stage import spack.util.path description = "manage bootstrap configuration" @@ -22,6 +27,38 @@ section = "system" level = "long" +# Tarball to be downloaded if binary packages are requested in a local mirror +BINARY_TARBALL = 'https://github.com/spack/spack-bootstrap-mirrors/releases/download/v0.2/bootstrap-buildcache.tar.gz' + +#: Subdirectory where to create the mirror +LOCAL_MIRROR_DIR = 'bootstrap_cache' + +# Metadata for a generated binary mirror +BINARY_METADATA = { + 'type': 'buildcache', + 'description': ('Buildcache copied from a public tarball available on Github.' + 'The sha256 checksum of binaries is checked before installation.'), + 'info': { + 'url': os.path.join('..', '..', LOCAL_MIRROR_DIR), + 'homepage': 'https://github.com/spack/spack-bootstrap-mirrors', + 'releases': 'https://github.com/spack/spack-bootstrap-mirrors/releases', + 'tarball': BINARY_TARBALL + } +} + +CLINGO_JSON = '$spack/share/spack/bootstrap/github-actions-v0.2/clingo.json' +GNUPG_JSON = '$spack/share/spack/bootstrap/github-actions-v0.2/gnupg.json' + +# Metadata for a generated source mirror +SOURCE_METADATA = { + 'type': 'install', + 'description': 'Mirror with software needed to bootstrap Spack', + 'info': { + 'url': os.path.join('..', '..', LOCAL_MIRROR_DIR) + } +} + + def _add_scope_option(parser): scopes = spack.config.scopes() scopes_metavar = spack.config.scopes_metavar @@ -67,24 +104,61 @@ def setup_parser(subparser): ) list = sp.add_parser( - 'list', help='list the methods available for bootstrapping' + 'list', help='list all the sources of software to bootstrap Spack' ) _add_scope_option(list) trust = sp.add_parser( - 'trust', help='trust a bootstrapping method' + 'trust', help='trust a bootstrapping source' ) _add_scope_option(trust) trust.add_argument( - 'name', help='name of the method to be trusted' + 'name', help='name of the source to be trusted' ) untrust = sp.add_parser( - 'untrust', help='untrust a bootstrapping method' + 'untrust', help='untrust a bootstrapping source' ) _add_scope_option(untrust) untrust.add_argument( - 'name', help='name of the method to be untrusted' + 'name', help='name of the source to be untrusted' + ) + + add = sp.add_parser( + 'add', help='add a new source for bootstrapping' + ) + _add_scope_option(add) + add.add_argument( + '--trust', action='store_true', + help='trust the source immediately upon addition') + add.add_argument( + 'name', help='name of the new source of software' + ) + add.add_argument( + 'metadata_dir', help='directory where to find metadata files' + ) + + remove = sp.add_parser( + 'remove', help='remove a bootstrapping source' + ) + remove.add_argument( + 'name', help='name of the source to be removed' + ) + + mirror = sp.add_parser( + 'mirror', help='create a local mirror to bootstrap Spack' + ) + mirror.add_argument( + '--binary-packages', action='store_true', + help='download public binaries in the mirror' + ) + mirror.add_argument( + '--dev', action='store_true', + help='download dev dependencies too' + ) + mirror.add_argument( + metavar='DIRECTORY', dest='root_dir', + help='root directory in which to create the mirror and metadata' ) @@ -137,10 +211,7 @@ def _root(args): def _list(args): - sources = spack.config.get( - 'bootstrap:sources', default=None, scope=args.scope - ) - + sources = spack.bootstrap.bootstrapping_sources(scope=args.scope) if not sources: llnl.util.tty.msg( "No method available for bootstrapping Spack's dependencies" @@ -249,6 +320,119 @@ def _status(args): print() +def _add(args): + initial_sources = spack.bootstrap.bootstrapping_sources() + names = [s['name'] for s in initial_sources] + + # If the name is already used error out + if args.name in names: + msg = 'a source named "{0}" already exist. Please choose a different name' + raise RuntimeError(msg.format(args.name)) + + # Check that the metadata file exists + metadata_dir = spack.util.path.canonicalize_path(args.metadata_dir) + if not os.path.exists(metadata_dir) or not os.path.isdir(metadata_dir): + raise RuntimeError( + 'the directory "{0}" does not exist'.format(args.metadata_dir) + ) + + file = os.path.join(metadata_dir, 'metadata.yaml') + if not os.path.exists(file): + raise RuntimeError('the file "{0}" does not exist'.format(file)) + + # Insert the new source as the highest priority one + write_scope = args.scope or spack.config.default_modify_scope(section='bootstrap') + sources = spack.config.get('bootstrap:sources', scope=write_scope) or [] + sources = [ + {'name': args.name, 'metadata': args.metadata_dir} + ] + sources + spack.config.set('bootstrap:sources', sources, scope=write_scope) + + msg = 'New bootstrapping source "{0}" added in the "{1}" configuration scope' + llnl.util.tty.msg(msg.format(args.name, write_scope)) + if args.trust: + _trust(args) + + +def _remove(args): + initial_sources = spack.bootstrap.bootstrapping_sources() + names = [s['name'] for s in initial_sources] + if args.name not in names: + msg = ('cannot find any bootstrapping source named "{0}". ' + 'Run `spack bootstrap list` to see available sources.') + raise RuntimeError(msg.format(args.name)) + + for current_scope in spack.config.scopes(): + sources = spack.config.get('bootstrap:sources', scope=current_scope) or [] + if args.name in [s['name'] for s in sources]: + sources = [s for s in sources if s['name'] != args.name] + spack.config.set('bootstrap:sources', sources, scope=current_scope) + msg = ('Removed the bootstrapping source named "{0}" from the ' + '"{1}" configuration scope.') + llnl.util.tty.msg(msg.format(args.name, current_scope)) + trusted = spack.config.get('bootstrap:trusted', scope=current_scope) or [] + if args.name in trusted: + trusted.pop(args.name) + spack.config.set('bootstrap:trusted', trusted, scope=current_scope) + msg = 'Deleting information on "{0}" from list of trusted sources' + llnl.util.tty.msg(msg.format(args.name)) + + +def _mirror(args): + mirror_dir = os.path.join(args.root_dir, LOCAL_MIRROR_DIR) + + # TODO: Here we are adding gnuconfig manually, but this can be fixed + # TODO: as soon as we have an option to add to a mirror all the possible + # TODO: dependencies of a spec + root_specs = spack.bootstrap.all_root_specs(development=args.dev) + ['gnuconfig'] + for spec_str in root_specs: + msg = 'Adding "{0}" and dependencies to the mirror at {1}' + llnl.util.tty.msg(msg.format(spec_str, mirror_dir)) + # Suppress tty from the call below for terser messages + llnl.util.tty.set_msg_enabled(False) + spec = spack.spec.Spec(spec_str).concretized() + for node in spec.traverse(): + spack.mirror.create(mirror_dir, [node]) + llnl.util.tty.set_msg_enabled(True) + + if args.binary_packages: + msg = 'Adding binary packages from "{0}" to the mirror at {1}' + llnl.util.tty.msg(msg.format(BINARY_TARBALL, mirror_dir)) + llnl.util.tty.set_msg_enabled(False) + stage = spack.stage.Stage(BINARY_TARBALL, path=tempfile.mkdtemp()) + stage.create() + stage.fetch() + stage.expand_archive() + build_cache_dir = os.path.join(stage.source_path, 'build_cache') + shutil.move(build_cache_dir, mirror_dir) + llnl.util.tty.set_msg_enabled(True) + + def write_metadata(subdir, metadata): + metadata_rel_dir = os.path.join('metadata', subdir) + metadata_yaml = os.path.join( + args.root_dir, metadata_rel_dir, 'metadata.yaml' + ) + llnl.util.filesystem.mkdirp(os.path.dirname(metadata_yaml)) + with open(metadata_yaml, mode='w') as f: + spack.util.spack_yaml.dump(metadata, stream=f) + return os.path.dirname(metadata_yaml), metadata_rel_dir + + instructions = ('\nTo register the mirror on the platform where it\'s supposed ' + 'to be used, move "{0}" to its final location and run the ' + 'following command(s):\n\n').format(args.root_dir) + cmd = ' % spack bootstrap add --trust {0} <final-path>/{1}\n' + _, rel_directory = write_metadata(subdir='sources', metadata=SOURCE_METADATA) + instructions += cmd.format('local-sources', rel_directory) + if args.binary_packages: + abs_directory, rel_directory = write_metadata( + subdir='binaries', metadata=BINARY_METADATA + ) + shutil.copy(spack.util.path.canonicalize_path(CLINGO_JSON), abs_directory) + shutil.copy(spack.util.path.canonicalize_path(GNUPG_JSON), abs_directory) + instructions += cmd.format('local-binaries', rel_directory) + print(instructions) + + def bootstrap(parser, args): callbacks = { 'status': _status, @@ -258,6 +442,9 @@ def bootstrap(parser, args): 'root': _root, 'list': _list, 'trust': _trust, - 'untrust': _untrust + 'untrust': _untrust, + 'add': _add, + 'remove': _remove, + 'mirror': _mirror } callbacks[args.subcommand](args) diff --git a/lib/spack/spack/schema/bootstrap.py b/lib/spack/spack/schema/bootstrap.py index b2f945f597..ee5cf98a9a 100644 --- a/lib/spack/spack/schema/bootstrap.py +++ b/lib/spack/spack/schema/bootstrap.py @@ -9,12 +9,10 @@ _source_schema = { 'type': 'object', 'properties': { 'name': {'type': 'string'}, - 'description': {'type': 'string'}, - 'type': {'type': 'string'}, - 'info': {'type': 'object'} + 'metadata': {'type': 'string'} }, 'additionalProperties': False, - 'required': ['name', 'description', 'type'] + 'required': ['name', 'metadata'] } properties = { diff --git a/lib/spack/spack/test/cmd/bootstrap.py b/lib/spack/spack/test/cmd/bootstrap.py index 6419829f69..e4cace7d89 100644 --- a/lib/spack/spack/test/cmd/bootstrap.py +++ b/lib/spack/spack/test/cmd/bootstrap.py @@ -10,6 +10,7 @@ import pytest import spack.config import spack.environment as ev import spack.main +import spack.mirror from spack.util.path import convert_to_posix_path _bootstrap = spack.main.SpackCommand('bootstrap') @@ -139,13 +140,82 @@ def test_trust_or_untrust_fails_with_no_method(mutable_config): def test_trust_or_untrust_fails_with_more_than_one_method(mutable_config): wrong_config = {'sources': [ {'name': 'github-actions', - 'type': 'buildcache', - 'description': ''}, + 'metadata': '$spack/share/spack/bootstrap/github-actions'}, {'name': 'github-actions', - 'type': 'buildcache', - 'description': 'Another entry'}], + 'metadata': '$spack/share/spack/bootstrap/github-actions'}], 'trusted': {} } with spack.config.override('bootstrap', wrong_config): with pytest.raises(RuntimeError, match='more than one'): _bootstrap('trust', 'github-actions') + + +@pytest.mark.parametrize('use_existing_dir', [True, False]) +def test_add_failures_for_non_existing_files(mutable_config, tmpdir, use_existing_dir): + metadata_dir = str(tmpdir) if use_existing_dir else '/foo/doesnotexist' + with pytest.raises(RuntimeError, match='does not exist'): + _bootstrap('add', 'mock-mirror', metadata_dir) + + +def test_add_failures_for_already_existing_name(mutable_config): + with pytest.raises(RuntimeError, match='already exist'): + _bootstrap('add', 'github-actions', 'some-place') + + +def test_remove_failure_for_non_existing_names(mutable_config): + with pytest.raises(RuntimeError, match='cannot find'): + _bootstrap('remove', 'mock-mirror') + + +def test_remove_and_add_a_source(mutable_config): + # Check we start with a single bootstrapping source + sources = spack.bootstrap.bootstrapping_sources() + assert len(sources) == 1 + + # Remove it and check the result + _bootstrap('remove', 'github-actions') + sources = spack.bootstrap.bootstrapping_sources() + assert not sources + + # Add it back and check we restored the initial state + _bootstrap( + 'add', 'github-actions', '$spack/share/spack/bootstrap/github-actions-v0.2' + ) + sources = spack.bootstrap.bootstrapping_sources() + assert len(sources) == 1 + + +@pytest.mark.maybeslow +@pytest.mark.skipif(sys.platform == 'win32', reason="Not supported on Windows (yet)") +def test_bootstrap_mirror_metadata(mutable_config, linux_os, monkeypatch, tmpdir): + """Test that `spack bootstrap mirror` creates a folder that can be ingested by + `spack bootstrap add`. Here we don't download data, since that would be an + expensive operation for a unit test. + """ + old_create = spack.mirror.create + monkeypatch.setattr(spack.mirror, 'create', lambda p, s: old_create(p, [])) + + # Create the mirror in a temporary folder + compilers = [{ + 'compiler': { + 'spec': 'gcc@12.0.1', + 'operating_system': '{0.name}{0.version}'.format(linux_os), + 'modules': [], + 'paths': { + 'cc': '/usr/bin', + 'cxx': '/usr/bin', + 'fc': '/usr/bin', + 'f77': '/usr/bin' + } + } + }] + with spack.config.override('compilers', compilers): + _bootstrap('mirror', str(tmpdir)) + + # Register the mirror + metadata_dir = tmpdir.join('metadata', 'sources') + _bootstrap('add', '--trust', 'test-mirror', str(metadata_dir)) + + assert _bootstrap.returncode == 0 + assert any(m['name'] == 'test-mirror' + for m in spack.bootstrap.bootstrapping_sources()) diff --git a/lib/spack/spack/test/data/config/bootstrap.yaml b/lib/spack/spack/test/data/config/bootstrap.yaml index 5ecff745cf..8929d7ff35 100644 --- a/lib/spack/spack/test/data/config/bootstrap.yaml +++ b/lib/spack/spack/test/data/config/bootstrap.yaml @@ -1,12 +1,5 @@ bootstrap: sources: - name: 'github-actions' - type: buildcache - description: | - Buildcache generated from a public workflow using Github Actions. - The sha256 checksum of binaries is checked before installation. - info: - url: file:///home/spack/production/spack/mirrors/clingo - homepage: https://github.com/alalazo/spack-bootstrap-mirrors - releases: https://github.com/alalazo/spack-bootstrap-mirrors/releases + metadata: $spack/share/spack/bootstrap/github-actions-v0.2 trusted: {} diff --git a/share/spack/bootstrap/github-actions-v0.1/metadata.yaml b/share/spack/bootstrap/github-actions-v0.1/metadata.yaml new file mode 100644 index 0000000000..b2439424b0 --- /dev/null +++ b/share/spack/bootstrap/github-actions-v0.1/metadata.yaml @@ -0,0 +1,8 @@ +type: buildcache +description: | + Buildcache generated from a public workflow using Github Actions. + The sha256 checksum of binaries is checked before installation. +info: + url: https://mirror.spack.io/bootstrap/github-actions/v0.1 + homepage: https://github.com/spack/spack-bootstrap-mirrors + releases: https://github.com/spack/spack-bootstrap-mirrors/releases diff --git a/share/spack/bootstrap/github-actions-v0.2/metadata.yaml b/share/spack/bootstrap/github-actions-v0.2/metadata.yaml new file mode 100644 index 0000000000..f786731aa8 --- /dev/null +++ b/share/spack/bootstrap/github-actions-v0.2/metadata.yaml @@ -0,0 +1,8 @@ +type: buildcache +description: | + Buildcache generated from a public workflow using Github Actions. + The sha256 checksum of binaries is checked before installation. +info: + url: https://mirror.spack.io/bootstrap/github-actions/v0.2 + homepage: https://github.com/spack/spack-bootstrap-mirrors + releases: https://github.com/spack/spack-bootstrap-mirrors/releases diff --git a/share/spack/bootstrap/spack-install/metadata.yaml b/share/spack/bootstrap/spack-install/metadata.yaml new file mode 100644 index 0000000000..c8ecaeb7e6 --- /dev/null +++ b/share/spack/bootstrap/spack-install/metadata.yaml @@ -0,0 +1,8 @@ +# This method is just Spack bootstrapping the software it needs from sources. +# It has been added here so that users can selectively disable bootstrapping +# from sources by "untrusting" it. +type: install +description: | + Specs built from sources downloaded from the Spack public mirror. +info: + url: https://mirror.spack.io diff --git a/share/spack/spack-completion.bash b/share/spack/spack-completion.bash index bfc44c444b..5e4498f395 100755 --- a/share/spack/spack-completion.bash +++ b/share/spack/spack-completion.bash @@ -434,7 +434,7 @@ _spack_bootstrap() { then SPACK_COMPREPLY="-h --help" else - SPACK_COMPREPLY="status enable disable reset root list trust untrust" + SPACK_COMPREPLY="status enable disable reset root list trust untrust add remove mirror" fi } @@ -485,6 +485,33 @@ _spack_bootstrap_untrust() { fi } +_spack_bootstrap_add() { + if $list_options + then + SPACK_COMPREPLY="-h --help --scope --trust" + else + SPACK_COMPREPLY="" + fi +} + +_spack_bootstrap_remove() { + if $list_options + then + SPACK_COMPREPLY="-h --help" + else + SPACK_COMPREPLY="" + fi +} + +_spack_bootstrap_mirror() { + if $list_options + then + SPACK_COMPREPLY="-h --help --binary-packages --dev" + else + SPACK_COMPREPLY="" + fi +} + _spack_build_env() { if $list_options then |