From 350372e3bf9c1ac5de2e7e74a3ccd8a46f7e1d5d Mon Sep 17 00:00:00 2001 From: Scott Wittenburg Date: Thu, 19 Aug 2021 12:15:40 -0600 Subject: buildcache: Add environment-aware buildcache sync command (#25470) --- lib/spack/spack/cmd/buildcache.py | 151 +++++++++++++++++++++++++++++++++ lib/spack/spack/test/cmd/buildcache.py | 77 +++++++++++++++++ share/spack/spack-completion.bash | 6 +- 3 files changed, 233 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/cmd/buildcache.py b/lib/spack/spack/cmd/buildcache.py index 601b8b1476..8fc0b6a04e 100644 --- a/lib/spack/spack/cmd/buildcache.py +++ b/lib/spack/spack/cmd/buildcache.py @@ -6,6 +6,7 @@ import argparse import os import shutil import sys +import tempfile import llnl.util.tty as tty @@ -15,6 +16,7 @@ import spack.cmd import spack.cmd.common.arguments as arguments import spack.config import spack.environment as ev +import spack.fetch_strategy as fs import spack.hash_types as ht import spack.mirror import spack.relocate @@ -23,9 +25,11 @@ import spack.spec import spack.store import spack.util.crypto import spack.util.url as url_util +import spack.util.web as web_util from spack.cmd import display_specs from spack.error import SpecError from spack.spec import Spec, save_dependency_spec_yamls +from spack.stage import Stage from spack.util.string import plural description = "create, download and install binary packages" @@ -226,6 +230,36 @@ def setup_parser(subparser): help='Destination mirror url') copy.set_defaults(func=buildcache_copy) + # Sync buildcache entries from one mirror to another + sync = subparsers.add_parser('sync', help=buildcache_sync.__doc__) + source = sync.add_mutually_exclusive_group(required=True) + source.add_argument('--src-directory', + metavar='DIRECTORY', + type=str, + help="Source mirror as a local file path") + source.add_argument('--src-mirror-name', + metavar='MIRROR_NAME', + type=str, + help="Name of the source mirror") + source.add_argument('--src-mirror-url', + metavar='MIRROR_URL', + type=str, + help="URL of the source mirror") + dest = sync.add_mutually_exclusive_group(required=True) + dest.add_argument('--dest-directory', + metavar='DIRECTORY', + type=str, + help="Destination mirror as a local file path") + dest.add_argument('--dest-mirror-name', + metavar='MIRROR_NAME', + type=str, + help="Name of the destination mirror") + dest.add_argument('--dest-mirror-url', + metavar='MIRROR_URL', + type=str, + help="URL of the destination mirror") + sync.set_defaults(func=buildcache_sync) + # Update buildcache index without copying any additional packages update_index = subparsers.add_parser( 'update-index', help=buildcache_update_index.__doc__) @@ -779,6 +813,123 @@ def buildcache_copy(args): shutil.copyfile(cdashid_src_path, cdashid_dest_path) +def buildcache_sync(args): + """ Syncs binaries (and associated metadata) from one mirror to another. + Requires an active environment in order to know which specs to sync. + + Args: + src (str): Source mirror URL + dest (str): Destination mirror URL + """ + # Figure out the source mirror + source_location = None + if args.src_directory: + source_location = args.src_directory + scheme = url_util.parse(source_location, scheme='').scheme + if scheme != '': + raise ValueError( + '"--src-directory" expected a local path; got a URL, instead') + # Ensure that the mirror lookup does not mistake this for named mirror + source_location = 'file://' + source_location + elif args.src_mirror_name: + source_location = args.src_mirror_name + result = spack.mirror.MirrorCollection().lookup(source_location) + if result.name == "": + raise ValueError( + 'no configured mirror named "{name}"'.format( + name=source_location)) + elif args.src_mirror_url: + source_location = args.src_mirror_url + scheme = url_util.parse(source_location, scheme='').scheme + if scheme == '': + raise ValueError( + '"{url}" is not a valid URL'.format(url=source_location)) + + src_mirror = spack.mirror.MirrorCollection().lookup(source_location) + src_mirror_url = url_util.format(src_mirror.fetch_url) + + # Figure out the destination mirror + dest_location = None + if args.dest_directory: + dest_location = args.dest_directory + scheme = url_util.parse(dest_location, scheme='').scheme + if scheme != '': + raise ValueError( + '"--dest-directory" expected a local path; got a URL, instead') + # Ensure that the mirror lookup does not mistake this for named mirror + dest_location = 'file://' + dest_location + elif args.dest_mirror_name: + dest_location = args.dest_mirror_name + result = spack.mirror.MirrorCollection().lookup(dest_location) + if result.name == "": + raise ValueError( + 'no configured mirror named "{name}"'.format( + name=dest_location)) + elif args.dest_mirror_url: + dest_location = args.dest_mirror_url + scheme = url_util.parse(dest_location, scheme='').scheme + if scheme == '': + raise ValueError( + '"{url}" is not a valid URL'.format(url=dest_location)) + + dest_mirror = spack.mirror.MirrorCollection().lookup(dest_location) + dest_mirror_url = url_util.format(dest_mirror.fetch_url) + + # Get the active environment + env = ev.get_env(args, 'buildcache sync', required=True) + + tty.msg('Syncing environment buildcache files from {0} to {1}'.format( + src_mirror_url, dest_mirror_url)) + + build_cache_dir = bindist.build_cache_relative_path() + buildcache_rel_paths = [] + + tty.debug('Syncing the following specs:') + for s in env.all_specs(): + tty.debug(' {0}{1}: {2}'.format( + '* ' if s in env.roots() else ' ', s.name, s.dag_hash())) + + buildcache_rel_paths.extend([ + os.path.join( + build_cache_dir, bindist.tarball_path_name(s, '.spack')), + os.path.join( + build_cache_dir, bindist.tarball_name(s, '.spec.yaml')), + os.path.join( + build_cache_dir, bindist.tarball_name(s, '.cdashid')) + ]) + + tmpdir = tempfile.mkdtemp() + + try: + for rel_path in buildcache_rel_paths: + src_url = url_util.join(src_mirror_url, rel_path) + local_path = os.path.join(tmpdir, rel_path) + dest_url = url_util.join(dest_mirror_url, rel_path) + + tty.debug('Copying {0} to {1} via {2}'.format( + src_url, dest_url, local_path)) + + stage = Stage(src_url, + name="temporary_file", + path=os.path.dirname(local_path), + keep=True) + + try: + stage.create() + stage.fetch() + web_util.push_to_url( + local_path, + dest_url, + keep_original=True) + except fs.FetchError as e: + tty.debug('spack buildcache unable to sync {0}'.format(rel_path)) + tty.debug(e) + finally: + stage.destroy() + finally: + shutil.rmtree(tmpdir) + + def update_index(mirror_url, update_keys=False): mirror = spack.mirror.MirrorCollection().lookup(mirror_url) outdir = url_util.format(mirror.push_url) diff --git a/lib/spack/spack/test/cmd/buildcache.py b/lib/spack/spack/test/cmd/buildcache.py index d67f709a6b..bf615634e7 100644 --- a/lib/spack/spack/test/cmd/buildcache.py +++ b/lib/spack/spack/test/cmd/buildcache.py @@ -6,6 +6,7 @@ import errno import os import platform +import shutil import pytest @@ -172,3 +173,79 @@ def test_update_key_index(tmpdir, mutable_mock_env_path, mirror('rm', 'test-mirror') assert 'index.json' in key_dir_list + + +def test_buildcache_sync(mutable_mock_env_path, install_mockery_mutable_config, + mock_packages, mock_fetch, mock_stage, tmpdir): + """ + Make sure buildcache sync works in an environment-aware manner, ignoring + any specs that may be in the mirror but not in the environment. + """ + working_dir = tmpdir.join('working_dir') + + src_mirror_dir = working_dir.join('src_mirror').strpath + src_mirror_url = 'file://{0}'.format(src_mirror_dir) + + dest_mirror_dir = working_dir.join('dest_mirror').strpath + dest_mirror_url = 'file://{0}'.format(dest_mirror_dir) + + in_env_pkg = 'trivial-install-test-package' + out_env_pkg = 'libdwarf' + + def verify_mirror_contents(): + dest_list = os.listdir( + os.path.join(dest_mirror_dir, 'build_cache')) + + found_pkg = False + + for p in dest_list: + assert(out_env_pkg not in p) + if in_env_pkg in p: + found_pkg = True + + if not found_pkg: + print('Expected to find {0} in {1}'.format( + in_env_pkg, dest_mirror_dir)) + assert(False) + + # Install a package and put it in the buildcache + s = Spec(out_env_pkg).concretized() + install(s.name) + buildcache( + 'create', '-u', '-f', '-a', '--mirror-url', src_mirror_url, s.name) + + env('create', 'test') + with ev.read('test'): + add(in_env_pkg) + install() + buildcache( + 'create', '-u', '-f', '-a', '--mirror-url', src_mirror_url, in_env_pkg) + + # Now run the spack buildcache sync command with all the various options + # for specifying mirrors + + # Use urls to specify mirrors + buildcache('sync', + '--src-mirror-url', src_mirror_url, + '--dest-mirror-url', dest_mirror_url) + + verify_mirror_contents() + shutil.rmtree(dest_mirror_dir) + + # Use local directory paths to specify fs locations + buildcache('sync', + '--src-directory', src_mirror_dir, + '--dest-directory', dest_mirror_dir) + + verify_mirror_contents() + shutil.rmtree(dest_mirror_dir) + + # Use mirror names to specify mirrors + mirror('add', 'src', src_mirror_url) + mirror('add', 'dest', dest_mirror_url) + + buildcache('sync', + '--src-mirror-name', 'src', + '--dest-mirror-name', 'dest') + + verify_mirror_contents() diff --git a/share/spack/spack-completion.bash b/share/spack/spack-completion.bash index 1281ca7349..ef15f38c6b 100755 --- a/share/spack/spack-completion.bash +++ b/share/spack/spack-completion.bash @@ -482,7 +482,7 @@ _spack_buildcache() { then SPACK_COMPREPLY="-h --help" else - SPACK_COMPREPLY="create install list keys preview check download get-buildcache-name save-yaml copy update-index" + SPACK_COMPREPLY="create install list keys preview check download get-buildcache-name save-yaml copy sync update-index" fi } @@ -546,6 +546,10 @@ _spack_buildcache_copy() { SPACK_COMPREPLY="-h --help --base-dir --spec-yaml --destination-url" } +_spack_buildcache_sync() { + SPACK_COMPREPLY="-h --help --src-directory --src-mirror-name --src-mirror-url --dest-directory --dest-mirror-name --dest-mirror-url" +} + _spack_buildcache_update_index() { SPACK_COMPREPLY="-h --help -d --mirror-url -k --keys" } -- cgit v1.2.3-60-g2f50