diff options
author | kwryankrattiger <80296582+kwryankrattiger@users.noreply.github.com> | 2022-09-29 13:48:06 -0500 |
---|---|---|
committer | GitHub <noreply@github.com> | 2022-09-29 13:48:06 -0500 |
commit | a01c36da455d0911c240152cf4b4b980b45bb60d (patch) | |
tree | f992733a73aae906529be5559b02a5862b0a8138 | |
parent | 7a25f416b8c641e8d3eb004d9c6349a647bab531 (diff) | |
download | spack-a01c36da455d0911c240152cf4b4b980b45bb60d.tar.gz spack-a01c36da455d0911c240152cf4b4b980b45bb60d.tar.bz2 spack-a01c36da455d0911c240152cf4b4b980b45bb60d.tar.xz spack-a01c36da455d0911c240152cf4b4b980b45bb60d.zip |
Install: Add use-buildcache option to install (#32537)
Install: Add use-buildcache option to install
* Allow differentiating between top level packages and dependencies when
determining whether to install from the cache or not.
* Add unit test for --use-buildcache
* Use metavar to display use-buildcache options.
* Update spack-completion
-rw-r--r-- | lib/spack/spack/cmd/install.py | 59 | ||||
-rw-r--r-- | lib/spack/spack/installer.py | 36 | ||||
-rw-r--r-- | lib/spack/spack/test/cmd/install.py | 79 | ||||
-rwxr-xr-x | share/spack/spack-completion.bash | 2 |
4 files changed, 167 insertions, 9 deletions
diff --git a/lib/spack/spack/cmd/install.py b/lib/spack/spack/cmd/install.py index dd3d5b8c9c..a6fdfd31f4 100644 --- a/lib/spack/spack/cmd/install.py +++ b/lib/spack/spack/cmd/install.py @@ -5,6 +5,7 @@ import argparse import os +import re import shutil import sys import textwrap @@ -31,10 +32,50 @@ section = "build" level = "short" +# Pass in the value string passed to use-buildcache and get back +# the package and dependencies values. +def parse_use_buildcache(opt): + bc_keys = ["package:", "dependencies:", ""] + bc_values = ["only", "never", "auto"] + kv_list = re.findall("([a-z]+:)?([a-z]+)", opt) + + # Verify keys and values + bc_map = {k: v for k, v in kv_list if k in bc_keys and v in bc_values} + if not len(kv_list) == len(bc_map): + tty.error("Unrecognized arguments passed to use-buildcache") + tty.error( + "Expected: --use-buildcache " + "[[auto|only|never],[package:[auto|only|never]],[dependencies:[auto|only|never]]]" + ) + exit(1) + + for _group in ["package:", "dependencies:"]: + if _group not in bc_map: + if "" in bc_map: + bc_map[_group] = bc_map[""] + else: + bc_map[_group] = "auto" + + return bc_map["package:"], bc_map["dependencies:"] + + +# Determine value of cache flag +def cache_opt(default_opt, use_buildcache): + if use_buildcache == "auto": + return default_opt + elif use_buildcache == "only": + return True + elif use_buildcache == "never": + return False + + def install_kwargs_from_args(args): """Translate command line arguments into a dictionary that will be passed to the package installer. """ + + pkg_use_bc, dep_use_bc = parse_use_buildcache(args.use_buildcache) + return { "fail_fast": args.fail_fast, "keep_prefix": args.keep_prefix, @@ -44,8 +85,10 @@ def install_kwargs_from_args(args): "verbose": args.verbose or args.install_verbose, "fake": args.fake, "dirty": args.dirty, - "use_cache": args.use_cache, - "cache_only": args.cache_only, + "package_use_cache": cache_opt(args.use_cache, pkg_use_bc), + "package_cache_only": cache_opt(args.cache_only, pkg_use_bc), + "dependencies_use_cache": cache_opt(args.use_cache, dep_use_bc), + "dependencies_cache_only": cache_opt(args.cache_only, dep_use_bc), "include_build_deps": args.include_build_deps, "explicit": True, # Use true as a default for install command "stop_at": args.until, @@ -123,6 +166,18 @@ the dependencies""", default=False, help="only install package from binary mirrors", ) + cache_group.add_argument( + "--use-buildcache", + dest="use_buildcache", + default="package:auto,dependencies:auto", + metavar="[{auto,only,never},][package:{auto,only,never},][dependencies:{auto,only,never}]", + help="""select the mode of buildcache for the 'package' and 'dependencies'. +Default: package:auto,dependencies:auto +- `auto` behaves like --use-cache +- `only` behaves like --cache-only +- `never` behaves like --no-cache +""", + ) subparser.add_argument( "--include-build-deps", diff --git a/lib/spack/spack/installer.py b/lib/spack/spack/installer.py index 2f0415b0c1..4150302b28 100644 --- a/lib/spack/spack/installer.py +++ b/lib/spack/spack/installer.py @@ -1182,12 +1182,12 @@ class PackageInstaller(object): Args: task (BuildTask): the installation build task for a package""" - install_args = task.request.install_args - cache_only = install_args.get("cache_only") explicit = task.explicit + install_args = task.request.install_args + cache_only = task.cache_only + use_cache = task.use_cache tests = install_args.get("tests") unsigned = install_args.get("unsigned") - use_cache = install_args.get("use_cache") pkg, pkg_id = task.pkg, task.pkg_id @@ -2220,7 +2220,29 @@ class BuildTask(object): @property def explicit(self): """The package was explicitly requested by the user.""" - return self.pkg == self.request.pkg and self.request.install_args.get("explicit", True) + return self.is_root and self.request.install_args.get("explicit", True) + + @property + def is_root(self): + """The package was requested directly, but may or may not be explicit + in an environment.""" + return self.pkg == self.request.pkg + + @property + def use_cache(self): + _use_cache = True + if self.is_root: + return self.request.install_args.get("package_use_cache", _use_cache) + else: + return self.request.install_args.get("dependencies_use_cache", _use_cache) + + @property + def cache_only(self): + _cache_only = False + if self.is_root: + return self.request.install_args.get("package_cache_only", _cache_only) + else: + return self.request.install_args.get("dependencies_cache_only", _cache_only) @property def key(self): @@ -2302,21 +2324,23 @@ class BuildRequest(object): def _add_default_args(self): """Ensure standard install options are set to at least the default.""" for arg, default in [ - ("cache_only", False), ("context", "build"), # installs *always* build + ("dependencies_cache_only", False), + ("dependencies_use_cache", True), ("dirty", False), ("fail_fast", False), ("fake", False), ("install_deps", True), ("install_package", True), ("install_source", False), + ("package_cache_only", False), + ("package_use_cache", True), ("keep_prefix", False), ("keep_stage", False), ("restage", False), ("skip_patch", False), ("tests", False), ("unsigned", False), - ("use_cache", True), ("verbose", False), ]: _ = self.install_args.setdefault(arg, default) diff --git a/lib/spack/spack/test/cmd/install.py b/lib/spack/spack/test/cmd/install.py index 91ef154001..e74f01db7f 100644 --- a/lib/spack/spack/test/cmd/install.py +++ b/lib/spack/spack/test/cmd/install.py @@ -5,6 +5,7 @@ import argparse import filecmp +import itertools import os import re import sys @@ -14,6 +15,7 @@ import pytest from six.moves import builtins import llnl.util.filesystem as fs +import llnl.util.tty as tty import spack.cmd.install import spack.compilers as compilers @@ -1090,3 +1092,80 @@ def test_install_callbacks_fail(install_mockery, mock_fetch, name, method): assert output.count(method) == 2 assert output.count("method not implemented") == 1 assert output.count("TestFailure: 1 tests failed") == 1 + + +def test_install_use_buildcache( + capsys, + mock_packages, + mock_fetch, + mock_archive, + mock_binary_index, + tmpdir, + install_mockery_mutable_config, +): + """ + Make sure installing with use-buildcache behaves correctly. + """ + + package_name = "dependent-install" + dependency_name = "dependency-install" + + def validate(mode, out, pkg): + def assert_auto(pkg, out): + assert "==> Extracting {0}".format(pkg) in out + + def assert_only(pkg, out): + assert "==> Extracting {0}".format(pkg) in out + + def assert_never(pkg, out): + assert "==> {0}: Executing phase: 'install'".format(pkg) in out + + if mode == "auto": + assert_auto(pkg, out) + elif mode == "only": + assert_only(pkg, out) + else: + assert_never(pkg, out) + + def install_use_buildcache(opt): + out = install( + "--no-check-signature", "--use-buildcache", opt, package_name, fail_on_error=True + ) + + pkg_opt, dep_opt = spack.cmd.install.parse_use_buildcache(opt) + validate(dep_opt, out, dependency_name) + validate(pkg_opt, out, package_name) + + # Clean up installed packages + uninstall("-y", "-a") + + # Setup the mirror + # Create a temp mirror directory for buildcache usage + mirror_dir = tmpdir.join("mirror_dir") + mirror_url = "file://{0}".format(mirror_dir.strpath) + + # Populate the buildcache + install(package_name) + buildcache("create", "-u", "-a", "-f", "-d", mirror_dir.strpath, package_name, dependency_name) + + # Uninstall the all of the packages for clean slate + uninstall("-y", "-a") + + # Configure the mirror where we put that buildcache w/ the compiler + mirror("add", "test-mirror", mirror_url) + + with capsys.disabled(): + # Install using the matrix of possible combinations with --use-buildcache + for pkg, deps in itertools.product(["auto", "only", "never"], repeat=2): + tty.debug( + "Testing `spack install --use-buildcache package:{0},dependencies:{1}`".format( + pkg, deps + ) + ) + install_use_buildcache("package:{0},dependencies:{1}".format(pkg, deps)) + install_use_buildcache("dependencies:{0},package:{1}".format(deps, pkg)) + + # Install using a default override option + # Alternative to --cache-only (always) or --no-cache (never) + for opt in ["auto", "only", "never"]: + install_use_buildcache(opt) diff --git a/share/spack/spack-completion.bash b/share/spack/spack-completion.bash index c91b888f09..1543cffd92 100755 --- a/share/spack/spack-completion.bash +++ b/share/spack/spack-completion.bash @@ -1188,7 +1188,7 @@ _spack_info() { _spack_install() { if $list_options then - SPACK_COMPREPLY="-h --help --only -u --until -j --jobs --overwrite --fail-fast --keep-prefix --keep-stage --dont-restage --use-cache --no-cache --cache-only --include-build-deps --no-check-signature --show-log-on-error --source -n --no-checksum --deprecated -v --verbose --fake --only-concrete --no-add -f --file --clean --dirty --test --log-format --log-file --help-cdash --cdash-upload-url --cdash-build --cdash-site --cdash-track --cdash-buildstamp -y --yes-to-all -U --fresh --reuse" + SPACK_COMPREPLY="-h --help --only -u --until -j --jobs --overwrite --fail-fast --keep-prefix --keep-stage --dont-restage --use-cache --no-cache --cache-only --use-buildcache --include-build-deps --no-check-signature --show-log-on-error --source -n --no-checksum --deprecated -v --verbose --fake --only-concrete --no-add -f --file --clean --dirty --test --log-format --log-file --help-cdash --cdash-upload-url --cdash-build --cdash-site --cdash-track --cdash-buildstamp -y --yes-to-all -U --fresh --reuse" else _all_packages fi |