diff options
author | Harmen Stoppels <harmenstoppels@gmail.com> | 2023-01-18 19:19:46 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2023-01-18 19:19:46 +0100 |
commit | f050b1cf7835fd31992b020e1061c52294ff7330 (patch) | |
tree | 150ca0a67be1b63d415146436e55b164f899f03d /lib | |
parent | 6cf32110b912056201d5a901ee2e99979dc601bf (diff) | |
download | spack-f050b1cf7835fd31992b020e1061c52294ff7330.tar.gz spack-f050b1cf7835fd31992b020e1061c52294ff7330.tar.bz2 spack-f050b1cf7835fd31992b020e1061c52294ff7330.tar.xz spack-f050b1cf7835fd31992b020e1061c52294ff7330.zip |
depfile: variable with all identifiers (#34678)
With the new variable [prefix/]SPACK_PACKAGE_IDS you can conveniently execute
things after each successful install.
For example push just-built packages to a buildcache
```
SPACK ?= spack
export SPACK_COLOR = always
MAKEFLAGS += -Orecurse
MY_BUILDCACHE := $(CURDIR)/cache
.PHONY: all clean
all: push
ifeq (,$(filter clean,$(MAKECMDGOALS)))
include env.mk
endif
# the relevant part: push has *all* example/push/<pkg identifier> as prereqs
push: $(addprefix example/push/,$(example/SPACK_PACKAGE_IDS))
$(SPACK) -e . buildcache update-index --directory $(MY_BUILDCACHE)
$(info Pushed everything, yay!)
# and each example/push/<pkg identifier> has the install target as prereq,
# and the body can use target local $(HASH) and $(SPEC) variables to do
# things, such as pushing to a build cache
example/push/%: example/install/%
@mkdir -p $(dir $@)
$(SPACK) -e . buildcache create --allow-root --only=package --unsigned --directory $(MY_BUILDCACHE) /$(HASH) # push $(SPEC)
@touch $@
spack.lock: spack.yaml
$(SPACK) -e . concretize -f
env.mk: spack.lock
$(SPACK) -e . env depfile -o $@ --make-target-prefix example
clean:
rm -rf spack.lock env.mk example/
``
Diffstat (limited to 'lib')
-rw-r--r-- | lib/spack/docs/environments.rst | 50 | ||||
-rw-r--r-- | lib/spack/spack/cmd/env.py | 15 | ||||
-rw-r--r-- | lib/spack/spack/test/cmd/env.py | 48 |
3 files changed, 112 insertions, 1 deletions
diff --git a/lib/spack/docs/environments.rst b/lib/spack/docs/environments.rst index 5c8b6b2fb4..ac49f7a523 100644 --- a/lib/spack/docs/environments.rst +++ b/lib/spack/docs/environments.rst @@ -1089,4 +1089,52 @@ output (``spack install --verbose``) while its dependencies are installed silent $ make -j16 install-deps/python-3.11.0-<hash> SPACK_INSTALL_FLAGS=--show-log-on-error # Install the root spec with verbose output. - $ make -j16 install/python-3.11.0-<hash> SPACK_INSTALL_FLAGS=--verbose
\ No newline at end of file + $ make -j16 install/python-3.11.0-<hash> SPACK_INSTALL_FLAGS=--verbose + +^^^^^^^^^^^^^^^^^^^^^^^^^ +Adding post-install hooks +^^^^^^^^^^^^^^^^^^^^^^^^^ + +Another advanced use-case of generated ``Makefile``\s is running a post-install +command for each package. These "hooks" could be anything from printing a +post-install message, running tests, or pushing just-built binaries to a buildcache. + +This can be accomplished through the generated ``[<prefix>/]SPACK_PACKAGE_IDS`` +variable. Assuming we have an active and concrete environment, we generate the +associated ``Makefile`` with a prefix ``example``: + +.. code:: console + + $ spack env depfile -o env.mk --make-target-prefix example + +And we now include it in a different ``Makefile``, in which we create a target +``example/push/%`` with ``%`` referring to a package identifier. This target +depends on the particular package installation. In this target we automatically +have the target-specific ``HASH`` and ``SPEC`` variables at our disposal. They +are respectively the spec hash (excluding leading ``/``), and a human-readable spec. +Finally, we have an entrypoint target ``push`` that will update the buildcache +index once every package is pushed. Note how this target uses the generated +``example/SPACK_PACKAGE_IDS`` variable to define its prerequisites. + +.. code:: Makefile + + SPACK ?= spack + BUILDCACHE_DIR = $(CURDIR)/tarballs + + .PHONY: all + + all: push + + include env.mk + + example/push/%: example/install/% + @mkdir -p $(dir $@) + $(info About to push $(SPEC) to a buildcache) + $(SPACK) -e . buildcache create --allow-root --only=package --directory $(BUILDCACHE_DIR) /$(HASH) + @touch $@ + + push: $(addprefix example/push/,$(example/SPACK_PACKAGE_IDS)) + $(info Updating the buildcache index) + $(SPACK) -e . buildcache update-index --directory $(BUILDCACHE_DIR) + $(info Done!) + @touch $@ diff --git a/lib/spack/spack/cmd/env.py b/lib/spack/spack/cmd/env.py index e4c62758c4..d555425bc5 100644 --- a/lib/spack/spack/cmd/env.py +++ b/lib/spack/spack/cmd/env.py @@ -735,6 +735,18 @@ def env_depfile(args): # Root specs without deps are the prereqs for the environment target root_install_targets = [get_install_target(h.format("{name}-{version}-{hash}")) for h in roots] + all_pkg_identifiers = [] + + # The SPACK_PACKAGE_IDS variable is "exported", which can be used when including + # generated makefiles to add post-install hooks, like pushing to a buildcache, + # running tests, etc. + # NOTE: GNU Make allows directory separators in variable names, so for consistency + # we can namespace this variable with the same prefix as targets. + if args.make_target_prefix is None: + pkg_identifier_variable = "SPACK_PACKAGE_IDS" + else: + pkg_identifier_variable = os.path.join(target_prefix, "SPACK_PACKAGE_IDS") + # All install and install-deps targets all_install_related_targets = [] @@ -744,6 +756,7 @@ def env_depfile(args): phony_convenience_targets = [] for tgt, _, _, _, _ in make_targets.adjacency_list: + all_pkg_identifiers.append(tgt) all_install_related_targets.append(get_install_target(tgt)) all_install_related_targets.append(get_install_deps_target(tgt)) if args.make_target_prefix is None: @@ -770,6 +783,8 @@ def env_depfile(args): "adjacency_list": make_targets.adjacency_list, "phony_convenience_targets": " ".join(phony_convenience_targets), "target_prefix": target_prefix, + "pkg_ids_variable": pkg_identifier_variable, + "pkg_ids": " ".join(all_pkg_identifiers), } ) diff --git a/lib/spack/spack/test/cmd/env.py b/lib/spack/spack/test/cmd/env.py index 81e69ab568..0fa5e4f390 100644 --- a/lib/spack/spack/test/cmd/env.py +++ b/lib/spack/spack/test/cmd/env.py @@ -3146,6 +3146,54 @@ def test_environment_depfile_out(tmpdir, mock_packages): assert stdout == f.read() +def test_spack_package_ids_variable(tmpdir, mock_packages): + # Integration test for post-install hooks through prefix/SPACK_PACKAGE_IDS + # variable + env("create", "test") + makefile_path = str(tmpdir.join("Makefile")) + include_path = str(tmpdir.join("include.mk")) + + # Create env and generate depfile in include.mk with prefix example/ + with ev.read("test"): + add("libdwarf") + concretize() + + with ev.read("test"): + env( + "depfile", + "-G", + "make", + "--make-disable-jobserver", + "--make-target-prefix=example", + "-o", + include_path, + ) + + # Include in Makefile and create target that depend on SPACK_PACKAGE_IDS + with open(makefile_path, "w") as f: + f.write( + r""" +all: post-install + +include include.mk + +example/post-install/%: example/install/% + $(info post-install: $(HASH)) # noqa: W191,E101 + +post-install: $(addprefix example/post-install/,$(example/SPACK_PACKAGE_IDS)) +""" + ) + make = Executable("make") + + # Do dry run. + out = make("-n", "-C", str(tmpdir), output=str) + + # post-install: <hash> should've been executed + with ev.read("test") as test: + for s in test.all_specs(): + assert "post-install: {}".format(s.dag_hash()) in out + + def test_unify_when_possible_works_around_conflicts(): e = ev.create("coconcretization") e.unify = "when_possible" |