From f050b1cf7835fd31992b020e1061c52294ff7330 Mon Sep 17 00:00:00 2001 From: Harmen Stoppels Date: Wed, 18 Jan 2023 19:19:46 +0100 Subject: depfile: variable with all identifiers (#34678) With the new variable [prefix/]SPACK_PACKAGE_IDS you can conveniently execute things after each successful install. For example push just-built packages to a buildcache ``` SPACK ?= spack export SPACK_COLOR = always MAKEFLAGS += -Orecurse MY_BUILDCACHE := $(CURDIR)/cache .PHONY: all clean all: push ifeq (,$(filter clean,$(MAKECMDGOALS))) include env.mk endif # the relevant part: push has *all* example/push/ as prereqs push: $(addprefix example/push/,$(example/SPACK_PACKAGE_IDS)) $(SPACK) -e . buildcache update-index --directory $(MY_BUILDCACHE) $(info Pushed everything, yay!) # and each example/push/ has the install target as prereq, # and the body can use target local $(HASH) and $(SPEC) variables to do # things, such as pushing to a build cache example/push/%: example/install/% @mkdir -p $(dir $@) $(SPACK) -e . buildcache create --allow-root --only=package --unsigned --directory $(MY_BUILDCACHE) /$(HASH) # push $(SPEC) @touch $@ spack.lock: spack.yaml $(SPACK) -e . concretize -f env.mk: spack.lock $(SPACK) -e . env depfile -o $@ --make-target-prefix example clean: rm -rf spack.lock env.mk example/ `` --- lib/spack/docs/environments.rst | 50 +++++++++++++++++++++++++++++++++- lib/spack/spack/cmd/env.py | 15 ++++++++++ lib/spack/spack/test/cmd/env.py | 48 ++++++++++++++++++++++++++++++++ share/spack/templates/depfile/Makefile | 3 ++ 4 files changed, 115 insertions(+), 1 deletion(-) diff --git a/lib/spack/docs/environments.rst b/lib/spack/docs/environments.rst index 5c8b6b2fb4..ac49f7a523 100644 --- a/lib/spack/docs/environments.rst +++ b/lib/spack/docs/environments.rst @@ -1089,4 +1089,52 @@ output (``spack install --verbose``) while its dependencies are installed silent $ make -j16 install-deps/python-3.11.0- SPACK_INSTALL_FLAGS=--show-log-on-error # Install the root spec with verbose output. - $ make -j16 install/python-3.11.0- SPACK_INSTALL_FLAGS=--verbose \ No newline at end of file + $ make -j16 install/python-3.11.0- SPACK_INSTALL_FLAGS=--verbose + +^^^^^^^^^^^^^^^^^^^^^^^^^ +Adding post-install hooks +^^^^^^^^^^^^^^^^^^^^^^^^^ + +Another advanced use-case of generated ``Makefile``\s is running a post-install +command for each package. These "hooks" could be anything from printing a +post-install message, running tests, or pushing just-built binaries to a buildcache. + +This can be accomplished through the generated ``[/]SPACK_PACKAGE_IDS`` +variable. Assuming we have an active and concrete environment, we generate the +associated ``Makefile`` with a prefix ``example``: + +.. code:: console + + $ spack env depfile -o env.mk --make-target-prefix example + +And we now include it in a different ``Makefile``, in which we create a target +``example/push/%`` with ``%`` referring to a package identifier. This target +depends on the particular package installation. In this target we automatically +have the target-specific ``HASH`` and ``SPEC`` variables at our disposal. They +are respectively the spec hash (excluding leading ``/``), and a human-readable spec. +Finally, we have an entrypoint target ``push`` that will update the buildcache +index once every package is pushed. Note how this target uses the generated +``example/SPACK_PACKAGE_IDS`` variable to define its prerequisites. + +.. code:: Makefile + + SPACK ?= spack + BUILDCACHE_DIR = $(CURDIR)/tarballs + + .PHONY: all + + all: push + + include env.mk + + example/push/%: example/install/% + @mkdir -p $(dir $@) + $(info About to push $(SPEC) to a buildcache) + $(SPACK) -e . buildcache create --allow-root --only=package --directory $(BUILDCACHE_DIR) /$(HASH) + @touch $@ + + push: $(addprefix example/push/,$(example/SPACK_PACKAGE_IDS)) + $(info Updating the buildcache index) + $(SPACK) -e . buildcache update-index --directory $(BUILDCACHE_DIR) + $(info Done!) + @touch $@ diff --git a/lib/spack/spack/cmd/env.py b/lib/spack/spack/cmd/env.py index e4c62758c4..d555425bc5 100644 --- a/lib/spack/spack/cmd/env.py +++ b/lib/spack/spack/cmd/env.py @@ -735,6 +735,18 @@ def env_depfile(args): # Root specs without deps are the prereqs for the environment target root_install_targets = [get_install_target(h.format("{name}-{version}-{hash}")) for h in roots] + all_pkg_identifiers = [] + + # The SPACK_PACKAGE_IDS variable is "exported", which can be used when including + # generated makefiles to add post-install hooks, like pushing to a buildcache, + # running tests, etc. + # NOTE: GNU Make allows directory separators in variable names, so for consistency + # we can namespace this variable with the same prefix as targets. + if args.make_target_prefix is None: + pkg_identifier_variable = "SPACK_PACKAGE_IDS" + else: + pkg_identifier_variable = os.path.join(target_prefix, "SPACK_PACKAGE_IDS") + # All install and install-deps targets all_install_related_targets = [] @@ -744,6 +756,7 @@ def env_depfile(args): phony_convenience_targets = [] for tgt, _, _, _, _ in make_targets.adjacency_list: + all_pkg_identifiers.append(tgt) all_install_related_targets.append(get_install_target(tgt)) all_install_related_targets.append(get_install_deps_target(tgt)) if args.make_target_prefix is None: @@ -770,6 +783,8 @@ def env_depfile(args): "adjacency_list": make_targets.adjacency_list, "phony_convenience_targets": " ".join(phony_convenience_targets), "target_prefix": target_prefix, + "pkg_ids_variable": pkg_identifier_variable, + "pkg_ids": " ".join(all_pkg_identifiers), } ) diff --git a/lib/spack/spack/test/cmd/env.py b/lib/spack/spack/test/cmd/env.py index 81e69ab568..0fa5e4f390 100644 --- a/lib/spack/spack/test/cmd/env.py +++ b/lib/spack/spack/test/cmd/env.py @@ -3146,6 +3146,54 @@ def test_environment_depfile_out(tmpdir, mock_packages): assert stdout == f.read() +def test_spack_package_ids_variable(tmpdir, mock_packages): + # Integration test for post-install hooks through prefix/SPACK_PACKAGE_IDS + # variable + env("create", "test") + makefile_path = str(tmpdir.join("Makefile")) + include_path = str(tmpdir.join("include.mk")) + + # Create env and generate depfile in include.mk with prefix example/ + with ev.read("test"): + add("libdwarf") + concretize() + + with ev.read("test"): + env( + "depfile", + "-G", + "make", + "--make-disable-jobserver", + "--make-target-prefix=example", + "-o", + include_path, + ) + + # Include in Makefile and create target that depend on SPACK_PACKAGE_IDS + with open(makefile_path, "w") as f: + f.write( + r""" +all: post-install + +include include.mk + +example/post-install/%: example/install/% + $(info post-install: $(HASH)) # noqa: W191,E101 + +post-install: $(addprefix example/post-install/,$(example/SPACK_PACKAGE_IDS)) +""" + ) + make = Executable("make") + + # Do dry run. + out = make("-n", "-C", str(tmpdir), output=str) + + # post-install: should've been executed + with ev.read("test") as test: + for s in test.all_specs(): + assert "post-install: {}".format(s.dag_hash()) in out + + def test_unify_when_possible_works_around_conflicts(): e = ev.create("coconcretization") e.unify = "when_possible" diff --git a/share/spack/templates/depfile/Makefile b/share/spack/templates/depfile/Makefile index 4ff7fff0a3..3e844176bc 100644 --- a/share/spack/templates/depfile/Makefile +++ b/share/spack/templates/depfile/Makefile @@ -1,6 +1,9 @@ SPACK ?= spack SPACK_INSTALL_FLAGS ?= +# This variable can be used to add post install hooks +{{ pkg_ids_variable }} := {{ pkg_ids }} + .PHONY: {{ all_target }} {{ clean_target }} {{ all_target }}: {{ env_target }} -- cgit v1.2.3-70-g09d2