From 5c4137baf19b9e271a2f13e886d6b875aab067dd Mon Sep 17 00:00:00 2001 From: Scott Wittenburg Date: Mon, 14 Nov 2022 19:37:23 -0700 Subject: gitlab: Add shared PR mirror to places pipelines look for binaries. (#33746) While binaries built for PRs that get merged must still be rebuilt in develop pipelines, they can be used by other PRs that find they would otherwise need to rebuild them. Now that spackbot is managing copying PR binaries from merged PRs into a shared location, keeping it pruned to a reasonable size, and making sure the indices are up to date, spack can use these mirrors as a potential source of binaries. --- lib/spack/spack/ci.py | 10 ++++++++++ lib/spack/spack/cmd/ci.py | 5 +++++ lib/spack/spack/test/cmd/ci.py | 2 ++ 3 files changed, 17 insertions(+) (limited to 'lib') diff --git a/lib/spack/spack/ci.py b/lib/spack/spack/ci.py index aa54b71b3d..d6113c0dc9 100644 --- a/lib/spack/spack/ci.py +++ b/lib/spack/spack/ci.py @@ -51,6 +51,7 @@ JOB_RETRY_CONDITIONS = [ TEMP_STORAGE_MIRROR_NAME = "ci_temporary_mirror" SPACK_RESERVED_TAGS = ["public", "protected", "notary"] +SHARED_PR_MIRROR_URL = "s3://spack-binaries-prs/shared_pr_mirror" spack_gpg = spack.main.SpackCommand("gpg") spack_compiler = spack.main.SpackCommand("compiler") @@ -731,6 +732,12 @@ def generate_gitlab_ci_yaml( # won't fetch its index and include in our local cache. spack.mirror.add("ci_pr_mirror", remote_mirror_override, cfg.default_modify_scope()) + shared_pr_mirror = None + if spack_pipeline_type == "spack_pull_request": + stack_name = os.environ.get("SPACK_CI_STACK_NAME", "") + shared_pr_mirror = url_util.join(SHARED_PR_MIRROR_URL, stack_name) + spack.mirror.add("ci_shared_pr_mirror", shared_pr_mirror, cfg.default_modify_scope()) + pipeline_artifacts_dir = artifacts_root if not pipeline_artifacts_dir: proj_dir = os.environ.get("CI_PROJECT_DIR", os.getcwd()) @@ -805,6 +812,8 @@ def generate_gitlab_ci_yaml( # Clean up remote mirror override if enabled if remote_mirror_override: spack.mirror.remove("ci_pr_mirror", cfg.default_modify_scope()) + if spack_pipeline_type == "spack_pull_request": + spack.mirror.remove("ci_shared_pr_mirror", cfg.default_modify_scope()) all_job_names = [] output_object = {} @@ -1294,6 +1303,7 @@ def generate_gitlab_ci_yaml( "SPACK_LOCAL_MIRROR_DIR": rel_local_mirror_dir, "SPACK_PIPELINE_TYPE": str(spack_pipeline_type), "SPACK_CI_STACK_NAME": os.environ.get("SPACK_CI_STACK_NAME", "None"), + "SPACK_CI_SHARED_PR_MIRROR_URL": shared_pr_mirror or "None", "SPACK_REBUILD_CHECK_UP_TO_DATE": str(prune_dag), "SPACK_REBUILD_EVERYTHING": str(rebuild_everything), } diff --git a/lib/spack/spack/cmd/ci.py b/lib/spack/spack/cmd/ci.py index 6f29da1f4d..168455bf9c 100644 --- a/lib/spack/spack/cmd/ci.py +++ b/lib/spack/spack/cmd/ci.py @@ -285,6 +285,7 @@ def ci_rebuild(args): remote_mirror_override = get_env_var("SPACK_REMOTE_MIRROR_OVERRIDE") remote_mirror_url = get_env_var("SPACK_REMOTE_MIRROR_URL") spack_ci_stack_name = get_env_var("SPACK_CI_STACK_NAME") + shared_pr_mirror_url = get_env_var("SPACK_CI_SHARED_PR_MIRROR_URL") rebuild_everything = get_env_var("SPACK_REBUILD_EVERYTHING") # Construct absolute paths relative to current $CI_PROJECT_DIR @@ -472,6 +473,10 @@ def ci_rebuild(args): spack.mirror.add("mirror_override", remote_mirror_override, cfg.default_modify_scope()) pipeline_mirrors.append(remote_mirror_override) + if spack_pipeline_type == "spack_pull_request": + if shared_pr_mirror_url != "None": + pipeline_mirrors.append(shared_pr_mirror_url) + matches = ( None if full_rebuild diff --git a/lib/spack/spack/test/cmd/ci.py b/lib/spack/spack/test/cmd/ci.py index 73b474c7df..d01e1e082d 100644 --- a/lib/spack/spack/test/cmd/ci.py +++ b/lib/spack/spack/test/cmd/ci.py @@ -706,6 +706,8 @@ spack: """ ) + monkeypatch.setattr(spack.ci, "SHARED_PR_MIRROR_URL", "https://fake.shared.pr.mirror") + with tmpdir.as_cwd(): env_cmd("create", "test", "./spack.yaml") outputfile = str(tmpdir.join(".gitlab-ci.yml")) -- cgit v1.2.3-70-g09d2