From efa1dba9e49c5089252f858844c5b60fcad83511 Mon Sep 17 00:00:00 2001 From: Harmen Stoppels Date: Mon, 28 Nov 2022 14:06:03 +0100 Subject: Revert "Revert "gitlab: Add shared PR mirror to places pipelines look for binaries. (#33746)" (#34087)" (#34153) This reverts commit 63e440651436fdc30993ef58875279e71f4851d1. --- lib/spack/spack/ci.py | 10 ++++++++++ lib/spack/spack/cmd/ci.py | 5 +++++ lib/spack/spack/test/cmd/ci.py | 2 ++ 3 files changed, 17 insertions(+) diff --git a/lib/spack/spack/ci.py b/lib/spack/spack/ci.py index 43e7d03226..6dcfb7a7f3 100644 --- a/lib/spack/spack/ci.py +++ b/lib/spack/spack/ci.py @@ -49,6 +49,7 @@ JOB_RETRY_CONDITIONS = [ TEMP_STORAGE_MIRROR_NAME = "ci_temporary_mirror" SPACK_RESERVED_TAGS = ["public", "protected", "notary"] +SHARED_PR_MIRROR_URL = "s3://spack-binaries-prs/shared_pr_mirror" spack_gpg = spack.main.SpackCommand("gpg") spack_compiler = spack.main.SpackCommand("compiler") @@ -729,6 +730,12 @@ def generate_gitlab_ci_yaml( # won't fetch its index and include in our local cache. spack.mirror.add("ci_pr_mirror", remote_mirror_override, cfg.default_modify_scope()) + shared_pr_mirror = None + if spack_pipeline_type == "spack_pull_request": + stack_name = os.environ.get("SPACK_CI_STACK_NAME", "") + shared_pr_mirror = url_util.join(SHARED_PR_MIRROR_URL, stack_name) + spack.mirror.add("ci_shared_pr_mirror", shared_pr_mirror, cfg.default_modify_scope()) + pipeline_artifacts_dir = artifacts_root if not pipeline_artifacts_dir: proj_dir = os.environ.get("CI_PROJECT_DIR", os.getcwd()) @@ -803,6 +810,8 @@ def generate_gitlab_ci_yaml( # Clean up remote mirror override if enabled if remote_mirror_override: spack.mirror.remove("ci_pr_mirror", cfg.default_modify_scope()) + if spack_pipeline_type == "spack_pull_request": + spack.mirror.remove("ci_shared_pr_mirror", cfg.default_modify_scope()) all_job_names = [] output_object = {} @@ -1292,6 +1301,7 @@ def generate_gitlab_ci_yaml( "SPACK_LOCAL_MIRROR_DIR": rel_local_mirror_dir, "SPACK_PIPELINE_TYPE": str(spack_pipeline_type), "SPACK_CI_STACK_NAME": os.environ.get("SPACK_CI_STACK_NAME", "None"), + "SPACK_CI_SHARED_PR_MIRROR_URL": shared_pr_mirror or "None", "SPACK_REBUILD_CHECK_UP_TO_DATE": str(prune_dag), "SPACK_REBUILD_EVERYTHING": str(rebuild_everything), } diff --git a/lib/spack/spack/cmd/ci.py b/lib/spack/spack/cmd/ci.py index 5c3e03c59b..f82dbba4ae 100644 --- a/lib/spack/spack/cmd/ci.py +++ b/lib/spack/spack/cmd/ci.py @@ -284,6 +284,7 @@ def ci_rebuild(args): remote_mirror_override = get_env_var("SPACK_REMOTE_MIRROR_OVERRIDE") remote_mirror_url = get_env_var("SPACK_REMOTE_MIRROR_URL") spack_ci_stack_name = get_env_var("SPACK_CI_STACK_NAME") + shared_pr_mirror_url = get_env_var("SPACK_CI_SHARED_PR_MIRROR_URL") rebuild_everything = get_env_var("SPACK_REBUILD_EVERYTHING") # Construct absolute paths relative to current $CI_PROJECT_DIR @@ -471,6 +472,10 @@ def ci_rebuild(args): spack.mirror.add("mirror_override", remote_mirror_override, cfg.default_modify_scope()) pipeline_mirrors.append(remote_mirror_override) + if spack_pipeline_type == "spack_pull_request": + if shared_pr_mirror_url != "None": + pipeline_mirrors.append(shared_pr_mirror_url) + matches = ( None if full_rebuild diff --git a/lib/spack/spack/test/cmd/ci.py b/lib/spack/spack/test/cmd/ci.py index 796bf5420f..640ef0d236 100644 --- a/lib/spack/spack/test/cmd/ci.py +++ b/lib/spack/spack/test/cmd/ci.py @@ -706,6 +706,8 @@ spack: """ ) + monkeypatch.setattr(spack.ci, "SHARED_PR_MIRROR_URL", "https://fake.shared.pr.mirror") + with tmpdir.as_cwd(): env_cmd("create", "test", "./spack.yaml") outputfile = str(tmpdir.join(".gitlab-ci.yml")) -- cgit v1.2.3-70-g09d2