From 84100afc9162869fb169e79c8dcd566aba380bdd Mon Sep 17 00:00:00 2001 From: Scott Wittenburg Date: Fri, 20 Aug 2021 07:45:23 -0600 Subject: Pipelines: use shared pr mirror for pipeline generation and builds (#25529) Once PR binary graduation is deployed, the shared PR mirror will contain binaries just built by a merged PR, before the subsequent develop pipeline has had time to finish. Using the shared PR mirror as a source of binaries will reduce the number of times we have to rebuild the same full hash. --- lib/spack/spack/ci.py | 9 +++++++-- lib/spack/spack/test/cmd/ci.py | 12 ++++++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/ci.py b/lib/spack/spack/ci.py index 0ada128f64..6fbddf6385 100644 --- a/lib/spack/spack/ci.py +++ b/lib/spack/spack/ci.py @@ -45,6 +45,8 @@ JOB_RETRY_CONDITIONS = [ ] SPACK_PR_MIRRORS_ROOT_URL = 's3://spack-binaries-prs' +SPACK_SHARED_PR_MIRROR_URL = url_util.join(SPACK_PR_MIRRORS_ROOT_URL, + 'shared_pr_mirror') TEMP_STORAGE_MIRROR_NAME = 'ci_temporary_mirror' spack_gpg = spack.main.SpackCommand('gpg') @@ -612,11 +614,14 @@ def generate_gitlab_ci_yaml(env, print_summary, output_file, 'strip-compilers': False, }) - # Add this mirror if it's enabled, as some specs might be up to date - # here and thus not need to be rebuilt. + # Add per-PR mirror (and shared PR mirror) if enabled, as some specs might + # be up to date in one of those and thus not need to be rebuilt. if pr_mirror_url: spack.mirror.add( 'ci_pr_mirror', pr_mirror_url, cfg.default_modify_scope()) + spack.mirror.add('ci_shared_pr_mirror', + SPACK_SHARED_PR_MIRROR_URL, + cfg.default_modify_scope()) pipeline_artifacts_dir = artifacts_root if not pipeline_artifacts_dir: diff --git a/lib/spack/spack/test/cmd/ci.py b/lib/spack/spack/test/cmd/ci.py index a9891796ac..fad1cd5351 100644 --- a/lib/spack/spack/test/cmd/ci.py +++ b/lib/spack/spack/test/cmd/ci.py @@ -617,6 +617,8 @@ spack: os.environ['SPACK_PR_BRANCH'] = 'fake-test-branch' monkeypatch.setattr( ci, 'SPACK_PR_MIRRORS_ROOT_URL', r"file:///fake/mirror") + monkeypatch.setattr( + ci, 'SPACK_SHARED_PR_MIRROR_URL', r"file:///fake/mirror_two") try: ci_cmd('generate', '--output-file', outputfile) finally: @@ -668,6 +670,8 @@ spack: with ev.read('test'): monkeypatch.setattr( ci, 'SPACK_PR_MIRRORS_ROOT_URL', r"file:///fake/mirror") + monkeypatch.setattr( + ci, 'SPACK_SHARED_PR_MIRROR_URL', r"file:///fake/mirror_two") ci_cmd('generate', '--output-file', outputfile) with open(outputfile) as f: @@ -1153,6 +1157,8 @@ spack: spack.main, 'get_version', lambda: '0.15.3-416-12ad69eb1') monkeypatch.setattr( ci, 'SPACK_PR_MIRRORS_ROOT_URL', r"file:///fake/mirror") + monkeypatch.setattr( + ci, 'SPACK_SHARED_PR_MIRROR_URL', r"file:///fake/mirror_two") ci_cmd('generate', '--output-file', outputfile) with open(outputfile) as f: @@ -1257,6 +1263,8 @@ spack: with ev.read('test'): monkeypatch.setattr( ci, 'SPACK_PR_MIRRORS_ROOT_URL', r"file:///fake/mirror") + monkeypatch.setattr( + ci, 'SPACK_SHARED_PR_MIRROR_URL', r"file:///fake/mirror_two") ci_cmd('generate', '--output-file', outputfile, '--dependencies') with open(outputfile) as f: @@ -1417,6 +1425,8 @@ spack: with ev.read('test'): monkeypatch.setattr( ci, 'SPACK_PR_MIRRORS_ROOT_URL', r"file:///fake/mirror") + monkeypatch.setattr( + ci, 'SPACK_SHARED_PR_MIRROR_URL', r"file:///fake/mirror_two") ci_cmd('generate', '--output-file', outputfile) @@ -1568,6 +1578,8 @@ spack: monkeypatch.setattr( ci, 'SPACK_PR_MIRRORS_ROOT_URL', r"file:///fake/mirror") + monkeypatch.setattr( + ci, 'SPACK_SHARED_PR_MIRROR_URL', r"file:///fake/mirror_two") with ev.read('test'): ci_cmd('generate', '--output-file', outputfile) -- cgit v1.2.3-60-g2f50