Pipelines: Move PR testing stacks (currently only E4S) into spack (#21714)

3 files changed, 61 insertions, 37 deletions

diff --git a/lib/spack/spack/ci.py b/lib/spack/spack/ci.py
index e12699fb5c..e853dc9f98 100644
--- a/lib/spack/spack/ci.py
+++ b/lib/spack/spack/ci.py
@@ -22,7 +22,7 @@ import llnl.util.tty as tty
 
 import spack
 import spack.binary_distribution as bindist
-import spack.cmd.buildcache as buildcache
+import spack.cmd
 import spack.compilers as compilers
 import spack.config as cfg
 import spack.environment as ev
@@ -41,7 +41,7 @@ JOB_RETRY_CONDITIONS = [
     'always',
 ]
 
-SPACK_PR_MIRRORS_ROOT_URL = 's3://spack-pr-mirrors'
+SPACK_PR_MIRRORS_ROOT_URL = 's3://spack-binaries-prs'
 TEMP_STORAGE_MIRROR_NAME = 'ci_temporary_mirror'
 
 spack_gpg = spack.main.SpackCommand('gpg')
@@ -943,7 +943,7 @@ def generate_gitlab_ci_yaml(env, print_summary, output_file, prune_dag=False,
 
             cleanup_job['stage'] = 'cleanup-temp-storage'
             cleanup_job['script'] = [
-                'spack mirror destroy --mirror-url {0}/$CI_PIPELINE_ID'.format(
+                'spack -d mirror destroy --mirror-url {0}/$CI_PIPELINE_ID'.format(
                     temp_storage_url_prefix)
             ]
             cleanup_job['when'] = 'always'
@@ -1262,16 +1262,35 @@ def read_cdashid_from_mirror(spec, mirror_url):
 def push_mirror_contents(env, spec, yaml_path, mirror_url, build_id,
                          sign_binaries):
     if mirror_url:
-        unsigned = not sign_binaries
-        tty.debug('Creating buildcache ({0})'.format(
-            'unsigned' if unsigned else 'signed'))
-        buildcache._createtarball(env, spec_yaml=yaml_path, add_deps=False,
-                                  output_location=mirror_url, force=True,
-                                  allow_root=True, unsigned=unsigned)
-        if build_id:
-            tty.debug('Writing cdashid ({0}) to remote mirror: {1}'.format(
-                build_id, mirror_url))
-            write_cdashid_to_mirror(build_id, spec, mirror_url)
+        try:
+            unsigned = not sign_binaries
+            tty.debug('Creating buildcache ({0})'.format(
+                'unsigned' if unsigned else 'signed'))
+            spack.cmd.buildcache._createtarball(
+                env, spec_yaml=yaml_path, add_deps=False,
+                output_location=mirror_url, force=True, allow_root=True,
+                unsigned=unsigned)
+            if build_id:
+                tty.debug('Writing cdashid ({0}) to remote mirror: {1}'.format(
+                    build_id, mirror_url))
+                write_cdashid_to_mirror(build_id, spec, mirror_url)
+        except Exception as inst:
+            # If the mirror we're pushing to is on S3 and there's some
+            # permissions problem, for example, we can't just target
+            # that exception type here, since users of the
+            # `spack ci rebuild' may not need or want any dependency
+            # on boto3.  So we use the first non-boto exception type
+            # in the heirarchy:
+            #     boto3.exceptions.S3UploadFailedError
+            #     boto3.exceptions.Boto3Error
+            #     Exception
+            #     BaseException
+            #     object
+            err_msg = 'Error msg: {0}'.format(inst)
+            if 'Access Denied' in err_msg:
+                tty.msg('Permission problem writing to {0}'.format(
+                    mirror_url))
+            tty.msg(err_msg)
 
 
 def copy_stage_logs_to_artifacts(job_spec, job_log_dir):
diff --git a/lib/spack/spack/cmd/ci.py b/lib/spack/spack/cmd/ci.py
index f2151708a8..bd42e10238 100644
--- a/lib/spack/spack/cmd/ci.py
+++ b/lib/spack/spack/cmd/ci.py
@@ -413,33 +413,18 @@ def ci_rebuild(args):
             else:
                 buildcache_mirror_url = remote_mirror_url
 
-            try:
-                spack_ci.push_mirror_contents(
-                    env, job_spec, job_spec_yaml_path, buildcache_mirror_url,
-                    cdash_build_id, sign_binaries)
-            except Exception as inst:
-                # If the mirror we're pushing to is on S3 and there's some
-                # permissions problem, for example, we can't just target
-                # that exception type here, since users of the
-                # `spack ci rebuild' may not need or want any dependency
-                # on boto3.  So we use the first non-boto exception type
-                # in the heirarchy:
-                #     boto3.exceptions.S3UploadFailedError
-                #     boto3.exceptions.Boto3Error
-                #     Exception
-                #     BaseException
-                #     object
-                err_msg = 'Error msg: {0}'.format(inst)
-                if 'Access Denied' in err_msg:
-                    tty.msg('Permission problem writing to mirror')
-                tty.msg(err_msg)
+            # Create buildcache in either the main remote mirror, or in the
+            # per-PR mirror, if this is a PR pipeline
+            spack_ci.push_mirror_contents(
+                env, job_spec, job_spec_yaml_path, buildcache_mirror_url,
+                cdash_build_id, sign_binaries)
 
             # Create another copy of that buildcache in the per-pipeline
             # temporary storage mirror (this is only done if either artifacts
             # buildcache is enabled or a temporary storage url prefix is set)
-            spack_ci.push_mirror_contents(env, job_spec, job_spec_yaml_path,
-                                          pipeline_mirror_url, cdash_build_id,
-                                          sign_binaries)
+            spack_ci.push_mirror_contents(
+                env, job_spec, job_spec_yaml_path, pipeline_mirror_url,
+                cdash_build_id, sign_binaries)
 
             # Relate this build to its dependencies on CDash (if enabled)
             if enable_cdash:
diff --git a/lib/spack/spack/test/cmd/ci.py b/lib/spack/spack/test/cmd/ci.py
index 3be182c7e5..24992d78c2 100644
--- a/lib/spack/spack/test/cmd/ci.py
+++ b/lib/spack/spack/test/cmd/ci.py
@@ -878,6 +878,26 @@ spack:
             assert(len(dl_dir_list) == 3)
 
 
+def test_push_mirror_contents_exceptions(monkeypatch, capsys):
+    def faked(env, spec_yaml=None, packages=None, add_spec=True,
+              add_deps=True, output_location=os.getcwd(),
+              signing_key=None, force=False, make_relative=False,
+              unsigned=False, allow_root=False, rebuild_index=False):
+        raise Exception('Error: Access Denied')
+
+    import spack.cmd.buildcache as buildcache
+    monkeypatch.setattr(buildcache, '_createtarball', faked)
+
+    url = 'fakejunk'
+    ci.push_mirror_contents(None, None, None, url, None, None)
+
+    captured = capsys.readouterr()
+    std_out = captured[0]
+    expect_msg = 'Permission problem writing to {0}'.format(url)
+
+    assert(expect_msg in std_out)
+
+
 def test_ci_generate_override_runner_attrs(tmpdir, mutable_mock_env_path,
                                            env_deactivate, install_mockery,
                                            mock_packages, monkeypatch):
@@ -1373,7 +1393,7 @@ spack:
                 assert('script' in cleanup_job)
                 cleanup_task = cleanup_job['script'][0]
 
-                assert(cleanup_task.startswith('spack mirror destroy'))
+                assert(cleanup_task.startswith('spack -d mirror destroy'))
 
                 assert('stages' in pipeline_doc)
                 stages = pipeline_doc['stages']