Pipelines: Move PR testing stacks (currently only E4S) into spack (#21714)

6 files changed, 273 insertions, 61 deletions

diff --git a/lib/spack/spack/ci.py b/lib/spack/spack/ci.py
index e12699fb5c..e853dc9f98 100644
--- a/lib/spack/spack/ci.py
+++ b/lib/spack/spack/ci.py
@@ -22,7 +22,7 @@ import llnl.util.tty as tty
 
 import spack
 import spack.binary_distribution as bindist
-import spack.cmd.buildcache as buildcache
+import spack.cmd
 import spack.compilers as compilers
 import spack.config as cfg
 import spack.environment as ev
@@ -41,7 +41,7 @@ JOB_RETRY_CONDITIONS = [
     'always',
 ]
 
-SPACK_PR_MIRRORS_ROOT_URL = 's3://spack-pr-mirrors'
+SPACK_PR_MIRRORS_ROOT_URL = 's3://spack-binaries-prs'
 TEMP_STORAGE_MIRROR_NAME = 'ci_temporary_mirror'
 
 spack_gpg = spack.main.SpackCommand('gpg')
@@ -943,7 +943,7 @@ def generate_gitlab_ci_yaml(env, print_summary, output_file, prune_dag=False,
 
             cleanup_job['stage'] = 'cleanup-temp-storage'
             cleanup_job['script'] = [
-                'spack mirror destroy --mirror-url {0}/$CI_PIPELINE_ID'.format(
+                'spack -d mirror destroy --mirror-url {0}/$CI_PIPELINE_ID'.format(
                     temp_storage_url_prefix)
             ]
             cleanup_job['when'] = 'always'
@@ -1262,16 +1262,35 @@ def read_cdashid_from_mirror(spec, mirror_url):
 def push_mirror_contents(env, spec, yaml_path, mirror_url, build_id,
                          sign_binaries):
     if mirror_url:
-        unsigned = not sign_binaries
-        tty.debug('Creating buildcache ({0})'.format(
-            'unsigned' if unsigned else 'signed'))
-        buildcache._createtarball(env, spec_yaml=yaml_path, add_deps=False,
-                                  output_location=mirror_url, force=True,
-                                  allow_root=True, unsigned=unsigned)
-        if build_id:
-            tty.debug('Writing cdashid ({0}) to remote mirror: {1}'.format(
-                build_id, mirror_url))
-            write_cdashid_to_mirror(build_id, spec, mirror_url)
+        try:
+            unsigned = not sign_binaries
+            tty.debug('Creating buildcache ({0})'.format(
+                'unsigned' if unsigned else 'signed'))
+            spack.cmd.buildcache._createtarball(
+                env, spec_yaml=yaml_path, add_deps=False,
+                output_location=mirror_url, force=True, allow_root=True,
+                unsigned=unsigned)
+            if build_id:
+                tty.debug('Writing cdashid ({0}) to remote mirror: {1}'.format(
+                    build_id, mirror_url))
+                write_cdashid_to_mirror(build_id, spec, mirror_url)
+        except Exception as inst:
+            # If the mirror we're pushing to is on S3 and there's some
+            # permissions problem, for example, we can't just target
+            # that exception type here, since users of the
+            # `spack ci rebuild' may not need or want any dependency
+            # on boto3.  So we use the first non-boto exception type
+            # in the heirarchy:
+            #     boto3.exceptions.S3UploadFailedError
+            #     boto3.exceptions.Boto3Error
+            #     Exception
+            #     BaseException
+            #     object
+            err_msg = 'Error msg: {0}'.format(inst)
+            if 'Access Denied' in err_msg:
+                tty.msg('Permission problem writing to {0}'.format(
+                    mirror_url))
+            tty.msg(err_msg)
 
 
 def copy_stage_logs_to_artifacts(job_spec, job_log_dir):
diff --git a/lib/spack/spack/cmd/ci.py b/lib/spack/spack/cmd/ci.py
index f2151708a8..bd42e10238 100644
--- a/lib/spack/spack/cmd/ci.py
+++ b/lib/spack/spack/cmd/ci.py
@@ -413,33 +413,18 @@ def ci_rebuild(args):
             else:
                 buildcache_mirror_url = remote_mirror_url
 
-            try:
-                spack_ci.push_mirror_contents(
-                    env, job_spec, job_spec_yaml_path, buildcache_mirror_url,
-                    cdash_build_id, sign_binaries)
-            except Exception as inst:
-                # If the mirror we're pushing to is on S3 and there's some
-                # permissions problem, for example, we can't just target
-                # that exception type here, since users of the
-                # `spack ci rebuild' may not need or want any dependency
-                # on boto3.  So we use the first non-boto exception type
-                # in the heirarchy:
-                #     boto3.exceptions.S3UploadFailedError
-                #     boto3.exceptions.Boto3Error
-                #     Exception
-                #     BaseException
-                #     object
-                err_msg = 'Error msg: {0}'.format(inst)
-                if 'Access Denied' in err_msg:
-                    tty.msg('Permission problem writing to mirror')
-                tty.msg(err_msg)
+            # Create buildcache in either the main remote mirror, or in the
+            # per-PR mirror, if this is a PR pipeline
+            spack_ci.push_mirror_contents(
+                env, job_spec, job_spec_yaml_path, buildcache_mirror_url,
+                cdash_build_id, sign_binaries)
 
             # Create another copy of that buildcache in the per-pipeline
             # temporary storage mirror (this is only done if either artifacts
             # buildcache is enabled or a temporary storage url prefix is set)
-            spack_ci.push_mirror_contents(env, job_spec, job_spec_yaml_path,
-                                          pipeline_mirror_url, cdash_build_id,
-                                          sign_binaries)
+            spack_ci.push_mirror_contents(
+                env, job_spec, job_spec_yaml_path, pipeline_mirror_url,
+                cdash_build_id, sign_binaries)
 
             # Relate this build to its dependencies on CDash (if enabled)
             if enable_cdash:
diff --git a/lib/spack/spack/test/cmd/ci.py b/lib/spack/spack/test/cmd/ci.py
index 3be182c7e5..24992d78c2 100644
--- a/lib/spack/spack/test/cmd/ci.py
+++ b/lib/spack/spack/test/cmd/ci.py
@@ -878,6 +878,26 @@ spack:
             assert(len(dl_dir_list) == 3)
 
 
+def test_push_mirror_contents_exceptions(monkeypatch, capsys):
+    def faked(env, spec_yaml=None, packages=None, add_spec=True,
+              add_deps=True, output_location=os.getcwd(),
+              signing_key=None, force=False, make_relative=False,
+              unsigned=False, allow_root=False, rebuild_index=False):
+        raise Exception('Error: Access Denied')
+
+    import spack.cmd.buildcache as buildcache
+    monkeypatch.setattr(buildcache, '_createtarball', faked)
+
+    url = 'fakejunk'
+    ci.push_mirror_contents(None, None, None, url, None, None)
+
+    captured = capsys.readouterr()
+    std_out = captured[0]
+    expect_msg = 'Permission problem writing to {0}'.format(url)
+
+    assert(expect_msg in std_out)
+
+
 def test_ci_generate_override_runner_attrs(tmpdir, mutable_mock_env_path,
                                            env_deactivate, install_mockery,
                                            mock_packages, monkeypatch):
@@ -1373,7 +1393,7 @@ spack:
                 assert('script' in cleanup_job)
                 cleanup_task = cleanup_job['script'][0]
 
-                assert(cleanup_task.startswith('spack mirror destroy'))
+                assert(cleanup_task.startswith('spack -d mirror destroy'))
 
                 assert('stages' in pipeline_doc)
                 stages = pipeline_doc['stages']
diff --git a/share/spack/gitlab/cloud_e4s_pipelines.yml b/share/spack/gitlab/cloud_e4s_pipelines.yml
deleted file mode 100644
index 67a74cf971..0000000000
--- a/share/spack/gitlab/cloud_e4s_pipelines.yml
+++ /dev/null
@@ -1,24 +0,0 @@
-pr_pipeline:
-  only:
-  - /^github\/pr[\d]+_.*$/
-  variables:
-    SPACK_REF: ${CI_COMMIT_SHA}
-    SPACK_PR_BRANCH: ${CI_COMMIT_REF_NAME}
-    SPACK_IS_PR_PIPELINE: "True"
-    AWS_ACCESS_KEY_ID: ${PR_MIRRORS_AWS_ACCESS_KEY_ID}
-    AWS_SECRET_ACCESS_KEY: ${PR_MIRRORS_AWS_SECRET_ACCESS_KEY}
-  trigger:
-    project: spack/e4s
-    strategy: depend
-
-develop_pipeline:
-  only:
-  - /^github\/develop$/
-  variables:
-    SPACK_REF: ${CI_COMMIT_SHA}
-    AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
-    AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
-    SPACK_SIGNING_KEY: ${SPACK_SIGNING_KEY}
-  trigger:
-    project: spack/e4s
-    strategy: depend
diff --git a/share/spack/gitlab/cloud_e4s_pipelines/.gitlab-ci.yml b/share/spack/gitlab/cloud_e4s_pipelines/.gitlab-ci.yml
new file mode 100644
index 0000000000..6f26eaf574
--- /dev/null
+++ b/share/spack/gitlab/cloud_e4s_pipelines/.gitlab-ci.yml
@@ -0,0 +1,61 @@
+stages: [generate, build]
+
+.generate:
+  stage: generate
+  script:
+    - . "./share/spack/setup-env.sh"
+    - spack --version
+    - cd share/spack/gitlab/cloud_e4s_pipelines/stacks/e4s
+    - spack env activate --without-view .
+    - spack -d ci generate --check-index-only
+      --output-file "${CI_PROJECT_DIR}/jobs_scratch_dir/e4s_pipeline.yml"
+  artifacts:
+    paths:
+      - "${CI_PROJECT_DIR}/jobs_scratch_dir/e4s_pipeline.yml"
+  tags: ["spack", "public", "medium"]
+
+e4s-pr-generate:
+  only:
+  - /^github\/pr[\d]+_.*$/
+  - move-pr-stack-into-spack
+  extends: .generate
+  before_script:
+    - export SPACK_PR_BRANCH=${CI_COMMIT_REF_NAME}
+    - export SPACK_IS_PR_PIPELINE="True"
+  image: { "name": "ghcr.io/scottwittenburg/ecpe4s-ubuntu18.04-runner-x86_64:2020-09-01", "entrypoint": [""] }
+
+e4s-develop-generate:
+  only:
+  - /^github\/develop$/
+  extends: .generate
+  image: { "name": "ghcr.io/scottwittenburg/ecpe4s-ubuntu18.04-runner-x86_64:2020-09-01", "entrypoint": [""] }
+
+e4s-pr-build:
+  only:
+  - /^github\/pr[\d]+_.*$/
+  - move-pr-stack-into-spack
+  stage: build
+  trigger:
+    include:
+      - artifact: "jobs_scratch_dir/e4s_pipeline.yml"
+        job: e4s-pr-generate
+    strategy: depend
+  variables:
+    AWS_ACCESS_KEY_ID: ${PR_MIRRORS_AWS_ACCESS_KEY_ID}
+    AWS_SECRET_ACCESS_KEY: ${PR_MIRRORS_AWS_SECRET_ACCESS_KEY}
+    SPACK_PR_BRANCH: ${CI_COMMIT_REF_NAME}
+    SPACK_IS_PR_PIPELINE: "True"
+
+e4s-develop-build:
+  only:
+  - /^github\/develop$/
+  stage: build
+  trigger:
+    include:
+      - artifact: "jobs_scratch_dir/e4s_pipeline.yml"
+        job: e4s-develop-generate
+    strategy: depend
+  variables:
+    AWS_ACCESS_KEY_ID: ${MIRRORS_AWS_ACCESS_KEY_ID}
+    AWS_SECRET_ACCESS_KEY: ${MIRRORS_AWS_SECRET_ACCESS_KEY}
+    SPACK_SIGNING_KEY: ${PACKAGE_SIGNING_KEY}
diff --git a/share/spack/gitlab/cloud_e4s_pipelines/stacks/e4s/spack.yaml b/share/spack/gitlab/cloud_e4s_pipelines/stacks/e4s/spack.yaml
new file mode 100644
index 0000000000..fc5dad423b
--- /dev/null
+++ b/share/spack/gitlab/cloud_e4s_pipelines/stacks/e4s/spack.yaml
@@ -0,0 +1,151 @@
+spack:
+  view: false
+  concretization: separately
+
+  config:
+    install_tree:
+      root: /home/software/spack
+      padded_length: 512
+      projections:
+        all: '{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'
+
+  packages:
+    all:
+      target: [x86_64]
+      providers:
+        blas:
+        - openblas
+        mpi:
+        - mpich
+      variants: +mpi
+    binutils:
+      variants: +gold+headers+libiberty~nls
+      version:
+      - 2.33.1
+    cmake:
+      version: [3.18.4]
+    hdf5:
+      variants: +fortran
+    mpich:
+      variants: ~wrapperrpath
+    openblas:
+      version: [0.3.10]
+    slepc:
+      version: [3.14.0]
+
+  definitions:
+  - e4s:
+    - adios
+    - adios2
+    - aml
+    - amrex
+    - arborx
+    - argobots
+    - ascent
+    - axom
+    - bolt
+    - caliper
+    - darshan-runtime
+    - darshan-util
+    - dyninst
+    - faodel
+    - flecsi+cinch
+    - flit
+    - gasnet
+    - ginkgo
+    - globalarrays
+    - gotcha
+    - hdf5
+    - hpctoolkit
+    - hpx
+    - hypre
+    - kokkos-kernels+openmp
+    - kokkos+openmp
+    - legion
+    - libnrm
+    - libquo
+    - magma cuda_arch=70 ^cuda@10.2.89
+    - mercury
+    - mfem
+    - mpifileutils@develop~xattr
+    - ninja
+    - omega-h
+    - openmpi
+    - openpmd-api
+    - papi
+    - papyrus@1.0.1
+    - parallel-netcdf
+    - pdt
+    - petsc
+    - phist
+    - plasma
+    - precice
+    - pumi
+    - py-jupyterhub
+    - py-libensemble
+    - py-petsc4py
+    - qthreads scheduler=distrib
+    - raja
+    - rempi
+    - scr
+    - slate ^openblas@0.3.5 threads=openmp ^cuda@10.2.89
+    - slepc
+    - stc
+    - strumpack ~slate
+    - sundials
+    - superlu
+    - superlu-dist
+    - swig
+    - sz
+    - tasmanian
+    - tau
+    - trilinos
+    - turbine
+    - umap
+    - umpire
+    - unifyfs
+    - upcxx
+    - veloc
+    - zfp
+  - arch:
+    - '%gcc@7.5.0 arch=linux-ubuntu18.04-x86_64'
+
+  specs:
+  - matrix:
+    - - $e4s
+    - - $arch
+
+  mirrors: { "mirror": "s3://spack-binaries-develop/e4s-new-cluster" }
+
+  gitlab-ci:
+    script:
+      - . "./share/spack/setup-env.sh"
+      - spack --version
+      - cd share/spack/gitlab/cloud_e4s_pipelines/stacks/e4s
+      - spack env activate --without-view .
+      - spack -d ci rebuild
+    mappings:
+      - match: [cuda, dyninst, hpx, precice, strumpack, sundials, trilinos, vtk-h, vtk-m]
+        runner-attributes:
+          image: { "name": "ghcr.io/scottwittenburg/ecpe4s-ubuntu18.04-runner-x86_64:2020-09-01", "entrypoint": [""] }
+          tags: ["spack", "public", "xlarge"]
+      - match: ['os=ubuntu18.04']
+        runner-attributes:
+          image: { "name": "ghcr.io/scottwittenburg/ecpe4s-ubuntu18.04-runner-x86_64:2020-09-01", "entrypoint": [""] }
+          tags: ["spack", "public", "large"]
+    temporary-storage-url-prefix: "s3://spack-binaries-prs/pipeline-storage"
+    service-job-attributes:
+      before_script:
+        - . "./share/spack/setup-env.sh"
+        - spack --version
+        - cd share/spack/gitlab/cloud_e4s_pipelines/stacks/e4s
+        - spack env activate --without-view .
+      image: { "name": "ghcr.io/scottwittenburg/ecpe4s-ubuntu18.04-runner-x86_64:2020-09-01", "entrypoint": [""] }
+      tags: ["spack", "public", "medium"]
+
+  cdash:
+    build-group: New PR testing workflow
+    url: https://cdash.spack.io
+    project: Spack Testing
+    site: Cloud Gitlab Infrastructure
+