Ci backwards compat (#36045)

* CI: Fixup docs for bootstrap.

* CI: Add compatibility shim

* Add an update method for CI

Update requires manually renaming section to `ci`. After
this patch, updating and using the deprecated `gitlab-ci` section
should be possible.

* Fix typos in generate warnings

* Fixup CI schema validation

* Add unit tests for legacy CI

* Add deprecated CI stack for continuous testing

* Allow updating gitlab-ci section directly with env update

* Make warning give good advice for updating gitlab-ci

* Fix typo in CI name

* Remove white space

* Remove unneeded component of deprected-ci

2 files changed, 167 insertions, 0 deletions

diff --git a/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml b/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml
index 9d3282feea..de8e5dc552 100644
--- a/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml
+++ b/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml
@@ -102,6 +102,37 @@ default:
   extends: [ ".generate-base" ]
   tags: ["spack", "public", "medium", "x86_64"]
 
+.generate-deprecated:
+  stage: generate
+  script:
+    - uname -a || true
+    - grep -E 'vendor|model name' /proc/cpuinfo 2>/dev/null | sort -u || head -n10 /proc/cpuinfo 2>/dev/null || true
+    - nproc || true
+    - . "./share/spack/setup-env.sh"
+    - spack --version
+    - cd share/spack/gitlab/cloud_pipelines/stacks/${SPACK_CI_STACK_NAME}
+    - spack env activate --without-view .
+    - spack
+      ci generate --check-index-only
+      --buildcache-destination "${SPACK_BUILDCACHE_DESTINATION}"
+      --artifacts-root "${CI_PROJECT_DIR}/jobs_scratch_dir"
+      --output-file "${CI_PROJECT_DIR}/jobs_scratch_dir/cloud-ci-pipeline.yml"
+  after_script:
+    - cat /proc/loadavg || true
+  artifacts:
+    paths:
+      - "${CI_PROJECT_DIR}/jobs_scratch_dir"
+  variables:
+    KUBERNETES_CPU_REQUEST: 4000m
+    KUBERNETES_MEMORY_REQUEST: 16G
+  interruptible: true
+  timeout: 60 minutes
+  retry:
+    max: 2
+    when:
+      - always
+  tags: ["spack", "public", "medium", "x86_64"]
+
 .generate-aarch64:
   extends: [ ".generate" ]
   tags: ["spack", "public", "medium", "aarch64"]
@@ -109,12 +140,18 @@ default:
 .pr-generate:
   extends: [ ".pr", ".generate" ]
 
+.pr-generate-deprecated:
+  extends: [ ".pr", ".generate-deprecated" ]
+
 .pr-generate-aarch64:
   extends: [ ".pr", ".generate-aarch64" ]
 
 .protected-generate:
   extends: [ ".protected", ".generate" ]
 
+.protected-generate-deprecated:
+  extends: [ ".protected", ".generate-deprecated" ]
+
 .protected-generate-aarch64:
   extends: [ ".protected", ".generate-aarch64" ]
 
@@ -978,3 +1015,39 @@ ml-linux-x86_64-rocm-protected-build:
   needs:
     - artifacts: True
       job: ml-linux-x86_64-rocm-protected-generate
+
+
+########################################
+# Deprecated CI testing
+########################################
+.deprecated-ci:
+  variables:
+    SPACK_CI_STACK_NAME: deprecated
+
+deprecated-ci-pr-generate:
+  extends: [ ".pr-generate-deprecated", ".deprecated-ci" ]
+
+deprecated-ci-protected-generate:
+  extends: [ ".protected-generate-deprecated", ".deprecated-ci" ]
+
+deprecated-ci-pr-build:
+  extends: [ ".pr-build", ".deprecated-ci" ]
+  trigger:
+    include:
+      - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
+        job: deprecated-ci-pr-generate
+    strategy: depend
+  needs:
+    - artifacts: True
+      job: deprecated-ci-pr-generate
+
+deprecated-ci-protected-build:
+  extends: [ ".protected-build", ".deprecated-ci" ]
+  trigger:
+    include:
+      - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
+        job: deprecated-ci-protected-generate
+    strategy: depend
+  needs:
+    - artifacts: True
+      job: deprecated-ci-protected-generate
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/deprecated/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/deprecated/spack.yaml
new file mode 100644
index 0000000000..edb27d5f4f
--- /dev/null
+++ b/share/spack/gitlab/cloud_pipelines/stacks/deprecated/spack.yaml
@@ -0,0 +1,94 @@
+###
+# Spack pipeline for testing deprecated gitlab-ci configuration
+###
+spack:
+  view: false
+  concretizer:
+    reuse: false
+    unify: false
+  config:
+    concretizer: clingo
+    db_lock_timeout: 120
+    install_tree:
+      padded_length: 256
+      projections:
+        all: '{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'
+    deprecated: true
+  packages:
+    all:
+      require: target=x86_64
+  specs:
+  - readline
+
+  mirrors:
+    mirror: s3://spack-binaries/develop/deprecated
+  gitlab-ci:
+    broken-tests-packages:
+    - gptune
+    broken-specs-url: s3://spack-binaries/broken-specs
+    image: ghcr.io/spack/tutorial-ubuntu-18.04:v2021-11-02
+    before_script:
+    - curl -LfsS "https://github.com/JuliaBinaryWrappers/GNUMake_jll.jl/releases/download/GNUMake-v4.3.0+1/GNUMake.v4.3.0.x86_64-linux-gnu.tar.gz"
+      -o gmake.tar.gz
+    - printf "fef1f59e56d2d11e6d700ba22d3444b6e583c663d6883fd0a4f63ab8bd280f0f gmake.tar.gz"
+      | sha256sum --check --strict --quiet
+    - tar -xzf gmake.tar.gz -C /usr bin/make 2> /dev/null
+    - uname -a || true
+    - grep -E "vendor|model name" /proc/cpuinfo 2>/dev/null | sort -u || head -n10
+      /proc/cpuinfo 2>/dev/null || true
+    - nproc
+    - . "./share/spack/setup-env.sh"
+    - spack --version
+    - spack arch
+    script:
+    - spack compiler find
+    - cd ${SPACK_CONCRETE_ENV_DIR}
+    - spack env activate --without-view .
+    - if [ -n "$SPACK_BUILD_JOBS" ]; then spack config add "config:build_jobs:$SPACK_BUILD_JOBS";
+      fi
+    - spack config add "config:install_tree:projections:${SPACK_JOB_SPEC_PKG_NAME}:'morepadding/{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'"
+    - mkdir -p ${SPACK_ARTIFACTS_ROOT}/user_data
+      # AWS runners mount E4S public key (verification), UO runners mount public/private (signing/verification)
+    - if [[ -r /mnt/key/e4s.gpg ]]; then spack gpg trust /mnt/key/e4s.gpg; fi
+      # UO runners mount intermediate ci public key (verification), AWS runners mount public/private (signing/verification)
+    - if [[ -r /mnt/key/intermediate_ci_signing_key.gpg ]]; then spack gpg trust /mnt/key/intermediate_ci_signing_key.gpg;
+      fi
+    - if [[ -r /mnt/key/spack_public_key.gpg ]]; then spack gpg trust /mnt/key/spack_public_key.gpg;
+      fi
+    - spack --color=always --backtrace ci rebuild --tests > >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_out.txt)
+      2> >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_err.txt >&2)
+    after_script:
+    - cat /proc/loadavg || true
+    match_behavior: first
+    mappings:
+    - match:
+      - '@:'
+      runner-attributes:
+        tags: [spack, public, small, x86_64]
+        variables:
+          CI_JOB_SIZE: small
+          SPACK_BUILD_JOBS: '1'
+          KUBERNETES_CPU_REQUEST: 500m
+          KUBERNETES_MEMORY_REQUEST: 500M
+    signing-job-attributes:
+      image: {name: 'ghcr.io/spack/notary:latest', entrypoint: ['']}
+      tags: [aws]
+      script:
+      - aws s3 sync --exclude "*" --include "*spec.json*" ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache
+        /tmp
+      - /sign.sh
+      - aws s3 sync --exclude "*" --include "*spec.json.sig*" /tmp ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache
+      - aws s3 cp /tmp/public_keys ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache/_pgp
+        --recursive --exclude "*" --include "*.pub"
+
+    service-job-attributes:
+      image: ghcr.io/spack/tutorial-ubuntu-18.04:v2021-11-02
+      before_script:
+      - . "./share/spack/setup-env.sh"
+      - spack --version
+      tags: [spack, public, x86_64]
+  cdash:
+    build-group: Spack Deprecated CI
+    url: https://cdash.spack.io
+    project: Spack Testing
+    site: Cloud Gitlab Infrastructure