path: root/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml

stages: [ "generate", "build", "publish" ]

variables:
  SPACK_DISABLE_LOCAL_CONFIG: "1"
  SPACK_USER_CACHE_PATH: "${CI_PROJECT_DIR}/tmp/_user_cache/"

default:
  image: { "name": "ghcr.io/spack/e4s-ubuntu-18.04:v2021-10-18", "entrypoint": [""] }

# CI Platform-Arch
.cray_rhel_zen4:
  variables:
    SPACK_TARGET_PLATFORM: "cray-rhel"
    SPACK_TARGET_ARCH: "zen4"

.cray_sles_zen4:
  variables:
    SPACK_TARGET_PLATFORM: "cray-sles"
    SPACK_TARGET_ARCH: "zen4"

.darwin_x86_64:
  variables:
    SPACK_TARGET_PLATFORM: "darwin"
    SPACK_TARGET_ARCH: "x86_64"

.darwin_aarch64:
  variables:
    SPACK_TARGET_PLATFORM: "darwin"
    SPACK_TARGET_ARCH: "aarch64"

.linux_x86_64_v3:
  variables:
    SPACK_TARGET_PLATFORM: "linux"
    SPACK_TARGET_ARCH: "x86_64_v3"

.linux_skylake:
  variables:
    SPACK_TARGET_PLATFORM: "linux"
    SPACK_TARGET_ARCH: "skylake_avx512"

.linux_icelake:
  variables:
    SPACK_TARGET_PLATFORM: "linux"
    SPACK_TARGET_ARCH: "icelake"

.linux_neoverse_n1:
  variables:
    SPACK_TARGET_PLATFORM: "linux"
    SPACK_TARGET_ARCH: "neoverse_n1"

.linux_neoverse_v1:
  variables:
    SPACK_TARGET_PLATFORM: "linux"
    SPACK_TARGET_ARCH: "neoverse_v1"

.linux_aarch64:
  variables:
    SPACK_TARGET_PLATFORM: "linux"
    SPACK_TARGET_ARCH: "aarch64"

.linux_power:
  variables:
    SPACK_TARGET_PLATFORM: "linux"
    SPACK_TARGET_ARCH: "ppc64le"

########################################
# Job templates
########################################
.base-job:
  variables:
    SPACK_BUILDCACHE_DESTINATION: "s3://spack-binaries/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}"

  rules:
    - if: $CI_COMMIT_REF_NAME == "develop"
      # Pipelines on develop only rebuild what is missing from the mirror
      when: always
      variables:
        SPACK_PIPELINE_TYPE: "spack_protected_branch"
        SPACK_COPY_BUILDCACHE: "s3://spack-binaries/${CI_COMMIT_REF_NAME}"
        SPACK_REQUIRE_SIGNING: "True"
        AWS_ACCESS_KEY_ID: ${PROTECTED_MIRRORS_AWS_ACCESS_KEY_ID}
        AWS_SECRET_ACCESS_KEY: ${PROTECTED_MIRRORS_AWS_SECRET_ACCESS_KEY}
        OIDC_TOKEN_AUDIENCE: "protected_binary_mirror"
    - if: $CI_COMMIT_REF_NAME =~ /^releases\/v.*/
      # Pipelines on release branches always rebuild everything
      when: always
      variables:
        SPACK_PIPELINE_TYPE: "spack_protected_branch"
        SPACK_COPY_BUILDCACHE: "s3://spack-binaries/${CI_COMMIT_REF_NAME}"
        SPACK_PRUNE_UNTOUCHED: "False"
        SPACK_PRUNE_UP_TO_DATE: "False"
        SPACK_REQUIRE_SIGNING: "True"
        AWS_ACCESS_KEY_ID: ${PROTECTED_MIRRORS_AWS_ACCESS_KEY_ID}
        AWS_SECRET_ACCESS_KEY: ${PROTECTED_MIRRORS_AWS_SECRET_ACCESS_KEY}
        OIDC_TOKEN_AUDIENCE: "protected_binary_mirror"
    - if: $CI_COMMIT_TAG =~ /^develop-[\d]{4}-[\d]{2}-[\d]{2}$/ || $CI_COMMIT_TAG =~ /^v.*/
      # Pipelines on tags (release or dev snapshots) only copy binaries from one mirror to another
      when: always
      variables:
        SPACK_PIPELINE_TYPE: "spack_copy_only"
        SPACK_SOURCE_MIRROR: "s3://spack-binaries/SPACK_REPLACE_VERSION/${SPACK_CI_STACK_NAME}"
        SPACK_COPY_BUILDCACHE: "s3://spack-binaries/${CI_COMMIT_REF_NAME}"
        AWS_ACCESS_KEY_ID: ${PROTECTED_MIRRORS_AWS_ACCESS_KEY_ID}
        AWS_SECRET_ACCESS_KEY: ${PROTECTED_MIRRORS_AWS_SECRET_ACCESS_KEY}
        OIDC_TOKEN_AUDIENCE: "protected_binary_mirror"
    - if: $CI_COMMIT_REF_NAME =~ /^pr[\d]+_.*$/
      # Pipelines on PR branches rebuild only what's missing, and do extra pruning
      when: always
      variables:
        SPACK_PIPELINE_TYPE: "spack_pull_request"
        SPACK_BUILDCACHE_DESTINATION: "s3://spack-binaries-prs/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}"
        SPACK_PRUNE_UNTOUCHED: "True"
        SPACK_PRUNE_UNTOUCHED_DEPENDENT_DEPTH: "1"
        AWS_ACCESS_KEY_ID: ${PR_MIRRORS_AWS_ACCESS_KEY_ID}
        AWS_SECRET_ACCESS_KEY: ${PR_MIRRORS_AWS_SECRET_ACCESS_KEY}
        OIDC_TOKEN_AUDIENCE: "pr_binary_mirror"

.generate-common:
  stage: generate
  script:
    - uname -a || true
    - grep -E 'vendor|model name' /proc/cpuinfo 2>/dev/null | sort -u || head -n10 /proc/cpuinfo 2>/dev/null || true
    - nproc || true
    - . "./share/spack/setup-env.sh"
    - spack --version
    - cd share/spack/gitlab/cloud_pipelines/stacks/${SPACK_CI_STACK_NAME}
    - spack env activate --without-view .
    - export SPACK_CI_CONFIG_ROOT="${SPACK_ROOT}/share/spack/gitlab/cloud_pipelines/configs"
    - spack
      --config-scope "${SPACK_CI_CONFIG_ROOT}"
      --config-scope "${SPACK_CI_CONFIG_ROOT}/${SPACK_TARGET_PLATFORM}"
      --config-scope "${SPACK_CI_CONFIG_ROOT}/${SPACK_TARGET_PLATFORM}/${SPACK_TARGET_ARCH}"
      ${CI_STACK_CONFIG_SCOPES}
      ci generate --check-index-only
      --buildcache-destination "${SPACK_BUILDCACHE_DESTINATION}"
      --artifacts-root "${CI_PROJECT_DIR}/jobs_scratch_dir"
      --output-file "${CI_PROJECT_DIR}/jobs_scratch_dir/cloud-ci-pipeline.yml"
  after_script:
    - cat /proc/loadavg || true
  artifacts:
    paths:
      - "${CI_PROJECT_DIR}/jobs_scratch_dir"
  variables:
    KUBERNETES_CPU_REQUEST: 4000m
    KUBERNETES_MEMORY_REQUEST: 16G
    # avoid moving targets like branches and tags
    SPACK_CONCRETIZER_REQUIRE_CHECKSUM: 1
    SPACK_BACKTRACE: 1
  interruptible: true
  timeout: 60 minutes
  retry:
    max: 2
    when:
      - always

# Generate without tags for cases using external runners
.generate-base:
  extends: [ ".base-job", ".generate-common" ]

.generate-x86_64:
  extends: [ ".generate-base" ]
  tags: ["spack", "public", "medium", "x86_64"]

.generate-aarch64:
  extends: [ ".generate-base" ]
  tags: ["spack", "public", "medium", "aarch64"]

.generate-deprecated:
  extends: [ ".base-job" ]
  stage: generate
  script:
    - uname -a || true
    - grep -E 'vendor|model name' /proc/cpuinfo 2>/dev/null | sort -u || head -n10 /proc/cpuinfo 2>/dev/null || true
    - nproc || true
    - . "./share/spack/setup-env.sh"
    - spack --version
    - cd share/spack/gitlab/cloud_pipelines/stacks/${SPACK_CI_STACK_NAME}
    - spack env activate --without-view .
    - spack
      ci generate --check-index-only
      --buildcache-destination "${SPACK_BUILDCACHE_DESTINATION}"
      --artifacts-root "${CI_PROJECT_DIR}/jobs_scratch_dir"
      --output-file "${CI_PROJECT_DIR}/jobs_scratch_dir/cloud-ci-pipeline.yml"
  after_script:
    - cat /proc/loadavg || true
  artifacts:
    paths:
      - "${CI_PROJECT_DIR}/jobs_scratch_dir"
  variables:
    KUBERNETES_CPU_REQUEST: 4000m
    KUBERNETES_MEMORY_REQUEST: 16G
  interruptible: true
  timeout: 60 minutes
  retry:
    max: 2
    when:
      - always
  tags: ["spack", "public", "medium", "x86_64"]

.build:
  extends: [ ".base-job" ]
  stage: build

protected-publish:
  # Copy binaries from stack-specific mirrors to a root mirror
  stage: publish
  only:
  - /^develop$/
  - /^releases\/v.*/
  - /^v.*/
  - /^develop-[\d]{4}-[\d]{2}-[\d]{2}$/
  image: "ghcr.io/spack/python-aws-bash:0.0.1"
  tags: ["spack", "public", "medium", "aws", "x86_64"]
  retry:
    max: 2
    when: ["runner_system_failure", "stuck_or_timeout_failure"]
  variables:
    SPACK_BUILDCACHE_DESTINATION: "s3://spack-binaries/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}"
    SPACK_COPY_BUILDCACHE: "s3://spack-binaries/${CI_COMMIT_REF_NAME}"
    SPACK_PIPELINE_TYPE: "spack_protected_branch"
    AWS_ACCESS_KEY_ID: ${PROTECTED_MIRRORS_AWS_ACCESS_KEY_ID}
    AWS_SECRET_ACCESS_KEY: ${PROTECTED_MIRRORS_AWS_SECRET_ACCESS_KEY}
    KUBERNETES_CPU_REQUEST: 4000m
    KUBERNETES_MEMORY_REQUEST: 16G
  script:
    - . "./share/spack/setup-env.sh"
    - spack --version
    - export COPY_SPECS_DIR=${CI_PROJECT_DIR}/jobs_scratch_dir/specs_to_copy
    - spack buildcache sync --manifest-glob "${COPY_SPECS_DIR}/*.json"
    - curl -fLsS https://spack.github.io/keys/spack-public-binary-key.pub -o /tmp/spack-public-binary-key.pub
    - aws s3 cp /tmp/spack-public-binary-key.pub "${SPACK_COPY_BUILDCACHE}/build_cache/_pgp/spack-public-binary-key.pub"
    - spack buildcache update-index --keys "${SPACK_COPY_BUILDCACHE}"
  id_tokens:
    GITLAB_OIDC_TOKEN:
      aud: "protected_binary_mirror"

########################################
# TEMPLATE FOR ADDING ANOTHER PIPELINE
########################################
#
# First add a new spack.yml defining the pipeline to run in
# share/spack/gitlab/cloud_pipelines/stacks/my-super-cool-stack/spack.yaml
#
# Then add the following entries at the bottom of this file.
#
# Note that when extending other jobs, gitlab does not merge lists (nor
# does it merge dictionary values in the case of key conflicts).  So lists
# and duplicated dictionary keys are simply overridden.  For this reason,
# you should inlclude your custom definitions at the end of the of the
# extends list.
#
# Also note that if extending .base-job, the mirror url given in your
# spack.yaml should take the form:
#
#     s3://spack-binaries/develop/${SPACK_CI_STACK_NAME}
#
########################################
# My Super Cool Pipeline
########################################
# .my-super-cool-stack:
#   extends: [ ".linux_x86_64_v3" ]
#   variables:
#     SPACK_CI_STACK_NAME: my-super-cool-stack
#     tags: [ "all", "tags", "your", "job", "needs"]
#
# my-super-cool-stack-generate:
#   extends: [ ".generate", ".my-super-cool-stack" ]
#
# my-super-cool-stack-build:
#   extends: [ ".build", ".my-super-cool-stack" ]
#   trigger:
#     include:
#       - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
#         job: my-super-cool-stack-generate
#     strategy: depend
#   needs:
#     - artifacts: True
#       job: my-super-cool-stack-generate

########################################
# E4S pipeline
########################################
.e4s:
  extends: [ ".linux_x86_64_v3" ]
  variables:
    SPACK_CI_STACK_NAME: e4s

e4s-generate:
  extends: [ ".e4s", ".generate-x86_64"]
  image: ghcr.io/spack/ubuntu20.04-runner-amd64-gcc-11.4:2023.08.01

e4s-build:
  extends: [ ".e4s", ".build" ]
  trigger:
    include:
      - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
        job: e4s-generate
    strategy: depend
  needs:
    - artifacts: True
      job: e4s-generate

########################################
# E4S ARM pipeline
########################################
.e4s-arm:
  extends: [ ".linux_aarch64" ]
  variables:
    SPACK_CI_STACK_NAME: e4s-arm

e4s-arm-generate:
  extends: [ ".e4s-arm", ".generate-aarch64" ]
  image: ghcr.io/spack/ubuntu20.04-runner-arm64-gcc-11.4:2023.08.01

e4s-arm-build:
  extends: [ ".e4s-arm", ".build" ]
  trigger:
    include:
      - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
        job: e4s-arm-generate
    strategy: depend
  needs:
    - artifacts: True
      job: e4s-arm-generate

########################################
# E4S ROCm External pipeline
########################################
.e4s-rocm-external:
  extends: [ ".linux_x86_64_v3" ]
  variables:
    SPACK_CI_STACK_NAME: e4s-rocm-external

e4s-rocm-external-generate:
  extends: [ ".e4s-rocm-external", ".generate-x86_64"]
  image: ghcr.io/spack/ubuntu20.04-runner-amd64-gcc-11.4-rocm5.4.3:2023.08.01

e4s-rocm-external-build:
  extends: [ ".e4s-rocm-external", ".build" ]
  trigger:
    include:
      - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
        job: e4s-rocm-external-generate
    strategy: depend
  needs:
    - artifacts: True
      job: e4s-rocm-external-generate

########################################
# GPU Testing Pipeline
########################################
.gpu-tests:
  extends: [ ".linux_x86_64_v3" ]
  variables:
    SPACK_CI_STACK_NAME: gpu-tests

gpu-tests-generate:
  extends: [ ".gpu-tests", ".generate-x86_64"]
  image: ghcr.io/spack/ubuntu20.04-runner-x86_64:2023-01-01

gpu-tests-build:
  extends: [ ".gpu-tests", ".build" ]
  trigger:
    include:
      - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
        job: gpu-tests-generate
    strategy: depend
  needs:
    - artifacts: True
      job: gpu-tests-generate

########################################
# E4S OneAPI Pipeline
########################################
.e4s-oneapi:
  extends: [ ".linux_x86_64_v3" ]
  variables:
    SPACK_CI_STACK_NAME: e4s-oneapi

e4s-oneapi-generate:
  extends: [ ".e4s-oneapi", ".generate-x86_64"]
  image: ghcr.io/spack/ubuntu20.04-runner-amd64-oneapi-2023.2.1:2023.08.01

e4s-oneapi-build:
  extends: [ ".e4s-oneapi", ".build" ]
  trigger:
    include:
      - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
        job: e4s-oneapi-generate
    strategy: depend
  needs:
    - artifacts: True
      job: e4s-oneapi-generate

########################################
# E4S on Power
########################################
.e4s-power-generate-tags-and-image:
  image: { "name": "ghcr.io/spack/ubuntu20.04-runner-ppc64-gcc-11.4:2023.08.01", "entrypoint": [""] }
  tags: ["spack", "public", "large", "ppc64le"]

.e4s-power:
  extends: [".linux_power"]
  variables:
    SPACK_CI_STACK_NAME: e4s-power

e4s-power-generate:
  extends: [ ".e4s-power", ".generate-x86_64", ".e4s-power-generate-tags-and-image"]

e4s-power-build:
  extends: [ ".e4s-power", ".build" ]
  trigger:
    include:
      - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
        job: e4s-power-generate
    strategy: depend
  needs:
    - artifacts: True
      job: e4s-power-generate

#########################################
# Build tests for different build-systems
#########################################
.build_systems:
  extends: [ ".linux_x86_64_v3" ]
  variables:
    SPACK_CI_STACK_NAME: build_systems

build_systems-generate:
  extends: [ ".build_systems", ".generate-x86_64"]

build_systems-build:
  extends: [ ".build_systems", ".build" ]
  trigger:
    include:
      - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
        job: build_systems-generate
    strategy: depend
  needs:
    - artifacts: True
      job: build_systems-generate

#########################################
# RADIUSS
#########################################
.radiuss:
  extends: [ ".linux_x86_64_v3" ]
  variables:
    SPACK_CI_STACK_NAME: radiuss

radiuss-generate:
  extends: [ ".radiuss", ".generate-x86_64" ]

radiuss-build:
  extends: [ ".radiuss", ".build" ]
  trigger:
    include:
      - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
        job: radiuss-generate
    strategy: depend
  needs:
    - artifacts: True
      job: radiuss-generate

########################################
# RADIUSS for AWS
########################################

.tags-x86_64_v4:
  tags: ["spack", "public", "aws", "medium", "x86_64_v4"]

# Include this AFTER .*-generate in "extends" list
.radiuss-aws-overrides:
  # This controls image for generate step; build step is controlled by spack.yaml
  # Note that generator emits OS info for build so these should be the same.
  image: { "name": "ghcr.io/spack/e4s-amazonlinux-2:v2023-03-09", "entrypoint": [""] }

.radiuss-aws:
  extends: [ ".linux_x86_64_v3" ]
  variables:
    SPACK_CI_STACK_NAME: radiuss-aws

radiuss-aws-generate:
  extends: [ ".radiuss-aws", ".generate-x86_64", ".radiuss-aws-overrides", ".tags-x86_64_v4" ]

radiuss-aws-build:
  extends: [ ".radiuss-aws", ".build" ]
  trigger:
    include:
      - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
        job: radiuss-aws-generate
    strategy: depend
  needs:
    - artifacts: True
      job: radiuss-aws-generate


# Parallel Pipeline for aarch64 (reuses override image, but generates and builds on aarch64)

.radiuss-aws-aarch64:
  extends: [ ".linux_aarch64" ]
  variables:
    SPACK_CI_STACK_NAME: radiuss-aws-aarch64

radiuss-aws-aarch64-generate:
  extends: [ ".radiuss-aws-aarch64", ".generate-aarch64", ".radiuss-aws-overrides" ]

radiuss-aws-aarch64-build:
  extends: [ ".radiuss-aws-aarch64", ".build" ]
  trigger:
    include:
      - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
        job: radiuss-aws-aarch64-generate
    strategy: depend
  needs:
    - artifacts: True
      job: radiuss-aws-aarch64-generate

########################################
# ECP Data & Vis SDK
########################################
.data-vis-sdk:
  extends: [ ".linux_x86_64_v3" ]
  variables:
    SPACK_CI_STACK_NAME: data-vis-sdk

data-vis-sdk-generate:
  extends: [ ".data-vis-sdk", ".generate-x86_64"]
  image: ghcr.io/spack/ubuntu20.04-runner-x86_64:2023-01-01

data-vis-sdk-build:
  extends: [ ".data-vis-sdk", ".build" ]
  trigger:
    include:
      - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
        job: data-vis-sdk-generate
    strategy: depend
  needs:
    - artifacts: True
      job: data-vis-sdk-generate

########################################
# AWS ISC Applications (x86_64)
########################################

# Call this AFTER .*-generate
.aws-isc-overrides:
  # This controls image for generate step; build step is controlled by spack.yaml
  # Note that generator emits OS info for build so these should be the same.
  image: { "name": "ghcr.io/spack/e4s-amazonlinux-2:v2023-03-09", "entrypoint": [""] }

.aws-isc:
  extends: [ ".linux_x86_64_v3" ]
  variables:
    SPACK_CI_STACK_NAME: aws-isc

aws-isc-generate:
  extends: [ ".aws-isc", ".generate-x86_64", ".aws-isc-overrides", ".tags-x86_64_v4" ]

aws-isc-build:
  extends: [ ".aws-isc", ".build" ]
  trigger:
    include:
      - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
        job: aws-isc-generate
    strategy: depend
  needs:
    - artifacts: True
      job: aws-isc-generate

# Parallel Pipeline for aarch64 (reuses override image, but generates and builds on aarch64)

.aws-isc-aarch64:
  extends: [ ".linux_aarch64" ]
  variables:
    SPACK_CI_STACK_NAME: aws-isc-aarch64

aws-isc-aarch64-generate:
  extends: [ ".aws-isc-aarch64", ".generate-aarch64", ".aws-isc-overrides" ]

aws-isc-aarch64-build:
  extends: [ ".aws-isc-aarch64", ".build" ]
  trigger:
    include:
      - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
        job: aws-isc-aarch64-generate
    strategy: depend
  needs:
    - artifacts: True
      job: aws-isc-aarch64-generate


########################################
# Spack Tutorial
########################################
.tutorial:
  extends: [ ".linux_x86_64_v3" ]
  variables:
    SPACK_CI_STACK_NAME: tutorial

tutorial-generate:
  extends: [ ".tutorial", ".generate-x86_64"]
  image: ghcr.io/spack/tutorial-ubuntu-22.04:v2023-05-07

tutorial-build:
  extends: [ ".tutorial", ".build" ]
  trigger:
    include:
      - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
        job: tutorial-generate
    strategy: depend
  needs:
    - artifacts: True
      job: tutorial-generate

#######################################
# Machine Learning - Linux x86_64 (CPU)
#######################################
.ml-linux-x86_64-cpu:
  extends: [ ".linux_x86_64_v3" ]
  variables:
    SPACK_CI_STACK_NAME: ml-linux-x86_64-cpu

ml-linux-x86_64-cpu-generate:
  extends: [ ".generate-x86_64", .ml-linux-x86_64-cpu, ".tags-x86_64_v4" ]
  image: ghcr.io/spack/linux-ubuntu22.04-x86_64_v2:v2023-07-01

ml-linux-x86_64-cpu-build:
  extends: [ ".build", ".ml-linux-x86_64-cpu" ]
  trigger:
    include:
      - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
        job: ml-linux-x86_64-cpu-generate
    strategy: depend
  needs:
    - artifacts: True
      job: ml-linux-x86_64-cpu-generate

########################################
# Machine Learning - Linux x86_64 (CUDA)
########################################
.ml-linux-x86_64-cuda:
  extends: [ ".linux_x86_64_v3" ]
  variables:
    SPACK_CI_STACK_NAME: ml-linux-x86_64-cuda

ml-linux-x86_64-cuda-generate:
  extends: [ ".generate-x86_64", .ml-linux-x86_64-cuda, ".tags-x86_64_v4" ]
  image: ghcr.io/spack/linux-ubuntu22.04-x86_64_v2:v2023-07-01

ml-linux-x86_64-cuda-build:
  extends: [ ".build", ".ml-linux-x86_64-cuda"  ]
  trigger:
    include:
      - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
        job: ml-linux-x86_64-cuda-generate
    strategy: depend
  needs:
    - artifacts: True
      job: ml-linux-x86_64-cuda-generate

########################################
# Machine Learning - Linux x86_64 (ROCm)
########################################
.ml-linux-x86_64-rocm:
  extends: [ ".linux_x86_64_v3" ]
  variables:
    SPACK_CI_STACK_NAME: ml-linux-x86_64-rocm

ml-linux-x86_64-rocm-generate:
  extends: [ ".generate-x86_64", .ml-linux-x86_64-rocm, ".tags-x86_64_v4" ]
  image: ghcr.io/spack/linux-ubuntu22.04-x86_64_v2:v2023-07-01

ml-linux-x86_64-rocm-build:
  extends: [ ".build", ".ml-linux-x86_64-rocm" ]
  trigger:
    include:
      - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
        job: ml-linux-x86_64-rocm-generate
    strategy: depend
  needs:
    - artifacts: True
      job: ml-linux-x86_64-rocm-generate

########################################
# Machine Learning - Darwin aarch64 (MPS)
########################################
.ml-darwin-aarch64-mps:
  extends: [".darwin_aarch64"]
  variables:
    SPACK_CI_STACK_NAME: ml-darwin-aarch64-mps

ml-darwin-aarch64-mps-generate:
  tags: [ "macos-ventura", "apple-clang-14", "aarch64-macos" ]
  extends: [ ".ml-darwin-aarch64-mps", ".generate-base"]

ml-darwin-aarch64-mps-build:
  extends: [ ".ml-darwin-aarch64-mps", ".build" ]
  trigger:
    include:
      - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
        job: ml-darwin-aarch64-mps-generate
    strategy: depend
  needs:
    - artifacts: True
      job: ml-darwin-aarch64-mps-generate

########################################
# Deprecated CI testing
########################################
.deprecated-ci:
  variables:
    SPACK_CI_STACK_NAME: deprecated

deprecated-ci-generate:
  extends: [ ".generate-deprecated", ".deprecated-ci" ]

deprecated-ci-build:
  extends: [ ".build", ".deprecated-ci" ]
  trigger:
    include:
      - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
        job: deprecated-ci-generate
    strategy: depend
  needs:
    - artifacts: True
      job: deprecated-ci-generate

########################################
# AWS PCLUSTER
########################################

.aws-pcluster-generate-image:
  image: { "name": "ghcr.io/spack/pcluster-amazonlinux-2:v2023-05-25", "entrypoint": [""] }

.aws-pcluster-generate:
  before_script:
    # Use gcc from local container buildcache
    - - . "./share/spack/setup-env.sh"
      - . /etc/profile.d/modules.sh
      - spack buildcache rebuild-index /bootstrap/local-cache/
      - spack mirror add local-cache /bootstrap/local-cache
      - spack gpg trust /bootstrap/public-key
      - cd "${CI_PROJECT_DIR}" && curl -sOL https://raw.githubusercontent.com/spack/spack-configs/main/AWS/parallelcluster/postinstall.sh
      - sed -i -e "s/spack arch -t/echo ${SPACK_TARGET_ARCH}/g" postinstall.sh
      - sed -i.bkp s/"spack install gcc"/"spack install --cache-only --reuse gcc"/ postinstall.sh
      - diff postinstall.sh postinstall.sh.bkp || echo Done
      - /bin/bash postinstall.sh -fg
      - spack config --scope site add "packages:all:target:[${SPACK_TARGET_ARCH}]"
  after_script:
    - - mv "${CI_PROJECT_DIR}/postinstall.sh" "${CI_PROJECT_DIR}/jobs_scratch_dir/"

# Icelake (one pipeline per target)
.aws-pcluster-icelake:
  variables:
    SPACK_CI_STACK_NAME: aws-pcluster-icelake

# aws-pcluster-generate-icelake:
#   extends: [ ".linux_icelake", ".aws-pcluster-icelake", ".generate-x86_64", ".tags-x86_64_v4", ".aws-pcluster-generate", ".aws-pcluster-generate-image" ]

# aws-pcluster-build-icelake:
#   extends: [ ".linux_icelake", ".aws-pcluster-icelake", ".build" ]
#   trigger:
#     include:
#       - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
#         job: aws-pcluster-generate-icelake
#     strategy: depend
#   needs:
#     - artifacts: True
#       job: aws-pcluster-generate-icelake

# Skylake_avx512 (one pipeline per target)
.aws-pcluster-skylake:
  variables:
    SPACK_CI_STACK_NAME: aws-pcluster-skylake

# aws-pcluster-generate-skylake:
#   extends: [ ".linux_skylake", ".aws-pcluster-skylake", ".generate-x86_64", ".tags-x86_64_v4", ".aws-pcluster-generate", ".aws-pcluster-generate-image" ]

# aws-pcluster-build-skylake:
#   extends: [ ".linux_skylake", ".aws-pcluster-skylake", ".build" ]
#   trigger:
#     include:
#       - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
#         job: aws-pcluster-generate-skylake
#     strategy: depend
#   needs:
#     - artifacts: True
#       job: aws-pcluster-generate-skylake

# Neoverse_n1 (one pipeline per target)
.aws-pcluster-neoverse_n1:
  variables:
    SPACK_CI_STACK_NAME: aws-pcluster-neoverse_n1

# aws-pcluster-generate-neoverse_n1:
#   extends: [ ".linux_neoverse_n1", ".aws-pcluster-neoverse_n1", ".generate-aarch64", ".aws-pcluster-generate", ".aws-pcluster-generate-image" ]

# aws-pcluster-build-neoverse_n1:
#   extends: [  ".linux_neoverse_n1", ".aws-pcluster-neoverse_n1", ".build" ]
#   trigger:
#     include:
#       - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
#         job: aws-pcluster-generate-neoverse_n1
#     strategy: depend
#   needs:
#     - artifacts: True
#       job: aws-pcluster-generate-neoverse_n1

# Neoverse_v1 (one pipeline per target)
.aws-pcluster-neoverse_v1:
  variables:
    SPACK_CI_STACK_NAME: aws-pcluster-neoverse_v1

# aws-pcluster-generate-neoverse_v1:
#   extends: [ ".linux_neoverse_v1", ".aws-pcluster-neoverse_v1", ".generate-aarch64", ".aws-pcluster-generate", ".aws-pcluster-generate-image" ]

# aws-pcluster-build-neoverse_v1:
#   extends: [  ".linux_neoverse_v1", ".aws-pcluster-neoverse_v1", ".build" ]
#   trigger:
#     include:
#       - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
#         job: aws-pcluster-generate-neoverse_v1
#     strategy: depend
#   needs:
#     - artifacts: True
#       job: aws-pcluster-generate-neoverse_v1

# Cray definitions
.generate-cray:
  extends: [ ".generate-common", ".base-job" ]
  before_script:
    - echo $PATH
    - module avail
    - module list

.generate-cray-rhel:
  tags: [ "cray-rhel-zen4", "public" ]
  extends: [ ".generate-cray" ]

.generate-cray-sles:
  tags: [ "cray-sles-zen4", "public" ]
  extends: [ ".generate-cray" ]


#######################################
# E4S - Cray RHEL
#######################################
.e4s-cray-rhel:
  extends: [ ".cray_rhel_zen4" ]
  variables:
    SPACK_CI_STACK_NAME: e4s-cray-rhel

e4s-cray-rhel-generate:
  extends: [ ".generate-cray-rhel", ".e4s-cray-rhel" ]

e4s-cray-rhel-build:
  extends: [ ".build", ".e4s-cray-rhel" ]
  trigger:
    include:
      - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
        job: e4s-cray-rhel-generate
    strategy: depend
  needs:
    - artifacts: True
      job: e4s-cray-rhel-generate

#######################################
# E4S - Cray SLES
#######################################
.e4s-cray-sles:
  extends: [ ".cray_sles_zen4" ]
  variables:
    SPACK_CI_STACK_NAME: e4s-cray-sles

# e4s-cray-sles-generate:
#   extends: [ ".generate-cray-sles", ".e4s-cray-sles" ]

# e4s-cray-sles-build:
#   extends: [ ".build", ".e4s-cray-sles" ]
#   trigger:
#     include:
#       - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
#         job: e4s-cray-sles-generate
#     strategy: depend
#   needs:
#     - artifacts: True
#       job: e4s-cray-sles-generate