From c08be95d5e40f9316b65014f437b7deb7c38a8f6 Mon Sep 17 00:00:00 2001 From: Scott Wittenburg Date: Fri, 12 May 2023 14:22:42 -0600 Subject: gitlab ci: release fixes and improvements (#37601) * gitlab ci: release fixes and improvements - use rules to reduce boilerplate in .gitlab-ci.yml - support copy-only pipeline jobs - make pipelines for release branches rebuild everything - make pipelines for protected tags copy-only * gitlab ci: remove url changes used in testing * gitlab ci: tag mirrors need public key Make sure that mirrors associated with release branches and tags contain the public key needed to verify the signed binaries. This also ensures that when stack-specific mirror contents are copied to the root, the root mirror has the public key as well. * review: be more specific about tags, curl flags * Make the check in ci.yaml consistent with the .gitlab-ci.yml --------- Co-authored-by: Ryan Krattiger --- lib/spack/spack/ci.py | 20 +- lib/spack/spack/schema/ci.py | 5 + share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml | 629 ++++++--------------- share/spack/gitlab/cloud_pipelines/configs/ci.yaml | 28 + 4 files changed, 218 insertions(+), 464 deletions(-) diff --git a/lib/spack/spack/ci.py b/lib/spack/spack/ci.py index f3d6398416..4589b5bca4 100644 --- a/lib/spack/spack/ci.py +++ b/lib/spack/spack/ci.py @@ -531,7 +531,7 @@ class SpackCI: """ self.ci_config = ci_config - self.named_jobs = ["any", "build", "cleanup", "noop", "reindex", "signing"] + self.named_jobs = ["any", "build", "copy", "cleanup", "noop", "reindex", "signing"] self.ir = { "jobs": {}, @@ -1207,7 +1207,7 @@ def generate_gitlab_ci_yaml( ).format(c_spec, release_spec) tty.debug(debug_msg) - if prune_dag and not rebuild_spec: + if prune_dag and not rebuild_spec and spack_pipeline_type != "spack_copy_only": tty.debug( "Pruning {0}/{1}, does not need rebuild.".format( release_spec.name, release_spec.dag_hash() @@ -1298,8 +1298,9 @@ def generate_gitlab_ci_yaml( max_length_needs = length_needs max_needs_job = job_name - output_object[job_name] = job_object - job_id += 1 + if spack_pipeline_type != "spack_copy_only": + output_object[job_name] = job_object + job_id += 1 if print_summary: for phase in phases: @@ -1329,6 +1330,17 @@ def generate_gitlab_ci_yaml( "when": ["runner_system_failure", "stuck_or_timeout_failure", "script_failure"], } + if spack_pipeline_type == "spack_copy_only": + stage_names.append("copy") + sync_job = copy.deepcopy(spack_ci_ir["jobs"]["copy"]["attributes"]) + sync_job["stage"] = "copy" + if artifacts_root: + sync_job["needs"] = [ + {"job": generate_job_name, "pipeline": "{0}".format(parent_pipeline_id)} + ] + output_object["copy"] = sync_job + job_id += 1 + if job_id > 0: if temp_storage_url_prefix: # There were some rebuild jobs scheduled, so we will need to diff --git a/lib/spack/spack/schema/ci.py b/lib/spack/spack/schema/ci.py index 2c22f071de..667960f372 100644 --- a/lib/spack/spack/schema/ci.py +++ b/lib/spack/spack/schema/ci.py @@ -89,6 +89,11 @@ named_attributes_schema = { "additionalProperties": False, "properties": {"build-job": attributes_schema, "build-job-remove": attributes_schema}, }, + { + "type": "object", + "additionalProperties": False, + "properties": {"copy-job": attributes_schema, "copy-job-remove": attributes_schema}, + }, { "type": "object", "additionalProperties": False, diff --git a/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml b/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml index bdd664b8d1..31c3ba8064 100644 --- a/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml +++ b/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml @@ -27,41 +27,47 @@ default: ######################################## # Job templates ######################################## - -.aws-pr-creds: - variables: - AWS_ACCESS_KEY_ID: ${PR_MIRRORS_AWS_ACCESS_KEY_ID} - AWS_SECRET_ACCESS_KEY: ${PR_MIRRORS_AWS_SECRET_ACCESS_KEY} - -.aws-protected-creds: - variables: - AWS_ACCESS_KEY_ID: ${PROTECTED_MIRRORS_AWS_ACCESS_KEY_ID} - AWS_SECRET_ACCESS_KEY: ${PROTECTED_MIRRORS_AWS_SECRET_ACCESS_KEY} - -.pr-refs: - only: - - /^pr[\d]+_.*$/ - -.pr: - extends: [ ".pr-refs" ] - variables: - SPACK_BUILDCACHE_DESTINATION: "s3://spack-binaries-prs/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}" - SPACK_PIPELINE_TYPE: "spack_pull_request" - SPACK_PRUNE_UNTOUCHED: "True" - SPACK_PRUNE_UNTOUCHED_DEPENDENT_DEPTH: "1" - -.protected-refs: - only: - - /^develop$/ - - /^releases\/v.*/ - - /^v.*/ - -.protected: - extends: [ ".protected-refs" ] +.base-job: variables: SPACK_BUILDCACHE_DESTINATION: "s3://spack-binaries/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}" - SPACK_COPY_BUILDCACHE: "s3://spack-binaries/${CI_COMMIT_REF_NAME}" - SPACK_PIPELINE_TYPE: "spack_protected_branch" + rules: + - if: $CI_COMMIT_REF_NAME == "develop" + # Pipelines on develop only rebuild what is missing from the mirror + when: always + variables: + SPACK_PIPELINE_TYPE: "spack_protected_branch" + SPACK_COPY_BUILDCACHE: "s3://spack-binaries/${CI_COMMIT_REF_NAME}" + AWS_ACCESS_KEY_ID: ${PROTECTED_MIRRORS_AWS_ACCESS_KEY_ID} + AWS_SECRET_ACCESS_KEY: ${PROTECTED_MIRRORS_AWS_SECRET_ACCESS_KEY} + - if: $CI_COMMIT_REF_NAME =~ /^releases\/v.*/ + # Pipelines on release branches always rebuild everything + when: always + variables: + SPACK_PIPELINE_TYPE: "spack_protected_branch" + SPACK_COPY_BUILDCACHE: "s3://spack-binaries/${CI_COMMIT_REF_NAME}" + SPACK_PRUNE_UNTOUCHED: "False" + SPACK_PRUNE_UP_TO_DATE: "False" + AWS_ACCESS_KEY_ID: ${PROTECTED_MIRRORS_AWS_ACCESS_KEY_ID} + AWS_SECRET_ACCESS_KEY: ${PROTECTED_MIRRORS_AWS_SECRET_ACCESS_KEY} + - if: $CI_COMMIT_TAG =~ /^develop-[\d]{4}-[\d]{2}-[\d]{2}$/ || $CI_COMMIT_TAG =~ /^v.*/ + # Pipelines on tags (release or dev snapshots) only copy binaries from one mirror to another + when: always + variables: + SPACK_PIPELINE_TYPE: "spack_copy_only" + SPACK_SOURCE_MIRROR: "s3://spack-binaries/SPACK_REPLACE_VERSION/${SPACK_CI_STACK_NAME}" + SPACK_COPY_BUILDCACHE: "s3://spack-binaries/${CI_COMMIT_REF_NAME}" + AWS_ACCESS_KEY_ID: ${PROTECTED_MIRRORS_AWS_ACCESS_KEY_ID} + AWS_SECRET_ACCESS_KEY: ${PROTECTED_MIRRORS_AWS_SECRET_ACCESS_KEY} + - if: $CI_COMMIT_REF_NAME =~ /^pr[\d]+_.*$/ + # Pipelines on PR branches rebuild only what's missing, and do extra pruning + when: always + variables: + SPACK_PIPELINE_TYPE: "spack_pull_request" + SPACK_BUILDCACHE_DESTINATION: "s3://spack-binaries-prs/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}" + SPACK_PRUNE_UNTOUCHED: "True" + SPACK_PRUNE_UNTOUCHED_DEPENDENT_DEPTH: "1" + AWS_ACCESS_KEY_ID: ${PR_MIRRORS_AWS_ACCESS_KEY_ID} + AWS_SECRET_ACCESS_KEY: ${PR_MIRRORS_AWS_SECRET_ACCESS_KEY} .generate-base: stage: generate @@ -99,10 +105,11 @@ default: - always .generate: - extends: [ ".generate-base" ] + extends: [ ".base-job", ".generate-base" ] tags: ["spack", "public", "medium", "x86_64"] .generate-deprecated: + extends: [ ".base-job" ] stage: generate script: - uname -a || true @@ -134,45 +141,30 @@ default: tags: ["spack", "public", "medium", "x86_64"] .generate-aarch64: - extends: [ ".generate" ] + extends: [ ".base-job", ".generate" ] tags: ["spack", "public", "medium", "aarch64"] -.pr-generate: - extends: [ ".pr", ".generate" ] - -.pr-generate-deprecated: - extends: [ ".pr", ".generate-deprecated" ] - -.pr-generate-aarch64: - extends: [ ".pr", ".generate-aarch64" ] - -.protected-generate: - extends: [ ".protected", ".generate" ] - -.protected-generate-deprecated: - extends: [ ".protected", ".generate-deprecated" ] - -.protected-generate-aarch64: - extends: [ ".protected", ".generate-aarch64" ] - .build: + extends: [ ".base-job" ] stage: build -.pr-build: - extends: [ ".pr", ".build", ".aws-pr-creds" ] - -.protected-build: - extends: [ ".protected", ".build", ".aws-protected-creds" ] - protected-publish: + # Copy binaries from stack-specific mirrors to a root mirror stage: publish - extends: [ ".protected", ".aws-protected-creds" ] + only: + - /^develop$/ + - /^releases\/v.*/ + - /^v.*/ + - /^develop-[\d]{4}-[\d]{2}-[\d]{2}$/ image: "ghcr.io/spack/python-aws-bash:0.0.1" tags: ["spack", "public", "medium", "aws", "x86_64"] retry: max: 2 when: ["runner_system_failure", "stuck_or_timeout_failure"] variables: + SPACK_BUILDCACHE_DESTINATION: "s3://spack-binaries/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}" + SPACK_COPY_BUILDCACHE: "s3://spack-binaries/${CI_COMMIT_REF_NAME}" + SPACK_PIPELINE_TYPE: "spack_protected_branch" AWS_ACCESS_KEY_ID: ${PROTECTED_MIRRORS_AWS_ACCESS_KEY_ID} AWS_SECRET_ACCESS_KEY: ${PROTECTED_MIRRORS_AWS_SECRET_ACCESS_KEY} KUBERNETES_CPU_REQUEST: 4000m @@ -182,7 +174,9 @@ protected-publish: - spack --version - export COPY_SPECS_DIR=${CI_PROJECT_DIR}/jobs_scratch_dir/specs_to_copy - spack buildcache sync --manifest-glob "${COPY_SPECS_DIR}/*.json" - - spack buildcache update-index "${SPACK_COPY_BUILDCACHE}" + - curl -fLsS https://spack.github.io/keys/spack-public-binary-key.pub -o /tmp/spack-public-binary-key.pub + - aws s3 cp /tmp/spack-public-binary-key.pub "${SPACK_COPY_BUILDCACHE}/build_cache/_pgp/spack-public-binary-key.pub" + - spack buildcache update-index --keys "${SPACK_COPY_BUILDCACHE}" ######################################## # TEMPLATE FOR ADDING ANOTHER PIPELINE @@ -208,33 +202,19 @@ protected-publish: # SPACK_CI_STACK_NAME: my-super-cool-stack # tags: [ "all", "tags", "your", "job", "needs"] # -# my-super-cool-stack-pr-generate: -# extends: [ ".pr-generate", ".my-super-cool-stack" ] -# -# my-super-cool-stack-protected-generate: -# extends: [ ".protected-generate", ".my-super-cool-stack" ] -# -# my-super-cool-stack-pr-build: -# extends: [ ".pr-build", ".my-super-cool-stack" ] -# trigger: -# include: -# - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml -# job: my-super-cool-stack-pr-generate -# strategy: depend -# needs: -# - artifacts: True -# job: my-super-cool-stack-pr-generate +# my-super-cool-stack-generate: +# extends: [ ".generate", ".my-super-cool-stack" ] # -# my-super-cool-stack-protected-build: -# extends: [ ".protected-build", ".my-super-cool-stack" ] +# my-super-cool-stack-build: +# extends: [ ".build", ".my-super-cool-stack" ] # trigger: # include: # - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml -# job: my-super-cool-stack-protected-generate +# job: my-super-cool-stack-generate # strategy: depend # needs: # - artifacts: True -# job: my-super-cool-stack-protected-generate +# job: my-super-cool-stack-generate ######################################## # E4S Mac Stack @@ -362,35 +342,20 @@ protected-publish: variables: SPACK_CI_STACK_NAME: e4s -e4s-pr-generate: - extends: [ ".e4s", ".pr-generate"] - image: ecpe4s/ubuntu20.04-runner-x86_64:2023-01-01 - -e4s-protected-generate: - extends: [ ".e4s", ".protected-generate"] +e4s-generate: + extends: [ ".e4s", ".generate"] image: ecpe4s/ubuntu20.04-runner-x86_64:2023-01-01 -e4s-pr-build: - extends: [ ".e4s", ".pr-build" ] - trigger: - include: - - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml - job: e4s-pr-generate - strategy: depend - needs: - - artifacts: True - job: e4s-pr-generate - -e4s-protected-build: - extends: [ ".e4s", ".protected-build" ] +e4s-build: + extends: [ ".e4s", ".build" ] trigger: include: - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml - job: e4s-protected-generate + job: e4s-generate strategy: depend needs: - artifacts: True - job: e4s-protected-generate + job: e4s-generate ######################################## # GPU Testing Pipeline @@ -400,35 +365,20 @@ e4s-protected-build: variables: SPACK_CI_STACK_NAME: gpu-tests -gpu-tests-pr-generate: - extends: [ ".gpu-tests", ".pr-generate"] - image: ecpe4s/ubuntu20.04-runner-x86_64:2023-01-01 - -gpu-tests-protected-generate: - extends: [ ".gpu-tests", ".protected-generate"] +gpu-tests-generate: + extends: [ ".gpu-tests", ".generate"] image: ecpe4s/ubuntu20.04-runner-x86_64:2023-01-01 -gpu-tests-pr-build: - extends: [ ".gpu-tests", ".pr-build" ] +gpu-tests-build: + extends: [ ".gpu-tests", ".build" ] trigger: include: - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml - job: gpu-tests-pr-generate + job: gpu-tests-generate strategy: depend needs: - artifacts: True - job: gpu-tests-pr-generate - -gpu-tests-protected-build: - extends: [ ".gpu-tests", ".protected-build" ] - trigger: - include: - - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml - job: gpu-tests-protected-generate - strategy: depend - needs: - - artifacts: True - job: gpu-tests-protected-generate + job: gpu-tests-generate ######################################## # E4S OneAPI Pipeline @@ -438,35 +388,20 @@ gpu-tests-protected-build: variables: SPACK_CI_STACK_NAME: e4s-oneapi -e4s-oneapi-pr-generate: - extends: [ ".e4s-oneapi", ".pr-generate"] +e4s-oneapi-generate: + extends: [ ".e4s-oneapi", ".generate"] image: ecpe4s/ubuntu20.04-runner-x86_64-oneapi:2023-01-01 -e4s-oneapi-protected-generate: - extends: [ ".e4s-oneapi", ".protected-generate"] - image: ecpe4s/ubuntu20.04-runner-x86_64-oneapi:2023-01-01 - -e4s-oneapi-pr-build: - extends: [ ".e4s-oneapi", ".pr-build" ] +e4s-oneapi-build: + extends: [ ".e4s-oneapi", ".build" ] trigger: include: - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml - job: e4s-oneapi-pr-generate + job: e4s-oneapi-generate strategy: depend needs: - artifacts: True - job: e4s-oneapi-pr-generate - -e4s-oneapi-protected-build: - extends: [ ".e4s-oneapi", ".protected-build" ] - trigger: - include: - - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml - job: e4s-oneapi-protected-generate - strategy: depend - needs: - - artifacts: True - job: e4s-oneapi-protected-generate + job: e4s-oneapi-generate ######################################## # E4S on Power @@ -480,33 +415,19 @@ e4s-oneapi-protected-build: variables: SPACK_CI_STACK_NAME: e4s-power -e4s-power-pr-generate: - extends: [ ".e4s-power", ".pr-generate", ".e4s-power-generate-tags-and-image"] +e4s-power-generate: + extends: [ ".e4s-power", ".generate", ".e4s-power-generate-tags-and-image"] -e4s-power-protected-generate: - extends: [ ".e4s-power", ".protected-generate", ".e4s-power-generate-tags-and-image"] - -e4s-power-pr-build: - extends: [ ".e4s-power", ".pr-build" ] +e4s-power-build: + extends: [ ".e4s-power", ".build" ] trigger: include: - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml - job: e4s-power-pr-generate + job: e4s-power-generate strategy: depend needs: - artifacts: True - job: e4s-power-pr-generate - -e4s-power-protected-build: - extends: [ ".e4s-power", ".protected-build" ] - trigger: - include: - - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml - job: e4s-power-protected-generate - strategy: depend - needs: - - artifacts: True - job: e4s-power-protected-generate + job: e4s-power-generate ######################################### # Build tests for different build-systems @@ -516,33 +437,19 @@ e4s-power-protected-build: variables: SPACK_CI_STACK_NAME: build_systems -build_systems-pr-generate: - extends: [ ".build_systems", ".pr-generate"] - -build_systems-protected-generate: - extends: [ ".build_systems", ".protected-generate"] - -build_systems-pr-build: - extends: [ ".build_systems", ".pr-build" ] - trigger: - include: - - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml - job: build_systems-pr-generate - strategy: depend - needs: - - artifacts: True - job: build_systems-pr-generate +build_systems-generate: + extends: [ ".build_systems", ".generate"] -build_systems-protected-build: - extends: [ ".build_systems", ".protected-build" ] +build_systems-build: + extends: [ ".build_systems", ".build" ] trigger: include: - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml - job: build_systems-protected-generate + job: build_systems-generate strategy: depend needs: - artifacts: True - job: build_systems-protected-generate + job: build_systems-generate ######################################### # RADIUSS @@ -552,35 +459,19 @@ build_systems-protected-build: variables: SPACK_CI_STACK_NAME: radiuss -# --------- PRs --------- -radiuss-pr-generate: - extends: [ ".radiuss", ".pr-generate" ] - -radiuss-pr-build: - extends: [ ".radiuss", ".pr-build" ] - trigger: - include: - - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml - job: radiuss-pr-generate - strategy: depend - needs: - - artifacts: True - job: radiuss-pr-generate - -# --------- Protected --------- -radiuss-protected-generate: - extends: [ ".radiuss", ".protected-generate" ] +radiuss-generate: + extends: [ ".radiuss", ".generate" ] -radiuss-protected-build: - extends: [ ".radiuss", ".protected-build" ] +radiuss-build: + extends: [ ".radiuss", ".build" ] trigger: include: - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml - job: radiuss-protected-generate + job: radiuss-generate strategy: depend needs: - artifacts: True - job: radiuss-protected-generate + job: radiuss-generate ######################################## # RADIUSS for AWS @@ -600,33 +491,19 @@ radiuss-protected-build: variables: SPACK_CI_STACK_NAME: radiuss-aws -radiuss-aws-pr-generate: - extends: [ ".radiuss-aws", ".pr-generate", ".radiuss-aws-overrides", ".tags-x86_64_v4" ] - -radiuss-aws-protected-generate: - extends: [ ".radiuss-aws", ".protected-generate", ".radiuss-aws-overrides", ".tags-x86_64_v4" ] - -radiuss-aws-pr-build: - extends: [ ".radiuss-aws", ".pr-build" ] - trigger: - include: - - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml - job: radiuss-aws-pr-generate - strategy: depend - needs: - - artifacts: True - job: radiuss-aws-pr-generate +radiuss-aws-generate: + extends: [ ".radiuss-aws", ".generate", ".radiuss-aws-overrides", ".tags-x86_64_v4" ] -radiuss-aws-protected-build: - extends: [ ".radiuss-aws", ".protected-build" ] +radiuss-aws-build: + extends: [ ".radiuss-aws", ".build" ] trigger: include: - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml - job: radiuss-aws-protected-generate + job: radiuss-aws-generate strategy: depend needs: - artifacts: True - job: radiuss-aws-protected-generate + job: radiuss-aws-generate # Parallel Pipeline for aarch64 (reuses override image, but generates and builds on aarch64) @@ -636,33 +513,19 @@ radiuss-aws-protected-build: variables: SPACK_CI_STACK_NAME: radiuss-aws-aarch64 -radiuss-aws-aarch64-pr-generate: - extends: [ ".radiuss-aws-aarch64", ".pr-generate-aarch64", ".radiuss-aws-overrides" ] +radiuss-aws-aarch64-generate: + extends: [ ".radiuss-aws-aarch64", ".generate-aarch64", ".radiuss-aws-overrides" ] -radiuss-aws-aarch64-protected-generate: - extends: [ ".radiuss-aws-aarch64", ".protected-generate-aarch64", ".radiuss-aws-overrides" ] - -radiuss-aws-aarch64-pr-build: - extends: [ ".radiuss-aws-aarch64", ".pr-build" ] +radiuss-aws-aarch64-build: + extends: [ ".radiuss-aws-aarch64", ".build" ] trigger: include: - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml - job: radiuss-aws-aarch64-pr-generate + job: radiuss-aws-aarch64-generate strategy: depend needs: - artifacts: True - job: radiuss-aws-aarch64-pr-generate - -radiuss-aws-aarch64-protected-build: - extends: [ ".radiuss-aws-aarch64", ".protected-build" ] - trigger: - include: - - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml - job: radiuss-aws-aarch64-protected-generate - strategy: depend - needs: - - artifacts: True - job: radiuss-aws-aarch64-protected-generate + job: radiuss-aws-aarch64-generate ######################################## # ECP Data & Vis SDK @@ -672,35 +535,20 @@ radiuss-aws-aarch64-protected-build: variables: SPACK_CI_STACK_NAME: data-vis-sdk -data-vis-sdk-pr-generate: - extends: [ ".data-vis-sdk", ".pr-generate"] +data-vis-sdk-generate: + extends: [ ".data-vis-sdk", ".generate"] image: ecpe4s/ubuntu20.04-runner-x86_64:2023-01-01 -data-vis-sdk-protected-generate: - extends: [ ".data-vis-sdk", ".protected-generate"] - image: ecpe4s/ubuntu20.04-runner-x86_64:2023-01-01 - -data-vis-sdk-pr-build: - extends: [ ".data-vis-sdk", ".pr-build" ] +data-vis-sdk-build: + extends: [ ".data-vis-sdk", ".build" ] trigger: include: - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml - job: data-vis-sdk-pr-generate + job: data-vis-sdk-generate strategy: depend needs: - artifacts: True - job: data-vis-sdk-pr-generate - -data-vis-sdk-protected-build: - extends: [ ".data-vis-sdk", ".protected-build" ] - trigger: - include: - - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml - job: data-vis-sdk-protected-generate - strategy: depend - needs: - - artifacts: True - job: data-vis-sdk-protected-generate + job: data-vis-sdk-generate ######################################## # AWS AHUG Applications (x86_64) @@ -717,34 +565,19 @@ data-vis-sdk-protected-build: variables: SPACK_CI_STACK_NAME: aws-ahug -aws-ahug-pr-generate: - extends: [ ".aws-ahug", ".pr-generate", ".aws-ahug-overrides", ".tags-x86_64_v4" ] +aws-ahug-generate: + extends: [ ".aws-ahug", ".generate", ".aws-ahug-overrides", ".tags-x86_64_v4" ] -aws-ahug-protected-generate: - extends: [ ".aws-ahug", ".protected-generate", ".aws-ahug-overrides", ".tags-x86_64_v4" ] - -aws-ahug-pr-build: - extends: [ ".aws-ahug", ".pr-build" ] +aws-ahug-build: + extends: [ ".aws-ahug", ".build" ] trigger: include: - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml - job: aws-ahug-pr-generate + job: aws-ahug-generate strategy: depend needs: - artifacts: True - job: aws-ahug-pr-generate - -aws-ahug-protected-build: - extends: [ ".aws-ahug", ".protected-build" ] - trigger: - include: - - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml - job: aws-ahug-protected-generate - strategy: depend - needs: - - artifacts: True - job: aws-ahug-protected-generate - + job: aws-ahug-generate # Parallel Pipeline for aarch64 (reuses override image, but generates and builds on aarch64) .aws-ahug-aarch64: @@ -752,33 +585,19 @@ aws-ahug-protected-build: variables: SPACK_CI_STACK_NAME: aws-ahug-aarch64 -aws-ahug-aarch64-pr-generate: - extends: [ ".aws-ahug-aarch64", ".pr-generate-aarch64", ".aws-ahug-overrides" ] - -aws-ahug-aarch64-protected-generate: - extends: [ ".aws-ahug-aarch64", ".protected-generate-aarch64", ".aws-ahug-overrides" ] - -aws-ahug-aarch64-pr-build: - extends: [ ".aws-ahug-aarch64", ".pr-build" ] - trigger: - include: - - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml - job: aws-ahug-aarch64-pr-generate - strategy: depend - needs: - - artifacts: True - job: aws-ahug-aarch64-pr-generate +aws-ahug-aarch64-generate: + extends: [ ".aws-ahug-aarch64", ".generate-aarch64", ".aws-ahug-overrides" ] -aws-ahug-aarch64-protected-build: - extends: [ ".aws-ahug-aarch64", ".protected-build" ] +aws-ahug-aarch64-build: + extends: [ ".aws-ahug-aarch64", ".build" ] trigger: include: - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml - job: aws-ahug-aarch64-protected-generate + job: aws-ahug-aarch64-generate strategy: depend needs: - artifacts: True - job: aws-ahug-aarch64-protected-generate + job: aws-ahug-aarch64-generate ######################################## # AWS ISC Applications (x86_64) @@ -795,34 +614,19 @@ aws-ahug-aarch64-protected-build: variables: SPACK_CI_STACK_NAME: aws-isc -aws-isc-pr-generate: - extends: [ ".aws-isc", ".pr-generate", ".aws-isc-overrides", ".tags-x86_64_v4" ] - -aws-isc-protected-generate: - extends: [ ".aws-isc", ".protected-generate", ".aws-isc-overrides", ".tags-x86_64_v4" ] +aws-isc-generate: + extends: [ ".aws-isc", ".generate", ".aws-isc-overrides", ".tags-x86_64_v4" ] -aws-isc-pr-build: - extends: [ ".aws-isc", ".pr-build" ] +aws-isc-build: + extends: [ ".aws-isc", ".build" ] trigger: include: - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml - job: aws-isc-pr-generate + job: aws-isc-generate strategy: depend needs: - artifacts: True - job: aws-isc-pr-generate - -aws-isc-protected-build: - extends: [ ".aws-isc", ".protected-build" ] - trigger: - include: - - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml - job: aws-isc-protected-generate - strategy: depend - needs: - - artifacts: True - job: aws-isc-protected-generate - + job: aws-isc-generate # Parallel Pipeline for aarch64 (reuses override image, but generates and builds on aarch64) @@ -831,33 +635,19 @@ aws-isc-protected-build: variables: SPACK_CI_STACK_NAME: aws-isc-aarch64 -aws-isc-aarch64-pr-generate: - extends: [ ".aws-isc-aarch64", ".pr-generate-aarch64", ".aws-isc-overrides" ] - -aws-isc-aarch64-protected-generate: - extends: [ ".aws-isc-aarch64", ".protected-generate-aarch64", ".aws-isc-overrides" ] - -aws-isc-aarch64-pr-build: - extends: [ ".aws-isc-aarch64", ".pr-build" ] - trigger: - include: - - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml - job: aws-isc-aarch64-pr-generate - strategy: depend - needs: - - artifacts: True - job: aws-isc-aarch64-pr-generate +aws-isc-aarch64-generate: + extends: [ ".aws-isc-aarch64", ".generate-aarch64", ".aws-isc-overrides" ] -aws-isc-aarch64-protected-build: - extends: [ ".aws-isc-aarch64", ".protected-build" ] +aws-isc-aarch64-build: + extends: [ ".aws-isc-aarch64", ".build" ] trigger: include: - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml - job: aws-isc-aarch64-protected-generate + job: aws-isc-aarch64-generate strategy: depend needs: - artifacts: True - job: aws-isc-aarch64-protected-generate + job: aws-isc-aarch64-generate ######################################## @@ -868,35 +658,20 @@ aws-isc-aarch64-protected-build: variables: SPACK_CI_STACK_NAME: tutorial -tutorial-pr-generate: - extends: [ ".tutorial", ".pr-generate"] +tutorial-generate: + extends: [ ".tutorial", ".generate"] image: ghcr.io/spack/tutorial-ubuntu-22.04:v2023-05-07 -tutorial-protected-generate: - extends: [ ".tutorial", ".protected-generate"] - image: ghcr.io/spack/tutorial-ubuntu-22.04:v2023-05-07 - -tutorial-pr-build: - extends: [ ".tutorial", ".pr-build" ] - trigger: - include: - - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml - job: tutorial-pr-generate - strategy: depend - needs: - - artifacts: True - job: tutorial-pr-generate - -tutorial-protected-build: - extends: [ ".tutorial", ".protected-build" ] +tutorial-build: + extends: [ ".tutorial", ".build" ] trigger: include: - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml - job: tutorial-protected-generate + job: tutorial-generate strategy: depend needs: - artifacts: True - job: tutorial-protected-generate + job: tutorial-generate ####################################### # Machine Learning - Linux x86_64 (CPU) @@ -906,37 +681,20 @@ tutorial-protected-build: variables: SPACK_CI_STACK_NAME: ml-linux-x86_64-cpu -.ml-linux-x86_64-cpu-generate: - extends: [ .ml-linux-x86_64-cpu, ".tags-x86_64_v4" ] +ml-linux-x86_64-cpu-generate: + extends: [ ".generate", .ml-linux-x86_64-cpu, ".tags-x86_64_v4" ] image: ghcr.io/spack/linux-ubuntu22.04-x86_64_v2:nightly -ml-linux-x86_64-cpu-pr-generate: - extends: [ ".pr-generate", ".ml-linux-x86_64-cpu-generate" ] - -ml-linux-x86_64-cpu-protected-generate: - extends: [ ".protected-generate", ".ml-linux-x86_64-cpu-generate" ] - -ml-linux-x86_64-cpu-pr-build: - extends: [ ".pr-build", ".ml-linux-x86_64-cpu" ] - trigger: - include: - - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml - job: ml-linux-x86_64-cpu-pr-generate - strategy: depend - needs: - - artifacts: True - job: ml-linux-x86_64-cpu-pr-generate - -ml-linux-x86_64-cpu-protected-build: - extends: [ ".protected-build", ".ml-linux-x86_64-cpu" ] +ml-linux-x86_64-cpu-build: + extends: [ ".build", ".ml-linux-x86_64-cpu" ] trigger: include: - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml - job: ml-linux-x86_64-cpu-protected-generate + job: ml-linux-x86_64-cpu-generate strategy: depend needs: - artifacts: True - job: ml-linux-x86_64-cpu-protected-generate + job: ml-linux-x86_64-cpu-generate ######################################## # Machine Learning - Linux x86_64 (CUDA) @@ -946,37 +704,20 @@ ml-linux-x86_64-cpu-protected-build: variables: SPACK_CI_STACK_NAME: ml-linux-x86_64-cuda -.ml-linux-x86_64-cuda-generate: - extends: [ .ml-linux-x86_64-cuda, ".tags-x86_64_v4" ] +ml-linux-x86_64-cuda-generate: + extends: [ ".generate", .ml-linux-x86_64-cuda, ".tags-x86_64_v4" ] image: ghcr.io/spack/linux-ubuntu22.04-x86_64_v2:nightly -ml-linux-x86_64-cuda-pr-generate: - extends: [ ".pr-generate", ".ml-linux-x86_64-cuda-generate" ] - -ml-linux-x86_64-cuda-protected-generate: - extends: [ ".protected-generate", ".ml-linux-x86_64-cuda-generate" ] - -ml-linux-x86_64-cuda-pr-build: - extends: [ ".pr-build", ".ml-linux-x86_64-cuda" ] +ml-linux-x86_64-cuda-build: + extends: [ ".build", ".ml-linux-x86_64-cuda" ] trigger: include: - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml - job: ml-linux-x86_64-cuda-pr-generate + job: ml-linux-x86_64-cuda-generate strategy: depend needs: - artifacts: True - job: ml-linux-x86_64-cuda-pr-generate - -ml-linux-x86_64-cuda-protected-build: - extends: [ ".protected-build", ".ml-linux-x86_64-cuda" ] - trigger: - include: - - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml - job: ml-linux-x86_64-cuda-protected-generate - strategy: depend - needs: - - artifacts: True - job: ml-linux-x86_64-cuda-protected-generate + job: ml-linux-x86_64-cuda-generate ######################################## # Machine Learning - Linux x86_64 (ROCm) @@ -986,38 +727,20 @@ ml-linux-x86_64-cuda-protected-build: variables: SPACK_CI_STACK_NAME: ml-linux-x86_64-rocm -.ml-linux-x86_64-rocm-generate: - extends: [ .ml-linux-x86_64-rocm, ".tags-x86_64_v4" ] +ml-linux-x86_64-rocm-generate: + extends: [ ".generate", .ml-linux-x86_64-rocm, ".tags-x86_64_v4" ] image: ghcr.io/spack/linux-ubuntu22.04-x86_64_v2:nightly -ml-linux-x86_64-rocm-pr-generate: - extends: [ ".pr-generate", ".ml-linux-x86_64-rocm-generate" ] - -ml-linux-x86_64-rocm-protected-generate: - extends: [ ".protected-generate", ".ml-linux-x86_64-rocm-generate" ] - -ml-linux-x86_64-rocm-pr-build: - extends: [ ".pr-build", ".ml-linux-x86_64-rocm" ] - trigger: - include: - - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml - job: ml-linux-x86_64-rocm-pr-generate - strategy: depend - needs: - - artifacts: True - job: ml-linux-x86_64-rocm-pr-generate - -ml-linux-x86_64-rocm-protected-build: - extends: [ ".protected-build", ".ml-linux-x86_64-rocm" ] +ml-linux-x86_64-rocm-build: + extends: [ ".build", ".ml-linux-x86_64-rocm" ] trigger: include: - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml - job: ml-linux-x86_64-rocm-protected-generate + job: ml-linux-x86_64-rocm-generate strategy: depend needs: - artifacts: True - job: ml-linux-x86_64-rocm-protected-generate - + job: ml-linux-x86_64-rocm-generate ######################################## # Deprecated CI testing @@ -1026,30 +749,16 @@ ml-linux-x86_64-rocm-protected-build: variables: SPACK_CI_STACK_NAME: deprecated -deprecated-ci-pr-generate: - extends: [ ".pr-generate-deprecated", ".deprecated-ci" ] - -deprecated-ci-protected-generate: - extends: [ ".protected-generate-deprecated", ".deprecated-ci" ] - -deprecated-ci-pr-build: - extends: [ ".pr-build", ".deprecated-ci" ] - trigger: - include: - - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml - job: deprecated-ci-pr-generate - strategy: depend - needs: - - artifacts: True - job: deprecated-ci-pr-generate +deprecated-ci-generate: + extends: [ ".generate-deprecated", ".deprecated-ci" ] -deprecated-ci-protected-build: - extends: [ ".protected-build", ".deprecated-ci" ] +deprecated-ci-build: + extends: [ ".build", ".deprecated-ci" ] trigger: include: - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml - job: deprecated-ci-protected-generate + job: deprecated-ci-generate strategy: depend needs: - artifacts: True - job: deprecated-ci-protected-generate + job: deprecated-ci-generate diff --git a/share/spack/gitlab/cloud_pipelines/configs/ci.yaml b/share/spack/gitlab/cloud_pipelines/configs/ci.yaml index bf9df5878d..f65760c923 100644 --- a/share/spack/gitlab/cloud_pipelines/configs/ci.yaml +++ b/share/spack/gitlab/cloud_pipelines/configs/ci.yaml @@ -36,6 +36,34 @@ ci: - aws s3 sync --exclude "*" --include "*spec.json.sig*" /tmp ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache - aws s3 cp /tmp/public_keys ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache/_pgp --recursive --exclude "*" --include "*.pub" + - copy-job: + tags: ["service", "x86_64"] + image: "ghcr.io/spack/python-aws-bash:0.0.1" + before_script: + - - if [[ $CI_COMMIT_TAG == "v"* ]]; then export SPACK_REPLACE_VERSION=$(echo "$CI_COMMIT_TAG" | sed 's/\(v[[:digit:]]\+\.[[:digit:]]\+\).*/releases\/\1/'); fi + - if [[ $CI_COMMIT_TAG == "develop-"* ]]; then export SPACK_REPLACE_VERSION=develop; fi + - export SPACK_BUILDCACHE_SOURCE=${SPACK_SOURCE_MIRROR//SPACK_REPLACE_VERSION/${SPACK_REPLACE_VERSION}} + script: + - - cd ${SPACK_CONCRETE_ENV_DIR} + - spack env activate --without-view . + - echo Copying environment specs from ${SRC_MIRROR} to ${SPACK_BUILDCACHE_DESTINATION} + - spack buildcache sync "${SPACK_BUILDCACHE_SOURCE}" "${SPACK_BUILDCACHE_DESTINATION}" + - curl -fLsS https://spack.github.io/keys/spack-public-binary-key.pub -o /tmp/spack-public-binary-key.pub + - aws s3 cp /tmp/spack-public-binary-key.pub "${SPACK_BUILDCACHE_DESTINATION}/build_cache/_pgp/spack-public-binary-key.pub" + - spack buildcache update-index --keys "${SPACK_BUILDCACHE_DESTINATION}" + when: "always" + retry: + max: 2 + when: + - "runner_system_failure" + - "stuck_or_timeout_failure" + - "script_failure" + interruptible: true + variables: + CI_JOB_SIZE: "medium" + KUBERNETES_CPU_REQUEST: "4000m" + KUBERNETES_MEMORY_REQUEST: "16G" + - reindex-job: tags: ["service"] variables: -- cgit v1.2.3-60-g2f50