From 85e13260cf2fc969daaf9bf6589b5a07255e074f Mon Sep 17 00:00:00 2001 From: Scott Wittenburg Date: Thu, 26 May 2022 08:31:22 -0600 Subject: ci: Support secure binary signing on protected pipelines (#30753) This PR supports the creation of securely signed binaries built from spack develop as well as release branches and tags. Specifically: - remove internal pr mirror url generation logic in favor of buildcache destination on command line - with a single mirror url specified in the spack.yaml, this makes it clearer where binaries from various pipelines are pushed - designate some tags as reserved: ['public', 'protected', 'notary'] - these tags are stripped from all jobs by default and provisioned internally based on pipeline type - update gitlab ci yaml to include pipelines on more protected branches than just develop (so include releases and tags) - binaries from all protected pipelines are pushed into mirrors including the branch name so releases, tags, and develop binaries are kept separate - update rebuild jobs running on protected pipelines to run on special runners provisioned with an intermediate signing key - protected rebuild jobs no longer use "SPACK_SIGNING_KEY" env var to obtain signing key (in fact, final signing key is nowhere available to rebuild jobs) - these intermediate signatures are verified at the end of each pipeline by a new signing job to ensure binaries were produced by a protected pipeline - optionallly schedule a signing/notary job at the end of the pipeline to sign all packges in the mirror - add signing-job-attributes to gitlab-ci section of spack environment to allow configuration - signing job runs on special runner (separate from protected rebuild runners) provisioned with public intermediate key and secret signing key --- share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml | 180 ++++++++++++++------- .../stacks/build_systems/spack.yaml | 23 ++- .../cloud_pipelines/stacks/data-vis-sdk/spack.yaml | 25 ++- .../cloud_pipelines/stacks/e4s-mac/spack.yaml | 6 +- .../gitlab/cloud_pipelines/stacks/e4s/spack.yaml | 25 ++- .../cloud_pipelines/stacks/radiuss/spack.yaml | 22 ++- .../cloud_pipelines/stacks/tutorial/spack.yaml | 23 ++- share/spack/spack-completion.bash | 2 +- 8 files changed, 212 insertions(+), 94 deletions(-) (limited to 'share/spack') diff --git a/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml b/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml index 321c2b6af6..b0936dfc4e 100644 --- a/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml +++ b/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml @@ -1,4 +1,4 @@ -stages: [ "generate", "build" ] +stages: [ "generate", "build", "publish" ] default: image: { "name": "ghcr.io/spack/e4s-ubuntu-18.04:v2021-10-18", "entrypoint": [""] } @@ -9,16 +9,25 @@ default: .pr: only: + - /^pr[\d]+_.*$/ - /^github\/pr[\d]+_.*$/ variables: - SPACK_PR_BRANCH: ${CI_COMMIT_REF_NAME} + SPACK_BUILDCACHE_DESTINATION: "s3://spack-binaries-prs/${CI_COMMIT_REF_NAME}" SPACK_PIPELINE_TYPE: "spack_pull_request" SPACK_PRUNE_UNTOUCHED: "True" -.develop: +.protected-refs: only: + - /^develop$/ + - /^releases\/v.*/ + - /^v.*/ - /^github\/develop$/ + +.protected: + extends: [ ".protected-refs" ] variables: + SPACK_BUILDCACHE_DESTINATION: "s3://spack-binaries/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}" + SPACK_COPY_BUILDCACHE: "s3://spack-binaries/${CI_COMMIT_REF_NAME}" SPACK_PIPELINE_TYPE: "spack_protected_branch" .generate: @@ -29,12 +38,13 @@ default: - cd share/spack/gitlab/cloud_pipelines/stacks/${SPACK_CI_STACK_NAME} - spack env activate --without-view . - spack ci generate --check-index-only + --buildcache-destination "${SPACK_BUILDCACHE_DESTINATION}" --artifacts-root "${CI_PROJECT_DIR}/jobs_scratch_dir" --output-file "${CI_PROJECT_DIR}/jobs_scratch_dir/cloud-ci-pipeline.yml" artifacts: paths: - "${CI_PROJECT_DIR}/jobs_scratch_dir" - tags: ["spack", "public", "medium", "x86_64"] + tags: ["spack", "aws", "public", "medium", "x86_64"] interruptible: true retry: max: 2 @@ -45,8 +55,8 @@ default: .pr-generate: extends: [ ".pr", ".generate" ] -.develop-generate: - extends: [ ".develop", ".generate" ] +.protected-generate: + extends: [ ".protected", ".generate" ] .build: stage: build @@ -57,12 +67,24 @@ default: AWS_ACCESS_KEY_ID: ${PR_MIRRORS_AWS_ACCESS_KEY_ID} AWS_SECRET_ACCESS_KEY: ${PR_MIRRORS_AWS_SECRET_ACCESS_KEY} -.develop-build: - extends: [ ".develop", ".build" ] +.protected-build: + extends: [ ".protected", ".build" ] variables: AWS_ACCESS_KEY_ID: ${PROTECTED_MIRRORS_AWS_ACCESS_KEY_ID} AWS_SECRET_ACCESS_KEY: ${PROTECTED_MIRRORS_AWS_SECRET_ACCESS_KEY} - SPACK_SIGNING_KEY: ${PACKAGE_SIGNING_KEY} + +protected-publish: + stage: publish + extends: [ ".protected-refs" ] + image: "ghcr.io/spack/python-aws-bash:0.0.1" + tags: ["spack", "public", "medium", "aws", "x86_64"] + variables: + AWS_ACCESS_KEY_ID: ${PROTECTED_MIRRORS_AWS_ACCESS_KEY_ID} + AWS_SECRET_ACCESS_KEY: ${PROTECTED_MIRRORS_AWS_SECRET_ACCESS_KEY} + script: + - . "./share/spack/setup-env.sh" + - spack --version + - spack buildcache update-index --mirror-url "s3://spack-binaries/${CI_COMMIT_REF_NAME}" ######################################## # TEMPLATE FOR ADDING ANOTHER PIPELINE @@ -83,8 +105,8 @@ default: # my-super-cool-stack-pr-generate: # extends: [ ".my-super-cool-stack", ".pr-generate"] # -# my-super-cool-stack-develop-generate: -# extends: [ ".my-super-cool-stack", ".develop-generate"] +# my-super-cool-stack-protected-generate: +# extends: [ ".my-super-cool-stack", ".protected-generate"] # # my-super-cool-stack-pr-build: # extends: [ ".my-super-cool-stack", ".pr-build" ] @@ -94,24 +116,62 @@ default: # job: my-super-cool-stack-pr-generate # strategy: depend # -# my-super-cool-stack-develop-build: -# extends: [ ".my-super-cool-stack", ".develop-build" ] +# my-super-cool-stack-protected-build: +# extends: [ ".my-super-cool-stack", ".protected-build" ] # trigger: # include: # - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml -# job: my-super-cool-stack-develop-generate +# job: my-super-cool-stack-protected-generate # strategy: depend ######################################## -# E4S Mac Stack +# E4S Mac Stack +# +# With no near-future plans to have +# protected aws runners running mac +# builds, it seems best to decouple +# them from the rest of the stacks for +# the time being. This way they can +# still run on UO runners and be signed +# using the previous approach. ######################################## .e4s-mac: variables: SPACK_CI_STACK_NAME: e4s-mac allow_failure: True +.mac-pr: + only: + - /^pr[\d]+_.*$/ + - /^github\/pr[\d]+_.*$/ + variables: + SPACK_BUILDCACHE_DESTINATION: "s3://spack-binaries-prs/${CI_COMMIT_REF_NAME}" + SPACK_PRUNE_UNTOUCHED: "True" + +.mac-protected: + only: + - /^develop$/ + - /^releases\/v.*/ + - /^v.*/ + - /^github\/develop$/ + variables: + SPACK_BUILDCACHE_DESTINATION: "s3://spack-binaries/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}" + +.mac-pr-build: + extends: [ ".mac-pr", ".build" ] + variables: + AWS_ACCESS_KEY_ID: ${PR_MIRRORS_AWS_ACCESS_KEY_ID} + AWS_SECRET_ACCESS_KEY: ${PR_MIRRORS_AWS_SECRET_ACCESS_KEY} + +.mac-protected-build: + extends: [ ".mac-protected", ".build" ] + variables: + AWS_ACCESS_KEY_ID: ${PROTECTED_MIRRORS_AWS_ACCESS_KEY_ID} + AWS_SECRET_ACCESS_KEY: ${PROTECTED_MIRRORS_AWS_SECRET_ACCESS_KEY} + SPACK_SIGNING_KEY: ${PACKAGE_SIGNING_KEY} + e4s-mac-pr-generate: - extends: [".e4s-mac", ".pr"] + extends: [".e4s-mac", ".mac-pr"] stage: generate script: - tmp="$(mktemp -d)"; export SPACK_USER_CONFIG_PATH="$tmp"; export SPACK_USER_CACHE_PATH="$tmp" @@ -135,8 +195,8 @@ e4s-mac-pr-generate: - stuck_or_timeout_failure timeout: 60 minutes -e4s-mac-develop-generate: - extends: [".e4s-mac", ".develop"] +e4s-mac-protected-generate: + extends: [".e4s-mac", ".mac-protected"] stage: generate script: - tmp="$(mktemp -d)"; export SPACK_USER_CONFIG_PATH="$tmp"; export SPACK_USER_CACHE_PATH="$tmp" @@ -161,7 +221,7 @@ e4s-mac-develop-generate: timeout: 60 minutes e4s-mac-pr-build: - extends: [ ".e4s-mac", ".pr-build" ] + extends: [ ".e4s-mac", ".mac-pr-build" ] trigger: include: - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml @@ -171,16 +231,16 @@ e4s-mac-pr-build: - artifacts: True job: e4s-mac-pr-generate -e4s-mac-develop-build: - extends: [ ".e4s-mac", ".develop-build" ] +e4s-mac-protected-build: + extends: [ ".e4s-mac", ".mac-protected-build" ] trigger: include: - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml - job: e4s-mac-develop-generate + job: e4s-mac-protected-generate strategy: depend needs: - artifacts: True - job: e4s-mac-develop-generate + job: e4s-mac-protected-generate ######################################## # E4S pipeline @@ -192,8 +252,8 @@ e4s-mac-develop-build: e4s-pr-generate: extends: [ ".e4s", ".pr-generate"] -e4s-develop-generate: - extends: [ ".e4s", ".develop-generate"] +e4s-protected-generate: + extends: [ ".e4s", ".protected-generate"] e4s-pr-build: extends: [ ".e4s", ".pr-build" ] @@ -206,16 +266,16 @@ e4s-pr-build: - artifacts: True job: e4s-pr-generate -e4s-develop-build: - extends: [ ".e4s", ".develop-build" ] +e4s-protected-build: + extends: [ ".e4s", ".protected-build" ] trigger: include: - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml - job: e4s-develop-generate + job: e4s-protected-generate strategy: depend needs: - artifacts: True - job: e4s-develop-generate + job: e4s-protected-generate ######################################## # E4S on Power @@ -231,8 +291,8 @@ e4s-develop-build: # e4s-on-power-pr-generate: # extends: [ ".e4s-on-power", ".pr-generate", ".power-e4s-generate-tags-and-image"] -# e4s-on-power-develop-generate: -# extends: [ ".e4s-on-power", ".develop-generate", ".power-e4s-generate-tags-and-image"] +# e4s-on-power-protected-generate: +# extends: [ ".e4s-on-power", ".protected-generate", ".power-e4s-generate-tags-and-image"] # e4s-on-power-pr-build: # extends: [ ".e4s-on-power", ".pr-build" ] @@ -245,16 +305,16 @@ e4s-develop-build: # - artifacts: True # job: e4s-on-power-pr-generate -# e4s-on-power-develop-build: -# extends: [ ".e4s-on-power", ".develop-build" ] +# e4s-on-power-protected-build: +# extends: [ ".e4s-on-power", ".protected-build" ] # trigger: # include: # - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml -# job: e4s-on-power-develop-generate +# job: e4s-on-power-protected-generate # strategy: depend # needs: # - artifacts: True -# job: e4s-on-power-develop-generate +# job: e4s-on-power-protected-generate ######################################### # Build tests for different build-systems @@ -266,8 +326,8 @@ e4s-develop-build: build_systems-pr-generate: extends: [ ".build_systems", ".pr-generate"] -build_systems-develop-generate: - extends: [ ".build_systems", ".develop-generate"] +build_systems-protected-generate: + extends: [ ".build_systems", ".protected-generate"] build_systems-pr-build: extends: [ ".build_systems", ".pr-build" ] @@ -280,16 +340,16 @@ build_systems-pr-build: - artifacts: True job: build_systems-pr-generate -build_systems-develop-build: - extends: [ ".build_systems", ".develop-build" ] +build_systems-protected-build: + extends: [ ".build_systems", ".protected-build" ] trigger: include: - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml - job: build_systems-develop-generate + job: build_systems-protected-generate strategy: depend needs: - artifacts: True - job: build_systems-develop-generate + job: build_systems-protected-generate ######################################### # RADIUSS @@ -313,20 +373,20 @@ radiuss-pr-build: - artifacts: True job: radiuss-pr-generate -# --------- Develop --------- -radiuss-develop-generate: - extends: [ ".radiuss", ".develop-generate" ] +# --------- Protected --------- +radiuss-protected-generate: + extends: [ ".radiuss", ".protected-generate" ] -radiuss-develop-build: - extends: [ ".radiuss", ".develop-build" ] +radiuss-protected-build: + extends: [ ".radiuss", ".protected-build" ] trigger: include: - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml - job: radiuss-develop-generate + job: radiuss-protected-generate strategy: depend needs: - artifacts: True - job: radiuss-develop-generate + job: radiuss-protected-generate ######################################## # ECP Data & Vis SDK @@ -338,8 +398,8 @@ radiuss-develop-build: data-vis-sdk-pr-generate: extends: [ ".data-vis-sdk", ".pr-generate"] -data-vis-sdk-develop-generate: - extends: [ ".data-vis-sdk", ".develop-generate"] +data-vis-sdk-protected-generate: + extends: [ ".data-vis-sdk", ".protected-generate"] data-vis-sdk-pr-build: extends: [ ".data-vis-sdk", ".pr-build" ] @@ -352,16 +412,16 @@ data-vis-sdk-pr-build: - artifacts: True job: data-vis-sdk-pr-generate -data-vis-sdk-develop-build: - extends: [ ".data-vis-sdk", ".develop-build" ] +data-vis-sdk-protected-build: + extends: [ ".data-vis-sdk", ".protected-build" ] trigger: include: - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml - job: data-vis-sdk-develop-generate + job: data-vis-sdk-protected-generate strategy: depend needs: - artifacts: True - job: data-vis-sdk-develop-generate + job: data-vis-sdk-protected-generate ######################################## # Spack Tutorial @@ -373,8 +433,8 @@ data-vis-sdk-develop-build: tutorial-pr-generate: extends: [ ".tutorial", ".pr-generate"] -tutorial-develop-generate: - extends: [ ".tutorial", ".develop-generate"] +tutorial-protected-generate: + extends: [ ".tutorial", ".protected-generate"] tutorial-pr-build: extends: [ ".tutorial", ".pr-build" ] @@ -387,13 +447,13 @@ tutorial-pr-build: - artifacts: True job: tutorial-pr-generate -tutorial-develop-build: - extends: [ ".tutorial", ".develop-build" ] +tutorial-protected-build: + extends: [ ".tutorial", ".protected-build" ] trigger: include: - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml - job: tutorial-develop-generate + job: tutorial-protected-generate strategy: depend needs: - artifacts: True - job: tutorial-develop-generate + job: tutorial-protected-generate diff --git a/share/spack/gitlab/cloud_pipelines/stacks/build_systems/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/build_systems/spack.yaml index 5a742718d4..a633d10099 100644 --- a/share/spack/gitlab/cloud_pipelines/stacks/build_systems/spack.yaml +++ b/share/spack/gitlab/cloud_pipelines/stacks/build_systems/spack.yaml @@ -29,7 +29,7 @@ spack: - - $default_specs - - $arch - mirrors: { "mirror": "s3://spack-binaries/build_systems" } + mirrors: { "mirror": "s3://spack-binaries/develop/build_systems" } gitlab-ci: script: @@ -38,6 +38,8 @@ spack: - cd ${SPACK_CONCRETE_ENV_DIR} - spack env activate --without-view . - spack config add "config:install_tree:projections:${SPACK_JOB_SPEC_PKG_NAME}:'morepadding/{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'" + - if [[ -r /mnt/key/intermediate_ci_signing_key.gpg ]]; then spack gpg trust /mnt/key/intermediate_ci_signing_key.gpg; fi + - if [[ -r /mnt/key/spack_public_key.gpg ]]; then spack gpg trust /mnt/key/spack_public_key.gpg; fi - spack -d ci rebuild image: @@ -48,7 +50,7 @@ spack: - match: - cmake runner-attributes: - tags: [ "spack", "public", "large", "x86_64"] + tags: [ "spack", "large", "x86_64"] variables: CI_JOB_SIZE: large KUBERNETES_CPU_REQUEST: 8000m @@ -61,7 +63,7 @@ spack: - openjpeg - sqlite runner-attributes: - tags: [ "spack", "public", "medium", "x86_64" ] + tags: [ "spack", "medium", "x86_64" ] variables: CI_JOB_SIZE: "medium" KUBERNETES_CPU_REQUEST: "2000m" @@ -85,7 +87,7 @@ spack: - xz - zlib runner-attributes: - tags: [ "spack", "public", "medium", "x86_64" ] + tags: [ "spack", "medium", "x86_64" ] variables: CI_JOB_SIZE: "small" KUBERNETES_CPU_REQUEST: "500m" @@ -94,18 +96,27 @@ spack: - match: - 'os=ubuntu18.04' runner-attributes: - tags: ["spack", "public", "x86_64"] + tags: ["spack", "x86_64"] variables: CI_JOB_SIZE: "default" + broken-specs-url: "s3://spack-binaries/broken-specs" - broken-specs-url: "s3://spack-binaries-develop/broken-specs" service-job-attributes: before_script: - . "./share/spack/setup-env.sh" - spack --version image: { "name": "ghcr.io/spack/e4s-ubuntu-18.04:v2021-10-18", "entrypoint": [""] } tags: ["spack", "public", "x86_64"] + + signing-job-attributes: + image: { "name": "ghcr.io/spack/notary:latest", "entrypoint": [""] } + tags: ["spack", "aws"] + script: + - aws s3 sync --exclude "*" --include "*spec.json*" ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache /tmp + - /sign.sh + - aws s3 sync --exclude "*" --include "*spec.json.sig*" /tmp ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache + cdash: build-group: Build tests for different build systems url: https://cdash.spack.io diff --git a/share/spack/gitlab/cloud_pipelines/stacks/data-vis-sdk/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/data-vis-sdk/spack.yaml index 85095e5619..ae9025e366 100644 --- a/share/spack/gitlab/cloud_pipelines/stacks/data-vis-sdk/spack.yaml +++ b/share/spack/gitlab/cloud_pipelines/stacks/data-vis-sdk/spack.yaml @@ -42,7 +42,7 @@ spack: +zfp +visit - mirrors: { "mirror": "s3://spack-binaries/data-vis-sdk" } + mirrors: { "mirror": "s3://spack-binaries/develop/data-vis-sdk" } gitlab-ci: image: { "name": "ghcr.io/spack/e4s-ubuntu-18.04:v2021-10-18", "entrypoint": [""] } @@ -52,13 +52,15 @@ spack: - cd ${SPACK_CONCRETE_ENV_DIR} - spack env activate --without-view . - spack config add "config:install_tree:projections:${SPACK_JOB_SPEC_PKG_NAME}:'morepadding/{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'" + - if [[ -r /mnt/key/intermediate_ci_signing_key.gpg ]]; then spack gpg trust /mnt/key/intermediate_ci_signing_key.gpg; fi + - if [[ -r /mnt/key/spack_public_key.gpg ]]; then spack gpg trust /mnt/key/spack_public_key.gpg; fi - spack -d ci rebuild mappings: - match: - llvm - qt runner-attributes: - tags: [ "spack", "public", "huge", "x86_64" ] + tags: [ "spack", "huge", "x86_64" ] variables: CI_JOB_SIZE: huge KUBERNETES_CPU_REQUEST: 11000m @@ -72,7 +74,7 @@ spack: - visit - vtk-m runner-attributes: - tags: [ "spack", "public", "large", "x86_64" ] + tags: [ "spack", "large", "x86_64" ] variables: CI_JOB_SIZE: large KUBERNETES_CPU_REQUEST: 8000m @@ -98,7 +100,7 @@ spack: - raja - vtk-h runner-attributes: - tags: [ "spack", "public", "medium", "x86_64" ] + tags: [ "spack", "medium", "x86_64" ] variables: CI_JOB_SIZE: "medium" KUBERNETES_CPU_REQUEST: "2000m" @@ -133,7 +135,7 @@ spack: - util-linux-uuid runner-attributes: - tags: [ "spack", "public", "small", "x86_64" ] + tags: [ "spack", "small", "x86_64" ] variables: CI_JOB_SIZE: "small" KUBERNETES_CPU_REQUEST: "500m" @@ -141,11 +143,12 @@ spack: - match: ['@:'] runner-attributes: - tags: ["spack", "public", "x86_64"] + tags: ["spack", "x86_64"] variables: CI_JOB_SIZE: "default" - broken-specs-url: "s3://spack-binaries-develop/broken-specs" + broken-specs-url: "s3://spack-binaries/broken-specs" + service-job-attributes: image: { "name": "ghcr.io/spack/e4s-ubuntu-18.04:v2021-10-18", "entrypoint": [""] } before_script: @@ -153,6 +156,14 @@ spack: - spack --version tags: ["spack", "public", "medium", "x86_64"] + signing-job-attributes: + image: { "name": "ghcr.io/spack/notary:latest", "entrypoint": [""] } + tags: ["spack", "aws"] + script: + - aws s3 sync --exclude "*" --include "*spec.json*" ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache /tmp + - /sign.sh + - aws s3 sync --exclude "*" --include "*spec.json.sig*" /tmp ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache + cdash: build-group: Data and Vis SDK url: https://cdash.spack.io diff --git a/share/spack/gitlab/cloud_pipelines/stacks/e4s-mac/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/e4s-mac/spack.yaml index dc86cb7dbe..6cfaae4a15 100644 --- a/share/spack/gitlab/cloud_pipelines/stacks/e4s-mac/spack.yaml +++ b/share/spack/gitlab/cloud_pipelines/stacks/e4s-mac/spack.yaml @@ -32,7 +32,7 @@ spack: - - $easy_specs - - $arch - mirrors: { "mirror": "s3://spack-binaries/e4s-mac" } + mirrors: { "mirror": "s3://spack-binaries/develop/e4s-mac" } gitlab-ci: @@ -51,7 +51,9 @@ spack: runner-attributes: tags: - omicron - broken-specs-url: "s3://spack-binaries-develop/broken-specs" + + broken-specs-url: "s3://spack-binaries/broken-specs" + service-job-attributes: before_script: - . "./share/spack/setup-env.sh" diff --git a/share/spack/gitlab/cloud_pipelines/stacks/e4s/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/e4s/spack.yaml index ce31a0156e..c4eaaddbae 100644 --- a/share/spack/gitlab/cloud_pipelines/stacks/e4s/spack.yaml +++ b/share/spack/gitlab/cloud_pipelines/stacks/e4s/spack.yaml @@ -222,7 +222,7 @@ spack: - - $cuda_specs - - $arch - mirrors: { "mirror": "s3://spack-binaries/e4s" } + mirrors: { "mirror": "s3://spack-binaries/develop/e4s" } gitlab-ci: @@ -233,6 +233,8 @@ spack: - spack env activate --without-view . - spack config add "config:install_tree:projections:${SPACK_JOB_SPEC_PKG_NAME}:'morepadding/{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'" - mkdir -p ${SPACK_ARTIFACTS_ROOT}/user_data + - if [[ -r /mnt/key/intermediate_ci_signing_key.gpg ]]; then spack gpg trust /mnt/key/intermediate_ci_signing_key.gpg; fi + - if [[ -r /mnt/key/spack_public_key.gpg ]]; then spack gpg trust /mnt/key/spack_public_key.gpg; fi - spack -d ci rebuild > >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_out.txt) 2> >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_err.txt >&2) image: { "name": "ghcr.io/spack/e4s-ubuntu-18.04:v2021-10-18", "entrypoint": [""] } @@ -240,7 +242,7 @@ spack: - match: - llvm runner-attributes: - tags: [ "spack", "public", "huge", "x86_64" ] + tags: [ "spack", "huge", "x86_64" ] variables: CI_JOB_SIZE: huge KUBERNETES_CPU_REQUEST: 11000m @@ -265,7 +267,7 @@ spack: - vtk-m - warpx runner-attributes: - tags: [ "spack", "public", "large", "x86_64" ] + tags: [ "spack", "large", "x86_64" ] variables: CI_JOB_SIZE: large KUBERNETES_CPU_REQUEST: 8000m @@ -333,7 +335,7 @@ spack: - vtk-h - zfp runner-attributes: - tags: [ "spack", "public", "medium", "x86_64" ] + tags: [ "spack", "medium", "x86_64" ] variables: CI_JOB_SIZE: "medium" KUBERNETES_CPU_REQUEST: "2000m" @@ -394,7 +396,7 @@ spack: - zlib - zstd runner-attributes: - tags: [ "spack", "public", "small", "x86_64" ] + tags: [ "spack", "small", "x86_64" ] variables: CI_JOB_SIZE: "small" KUBERNETES_CPU_REQUEST: "500m" @@ -402,11 +404,12 @@ spack: - match: ['os=ubuntu18.04'] runner-attributes: - tags: ["spack", "public", "x86_64"] + tags: ["spack", "x86_64"] variables: CI_JOB_SIZE: "default" - broken-specs-url: "s3://spack-binaries-develop/broken-specs" + broken-specs-url: "s3://spack-binaries/broken-specs" + service-job-attributes: before_script: - . "./share/spack/setup-env.sh" @@ -414,6 +417,14 @@ spack: image: { "name": "ghcr.io/spack/e4s-ubuntu-18.04:v2021-10-18", "entrypoint": [""] } tags: ["spack", "public", "x86_64"] + signing-job-attributes: + image: { "name": "ghcr.io/spack/notary:latest", "entrypoint": [""] } + tags: ["spack", "aws"] + script: + - aws s3 sync --exclude "*" --include "*spec.json*" ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache /tmp + - /sign.sh + - aws s3 sync --exclude "*" --include "*spec.json.sig*" /tmp ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache + cdash: build-group: New PR testing workflow url: https://cdash.spack.io diff --git a/share/spack/gitlab/cloud_pipelines/stacks/radiuss/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/radiuss/spack.yaml index 846b45f707..ebb15eb7ca 100644 --- a/share/spack/gitlab/cloud_pipelines/stacks/radiuss/spack.yaml +++ b/share/spack/gitlab/cloud_pipelines/stacks/radiuss/spack.yaml @@ -54,7 +54,7 @@ spack: - zfp mirrors: - mirror: "s3://spack-binaries/radiuss" + mirror: "s3://spack-binaries/develop/radiuss" specs: - matrix: @@ -69,6 +69,8 @@ spack: - cd ${SPACK_CONCRETE_ENV_DIR} - spack env activate --without-view . - spack config add "config:install_tree:projections:${SPACK_JOB_SPEC_PKG_NAME}:'morepadding/{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'" + - if [[ -r /mnt/key/intermediate_ci_signing_key.gpg ]]; then spack gpg trust /mnt/key/intermediate_ci_signing_key.gpg; fi + - if [[ -r /mnt/key/spack_public_key.gpg ]]; then spack gpg trust /mnt/key/spack_public_key.gpg; fi - spack -d ci rebuild mappings: - match: @@ -76,7 +78,7 @@ spack: - openblas - rust runner-attributes: - tags: ["spack", "public", "large", "x86_64"] + tags: ["spack", "large", "x86_64"] variables: CI_JOB_SIZE: large KUBERNETES_CPU_REQUEST: 8000m @@ -96,7 +98,7 @@ spack: - vtk-h - vtk-m runner-attributes: - tags: ["spack", "public", "medium", "x86_64"] + tags: ["spack", "medium", "x86_64"] variables: CI_JOB_SIZE: "medium" KUBERNETES_CPU_REQUEST: "2000m" @@ -150,7 +152,7 @@ spack: - zfp - zlib runner-attributes: - tags: ["spack", "public", "small", "x86_64"] + tags: ["spack", "small", "x86_64"] variables: CI_JOB_SIZE: "small" KUBERNETES_CPU_REQUEST: "500m" @@ -158,10 +160,12 @@ spack: - match: ['os=ubuntu18.04'] runner-attributes: - tags: ["spack", "public", "x86_64"] + tags: ["spack", "x86_64"] variables: CI_JOB_SIZE: "default" + broken-specs-url: "s3://spack-binaries/broken-specs" + service-job-attributes: before_script: - . "./share/spack/setup-env.sh" @@ -169,6 +173,14 @@ spack: image: { "name": "ghcr.io/spack/e4s-ubuntu-18.04:v2021-10-18", "entrypoint": [""] } tags: ["spack", "public", "x86_64"] + signing-job-attributes: + image: { "name": "ghcr.io/spack/notary:latest", "entrypoint": [""] } + tags: ["spack", "aws"] + script: + - aws s3 sync --exclude "*" --include "*spec.json*" ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache /tmp + - /sign.sh + - aws s3 sync --exclude "*" --include "*spec.json.sig*" /tmp ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache + cdash: build-group: RADIUSS url: https://cdash.spack.io diff --git a/share/spack/gitlab/cloud_pipelines/stacks/tutorial/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/tutorial/spack.yaml index 14bd78b3b6..504b4209d2 100644 --- a/share/spack/gitlab/cloud_pipelines/stacks/tutorial/spack.yaml +++ b/share/spack/gitlab/cloud_pipelines/stacks/tutorial/spack.yaml @@ -59,7 +59,7 @@ spack: - $gcc_spack_built_packages mirrors: - mirror: 's3://spack-binaries/tutorial' + mirror: 's3://spack-binaries/develop/tutorial' gitlab-ci: script: @@ -69,6 +69,8 @@ spack: - cd ${SPACK_CONCRETE_ENV_DIR} - spack env activate --without-view . - spack config add "config:install_tree:projections:${SPACK_JOB_SPEC_PKG_NAME}:'morepadding/{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'" + - if [[ -r /mnt/key/intermediate_ci_signing_key.gpg ]]; then spack gpg trust /mnt/key/intermediate_ci_signing_key.gpg; fi + - if [[ -r /mnt/key/spack_public_key.gpg ]]; then spack gpg trust /mnt/key/spack_public_key.gpg; fi - spack -d ci rebuild image: { "name": "ghcr.io/spack/tutorial-ubuntu-18.04:v2021-11-02", "entrypoint": [""] } @@ -81,7 +83,7 @@ spack: - netlib-lapack - trilinos runner-attributes: - tags: ["spack", "public", "large", "x86_64"] + tags: ["spack", "large", "x86_64"] variables: CI_JOB_SIZE: large KUBERNETES_CPU_REQUEST: 8000m @@ -99,7 +101,7 @@ spack: - py-scipy - slurm runner-attributes: - tags: ["spack", "public", "medium", "x86_64"] + tags: ["spack", "medium", "x86_64"] variables: CI_JOB_SIZE: "medium" KUBERNETES_CPU_REQUEST: "2000m" @@ -129,7 +131,7 @@ spack: - tar - util-linux-uuid runner-attributes: - tags: ["spack", "public", "small", "x86_64"] + tags: ["spack", "small", "x86_64"] variables: CI_JOB_SIZE: "small" KUBERNETES_CPU_REQUEST: "500m" @@ -137,11 +139,12 @@ spack: - match: ['@:'] runner-attributes: - tags: ["spack", "public", "x86_64"] + tags: ["spack", "x86_64"] variables: CI_JOB_SIZE: default - broken-specs-url: "s3://spack-binaries-develop/broken-specs" + broken-specs-url: "s3://spack-binaries/broken-specs" + service-job-attributes: image: { "name": "ghcr.io/spack/tutorial-ubuntu-18.04:v2021-11-02", "entrypoint": [""] } before_script: @@ -149,6 +152,14 @@ spack: - spack --version tags: ["spack", "public", "x86_64"] + signing-job-attributes: + image: { "name": "ghcr.io/spack/notary:latest", "entrypoint": [""] } + tags: ["spack", "aws"] + script: + - aws s3 sync --exclude "*" --include "*spec.json*" ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache /tmp + - /sign.sh + - aws s3 sync --exclude "*" --include "*spec.json.sig*" /tmp ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache + cdash: build-group: Spack Tutorial url: https://cdash.spack.io diff --git a/share/spack/spack-completion.bash b/share/spack/spack-completion.bash index 5e4498f395..19508dd17c 100755 --- a/share/spack/spack-completion.bash +++ b/share/spack/spack-completion.bash @@ -626,7 +626,7 @@ _spack_ci() { } _spack_ci_generate() { - SPACK_COMPREPLY="-h --help --output-file --copy-to --optimize --dependencies --prune-dag --no-prune-dag --check-index-only --artifacts-root" + SPACK_COMPREPLY="-h --help --output-file --copy-to --optimize --dependencies --buildcache-destination --prune-dag --no-prune-dag --check-index-only --artifacts-root" } _spack_ci_rebuild_index() { -- cgit v1.2.3-70-g09d2