diff options
| -rw-r--r-- | lib/spack/docs/pipelines.rst | 10 | ||||
| -rw-r--r-- | lib/spack/spack/ci.py | 81 | ||||
| -rw-r--r-- | lib/spack/spack/schema/ci.py | 57 | ||||
| -rw-r--r-- | lib/spack/spack/schema/env.py | 12 | ||||
| -rw-r--r-- | lib/spack/spack/test/cmd/ci.py | 131 | ||||
| -rw-r--r-- | share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml | 73 | ||||
| -rw-r--r-- | share/spack/gitlab/cloud_pipelines/stacks/deprecated/spack.yaml | 94 |
7 files changed, 433 insertions, 25 deletions
diff --git a/lib/spack/docs/pipelines.rst b/lib/spack/docs/pipelines.rst index 5d06accc4a..d594879aab 100644 --- a/lib/spack/docs/pipelines.rst +++ b/lib/spack/docs/pipelines.rst @@ -632,18 +632,18 @@ Here's an example of what bootstrapping some compilers might look like: exclude: - '%gcc@7.3.0 os=centos7' - '%gcc@5.5.0 os=ubuntu18.04' - gitlab-ci: + ci: bootstrap: - name: compiler-pkgs compiler-agnostic: true - mappings: - # mappings similar to the example higher up in this description + pipeline-gen: + # similar to the example higher up in this description ... The example above adds a list to the ``definitions`` called ``compiler-pkgs`` (you can add any number of these), which lists compiler packages that should be staged ahead of the full matrix of release specs (in this example, only -readline). Then within the ``gitlab-ci`` section, note the addition of a +readline). Then within the ``ci`` section, note the addition of a ``bootstrap`` section, which can contain a list of items, each referring to a list in the ``definitions`` section. These items can either be a dictionary or a string. If you supply a dictionary, it must have a name @@ -709,7 +709,7 @@ be reported. Take a look at the `schema <https://github.com/spack/spack/blob/develop/lib/spack/spack/schema/ci.py>`_ -for the gitlab-ci section of the spack environment file, to see precisely what +for the ci section of the spack environment file, to see precisely what syntax is allowed there. .. _reserved_tags: diff --git a/lib/spack/spack/ci.py b/lib/spack/spack/ci.py index c8aa29d0bd..f7f13cb64e 100644 --- a/lib/spack/spack/ci.py +++ b/lib/spack/spack/ci.py @@ -756,10 +756,20 @@ def generate_gitlab_ci_yaml( ci_config = cfg.get("ci") if not ci_config: - tty.die('Environment yaml does not have "ci" section') + tty.warn("Environment does not have `ci` a configuration") + gitlabci_config = yaml_root.get("gitlab-ci") + if not gitlabci_config: + tty.die("Environment yaml does not have `gitlab-ci` config section. Cannot recover.") + + tty.warn( + "The `gitlab-ci` configuration is deprecated in favor of `ci`.\n", + "To update run \n\t$ spack env update /path/to/ci/spack.yaml", + ) + translate_deprecated_config(gitlabci_config) + ci_config = gitlabci_config # Default target is gitlab...and only target is gitlab - if "target" in ci_config and ci_config["target"] != "gitlab": + if not ci_config.get("target", "gitlab") == "gitlab": tty.die('Spack CI module only generates target "gitlab"') cdash_config = cfg.get("cdash") @@ -938,6 +948,10 @@ def generate_gitlab_ci_yaml( env_includes.extend(include_scopes) env_yaml_root["spack"]["include"] = env_includes + if "gitlab-ci" in env_yaml_root["spack"] and "ci" not in env_yaml_root["spack"]: + env_yaml_root["spack"]["ci"] = env_yaml_root["spack"].pop("gitlab-ci") + translate_deprecated_config(env_yaml_root["spack"]["ci"]) + with open(os.path.join(concrete_env_dir, "spack.yaml"), "w") as fd: fd.write(syaml.dump_config(env_yaml_root, default_flow_style=False)) @@ -2474,3 +2488,66 @@ class CDashHandler(object): ) reporter = CDash(configuration=configuration) reporter.test_skipped_report(directory_name, spec, reason) + + +def translate_deprecated_config(config): + # Remove all deprecated keys from config + mappings = config.pop("mappings", []) + match_behavior = config.pop("match_behavior", "first") + + build_job = {} + if "image" in config: + build_job["image"] = config.pop("image") + if "tags" in config: + build_job["tags"] = config.pop("tags") + if "variables" in config: + build_job["variables"] = config.pop("variables") + if "before_script" in config: + build_job["before_script"] = config.pop("before_script") + if "script" in config: + build_job["script"] = config.pop("script") + if "after_script" in config: + build_job["after_script"] = config.pop("after_script") + + signing_job = None + if "signing-job-attributes" in config: + signing_job = {"signing-job": config.pop("signing-job-attributes")} + + service_job_attributes = None + if "service-job-attributes" in config: + service_job_attributes = config.pop("service-job-attributes") + + # If this config already has pipeline-gen do not more + if "pipeline-gen" in config: + return True if mappings or build_job or signing_job or service_job_attributes else False + + config["target"] = "gitlab" + + config["pipeline-gen"] = [] + pipeline_gen = config["pipeline-gen"] + + # Build Job + submapping = [] + for section in mappings: + submapping_section = {"match": section["match"]} + if "runner-attributes" in section: + submapping_section["build-job"] = section["runner-attributes"] + if "remove-attributes" in section: + submapping_section["build-job-remove"] = section["remove-attributes"] + submapping.append(submapping_section) + pipeline_gen.append({"submapping": submapping, "match_behavior": match_behavior}) + + if build_job: + pipeline_gen.append({"build-job": build_job}) + + # Signing Job + if signing_job: + pipeline_gen.append(signing_job) + + # Service Jobs + if service_job_attributes: + pipeline_gen.append({"reindex-job": service_job_attributes}) + pipeline_gen.append({"noop-job": service_job_attributes}) + pipeline_gen.append({"cleanup-job": service_job_attributes}) + + return True diff --git a/lib/spack/spack/schema/ci.py b/lib/spack/spack/schema/ci.py index 3fcb7fc164..59d1ac615c 100644 --- a/lib/spack/spack/schema/ci.py +++ b/lib/spack/spack/schema/ci.py @@ -11,6 +11,8 @@ from llnl.util.lang import union_dicts +import spack.schema.gitlab_ci + # Schema for script fields # List of lists and/or strings # This is similar to what is allowed in @@ -20,24 +22,27 @@ script_schema = { "items": {"anyOf": [{"type": "string"}, {"type": "array", "items": {"type": "string"}}]}, } +# Schema for CI image +image_schema = { + "oneOf": [ + {"type": "string"}, + { + "type": "object", + "properties": { + "name": {"type": "string"}, + "entrypoint": {"type": "array", "items": {"type": "string"}}, + }, + }, + ] +} + # Additional attributes are allow # and will be forwarded directly to the # CI target YAML for each job. attributes_schema = { "type": "object", "properties": { - "image": { - "oneOf": [ - {"type": "string"}, - { - "type": "object", - "properties": { - "name": {"type": "string"}, - "entrypoint": {"type": "array", "items": {"type": "string"}}, - }, - }, - ] - }, + "image": image_schema, "tags": {"type": "array", "items": {"type": "string"}}, "variables": { "type": "object", @@ -169,7 +174,15 @@ ci_properties = { } #: Properties for inclusion in other schemas -properties = {"ci": ci_properties} +properties = { + "ci": { + "oneOf": [ + ci_properties, + # Allow legacy format under `ci` for `config update ci` + spack.schema.gitlab_ci.gitlab_ci_properties, + ] + } +} #: Full schema with metadata schema = { @@ -179,3 +192,21 @@ schema = { "additionalProperties": False, "properties": properties, } + + +def update(data): + import llnl.util.tty as tty + + import spack.ci + import spack.environment as ev + + # Warn if deprecated section is still in the environment + ci_env = ev.active_environment() + if ci_env: + env_config = ev.config_dict(ci_env.yaml) + if "gitlab-ci" in env_config: + tty.die("Error: `gitlab-ci` section detected with `ci`, these are not compatible") + + # Detect if the ci section is using the new pipeline-gen + # If it is, assume it has already been converted + return spack.ci.translate_deprecated_config(data) diff --git a/lib/spack/spack/schema/env.py b/lib/spack/spack/schema/env.py index 88c1ef1c85..93f0f9129f 100644 --- a/lib/spack/spack/schema/env.py +++ b/lib/spack/spack/schema/env.py @@ -10,6 +10,7 @@ """ from llnl.util.lang import union_dicts +import spack.schema.gitlab_ci # DEPRECATED import spack.schema.merged import spack.schema.packages import spack.schema.projections @@ -52,6 +53,8 @@ schema = { "default": {}, "additionalProperties": False, "properties": union_dicts( + # Include deprecated "gitlab-ci" section + spack.schema.gitlab_ci.properties, # merged configuration scope schemas spack.schema.merged.properties, # extra environment schema properties @@ -130,6 +133,15 @@ def update(data): Returns: True if data was changed, False otherwise """ + + import spack.ci + + if "gitlab-ci" in data: + data["ci"] = data.pop("gitlab-ci") + + if "ci" in data: + return spack.ci.translate_deprecated_config(data["ci"]) + # There are not currently any deprecated attributes in this section # that have not been removed return False diff --git a/lib/spack/spack/test/cmd/ci.py b/lib/spack/spack/test/cmd/ci.py index c2c14410ca..44f195bc3b 100644 --- a/lib/spack/spack/test/cmd/ci.py +++ b/lib/spack/spack/test/cmd/ci.py @@ -33,6 +33,7 @@ from spack.schema.database_index import schema as db_idx_schema from spack.spec import CompilerSpec, Spec from spack.util.pattern import Bunch +config_cmd = spack.main.SpackCommand("config") ci_cmd = spack.main.SpackCommand("ci") env_cmd = spack.main.SpackCommand("env") mirror_cmd = spack.main.SpackCommand("mirror") @@ -412,7 +413,7 @@ spack: """ ) - expect_out = 'Error: Environment yaml does not have "ci" section' + expect_out = "Environment does not have `ci` a configuration" with tmpdir.as_cwd(): env_cmd("create", "test", "./spack.yaml") @@ -1842,12 +1843,11 @@ def test_ci_generate_prune_env_vars( spack: specs: - libelf - ci: - pipeline-gen: - - submapping: + gitlab-ci: + mappings: - match: - arch=test-debian6-core2 - build-job: + runner-attributes: tags: - donotcare image: donotcare @@ -2290,3 +2290,124 @@ def test_cmd_first_line(): ) assert spack.cmd.first_line(doc) == first + + +legacy_spack_yaml_contents = """ +spack: + definitions: + - bootstrap: + - cmake@3.4.3 + - old-gcc-pkgs: + - archive-files + - callpath + # specify ^openblas-with-lapack to ensure that builtin.mock repo flake8 + # package (which can also provide lapack) is not chosen, as it violates + # a package-level check which requires exactly one fetch strategy (this + # is apparently not an issue for other tests that use it). + - hypre@0.2.15 ^openblas-with-lapack + specs: + - matrix: + - [$old-gcc-pkgs] + mirrors: + test-mirror: file:///some/fake/mirror + {0}: + bootstrap: + - name: bootstrap + compiler-agnostic: true + match_behavior: first + mappings: + - match: + - arch=test-debian6-core2 + runner-attributes: + tags: + - donotcare + image: donotcare + - match: + - arch=test-debian6-m1 + runner-attributes: + tags: + - donotcare + image: donotcare + service-job-attributes: + image: donotcare + tags: [donotcare] + cdash: + build-group: Not important + url: https://my.fake.cdash + project: Not used + site: Nothing +""" + + +@pytest.mark.regression("36409") +def test_gitlab_ci_deprecated( + tmpdir, + mutable_mock_env_path, + install_mockery, + mock_packages, + monkeypatch, + ci_base_environment, + mock_binary_index, +): + mirror_url = "file:///some/fake/mirror" + filename = str(tmpdir.join("spack.yaml")) + with open(filename, "w") as f: + f.write(legacy_spack_yaml_contents.format("gitlab-ci")) + + with tmpdir.as_cwd(): + env_cmd("create", "test", "./spack.yaml") + outputfile = "generated-pipeline.yaml" + + with ev.read("test"): + ci_cmd("generate", "--output-file", outputfile) + + with open(outputfile) as f: + contents = f.read() + yaml_contents = syaml.load(contents) + + found_spec = False + for ci_key in yaml_contents.keys(): + if "(bootstrap)" in ci_key: + found_spec = True + assert "cmake" in ci_key + assert found_spec + assert "stages" in yaml_contents + assert len(yaml_contents["stages"]) == 6 + assert yaml_contents["stages"][0] == "stage-0" + assert yaml_contents["stages"][5] == "stage-rebuild-index" + + assert "rebuild-index" in yaml_contents + rebuild_job = yaml_contents["rebuild-index"] + expected = "spack buildcache update-index --keys --mirror-url {0}".format(mirror_url) + assert rebuild_job["script"][0] == expected + + assert "variables" in yaml_contents + assert "SPACK_ARTIFACTS_ROOT" in yaml_contents["variables"] + artifacts_root = yaml_contents["variables"]["SPACK_ARTIFACTS_ROOT"] + assert artifacts_root == "jobs_scratch_dir" + + +@pytest.mark.regression("36045") +def test_gitlab_ci_update( + tmpdir, + mutable_mock_env_path, + install_mockery, + mock_packages, + monkeypatch, + ci_base_environment, + mock_binary_index, +): + filename = str(tmpdir.join("spack.yaml")) + with open(filename, "w") as f: + f.write(legacy_spack_yaml_contents.format("ci")) + + with tmpdir.as_cwd(): + env_cmd("update", "-y", ".") + + with open("spack.yaml") as f: + contents = f.read() + yaml_contents = syaml.load(contents) + + ci_root = yaml_contents["spack"]["ci"] + + assert "pipeline-gen" in ci_root diff --git a/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml b/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml index 9d3282feea..de8e5dc552 100644 --- a/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml +++ b/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml @@ -102,6 +102,37 @@ default: extends: [ ".generate-base" ] tags: ["spack", "public", "medium", "x86_64"] +.generate-deprecated: + stage: generate + script: + - uname -a || true + - grep -E 'vendor|model name' /proc/cpuinfo 2>/dev/null | sort -u || head -n10 /proc/cpuinfo 2>/dev/null || true + - nproc || true + - . "./share/spack/setup-env.sh" + - spack --version + - cd share/spack/gitlab/cloud_pipelines/stacks/${SPACK_CI_STACK_NAME} + - spack env activate --without-view . + - spack + ci generate --check-index-only + --buildcache-destination "${SPACK_BUILDCACHE_DESTINATION}" + --artifacts-root "${CI_PROJECT_DIR}/jobs_scratch_dir" + --output-file "${CI_PROJECT_DIR}/jobs_scratch_dir/cloud-ci-pipeline.yml" + after_script: + - cat /proc/loadavg || true + artifacts: + paths: + - "${CI_PROJECT_DIR}/jobs_scratch_dir" + variables: + KUBERNETES_CPU_REQUEST: 4000m + KUBERNETES_MEMORY_REQUEST: 16G + interruptible: true + timeout: 60 minutes + retry: + max: 2 + when: + - always + tags: ["spack", "public", "medium", "x86_64"] + .generate-aarch64: extends: [ ".generate" ] tags: ["spack", "public", "medium", "aarch64"] @@ -109,12 +140,18 @@ default: .pr-generate: extends: [ ".pr", ".generate" ] +.pr-generate-deprecated: + extends: [ ".pr", ".generate-deprecated" ] + .pr-generate-aarch64: extends: [ ".pr", ".generate-aarch64" ] .protected-generate: extends: [ ".protected", ".generate" ] +.protected-generate-deprecated: + extends: [ ".protected", ".generate-deprecated" ] + .protected-generate-aarch64: extends: [ ".protected", ".generate-aarch64" ] @@ -978,3 +1015,39 @@ ml-linux-x86_64-rocm-protected-build: needs: - artifacts: True job: ml-linux-x86_64-rocm-protected-generate + + +######################################## +# Deprecated CI testing +######################################## +.deprecated-ci: + variables: + SPACK_CI_STACK_NAME: deprecated + +deprecated-ci-pr-generate: + extends: [ ".pr-generate-deprecated", ".deprecated-ci" ] + +deprecated-ci-protected-generate: + extends: [ ".protected-generate-deprecated", ".deprecated-ci" ] + +deprecated-ci-pr-build: + extends: [ ".pr-build", ".deprecated-ci" ] + trigger: + include: + - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml + job: deprecated-ci-pr-generate + strategy: depend + needs: + - artifacts: True + job: deprecated-ci-pr-generate + +deprecated-ci-protected-build: + extends: [ ".protected-build", ".deprecated-ci" ] + trigger: + include: + - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml + job: deprecated-ci-protected-generate + strategy: depend + needs: + - artifacts: True + job: deprecated-ci-protected-generate diff --git a/share/spack/gitlab/cloud_pipelines/stacks/deprecated/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/deprecated/spack.yaml new file mode 100644 index 0000000000..edb27d5f4f --- /dev/null +++ b/share/spack/gitlab/cloud_pipelines/stacks/deprecated/spack.yaml @@ -0,0 +1,94 @@ +### +# Spack pipeline for testing deprecated gitlab-ci configuration +### +spack: + view: false + concretizer: + reuse: false + unify: false + config: + concretizer: clingo + db_lock_timeout: 120 + install_tree: + padded_length: 256 + projections: + all: '{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}' + deprecated: true + packages: + all: + require: target=x86_64 + specs: + - readline + + mirrors: + mirror: s3://spack-binaries/develop/deprecated + gitlab-ci: + broken-tests-packages: + - gptune + broken-specs-url: s3://spack-binaries/broken-specs + image: ghcr.io/spack/tutorial-ubuntu-18.04:v2021-11-02 + before_script: + - curl -LfsS "https://github.com/JuliaBinaryWrappers/GNUMake_jll.jl/releases/download/GNUMake-v4.3.0+1/GNUMake.v4.3.0.x86_64-linux-gnu.tar.gz" + -o gmake.tar.gz + - printf "fef1f59e56d2d11e6d700ba22d3444b6e583c663d6883fd0a4f63ab8bd280f0f gmake.tar.gz" + | sha256sum --check --strict --quiet + - tar -xzf gmake.tar.gz -C /usr bin/make 2> /dev/null + - uname -a || true + - grep -E "vendor|model name" /proc/cpuinfo 2>/dev/null | sort -u || head -n10 + /proc/cpuinfo 2>/dev/null || true + - nproc + - . "./share/spack/setup-env.sh" + - spack --version + - spack arch + script: + - spack compiler find + - cd ${SPACK_CONCRETE_ENV_DIR} + - spack env activate --without-view . + - if [ -n "$SPACK_BUILD_JOBS" ]; then spack config add "config:build_jobs:$SPACK_BUILD_JOBS"; + fi + - spack config add "config:install_tree:projections:${SPACK_JOB_SPEC_PKG_NAME}:'morepadding/{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'" + - mkdir -p ${SPACK_ARTIFACTS_ROOT}/user_data + # AWS runners mount E4S public key (verification), UO runners mount public/private (signing/verification) + - if [[ -r /mnt/key/e4s.gpg ]]; then spack gpg trust /mnt/key/e4s.gpg; fi + # UO runners mount intermediate ci public key (verification), AWS runners mount public/private (signing/verification) + - if [[ -r /mnt/key/intermediate_ci_signing_key.gpg ]]; then spack gpg trust /mnt/key/intermediate_ci_signing_key.gpg; + fi + - if [[ -r /mnt/key/spack_public_key.gpg ]]; then spack gpg trust /mnt/key/spack_public_key.gpg; + fi + - spack --color=always --backtrace ci rebuild --tests > >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_out.txt) + 2> >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_err.txt >&2) + after_script: + - cat /proc/loadavg || true + match_behavior: first + mappings: + - match: + - '@:' + runner-attributes: + tags: [spack, public, small, x86_64] + variables: + CI_JOB_SIZE: small + SPACK_BUILD_JOBS: '1' + KUBERNETES_CPU_REQUEST: 500m + KUBERNETES_MEMORY_REQUEST: 500M + signing-job-attributes: + image: {name: 'ghcr.io/spack/notary:latest', entrypoint: ['']} + tags: [aws] + script: + - aws s3 sync --exclude "*" --include "*spec.json*" ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache + /tmp + - /sign.sh + - aws s3 sync --exclude "*" --include "*spec.json.sig*" /tmp ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache + - aws s3 cp /tmp/public_keys ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache/_pgp + --recursive --exclude "*" --include "*.pub" + + service-job-attributes: + image: ghcr.io/spack/tutorial-ubuntu-18.04:v2021-11-02 + before_script: + - . "./share/spack/setup-env.sh" + - spack --version + tags: [spack, public, x86_64] + cdash: + build-group: Spack Deprecated CI + url: https://cdash.spack.io + project: Spack Testing + site: Cloud Gitlab Infrastructure |