diff options
| -rw-r--r-- | lib/spack/spack/binary_distribution.py | 18 | ||||
| -rw-r--r-- | lib/spack/spack/ci.py | 182 | ||||
| -rw-r--r-- | lib/spack/spack/cmd/ci.py | 44 | ||||
| -rw-r--r-- | lib/spack/spack/schema/gitlab_ci.py | 1 | ||||
| -rw-r--r-- | lib/spack/spack/test/cmd/ci.py | 155 | ||||
| -rw-r--r-- | share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml | 180 | ||||
| -rw-r--r-- | share/spack/gitlab/cloud_pipelines/stacks/build_systems/spack.yaml | 23 | ||||
| -rw-r--r-- | share/spack/gitlab/cloud_pipelines/stacks/data-vis-sdk/spack.yaml | 25 | ||||
| -rw-r--r-- | share/spack/gitlab/cloud_pipelines/stacks/e4s-mac/spack.yaml | 6 | ||||
| -rw-r--r-- | share/spack/gitlab/cloud_pipelines/stacks/e4s/spack.yaml | 25 | ||||
| -rw-r--r-- | share/spack/gitlab/cloud_pipelines/stacks/radiuss/spack.yaml | 22 | ||||
| -rw-r--r-- | share/spack/gitlab/cloud_pipelines/stacks/tutorial/spack.yaml | 23 | ||||
| -rwxr-xr-x | share/spack/spack-completion.bash | 2 |
13 files changed, 534 insertions, 172 deletions
diff --git a/lib/spack/spack/binary_distribution.py b/lib/spack/spack/binary_distribution.py index ac1723a94c..fb4c6ce472 100644 --- a/lib/spack/spack/binary_distribution.py +++ b/lib/spack/spack/binary_distribution.py @@ -210,7 +210,7 @@ class BinaryCacheIndex(object): return spec_list - def find_built_spec(self, spec): + def find_built_spec(self, spec, mirrors_to_check=None): """Look in our cache for the built spec corresponding to ``spec``. If the spec can be found among the configured binary mirrors, a @@ -225,6 +225,8 @@ class BinaryCacheIndex(object): Args: spec (spack.spec.Spec): Concrete spec to find + mirrors_to_check: Optional mapping containing mirrors to check. If + None, just assumes all configured mirrors. Returns: An list of objects containing the found specs and mirror url where @@ -240,17 +242,23 @@ class BinaryCacheIndex(object): ] """ self.regenerate_spec_cache() - return self.find_by_hash(spec.dag_hash()) + return self.find_by_hash(spec.dag_hash(), mirrors_to_check=mirrors_to_check) - def find_by_hash(self, find_hash): + def find_by_hash(self, find_hash, mirrors_to_check=None): """Same as find_built_spec but uses the hash of a spec. Args: find_hash (str): hash of the spec to search + mirrors_to_check: Optional mapping containing mirrors to check. If + None, just assumes all configured mirrors. """ if find_hash not in self._mirrors_for_spec: return None - return self._mirrors_for_spec[find_hash] + results = self._mirrors_for_spec[find_hash] + if not mirrors_to_check: + return results + mirror_urls = mirrors_to_check.values() + return [r for r in results if r['mirror_url'] in mirror_urls] def update_spec(self, spec, found_list): """ @@ -1822,7 +1830,7 @@ def get_mirrors_for_spec(spec=None, mirrors_to_check=None, index_only=False): tty.debug("No Spack mirrors are currently configured") return {} - results = binary_index.find_built_spec(spec) + results = binary_index.find_built_spec(spec, mirrors_to_check=mirrors_to_check) # Maybe we just didn't have the latest information from the mirror, so # try to fetch directly, unless we are only considering the indices. diff --git a/lib/spack/spack/ci.py b/lib/spack/spack/ci.py index d2eb8634ea..254821cb3d 100644 --- a/lib/spack/spack/ci.py +++ b/lib/spack/spack/ci.py @@ -33,7 +33,6 @@ import spack.repo import spack.util.executable as exe import spack.util.gpg as gpg_util import spack.util.spack_yaml as syaml -import spack.util.url as url_util import spack.util.web as web_util from spack.error import SpackError from spack.spec import Spec @@ -42,10 +41,8 @@ JOB_RETRY_CONDITIONS = [ 'always', ] -SPACK_PR_MIRRORS_ROOT_URL = 's3://spack-binaries-prs' -SPACK_SHARED_PR_MIRROR_URL = url_util.join(SPACK_PR_MIRRORS_ROOT_URL, - 'shared_pr_mirror') TEMP_STORAGE_MIRROR_NAME = 'ci_temporary_mirror' +SPACK_RESERVED_TAGS = ["public", "protected", "notary"] spack_gpg = spack.main.SpackCommand('gpg') spack_compiler = spack.main.SpackCommand('compiler') @@ -199,6 +196,11 @@ def _get_cdash_build_name(spec, build_group): spec.name, spec.version, spec.compiler, spec.architecture, build_group) +def _remove_reserved_tags(tags): + """Convenience function to strip reserved tags from jobs""" + return [tag for tag in tags if tag not in SPACK_RESERVED_TAGS] + + def _get_spec_string(spec): format_elements = [ '{name}{@version}', @@ -231,8 +233,10 @@ def _add_dependency(spec_label, dep_label, deps): deps[spec_label].add(dep_label) -def _get_spec_dependencies(specs, deps, spec_labels, check_index_only=False): - spec_deps_obj = _compute_spec_deps(specs, check_index_only=check_index_only) +def _get_spec_dependencies(specs, deps, spec_labels, check_index_only=False, + mirrors_to_check=None): + spec_deps_obj = _compute_spec_deps(specs, check_index_only=check_index_only, + mirrors_to_check=mirrors_to_check) if spec_deps_obj: dependencies = spec_deps_obj['dependencies'] @@ -249,7 +253,7 @@ def _get_spec_dependencies(specs, deps, spec_labels, check_index_only=False): _add_dependency(entry['spec'], entry['depends'], deps) -def stage_spec_jobs(specs, check_index_only=False): +def stage_spec_jobs(specs, check_index_only=False, mirrors_to_check=None): """Take a set of release specs and generate a list of "stages", where the jobs in any stage are dependent only on jobs in previous stages. This allows us to maximize build parallelism within the gitlab-ci framework. @@ -261,6 +265,8 @@ def stage_spec_jobs(specs, check_index_only=False): are up to date on those mirrors. This flag limits that search to the binary cache indices on those mirrors to speed the process up, even though there is no garantee the index is up to date. + mirrors_to_checK: Optional mapping giving mirrors to check instead of + any configured mirrors. Returns: A tuple of information objects describing the specs, dependencies and stages: @@ -297,8 +303,8 @@ def stage_spec_jobs(specs, check_index_only=False): deps = {} spec_labels = {} - _get_spec_dependencies( - specs, deps, spec_labels, check_index_only=check_index_only) + _get_spec_dependencies(specs, deps, spec_labels, check_index_only=check_index_only, + mirrors_to_check=mirrors_to_check) # Save the original deps, as we need to return them at the end of the # function. In the while loop below, the "dependencies" variable is @@ -340,7 +346,7 @@ def _print_staging_summary(spec_labels, dependencies, stages): _get_spec_string(s))) -def _compute_spec_deps(spec_list, check_index_only=False): +def _compute_spec_deps(spec_list, check_index_only=False, mirrors_to_check=None): """ Computes all the dependencies for the spec(s) and generates a JSON object which provides both a list of unique spec names as well as a @@ -413,7 +419,7 @@ def _compute_spec_deps(spec_list, check_index_only=False): continue up_to_date_mirrors = bindist.get_mirrors_for_spec( - spec=s, index_only=check_index_only) + spec=s, mirrors_to_check=mirrors_to_check, index_only=check_index_only) skey = _spec_deps_key(s) spec_labels[skey] = { @@ -602,8 +608,8 @@ def get_spec_filter_list(env, affected_pkgs, dependencies=True, dependents=True) def generate_gitlab_ci_yaml(env, print_summary, output_file, prune_dag=False, check_index_only=False, run_optimizer=False, use_dependencies=False, - artifacts_root=None): - """ Generate a gitlab yaml file to run a dynamic chile pipeline from + artifacts_root=None, remote_mirror_override=None): + """ Generate a gitlab yaml file to run a dynamic child pipeline from the spec matrix in the active environment. Arguments: @@ -629,6 +635,10 @@ def generate_gitlab_ci_yaml(env, print_summary, output_file, artifacts_root (str): Path where artifacts like logs, environment files (spack.yaml, spack.lock), etc should be written. GitLab requires this to be within the project directory. + remote_mirror_override (str): Typically only needed when one spack.yaml + is used to populate several mirrors with binaries, based on some + criteria. Spack protected pipelines populate different mirrors based + on branch name, facilitated by this option. """ with spack.concretize.disable_compiler_existence_check(): with env.write_transaction(): @@ -678,17 +688,19 @@ def generate_gitlab_ci_yaml(env, print_summary, output_file, for s in affected_specs: tty.debug(' {0}'.format(s.name)) - generate_job_name = os.environ.get('CI_JOB_NAME', None) - parent_pipeline_id = os.environ.get('CI_PIPELINE_ID', None) + # Downstream jobs will "need" (depend on, for both scheduling and + # artifacts, which include spack.lock file) this pipeline generation + # job by both name and pipeline id. If those environment variables + # do not exist, then maybe this is just running in a shell, in which + # case, there is no expectation gitlab will ever run the generated + # pipeline and those environment variables do not matter. + generate_job_name = os.environ.get('CI_JOB_NAME', 'job-does-not-exist') + parent_pipeline_id = os.environ.get('CI_PIPELINE_ID', 'pipeline-does-not-exist') + # Values: "spack_pull_request", "spack_protected_branch", or not set spack_pipeline_type = os.environ.get('SPACK_PIPELINE_TYPE', None) - is_pr_pipeline = spack_pipeline_type == 'spack_pull_request' - spack_pr_branch = os.environ.get('SPACK_PR_BRANCH', None) - pr_mirror_url = None - if spack_pr_branch: - pr_mirror_url = url_util.join(SPACK_PR_MIRRORS_ROOT_URL, - spack_pr_branch) + spack_buildcache_copy = os.environ.get('SPACK_COPY_BUILDCACHE', None) if 'mirrors' not in yaml_root or len(yaml_root['mirrors'].values()) < 1: tty.die('spack ci generate requires an env containing a mirror') @@ -743,14 +755,25 @@ def generate_gitlab_ci_yaml(env, print_summary, output_file, 'strip-compilers': False, }) - # Add per-PR mirror (and shared PR mirror) if enabled, as some specs might - # be up to date in one of those and thus not need to be rebuilt. - if pr_mirror_url: - spack.mirror.add( - 'ci_pr_mirror', pr_mirror_url, cfg.default_modify_scope()) - spack.mirror.add('ci_shared_pr_mirror', - SPACK_SHARED_PR_MIRROR_URL, - cfg.default_modify_scope()) + # If a remote mirror override (alternate buildcache destination) was + # specified, add it here in case it has already built hashes we might + # generate. + mirrors_to_check = None + if remote_mirror_override: + if spack_pipeline_type == 'spack_protected_branch': + # Overriding the main mirror in this case might result + # in skipping jobs on a release pipeline because specs are + # up to date in develop. Eventually we want to notice and take + # advantage of this by scheduling a job to copy the spec from + # develop to the release, but until we have that, this makes + # sure we schedule a rebuild job if the spec isn't already in + # override mirror. + mirrors_to_check = { + 'override': remote_mirror_override + } + else: + spack.mirror.add( + 'ci_pr_mirror', remote_mirror_override, cfg.default_modify_scope()) pipeline_artifacts_dir = artifacts_root if not pipeline_artifacts_dir: @@ -825,11 +848,13 @@ def generate_gitlab_ci_yaml(env, print_summary, output_file, phase_spec.concretize() staged_phases[phase_name] = stage_spec_jobs( concrete_phase_specs, - check_index_only=check_index_only) + check_index_only=check_index_only, + mirrors_to_check=mirrors_to_check) finally: - # Clean up PR mirror if enabled - if pr_mirror_url: - spack.mirror.remove('ci_pr_mirror', cfg.default_modify_scope()) + # Clean up remote mirror override if enabled + if remote_mirror_override: + if spack_pipeline_type != 'spack_protected_branch': + spack.mirror.remove('ci_pr_mirror', cfg.default_modify_scope()) all_job_names = [] output_object = {} @@ -889,6 +914,14 @@ def generate_gitlab_ci_yaml(env, print_summary, output_file, tags = [tag for tag in runner_attribs['tags']] + if spack_pipeline_type is not None: + # For spack pipelines "public" and "protected" are reserved tags + tags = _remove_reserved_tags(tags) + if spack_pipeline_type == 'spack_protected_branch': + tags.extend(['aws', 'protected']) + elif spack_pipeline_type == 'spack_pull_request': + tags.extend(['public']) + variables = {} if 'variables' in runner_attribs: variables.update(runner_attribs['variables']) @@ -1174,6 +1207,10 @@ def generate_gitlab_ci_yaml(env, print_summary, output_file, service_job_config, cleanup_job) + if 'tags' in cleanup_job: + service_tags = _remove_reserved_tags(cleanup_job['tags']) + cleanup_job['tags'] = service_tags + cleanup_job['stage'] = 'cleanup-temp-storage' cleanup_job['script'] = [ 'spack -d mirror destroy --mirror-url {0}/$CI_PIPELINE_ID'.format( @@ -1181,9 +1218,74 @@ def generate_gitlab_ci_yaml(env, print_summary, output_file, ] cleanup_job['when'] = 'always' cleanup_job['retry'] = service_job_retries + cleanup_job['interruptible'] = True output_object['cleanup'] = cleanup_job + if ('signing-job-attributes' in gitlab_ci and + spack_pipeline_type == 'spack_protected_branch'): + # External signing: generate a job to check and sign binary pkgs + stage_names.append('stage-sign-pkgs') + signing_job_config = gitlab_ci['signing-job-attributes'] + signing_job = {} + + signing_job_attrs_to_copy = [ + 'image', + 'tags', + 'variables', + 'before_script', + 'script', + 'after_script', + ] + + _copy_attributes(signing_job_attrs_to_copy, + signing_job_config, + signing_job) + + signing_job_tags = [] + if 'tags' in signing_job: + signing_job_tags = _remove_reserved_tags(signing_job['tags']) + + for tag in ['aws', 'protected', 'notary']: + if tag not in signing_job_tags: + signing_job_tags.append(tag) + signing_job['tags'] = signing_job_tags + + signing_job['stage'] = 'stage-sign-pkgs' + signing_job['when'] = 'always' + signing_job['retry'] = { + 'max': 2, + 'when': ['always'] + } + signing_job['interruptible'] = True + + output_object['sign-pkgs'] = signing_job + + if spack_buildcache_copy: + # Generate a job to copy the contents from wherever the builds are getting + # pushed to the url specified in the "SPACK_BUILDCACHE_COPY" environment + # variable. + src_url = remote_mirror_override or remote_mirror_url + dest_url = spack_buildcache_copy + + stage_names.append('stage-copy-buildcache') + copy_job = { + 'stage': 'stage-copy-buildcache', + 'tags': ['spack', 'public', 'medium', 'aws', 'x86_64'], + 'image': 'ghcr.io/spack/python-aws-bash:0.0.1', + 'when': 'on_success', + 'interruptible': True, + 'retry': service_job_retries, + 'script': [ + '. ./share/spack/setup-env.sh', + 'spack --version', + 'aws s3 sync --exclude *index.json* --exclude *pgp* {0} {1}'.format( + src_url, dest_url) + ] + } + + output_object['copy-mirror'] = copy_job + if rebuild_index_enabled: # Add a final job to regenerate the index stage_names.append('stage-rebuild-index') @@ -1194,9 +1296,13 @@ def generate_gitlab_ci_yaml(env, print_summary, output_file, service_job_config, final_job) + if 'tags' in final_job: + service_tags = _remove_reserved_tags(final_job['tags']) + final_job['tags'] = service_tags + index_target_mirror = mirror_urls[0] - if is_pr_pipeline: - index_target_mirror = pr_mirror_url + if remote_mirror_override: + index_target_mirror = remote_mirror_override final_job['stage'] = 'stage-rebuild-index' final_job['script'] = [ @@ -1205,6 +1311,7 @@ def generate_gitlab_ci_yaml(env, print_summary, output_file, ] final_job['when'] = 'always' final_job['retry'] = service_job_retries + final_job['interruptible'] = True output_object['rebuild-index'] = final_job @@ -1237,8 +1344,9 @@ def generate_gitlab_ci_yaml(env, print_summary, output_file, 'SPACK_PIPELINE_TYPE': str(spack_pipeline_type) } - if pr_mirror_url: - output_object['variables']['SPACK_PR_MIRROR_URL'] = pr_mirror_url + if remote_mirror_override: + (output_object['variables'] + ['SPACK_REMOTE_MIRROR_OVERRIDE']) = remote_mirror_override spack_stack_name = os.environ.get('SPACK_CI_STACK_NAME', None) if spack_stack_name: diff --git a/lib/spack/spack/cmd/ci.py b/lib/spack/spack/cmd/ci.py index 9687aceac8..fc30c1c1bf 100644 --- a/lib/spack/spack/cmd/ci.py +++ b/lib/spack/spack/cmd/ci.py @@ -64,6 +64,11 @@ def setup_parser(subparser): '--dependencies', action='store_true', default=False, help="(Experimental) disable DAG scheduling; use " ' "plain" dependencies.') + generate.add_argument( + '--buildcache-destination', default=None, + help="Override the mirror configured in the environment (spack.yaml) " + + "in order to push binaries from the generated pipeline to a " + + "different location.") prune_group = generate.add_mutually_exclusive_group() prune_group.add_argument( '--prune-dag', action='store_true', dest='prune_dag', @@ -127,6 +132,7 @@ def ci_generate(args): prune_dag = args.prune_dag index_only = args.index_only artifacts_root = args.artifacts_root + buildcache_destination = args.buildcache_destination if not output_file: output_file = os.path.abspath(".gitlab-ci.yml") @@ -140,7 +146,8 @@ def ci_generate(args): spack_ci.generate_gitlab_ci_yaml( env, True, output_file, prune_dag=prune_dag, check_index_only=index_only, run_optimizer=run_optimizer, - use_dependencies=use_dependencies, artifacts_root=artifacts_root) + use_dependencies=use_dependencies, artifacts_root=artifacts_root, + remote_mirror_override=buildcache_destination) if copy_yaml_to: copy_to_dir = os.path.dirname(copy_yaml_to) @@ -180,6 +187,9 @@ def ci_rebuild(args): if not gitlab_ci: tty.die('spack ci rebuild requires an env containing gitlab-ci cfg') + tty.msg('SPACK_BUILDCACHE_DESTINATION={0}'.format( + os.environ.get('SPACK_BUILDCACHE_DESTINATION', None))) + # Grab the environment variables we need. These either come from the # pipeline generation step ("spack ci generate"), where they were written # out as variables, or else provided by GitLab itself. @@ -196,7 +206,7 @@ def ci_rebuild(args): compiler_action = get_env_var('SPACK_COMPILER_ACTION') cdash_build_name = get_env_var('SPACK_CDASH_BUILD_NAME') spack_pipeline_type = get_env_var('SPACK_PIPELINE_TYPE') - pr_mirror_url = get_env_var('SPACK_PR_MIRROR_URL') + remote_mirror_override = get_env_var('SPACK_REMOTE_MIRROR_OVERRIDE') remote_mirror_url = get_env_var('SPACK_REMOTE_MIRROR_URL') # Construct absolute paths relative to current $CI_PROJECT_DIR @@ -244,6 +254,10 @@ def ci_rebuild(args): tty.debug('Pipeline type - PR: {0}, develop: {1}'.format( spack_is_pr_pipeline, spack_is_develop_pipeline)) + # If no override url exists, then just push binary package to the + # normal remote mirror url. + buildcache_mirror_url = remote_mirror_override or remote_mirror_url + # Figure out what is our temporary storage mirror: Is it artifacts # buildcache? Or temporary-storage-url-prefix? In some cases we need to # force something or pipelines might not have a way to propagate build @@ -373,7 +387,24 @@ def ci_rebuild(args): cfg.default_modify_scope()) # Check configured mirrors for a built spec with a matching hash - matches = bindist.get_mirrors_for_spec(job_spec, index_only=False) + mirrors_to_check = None + if remote_mirror_override and spack_pipeline_type == 'spack_protected_branch': + # Passing "mirrors_to_check" below means we *only* look in the override + # mirror to see if we should skip building, which is what we want. + mirrors_to_check = { + 'override': remote_mirror_override + } + + # Adding this mirror to the list of configured mirrors means dependencies + # could be installed from either the override mirror or any other configured + # mirror (e.g. remote_mirror_url which is defined in the environment or + # pipeline_mirror_url), which is also what we want. + spack.mirror.add('mirror_override', + remote_mirror_override, + cfg.default_modify_scope()) + + matches = bindist.get_mirrors_for_spec( + job_spec, mirrors_to_check=mirrors_to_check, index_only=False) if matches: # Got a hash match on at least one configured mirror. All @@ -517,13 +548,6 @@ def ci_rebuild(args): # any logs from the staging directory to artifacts now spack_ci.copy_stage_logs_to_artifacts(job_spec, job_log_dir) - # Create buildcache on remote mirror, either on pr-specific mirror or - # on the main mirror defined in the gitlab-enabled spack environment - if spack_is_pr_pipeline: - buildcache_mirror_url = pr_mirror_url - else: - buildcache_mirror_url = remote_mirror_url - # If the install succeeded, create a buildcache entry for this job spec # and push it to one or more mirrors. If the install did not succeed, # print out some instructions on how to reproduce this build failure diff --git a/lib/spack/spack/schema/gitlab_ci.py b/lib/spack/spack/schema/gitlab_ci.py index 4a08f8d6ce..80e40dd72b 100644 --- a/lib/spack/spack/schema/gitlab_ci.py +++ b/lib/spack/spack/schema/gitlab_ci.py @@ -110,6 +110,7 @@ core_shared_properties = union_dicts( }, }, 'service-job-attributes': runner_selector_schema, + 'signing-job-attributes': runner_selector_schema, 'rebuild-index': {'type': 'boolean'}, 'broken-specs-url': {'type': 'string'}, }, diff --git a/lib/spack/spack/test/cmd/ci.py b/lib/spack/spack/test/cmd/ci.py index ba422f0923..f6b6bb8c19 100644 --- a/lib/spack/spack/test/cmd/ci.py +++ b/lib/spack/spack/test/cmd/ci.py @@ -635,10 +635,6 @@ spack: outputfile = str(tmpdir.join('.gitlab-ci.yml')) with ev.read('test'): - monkeypatch.setattr( - ci, 'SPACK_PR_MIRRORS_ROOT_URL', r"file:///fake/mirror") - monkeypatch.setattr( - ci, 'SPACK_SHARED_PR_MIRROR_URL', r"file:///fake/mirror_two") ci_cmd('generate', '--output-file', outputfile) with open(outputfile) as f: @@ -683,10 +679,6 @@ spack: outputfile = str(tmpdir.join('.gitlab-ci.yml')) with ev.read('test'): - monkeypatch.setattr( - ci, 'SPACK_PR_MIRRORS_ROOT_URL', r"file:///fake/mirror") - monkeypatch.setattr( - ci, 'SPACK_SHARED_PR_MIRROR_URL', r"file:///fake/mirror_two") ci_cmd('generate', '--output-file', outputfile) with open(outputfile) as f: @@ -920,6 +912,77 @@ spack: env_cmd('deactivate') +def test_ci_generate_mirror_override(tmpdir, mutable_mock_env_path, + install_mockery_mutable_config, mock_packages, + mock_fetch, mock_stage, mock_binary_index, + ci_base_environment): + """Ensure that protected pipelines using --buildcache-destination do not + skip building specs that are not in the override mirror when they are + found in the main mirror.""" + os.environ.update({ + 'SPACK_PIPELINE_TYPE': 'spack_protected_branch', + }) + + working_dir = tmpdir.join('working_dir') + + mirror_dir = working_dir.join('mirror') + mirror_url = 'file://{0}'.format(mirror_dir.strpath) + + spack_yaml_contents = """ +spack: + definitions: + - packages: [patchelf] + specs: + - $packages + mirrors: + test-mirror: {0} + gitlab-ci: + mappings: + - match: + - patchelf + runner-attributes: + tags: + - donotcare + image: donotcare + service-job-attributes: + tags: + - nonbuildtag + image: basicimage +""".format(mirror_url) + + filename = str(tmpdir.join('spack.yaml')) + with open(filename, 'w') as f: + f.write(spack_yaml_contents) + + with tmpdir.as_cwd(): + env_cmd('create', 'test', './spack.yaml') + first_ci_yaml = str(tmpdir.join('.gitlab-ci-1.yml')) + second_ci_yaml = str(tmpdir.join('.gitlab-ci-2.yml')) + with ev.read('test'): + install_cmd() + buildcache_cmd('create', '-u', '--mirror-url', mirror_url, 'patchelf') + buildcache_cmd('update-index', '--mirror-url', mirror_url, output=str) + + # This generate should not trigger a rebuild of patchelf, since it's in + # the main mirror referenced in the environment. + ci_cmd('generate', '--check-index-only', '--output-file', first_ci_yaml) + + # Because we used a mirror override (--buildcache-destination) on a + # spack protected pipeline, we expect to only look in the override + # mirror for the spec, and thus the patchelf job should be generated in + # this pipeline + ci_cmd('generate', '--check-index-only', '--output-file', second_ci_yaml, + '--buildcache-destination', 'file:///mirror/not/exist') + + with open(first_ci_yaml) as fd1: + first_yaml = fd1.read() + assert 'no-specs-to-rebuild' in first_yaml + + with open(second_ci_yaml) as fd2: + second_yaml = fd2.read() + assert 'no-specs-to-rebuild' not in second_yaml + + @pytest.mark.disable_clean_stage_check def test_push_mirror_contents(tmpdir, mutable_mock_env_path, install_mockery_mutable_config, mock_packages, @@ -1151,10 +1214,6 @@ spack: with ev.read('test'): monkeypatch.setattr( spack.main, 'get_version', lambda: '0.15.3-416-12ad69eb1') - monkeypatch.setattr( - ci, 'SPACK_PR_MIRRORS_ROOT_URL', r"file:///fake/mirror") - monkeypatch.setattr( - ci, 'SPACK_SHARED_PR_MIRROR_URL', r"file:///fake/mirror_two") ci_cmd('generate', '--output-file', outputfile) with open(outputfile) as f: @@ -1256,10 +1315,6 @@ spack: outputfile = str(tmpdir.join('.gitlab-ci.yml')) with ev.read('test'): - monkeypatch.setattr( - ci, 'SPACK_PR_MIRRORS_ROOT_URL', r"file:///fake/mirror") - monkeypatch.setattr( - ci, 'SPACK_SHARED_PR_MIRROR_URL', r"file:///fake/mirror_two") ci_cmd('generate', '--output-file', outputfile, '--dependencies') with open(outputfile) as f: @@ -1417,11 +1472,6 @@ spack: outputfile = str(tmpdir.join('.gitlab-ci.yml')) with ev.read('test'): - monkeypatch.setattr( - ci, 'SPACK_PR_MIRRORS_ROOT_URL', r"file:///fake/mirror") - monkeypatch.setattr( - ci, 'SPACK_SHARED_PR_MIRROR_URL', r"file:///fake/mirror_two") - ci_cmd('generate', '--output-file', outputfile) with open(outputfile) as of: @@ -1630,11 +1680,6 @@ spack: env_cmd('create', 'test', './spack.yaml') outputfile = str(tmpdir.join('.gitlab-ci.yml')) - monkeypatch.setattr( - ci, 'SPACK_PR_MIRRORS_ROOT_URL', r"file:///fake/mirror") - monkeypatch.setattr( - ci, 'SPACK_SHARED_PR_MIRROR_URL', r"file:///fake/mirror_two") - with ev.read('test'): ci_cmd('generate', '--output-file', outputfile) @@ -1715,6 +1760,64 @@ spack: assert(ex not in output) +def test_ci_generate_external_signing_job(tmpdir, mutable_mock_env_path, + install_mockery, + mock_packages, monkeypatch, + ci_base_environment): + """Verify that in external signing mode: 1) each rebuild jobs includes + the location where the binary hash information is written and 2) we + properly generate a final signing job in the pipeline.""" + os.environ.update({ + 'SPACK_PIPELINE_TYPE': 'spack_protected_branch' + }) + filename = str(tmpdir.join('spack.yaml')) + with open(filename, 'w') as f: + f.write("""\ +spack: + specs: + - archive-files + mirrors: + some-mirror: https://my.fake.mirror + gitlab-ci: + temporary-storage-url-prefix: file:///work/temp/mirror + mappings: + - match: + - archive-files + runner-attributes: + tags: + - donotcare + image: donotcare + signing-job-attributes: + tags: + - nonbuildtag + - secretrunner + image: + name: customdockerimage + entrypoint: [] + variables: + IMPORTANT_INFO: avalue + script: + - echo hello +""") + + with tmpdir.as_cwd(): + env_cmd('create', 'test', './spack.yaml') + outputfile = str(tmpdir.join('.gitlab-ci.yml')) + + with ev.read('test'): + ci_cmd('generate', '--output-file', outputfile) + + with open(outputfile) as of: + pipeline_doc = syaml.load(of.read()) + + assert 'sign-pkgs' in pipeline_doc + signing_job = pipeline_doc['sign-pkgs'] + assert 'tags' in signing_job + signing_job_tags = signing_job['tags'] + for expected_tag in ['notary', 'protected', 'aws']: + assert expected_tag in signing_job_tags + + def test_ci_reproduce(tmpdir, mutable_mock_env_path, install_mockery, mock_packages, monkeypatch, last_two_git_commits, ci_base_environment, mock_binary_index): diff --git a/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml b/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml index 321c2b6af6..b0936dfc4e 100644 --- a/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml +++ b/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml @@ -1,4 +1,4 @@ -stages: [ "generate", "build" ] +stages: [ "generate", "build", "publish" ] default: image: { "name": "ghcr.io/spack/e4s-ubuntu-18.04:v2021-10-18", "entrypoint": [""] } @@ -9,16 +9,25 @@ default: .pr: only: + - /^pr[\d]+_.*$/ - /^github\/pr[\d]+_.*$/ variables: - SPACK_PR_BRANCH: ${CI_COMMIT_REF_NAME} + SPACK_BUILDCACHE_DESTINATION: "s3://spack-binaries-prs/${CI_COMMIT_REF_NAME}" SPACK_PIPELINE_TYPE: "spack_pull_request" SPACK_PRUNE_UNTOUCHED: "True" -.develop: +.protected-refs: only: + - /^develop$/ + - /^releases\/v.*/ + - /^v.*/ - /^github\/develop$/ + +.protected: + extends: [ ".protected-refs" ] variables: + SPACK_BUILDCACHE_DESTINATION: "s3://spack-binaries/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}" + SPACK_COPY_BUILDCACHE: "s3://spack-binaries/${CI_COMMIT_REF_NAME}" SPACK_PIPELINE_TYPE: "spack_protected_branch" .generate: @@ -29,12 +38,13 @@ default: - cd share/spack/gitlab/cloud_pipelines/stacks/${SPACK_CI_STACK_NAME} - spack env activate --without-view . - spack ci generate --check-index-only + --buildcache-destination "${SPACK_BUILDCACHE_DESTINATION}" --artifacts-root "${CI_PROJECT_DIR}/jobs_scratch_dir" --output-file "${CI_PROJECT_DIR}/jobs_scratch_dir/cloud-ci-pipeline.yml" artifacts: paths: - "${CI_PROJECT_DIR}/jobs_scratch_dir" - tags: ["spack", "public", "medium", "x86_64"] + tags: ["spack", "aws", "public", "medium", "x86_64"] interruptible: true retry: max: 2 @@ -45,8 +55,8 @@ default: .pr-generate: extends: [ ".pr", ".generate" ] -.develop-generate: - extends: [ ".develop", ".generate" ] +.protected-generate: + extends: [ ".protected", ".generate" ] .build: stage: build @@ -57,12 +67,24 @@ default: AWS_ACCESS_KEY_ID: ${PR_MIRRORS_AWS_ACCESS_KEY_ID} AWS_SECRET_ACCESS_KEY: ${PR_MIRRORS_AWS_SECRET_ACCESS_KEY} -.develop-build: - extends: [ ".develop", ".build" ] +.protected-build: + extends: [ ".protected", ".build" ] variables: AWS_ACCESS_KEY_ID: ${PROTECTED_MIRRORS_AWS_ACCESS_KEY_ID} AWS_SECRET_ACCESS_KEY: ${PROTECTED_MIRRORS_AWS_SECRET_ACCESS_KEY} - SPACK_SIGNING_KEY: ${PACKAGE_SIGNING_KEY} + +protected-publish: + stage: publish + extends: [ ".protected-refs" ] + image: "ghcr.io/spack/python-aws-bash:0.0.1" + tags: ["spack", "public", "medium", "aws", "x86_64"] + variables: + AWS_ACCESS_KEY_ID: ${PROTECTED_MIRRORS_AWS_ACCESS_KEY_ID} + AWS_SECRET_ACCESS_KEY: ${PROTECTED_MIRRORS_AWS_SECRET_ACCESS_KEY} + script: + - . "./share/spack/setup-env.sh" + - spack --version + - spack buildcache update-index --mirror-url "s3://spack-binaries/${CI_COMMIT_REF_NAME}" ######################################## # TEMPLATE FOR ADDING ANOTHER PIPELINE @@ -83,8 +105,8 @@ default: # my-super-cool-stack-pr-generate: # extends: [ ".my-super-cool-stack", ".pr-generate"] # -# my-super-cool-stack-develop-generate: -# extends: [ ".my-super-cool-stack", ".develop-generate"] +# my-super-cool-stack-protected-generate: +# extends: [ ".my-super-cool-stack", ".protected-generate"] # # my-super-cool-stack-pr-build: # extends: [ ".my-super-cool-stack", ".pr-build" ] @@ -94,24 +116,62 @@ default: # job: my-super-cool-stack-pr-generate # strategy: depend # -# my-super-cool-stack-develop-build: -# extends: [ ".my-super-cool-stack", ".develop-build" ] +# my-super-cool-stack-protected-build: +# extends: [ ".my-super-cool-stack", ".protected-build" ] # trigger: # include: # - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml -# job: my-super-cool-stack-develop-generate +# job: my-super-cool-stack-protected-generate # strategy: depend ######################################## -# E4S Mac Stack +# E4S Mac Stack +# +# With no near-future plans to have +# protected aws runners running mac +# builds, it seems best to decouple +# them from the rest of the stacks for +# the time being. This way they can +# still run on UO runners and be signed +# using the previous approach. ######################################## .e4s-mac: variables: SPACK_CI_STACK_NAME: e4s-mac allow_failure: True +.mac-pr: + only: + - /^pr[\d]+_.*$/ + - /^github\/pr[\d]+_.*$/ + variables: + SPACK_BUILDCACHE_DESTINATION: "s3://spack-binaries-prs/${CI_COMMIT_REF_NAME}" + SPACK_PRUNE_UNTOUCHED: "True" + +.mac-protected: + only: + - /^develop$/ + - /^releases\/v.*/ + - /^v.*/ + - /^github\/develop$/ + variables: + SPACK_BUILDCACHE_DESTINATION: "s3://spack-binaries/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}" + +.mac-pr-build: + extends: [ ".mac-pr", ".build" ] + variables: + AWS_ACCESS_KEY_ID: ${PR_MIRRORS_AWS_ACCESS_KEY_ID} + AWS_SECRET_ACCESS_KEY: ${PR_MIRRORS_AWS_SECRET_ACCESS_KEY} + +.mac-protected-build: + extends: [ ".mac-protected", ".build" ] + variables: + AWS_ACCESS_KEY_ID: ${PROTECTED_MIRRORS_AWS_ACCESS_KEY_ID} + AWS_SECRET_ACCESS_KEY: ${PROTECTED_MIRRORS_AWS_SECRET_ACCESS_KEY} + SPACK_SIGNING_KEY: ${PACKAGE_SIGNING_KEY} + e4s-mac-pr-generate: - extends: [".e4s-mac", ".pr"] + extends: [".e4s-mac", ".mac-pr"] stage: generate script: - tmp="$(mktemp -d)"; export SPACK_USER_CONFIG_PATH="$tmp"; export SPACK_USER_CACHE_PATH="$tmp" @@ -135,8 +195,8 @@ e4s-mac-pr-generate: - stuck_or_timeout_failure timeout: 60 minutes -e4s-mac-develop-generate: - extends: [".e4s-mac", ".develop"] +e4s-mac-protected-generate: + extends: [".e4s-mac", ".mac-protected"] stage: generate script: - tmp="$(mktemp -d)"; export SPACK_USER_CONFIG_PATH="$tmp"; export SPACK_USER_CACHE_PATH="$tmp" @@ -161,7 +221,7 @@ e4s-mac-develop-generate: timeout: 60 minutes e4s-mac-pr-build: - extends: [ ".e4s-mac", ".pr-build" ] + extends: [ ".e4s-mac", ".mac-pr-build" ] trigger: include: - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml @@ -171,16 +231,16 @@ e4s-mac-pr-build: - artifacts: True job: e4s-mac-pr-generate -e4s-mac-develop-build: - extends: [ ".e4s-mac", ".develop-build" ] +e4s-mac-protected-build: + extends: [ ".e4s-mac", ".mac-protected-build" ] trigger: include: - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml - job: e4s-mac-develop-generate + job: e4s-mac-protected-generate strategy: depend needs: - artifacts: True - job: e4s-mac-develop-generate + job: e4s-mac-protected-generate ######################################## # E4S pipeline @@ -192,8 +252,8 @@ e4s-mac-develop-build: e4s-pr-generate: extends: [ ".e4s", ".pr-generate"] -e4s-develop-generate: - extends: [ ".e4s", ".develop-generate"] +e4s-protected-generate: + extends: [ ".e4s", ".protected-generate"] e4s-pr-build: extends: [ ".e4s", ".pr-build" ] @@ -206,16 +266,16 @@ e4s-pr-build: - artifacts: True job: e4s-pr-generate -e4s-develop-build: - extends: [ ".e4s", ".develop-build" ] +e4s-protected-build: + extends: [ ".e4s", ".protected-build" ] trigger: include: - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml - job: e4s-develop-generate + job: e4s-protected-generate strategy: depend needs: - artifacts: True - job: e4s-develop-generate + job: e4s-protected-generate ######################################## # E4S on Power @@ -231,8 +291,8 @@ e4s-develop-build: # e4s-on-power-pr-generate: # extends: [ ".e4s-on-power", ".pr-generate", ".power-e4s-generate-tags-and-image"] -# e4s-on-power-develop-generate: -# extends: [ ".e4s-on-power", ".develop-generate", ".power-e4s-generate-tags-and-image"] +# e4s-on-power-protected-generate: +# extends: [ ".e4s-on-power", ".protected-generate", ".power-e4s-generate-tags-and-image"] # e4s-on-power-pr-build: # extends: [ ".e4s-on-power", ".pr-build" ] @@ -245,16 +305,16 @@ e4s-develop-build: # - artifacts: True # job: e4s-on-power-pr-generate -# e4s-on-power-develop-build: -# extends: [ ".e4s-on-power", ".develop-build" ] +# e4s-on-power-protected-build: +# extends: [ ".e4s-on-power", ".protected-build" ] # trigger: # include: # - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml -# job: e4s-on-power-develop-generate +# job: e4s-on-power-protected-generate # strategy: depend # needs: # - artifacts: True -# job: e4s-on-power-develop-generate +# job: e4s-on-power-protected-generate ######################################### # Build tests for different build-systems @@ -266,8 +326,8 @@ e4s-develop-build: build_systems-pr-generate: extends: [ ".build_systems", ".pr-generate"] -build_systems-develop-generate: - extends: [ ".build_systems", ".develop-generate"] +build_systems-protected-generate: + extends: [ ".build_systems", ".protected-generate"] build_systems-pr-build: extends: [ ".build_systems", ".pr-build" ] @@ -280,16 +340,16 @@ build_systems-pr-build: - artifacts: True job: build_systems-pr-generate -build_systems-develop-build: - extends: [ ".build_systems", ".develop-build" ] +build_systems-protected-build: + extends: [ ".build_systems", ".protected-build" ] trigger: include: - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml - job: build_systems-develop-generate + job: build_systems-protected-generate strategy: depend needs: - artifacts: True - job: build_systems-develop-generate + job: build_systems-protected-generate ######################################### # RADIUSS @@ -313,20 +373,20 @@ radiuss-pr-build: - artifacts: True job: radiuss-pr-generate -# --------- Develop --------- -radiuss-develop-generate: - extends: [ ".radiuss", ".develop-generate" ] +# --------- Protected --------- +radiuss-protected-generate: + extends: [ ".radiuss", ".protected-generate" ] -radiuss-develop-build: - extends: [ ".radiuss", ".develop-build" ] +radiuss-protected-build: + extends: [ ".radiuss", ".protected-build" ] trigger: include: - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml - job: radiuss-develop-generate + job: radiuss-protected-generate strategy: depend needs: - artifacts: True - job: radiuss-develop-generate + job: radiuss-protected-generate ######################################## # ECP Data & Vis SDK @@ -338,8 +398,8 @@ radiuss-develop-build: data-vis-sdk-pr-generate: extends: [ ".data-vis-sdk", ".pr-generate"] -data-vis-sdk-develop-generate: - extends: [ ".data-vis-sdk", ".develop-generate"] +data-vis-sdk-protected-generate: + extends: [ ".data-vis-sdk", ".protected-generate"] data-vis-sdk-pr-build: extends: [ ".data-vis-sdk", ".pr-build" ] @@ -352,16 +412,16 @@ data-vis-sdk-pr-build: - artifacts: True job: data-vis-sdk-pr-generate -data-vis-sdk-develop-build: - extends: [ ".data-vis-sdk", ".develop-build" ] +data-vis-sdk-protected-build: + extends: [ ".data-vis-sdk", ".protected-build" ] trigger: include: - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml - job: data-vis-sdk-develop-generate + job: data-vis-sdk-protected-generate strategy: depend needs: - artifacts: True - job: data-vis-sdk-develop-generate + job: data-vis-sdk-protected-generate ######################################## # Spack Tutorial @@ -373,8 +433,8 @@ data-vis-sdk-develop-build: tutorial-pr-generate: extends: [ ".tutorial", ".pr-generate"] -tutorial-develop-generate: - extends: [ ".tutorial", ".develop-generate"] +tutorial-protected-generate: + extends: [ ".tutorial", ".protected-generate"] tutorial-pr-build: extends: [ ".tutorial", ".pr-build" ] @@ -387,13 +447,13 @@ tutorial-pr-build: - artifacts: True job: tutorial-pr-generate -tutorial-develop-build: - extends: [ ".tutorial", ".develop-build" ] +tutorial-protected-build: + extends: [ ".tutorial", ".protected-build" ] trigger: include: - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml - job: tutorial-develop-generate + job: tutorial-protected-generate strategy: depend needs: - artifacts: True - job: tutorial-develop-generate + job: tutorial-protected-generate diff --git a/share/spack/gitlab/cloud_pipelines/stacks/build_systems/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/build_systems/spack.yaml index 5a742718d4..a633d10099 100644 --- a/share/spack/gitlab/cloud_pipelines/stacks/build_systems/spack.yaml +++ b/share/spack/gitlab/cloud_pipelines/stacks/build_systems/spack.yaml @@ -29,7 +29,7 @@ spack: - - $default_specs - - $arch - mirrors: { "mirror": "s3://spack-binaries/build_systems" } + mirrors: { "mirror": "s3://spack-binaries/develop/build_systems" } gitlab-ci: script: @@ -38,6 +38,8 @@ spack: - cd ${SPACK_CONCRETE_ENV_DIR} - spack env activate --without-view . - spack config add "config:install_tree:projections:${SPACK_JOB_SPEC_PKG_NAME}:'morepadding/{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'" + - if [[ -r /mnt/key/intermediate_ci_signing_key.gpg ]]; then spack gpg trust /mnt/key/intermediate_ci_signing_key.gpg; fi + - if [[ -r /mnt/key/spack_public_key.gpg ]]; then spack gpg trust /mnt/key/spack_public_key.gpg; fi - spack -d ci rebuild image: @@ -48,7 +50,7 @@ spack: - match: - cmake runner-attributes: - tags: [ "spack", "public", "large", "x86_64"] + tags: [ "spack", "large", "x86_64"] variables: CI_JOB_SIZE: large KUBERNETES_CPU_REQUEST: 8000m @@ -61,7 +63,7 @@ spack: - openjpeg - sqlite runner-attributes: - tags: [ "spack", "public", "medium", "x86_64" ] + tags: [ "spack", "medium", "x86_64" ] variables: CI_JOB_SIZE: "medium" KUBERNETES_CPU_REQUEST: "2000m" @@ -85,7 +87,7 @@ spack: - xz - zlib runner-attributes: - tags: [ "spack", "public", "medium", "x86_64" ] + tags: [ "spack", "medium", "x86_64" ] variables: CI_JOB_SIZE: "small" KUBERNETES_CPU_REQUEST: "500m" @@ -94,18 +96,27 @@ spack: - match: - 'os=ubuntu18.04' runner-attributes: - tags: ["spack", "public", "x86_64"] + tags: ["spack", "x86_64"] variables: CI_JOB_SIZE: "default" + broken-specs-url: "s3://spack-binaries/broken-specs" - broken-specs-url: "s3://spack-binaries-develop/broken-specs" service-job-attributes: before_script: - . "./share/spack/setup-env.sh" - spack --version image: { "name": "ghcr.io/spack/e4s-ubuntu-18.04:v2021-10-18", "entrypoint": [""] } tags: ["spack", "public", "x86_64"] + + signing-job-attributes: + image: { "name": "ghcr.io/spack/notary:latest", "entrypoint": [""] } + tags: ["spack", "aws"] + script: + - aws s3 sync --exclude "*" --include "*spec.json*" ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache /tmp + - /sign.sh + - aws s3 sync --exclude "*" --include "*spec.json.sig*" /tmp ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache + cdash: build-group: Build tests for different build systems url: https://cdash.spack.io diff --git a/share/spack/gitlab/cloud_pipelines/stacks/data-vis-sdk/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/data-vis-sdk/spack.yaml index 85095e5619..ae9025e366 100644 --- a/share/spack/gitlab/cloud_pipelines/stacks/data-vis-sdk/spack.yaml +++ b/share/spack/gitlab/cloud_pipelines/stacks/data-vis-sdk/spack.yaml @@ -42,7 +42,7 @@ spack: +zfp +visit - mirrors: { "mirror": "s3://spack-binaries/data-vis-sdk" } + mirrors: { "mirror": "s3://spack-binaries/develop/data-vis-sdk" } gitlab-ci: image: { "name": "ghcr.io/spack/e4s-ubuntu-18.04:v2021-10-18", "entrypoint": [""] } @@ -52,13 +52,15 @@ spack: - cd ${SPACK_CONCRETE_ENV_DIR} - spack env activate --without-view . - spack config add "config:install_tree:projections:${SPACK_JOB_SPEC_PKG_NAME}:'morepadding/{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'" + - if [[ -r /mnt/key/intermediate_ci_signing_key.gpg ]]; then spack gpg trust /mnt/key/intermediate_ci_signing_key.gpg; fi + - if [[ -r /mnt/key/spack_public_key.gpg ]]; then spack gpg trust /mnt/key/spack_public_key.gpg; fi - spack -d ci rebuild mappings: - match: - llvm - qt runner-attributes: - tags: [ "spack", "public", "huge", "x86_64" ] + tags: [ "spack", "huge", "x86_64" ] variables: CI_JOB_SIZE: huge KUBERNETES_CPU_REQUEST: 11000m @@ -72,7 +74,7 @@ spack: - visit - vtk-m runner-attributes: - tags: [ "spack", "public", "large", "x86_64" ] + tags: [ "spack", "large", "x86_64" ] variables: CI_JOB_SIZE: large KUBERNETES_CPU_REQUEST: 8000m @@ -98,7 +100,7 @@ spack: - raja - vtk-h runner-attributes: - tags: [ "spack", "public", "medium", "x86_64" ] + tags: [ "spack", "medium", "x86_64" ] variables: CI_JOB_SIZE: "medium" KUBERNETES_CPU_REQUEST: "2000m" @@ -133,7 +135,7 @@ spack: - util-linux-uuid runner-attributes: - tags: [ "spack", "public", "small", "x86_64" ] + tags: [ "spack", "small", "x86_64" ] variables: CI_JOB_SIZE: "small" KUBERNETES_CPU_REQUEST: "500m" @@ -141,11 +143,12 @@ spack: - match: ['@:'] runner-attributes: - tags: ["spack", "public", "x86_64"] + tags: ["spack", "x86_64"] variables: CI_JOB_SIZE: "default" - broken-specs-url: "s3://spack-binaries-develop/broken-specs" + broken-specs-url: "s3://spack-binaries/broken-specs" + service-job-attributes: image: { "name": "ghcr.io/spack/e4s-ubuntu-18.04:v2021-10-18", "entrypoint": [""] } before_script: @@ -153,6 +156,14 @@ spack: - spack --version tags: ["spack", "public", "medium", "x86_64"] + signing-job-attributes: + image: { "name": "ghcr.io/spack/notary:latest", "entrypoint": [""] } + tags: ["spack", "aws"] + script: + - aws s3 sync --exclude "*" --include "*spec.json*" ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache /tmp + - /sign.sh + - aws s3 sync --exclude "*" --include "*spec.json.sig*" /tmp ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache + cdash: build-group: Data and Vis SDK url: https://cdash.spack.io diff --git a/share/spack/gitlab/cloud_pipelines/stacks/e4s-mac/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/e4s-mac/spack.yaml index dc86cb7dbe..6cfaae4a15 100644 --- a/share/spack/gitlab/cloud_pipelines/stacks/e4s-mac/spack.yaml +++ b/share/spack/gitlab/cloud_pipelines/stacks/e4s-mac/spack.yaml @@ -32,7 +32,7 @@ spack: - - $easy_specs - - $arch - mirrors: { "mirror": "s3://spack-binaries/e4s-mac" } + mirrors: { "mirror": "s3://spack-binaries/develop/e4s-mac" } gitlab-ci: @@ -51,7 +51,9 @@ spack: runner-attributes: tags: - omicron - broken-specs-url: "s3://spack-binaries-develop/broken-specs" + + broken-specs-url: "s3://spack-binaries/broken-specs" + service-job-attributes: before_script: - . "./share/spack/setup-env.sh" diff --git a/share/spack/gitlab/cloud_pipelines/stacks/e4s/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/e4s/spack.yaml index ce31a0156e..c4eaaddbae 100644 --- a/share/spack/gitlab/cloud_pipelines/stacks/e4s/spack.yaml +++ b/share/spack/gitlab/cloud_pipelines/stacks/e4s/spack.yaml @@ -222,7 +222,7 @@ spack: - - $cuda_specs - - $arch - mirrors: { "mirror": "s3://spack-binaries/e4s" } + mirrors: { "mirror": "s3://spack-binaries/develop/e4s" } gitlab-ci: @@ -233,6 +233,8 @@ spack: - spack env activate --without-view . - spack config add "config:install_tree:projections:${SPACK_JOB_SPEC_PKG_NAME}:'morepadding/{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'" - mkdir -p ${SPACK_ARTIFACTS_ROOT}/user_data + - if [[ -r /mnt/key/intermediate_ci_signing_key.gpg ]]; then spack gpg trust /mnt/key/intermediate_ci_signing_key.gpg; fi + - if [[ -r /mnt/key/spack_public_key.gpg ]]; then spack gpg trust /mnt/key/spack_public_key.gpg; fi - spack -d ci rebuild > >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_out.txt) 2> >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_err.txt >&2) image: { "name": "ghcr.io/spack/e4s-ubuntu-18.04:v2021-10-18", "entrypoint": [""] } @@ -240,7 +242,7 @@ spack: - match: - llvm runner-attributes: - tags: [ "spack", "public", "huge", "x86_64" ] + tags: [ "spack", "huge", "x86_64" ] variables: CI_JOB_SIZE: huge KUBERNETES_CPU_REQUEST: 11000m @@ -265,7 +267,7 @@ spack: - vtk-m - warpx runner-attributes: - tags: [ "spack", "public", "large", "x86_64" ] + tags: [ "spack", "large", "x86_64" ] variables: CI_JOB_SIZE: large KUBERNETES_CPU_REQUEST: 8000m @@ -333,7 +335,7 @@ spack: - vtk-h - zfp runner-attributes: - tags: [ "spack", "public", "medium", "x86_64" ] + tags: [ "spack", "medium", "x86_64" ] variables: CI_JOB_SIZE: "medium" KUBERNETES_CPU_REQUEST: "2000m" @@ -394,7 +396,7 @@ spack: - zlib - zstd runner-attributes: - tags: [ "spack", "public", "small", "x86_64" ] + tags: [ "spack", "small", "x86_64" ] variables: CI_JOB_SIZE: "small" KUBERNETES_CPU_REQUEST: "500m" @@ -402,11 +404,12 @@ spack: - match: ['os=ubuntu18.04'] runner-attributes: - tags: ["spack", "public", "x86_64"] + tags: ["spack", "x86_64"] variables: CI_JOB_SIZE: "default" - broken-specs-url: "s3://spack-binaries-develop/broken-specs" + broken-specs-url: "s3://spack-binaries/broken-specs" + service-job-attributes: before_script: - . "./share/spack/setup-env.sh" @@ -414,6 +417,14 @@ spack: image: { "name": "ghcr.io/spack/e4s-ubuntu-18.04:v2021-10-18", "entrypoint": [""] } tags: ["spack", "public", "x86_64"] + signing-job-attributes: + image: { "name": "ghcr.io/spack/notary:latest", "entrypoint": [""] } + tags: ["spack", "aws"] + script: + - aws s3 sync --exclude "*" --include "*spec.json*" ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache /tmp + - /sign.sh + - aws s3 sync --exclude "*" --include "*spec.json.sig*" /tmp ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache + cdash: build-group: New PR testing workflow url: https://cdash.spack.io diff --git a/share/spack/gitlab/cloud_pipelines/stacks/radiuss/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/radiuss/spack.yaml index 846b45f707..ebb15eb7ca 100644 --- a/share/spack/gitlab/cloud_pipelines/stacks/radiuss/spack.yaml +++ b/share/spack/gitlab/cloud_pipelines/stacks/radiuss/spack.yaml @@ -54,7 +54,7 @@ spack: - zfp mirrors: - mirror: "s3://spack-binaries/radiuss" + mirror: "s3://spack-binaries/develop/radiuss" specs: - matrix: @@ -69,6 +69,8 @@ spack: - cd ${SPACK_CONCRETE_ENV_DIR} - spack env activate --without-view . - spack config add "config:install_tree:projections:${SPACK_JOB_SPEC_PKG_NAME}:'morepadding/{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'" + - if [[ -r /mnt/key/intermediate_ci_signing_key.gpg ]]; then spack gpg trust /mnt/key/intermediate_ci_signing_key.gpg; fi + - if [[ -r /mnt/key/spack_public_key.gpg ]]; then spack gpg trust /mnt/key/spack_public_key.gpg; fi - spack -d ci rebuild mappings: - match: @@ -76,7 +78,7 @@ spack: - openblas - rust runner-attributes: - tags: ["spack", "public", "large", "x86_64"] + tags: ["spack", "large", "x86_64"] variables: CI_JOB_SIZE: large KUBERNETES_CPU_REQUEST: 8000m @@ -96,7 +98,7 @@ spack: - vtk-h - vtk-m runner-attributes: - tags: ["spack", "public", "medium", "x86_64"] + tags: ["spack", "medium", "x86_64"] variables: CI_JOB_SIZE: "medium" KUBERNETES_CPU_REQUEST: "2000m" @@ -150,7 +152,7 @@ spack: - zfp - zlib runner-attributes: - tags: ["spack", "public", "small", "x86_64"] + tags: ["spack", "small", "x86_64"] variables: CI_JOB_SIZE: "small" KUBERNETES_CPU_REQUEST: "500m" @@ -158,10 +160,12 @@ spack: - match: ['os=ubuntu18.04'] runner-attributes: - tags: ["spack", "public", "x86_64"] + tags: ["spack", "x86_64"] variables: CI_JOB_SIZE: "default" + broken-specs-url: "s3://spack-binaries/broken-specs" + service-job-attributes: before_script: - . "./share/spack/setup-env.sh" @@ -169,6 +173,14 @@ spack: image: { "name": "ghcr.io/spack/e4s-ubuntu-18.04:v2021-10-18", "entrypoint": [""] } tags: ["spack", "public", "x86_64"] + signing-job-attributes: + image: { "name": "ghcr.io/spack/notary:latest", "entrypoint": [""] } + tags: ["spack", "aws"] + script: + - aws s3 sync --exclude "*" --include "*spec.json*" ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache /tmp + - /sign.sh + - aws s3 sync --exclude "*" --include "*spec.json.sig*" /tmp ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache + cdash: build-group: RADIUSS url: https://cdash.spack.io diff --git a/share/spack/gitlab/cloud_pipelines/stacks/tutorial/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/tutorial/spack.yaml index 14bd78b3b6..504b4209d2 100644 --- a/share/spack/gitlab/cloud_pipelines/stacks/tutorial/spack.yaml +++ b/share/spack/gitlab/cloud_pipelines/stacks/tutorial/spack.yaml @@ -59,7 +59,7 @@ spack: - $gcc_spack_built_packages mirrors: - mirror: 's3://spack-binaries/tutorial' + mirror: 's3://spack-binaries/develop/tutorial' gitlab-ci: script: @@ -69,6 +69,8 @@ spack: - cd ${SPACK_CONCRETE_ENV_DIR} - spack env activate --without-view . - spack config add "config:install_tree:projections:${SPACK_JOB_SPEC_PKG_NAME}:'morepadding/{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'" + - if [[ -r /mnt/key/intermediate_ci_signing_key.gpg ]]; then spack gpg trust /mnt/key/intermediate_ci_signing_key.gpg; fi + - if [[ -r /mnt/key/spack_public_key.gpg ]]; then spack gpg trust /mnt/key/spack_public_key.gpg; fi - spack -d ci rebuild image: { "name": "ghcr.io/spack/tutorial-ubuntu-18.04:v2021-11-02", "entrypoint": [""] } @@ -81,7 +83,7 @@ spack: - netlib-lapack - trilinos runner-attributes: - tags: ["spack", "public", "large", "x86_64"] + tags: ["spack", "large", "x86_64"] variables: CI_JOB_SIZE: large KUBERNETES_CPU_REQUEST: 8000m @@ -99,7 +101,7 @@ spack: - py-scipy - slurm runner-attributes: - tags: ["spack", "public", "medium", "x86_64"] + tags: ["spack", "medium", "x86_64"] variables: CI_JOB_SIZE: "medium" KUBERNETES_CPU_REQUEST: "2000m" @@ -129,7 +131,7 @@ spack: - tar - util-linux-uuid runner-attributes: - tags: ["spack", "public", "small", "x86_64"] + tags: ["spack", "small", "x86_64"] variables: CI_JOB_SIZE: "small" KUBERNETES_CPU_REQUEST: "500m" @@ -137,11 +139,12 @@ spack: - match: ['@:'] runner-attributes: - tags: ["spack", "public", "x86_64"] + tags: ["spack", "x86_64"] variables: CI_JOB_SIZE: default - broken-specs-url: "s3://spack-binaries-develop/broken-specs" + broken-specs-url: "s3://spack-binaries/broken-specs" + service-job-attributes: image: { "name": "ghcr.io/spack/tutorial-ubuntu-18.04:v2021-11-02", "entrypoint": [""] } before_script: @@ -149,6 +152,14 @@ spack: - spack --version tags: ["spack", "public", "x86_64"] + signing-job-attributes: + image: { "name": "ghcr.io/spack/notary:latest", "entrypoint": [""] } + tags: ["spack", "aws"] + script: + - aws s3 sync --exclude "*" --include "*spec.json*" ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache /tmp + - /sign.sh + - aws s3 sync --exclude "*" --include "*spec.json.sig*" /tmp ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache + cdash: build-group: Spack Tutorial url: https://cdash.spack.io diff --git a/share/spack/spack-completion.bash b/share/spack/spack-completion.bash index 5e4498f395..19508dd17c 100755 --- a/share/spack/spack-completion.bash +++ b/share/spack/spack-completion.bash @@ -626,7 +626,7 @@ _spack_ci() { } _spack_ci_generate() { - SPACK_COMPREPLY="-h --help --output-file --copy-to --optimize --dependencies --prune-dag --no-prune-dag --check-index-only --artifacts-root" + SPACK_COMPREPLY="-h --help --output-file --copy-to --optimize --dependencies --buildcache-destination --prune-dag --no-prune-dag --check-index-only --artifacts-root" } _spack_ci_rebuild_index() { |