diff options
| -rw-r--r-- | lib/spack/spack/binary_distribution.py | 58 | ||||
| -rw-r--r-- | lib/spack/spack/ci.py | 30 | ||||
| -rw-r--r-- | lib/spack/spack/cmd/ci.py | 204 | ||||
| -rw-r--r-- | lib/spack/spack/test/bindist.py | 49 | ||||
| -rw-r--r-- | lib/spack/spack/test/cmd/ci.py | 4 | ||||
| -rw-r--r-- | share/spack/gitlab/cloud_e4s_pipelines.yml | 24 | ||||
| -rw-r--r-- | share/spack/gitlab/pr_pipeline.yml | 18 |
7 files changed, 261 insertions, 126 deletions
diff --git a/lib/spack/spack/binary_distribution.py b/lib/spack/spack/binary_distribution.py index ed1ce02aac..d643bde7b3 100644 --- a/lib/spack/spack/binary_distribution.py +++ b/lib/spack/spack/binary_distribution.py @@ -698,10 +698,23 @@ def generate_package_index(cache_prefix): enable_transaction_locking=False, record_fields=['spec', 'ref_count']) - file_list = ( - entry - for entry in web_util.list_url(cache_prefix) - if entry.endswith('.yaml')) + try: + file_list = ( + entry + for entry in web_util.list_url(cache_prefix) + if entry.endswith('.yaml')) + except KeyError as inst: + msg = 'No packages at {0}: {1}'.format(cache_prefix, inst) + tty.warn(msg) + return + except Exception as err: + # If we got some kind of S3 (access denied or other connection + # error), the first non boto-specific class in the exception + # hierarchy is Exception. Just print a warning and return + msg = 'Encountered problem listing packages at {0}: {1}'.format( + cache_prefix, err) + tty.warn(msg) + return tty.debug('Retrieving spec.yaml files from {0} to build index'.format( cache_prefix)) @@ -763,10 +776,23 @@ def generate_key_index(key_prefix, tmpdir=None): url_util.format(key_prefix), 'to build key index'))) - fingerprints = ( - entry[:-4] - for entry in web_util.list_url(key_prefix, recursive=False) - if entry.endswith('.pub')) + try: + fingerprints = ( + entry[:-4] + for entry in web_util.list_url(key_prefix, recursive=False) + if entry.endswith('.pub')) + except KeyError as inst: + msg = 'No keys at {0}: {1}'.format(key_prefix, inst) + tty.warn(msg) + return + except Exception as err: + # If we got some kind of S3 (access denied or other connection + # error), the first non boto-specific class in the exception + # hierarchy is Exception. Just print a warning and return + msg = 'Encountered problem listing keys at {0}: {1}'.format( + key_prefix, err) + tty.warn(msg) + return remove_tmpdir = False @@ -1284,15 +1310,16 @@ def extract_tarball(spec, filename, allow_root=False, unsigned=False, os.remove(filename) -def try_direct_fetch(spec, force=False, full_hash_match=False): +def try_direct_fetch(spec, force=False, full_hash_match=False, mirrors=None): """ Try to find the spec directly on the configured mirrors """ specfile_name = tarball_name(spec, '.spec.yaml') lenient = not full_hash_match found_specs = [] + spec_full_hash = spec.full_hash() - for mirror in spack.mirror.MirrorCollection().values(): + for mirror in spack.mirror.MirrorCollection(mirrors=mirrors).values(): buildcache_fetch_url = url_util.join( mirror.fetch_url, _build_cache_relative_path, specfile_name) @@ -1312,7 +1339,7 @@ def try_direct_fetch(spec, force=False, full_hash_match=False): # Do not recompute the full hash for the fetched spec, instead just # read the property. - if lenient or fetched_spec._full_hash == spec.full_hash(): + if lenient or fetched_spec._full_hash == spec_full_hash: found_specs.append({ 'mirror_url': mirror.fetch_url, 'spec': fetched_spec, @@ -1321,7 +1348,8 @@ def try_direct_fetch(spec, force=False, full_hash_match=False): return found_specs -def get_mirrors_for_spec(spec=None, force=False, full_hash_match=False): +def get_mirrors_for_spec(spec=None, force=False, full_hash_match=False, + mirrors_to_check=None): """ Check if concrete spec exists on mirrors and return a list indicating the mirrors on which it can be found @@ -1329,7 +1357,7 @@ def get_mirrors_for_spec(spec=None, force=False, full_hash_match=False): if spec is None: return [] - if not spack.mirror.MirrorCollection(): + if not spack.mirror.MirrorCollection(mirrors=mirrors_to_check): tty.debug("No Spack mirrors are currently configured") return {} @@ -1354,7 +1382,9 @@ def get_mirrors_for_spec(spec=None, force=False, full_hash_match=False): if not results: results = try_direct_fetch(spec, force=force, - full_hash_match=full_hash_match) + full_hash_match=full_hash_match, + mirrors=mirrors_to_check) + if results: binary_index.update_spec(spec, results) diff --git a/lib/spack/spack/ci.py b/lib/spack/spack/ci.py index c9054052e6..a3a2248238 100644 --- a/lib/spack/spack/ci.py +++ b/lib/spack/spack/ci.py @@ -33,12 +33,16 @@ import spack.repo from spack.spec import Spec import spack.util.spack_yaml as syaml import spack.util.web as web_util +import spack.util.gpg as gpg_util +import spack.util.url as url_util JOB_RETRY_CONDITIONS = [ 'always', ] +SPACK_PR_MIRRORS_ROOT_URL = 's3://spack-pr-mirrors' + spack_gpg = spack.main.SpackCommand('gpg') spack_compiler = spack.main.SpackCommand('compiler') @@ -529,6 +533,12 @@ def generate_gitlab_ci_yaml(env, print_summary, output_file, os.environ.get('SPACK_IS_PR_PIPELINE', '').lower() == 'true' ) + spack_pr_branch = os.environ.get('SPACK_PR_BRANCH', None) + pr_mirror_url = None + if spack_pr_branch: + pr_mirror_url = url_util.join(SPACK_PR_MIRRORS_ROOT_URL, + spack_pr_branch) + ci_mirrors = yaml_root['mirrors'] mirror_urls = [url for url in ci_mirrors.values()] @@ -858,6 +868,9 @@ def generate_gitlab_ci_yaml(env, print_summary, output_file, 'SPACK_CHECKOUT_VERSION': version_to_clone, } + if pr_mirror_url: + output_object['variables']['SPACK_PR_MIRROR_URL'] = pr_mirror_url + sorted_output = {} for output_key, output_value in sorted(output_object.items()): sorted_output[output_key] = output_value @@ -920,6 +933,14 @@ def import_signing_key(base64_signing_key): tty.debug(signing_keys_output) +def can_sign_binaries(): + return len(gpg_util.signing_keys()) == 1 + + +def can_verify_binaries(): + return len(gpg_util.public_keys()) >= 1 + + def configure_compilers(compiler_action, scope=None): if compiler_action == 'INSTALL_MISSING': tty.debug('Make sure bootstrapped compiler will be installed') @@ -1095,12 +1116,15 @@ def read_cdashid_from_mirror(spec, mirror_url): return int(contents) -def push_mirror_contents(env, spec, yaml_path, mirror_url, build_id): +def push_mirror_contents(env, spec, yaml_path, mirror_url, build_id, + sign_binaries): if mirror_url: - tty.debug('Creating buildcache') + unsigned = not sign_binaries + tty.debug('Creating buildcache ({0})'.format( + 'unsigned' if unsigned else 'signed')) buildcache._createtarball(env, spec_yaml=yaml_path, add_deps=False, output_location=mirror_url, force=True, - allow_root=True) + allow_root=True, unsigned=unsigned) if build_id: tty.debug('Writing cdashid ({0}) to remote mirror: {1}'.format( build_id, mirror_url)) diff --git a/lib/spack/spack/cmd/ci.py b/lib/spack/spack/cmd/ci.py index e0b9b6e0fa..3dd97e53b0 100644 --- a/lib/spack/spack/cmd/ci.py +++ b/lib/spack/spack/cmd/ci.py @@ -138,6 +138,7 @@ def ci_rebuild(args): cdash_build_name = get_env_var('SPACK_CDASH_BUILD_NAME') related_builds = get_env_var('SPACK_RELATED_BUILDS_CDASH') pr_env_var = get_env_var('SPACK_IS_PR_PIPELINE') + pr_mirror_url = get_env_var('SPACK_PR_MIRROR_URL') gitlab_ci = None if 'gitlab-ci' in yaml_root: @@ -180,8 +181,6 @@ def ci_rebuild(args): tty.debug('job_spec_pkg_name = {0}'.format(job_spec_pkg_name)) tty.debug('compiler_action = {0}'.format(compiler_action)) - spack_cmd = exe.which('spack') - cdash_report_dir = os.path.join(ci_artifact_dir, 'cdash_report') temp_dir = os.path.join(ci_artifact_dir, 'jobs_scratch_dir') job_log_dir = os.path.join(temp_dir, 'logs') @@ -235,20 +234,17 @@ def ci_rebuild(args): for next_entry in directory_list: tty.debug(' {0}'.format(next_entry)) - # Make a copy of the environment file, so we can overwrite the changed - # version in between the two invocations of "spack install" - env_src_path = env.manifest_path - env_dirname = os.path.dirname(env_src_path) - env_filename = os.path.basename(env_src_path) - env_copyname = '{0}_BACKUP'.format(env_filename) - env_dst_path = os.path.join(env_dirname, env_copyname) - shutil.copyfile(env_src_path, env_dst_path) - tty.debug('job concrete spec path: {0}'.format(job_spec_yaml_path)) if signing_key: spack_ci.import_signing_key(signing_key) + can_sign = spack_ci.can_sign_binaries() + sign_binaries = can_sign and spack_is_pr_pipeline is False + + can_verify = spack_ci.can_verify_binaries() + verify_binaries = can_verify and spack_is_pr_pipeline is False + spack_ci.configure_compilers(compiler_action) spec_map = spack_ci.get_concrete_specs( @@ -273,27 +269,76 @@ def ci_rebuild(args): with open(root_spec_yaml_path, 'w') as fd: fd.write(spec_map['root'].to_yaml(hash=ht.build_hash)) - if bindist.needs_rebuild(job_spec, remote_mirror_url, True): - # Binary on remote mirror is not up to date, we need to rebuild - # it. - # - # FIXME: ensure mirror precedence causes this local mirror to - # be chosen ahead of the remote one when installing deps + # TODO: Refactor the spack install command so it's easier to use from + # python modules. Currently we use "exe.which('spack')" to make it + # easier to install packages from here, but it introduces some + # problems, e.g. if we want the spack command to have access to the + # mirrors we're configuring, then we have to use the "spack" command + # to add the mirrors too, which in turn means that any code here *not* + # using the spack command does *not* have access to the mirrors. + spack_cmd = exe.which('spack') + mirrors_to_check = { + 'ci_remote_mirror': remote_mirror_url, + } + + def add_mirror(mirror_name, mirror_url): + m_args = ['mirror', 'add', mirror_name, mirror_url] + tty.debug('Adding mirror: spack {0}'.format(m_args)) + mirror_add_output = spack_cmd(*m_args) + # Workaround: Adding the mirrors above, using "spack_cmd" makes + # sure they're available later when we use "spack_cmd" to install + # the package. But then we also need to add them to this dict + # below, so they're available in this process (we end up having to + # pass them to "bindist.get_mirrors_for_spec()") + mirrors_to_check[mirror_name] = mirror_url + tty.debug('spack mirror add output: {0}'.format(mirror_add_output)) + + # Configure mirrors + if pr_mirror_url: + add_mirror('ci_pr_mirror', pr_mirror_url) + + if enable_artifacts_mirror: + add_mirror('ci_artifact_mirror', artifact_mirror_url) + + tty.debug('listing spack mirrors:') + spack_cmd('mirror', 'list') + spack_cmd('config', 'blame', 'mirrors') + + # Checks all mirrors for a built spec with a matching full hash + matches = bindist.get_mirrors_for_spec( + job_spec, force=False, full_hash_match=True, + mirrors_to_check=mirrors_to_check) + + if matches: + # Got at full hash match on at least one configured mirror. All + # matches represent the fully up-to-date spec, so should all be + # equivalent. If artifacts mirror is enabled, we just pick one + # of the matches and download the buildcache files from there to + # the artifacts, so they're available to be used by dependent + # jobs in subsequent stages. + tty.debug('No need to rebuild {0}'.format(job_spec_pkg_name)) if enable_artifacts_mirror: - mirror_add_output = spack_cmd( - 'mirror', 'add', 'local_mirror', artifact_mirror_url) - tty.debug('spack mirror add:') - tty.debug(mirror_add_output) + matching_mirror = matches[0]['mirror_url'] + tty.debug('Getting {0} buildcache from {1}'.format( + job_spec_pkg_name, matching_mirror)) + tty.debug('Downloading to {0}'.format(build_cache_dir)) + buildcache.download_buildcache_files( + job_spec, build_cache_dir, True, matching_mirror) + else: + # No full hash match anywhere means we need to rebuild spec - mirror_list_output = spack_cmd('mirror', 'list') - tty.debug('listing spack mirrors:') - tty.debug(mirror_list_output) + # Build up common install arguments + install_args = [ + '-d', '-v', '-k', 'install', + '--keep-stage', + '--require-full-hash-match', + ] - # 2) build up install arguments - install_args = ['-d', '-v', '-k', 'install', '--keep-stage'] + if not verify_binaries: + install_args.append('--no-check-signature') - # 3) create/register a new build on CDash (if enabled) - cdash_args = [] + # Add arguments to create + register a new build on CDash (if + # enabled) if enable_cdash: tty.debug('Registering build with CDash') (cdash_build_id, @@ -304,82 +349,63 @@ def ci_rebuild(args): cdash_upload_url = '{0}/submit.php?project={1}'.format( cdash_base_url, cdash_project_enc) - cdash_args = [ + install_args.extend([ '--cdash-upload-url', cdash_upload_url, '--cdash-build', cdash_build_name, '--cdash-site', cdash_site, '--cdash-buildstamp', cdash_build_stamp, - ] + ]) - spec_cli_arg = [job_spec_yaml_path] + install_args.append(job_spec_yaml_path) - tty.debug('Installing package') + tty.debug('Installing {0} from source'.format(job_spec.name)) try: - # Two-pass install is intended to avoid spack trying to - # install from buildcache even though the locally computed - # full hash is different than the one stored in the spec.yaml - # file on the remote mirror. - first_pass_args = install_args + [ - '--cache-only', - '--only', - 'dependencies', - ] - first_pass_args.extend(spec_cli_arg) - tty.debug('First pass install arguments: {0}'.format( - first_pass_args)) - spack_cmd(*first_pass_args) - - # Overwrite the changed environment file so it doesn't break - # the next install invocation. - tty.debug('Copying {0} to {1}'.format( - env_dst_path, env_src_path)) - shutil.copyfile(env_dst_path, env_src_path) - - second_pass_args = install_args + [ - '--no-cache', - '--only', - 'package', - ] - second_pass_args.extend(cdash_args) - second_pass_args.extend(spec_cli_arg) - tty.debug('Second pass install arguments: {0}'.format( - second_pass_args)) - spack_cmd(*second_pass_args) - except Exception as inst: - tty.error('Caught exception during install:') - tty.error(inst) + tty.debug('spack install arguments: {0}'.format( + install_args)) + spack_cmd(*install_args) + finally: + spack_ci.copy_stage_logs_to_artifacts(job_spec, job_log_dir) + + # Create buildcache on remote mirror, either on pr-specific + # mirror or on mirror defined in spack environment + if spack_is_pr_pipeline: + buildcache_mirror_url = pr_mirror_url + else: + buildcache_mirror_url = remote_mirror_url - spack_ci.copy_stage_logs_to_artifacts(job_spec, job_log_dir) - - # 4) create buildcache on remote mirror, but not if this is - # running to test a spack PR - if not spack_is_pr_pipeline: + try: spack_ci.push_mirror_contents( - env, job_spec, job_spec_yaml_path, remote_mirror_url, - cdash_build_id) - - # 5) create another copy of that buildcache on "local artifact - # mirror" (only done if cash reporting is enabled) + env, job_spec, job_spec_yaml_path, buildcache_mirror_url, + cdash_build_id, sign_binaries) + except Exception as inst: + # If the mirror we're pushing to is on S3 and there's some + # permissions problem, for example, we can't just target + # that exception type here, since users of the + # `spack ci rebuild' may not need or want any dependency + # on boto3. So we use the first non-boto exception type + # in the heirarchy: + # boto3.exceptions.S3UploadFailedError + # boto3.exceptions.Boto3Error + # Exception + # BaseException + # object + err_msg = 'Error msg: {0}'.format(inst) + if 'Access Denied' in err_msg: + tty.msg('Permission problem writing to mirror') + tty.msg(err_msg) + + # Create another copy of that buildcache on "local artifact + # mirror" (only done if artifacts buildcache is enabled) spack_ci.push_mirror_contents(env, job_spec, job_spec_yaml_path, - artifact_mirror_url, cdash_build_id) + artifact_mirror_url, cdash_build_id, + sign_binaries) - # 6) relate this build to its dependencies on CDash (if enabled) + # Relate this build to its dependencies on CDash (if enabled) if enable_cdash: spack_ci.relate_cdash_builds( spec_map, cdash_base_url, cdash_build_id, cdash_project, - artifact_mirror_url or remote_mirror_url) - else: - # There is nothing to do here unless "local artifact mirror" is - # enabled, in which case, we need to download the buildcache to - # the local artifacts directory to be used by dependent jobs in - # subsequent stages - tty.debug('No need to rebuild {0}'.format(job_spec_pkg_name)) - if enable_artifacts_mirror: - tty.debug('Getting {0} buildcache'.format(job_spec_pkg_name)) - tty.debug('Downloading to {0}'.format(build_cache_dir)) - buildcache.download_buildcache_files( - job_spec, build_cache_dir, True, remote_mirror_url) + artifact_mirror_url or pr_mirror_url or remote_mirror_url) def ci(parser, args): diff --git a/lib/spack/spack/test/bindist.py b/lib/spack/spack/test/bindist.py index ef54e658f0..974c50c260 100644 --- a/lib/spack/spack/test/bindist.py +++ b/lib/spack/spack/test/bindist.py @@ -22,6 +22,7 @@ import spack.cmd.mirror as mirror from spack.main import SpackCommand import spack.mirror import spack.util.gpg +import spack.util.web as web_util from spack.directory_layout import YamlDirectoryLayout from spack.spec import Spec @@ -622,3 +623,51 @@ def test_spec_needs_rebuild(install_mockery_mutable_config, mock_packages, rebuild = bindist.needs_rebuild(s, mirror_url, rebuild_on_errors=True) assert rebuild + + +def test_generate_indices_key_error(monkeypatch, capfd): + + def mock_list_url(url, recursive=False): + print('mocked list_url({0}, {1})'.format(url, recursive)) + raise KeyError('Test KeyError handling') + + monkeypatch.setattr(web_util, 'list_url', mock_list_url) + + test_url = 'file:///fake/keys/dir' + + # Make sure generate_key_index handles the KeyError + bindist.generate_key_index(test_url) + + err = capfd.readouterr()[1] + assert 'Warning: No keys at {0}'.format(test_url) in err + + # Make sure generate_package_index handles the KeyError + bindist.generate_package_index(test_url) + + err = capfd.readouterr()[1] + assert 'Warning: No packages at {0}'.format(test_url) in err + + +def test_generate_indices_exception(monkeypatch, capfd): + + def mock_list_url(url, recursive=False): + print('mocked list_url({0}, {1})'.format(url, recursive)) + raise Exception('Test Exception handling') + + monkeypatch.setattr(web_util, 'list_url', mock_list_url) + + test_url = 'file:///fake/keys/dir' + + # Make sure generate_key_index handles the Exception + bindist.generate_key_index(test_url) + + err = capfd.readouterr()[1] + expect = 'Encountered problem listing keys at {0}'.format(test_url) + assert expect in err + + # Make sure generate_package_index handles the Exception + bindist.generate_package_index(test_url) + + err = capfd.readouterr()[1] + expect = 'Encountered problem listing packages at {0}'.format(test_url) + assert expect in err diff --git a/lib/spack/spack/test/cmd/ci.py b/lib/spack/spack/test/cmd/ci.py index 4e19cf384d..82734c922a 100644 --- a/lib/spack/spack/test/cmd/ci.py +++ b/lib/spack/spack/test/cmd/ci.py @@ -733,9 +733,9 @@ spack: install_cmd('--keep-stage', yaml_path) - # env, spec, yaml_path, mirror_url, build_id + # env, spec, yaml_path, mirror_url, build_id, sign_binaries ci.push_mirror_contents( - env, concrete_spec, yaml_path, mirror_url, '42') + env, concrete_spec, yaml_path, mirror_url, '42', True) buildcache_path = os.path.join(mirror_dir.strpath, 'build_cache') diff --git a/share/spack/gitlab/cloud_e4s_pipelines.yml b/share/spack/gitlab/cloud_e4s_pipelines.yml new file mode 100644 index 0000000000..67a74cf971 --- /dev/null +++ b/share/spack/gitlab/cloud_e4s_pipelines.yml @@ -0,0 +1,24 @@ +pr_pipeline: + only: + - /^github\/pr[\d]+_.*$/ + variables: + SPACK_REF: ${CI_COMMIT_SHA} + SPACK_PR_BRANCH: ${CI_COMMIT_REF_NAME} + SPACK_IS_PR_PIPELINE: "True" + AWS_ACCESS_KEY_ID: ${PR_MIRRORS_AWS_ACCESS_KEY_ID} + AWS_SECRET_ACCESS_KEY: ${PR_MIRRORS_AWS_SECRET_ACCESS_KEY} + trigger: + project: spack/e4s + strategy: depend + +develop_pipeline: + only: + - /^github\/develop$/ + variables: + SPACK_REF: ${CI_COMMIT_SHA} + AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID} + AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY} + SPACK_SIGNING_KEY: ${SPACK_SIGNING_KEY} + trigger: + project: spack/e4s + strategy: depend diff --git a/share/spack/gitlab/pr_pipeline.yml b/share/spack/gitlab/pr_pipeline.yml deleted file mode 100644 index 367a4b9077..0000000000 --- a/share/spack/gitlab/pr_pipeline.yml +++ /dev/null @@ -1,18 +0,0 @@ -pr_pipeline: - only: - - external_pull_requests - variables: - SPACK_REF: ${CI_EXTERNAL_PULL_REQUEST_SOURCE_BRANCH_NAME} - SPACK_IS_PR_PIPELINE: "True" - trigger: - project: spack/e4s - strategy: depend - -merge_pipeline: - only: - - develop - variables: - SPACK_REF: ${CI_COMMIT_SHA} - trigger: - project: spack/e4s - strategy: depend
\ No newline at end of file |