diff options
Diffstat (limited to 'lib')
| -rw-r--r-- | lib/spack/spack/cmd/release_jobs.py | 414 | ||||
| -rw-r--r-- | lib/spack/spack/schema/os_container_mapping.py | 50 |
2 files changed, 173 insertions, 291 deletions
diff --git a/lib/spack/spack/cmd/release_jobs.py b/lib/spack/spack/cmd/release_jobs.py index fd79acd573..e982c76562 100644 --- a/lib/spack/spack/cmd/release_jobs.py +++ b/lib/spack/spack/cmd/release_jobs.py @@ -3,26 +3,20 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) -import argparse import json -import os -import shutil -import tempfile +import sys -import subprocess from jsonschema import validate, ValidationError from six import iteritems +from six.moves.urllib.request import build_opener, HTTPHandler, Request +from six.moves.urllib.parse import urlencode import llnl.util.tty as tty -from spack.architecture import sys_type +import spack.environment as ev from spack.dependency import all_deptypes -from spack.spec import Spec, CompilerSpec -from spack.paths import spack_root -from spack.error import SpackError -from spack.schema.os_container_mapping import schema as mapping_schema +from spack.spec import Spec from spack.schema.specs_deps import schema as specs_deps_schema -from spack.spec_set import CombinatorialSpecSet import spack.util.spack_yaml as syaml description = "generate release build set as .gitlab-ci.yml" @@ -32,57 +26,101 @@ level = "long" def setup_parser(subparser): subparser.add_argument( - '-s', '--spec-set', default=None, - help="path to release spec-set yaml file") - - subparser.add_argument( - '-m', '--mirror-url', default=None, - help="url of binary mirror where builds should be pushed") + '-f', '--force', action='store_true', default=False, + help="Force re-concretization of environment first") subparser.add_argument( '-o', '--output-file', default=".gitlab-ci.yml", help="path to output file to write") subparser.add_argument( - '-t', '--shared-runner-tag', default=None, - help="tag to add to jobs for shared runner selection") - - subparser.add_argument( '-k', '--signing-key', default=None, help="hash of gpg key to use for package signing") subparser.add_argument( - '-c', '--cdash-url', default='https://cdash.spack.io', - help="Base url of CDash instance jobs should communicate with") - - subparser.add_argument( '-p', '--print-summary', action='store_true', default=False, help="Print summary of staged jobs to standard output") subparser.add_argument( - '--resolve-deps-locally', action='store_true', default=False, - help="Use only the current machine to concretize specs, " + - "instead of iterating over items in os-container-mapping.yaml " + - "and using docker run. Assumes the current machine architecure " + - "is listed in the os-container-mapping.yaml config file.") + '-c', '--cdash-credentials', default=None, + help="Path to file containing CDash authentication token") - subparser.add_argument( - '--specs-deps-output', default='/dev/stdout', - help="A file path to which spec deps should be written. This " + - "argument is generally for internal use, and should not be " + - "provided by end-users under normal conditions.") - subparser.add_argument( - 'specs', nargs=argparse.REMAINDER, - help="These positional arguments are generally for internal use. " + - "The --spec-set argument should be used to identify a yaml " + - "file describing the set of release specs to include in the " + - ".gitlab-ci.yml file.") +def _create_buildgroup(opener, headers, url, project, group_name, group_type): + data = { + "newbuildgroup": group_name, + "project": project, + "type": group_type + } + + request = Request(url, data=json.dumps(data), headers=headers) + + response = opener.open(request) + response_code = response.getcode() + + if response_code != 200 and response_code != 201: + print('Creating buildgroup failed (response code = {0}'.format( + response_code)) + return None + + response_text = response.read() + response_json = json.loads(response_text) + build_group_id = response_json['id'] + + return build_group_id + +def populate_buildgroup(job_names, group_name, project, site, + credentials, cdash_url, exit_on_fail=False): + url = "{0}/api/v1/buildgroup.php".format(cdash_url) -def get_job_name(spec, osarch): - return '{0} {1} {2} {3}'.format(spec.name, spec.version, - spec.compiler, osarch) + headers = { + 'Authorization': 'Bearer {0}'.format(credentials), + 'Content-Type': 'application/json', + } + + opener = build_opener(HTTPHandler) + + parent_group_id = _create_buildgroup( + opener, headers, url, project, group_name, 'Daily') + group_id = _create_buildgroup( + opener, headers, url, project, 'Latest {0}'.format(group_name), + 'Latest') + + if not parent_group_id or not group_id: + print('Unable to create or retrieve the build groups') + sys.exit(1) + + data = { + 'project': project, + 'buildgroupid': group_id, + 'dynamiclist': [{ + 'match': name, + 'parentgroupid': parent_group_id, + 'site': site + } for name in job_names] + } + + request = Request(url, data=json.dumps(data), headers=headers) + request.get_method = lambda: 'PUT' + + response = opener.open(request) + response_code = response.getcode() + + if response_code != 200 and exit_on_fail: + print('Unexpected response ({0}) when populating buildgroup'.format( + response_code)) + sys.exit(1) + + +def get_job_name(spec, osarch, build_group): + return '{0} {1} {2} {3} {4}'.format( + spec.name, spec.version, spec.compiler, osarch, build_group) + + +def get_cdash_build_name(spec, build_group): + return '{0}@{1}%{2} arch={3} ({4})'.format( + spec.name, spec.version, spec.compiler, spec.architecture, build_group) def get_spec_string(spec): @@ -109,84 +147,8 @@ def _add_dependency(spec_label, dep_label, deps): deps[spec_label].add(dep_label) -def get_deps_using_container(specs, image): - image_home_dir = '/home/spackuser' - repo_mount_location = '{0}/spack'.format(image_home_dir) - temp_dir = tempfile.mkdtemp(dir='/tmp') - - # The paths this module will see (from outside the container) - temp_file = os.path.join(temp_dir, 'spec_deps.json') - temp_err = os.path.join(temp_dir, 'std_err.log') - - # The paths the bash_command will see inside the container - json_output = '/work/spec_deps.json' - std_error = '/work/std_err.log' - - specs_arg = ' '.join([str(spec) for spec in specs]) - - bash_command = " ".join(["source {0}/share/spack/setup-env.sh ;", - "spack release-jobs", - "--specs-deps-output {1}", - "{2}", - "2> {3}"]).format( - repo_mount_location, json_output, specs_arg, std_error) - - docker_cmd_to_run = [ - 'docker', 'run', '--rm', - '-v', '{0}:{1}'.format(spack_root, repo_mount_location), - '-v', '{0}:{1}'.format(temp_dir, '/work'), - '--entrypoint', 'bash', - '-t', str(image), - '-c', - bash_command, - ] - - tty.debug('Running subprocess command:') - tty.debug(' '.join(docker_cmd_to_run)) - - # Docker is going to merge the stdout/stderr from the script and write it - # all to the stdout of the running container. For this reason, we won't - # pipe any stdout/stderr from the docker command, but rather write the - # output we care about to a file in a mounted directory. Similarly, any - # errors from running the spack command inside the container are redirected - # to another file in the mounted directory. - proc = subprocess.Popen(docker_cmd_to_run) - proc.wait() - - # Check for errors from spack command - if os.path.exists(temp_err) and os.path.getsize(temp_err) > 0: - # Spack wrote something to stderr inside the container. We will - # print out whatever it is, but attempt to carry on with the process. - tty.error('Encountered spack error running command in container:') - with open(temp_err, 'r') as err: - tty.error(err.read()) - - spec_deps_obj = {} - - try: - # Finally, try to read/parse the output we really care about: the - # specs and dependency edges for the provided spec, as it was - # concretized in the appropriate container. - with open(temp_file, 'r') as fd: - spec_deps_obj = json.loads(fd.read()) - - except ValueError as val_err: - tty.error('Failed to read json object from spec-deps output file:') - tty.error(str(val_err)) - except IOError as io_err: - tty.error('Problem reading from spec-deps json output file:') - tty.error(str(io_err)) - finally: - shutil.rmtree(temp_dir) - - return spec_deps_obj - - -def get_spec_dependencies(specs, deps, spec_labels, image=None): - if image: - spec_deps_obj = get_deps_using_container(specs, image) - else: - spec_deps_obj = compute_spec_deps(specs) +def get_spec_dependencies(specs, deps, spec_labels): + spec_deps_obj = compute_spec_deps(specs) try: validate(spec_deps_obj, specs_deps_schema) @@ -210,7 +172,7 @@ def get_spec_dependencies(specs, deps, spec_labels, image=None): _add_dependency(entry['spec'], entry['depends'], deps) -def stage_spec_jobs(spec_set, containers, current_system=None): +def stage_spec_jobs(specs): """Take a set of release specs along with a dictionary describing the available docker containers and what compilers they have, and generate a list of "stages", where the jobs in any stage are dependent only on @@ -265,46 +227,7 @@ def stage_spec_jobs(spec_set, containers, current_system=None): deps = {} spec_labels = {} - if current_system: - if current_system not in containers: - error_msg = ' '.join(['Current system ({0}) does not appear in', - 'os_container_mapping.yaml, ignoring', - 'request']).format( - current_system) - raise SpackError(error_msg) - os_names = [current_system] - else: - os_names = [name for name in containers] - - container_specs = {} - for name in os_names: - container_specs[name] = {'image': None, 'specs': []} - - # Collect together all the specs that should be concretized in each - # container so they can all be done at once, avoiding the need to - # run the docker container for each spec separately. - for spec in spec_set: - for osname in os_names: - container_info = containers[osname] - image = None if current_system else container_info['image'] - if image: - container_specs[osname]['image'] = image - if 'compilers' in container_info: - found_at_least_one = False - for item in container_info['compilers']: - container_compiler_spec = CompilerSpec(item['name']) - if spec.compiler == container_compiler_spec: - container_specs[osname]['specs'].append(spec) - found_at_least_one = True - if not found_at_least_one: - tty.warn('No compiler in {0} satisfied {1}'.format( - osname, spec.compiler)) - - for osname in container_specs: - if container_specs[osname]['specs']: - image = container_specs[osname]['image'] - specs = container_specs[osname]['specs'] - get_spec_dependencies(specs, deps, spec_labels, image) + get_spec_dependencies(specs, deps, spec_labels) # Save the original deps, as we need to return them at the end of the # function. In the while loop below, the "dependencies" variable is @@ -452,49 +375,48 @@ def compute_spec_deps(spec_list, stream_like=None): return deps_json_obj -def release_jobs(parser, args): - share_path = os.path.join(spack_root, 'share', 'spack', 'docker') - os_container_mapping_path = os.path.join( - share_path, 'os-container-mapping.yaml') +def spec_matches(spec, match_string): + return spec.satisfies(match_string) - with open(os_container_mapping_path, 'r') as fin: - os_container_mapping = syaml.load(fin) - try: - validate(os_container_mapping, mapping_schema) - except ValidationError as val_err: - tty.error('Ill-formed os-container-mapping configuration object') - tty.error(os_container_mapping) - tty.debug(val_err) - return +def find_matching_config(spec, ci_mappings): + for ci_mapping in ci_mappings: + for match_string in ci_mapping['match']: + if spec_matches(spec, match_string): + return ci_mapping['runner-attributes'] + return None - containers = os_container_mapping['containers'] - if args.specs: - # Just print out the spec labels and all dependency edges in - # a json format. - spec_list = [Spec(s) for s in args.specs] - with open(args.specs_deps_output, 'w') as out: - compute_spec_deps(spec_list, out) - return +def release_jobs(parser, args): + env = ev.get_env(args, 'release-jobs', required=True) + env.concretize(force=args.force) - current_system = sys_type() if args.resolve_deps_locally else None + yaml_root = env.yaml['spack'] - release_specs_path = args.spec_set - if not release_specs_path: - raise SpackError('Must provide path to release spec-set') + if 'gitlab-ci' not in yaml_root: + tty.die('Environment yaml does not have "gitlab-ci" section') - release_spec_set = CombinatorialSpecSet.from_file(release_specs_path) + ci_mappings = yaml_root['gitlab-ci']['mappings'] - mirror_url = args.mirror_url + ci_cdash = yaml_root['cdash'] + build_group = ci_cdash['build-group'] + cdash_url = ci_cdash['url'] + cdash_project = ci_cdash['project'] + proj_enc = urlencode({'project': cdash_project}) + eq_idx = proj_enc.find('=') + 1 + cdash_project_enc = proj_enc[eq_idx:] + cdash_site = ci_cdash['site'] + cdash_auth_token = None - if not mirror_url: - raise SpackError('Must provide url of target binary mirror') + if args.cdash_credentials: + with open(args.cdash_credentials) as fd: + cdash_auth_token = fd.read() + cdash_auth_token = cdash_auth_token.strip() - cdash_url = args.cdash_url + ci_mirrors = yaml_root['mirrors'] + mirror_urls = ci_mirrors.values() - spec_labels, dependencies, stages = stage_spec_jobs( - release_spec_set, containers, current_system) + spec_labels, dependencies, stages = stage_spec_jobs(env.all_specs()) if not stages: tty.msg('No jobs staged, exiting.') @@ -503,6 +425,7 @@ def release_jobs(parser, args): if args.print_summary: print_staging_summary(spec_labels, dependencies, stages) + all_job_names = [] output_object = {} job_count = 0 @@ -516,37 +439,56 @@ def release_jobs(parser, args): release_spec = spec_labels[spec_label]['spec'] root_spec = spec_labels[spec_label]['rootSpec'] - pkg_compiler = release_spec.compiler - pkg_hash = release_spec.dag_hash() + runner_attribs = find_matching_config(release_spec, ci_mappings) + + if not runner_attribs: + tty.warn('No match found for {0}, skipping it'.format( + release_spec)) + continue + + tags = [tag for tag in runner_attribs['tags']] + + variables = {} + if 'variables' in runner_attribs: + variables.update(runner_attribs['variables']) + + build_image = None + if 'image' in runner_attribs: + build_image = runner_attribs['image'] osname = str(release_spec.architecture) - job_name = get_job_name(release_spec, osname) - container_info = containers[osname] - build_image = container_info['image'] + job_name = get_job_name(release_spec, osname, build_group) + cdash_build_name = get_cdash_build_name(release_spec, build_group) - job_scripts = ['./bin/rebuild-package.sh'] + all_job_names.append(cdash_build_name) - if 'setup_script' in container_info: - job_scripts.insert( - 0, container_info['setup_script'] % pkg_compiler) + job_scripts = ['./bin/rebuild-package.sh'] job_dependencies = [] if spec_label in dependencies: job_dependencies = ( - [get_job_name(spec_labels[dep_label]['spec'], osname) - for dep_label in dependencies[spec_label]]) + [get_job_name(spec_labels[d]['spec'], osname, build_group) + for d in dependencies[spec_label]]) + + job_variables = { + 'MIRROR_URL': mirror_urls[0], + 'CDASH_BASE_URL': cdash_url, + 'CDASH_PROJECT': cdash_project, + 'CDASH_PROJECT_ENC': cdash_project_enc, + 'CDASH_BUILD_NAME': cdash_build_name, + 'DEPENDENCIES': ';'.join(job_dependencies), + 'ROOT_SPEC': str(root_spec), + } + + if args.signing_key: + job_variables['SIGN_KEY_HASH'] = args.signing_key + + variables.update(job_variables) job_object = { 'stage': stage_name, - 'variables': { - 'MIRROR_URL': mirror_url, - 'CDASH_BASE_URL': cdash_url, - 'HASH': pkg_hash, - 'DEPENDENCIES': ';'.join(job_dependencies), - 'ROOT_SPEC': str(root_spec), - }, + 'variables': variables, 'script': job_scripts, - 'image': build_image, 'artifacts': { 'paths': [ 'local_mirror/build_cache', @@ -556,51 +498,41 @@ def release_jobs(parser, args): 'when': 'always', }, 'dependencies': job_dependencies, + 'tags': tags, } - # If we see 'compilers' in the container iformation, it's a - # filter for the compilers this container can handle, else we - # assume it can handle any compiler - if 'compilers' in container_info: - do_job = False - for item in container_info['compilers']: - container_compiler_spec = CompilerSpec(item['name']) - if pkg_compiler == container_compiler_spec: - do_job = True - else: - do_job = True - - if args.shared_runner_tag: - job_object['tags'] = [args.shared_runner_tag] + if build_image: + job_object['image'] = build_image - if args.signing_key: - job_object['variables']['SIGN_KEY_HASH'] = args.signing_key - - if do_job: - output_object[job_name] = job_object - job_count += 1 + output_object[job_name] = job_object + job_count += 1 stage += 1 tty.msg('{0} build jobs generated in {1} stages'.format( job_count, len(stages))) - final_stage = 'stage-rebuild-index' + # Use "all_job_names" to populate the build group for this set + if cdash_auth_token: + populate_buildgroup(all_job_names, build_group, cdash_project, + cdash_site, cdash_auth_token, cdash_url) + else: + tty.warn('Unable to populate buildgroup without CDash credentials') + # Add an extra, final job to regenerate the index + final_stage = 'stage-rebuild-index' final_job = { 'stage': final_stage, 'variables': { - 'MIRROR_URL': mirror_url, + 'MIRROR_URL': mirror_urls[0], }, - 'image': build_image, + 'image': 'scottwittenburg/spack_ci_generator_alpine', # just needs some basic python image 'script': './bin/rebuild-index.sh', + 'tags': ['spack-k8s'] # may want a runner to handle this } - - if args.shared_runner_tag: - final_job['tags'] = [args.shared_runner_tag] - output_object['rebuild-index'] = final_job stage_names.append(final_stage) + output_object['stages'] = stage_names with open(args.output_file, 'w') as outf: diff --git a/lib/spack/spack/schema/os_container_mapping.py b/lib/spack/spack/schema/os_container_mapping.py deleted file mode 100644 index a69549d51d..0000000000 --- a/lib/spack/spack/schema/os_container_mapping.py +++ /dev/null @@ -1,50 +0,0 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) - -"""Schema for os-container-mapping.yaml configuration file. - -.. literalinclude:: _spack_root/lib/spack/spack/schema/os_container_mapping.py - :lines: 32- -""" - - -schema = { - '$schema': 'http://json-schema.org/schema#', - 'title': 'Spack release builds os/container mapping config file schema', - 'type': 'object', - 'additionalProperties': False, - 'patternProperties': { - r'containers': { - 'type': 'object', - 'default': {}, - 'patternProperties': { - r'[\w\d\-_\.]+': { - 'type': 'object', - 'default': {}, - 'additionalProperties': False, - 'required': ['image'], - 'properties': { - 'image': {'type': 'string'}, - 'setup_script': {'type': 'string'}, - 'compilers': { - 'type': 'array', - 'default': [], - 'items': { - 'type': 'object', - 'default': {}, - 'additionalProperties': False, - 'required': ['name'], - 'properties': { - 'name': {'type': 'string'}, - 'path': {'type': 'string'}, - }, - }, - }, - }, - }, - }, - }, - }, -} |