From a6e8e889b291a5a048cbdf5497f19216a4acb25e Mon Sep 17 00:00:00 2001 From: Scott Wittenburg Date: Mon, 17 Dec 2018 17:35:47 -0700 Subject: release workflow: Add spack command for generating the .gitlab-ci.yml This spack command adds a new schema for a file which describes the builder containers available, along with the compilers availabe on each builder. The release-jobs command then generates the .gitlab-ci.yml file by first expanding the release spec set, concretizing each spec (in an appropriate docker container if --this-machine-only argument is not provided on command line), and then combining and staging all the concrete specs as jobs to be run by gitlab. --- lib/spack/spack/cmd/release_jobs.py | 607 +++++++++++++++++++++++++ lib/spack/spack/schema/os_container_mapping.py | 50 ++ lib/spack/spack/schema/specs_deps.py | 48 ++ lib/spack/spack/test/cmd/release_jobs.py | 118 +++++ 4 files changed, 823 insertions(+) create mode 100644 lib/spack/spack/cmd/release_jobs.py create mode 100644 lib/spack/spack/schema/os_container_mapping.py create mode 100644 lib/spack/spack/schema/specs_deps.py create mode 100644 lib/spack/spack/test/cmd/release_jobs.py (limited to 'lib') diff --git a/lib/spack/spack/cmd/release_jobs.py b/lib/spack/spack/cmd/release_jobs.py new file mode 100644 index 0000000000..78baf81263 --- /dev/null +++ b/lib/spack/spack/cmd/release_jobs.py @@ -0,0 +1,607 @@ +# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import argparse +import json +import os +import shutil +import tempfile + +import subprocess +from jsonschema import validate, ValidationError +from six import iteritems + +import llnl.util.tty as tty + +from spack.architecture import sys_type +from spack.dependency import all_deptypes +from spack.spec import Spec, CompilerSpec +from spack.paths import spack_root +from spack.error import SpackError +from spack.schema.os_container_mapping import schema as mapping_schema +from spack.schema.specs_deps import schema as specs_deps_schema +from spack.spec_set import CombinatorialSpecSet +import spack.util.spack_yaml as syaml + +description = "generate release build set as .gitlab-ci.yml" +section = "build" +level = "long" + + +def setup_parser(subparser): + subparser.add_argument( + '-s', '--spec-set', default=None, + help="path to release spec-set yaml file") + + subparser.add_argument( + '-m', '--mirror-url', default=None, + help="url of binary mirror where builds should be pushed") + + subparser.add_argument( + '-o', '--output-file', default=".gitlab-ci.yml", + help="path to output file to write") + + subparser.add_argument( + '-t', '--shared-runner-tag', default=None, + help="tag to add to jobs for shared runner selection") + + subparser.add_argument( + '-k', '--signing-key', default=None, + help="hash of gpg key to use for package signing") + + subparser.add_argument( + '-c', '--cdash-url', default='https://cdash.spack.io', + help="Base url of CDash instance jobs should communicate with") + + subparser.add_argument( + '-p', '--print-summary', action='store_true', default=False, + help="Print summary of staged jobs to standard output") + + subparser.add_argument( + '--resolve-deps-locally', action='store_true', default=False, + help="Use only the current machine to concretize specs, " + + "instead of iterating over items in os-container-mapping.yaml " + + "and using docker run. Assumes the current machine architecure " + + "is listed in the os-container-mapping.yaml config file.") + + subparser.add_argument( + '--specs-deps-output', default='/dev/stdout', + help="A file path to which spec deps should be written. This " + + "argument is generally for internal use, and should not be " + + "provided by end-users under normal conditions.") + + subparser.add_argument( + 'specs', nargs=argparse.REMAINDER, + help="These positional arguments are generally for internal use. " + + "The --spec-set argument should be used to identify a yaml " + + "file describing the set of release specs to include in the " + + ".gitlab-ci.yml file.") + + +def get_job_name(spec, osarch): + return '{0} {1} {2} {3}'.format(spec.name, spec.version, + spec.compiler, osarch) + + +def get_spec_string(spec): + format_elements = [ + '${package}@${version}', + '%${compilername}@${compilerversion}', + ] + + if spec.architecture: + format_elements.append(' arch=${architecture}') + + return spec.format(''.join(format_elements)) + + +def spec_deps_key_label(s): + return s.dag_hash(), "%s/%s" % (s.name, s.dag_hash(7)) + + +def _add_dependency(spec_label, dep_label, deps): + if spec_label == dep_label: + return + if spec_label not in deps: + deps[spec_label] = set() + deps[spec_label].add(dep_label) + + +def get_deps_using_container(specs, image): + image_home_dir = '/home/spackuser' + repo_mount_location = '{0}/spack'.format(image_home_dir) + temp_dir = tempfile.mkdtemp(dir='/tmp') + + # The paths this module will see (from outside the container) + temp_file = os.path.join(temp_dir, 'spec_deps.json') + temp_err = os.path.join(temp_dir, 'std_err.log') + + # The paths the bash_command will see inside the container + json_output = '/work/spec_deps.json' + std_error = '/work/std_err.log' + + specs_arg = ' '.join([str(spec) for spec in specs]) + + bash_command = " ".join(["source {0}/share/spack/setup-env.sh ;", + "spack release-jobs", + "--specs-deps-output {1}", + "{2}", + "2> {3}"]).format( + repo_mount_location, json_output, specs_arg, std_error) + + docker_cmd_to_run = [ + 'docker', 'run', '--rm', + '-v', '{0}:{1}'.format(spack_root, repo_mount_location), + '-v', '{0}:{1}'.format(temp_dir, '/work'), + '--entrypoint', 'bash', + '-t', str(image), + '-c', + bash_command, + ] + + tty.debug('Running subprocess command:') + tty.debug(' '.join(docker_cmd_to_run)) + + # Docker is going to merge the stdout/stderr from the script and write it + # all to the stdout of the running container. For this reason, we won't + # pipe any stdout/stderr from the docker command, but rather write the + # output we care about to a file in a mounted directory. Similarly, any + # errors from running the spack command inside the container are redirected + # to another file in the mounted directory. + proc = subprocess.Popen(docker_cmd_to_run) + proc.wait() + + # Check for errors from spack command + if os.path.exists(temp_err) and os.path.getsize(temp_err) > 0: + # Spack wrote something to stderr inside the container. We will + # print out whatever it is, but attempt to carry on with the process. + tty.error('Encountered spack error running command in container:') + with open(temp_err, 'r') as err: + tty.error(err.read()) + + spec_deps_obj = {} + + try: + # Finally, try to read/parse the output we really care about: the + # specs and dependency edges for the provided spec, as it was + # concretized in the appropriate container. + with open(temp_file, 'r') as fd: + spec_deps_obj = json.loads(fd.read()) + + except ValueError as val_err: + tty.error('Failed to read json object from spec-deps output file:') + tty.error(str(val_err)) + except IOError as io_err: + tty.error('Problem reading from spec-deps json output file:') + tty.error(str(io_err)) + finally: + shutil.rmtree(temp_dir) + + return spec_deps_obj + + +def get_spec_dependencies(specs, deps, spec_labels, image=None): + if image: + spec_deps_obj = get_deps_using_container(specs, image) + else: + spec_deps_obj = compute_spec_deps(specs) + + try: + validate(spec_deps_obj, specs_deps_schema) + except ValidationError as val_err: + tty.error('Ill-formed specs dependencies JSON object') + tty.error(spec_deps_obj) + tty.debug(val_err) + return + + if spec_deps_obj: + dependencies = spec_deps_obj['dependencies'] + specs = spec_deps_obj['specs'] + + for entry in specs: + spec_labels[entry['label']] = { + 'spec': Spec(entry['spec']), + 'rootSpec': entry['root_spec'], + } + + for entry in dependencies: + _add_dependency(entry['spec'], entry['depends'], deps) + + +def stage_spec_jobs(spec_set, containers, current_system=None): + """Take a set of release specs along with a dictionary describing the + available docker containers and what compilers they have, and generate + a list of "stages", where the jobs in any stage are dependent only on + jobs in previous stages. This allows us to maximize build parallelism + within the gitlab-ci framework. + + Arguments: + spec_set (CombinatorialSpecSet): Iterable containing all the specs + to build. + containers (dict): Describes the docker containers available to use + for concretizing specs (and also for the gitlab runners to use + for building packages). The schema can be found at + "lib/spack/spack/schema/os_container_mapping.py" + current_system (string): If provided, this indicates not to use the + containers for concretizing the release specs, but rather just + assume the current system is in the "containers" dictionary. A + SpackError will be raised if the current system is not in that + dictionary. + + Returns: A tuple of information objects describing the specs, dependencies + and stages: + + spec_labels: A dictionary mapping the spec labels which are made of + (pkg-name/hash-prefix), to objects containing "rootSpec" and "spec" + keys. The root spec is the spec of which this spec is a dependency + and the spec is the formatted spec string for this spec. + + deps: A dictionary where the keys should also have appeared as keys in + the spec_labels dictionary, and the values are the set of + dependencies for that spec. + + stages: An ordered list of sets, each of which contains all the jobs to + built in that stage. The jobs are expressed in the same format as + the keys in the spec_labels and deps objects. + + """ + + # The convenience method below, "remove_satisfied_deps()", does not modify + # the "deps" parameter. Instead, it returns a new dictionary where only + # dependencies which have not yet been satisfied are included in the + # return value. + def remove_satisfied_deps(deps, satisfied_list): + new_deps = {} + + for key, value in iteritems(deps): + new_value = set([v for v in value if v not in satisfied_list]) + if new_value: + new_deps[key] = new_value + + return new_deps + + deps = {} + spec_labels = {} + + if current_system: + if current_system not in containers: + error_msg = ' '.join(['Current system ({0}) does not appear in', + 'os_container_mapping.yaml, ignoring', + 'request']).format( + current_system) + raise SpackError(error_msg) + os_names = [current_system] + else: + os_names = [name for name in containers] + + container_specs = {} + for name in os_names: + container_specs[name] = {'image': None, 'specs': []} + + # Collect together all the specs that should be concretized in each + # container so they can all be done at once, avoiding the need to + # run the docker container for each spec separately. + for spec in spec_set: + for osname in os_names: + container_info = containers[osname] + image = None if current_system else container_info['image'] + if image: + container_specs[osname]['image'] = image + if 'compilers' in container_info: + found_at_least_one = False + for item in container_info['compilers']: + container_compiler_spec = CompilerSpec(item['name']) + if spec.compiler == container_compiler_spec: + container_specs[osname]['specs'].append(spec) + found_at_least_one = True + if not found_at_least_one: + tty.warn('No compiler in {0} satisfied {1}'.format( + osname, spec.compiler)) + + for osname in container_specs: + if container_specs[osname]['specs']: + image = container_specs[osname]['image'] + specs = container_specs[osname]['specs'] + get_spec_dependencies(specs, deps, spec_labels, image) + + # Save the original deps, as we need to return them at the end of the + # function. In the while loop below, the "dependencies" variable is + # overwritten rather than being modified each time through the loop, + # thus preserving the original value of "deps" saved here. + dependencies = deps + unstaged = set(spec_labels.keys()) + stages = [] + + while dependencies: + dependents = set(dependencies.keys()) + next_stage = unstaged.difference(dependents) + stages.append(next_stage) + unstaged.difference_update(next_stage) + # Note that "dependencies" is a dictionary mapping each dependent + # package to the set of not-yet-handled dependencies. The final step + # below removes all the dependencies that are handled by this stage. + dependencies = remove_satisfied_deps(dependencies, next_stage) + + if unstaged: + stages.append(unstaged.copy()) + + return spec_labels, deps, stages + + +def print_staging_summary(spec_labels, dependencies, stages): + if not stages: + return + + tty.msg('Staging summary:') + stage_index = 0 + for stage in stages: + tty.msg(' stage {0} ({1} jobs):'.format(stage_index, len(stage))) + + for job in sorted(stage): + s = spec_labels[job]['spec'] + tty.msg(' {0} -> {1}'.format(job, get_spec_string(s))) + + stage_index += 1 + + +def compute_spec_deps(spec_list, stream_like=None): + """ + Computes all the dependencies for the spec(s) and generates a JSON + object which provides both a list of unique spec names as well as a + comprehensive list of all the edges in the dependency graph. For + example, given a single spec like 'readline@7.0', this function + generates the following JSON object: + + .. code-block:: JSON + + { + "dependencies": [ + { + "depends": "readline/ip6aiun", + "spec": "readline/ip6aiun" + }, + { + "depends": "ncurses/y43rifz", + "spec": "readline/ip6aiun" + }, + { + "depends": "ncurses/y43rifz", + "spec": "readline/ip6aiun" + }, + { + "depends": "pkgconf/eg355zb", + "spec": "ncurses/y43rifz" + }, + { + "depends": "pkgconf/eg355zb", + "spec": "readline/ip6aiun" + } + ], + "specs": [ + { + "root_spec": "readline@7.0%clang@9.1.0-apple arch=darwin-...", + "spec": "readline@7.0%clang@9.1.0-apple arch=darwin-highs...", + "label": "readline/ip6aiun" + }, + { + "root_spec": "readline@7.0%clang@9.1.0-apple arch=darwin-...", + "spec": "ncurses@6.1%clang@9.1.0-apple arch=darwin-highsi...", + "label": "ncurses/y43rifz" + }, + { + "root_spec": "readline@7.0%clang@9.1.0-apple arch=darwin-...", + "spec": "pkgconf@1.5.4%clang@9.1.0-apple arch=darwin-high...", + "label": "pkgconf/eg355zb" + } + ] + } + + The object can be optionally written out to some stream. This is + useful, for example, when we need to concretize and generate the + dependencies of a spec in a specific docker container. + + """ + deptype = all_deptypes + spec_labels = {} + + specs = [] + dependencies = [] + + def append_dep(s, d): + dependencies.append({ + 'spec': s, + 'depends': d, + }) + + for spec in spec_list: + spec.concretize() + + root_spec = get_spec_string(spec) + + rkey, rlabel = spec_deps_key_label(spec) + + for s in spec.traverse(deptype=deptype): + skey, slabel = spec_deps_key_label(s) + spec_labels[slabel] = { + 'spec': get_spec_string(s), + 'root': root_spec, + } + append_dep(rlabel, slabel) + + for d in s.dependencies(deptype=deptype): + dkey, dlabel = spec_deps_key_label(d) + append_dep(slabel, dlabel) + + for l, d in spec_labels.items(): + specs.append({ + 'label': l, + 'spec': d['spec'], + 'root_spec': d['root'], + }) + + deps_json_obj = { + 'specs': specs, + 'dependencies': dependencies, + } + + if stream_like: + stream_like.write(json.dumps(deps_json_obj)) + + return deps_json_obj + + +def release_jobs(parser, args): + share_path = os.path.join(spack_root, 'share', 'spack', 'docker') + os_container_mapping_path = os.path.join( + share_path, 'os-container-mapping.yaml') + + with open(os_container_mapping_path, 'r') as fin: + os_container_mapping = syaml.load(fin) + + try: + validate(os_container_mapping, mapping_schema) + except ValidationError as val_err: + tty.error('Ill-formed os-container-mapping configuration object') + tty.error(os_container_mapping) + tty.debug(val_err) + return + + containers = os_container_mapping['containers'] + + if args.specs: + # Just print out the spec labels and all dependency edges in + # a json format. + spec_list = [Spec(s) for s in args.specs] + with open(args.specs_deps_output, 'w') as out: + compute_spec_deps(spec_list, out) + return + + current_system = sys_type() if args.resolve_deps_locally else None + + release_specs_path = args.spec_set + if not release_specs_path: + raise SpackError('Must provide path to release spec-set') + + release_spec_set = CombinatorialSpecSet.from_file(release_specs_path) + + mirror_url = args.mirror_url + + if not mirror_url: + raise SpackError('Must provide url of target binary mirror') + + cdash_url = args.cdash_url + + spec_labels, dependencies, stages = stage_spec_jobs( + release_spec_set, containers, current_system) + + if not stages: + tty.msg('No jobs staged, exiting.') + return + + if args.print_summary: + print_staging_summary(spec_labels, dependencies, stages) + + output_object = {} + job_count = 0 + + stage_names = ['stage-{0}'.format(i) for i in range(len(stages))] + stage = 0 + + for stage_jobs in stages: + stage_name = stage_names[stage] + + for spec_label in stage_jobs: + release_spec = spec_labels[spec_label]['spec'] + root_spec = spec_labels[spec_label]['rootSpec'] + + pkg_compiler = release_spec.compiler + pkg_hash = release_spec.dag_hash() + + osname = str(release_spec.architecture) + job_name = get_job_name(release_spec, osname) + container_info = containers[osname] + build_image = container_info['image'] + + job_scripts = ['./bin/rebuild-package.sh'] + + if 'setup_script' in container_info: + job_scripts.insert( + 0, container_info['setup_script'] % pkg_compiler) + + job_dependencies = [] + if spec_label in dependencies: + job_dependencies = ( + [get_job_name(spec_labels[dep_label]['spec'], osname) + for dep_label in dependencies[spec_label]]) + + job_object = { + 'stage': stage_name, + 'variables': { + 'MIRROR_URL': mirror_url, + 'CDASH_BASE_URL': cdash_url, + 'HASH': pkg_hash, + 'DEPENDENCIES': ';'.join(job_dependencies), + 'ROOT_SPEC': str(root_spec), + }, + 'script': job_scripts, + 'image': build_image, + 'artifacts': { + 'paths': [ + 'local_mirror/build_cache', + 'jobs_scratch_dir', + 'cdash_report', + ], + 'when': 'always', + }, + 'dependencies': job_dependencies, + } + + # If we see 'compilers' in the container iformation, it's a + # filter for the compilers this container can handle, else we + # assume it can handle any compiler + if 'compilers' in container_info: + do_job = False + for item in container_info['compilers']: + container_compiler_spec = CompilerSpec(item['name']) + if pkg_compiler == container_compiler_spec: + do_job = True + else: + do_job = True + + if args.shared_runner_tag: + job_object['tags'] = [args.shared_runner_tag] + + if args.signing_key: + job_object['variables']['SIGN_KEY_HASH'] = args.signing_key + + if do_job: + output_object[job_name] = job_object + job_count += 1 + + stage += 1 + + tty.msg('{0} build jobs generated in {1} stages'.format( + job_count, len(stages))) + + final_stage = 'stage-rebuild-index' + + final_job = { + 'stage': final_stage, + 'variables': { + 'MIRROR_URL': mirror_url, + }, + 'image': build_image, + 'script': './bin/rebuild-index.sh', + } + + if args.shared_runner_tag: + final_job['tags'] = [args.shared_runner_tag] + + output_object['rebuild-index'] = final_job + stage_names.append(final_stage) + output_object['stages'] = stage_names + + with open(args.output_file, 'w') as outf: + outf.write(syaml.dump(output_object)) diff --git a/lib/spack/spack/schema/os_container_mapping.py b/lib/spack/spack/schema/os_container_mapping.py new file mode 100644 index 0000000000..0476e98945 --- /dev/null +++ b/lib/spack/spack/schema/os_container_mapping.py @@ -0,0 +1,50 @@ +# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +"""Schema for os-container-mapping.yaml configuration file. + +.. literalinclude:: ../spack/schema/os_container_mapping.py + :lines: 32- +""" + + +schema = { + '$schema': 'http://json-schema.org/schema#', + 'title': 'Spack release builds os/container mapping config file schema', + 'type': 'object', + 'additionalProperties': False, + 'patternProperties': { + r'containers': { + 'type': 'object', + 'default': {}, + 'patternProperties': { + r'[\w\d\-_\.]+': { + 'type': 'object', + 'default': {}, + 'additionalProperties': False, + 'required': ['image'], + 'properties': { + 'image': {'type': 'string'}, + 'setup_script': {'type': 'string'}, + 'compilers': { + 'type': 'array', + 'default': [], + 'items': { + 'type': 'object', + 'default': {}, + 'additionalProperties': False, + 'required': ['name'], + 'properties': { + 'name': {'type': 'string'}, + 'path': {'type': 'string'}, + }, + }, + }, + }, + }, + }, + }, + }, +} diff --git a/lib/spack/spack/schema/specs_deps.py b/lib/spack/spack/schema/specs_deps.py new file mode 100644 index 0000000000..c6981837ee --- /dev/null +++ b/lib/spack/spack/schema/specs_deps.py @@ -0,0 +1,48 @@ +# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +"""Schema for expressing dependencies of a set of specs in a JSON file + +.. literalinclude:: ../spack/schema/specs_deps.py + :lines: 32- +""" + + +schema = { + '$schema': 'http://json-schema.org/schema#', + 'title': 'Spack schema for the dependencies of a set of specs', + 'type': 'object', + 'additionalProperties': False, + 'required': ['specs'], + 'properties': { + r'dependencies': { + 'type': 'array', + 'default': [], + 'items': { + 'type': 'object', + 'additionalProperties': False, + 'required': ['depends', 'spec'], + 'properties': { + r'depends': {'type': 'string'}, + r'spec': {'type': 'string'}, + }, + }, + }, + r'specs': { + 'type': 'array', + 'default': [], + 'items': { + 'type': 'object', + 'additionalProperties': False, + 'required': ['root_spec', 'spec', 'label'], + 'properties': { + r'root_spec': {'type': 'string'}, + r'spec': {'type': 'string'}, + r'label': {'type': 'string'}, + } + }, + }, + }, +} diff --git a/lib/spack/spack/test/cmd/release_jobs.py b/lib/spack/spack/test/cmd/release_jobs.py new file mode 100644 index 0000000000..7768b7d8c1 --- /dev/null +++ b/lib/spack/spack/test/cmd/release_jobs.py @@ -0,0 +1,118 @@ +# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import json + +from jsonschema import validate + +from spack import repo +from spack.architecture import sys_type +from spack.cmd.release_jobs import stage_spec_jobs, spec_deps_key_label +from spack.main import SpackCommand +from spack.schema.specs_deps import schema as specs_deps_schema +from spack.spec import Spec +from spack.test.conftest import MockPackage, MockPackageMultiRepo + + +release_jobs = SpackCommand('release-jobs') + + +def test_specs_deps(tmpdir, config): + """If we ask for the specs dependencies to be written to disk, then make + sure we get a file of the correct format.""" + + output_path = str(tmpdir.mkdir('json').join('spec_deps.json')) + release_jobs('--specs-deps-output', output_path, 'readline') + + deps_object = None + + with open(output_path, 'r') as fd: + deps_object = json.loads(fd.read()) + + assert (deps_object is not None) + + validate(deps_object, specs_deps_schema) + + +def test_specs_staging(config): + """Make sure we achieve the best possible staging for the following +spec DAG:: + + a + /| + c b + |\ + e d + |\ + f g + +In this case, we would expect 'c', 'e', 'f', and 'g' to be in the first stage, +and then 'd', 'b', and 'a' to be put in the next three stages, respectively. + +""" + current_system = sys_type() + + config_compilers = config.get_config('compilers') + first_compiler = config_compilers[0] + compiler_spec = first_compiler['compiler']['spec'] + + # Whatever that first compiler in the configuration was, let's make sure + # we mock up an entry like we'd find in os-container-mapping.yaml which + # has that compiler. + mock_containers = {} + mock_containers[current_system] = { + "image": "dontcare", + "compilers": [ + { + "name": compiler_spec, + } + ], + } + + default = ('build', 'link') + + g = MockPackage('g', [], []) + f = MockPackage('f', [], []) + e = MockPackage('e', [], []) + d = MockPackage('d', [f, g], [default, default]) + c = MockPackage('c', [], []) + b = MockPackage('b', [d, e], [default, default]) + a = MockPackage('a', [b, c], [default, default]) + + mock_repo = MockPackageMultiRepo([a, b, c, d, e, f, g]) + + with repo.swap(mock_repo): + # Now we'll ask for the root package to be compiled with whatever that + # first compiler in the configuration was. + spec_a = Spec('a%{0}'.format(compiler_spec)) + spec_a.concretize() + + spec_a_label = spec_deps_key_label(spec_a)[1] + spec_b_label = spec_deps_key_label(spec_a['b'])[1] + spec_c_label = spec_deps_key_label(spec_a['c'])[1] + spec_d_label = spec_deps_key_label(spec_a['d'])[1] + spec_e_label = spec_deps_key_label(spec_a['e'])[1] + spec_f_label = spec_deps_key_label(spec_a['f'])[1] + spec_g_label = spec_deps_key_label(spec_a['g'])[1] + + spec_labels, dependencies, stages = stage_spec_jobs( + [spec_a], mock_containers, current_system) + + assert (len(stages) == 4) + + assert (len(stages[0]) == 4) + assert (spec_c_label in stages[0]) + assert (spec_e_label in stages[0]) + assert (spec_f_label in stages[0]) + assert (spec_g_label in stages[0]) + + assert (len(stages[1]) == 1) + assert (spec_d_label in stages[1]) + + assert (len(stages[2]) == 1) + assert (spec_b_label in stages[2]) + + assert (len(stages[3]) == 1) + assert (spec_a_label in stages[3]) -- cgit v1.2.3-70-g09d2