summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorOmar Padron <omar.padron@kitware.com>2020-06-25 14:27:20 -0400
committerGitHub <noreply@github.com>2020-06-25 14:27:20 -0400
commit7c54aa2eb088398d626b8ce4354623d62f3a4638 (patch)
tree7f33430924096b93bfbd2a52f45a3e5745321b26
parent096bd69a949225590feeaeaf16545d673c4cee6d (diff)
downloadspack-7c54aa2eb088398d626b8ce4354623d62f3a4638.tar.gz
spack-7c54aa2eb088398d626b8ce4354623d62f3a4638.tar.bz2
spack-7c54aa2eb088398d626b8ce4354623d62f3a4638.tar.xz
spack-7c54aa2eb088398d626b8ce4354623d62f3a4638.zip
add workaround for gitlab ci needs limit (#17219)
* add workaround for gitlab ci needs limit * fix style/address review comments * convert filter obj to list * update command completion * remove dict comprehension * add workaround tests * fix sorting issue between disparate types * add indeces to format
-rw-r--r--lib/spack/spack/ci.py7
-rw-r--r--lib/spack/spack/ci_needs_workaround.py47
-rw-r--r--lib/spack/spack/ci_optimization.py8
-rw-r--r--lib/spack/spack/cmd/ci.py10
-rw-r--r--lib/spack/spack/test/ci.py170
-rwxr-xr-xshare/spack/spack-completion.bash2
6 files changed, 236 insertions, 8 deletions
diff --git a/lib/spack/spack/ci.py b/lib/spack/spack/ci.py
index 230f25ccee..580dbf29d2 100644
--- a/lib/spack/spack/ci.py
+++ b/lib/spack/spack/ci.py
@@ -450,7 +450,7 @@ def format_job_needs(phase_name, strip_compilers, dep_jobs,
def generate_gitlab_ci_yaml(env, print_summary, output_file,
custom_spack_repo=None, custom_spack_ref=None,
- run_optimizer=False):
+ run_optimizer=False, use_dependencies=False):
# FIXME: What's the difference between one that opens with 'spack'
# and one that opens with 'env'? This will only handle the former.
with spack.concretize.disable_compiler_existence_check():
@@ -794,6 +794,11 @@ def generate_gitlab_ci_yaml(env, print_summary, output_file,
import spack.ci_optimization as ci_opt
sorted_output = ci_opt.optimizer(sorted_output)
+ # TODO(opadron): remove this or refactor
+ if use_dependencies:
+ import spack.ci_needs_workaround as cinw
+ sorted_output = cinw.needs_to_dependencies(sorted_output)
+
with open(output_file, 'w') as outf:
outf.write(syaml.dump_config(sorted_output, default_flow_style=True))
diff --git a/lib/spack/spack/ci_needs_workaround.py b/lib/spack/spack/ci_needs_workaround.py
new file mode 100644
index 0000000000..ba9f7c62af
--- /dev/null
+++ b/lib/spack/spack/ci_needs_workaround.py
@@ -0,0 +1,47 @@
+# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+import collections
+
+try:
+ # dynamically import to keep vermin from complaining
+ collections_abc = __import__('collections.abc')
+except ImportError:
+ collections_abc = collections
+
+
+get_job_name = lambda needs_entry: (
+ needs_entry.get('job') if (
+ isinstance(needs_entry, collections_abc.Mapping) and
+ needs_entry.get('artifacts', True))
+
+ else
+
+ needs_entry if isinstance(needs_entry, str)
+
+ else None)
+
+
+def convert_job(job_entry):
+ if not isinstance(job_entry, collections_abc.Mapping):
+ return job_entry
+
+ needs = job_entry.get('needs')
+ if needs is None:
+ return job_entry
+
+ new_job = {}
+ new_job.update(job_entry)
+ del new_job['needs']
+
+ new_job['dependencies'] = list(filter(
+ (lambda x: x is not None),
+ (get_job_name(needs_entry) for needs_entry in needs)))
+
+ return new_job
+
+
+def needs_to_dependencies(yaml):
+ return dict((k, convert_job(v)) for k, v in yaml.items())
diff --git a/lib/spack/spack/ci_optimization.py b/lib/spack/spack/ci_optimization.py
index 693802d06d..d938a681e3 100644
--- a/lib/spack/spack/ci_optimization.py
+++ b/lib/spack/spack/ci_optimization.py
@@ -190,10 +190,10 @@ def print_delta(name, old, new, applied=None):
applied = (new <= old)
print('\n'.join((
- '{} {}:',
- ' before: {: 10d}',
- ' after : {: 10d}',
- ' delta : {:+10d} ({:=+3d}.{}%)',
+ '{0} {1}:',
+ ' before: {2: 10d}',
+ ' after : {3: 10d}',
+ ' delta : {4:+10d} ({5:=+3d}.{6}%)',
)).format(
name,
('+' if applied else 'x'),
diff --git a/lib/spack/spack/cmd/ci.py b/lib/spack/spack/cmd/ci.py
index 3e57c6656a..d42a3e7ae6 100644
--- a/lib/spack/spack/cmd/ci.py
+++ b/lib/spack/spack/cmd/ci.py
@@ -55,10 +55,14 @@ def setup_parser(subparser):
"should be checked out as a step in each generated job. " +
"This argument is ignored if no --spack-repo is provided.")
generate.add_argument(
- '--optimize', action='store_true',
+ '--optimize', action='store_true', default=False,
help="(Experimental) run the generated document through a series of "
"optimization passes designed to reduce the size of the "
"generated file.")
+ generate.add_argument(
+ '--dependencies', action='store_true', default=False,
+ help="(Experimental) disable DAG scheduling; use "
+ ' "plain" dependencies.')
generate.set_defaults(func=ci_generate)
# Check a spec against mirror. Rebuild, create buildcache and push to
@@ -81,6 +85,7 @@ def ci_generate(args):
spack_repo = args.spack_repo
spack_ref = args.spack_ref
run_optimizer = args.optimize
+ use_dependencies = args.dependencies
if not output_file:
gen_ci_dir = os.getcwd()
@@ -93,7 +98,8 @@ def ci_generate(args):
# Generate the jobs
spack_ci.generate_gitlab_ci_yaml(
env, True, output_file, spack_repo, spack_ref,
- run_optimizer=run_optimizer)
+ run_optimizer=run_optimizer,
+ use_dependencies=use_dependencies)
if copy_yaml_to:
copy_to_dir = os.path.dirname(copy_yaml_to)
diff --git a/lib/spack/spack/test/ci.py b/lib/spack/spack/test/ci.py
index a153437ffa..9b3c48654a 100644
--- a/lib/spack/spack/test/ci.py
+++ b/lib/spack/spack/test/ci.py
@@ -15,6 +15,17 @@ import spack.spec as spec
import spack.util.web as web_util
import spack.util.gpg
+import spack.ci_optimization as ci_opt
+import spack.ci_needs_workaround as cinw
+import spack.util.spack_yaml as syaml
+import itertools as it
+import collections
+try:
+ # dynamically import to keep vermin from complaining
+ collections_abc = __import__('collections.abc')
+except ImportError:
+ collections_abc = collections
+
@pytest.fixture
def tmp_scope():
@@ -166,3 +177,162 @@ def test_read_write_cdash_ids(config, tmp_scope, tmpdir, mock_packages):
read_cdashid = ci.read_cdashid_from_mirror(mock_spec, mirror_url)
assert(str(read_cdashid) == orig_cdashid)
+
+
+def test_ci_workarounds():
+ fake_root_spec = 'x' * 544
+ fake_spack_ref = 'x' * 40
+
+ common_variables = {
+ 'SPACK_COMPILER_ACTION': 'NONE',
+ 'SPACK_IS_PR_PIPELINE': 'False',
+ }
+
+ common_script = ['spack ci rebuild']
+
+ common_before_script = [
+ 'git clone "https://github.com/spack/spack"',
+ ' && '.join((
+ 'pushd ./spack',
+ 'git checkout "{ref}"'.format(ref=fake_spack_ref),
+ 'popd')),
+ '. "./spack/share/spack/setup-env.sh"'
+ ]
+
+ def make_build_job(name, deps, stage, use_artifact_buildcache, optimize,
+ use_dependencies):
+ variables = common_variables.copy()
+ variables['SPACK_JOB_SPEC_PKG_NAME'] = name
+
+ result = {
+ 'stage': stage,
+ 'tags': ['tag-0', 'tag-1'],
+ 'artifacts': {
+ 'paths': [
+ 'jobs_scratch_dir',
+ 'cdash_report',
+ name + '.spec.yaml',
+ name + '.cdashid',
+ name
+ ],
+ 'when': 'always'
+ },
+ 'retry': {'max': 2, 'when': ['always']},
+ 'after_script': ['rm -rf "./spack"'],
+ 'image': {'name': 'spack/centos7', 'entrypoint': ['']},
+ }
+
+ if optimize:
+ result['extends'] = ['.c0', '.c1', '.c2']
+ else:
+ variables['SPACK_ROOT_SPEC'] = fake_root_spec
+ result['script'] = common_script
+ result['before_script'] = common_before_script
+
+ result['variables'] = variables
+
+ if use_dependencies:
+ result['dependencies'] = (
+ list(deps) if use_artifact_buildcache
+ else [])
+ else:
+ result['needs'] = [
+ {'job': dep, 'artifacts': use_artifact_buildcache}
+ for dep in deps]
+
+ return {name: result}
+
+ def make_rebuild_index_job(
+ use_artifact_buildcache, optimize, use_dependencies):
+
+ result = {
+ 'stage': 'stage-rebuild-index',
+ 'script': 'spack buildcache update-index -d s3://mirror',
+ 'tags': ['tag-0', 'tag-1'],
+ 'image': {'name': 'spack/centos7', 'entrypoint': ['']},
+ 'after_script': ['rm -rf "./spack"'],
+ }
+
+ if optimize:
+ result['extends'] = '.c1'
+ else:
+ result['before_script'] = common_before_script
+
+ return {'rebuild-index': result}
+
+ def make_factored_jobs(optimize):
+ return {
+ '.c0': {'script': common_script},
+ '.c1': {'before_script': common_before_script},
+ '.c2': {'variables': {'SPACK_ROOT_SPEC': fake_root_spec}}
+ } if optimize else {}
+
+ def make_yaml_obj(use_artifact_buildcache, optimize, use_dependencies):
+ result = {}
+
+ result.update(make_build_job(
+ 'pkg-a', [], 'stage-0', use_artifact_buildcache, optimize,
+ use_dependencies))
+
+ result.update(make_build_job(
+ 'pkg-b', ['pkg-a'], 'stage-1', use_artifact_buildcache, optimize,
+ use_dependencies))
+
+ result.update(make_build_job(
+ 'pkg-c', ['pkg-a', 'pkg-b'], 'stage-2', use_artifact_buildcache,
+ optimize, use_dependencies))
+
+ result.update(make_rebuild_index_job(
+ use_artifact_buildcache, optimize, use_dependencies))
+
+ result.update(make_factored_jobs(optimize))
+
+ return result
+
+ def sort_yaml_obj(obj):
+ if isinstance(obj, collections_abc.Mapping):
+ result = syaml.syaml_dict()
+ for k in sorted(obj.keys(), key=str):
+ result[k] = sort_yaml_obj(obj[k])
+ return result
+
+ if (isinstance(obj, collections_abc.Sequence) and
+ not isinstance(obj, str)):
+ return syaml.syaml_list(sorted(
+ (sort_yaml_obj(x) for x in obj), key=str))
+
+ return obj
+
+ # test every combination of:
+ # use artifact buildcache: true or false
+ # run optimization pass: true or false
+ # convert needs to dependencies: true or false
+ for use_ab in (False, True):
+ original = make_yaml_obj(
+ use_artifact_buildcache=use_ab,
+ optimize=False,
+ use_dependencies=False)
+
+ for opt, deps in it.product(*(((False, True),) * 2)):
+ # neither optimizing nor converting needs->dependencies
+ if not (opt or deps):
+ # therefore, nothing to test
+ continue
+
+ predicted = make_yaml_obj(
+ use_artifact_buildcache=use_ab,
+ optimize=opt,
+ use_dependencies=deps)
+
+ actual = original.copy()
+ if opt:
+ actual = ci_opt.optimizer(actual)
+ if deps:
+ actual = cinw.needs_to_dependencies(actual)
+
+ predicted = syaml.dump_config(
+ sort_yaml_obj(predicted), default_flow_style=True)
+ actual = syaml.dump_config(
+ sort_yaml_obj(actual), default_flow_style=True)
+
+ assert(predicted == actual)
diff --git a/share/spack/spack-completion.bash b/share/spack/spack-completion.bash
index ffd7596416..eff9e15b2c 100755
--- a/share/spack/spack-completion.bash
+++ b/share/spack/spack-completion.bash
@@ -474,7 +474,7 @@ _spack_ci() {
}
_spack_ci_generate() {
- SPACK_COMPREPLY="-h --help --output-file --copy-to --spack-repo --spack-ref --optimize"
+ SPACK_COMPREPLY="-h --help --output-file --copy-to --spack-repo --spack-ref --optimize --dependencies"
}
_spack_ci_rebuild() {