diff options
| -rw-r--r-- | lib/spack/spack/ci.py | 39 | ||||
| -rw-r--r-- | lib/spack/spack/test/ci.py | 41 | ||||
| -rw-r--r-- | lib/spack/spack/test/cmd/ci.py | 91 |
3 files changed, 149 insertions, 22 deletions
diff --git a/lib/spack/spack/ci.py b/lib/spack/spack/ci.py index 13b97c0e0a..8baea51d75 100644 --- a/lib/spack/spack/ci.py +++ b/lib/spack/spack/ci.py @@ -490,16 +490,28 @@ def compute_affected_packages(rev1="HEAD^", rev2="HEAD"): return spack.repo.get_all_package_diffs("ARC", rev1=rev1, rev2=rev2) -def get_spec_filter_list(env, affected_pkgs): +def get_spec_filter_list(env, affected_pkgs, dependent_traverse_depth=None): """Given a list of package names and an active/concretized environment, return the set of all concrete specs from the environment that could have been affected by changing the list of packages. + If a ``dependent_traverse_depth`` is given, it is used to limit + upward (in the parent direction) traversal of specs of touched + packages. E.g. if 1 is provided, then only direct dependents + of touched package specs are traversed to produce specs that + could have been affected by changing the package, while if 0 is + provided, only the changed specs themselves are traversed. If ``None`` + is given, upward traversal of touched package specs is done all + the way to the environment roots. Providing a negative number + results in no traversals at all, yielding an empty set. + Arguments: env (spack.environment.Environment): Active concrete environment affected_pkgs (List[str]): Affected package names + dependent_traverse_depth: Optional integer to limit dependent + traversal, or None to disable the limit. Returns: @@ -516,10 +528,11 @@ def get_spec_filter_list(env, affected_pkgs): visited = set() dag_hash = lambda s: s.dag_hash() for match in env_matches: - for parent in match.traverse(direction="parents", key=dag_hash): - affected_specs.update( - parent.traverse(direction="children", visited=visited, key=dag_hash) - ) + for dep_level, parent in match.traverse(direction="parents", key=dag_hash, depth=True): + if dependent_traverse_depth is None or dep_level <= dependent_traverse_depth: + affected_specs.update( + parent.traverse(direction="children", visited=visited, key=dag_hash) + ) return affected_specs @@ -580,6 +593,18 @@ def generate_gitlab_ci_yaml( cdash_handler = CDashHandler(yaml_root.get("cdash")) if "cdash" in yaml_root else None build_group = cdash_handler.build_group if cdash_handler else None + dependent_depth = os.environ.get("SPACK_PRUNE_UNTOUCHED_DEPENDENT_DEPTH", None) + if dependent_depth is not None: + try: + dependent_depth = int(dependent_depth) + except (TypeError, ValueError): + tty.warn( + "Unrecognized value ({0}) ".format(dependent_depth), + "provide forSPACK_PRUNE_UNTOUCHED_DEPENDENT_DEPTH, ", + "ignoring it.", + ) + dependent_depth = None + prune_untouched_packages = False spack_prune_untouched = os.environ.get("SPACK_PRUNE_UNTOUCHED", None) if spack_prune_untouched is not None and spack_prune_untouched.lower() == "true": @@ -595,7 +620,9 @@ def generate_gitlab_ci_yaml( tty.debug("affected pkgs:") for p in affected_pkgs: tty.debug(" {0}".format(p)) - affected_specs = get_spec_filter_list(env, affected_pkgs) + affected_specs = get_spec_filter_list( + env, affected_pkgs, dependent_traverse_depth=dependent_depth + ) tty.debug("all affected specs:") for s in affected_specs: tty.debug(" {0}/{1}".format(s.name, s.dag_hash()[:7])) diff --git a/lib/spack/spack/test/ci.py b/lib/spack/spack/test/ci.py index 7d49c15c8d..7406f171da 100644 --- a/lib/spack/spack/test/ci.py +++ b/lib/spack/spack/test/ci.py @@ -408,19 +408,36 @@ def test_get_spec_filter_list(mutable_mock_env_path, config, mutable_mock_repo): touched = ["libdwarf"] - # traversing both directions from libdwarf in the graphs depicted - # above (and additionally including dependencies of dependents of - # libdwarf) results in the following possibly affected env specs: - # mpileaks, callpath, dyninst, libdwarf, libelf, and mpich. - # Unaffected specs are hypre and it's dependencies. - - affected_specs = ci.get_spec_filter_list(e1, touched) - affected_pkg_names = set([s.name for s in affected_specs]) - expected_affected_pkg_names = set( - ["mpileaks", "mpich", "callpath", "dyninst", "libdwarf", "libelf"] - ) + # Make sure we return the correct set of possibly affected specs, + # given a dependent traversal depth and the fact that the touched + # package is libdwarf. Passing traversal depth of None or something + # equal to or larger than the greatest depth in the graph are + # equivalent and result in traversal of all specs from the touched + # package to the root. Passing negative traversal depth results in + # no spec traversals. Passing any other number yields differing + # numbers of possibly affected specs. + + full_set = set(["mpileaks", "mpich", "callpath", "dyninst", "libdwarf", "libelf"]) + empty_set = set([]) + depth_2_set = set(["mpich", "callpath", "dyninst", "libdwarf", "libelf"]) + depth_1_set = set(["dyninst", "libdwarf", "libelf"]) + depth_0_set = set(["libdwarf", "libelf"]) + + expectations = { + None: full_set, + 3: full_set, + 100: full_set, + -1: empty_set, + 0: depth_0_set, + 1: depth_1_set, + 2: depth_2_set, + } - assert affected_pkg_names == expected_affected_pkg_names + for key, val in expectations.items(): + affected_specs = ci.get_spec_filter_list(e1, touched, dependent_traverse_depth=key) + affected_pkg_names = set([s.name for s in affected_specs]) + print(f"{key}: {affected_pkg_names}") + assert affected_pkg_names == val @pytest.mark.regression("29947") diff --git a/lib/spack/spack/test/cmd/ci.py b/lib/spack/spack/test/cmd/ci.py index 5a05aa1d94..4c39dc2869 100644 --- a/lib/spack/spack/test/cmd/ci.py +++ b/lib/spack/spack/test/cmd/ci.py @@ -1755,6 +1755,12 @@ spack: mirror_url ) ) + + # Dependency graph rooted at callpath + # callpath -> dyninst -> libelf + # -> libdwarf -> libelf + # -> mpich + with tmpdir.as_cwd(): env_cmd("create", "test", "./spack.yaml") outputfile = str(tmpdir.join(".gitlab-ci.yml")) @@ -1765,19 +1771,96 @@ spack: def fake_stack_changed(env_path, rev1="HEAD^", rev2="HEAD"): return False - with ev.read("test"): + env_hashes = {} + + with ev.read("test") as active_env: monkeypatch.setattr(ci, "compute_affected_packages", fake_compute_affected) monkeypatch.setattr(ci, "get_stack_changed", fake_stack_changed) + + active_env.concretize() + + for s in active_env.all_specs(): + env_hashes[s.name] = s.dag_hash() + ci_cmd("generate", "--output-file", outputfile) with open(outputfile) as f: contents = f.read() + print(contents) yaml_contents = syaml.load(contents) + generated_hashes = [] + for ci_key in yaml_contents.keys(): - if "archive-files" in ci_key: - print("Error: archive-files should have been pruned") - assert False + if ci_key.startswith("(specs)"): + generated_hashes.append( + yaml_contents[ci_key]["variables"]["SPACK_JOB_SPEC_DAG_HASH"] + ) + + assert env_hashes["archive-files"] not in generated_hashes + for spec_name in ["callpath", "dyninst", "mpich", "libdwarf", "libelf"]: + assert env_hashes[spec_name] in generated_hashes + + +def test_ci_generate_prune_env_vars( + tmpdir, mutable_mock_env_path, install_mockery, mock_packages, ci_base_environment, monkeypatch +): + """Make sure environment variables controlling untouched spec + pruning behave as expected.""" + os.environ.update( + { + "SPACK_PRUNE_UNTOUCHED": "TRUE", # enables pruning of untouched specs + } + ) + filename = str(tmpdir.join("spack.yaml")) + with open(filename, "w") as f: + f.write( + """\ +spack: + specs: + - libelf + gitlab-ci: + mappings: + - match: + - arch=test-debian6-core2 + runner-attributes: + tags: + - donotcare + image: donotcare +""" + ) + + with tmpdir.as_cwd(): + env_cmd("create", "test", "./spack.yaml") + + def fake_compute_affected(r1=None, r2=None): + return ["libdwarf"] + + def fake_stack_changed(env_path, rev1="HEAD^", rev2="HEAD"): + return False + + expected_depth_param = None + + def check_get_spec_filter_list(env, affected_pkgs, dependent_traverse_depth=None): + assert dependent_traverse_depth == expected_depth_param + return set() + + monkeypatch.setattr(ci, "compute_affected_packages", fake_compute_affected) + monkeypatch.setattr(ci, "get_stack_changed", fake_stack_changed) + monkeypatch.setattr(ci, "get_spec_filter_list", check_get_spec_filter_list) + + expectations = {"-1": -1, "0": 0, "True": None} + + for key, val in expectations.items(): + with ev.read("test"): + os.environ.update({"SPACK_PRUNE_UNTOUCHED_DEPENDENT_DEPTH": key}) + expected_depth_param = val + # Leaving out the mirror in the spack.yaml above means the + # pipeline generation command will fail, pretty much immediately. + # But for this test, we only care how the environment variables + # for pruning are handled, the faster the better. So allow the + # spack command to fail. + ci_cmd("generate", fail_on_error=False) def test_ci_subcommands_without_mirror( |