Ci generate on change (#47318)

* don't concretize in CI if changed packages are not in stacks

Signed-off-by: Todd Gamblin <tgamblin@llnl.gov>

* Generate noop job when no specs to rebuild due to untouched pruning

* Add test to verify skipping generate creates a noop job

* Changed debug for early exit

---------

Signed-off-by: Todd Gamblin <tgamblin@llnl.gov>
Co-authored-by: Todd Gamblin <tgamblin@llnl.gov>

2 files changed, 109 insertions, 27 deletions

diff --git a/lib/spack/spack/ci.py b/lib/spack/spack/ci.py
index 5a8b2ae1e7..8a23bc3ae1 100644
--- a/lib/spack/spack/ci.py
+++ b/lib/spack/spack/ci.py
@@ -37,6 +37,7 @@ import spack.config as cfg
 import spack.error
 import spack.main
 import spack.mirror
+import spack.package_base
 import spack.paths
 import spack.repo
 import spack.spec
@@ -264,14 +265,22 @@ def _format_job_needs(dep_jobs, build_group, prune_dag, rebuild_decisions):
 def get_change_revisions():
     """If this is a git repo get the revisions to use when checking
     for changed packages and spack core modules."""
+    rev1 = None
+    rev2 = None
+
+    # Note: git_dir may be a file in a worktree. If it exists, attempt to use git
+    #       to determine if there is a revision
     git_dir = os.path.join(spack.paths.prefix, ".git")
-    if os.path.exists(git_dir) and os.path.isdir(git_dir):
-        # TODO: This will only find changed packages from the last
-        # TODO: commit.  While this may work for single merge commits
-        # TODO: when merging the topic branch into the base, it will
-        # TODO: require more thought outside of that narrow case.
-        return "HEAD^", "HEAD"
-    return None, None
+    if os.path.exists(git_dir):
+        # The default will only find changed packages from the last
+        # commit. When the commit is a merge commit, this is will return all of the
+        # changes on the topic.
+        # TODO: Handle the case where the clone is not shallow clone of a merge commit
+        #       using `git merge-base`
+        rev1 = "HEAD^"
+        rev2 = "HEAD"
+
+    return rev1, rev2
 
 
 def get_stack_changed(env_path, rev1="HEAD^", rev2="HEAD"):
@@ -390,7 +399,7 @@ class SpackCI:
     used by the CI generator(s).
     """
 
-    def __init__(self, ci_config, spec_labels, stages):
+    def __init__(self, ci_config, spec_labels=None, stages=None):
         """Given the information from the ci section of the config
         and the staged jobs, set up meta data needed for generating Spack
         CI IR.
@@ -408,8 +417,9 @@ class SpackCI:
         }
         jobs = self.ir["jobs"]
 
-        for spec, dag_hash in _build_jobs(spec_labels, stages):
-            jobs[dag_hash] = self.__init_job(spec)
+        if spec_labels and stages:
+            for spec, dag_hash in _build_jobs(spec_labels, stages):
+                jobs[dag_hash] = self.__init_job(spec)
 
         for name in self.named_jobs:
             # Skip the special named jobs
@@ -705,14 +715,53 @@ def generate_gitlab_ci_yaml(
             files (spack.yaml, spack.lock), etc should be written.  GitLab
             requires this to be within the project directory.
     """
+    rev1, rev2 = get_change_revisions()
+    tty.debug(f"Got following revisions: rev1={rev1}, rev2={rev2}")
+
+    # Get the joined "ci" config with all of the current scopes resolved
+    ci_config = cfg.get("ci")
+    spack_prune_untouched = os.environ.get("SPACK_PRUNE_UNTOUCHED", None)
+
+    changed = rev1 and rev2
+    affected_pkgs = None
+    if spack_prune_untouched and changed:
+        affected_pkgs = compute_affected_packages(rev1, rev2)
+        tty.debug("affected pkgs:")
+        if affected_pkgs:
+            for p in affected_pkgs:
+                tty.debug(f"  {p}")
+        else:
+            tty.debug("  no affected packages...")
+
+        possible_builds = spack.package_base.possible_dependencies(*env.user_specs)
+        changed = any((spec in p for p in possible_builds.values()) for spec in affected_pkgs)
+
+        if not changed:
+            spack_ci = SpackCI(ci_config)
+            spack_ci_ir = spack_ci.generate_ir()
+
+            # No jobs should be generated.
+            noop_job = spack_ci_ir["jobs"]["noop"]["attributes"]
+            # If this job fails ignore the status and carry on
+            noop_job["retry"] = 0
+            noop_job["allow_failure"] = True
+
+            tty.msg("Skipping concretization, generating no-op job")
+            output_object = {"no-specs-to-rebuild": noop_job}
+
+            # Ensure the child pipeline always runs
+            output_object["workflow"] = {"rules": [{"when": "always"}]}
+
+            with open(output_file, "w") as f:
+                ruamel.yaml.YAML().dump(output_object, f)
+
+            return
+
     with spack.concretize.disable_compiler_existence_check():
         with env.write_transaction():
             env.concretize()
             env.write()
 
-    # Get the joined "ci" config with all of the current scopes resolved
-    ci_config = cfg.get("ci")
-
     if not ci_config:
         raise SpackCIError("Environment does not have a `ci` configuration")
 
@@ -737,20 +786,13 @@ def generate_gitlab_ci_yaml(
             dependent_depth = None
 
     prune_untouched_packages = False
-    spack_prune_untouched = os.environ.get("SPACK_PRUNE_UNTOUCHED", None)
     if spack_prune_untouched is not None and spack_prune_untouched.lower() == "true":
         # Requested to prune untouched packages, but assume we won't do that
         # unless we're actually in a git repo.
-        rev1, rev2 = get_change_revisions()
-        tty.debug(f"Got following revisions: rev1={rev1}, rev2={rev2}")
-        if rev1 and rev2:
+        if changed:
             # If the stack file itself did not change, proceed with pruning
             if not get_stack_changed(env.manifest_path, rev1, rev2):
                 prune_untouched_packages = True
-                affected_pkgs = compute_affected_packages(rev1, rev2)
-                tty.debug("affected pkgs:")
-                for p in affected_pkgs:
-                    tty.debug(f"  {p}")
                 affected_specs = get_spec_filter_list(
                     env, affected_pkgs, dependent_traverse_depth=dependent_depth
                 )
@@ -1098,11 +1140,6 @@ def generate_gitlab_ci_yaml(
         # warn only if there was actually a CDash configuration.
         tty.warn("Unable to populate buildgroup without CDash credentials")
 
-    service_job_retries = {
-        "max": 2,
-        "when": ["runner_system_failure", "stuck_or_timeout_failure", "script_failure"],
-    }
-
     if copy_only_pipeline:
         stage_names.append("copy")
         sync_job = copy.deepcopy(spack_ci_ir["jobs"]["copy"]["attributes"])
@@ -1162,7 +1199,10 @@ def generate_gitlab_ci_yaml(
             )
 
             final_job["when"] = "always"
-            final_job["retry"] = service_job_retries
+            final_job["retry"] = {
+                "max": 2,
+                "when": ["runner_system_failure", "stuck_or_timeout_failure", "script_failure"],
+            }
             final_job["interruptible"] = True
             final_job["dependencies"] = []
 
diff --git a/lib/spack/spack/test/cmd/ci.py b/lib/spack/spack/test/cmd/ci.py
index 36aa992c63..2f0e053265 100644
--- a/lib/spack/spack/test/cmd/ci.py
+++ b/lib/spack/spack/test/cmd/ci.py
@@ -1650,3 +1650,45 @@ def test_ci_dynamic_mapping_full(
                 assert job.get("variables", {}).get("MY_VAR") == "hello"
                 assert "ignored_field" not in job
                 assert "unallowed_field" not in job
+
+
+def test_ci_generate_noop_no_concretize(
+    tmpdir,
+    working_env,
+    mutable_mock_env_path,
+    install_mockery,
+    mock_packages,
+    monkeypatch,
+    ci_base_environment,
+):
+    # Write the enviroment file
+    filename = str(tmpdir.join("spack.yaml"))
+    with open(filename, "w") as f:
+        f.write(
+            """\
+spack:
+  specs:
+    - pkg-a
+  mirrors:
+    buildcache-destination: https://my.fake.mirror
+  ci:
+    type: gitlab
+"""
+        )
+
+    def fake_compute_affected(r1=None, r2=None):
+        return []
+
+    monkeypatch.setattr(ci, "compute_affected_packages", fake_compute_affected)
+    monkeypatch.setenv("SPACK_PRUNE_UNTOUCHED", "TRUE")  # enables pruning of untouched specs
+
+    with tmpdir.as_cwd():
+        env_cmd("create", "test", "./spack.yaml")
+        outputfile = str(tmpdir.join(".gitlab-ci.yml"))
+
+        with ev.read("test"):
+            ci_cmd("generate", "--output-file", outputfile)
+
+            with open(outputfile) as of:
+                pipeline_doc = syaml.load(of.read())
+                assert "no-specs-to-rebuild" in pipeline_doc