Fix bugs in spack monitor (#27511)

Updates to installer.py did not account for spack monitor, so as currently implemented
there are three cases of failure that spack monitor will not account for. To fix this we add additional
hooks, including an on cancel and also do a custom action on concretization fail.

Signed-off-by: vsoch <vsoch@users.noreply.github.com>

Co-authored-by: vsoch <vsoch@users.noreply.github.com>

7 files changed, 77 insertions, 3 deletions

diff --git a/lib/spack/docs/analyze.rst b/lib/spack/docs/analyze.rst
index 38af77cd7f..5936a79f24 100644
--- a/lib/spack/docs/analyze.rst
+++ b/lib/spack/docs/analyze.rst
@@ -59,7 +59,7 @@ are available:
     install_files            : install file listing read from install_manifest.json
     environment_variables    : environment variables parsed from spack-build-env.txt
     config_args              : config args loaded from spack-configure-args.txt
-    abigail                  : Application Binary Interface (ABI) features for objects
+    libabigail                  : Application Binary Interface (ABI) features for objects
 
 
 In the above, the first three are fairly simple - parsing metadata files from
diff --git a/lib/spack/docs/developer_guide.rst b/lib/spack/docs/developer_guide.rst
index 4e683217b9..e62b9fae5a 100644
--- a/lib/spack/docs/developer_guide.rst
+++ b/lib/spack/docs/developer_guide.rst
@@ -671,6 +671,13 @@ If you need to write a hook that is relevant to a failure within a build
 process, you would want to instead use ``on_phase_failure``.
 
 
+"""""""""""""""""""""""""""
+``on_install_cancel(spec)``
+"""""""""""""""""""""""""""
+
+The same, but triggered if a spec install is cancelled for any reason.
+
+
 """""""""""""""""""""""""""""""""""""""""""""""
 ``on_phase_success(pkg, phase_name, log_file)``
 """""""""""""""""""""""""""""""""""""""""""""""
diff --git a/lib/spack/spack/cmd/install.py b/lib/spack/spack/cmd/install.py
index 8050d9335d..2c69069ef0 100644
--- a/lib/spack/spack/cmd/install.py
+++ b/lib/spack/spack/cmd/install.py
@@ -403,6 +403,10 @@ environment variables:
     except SpackError as e:
         tty.debug(e)
         reporter.concretization_report(e.message)
+
+        # Tell spack monitor about it
+        if args.use_monitor and abstract_specs:
+            monitor.failed_concretization(abstract_specs)
         raise
 
     # 2. Concrete specs from yaml files
diff --git a/lib/spack/spack/hooks/__init__.py b/lib/spack/spack/hooks/__init__.py
index e802fa0197..ca6a5fade9 100644
--- a/lib/spack/spack/hooks/__init__.py
+++ b/lib/spack/spack/hooks/__init__.py
@@ -90,6 +90,7 @@ on_phase_error = _HookRunner('on_phase_error')
 on_install_start = _HookRunner('on_install_start')
 on_install_success = _HookRunner('on_install_success')
 on_install_failure = _HookRunner('on_install_failure')
+on_install_cancel = _HookRunner('on_install_cancel')
 
 # Analyzer hooks
 on_analyzer_save = _HookRunner('on_analyzer_save')
diff --git a/lib/spack/spack/hooks/monitor.py b/lib/spack/spack/hooks/monitor.py
index 5ddc1223e8..9da5012593 100644
--- a/lib/spack/spack/hooks/monitor.py
+++ b/lib/spack/spack/hooks/monitor.py
@@ -41,6 +41,17 @@ def on_install_failure(spec):
     tty.verbose(result.get('message'))
 
 
+def on_install_cancel(spec):
+    """Triggered on cancel of an install
+    """
+    if not spack.monitor.cli:
+        return
+
+    tty.debug("Running on_install_cancel for %s" % spec)
+    result = spack.monitor.cli.cancel_task(spec)
+    tty.verbose(result.get('message'))
+
+
 def on_phase_success(pkg, phase_name, log_file):
     """Triggered on a phase success
     """
diff --git a/lib/spack/spack/installer.py b/lib/spack/spack/installer.py
index 196858830a..d464817f92 100644
--- a/lib/spack/spack/installer.py
+++ b/lib/spack/spack/installer.py
@@ -1198,6 +1198,7 @@ class PackageInstaller(object):
         except spack.build_environment.StopPhase as e:
             # A StopPhase exception means that do_install was asked to
             # stop early from clients, and is not an error at this point
+            spack.hooks.on_install_failure(task.request.pkg.spec)
             pid = '{0}: '.format(self.pid) if tty.show_pid() else ''
             tty.debug('{0}{1}'.format(pid, str(e)))
             tty.debug('Package stage directory: {0}' .format(pkg.stage.source_path))
@@ -1654,7 +1655,7 @@ class PackageInstaller(object):
                 err = 'Failed to install {0} due to {1}: {2}'
                 tty.error(err.format(pkg.name, exc.__class__.__name__,
                           str(exc)))
-                spack.hooks.on_install_failure(task.request.pkg.spec)
+                spack.hooks.on_install_cancel(task.request.pkg.spec)
                 raise
 
             except (Exception, SystemExit) as exc:
@@ -1918,6 +1919,9 @@ class BuildProcessInstaller(object):
                 except BaseException:
                     combine_phase_logs(pkg.phase_log_files, pkg.log_path)
                     spack.hooks.on_phase_error(pkg, phase_name, log_file)
+
+                    # phase error indicates install error
+                    spack.hooks.on_install_failure(pkg.spec)
                     raise
 
                 # We assume loggers share echo True/False
diff --git a/lib/spack/spack/monitor.py b/lib/spack/spack/monitor.py
index 54c14e905c..dd967d793d 100644
--- a/lib/spack/spack/monitor.py
+++ b/lib/spack/spack/monitor.py
@@ -118,13 +118,16 @@ class SpackMonitorClient:
 
     def __init__(self, host=None, prefix="ms1", allow_fail=False, tags=None,
                  save_local=False):
+        # We can control setting an arbitrary version if needed
+        sv = spack.main.get_version()
+        self.spack_version = os.environ.get("SPACKMON_SPACK_VERSION") or sv
+
         self.host = host or "http://127.0.0.1"
         self.baseurl = "%s/%s" % (self.host, prefix.strip("/"))
         self.token = os.environ.get("SPACKMON_TOKEN")
         self.username = os.environ.get("SPACKMON_USER")
         self.headers = {}
         self.allow_fail = allow_fail
-        self.spack_version = spack.main.get_version()
         self.capture_build_environment()
         self.tags = tags
         self.save_local = save_local
@@ -200,6 +203,14 @@ class SpackMonitorClient:
         """
         from spack.util.environment import get_host_environment_metadata
         self.build_environment = get_host_environment_metadata()
+        keys = list(self.build_environment.keys())
+
+        # Allow to customize any of these values via the environment
+        for key in keys:
+            envar_name = "SPACKMON_%s" % key.upper()
+            envar = os.environ.get(envar_name)
+            if envar:
+                self.build_environment[key] = envar
 
     def require_auth(self):
         """
@@ -413,6 +424,37 @@ class SpackMonitorClient:
 
         return configs
 
+    def failed_concretization(self, specs):
+        """
+        Given a list of abstract specs, tell spack monitor concretization failed.
+        """
+        configs = {}
+
+        # There should only be one spec generally (what cases would have >1?)
+        for spec in specs:
+
+            # update the spec to have build hash indicating that cannot be built
+            meta = spec.to_dict()['spec']
+            nodes = []
+            for node in meta.get("nodes", []):
+                for hashtype in ["build_hash", "full_hash"]:
+                    node[hashtype] = "FAILED_CONCRETIZATION"
+                nodes.append(node)
+            meta['nodes'] = nodes
+
+            # We can't concretize / hash
+            as_dict = {"spec": meta,
+                       "spack_version": self.spack_version}
+
+            if self.save_local:
+                filename = "spec-%s-%s-config.json" % (spec.name, spec.version)
+                self.save(as_dict, filename)
+            else:
+                response = self.do_request("specs/new/", data=sjson.dump(as_dict))
+                configs[spec.package.name] = response.get('data', {})
+
+        return configs
+
     def new_build(self, spec):
         """
         Create a new build.
@@ -503,6 +545,11 @@ class SpackMonitorClient:
         """
         return self.update_build(spec, status="FAILED")
 
+    def cancel_task(self, spec):
+        """Given a spec, mark it as cancelled.
+        """
+        return self.update_build(spec, status="CANCELLED")
+
     def send_analyze_metadata(self, pkg, metadata):
         """
         Send spack analyzer metadata to the spack monitor server.