diff options
Diffstat (limited to 'lib')
| -rw-r--r-- | lib/spack/docs/monitoring.rst | 16 | ||||
| -rw-r--r-- | lib/spack/spack/cmd/install.py | 1 | ||||
| -rw-r--r-- | lib/spack/spack/monitor.py | 115 | ||||
| -rw-r--r-- | lib/spack/spack/paths.py | 2 | ||||
| -rw-r--r-- | lib/spack/spack/test/monitor.py | 42 |
5 files changed, 154 insertions, 22 deletions
diff --git a/lib/spack/docs/monitoring.rst b/lib/spack/docs/monitoring.rst index b1d491c563..41a7a4a2d2 100644 --- a/lib/spack/docs/monitoring.rst +++ b/lib/spack/docs/monitoring.rst @@ -102,3 +102,19 @@ more tags to your build, you can do: # Add two tags, "pizza" and "pasta" $ spack install --monitor --monitor-tags pizza,pasta hdf5 + +------------------ +Monitoring Offline +------------------ + +In the case that you want to save monitor results to your filesystem +and then upload them later (perhaps you are in an environment where you don't +have credentials or it isn't safe to use them) you can use the ``--monitor-save-local`` +flag. + +.. code-block:: console + + $ spack install --monitor --monitor-save-local hdf5 + +This will save results in a subfolder, "monitor" in your designated spack +reports folder, which defaults to ``$HOME/.spack/reports/monitor``. diff --git a/lib/spack/spack/cmd/install.py b/lib/spack/spack/cmd/install.py index 587f039656..5a0b47215a 100644 --- a/lib/spack/spack/cmd/install.py +++ b/lib/spack/spack/cmd/install.py @@ -306,6 +306,7 @@ environment variables: prefix=args.monitor_prefix, disable_auth=args.monitor_disable_auth, tags=args.monitor_tags, + save_local=args.monitor_save_local, ) reporter = spack.report.collect_info( diff --git a/lib/spack/spack/monitor.py b/lib/spack/spack/monitor.py index 5d6246ebf1..a6ca3be316 100644 --- a/lib/spack/spack/monitor.py +++ b/lib/spack/spack/monitor.py @@ -7,6 +7,8 @@ https://github.com/spack/spack-monitor/blob/main/script/spackmoncli.py """ +from datetime import datetime +import hashlib import base64 import os import re @@ -18,11 +20,13 @@ except ImportError: from urllib2 import urlopen, Request, URLError # type: ignore # novm import spack +import spack.config import spack.hash_types as ht import spack.main import spack.store import spack.util.spack_json as sjson import spack.util.spack_yaml as syaml +import spack.util.path import llnl.util.tty as tty from copy import deepcopy @@ -31,7 +35,8 @@ from copy import deepcopy cli = None -def get_client(host, prefix="ms1", disable_auth=False, allow_fail=False, tags=None): +def get_client(host, prefix="ms1", disable_auth=False, allow_fail=False, tags=None, + save_local=False): """ Get a monitor client for a particular host and prefix. @@ -47,26 +52,25 @@ def get_client(host, prefix="ms1", disable_auth=False, allow_fail=False, tags=No """ global cli cli = SpackMonitorClient(host=host, prefix=prefix, allow_fail=allow_fail, - tags=tags) + tags=tags, save_local=save_local) # If we don't disable auth, environment credentials are required - if not disable_auth: + if not disable_auth and not save_local: cli.require_auth() - # We will exit early if the monitoring service is not running - info = cli.service_info() - - # If we allow failure, the response will be done - if info: - tty.debug("%s v.%s has status %s" % ( - info['id'], - info['version'], - info['status']) - ) - return cli - - else: - tty.debug("spack-monitor server not found, continuing as allow_fail is True.") + # We will exit early if the monitoring service is not running, but + # only if we aren't doing a local save + if not save_local: + info = cli.service_info() + + # If we allow failure, the response will be done + if info: + tty.debug("%s v.%s has status %s" % ( + info['id'], + info['version'], + info['status']) + ) + return cli def get_monitor_group(subparser): @@ -83,6 +87,9 @@ def get_monitor_group(subparser): '--monitor', action='store_true', dest='use_monitor', default=False, help="interact with a montor server during builds.") monitor_group.add_argument( + '--monitor-save-local', action='store_true', dest='monitor_save_local', + default=False, help="save monitor results to .spack instead of server.") + monitor_group.add_argument( '--monitor-no-auth', action='store_true', dest='monitor_disable_auth', default=False, help="the monitoring server does not require auth.") monitor_group.add_argument( @@ -110,7 +117,8 @@ class SpackMonitorClient: to the client on init. """ - def __init__(self, host=None, prefix="ms1", allow_fail=False, tags=None): + def __init__(self, host=None, prefix="ms1", allow_fail=False, tags=None, + save_local=False): self.host = host or "http://127.0.0.1" self.baseurl = "%s/%s" % (self.host, prefix.strip("/")) self.token = os.environ.get("SPACKMON_TOKEN") @@ -120,9 +128,34 @@ class SpackMonitorClient: self.spack_version = spack.main.get_version() self.capture_build_environment() self.tags = tags + self.save_local = save_local # We keey lookup of build_id by full_hash self.build_ids = {} + self.setup_save() + + def setup_save(self): + """Given a local save "save_local" ensure the output directory exists. + """ + if not self.save_local: + return + + save_dir = spack.util.path.canonicalize_path( + spack.config.get('config:monitor_dir', '~/.spack/reports/monitor')) + + # Name based on timestamp + now = datetime.now().strftime('%Y-%m-%d-%H-%M-%S-%s') + self.save_dir = os.path.join(save_dir, now) + if not os.path.exists(self.save_dir): + os.makedirs(self.save_dir) + + def save(self, obj, filename): + """ + Save a monitor json result to the save directory. + """ + filename = os.path.join(self.save_dir, filename) + write_json(obj, filename) + return {"message": "Build saved locally to %s" % filename} def load_build_environment(self, spec): """ @@ -174,7 +207,7 @@ class SpackMonitorClient: The token and username must not be unset """ - if not self.token or not self.username: + if not self.save_local and (not self.token or not self.username): tty.die("You are required to export SPACKMON_TOKEN and SPACKMON_USER") def set_header(self, name, value): @@ -346,8 +379,14 @@ class SpackMonitorClient: spec.concretize() as_dict = {"spec": spec.to_dict(hash=ht.full_hash), "spack_version": self.spack_version} - response = self.do_request("specs/new/", data=sjson.dump(as_dict)) - configs[spec.package.name] = response.get('data', {}) + + if self.save_local: + filename = "spec-%s-%s-config.json" % (spec.name, spec.version) + self.save(sjson.dump(as_dict), filename) + else: + response = self.do_request("specs/new/", data=sjson.dump(as_dict)) + configs[spec.package.name] = response.get('data', {}) + return configs def new_build(self, spec): @@ -384,6 +423,27 @@ class SpackMonitorClient: spec_file = os.path.join(meta_dir, "spec.yaml") data['spec'] = syaml.load(read_file(spec_file)) + if self.save_local: + return self.get_local_build_id(data, full_hash, return_response) + return self.get_server_build_id(data, full_hash, return_response) + + def get_local_build_id(self, data, full_hash, return_response): + """ + Generate a local build id based on hashing the expected data + """ + hasher = hashlib.md5() + hasher.update(str(data).encode('utf-8')) + bid = hasher.hexdigest() + filename = "build-metadata-%s.json" % full_hash + response = self.save(sjson.dump(data), filename) + if return_response: + return response + return bid + + def get_server_build_id(self, data, full_hash, return_response=False): + """ + Retrieve a build id from the spack monitor server + """ response = self.do_request("builds/new/", data=sjson.dump(data)) # Add the build id to the lookup @@ -403,6 +463,10 @@ class SpackMonitorClient: successful install. This endpoint can take a general status to update. """ data = {"build_id": self.get_build_id(spec), "status": status} + if self.save_local: + filename = "build-%s-status.json" % data['build_id'] + return self.save(sjson.dump(data), filename) + return self.do_request("builds/update/", data=sjson.dump(data)) def fail_task(self, spec): @@ -444,6 +508,10 @@ class SpackMonitorClient: "output": read_file(phase_output_file), "phase_name": phase_name}) + if self.save_local: + filename = "build-%s-phase-%s.json" % (data['build_id'], phase_name) + return self.save(sjson.dump(data), filename) + return self.do_request("builds/phases/update/", data=sjson.dump(data)) def upload_specfile(self, filename): @@ -459,6 +527,11 @@ class SpackMonitorClient: # We load as json just to validate it spec = read_json(filename) data = {"spec": spec, "spack_verison": self.spack_version} + + if self.save_local: + filename = "spec-%s-%s.json" % (spec.name, spec.version) + return self.save(sjson.dump(data), filename) + return self.do_request("specs/new/", data=sjson.dump(data)) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index b0ff031dd1..c45a5a6b0f 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -54,7 +54,7 @@ user_config_path = os.path.expanduser('~/.spack') user_bootstrap_path = os.path.join(user_config_path, 'bootstrap') user_bootstrap_store = os.path.join(user_bootstrap_path, 'store') reports_path = os.path.join(user_config_path, "reports") - +monitor_path = os.path.join(reports_path, "monitor") opt_path = os.path.join(prefix, "opt") etc_path = os.path.join(prefix, "etc") diff --git a/lib/spack/spack/test/monitor.py b/lib/spack/spack/test/monitor.py new file mode 100644 index 0000000000..e8b466ab1a --- /dev/null +++ b/lib/spack/spack/test/monitor.py @@ -0,0 +1,42 @@ +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import spack.config +import spack.spec +from spack.main import SpackCommand +import pytest +import os + +install = SpackCommand('install') + + +@pytest.fixture(scope='session') +def test_install_monitor_save_local(install_mockery_mutable_config, + mock_fetch, tmpdir_factory): + """ + Mock installing and saving monitor results to file. + """ + reports_dir = tmpdir_factory.mktemp('reports') + spack.config.set('config:monitor_dir', str(reports_dir)) + out = install('--monitor', '--monitor-save-local', 'dttop') + assert "Successfully installed dttop" in out + + # The reports directory should not be empty (timestamped folders) + assert os.listdir(str(reports_dir)) + + # Get the spec name + spec = spack.spec.Spec("dttop") + spec.concretize() + full_hash = spec.full_hash() + + # Ensure we have monitor results saved + for dirname in os.listdir(str(reports_dir)): + dated_dir = os.path.join(str(reports_dir), dirname) + build_metadata = "build-metadata-%s.json" % full_hash + assert build_metadata in os.listdir(dated_dir) + spec_file = "spec-dttop-%s-config.json" % spec.version + assert spec_file in os.listdir(dated_dir) + + spack.config.set('config:monitor_dir', "~/.spack/reports/monitor") |