diff options
author | Vanessasaurus <814322+vsoch@users.noreply.github.com> | 2021-05-25 12:29:34 -0600 |
---|---|---|
committer | GitHub <noreply@github.com> | 2021-05-25 11:29:34 -0700 |
commit | b44bb952eb29572fe8bcf9eab65d8baa24d3b994 (patch) | |
tree | ade7832c8027b6facf526a18d65c86d12198afb0 | |
parent | e22da8df057c607ddc586956caa60a76ff25574d (diff) | |
download | spack-b44bb952eb29572fe8bcf9eab65d8baa24d3b994.tar.gz spack-b44bb952eb29572fe8bcf9eab65d8baa24d3b994.tar.bz2 spack-b44bb952eb29572fe8bcf9eab65d8baa24d3b994.tar.xz spack-b44bb952eb29572fe8bcf9eab65d8baa24d3b994.zip |
first set of work to allow for saving local results with spack monitor (#23804)
This work will come in two phases. The first here is to allow saving of a local result
with spack monitor, and the second will add a spack monitor command so the user can
do spack monitor upload.
Signed-off-by: vsoch <vsoch@users.noreply.github.com>
Co-authored-by: vsoch <vsoch@users.noreply.github.com>
-rw-r--r-- | lib/spack/docs/monitoring.rst | 16 | ||||
-rw-r--r-- | lib/spack/spack/cmd/install.py | 1 | ||||
-rw-r--r-- | lib/spack/spack/monitor.py | 115 | ||||
-rw-r--r-- | lib/spack/spack/paths.py | 2 | ||||
-rw-r--r-- | lib/spack/spack/test/monitor.py | 42 | ||||
-rwxr-xr-x | share/spack/spack-completion.bash | 4 |
6 files changed, 156 insertions, 24 deletions
diff --git a/lib/spack/docs/monitoring.rst b/lib/spack/docs/monitoring.rst index b1d491c563..41a7a4a2d2 100644 --- a/lib/spack/docs/monitoring.rst +++ b/lib/spack/docs/monitoring.rst @@ -102,3 +102,19 @@ more tags to your build, you can do: # Add two tags, "pizza" and "pasta" $ spack install --monitor --monitor-tags pizza,pasta hdf5 + +------------------ +Monitoring Offline +------------------ + +In the case that you want to save monitor results to your filesystem +and then upload them later (perhaps you are in an environment where you don't +have credentials or it isn't safe to use them) you can use the ``--monitor-save-local`` +flag. + +.. code-block:: console + + $ spack install --monitor --monitor-save-local hdf5 + +This will save results in a subfolder, "monitor" in your designated spack +reports folder, which defaults to ``$HOME/.spack/reports/monitor``. diff --git a/lib/spack/spack/cmd/install.py b/lib/spack/spack/cmd/install.py index 587f039656..5a0b47215a 100644 --- a/lib/spack/spack/cmd/install.py +++ b/lib/spack/spack/cmd/install.py @@ -306,6 +306,7 @@ environment variables: prefix=args.monitor_prefix, disable_auth=args.monitor_disable_auth, tags=args.monitor_tags, + save_local=args.monitor_save_local, ) reporter = spack.report.collect_info( diff --git a/lib/spack/spack/monitor.py b/lib/spack/spack/monitor.py index 5d6246ebf1..a6ca3be316 100644 --- a/lib/spack/spack/monitor.py +++ b/lib/spack/spack/monitor.py @@ -7,6 +7,8 @@ https://github.com/spack/spack-monitor/blob/main/script/spackmoncli.py """ +from datetime import datetime +import hashlib import base64 import os import re @@ -18,11 +20,13 @@ except ImportError: from urllib2 import urlopen, Request, URLError # type: ignore # novm import spack +import spack.config import spack.hash_types as ht import spack.main import spack.store import spack.util.spack_json as sjson import spack.util.spack_yaml as syaml +import spack.util.path import llnl.util.tty as tty from copy import deepcopy @@ -31,7 +35,8 @@ from copy import deepcopy cli = None -def get_client(host, prefix="ms1", disable_auth=False, allow_fail=False, tags=None): +def get_client(host, prefix="ms1", disable_auth=False, allow_fail=False, tags=None, + save_local=False): """ Get a monitor client for a particular host and prefix. @@ -47,26 +52,25 @@ def get_client(host, prefix="ms1", disable_auth=False, allow_fail=False, tags=No """ global cli cli = SpackMonitorClient(host=host, prefix=prefix, allow_fail=allow_fail, - tags=tags) + tags=tags, save_local=save_local) # If we don't disable auth, environment credentials are required - if not disable_auth: + if not disable_auth and not save_local: cli.require_auth() - # We will exit early if the monitoring service is not running - info = cli.service_info() - - # If we allow failure, the response will be done - if info: - tty.debug("%s v.%s has status %s" % ( - info['id'], - info['version'], - info['status']) - ) - return cli - - else: - tty.debug("spack-monitor server not found, continuing as allow_fail is True.") + # We will exit early if the monitoring service is not running, but + # only if we aren't doing a local save + if not save_local: + info = cli.service_info() + + # If we allow failure, the response will be done + if info: + tty.debug("%s v.%s has status %s" % ( + info['id'], + info['version'], + info['status']) + ) + return cli def get_monitor_group(subparser): @@ -83,6 +87,9 @@ def get_monitor_group(subparser): '--monitor', action='store_true', dest='use_monitor', default=False, help="interact with a montor server during builds.") monitor_group.add_argument( + '--monitor-save-local', action='store_true', dest='monitor_save_local', + default=False, help="save monitor results to .spack instead of server.") + monitor_group.add_argument( '--monitor-no-auth', action='store_true', dest='monitor_disable_auth', default=False, help="the monitoring server does not require auth.") monitor_group.add_argument( @@ -110,7 +117,8 @@ class SpackMonitorClient: to the client on init. """ - def __init__(self, host=None, prefix="ms1", allow_fail=False, tags=None): + def __init__(self, host=None, prefix="ms1", allow_fail=False, tags=None, + save_local=False): self.host = host or "http://127.0.0.1" self.baseurl = "%s/%s" % (self.host, prefix.strip("/")) self.token = os.environ.get("SPACKMON_TOKEN") @@ -120,9 +128,34 @@ class SpackMonitorClient: self.spack_version = spack.main.get_version() self.capture_build_environment() self.tags = tags + self.save_local = save_local # We keey lookup of build_id by full_hash self.build_ids = {} + self.setup_save() + + def setup_save(self): + """Given a local save "save_local" ensure the output directory exists. + """ + if not self.save_local: + return + + save_dir = spack.util.path.canonicalize_path( + spack.config.get('config:monitor_dir', '~/.spack/reports/monitor')) + + # Name based on timestamp + now = datetime.now().strftime('%Y-%m-%d-%H-%M-%S-%s') + self.save_dir = os.path.join(save_dir, now) + if not os.path.exists(self.save_dir): + os.makedirs(self.save_dir) + + def save(self, obj, filename): + """ + Save a monitor json result to the save directory. + """ + filename = os.path.join(self.save_dir, filename) + write_json(obj, filename) + return {"message": "Build saved locally to %s" % filename} def load_build_environment(self, spec): """ @@ -174,7 +207,7 @@ class SpackMonitorClient: The token and username must not be unset """ - if not self.token or not self.username: + if not self.save_local and (not self.token or not self.username): tty.die("You are required to export SPACKMON_TOKEN and SPACKMON_USER") def set_header(self, name, value): @@ -346,8 +379,14 @@ class SpackMonitorClient: spec.concretize() as_dict = {"spec": spec.to_dict(hash=ht.full_hash), "spack_version": self.spack_version} - response = self.do_request("specs/new/", data=sjson.dump(as_dict)) - configs[spec.package.name] = response.get('data', {}) + + if self.save_local: + filename = "spec-%s-%s-config.json" % (spec.name, spec.version) + self.save(sjson.dump(as_dict), filename) + else: + response = self.do_request("specs/new/", data=sjson.dump(as_dict)) + configs[spec.package.name] = response.get('data', {}) + return configs def new_build(self, spec): @@ -384,6 +423,27 @@ class SpackMonitorClient: spec_file = os.path.join(meta_dir, "spec.yaml") data['spec'] = syaml.load(read_file(spec_file)) + if self.save_local: + return self.get_local_build_id(data, full_hash, return_response) + return self.get_server_build_id(data, full_hash, return_response) + + def get_local_build_id(self, data, full_hash, return_response): + """ + Generate a local build id based on hashing the expected data + """ + hasher = hashlib.md5() + hasher.update(str(data).encode('utf-8')) + bid = hasher.hexdigest() + filename = "build-metadata-%s.json" % full_hash + response = self.save(sjson.dump(data), filename) + if return_response: + return response + return bid + + def get_server_build_id(self, data, full_hash, return_response=False): + """ + Retrieve a build id from the spack monitor server + """ response = self.do_request("builds/new/", data=sjson.dump(data)) # Add the build id to the lookup @@ -403,6 +463,10 @@ class SpackMonitorClient: successful install. This endpoint can take a general status to update. """ data = {"build_id": self.get_build_id(spec), "status": status} + if self.save_local: + filename = "build-%s-status.json" % data['build_id'] + return self.save(sjson.dump(data), filename) + return self.do_request("builds/update/", data=sjson.dump(data)) def fail_task(self, spec): @@ -444,6 +508,10 @@ class SpackMonitorClient: "output": read_file(phase_output_file), "phase_name": phase_name}) + if self.save_local: + filename = "build-%s-phase-%s.json" % (data['build_id'], phase_name) + return self.save(sjson.dump(data), filename) + return self.do_request("builds/phases/update/", data=sjson.dump(data)) def upload_specfile(self, filename): @@ -459,6 +527,11 @@ class SpackMonitorClient: # We load as json just to validate it spec = read_json(filename) data = {"spec": spec, "spack_verison": self.spack_version} + + if self.save_local: + filename = "spec-%s-%s.json" % (spec.name, spec.version) + return self.save(sjson.dump(data), filename) + return self.do_request("specs/new/", data=sjson.dump(data)) diff --git a/lib/spack/spack/paths.py b/lib/spack/spack/paths.py index b0ff031dd1..c45a5a6b0f 100644 --- a/lib/spack/spack/paths.py +++ b/lib/spack/spack/paths.py @@ -54,7 +54,7 @@ user_config_path = os.path.expanduser('~/.spack') user_bootstrap_path = os.path.join(user_config_path, 'bootstrap') user_bootstrap_store = os.path.join(user_bootstrap_path, 'store') reports_path = os.path.join(user_config_path, "reports") - +monitor_path = os.path.join(reports_path, "monitor") opt_path = os.path.join(prefix, "opt") etc_path = os.path.join(prefix, "etc") diff --git a/lib/spack/spack/test/monitor.py b/lib/spack/spack/test/monitor.py new file mode 100644 index 0000000000..e8b466ab1a --- /dev/null +++ b/lib/spack/spack/test/monitor.py @@ -0,0 +1,42 @@ +# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import spack.config +import spack.spec +from spack.main import SpackCommand +import pytest +import os + +install = SpackCommand('install') + + +@pytest.fixture(scope='session') +def test_install_monitor_save_local(install_mockery_mutable_config, + mock_fetch, tmpdir_factory): + """ + Mock installing and saving monitor results to file. + """ + reports_dir = tmpdir_factory.mktemp('reports') + spack.config.set('config:monitor_dir', str(reports_dir)) + out = install('--monitor', '--monitor-save-local', 'dttop') + assert "Successfully installed dttop" in out + + # The reports directory should not be empty (timestamped folders) + assert os.listdir(str(reports_dir)) + + # Get the spec name + spec = spack.spec.Spec("dttop") + spec.concretize() + full_hash = spec.full_hash() + + # Ensure we have monitor results saved + for dirname in os.listdir(str(reports_dir)): + dated_dir = os.path.join(str(reports_dir), dirname) + build_metadata = "build-metadata-%s.json" % full_hash + assert build_metadata in os.listdir(dated_dir) + spec_file = "spec-dttop-%s-config.json" % spec.version + assert spec_file in os.listdir(dated_dir) + + spack.config.set('config:monitor_dir', "~/.spack/reports/monitor") diff --git a/share/spack/spack-completion.bash b/share/spack/spack-completion.bash index 6fb681afe3..5d9a01737f 100755 --- a/share/spack/spack-completion.bash +++ b/share/spack/spack-completion.bash @@ -358,7 +358,7 @@ _spack_add() { _spack_analyze() { if $list_options then - SPACK_COMPREPLY="-h --help --monitor --monitor-no-auth --monitor-tags --monitor-keep-going --monitor-host --monitor-prefix" + SPACK_COMPREPLY="-h --help --monitor --monitor-save-local --monitor-no-auth --monitor-tags --monitor-keep-going --monitor-host --monitor-prefix" else SPACK_COMPREPLY="list-analyzers run" fi @@ -1063,7 +1063,7 @@ _spack_info() { _spack_install() { if $list_options then - SPACK_COMPREPLY="-h --help --only -u --until -j --jobs --overwrite --fail-fast --keep-prefix --keep-stage --dont-restage --use-cache --no-cache --cache-only --monitor --monitor-no-auth --monitor-tags --monitor-keep-going --monitor-host --monitor-prefix --include-build-deps --no-check-signature --require-full-hash-match --show-log-on-error --source -n --no-checksum --deprecated -v --verbose --fake --only-concrete --no-add -f --file --clean --dirty --test --run-tests --log-format --log-file --help-cdash --cdash-upload-url --cdash-build --cdash-site --cdash-track --cdash-buildstamp -y --yes-to-all" + SPACK_COMPREPLY="-h --help --only -u --until -j --jobs --overwrite --fail-fast --keep-prefix --keep-stage --dont-restage --use-cache --no-cache --cache-only --monitor --monitor-save-local --monitor-no-auth --monitor-tags --monitor-keep-going --monitor-host --monitor-prefix --include-build-deps --no-check-signature --require-full-hash-match --show-log-on-error --source -n --no-checksum --deprecated -v --verbose --fake --only-concrete --no-add -f --file --clean --dirty --test --run-tests --log-format --log-file --help-cdash --cdash-upload-url --cdash-build --cdash-site --cdash-track --cdash-buildstamp -y --yes-to-all" else _all_packages fi |