summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--lib/spack/docs/monitoring.rst134
-rw-r--r--lib/spack/spack/cmd/containerize.py12
-rw-r--r--lib/spack/spack/cmd/install.py4
-rw-r--r--lib/spack/spack/container/writers/__init__.py23
-rw-r--r--lib/spack/spack/monitor.py8
-rw-r--r--lib/spack/spack/test/monitor.py237
-rwxr-xr-xshare/spack/spack-completion.bash2
-rw-r--r--share/spack/templates/container/Dockerfile2
-rw-r--r--share/spack/templates/container/singularity.def2
9 files changed, 420 insertions, 4 deletions
diff --git a/lib/spack/docs/monitoring.rst b/lib/spack/docs/monitoring.rst
index 97f4fc4cd8..41c79cf2b6 100644
--- a/lib/spack/docs/monitoring.rst
+++ b/lib/spack/docs/monitoring.rst
@@ -103,6 +103,140 @@ more tags to your build, you can do:
$ spack install --monitor --monitor-tags pizza,pasta hdf5
+----------------------------
+Monitoring with Containerize
+----------------------------
+
+The same argument group is available to add to a containerize command.
+
+^^^^^^
+Docker
+^^^^^^
+
+To add monitoring to a Docker container recipe generation using the defaults,
+and assuming a monitor server running on localhost, you would
+start with a spack.yaml in your present working directory:
+
+.. code-block:: yaml
+
+ spack:
+ specs:
+ - samtools
+
+And then do:
+
+.. code-block:: console
+
+ # preview first
+ spack containerize --monitor
+
+ # and then write to a Dockerfile
+ spack containerize --monitor > Dockerfile
+
+
+The install command will be edited to include commands for enabling monitoring.
+However, getting secrets into the container for your monitor server is something
+that should be done carefully. Specifically you should:
+
+ - Never try to define secrets as ENV, ARG, or using ``--build-arg``
+ - Do not try to get the secret into the container via a "temporary" file that you remove (it in fact will still exist in a layer)
+
+Instead, it's recommended to use buildkit `as explained here <https://pythonspeed.com/articles/docker-build-secrets/>`_.
+You'll need to again export environment variables for your spack monitor server:
+
+.. code-block:: console
+
+ $ export SPACKMON_TOKEN=50445263afd8f67e59bd79bff597836ee6c05438
+ $ export SPACKMON_USER=spacky
+
+And then use buildkit along with your build and identifying the name of the secret:
+
+.. code-block:: console
+
+ $ DOCKER_BUILDKIT=1 docker build --secret id=st,env=SPACKMON_TOKEN --secret id=su,env=SPACKMON_USER -t spack/container .
+
+The secrets are expected to come from your environment, and then will be temporarily mounted and available
+at ``/run/secrets/<name>``. If you forget to supply them (and authentication is required) the build
+will fail. If you need to build on your host (and interact with a spack monitor at localhost) you'll
+need to tell Docker to use the host network:
+
+.. code-block:: console
+
+ $ DOCKER_BUILDKIT=1 docker build --network="host" --secret id=st,env=SPACKMON_TOKEN --secret id=su,env=SPACKMON_USER -t spack/container .
+
+
+^^^^^^^^^^^
+Singularity
+^^^^^^^^^^^
+
+To add monitoring to a Singularity container build, the spack.yaml needs to
+be modified slightly to specify wanting a different format:
+
+
+.. code-block:: yaml
+
+ spack:
+ specs:
+ - samtools
+ container:
+ format: singularity
+
+
+Again, generate the recipe:
+
+
+.. code-block:: console
+
+ # preview first
+ $ spack containerize --monitor
+
+ # then write to a Singularity recipe
+ $ spack containerize --monitor > Singularity
+
+
+Singularity doesn't have a direct way to define secrets at build time, so we have
+to do a bit of a manual command to add a file, source secrets in it, and remove it.
+Since Singularity doesn't have layers like Docker, deleting a file will truly
+remove it from the container and history. So let's say we have this file,
+``secrets.sh``:
+
+.. code-block:: console
+
+ # secrets.sh
+ export SPACKMON_USER=spack
+ export SPACKMON_TOKEN=50445263afd8f67e59bd79bff597836ee6c05438
+
+
+We would then generate the Singularity recipe, and add a files section,
+a source of that file at the start of ``%post``, and **importantly**
+a removal of the final at the end of that same section.
+
+.. code-block::
+
+ Bootstrap: docker
+ From: spack/ubuntu-bionic:latest
+ Stage: build
+
+ %files
+ secrets.sh /opt/secrets.sh
+
+ %post
+ . /opt/secrets.sh
+
+ # spack install commands are here
+ ...
+
+ # Don't forget to remove here!
+ rm /opt/secrets.sh
+
+
+You can then build the container as your normally would.
+
+.. code-block:: console
+
+ $ sudo singularity build container.sif Singularity
+
+
------------------
Monitoring Offline
------------------
diff --git a/lib/spack/spack/cmd/containerize.py b/lib/spack/spack/cmd/containerize.py
index 27ef988f69..a145558bd7 100644
--- a/lib/spack/spack/cmd/containerize.py
+++ b/lib/spack/spack/cmd/containerize.py
@@ -5,6 +5,7 @@
import os
import os.path
import spack.container
+import spack.monitor
description = ("creates recipes to build images for different"
" container runtimes")
@@ -12,6 +13,10 @@ section = "container"
level = "long"
+def setup_parser(subparser):
+ monitor_group = spack.monitor.get_monitor_group(subparser) # noqa
+
+
def containerize(parser, args):
config_dir = args.env_dir or os.getcwd()
config_file = os.path.abspath(os.path.join(config_dir, 'spack.yaml'))
@@ -21,5 +26,12 @@ def containerize(parser, args):
config = spack.container.validate(config_file)
+ # If we have a monitor request, add monitor metadata to config
+ if args.use_monitor:
+ config['spack']['monitor'] = {"disable_auth": args.monitor_disable_auth,
+ "host": args.monitor_host,
+ "keep_going": args.monitor_keep_going,
+ "prefix": args.monitor_prefix,
+ "tags": args.monitor_tags}
recipe = spack.container.recipe(config)
print(recipe)
diff --git a/lib/spack/spack/cmd/install.py b/lib/spack/spack/cmd/install.py
index 16026bd5f2..4f3b3222e4 100644
--- a/lib/spack/spack/cmd/install.py
+++ b/lib/spack/spack/cmd/install.py
@@ -347,6 +347,10 @@ environment variables:
reporter.filename = default_log_file(specs[0])
reporter.specs = specs
+ # Tell the monitor about the specs
+ if args.use_monitor and specs:
+ monitor.new_configuration(specs)
+
tty.msg("Installing environment {0}".format(env.name))
with reporter('build'):
env.install_all(args, **kwargs)
diff --git a/lib/spack/spack/container/writers/__init__.py b/lib/spack/spack/container/writers/__init__.py
index b1c82a7bdf..4c43e3db35 100644
--- a/lib/spack/spack/container/writers/__init__.py
+++ b/lib/spack/spack/container/writers/__init__.py
@@ -111,12 +111,35 @@ class PathContext(tengine.Context):
)
@tengine.context_property
+ def monitor(self):
+ """Enable using spack monitor during build."""
+ Monitor = collections.namedtuple('Monitor', [
+ 'enabled', 'host', 'disable_auth', 'prefix', 'keep_going', 'tags'
+ ])
+ monitor = self.config.get("monitor")
+
+ # If we don't have a monitor group, cut out early.
+ if not monitor:
+ return Monitor(False, None, None, None, None, None)
+
+ return Monitor(
+ enabled=True,
+ host=monitor.get('host'),
+ prefix=monitor.get('prefix'),
+ disable_auth=monitor.get("disable_auth"),
+ keep_going=monitor.get("keep_going"),
+ tags=monitor.get('tags')
+ )
+
+ @tengine.context_property
def manifest(self):
"""The spack.yaml file that should be used in the image"""
import jsonschema
# Copy in the part of spack.yaml prescribed in the configuration file
manifest = copy.deepcopy(self.config)
manifest.pop('container')
+ if "monitor" in manifest:
+ manifest.pop("monitor")
# Ensure that a few paths are where they need to be
manifest.setdefault('config', syaml.syaml_dict())
diff --git a/lib/spack/spack/monitor.py b/lib/spack/spack/monitor.py
index f7d108cb75..1b44d0a032 100644
--- a/lib/spack/spack/monitor.py
+++ b/lib/spack/spack/monitor.py
@@ -172,7 +172,7 @@ class SpackMonitorClient:
env_file = os.path.join(pkg_dir, "install_environment.json")
build_environment = read_json(env_file)
if not build_environment:
- tty.warning(
+ tty.warn(
"install_environment.json not found in package folder. "
" This means that the current environment metadata will be used."
)
@@ -283,6 +283,12 @@ class SpackMonitorClient:
elif hasattr(e, 'code'):
msg = e.code
+ # If we can parse the message, try it
+ try:
+ msg += "\n%s" % e.read().decode("utf8", 'ignore')
+ except Exception:
+ pass
+
if self.allow_fail:
tty.warning("Request to %s was not successful, but continuing." % e.url)
return
diff --git a/lib/spack/spack/test/monitor.py b/lib/spack/spack/test/monitor.py
index e8b466ab1a..77c2754d44 100644
--- a/lib/spack/spack/test/monitor.py
+++ b/lib/spack/spack/test/monitor.py
@@ -6,12 +6,249 @@
import spack.config
import spack.spec
from spack.main import SpackCommand
+from spack.monitor import SpackMonitorClient
+import llnl.util.tty as tty
+import spack.monitor
import pytest
import os
install = SpackCommand('install')
+def get_client(host, prefix="ms1", disable_auth=False, allow_fail=False, tags=None,
+ save_local=False):
+ """
+ We replicate this function to not generate a global client.
+ """
+ cli = SpackMonitorClient(host=host, prefix=prefix, allow_fail=allow_fail,
+ tags=tags, save_local=save_local)
+
+ # If we don't disable auth, environment credentials are required
+ if not disable_auth and not save_local:
+ cli.require_auth()
+
+ # We will exit early if the monitoring service is not running, but
+ # only if we aren't doing a local save
+ if not save_local:
+ info = cli.service_info()
+
+ # If we allow failure, the response will be done
+ if info:
+ tty.debug("%s v.%s has status %s" % (
+ info['id'],
+ info['version'],
+ info['status'])
+ )
+ return cli
+
+
+@pytest.fixture
+def mock_monitor_request(monkeypatch):
+ """
+ Monitor requests that are shared across tests go here
+ """
+ def mock_do_request(self, endpoint, *args, **kwargs):
+
+ build = {"build_id": 1,
+ "spec_full_hash": "bpfvysmqndtmods4rmy6d6cfquwblngp",
+ "spec_name": "dttop"}
+
+ # Service Info
+ if endpoint == "":
+ organization = {"name": "spack", "url": "https://github.com/spack"}
+ return {"id": "spackmon", "status": "running",
+ "name": "Spack Monitor (Spackmon)",
+ "description": "The best spack monitor",
+ "organization": organization,
+ "contactUrl": "https://github.com/spack/spack-monitor/issues",
+ "documentationUrl": "https://spack-monitor.readthedocs.io",
+ "createdAt": "2021-04-09T21:54:51Z",
+ "updatedAt": "2021-05-24T15:06:46Z",
+ "environment": "test",
+ "version": "0.0.1",
+ "auth_instructions_url": "url"}
+
+ # New Build
+ elif endpoint == "builds/new/":
+ return {"message": "Build get or create was successful.",
+ "data": {
+ "build_created": True,
+ "build_environment_created": True,
+ "build": build
+ },
+ "code": 201}
+
+ # Update Build
+ elif endpoint == "builds/update/":
+ return {"message": "Status updated",
+ "data": {"build": build},
+ "code": 200}
+
+ # Send Analyze Metadata
+ elif endpoint == "analyze/builds/":
+ return {"message": "Metadata updated",
+ "data": {"build": build},
+ "code": 200}
+
+ # Update Build Phase
+ elif endpoint == "builds/phases/update/":
+ return {"message": "Phase autoconf was successfully updated.",
+ "code": 200,
+ "data": {
+ "build_phase": {
+ "id": 1,
+ "status": "SUCCESS",
+ "name": "autoconf"
+ }
+ }}
+
+ # Update Phase Status
+ elif endpoint == "phases/update/":
+ return {"message": "Status updated",
+ "data": {"build": build},
+ "code": 200}
+
+ # New Spec
+ elif endpoint == "specs/new/":
+ return {"message": "success",
+ "data": {
+ "full_hash": "bpfvysmqndtmods4rmy6d6cfquwblngp",
+ "name": "dttop",
+ "version": "1.0",
+ "spack_version": "0.16.0-1379-7a5351d495",
+ "specs": {
+ "dtbuild1": "btcmljubs4njhdjqt2ebd6nrtn6vsrks",
+ "dtlink1": "x4z6zv6lqi7cf6l4twz4bg7hj3rkqfmk",
+ "dtrun1": "i6inyro74p5yqigllqk5ivvwfjfsw6qz"
+ }
+ }}
+ else:
+ pytest.fail("bad endpoint: %s" % endpoint)
+ monkeypatch.setattr(spack.monitor.SpackMonitorClient, "do_request", mock_do_request)
+
+
+def test_spack_monitor_auth(mock_monitor_request):
+ with pytest.raises(SystemExit):
+ get_client(host="http://127.0.0.1")
+
+ os.environ["SPACKMON_TOKEN"] = "xxxxxxxxxxxxxxxxx"
+ os.environ["SPACKMON_USER"] = "spackuser"
+ get_client(host="http://127.0.0.1")
+
+
+def test_spack_monitor_without_auth(mock_monitor_request):
+ get_client(host="hostname", disable_auth=True)
+
+
+def test_spack_monitor_build_env(mock_monitor_request, install_mockery_mutable_config):
+ monitor = get_client(host="hostname", disable_auth=True)
+ assert hasattr(monitor, "build_environment")
+ for key in ["host_os", "platform", "host_target", "hostname", "spack_version",
+ "kernel_version"]:
+ assert key in monitor.build_environment
+
+ spec = spack.spec.Spec("dttop")
+ spec.concretize()
+ # Loads the build environment from the spec install folder
+ monitor.load_build_environment(spec)
+
+
+def test_spack_monitor_basic_auth(mock_monitor_request):
+ monitor = get_client(host="hostname", disable_auth=True)
+
+ # Headers should be empty
+ assert not monitor.headers
+ monitor.set_basic_auth("spackuser", "password")
+ assert "Authorization" in monitor.headers
+ assert monitor.headers['Authorization'].startswith("Basic")
+
+
+def test_spack_monitor_new_configuration(mock_monitor_request, install_mockery):
+ monitor = get_client(host="hostname", disable_auth=True)
+ spec = spack.spec.Spec("dttop")
+ spec.concretize()
+ response = monitor.new_configuration([spec])
+
+ # The response is a lookup of specs
+ assert "dttop" in response
+
+
+def test_spack_monitor_new_build(mock_monitor_request, install_mockery_mutable_config,
+ install_mockery):
+ monitor = get_client(host="hostname", disable_auth=True)
+ spec = spack.spec.Spec("dttop")
+ spec.concretize()
+ response = monitor.new_build(spec)
+ assert "message" in response and "data" in response and "code" in response
+ assert response['code'] == 201
+ # We should be able to get a build id
+ monitor.get_build_id(spec)
+
+
+def test_spack_monitor_update_build(mock_monitor_request, install_mockery,
+ install_mockery_mutable_config):
+ monitor = get_client(host="hostname", disable_auth=True)
+ spec = spack.spec.Spec("dttop")
+ spec.concretize()
+ response = monitor.update_build(spec, status="SUCCESS")
+ assert "message" in response and "data" in response and "code" in response
+ assert response['code'] == 200
+
+
+def test_spack_monitor_fail_task(mock_monitor_request, install_mockery,
+ install_mockery_mutable_config):
+ monitor = get_client(host="hostname", disable_auth=True)
+ spec = spack.spec.Spec("dttop")
+ spec.concretize()
+ response = monitor.fail_task(spec)
+ assert "message" in response and "data" in response and "code" in response
+ assert response['code'] == 200
+
+
+def test_spack_monitor_send_analyze_metadata(monkeypatch, mock_monitor_request,
+ install_mockery,
+ install_mockery_mutable_config):
+
+ def buildid(*args, **kwargs):
+ return 1
+ monkeypatch.setattr(spack.monitor.SpackMonitorClient, "get_build_id", buildid)
+ monitor = get_client(host="hostname", disable_auth=True)
+ spec = spack.spec.Spec("dttop")
+ spec.concretize()
+ response = monitor.send_analyze_metadata(spec.package, metadata={"boop": "beep"})
+ assert "message" in response and "data" in response and "code" in response
+ assert response['code'] == 200
+
+
+def test_spack_monitor_send_phase(mock_monitor_request, install_mockery,
+ install_mockery_mutable_config):
+
+ monitor = get_client(host="hostname", disable_auth=True)
+
+ def get_build_id(*args, **kwargs):
+ return 1
+
+ spec = spack.spec.Spec("dttop")
+ spec.concretize()
+ response = monitor.send_phase(spec.package, "autoconf",
+ spec.package.install_log_path,
+ "SUCCESS")
+ assert "message" in response and "data" in response and "code" in response
+ assert response['code'] == 200
+
+
+def test_spack_monitor_info(mock_monitor_request):
+ os.environ["SPACKMON_TOKEN"] = "xxxxxxxxxxxxxxxxx"
+ os.environ["SPACKMON_USER"] = "spackuser"
+ monitor = get_client(host="http://127.0.0.1")
+ info = monitor.service_info()
+
+ for key in ['id', 'status', 'name', 'description', 'organization',
+ 'contactUrl', 'documentationUrl', 'createdAt', 'updatedAt',
+ 'environment', 'version', 'auth_instructions_url']:
+ assert key in info
+
+
@pytest.fixture(scope='session')
def test_install_monitor_save_local(install_mockery_mutable_config,
mock_fetch, tmpdir_factory):
diff --git a/share/spack/spack-completion.bash b/share/spack/spack-completion.bash
index 8b41436283..26dd77aeed 100755
--- a/share/spack/spack-completion.bash
+++ b/share/spack/spack-completion.bash
@@ -703,7 +703,7 @@ _spack_config_revert() {
}
_spack_containerize() {
- SPACK_COMPREPLY="-h --help"
+ SPACK_COMPREPLY="-h --help --monitor --monitor-save-local --monitor-no-auth --monitor-tags --monitor-keep-going --monitor-host --monitor-prefix"
}
_spack_create() {
diff --git a/share/spack/templates/container/Dockerfile b/share/spack/templates/container/Dockerfile
index 3623a7ba0b..875702979d 100644
--- a/share/spack/templates/container/Dockerfile
+++ b/share/spack/templates/container/Dockerfile
@@ -14,7 +14,7 @@ RUN mkdir {{ paths.environment }} \
{{ manifest }} > {{ paths.environment }}/spack.yaml
# Install the software, remove unnecessary deps
-RUN cd {{ paths.environment }} && spack env activate . && spack install --fail-fast && spack gc -y
+RUN {% if monitor.enabled %}--mount=type=secret,id=su --mount=type=secret,id=st{% endif %} cd {{ paths.environment }} && spack env activate . {% if not monitor.disable_auth %}&& export SPACKMON_USER=$(cat /run/secrets/su) && export SPACKMON_TOKEN=$(cat /run/secrets/st) {% endif %}&& spack install {% if monitor.enabled %}--monitor {% if monitor.prefix %}--monitor-prefix {{ monitor.prefix }} {% endif %}{% if monitor.tags %}--monitor-tags {{ monitor.tags }} {% endif %}{% if monitor.keep_going %}--monitor-keep-going {% endif %}{% if monitor.host %}--monitor-host {{ monitor.host }} {% endif %}{% if monitor.disable_auth %}--monitor-disable-auth {% endif %}{% endif %}--fail-fast && spack gc -y
{% if strip %}
# Strip all the binaries
diff --git a/share/spack/templates/container/singularity.def b/share/spack/templates/container/singularity.def
index 33d775b024..de0392b718 100644
--- a/share/spack/templates/container/singularity.def
+++ b/share/spack/templates/container/singularity.def
@@ -21,7 +21,7 @@ EOF
# Install all the required software
. /opt/spack/share/spack/setup-env.sh
spack env activate .
- spack install --fail-fast
+ spack install {% if monitor.enabled %}--monitor {% if monitor.prefix %}--monitor-prefix {{ monitor.prefix }} {% endif %}{% if monitor.tags %}--monitor-tags {{ monitor.tags }} {% endif %}{% if monitor.keep_going %}--monitor-keep-going {% endif %}{% if monitor.host %}--monitor-host {{ monitor.host }} {% endif %}{% if monitor.disable_auth %}--monitor-disable-auth {% endif %}{% endif %}--fail-fast
spack gc -y
spack env deactivate
spack env activate --sh -d . >> {{ paths.environment }}/environment_modifications.sh
generated by cgit v1.2.3-70-g09d2 (git 2.46.0) at 2024-12-04 00:22:49 +0000