diff options
Diffstat (limited to 'lib')
| -rw-r--r-- | lib/spack/docs/developer_guide.rst | 268 | ||||
| -rw-r--r-- | lib/spack/spack/analyzers/__init__.py | 42 | ||||
| -rw-r--r-- | lib/spack/spack/analyzers/analyzer_base.py | 116 | ||||
| -rw-r--r-- | lib/spack/spack/analyzers/config_args.py | 33 | ||||
| -rw-r--r-- | lib/spack/spack/analyzers/environment_variables.py | 54 | ||||
| -rw-r--r-- | lib/spack/spack/analyzers/install_files.py | 31 | ||||
| -rw-r--r-- | lib/spack/spack/analyzers/libabigail.py | 114 | ||||
| -rw-r--r-- | lib/spack/spack/cmd/analyze.py | 116 | ||||
| -rw-r--r-- | lib/spack/spack/cmd/containerize.py | 11 | ||||
| -rw-r--r-- | lib/spack/spack/cmd/install.py | 33 | ||||
| -rw-r--r-- | lib/spack/spack/cmd/monitor.py | 33 | ||||
| -rw-r--r-- | lib/spack/spack/container/writers/__init__.py | 22 | ||||
| -rw-r--r-- | lib/spack/spack/hooks/__init__.py | 4 | ||||
| -rw-r--r-- | lib/spack/spack/hooks/monitor.py | 85 | ||||
| -rw-r--r-- | lib/spack/spack/installer.py | 1 | ||||
| -rw-r--r-- | lib/spack/spack/monitor.py | 738 | ||||
| -rw-r--r-- | lib/spack/spack/test/cmd/analyze.py | 180 | ||||
| -rw-r--r-- | lib/spack/spack/test/monitor.py | 278 |
18 files changed, 0 insertions, 2159 deletions
diff --git a/lib/spack/docs/developer_guide.rst b/lib/spack/docs/developer_guide.rst index 671e6900f5..761599c5d9 100644 --- a/lib/spack/docs/developer_guide.rst +++ b/lib/spack/docs/developer_guide.rst @@ -107,7 +107,6 @@ with a high level view of Spack's directory structure: llnl/ <- some general-use libraries spack/ <- spack module; contains Python code - analyzers/ <- modules to run analysis on installed packages build_systems/ <- modules for different build systems cmd/ <- each file in here is a spack subcommand compilers/ <- compiler description files @@ -242,22 +241,6 @@ Unit tests Implements Spack's test suite. Add a module and put its name in the test suite in ``__init__.py`` to add more unit tests. -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Research and Monitoring Modules -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -:mod:`spack.monitor` - Contains :class:`~spack.monitor.SpackMonitorClient`. This is accessed from - the ``spack install`` and ``spack analyze`` commands to send build and - package metadata up to a `Spack Monitor - <https://github.com/spack/spack-monitor>`_ server. - - -:mod:`spack.analyzers` - A module folder with a :class:`~spack.analyzers.analyzer_base.AnalyzerBase` - that provides base functions to run, save, and (optionally) upload analysis - results to a `Spack Monitor <https://github.com/spack/spack-monitor>`_ server. - ^^^^^^^^^^^^^ Other Modules @@ -301,240 +284,6 @@ Most spack commands look something like this: The information in Package files is used at all stages in this process. -Conceptually, packages are overloaded. They contain: - -------------- -Stage objects -------------- - - -.. _writing-analyzers: - ------------------ -Writing analyzers ------------------ - -To write an analyzer, you should add a new python file to the -analyzers module directory at ``lib/spack/spack/analyzers`` . -Your analyzer should be a subclass of the :class:`AnalyzerBase <spack.analyzers.analyzer_base.AnalyzerBase>`. For example, if you want -to add an analyzer class ``Myanalyzer`` you would write to -``spack/analyzers/myanalyzer.py`` and import and -use the base as follows: - -.. code-block:: python - - from .analyzer_base import AnalyzerBase - - class Myanalyzer(AnalyzerBase): - - -Note that the class name is your module file name, all lowercase -except for the first capital letter. You can look at other analyzers in -that analyzer directory for examples. The guide here will tell you about the basic functions needed. - -^^^^^^^^^^^^^^^^^^^^^^^^^ -Analyzer Output Directory -^^^^^^^^^^^^^^^^^^^^^^^^^ - -By default, when you run ``spack analyze run`` an analyzer output directory will -be created in your spack user directory in your ``$HOME``. The reason we output here -is because the install directory might not always be writable. - -.. code-block:: console - - ~/.spack/ - analyzers - -Result files will be written here, organized in subfolders in the same structure -as the package, with each analyzer owning it's own subfolder. for example: - - -.. code-block:: console - - $ tree ~/.spack/analyzers/ - /home/spackuser/.spack/analyzers/ - └── linux-ubuntu20.04-skylake - └── gcc-9.3.0 - └── zlib-1.2.11-sl7m27mzkbejtkrajigj3a3m37ygv4u2 - ├── environment_variables - │ └── spack-analyzer-environment-variables.json - ├── install_files - │ └── spack-analyzer-install-files.json - └── libabigail - └── lib - └── spack-analyzer-libabigail-libz.so.1.2.11.xml - - -Notice that for the libabigail analyzer, since results are generated per object, -we honor the object's folder in case there are equivalently named files in -different folders. The result files are typically written as json so they can be easily read and uploaded in a future interaction with a monitor. - - -^^^^^^^^^^^^^^^^^ -Analyzer Metadata -^^^^^^^^^^^^^^^^^ - -Your analyzer is required to have the class attributes ``name``, ``outfile``, -and ``description``. These are printed to the user with they use the subcommand -``spack analyze list-analyzers``. Here is an example. -As we mentioned above, note that this analyzer would live in a module named -``libabigail.py`` in the analyzers folder so that the class can be discovered. - - -.. code-block:: python - - class Libabigail(AnalyzerBase): - - name = "libabigail" - outfile = "spack-analyzer-libabigail.json" - description = "Application Binary Interface (ABI) features for objects" - - -This means that the name and output file should be unique for your analyzer. -Note that "all" cannot be the name of an analyzer, as this key is used to indicate -that the user wants to run all analyzers. - -.. _analyzer_run_function: - - -^^^^^^^^^^^^^^^^^^^^^^^^ -An analyzer run Function -^^^^^^^^^^^^^^^^^^^^^^^^ - -The core of an analyzer is its ``run()`` function, which should accept no -arguments. You can assume your analyzer has the package spec of interest at ``self.spec`` -and it's up to the run function to generate whatever analysis data you need, -and then return the object with a key as the analyzer name. The result data -should be a list of objects, each with a name, ``analyzer_name``, ``install_file``, -and one of ``value`` or ``binary_value``. The install file should be for a relative -path, and not the absolute path. For example, let's say we extract a metric called -``metric`` for ``bin/wget`` using our analyzer ``thebest-analyzer``. -We might have data that looks like this: - -.. code-block:: python - - result = {"name": "metric", "analyzer_name": "thebest-analyzer", "value": "1", "install_file": "bin/wget"} - - -We'd then return it as follows - note that they key is the analyzer name at ``self.name``. - -.. code-block:: python - - return {self.name: result} - -This will save the complete result to the analyzer metadata folder, as described -previously. If you want support for adding a different kind of metadata (e.g., -not associated with an install file) then the monitor server would need to be updated -to support this first. - - -^^^^^^^^^^^^^^^^^^^^^^^^^ -An analyzer init Function -^^^^^^^^^^^^^^^^^^^^^^^^^ - -If you don't need any extra dependencies or checks, you can skip defining an analyzer -init function, as the base class will handle it. Typically, it will accept -a spec, and an optional output directory (if the user does not want the default -metadata folder for analyzer results). The analyzer init function should call -it's parent init, and then do any extra checks or validation that are required to -work. For example: - -.. code-block:: python - - def __init__(self, spec, dirname=None): - super(Myanalyzer, self).__init__(spec, dirname) - - # install extra dependencies, do extra preparation and checks here - - -At the end of the init, you will have available to you: - - - **self.spec**: the spec object - - **self.dirname**: an optional directory name the user as provided at init to save - - **self.output_dir**: the analyzer metadata directory, where we save by default - - **self.meta_dir**: the path to the package metadata directory (.spack) if you need it - -And can proceed to write your analyzer. - - -^^^^^^^^^^^^^^^^^^^^^^^ -Saving Analyzer Results -^^^^^^^^^^^^^^^^^^^^^^^ - -The analyzer will have ``save_result`` called, with the result object generated -to save it to the filesystem, and if the user has added the ``--monitor`` flag -to upload it to a monitor server. If your result follows an accepted result -format and you don't need to parse it further, you don't need to add this -function to your class. However, if your result data is large or otherwise -needs additional parsing, you can define it. If you define the function, it -is useful to know about the ``output_dir`` property, which you can join -with your output file relative path of choice: - -.. code-block:: python - - outfile = os.path.join(self.output_dir, "my-output-file.txt") - - -The directory will be provided by the ``output_dir`` property but it won't exist, -so you should create it: - - -.. code::block:: python - - # Create the output directory - if not os.path.exists(self._output_dir): - os.makedirs(self._output_dir) - - -If you are generating results that match to specific files in the package -install directory, you should try to maintain those paths in the case that -there are equivalently named files in different directories that would -overwrite one another. As an example of an analyzer with a custom save, -the Libabigail analyzer saves ``*.xml`` files to the analyzer metadata -folder in ``run()``, as they are either binaries, or as xml (text) would -usually be too big to pass in one request. For this reason, the files -are saved during ``run()`` and the filenames added to the result object, -and then when the result object is passed back into ``save_result()``, -we skip saving to the filesystem, and instead read the file and send -each one (separately) to the monitor: - - -.. code-block:: python - - def save_result(self, result, monitor=None, overwrite=False): - """ABI results are saved to individual files, so each one needs to be - read and uploaded. Result here should be the lookup generated in run(), - the key is the analyzer name, and each value is the result file. - We currently upload the entire xml as text because libabigail can't - easily read gzipped xml, but this will be updated when it can. - """ - if not monitor: - return - - name = self.spec.package.name - - for obj, filename in result.get(self.name, {}).items(): - - # Don't include the prefix - rel_path = obj.replace(self.spec.prefix + os.path.sep, "") - - # We've already saved the results to file during run - content = spack.monitor.read_file(filename) - - # A result needs an analyzer, value or binary_value, and name - data = {"value": content, "install_file": rel_path, "name": "abidw-xml"} - tty.info("Sending result for %s %s to monitor." % (name, rel_path)) - monitor.send_analyze_metadata(self.spec.package, {"libabigail": [data]}) - - - -Notice that this function, if you define it, requires a result object (generated by -``run()``, a monitor (if you want to send), and a boolean ``overwrite`` to be used -to check if a result exists first, and not write to it if the result exists and -overwrite is False. Also notice that since we already saved these files to the analyzer metadata folder, we return early if a monitor isn't defined, because this function serves to send results to the monitor. If you haven't saved anything to the analyzer metadata folder -yet, you might want to do that here. You should also use ``tty.info`` to give -the user a message of "Writing result to $DIRNAME." - .. _writing-commands: @@ -699,23 +448,6 @@ with a hook, and this is the purpose of this particular hook. Akin to ``on_phase_success`` we require the same variables - the package that failed, the name of the phase, and the log file where we might find errors. -""""""""""""""""""""""""""""""""" -``on_analyzer_save(pkg, result)`` -""""""""""""""""""""""""""""""""" - -After an analyzer has saved some result for a package, this hook is called, -and it provides the package that we just ran the analysis for, along with -the loaded result. Typically, a result is structured to have the name -of the analyzer as key, and the result object that is defined in detail in -:ref:`analyzer_run_function`. - -.. code-block:: python - - def on_analyzer_save(pkg, result): - """given a package and a result... - """ - print('Do something extra with a package analysis result here') - ^^^^^^^^^^^^^^^^^^^^^^ Adding a New Hook Type diff --git a/lib/spack/spack/analyzers/__init__.py b/lib/spack/spack/analyzers/__init__.py deleted file mode 100644 index 842449dbbe..0000000000 --- a/lib/spack/spack/analyzers/__init__.py +++ /dev/null @@ -1,42 +0,0 @@ -# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) - -"""This package contains code for creating analyzers to extract Application -Binary Interface (ABI) information, along with simple analyses that just load -existing metadata. -""" - -from __future__ import absolute_import - -import llnl.util.tty as tty - -import spack.paths -import spack.util.classes - -mod_path = spack.paths.analyzers_path -analyzers = spack.util.classes.list_classes("spack.analyzers", mod_path) - -# The base analyzer does not have a name, and cannot do dict comprehension -analyzer_types = {} -for a in analyzers: - if not hasattr(a, "name"): - continue - analyzer_types[a.name] = a - - -def list_all(): - """A helper function to list all analyzers and their descriptions - """ - for name, analyzer in analyzer_types.items(): - print("%-25s: %-35s" % (name, analyzer.description)) - - -def get_analyzer(name): - """Courtesy function to retrieve an analyzer, and exit on error if it - does not exist. - """ - if name in analyzer_types: - return analyzer_types[name] - tty.die("Analyzer %s does not exist" % name) diff --git a/lib/spack/spack/analyzers/analyzer_base.py b/lib/spack/spack/analyzers/analyzer_base.py deleted file mode 100644 index 52b02c6a04..0000000000 --- a/lib/spack/spack/analyzers/analyzer_base.py +++ /dev/null @@ -1,116 +0,0 @@ -# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) - -"""An analyzer base provides basic functions to run the analysis, save results, -and (optionally) interact with a Spack Monitor -""" - -import os - -import llnl.util.tty as tty - -import spack.config -import spack.hooks -import spack.monitor -import spack.util.path - - -def get_analyzer_dir(spec, analyzer_dir=None): - """ - Given a spec, return the directory to save analyzer results. - - We create the directory if it does not exist. We also check that the - spec has an associated package. An analyzer cannot be run if the spec isn't - associated with a package. If the user provides a custom analyzer_dir, - we use it over checking the config and the default at ~/.spack/analyzers - """ - # An analyzer cannot be run if the spec isn't associated with a package - if not hasattr(spec, "package") or not spec.package: - tty.die("A spec can only be analyzed with an associated package.") - - # The top level directory is in the user home, or a custom location - if not analyzer_dir: - analyzer_dir = spack.util.path.canonicalize_path( - spack.config.get('config:analyzers_dir', '~/.spack/analyzers')) - - # We follow the same convention as the spec install (this could be better) - package_prefix = os.sep.join(spec.package.prefix.split('/')[-3:]) - meta_dir = os.path.join(analyzer_dir, package_prefix) - return meta_dir - - -class AnalyzerBase(object): - - def __init__(self, spec, dirname=None): - """ - Verify that the analyzer has correct metadata. - - An Analyzer is intended to run on one spec install, so the spec - with its associated package is required on init. The child analyzer - class should define an init function that super's the init here, and - also check that the analyzer has all dependencies that it - needs. If an analyzer subclass does not have dependencies, it does not - need to define an init. An Analyzer should not be allowed to proceed - if one or more dependencies are missing. The dirname, if defined, - is an optional directory name to save to (instead of the default meta - spack directory). - """ - self.spec = spec - self.dirname = dirname - self.meta_dir = os.path.dirname(spec.package.install_log_path) - - for required in ["name", "outfile", "description"]: - if not hasattr(self, required): - tty.die("Please add a %s attribute on the analyzer." % required) - - def run(self): - """ - Given a spec with an installed package, run the analyzer on it. - """ - raise NotImplementedError - - @property - def output_dir(self): - """ - The full path to the output directory. - - This includes the nested analyzer directory structure. This function - does not create anything. - """ - if not hasattr(self, "_output_dir"): - output_dir = get_analyzer_dir(self.spec, self.dirname) - self._output_dir = os.path.join(output_dir, self.name) - - return self._output_dir - - def save_result(self, result, overwrite=False): - """ - Save a result to the associated spack monitor, if defined. - - This function is on the level of the analyzer because it might be - the case that the result is large (appropriate for a single request) - or that the data is organized differently (e.g., more than one - request per result). If an analyzer subclass needs to over-write - this function with a custom save, that is appropriate to do (see abi). - """ - # We maintain the structure in json with the analyzer as key so - # that in the future, we could upload to a monitor server - if result[self.name]: - - outfile = os.path.join(self.output_dir, self.outfile) - - # Only try to create the results directory if we have a result - if not os.path.exists(self._output_dir): - os.makedirs(self._output_dir) - - # Don't overwrite an existing result if overwrite is False - if os.path.exists(outfile) and not overwrite: - tty.info("%s exists and overwrite is False, skipping." % outfile) - else: - tty.info("Writing result to %s" % outfile) - spack.monitor.write_json(result[self.name], outfile) - - # This hook runs after a save result - spack.hooks.on_analyzer_save(self.spec.package, result) diff --git a/lib/spack/spack/analyzers/config_args.py b/lib/spack/spack/analyzers/config_args.py deleted file mode 100644 index 2e41576cc6..0000000000 --- a/lib/spack/spack/analyzers/config_args.py +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) - -"""A configargs analyzer is a class of analyzer that typically just uploads -already existing metadata about config args from a package spec install -directory.""" - - -import os - -import spack.monitor - -from .analyzer_base import AnalyzerBase - - -class ConfigArgs(AnalyzerBase): - - name = "config_args" - outfile = "spack-analyzer-config-args.json" - description = "config args loaded from spack-configure-args.txt" - - def run(self): - """ - Load the configure-args.txt and save in json. - - The run function will find the spack-config-args.txt file in the - package install directory, and read it into a json structure that has - the name of the analyzer as the key. - """ - config_file = os.path.join(self.meta_dir, "spack-configure-args.txt") - return {self.name: spack.monitor.read_file(config_file)} diff --git a/lib/spack/spack/analyzers/environment_variables.py b/lib/spack/spack/analyzers/environment_variables.py deleted file mode 100644 index fc73896ba7..0000000000 --- a/lib/spack/spack/analyzers/environment_variables.py +++ /dev/null @@ -1,54 +0,0 @@ -# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) - -"""An environment analyzer will read and parse the environment variables -file in the installed package directory, generating a json file that has -an index of key, value pairs for environment variables.""" - - -import os - -import llnl.util.tty as tty - -from spack.util.environment import EnvironmentModifications - -from .analyzer_base import AnalyzerBase - - -class EnvironmentVariables(AnalyzerBase): - - name = "environment_variables" - outfile = "spack-analyzer-environment-variables.json" - description = "environment variables parsed from spack-build-env.txt" - - def run(self): - """ - Load, parse, and save spack-build-env.txt to analyzers. - - Read in the spack-build-env.txt file from the package install - directory and parse the environment variables into key value pairs. - The result should have the key for the analyzer, the name. - """ - env_file = os.path.join(self.meta_dir, "spack-build-env.txt") - return {self.name: self._read_environment_file(env_file)} - - def _read_environment_file(self, filename): - """ - Read and parse the environment file. - - Given an environment file, we want to read it, split by semicolons - and new lines, and then parse down to the subset of SPACK_* variables. - We assume that all spack prefix variables are not secrets, and unlike - the install_manifest.json, we don't (at least to start) parse the values - to remove path prefixes specific to user systems. - """ - if not os.path.exists(filename): - tty.warn("No environment file available") - return - - mods = EnvironmentModifications.from_sourcing_file(filename) - env = {} - mods.apply_modifications(env) - return env diff --git a/lib/spack/spack/analyzers/install_files.py b/lib/spack/spack/analyzers/install_files.py deleted file mode 100644 index 44ef66b0bc..0000000000 --- a/lib/spack/spack/analyzers/install_files.py +++ /dev/null @@ -1,31 +0,0 @@ -# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) - -"""The install files json file (install_manifest.json) already exists in -the package install folder, so this analyzer simply moves it to the user -analyzer folder for further processing.""" - - -import os - -import spack.monitor - -from .analyzer_base import AnalyzerBase - - -class InstallFiles(AnalyzerBase): - - name = "install_files" - outfile = "spack-analyzer-install-files.json" - description = "install file listing read from install_manifest.json" - - def run(self): - """ - Load in the install_manifest.json and save to analyzers. - - We write it out to the analyzers folder, with key as the analyzer name. - """ - manifest_file = os.path.join(self.meta_dir, "install_manifest.json") - return {self.name: spack.monitor.read_json(manifest_file)} diff --git a/lib/spack/spack/analyzers/libabigail.py b/lib/spack/spack/analyzers/libabigail.py deleted file mode 100644 index 92e2faee0e..0000000000 --- a/lib/spack/spack/analyzers/libabigail.py +++ /dev/null @@ -1,114 +0,0 @@ -# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) -import os - -import llnl.util.tty as tty - -import spack -import spack.binary_distribution -import spack.bootstrap -import spack.error -import spack.hooks -import spack.monitor -import spack.package_base -import spack.repo -import spack.util.executable - -from .analyzer_base import AnalyzerBase - - -class Libabigail(AnalyzerBase): - - name = "libabigail" - outfile = "spack-analyzer-libabigail.json" - description = "Application Binary Interface (ABI) features for objects" - - def __init__(self, spec, dirname=None): - """ - init for an analyzer ensures we have all needed dependencies. - - For the libabigail analyzer, this means Libabigail. - Since the output for libabigail is one file per object, we communicate - with the monitor multiple times. - """ - super(Libabigail, self).__init__(spec, dirname) - - # This doesn't seem to work to import on the module level - tty.debug("Preparing to use Libabigail, will install if missing.") - - with spack.bootstrap.ensure_bootstrap_configuration(): - # libabigail won't install lib/bin/share without docs - spec = spack.spec.Spec("libabigail+docs") - spack.bootstrap.ensure_executables_in_path_or_raise( - ["abidw"], abstract_spec=spec - ) - self.abidw = spack.util.executable.which('abidw') - - def run(self): - """ - Run libabigail, and save results to filename. - - This run function differs in that we write as we generate and then - return a dict with the analyzer name as the key, and the value of a - dict of results, where the key is the object name, and the value is - the output file written to. - """ - manifest = spack.binary_distribution.get_buildfile_manifest(self.spec) - - # This result will store a path to each file - result = {} - - # Generate an output file for each binary or object - for obj in manifest.get("binary_to_relocate_fullpath", []): - - # We want to preserve the path in the install directory in case - # a library has an equivalenly named lib or executable, for example - outdir = os.path.dirname(obj.replace(self.spec.package.prefix, - '').strip(os.path.sep)) - outfile = "spack-analyzer-libabigail-%s.xml" % os.path.basename(obj) - outfile = os.path.join(self.output_dir, outdir, outfile) - outdir = os.path.dirname(outfile) - - # Create the output directory - if not os.path.exists(outdir): - os.makedirs(outdir) - - # Sometimes libabigail segfaults and dumps - try: - self.abidw(obj, "--out-file", outfile) - result[obj] = outfile - tty.info("Writing result to %s" % outfile) - except spack.error.SpackError: - tty.warn("Issue running abidw for %s" % obj) - - return {self.name: result} - - def save_result(self, result, overwrite=False): - """ - Read saved ABI results and upload to monitor server. - - ABI results are saved to individual files, so each one needs to be - read and uploaded. Result here should be the lookup generated in run(), - the key is the analyzer name, and each value is the result file. - We currently upload the entire xml as text because libabigail can't - easily read gzipped xml, but this will be updated when it can. - """ - if not spack.monitor.cli: - return - - name = self.spec.package.name - - for obj, filename in result.get(self.name, {}).items(): - - # Don't include the prefix - rel_path = obj.replace(self.spec.prefix + os.path.sep, "") - - # We've already saved the results to file during run - content = spack.monitor.read_file(filename) - - # A result needs an analyzer, value or binary_value, and name - data = {"value": content, "install_file": rel_path, "name": "abidw-xml"} - tty.info("Sending result for %s %s to monitor." % (name, rel_path)) - spack.hooks.on_analyzer_save(self.spec.package, {"libabigail": [data]}) diff --git a/lib/spack/spack/cmd/analyze.py b/lib/spack/spack/cmd/analyze.py deleted file mode 100644 index 3112c72d67..0000000000 --- a/lib/spack/spack/cmd/analyze.py +++ /dev/null @@ -1,116 +0,0 @@ -# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) - -import sys - -import llnl.util.tty as tty - -import spack.analyzers -import spack.build_environment -import spack.cmd -import spack.cmd.common.arguments as arguments -import spack.environment as ev -import spack.fetch_strategy -import spack.monitor -import spack.paths -import spack.report - -description = "run analyzers on installed packages" -section = "analysis" -level = "long" - - -def setup_parser(subparser): - sp = subparser.add_subparsers(metavar='SUBCOMMAND', dest='analyze_command') - - sp.add_parser('list-analyzers', - description="list available analyzers", - help="show list of analyzers that are available to run.") - - # This adds the monitor group to the subparser - spack.monitor.get_monitor_group(subparser) - - # Run Parser - run_parser = sp.add_parser('run', description="run an analyzer", - help="provide the name of the analyzer to run.") - - run_parser.add_argument( - '--overwrite', action='store_true', - help="re-analyze even if the output file already exists.") - run_parser.add_argument( - '-p', '--path', default=None, - dest='path', - help="write output to a different directory than ~/.spack/analyzers") - run_parser.add_argument( - '-a', '--analyzers', default=None, - dest="analyzers", action="append", - help="add an analyzer (defaults to all available)") - arguments.add_common_arguments(run_parser, ['spec']) - - -def analyze_spec(spec, analyzers=None, outdir=None, monitor=None, overwrite=False): - """ - Do an analysis for a spec, optionally adding monitoring. - - We also allow the user to specify a custom output directory. - analyze_spec(spec, args.analyzers, args.outdir, monitor) - - Args: - spec (spack.spec.Spec): spec object of installed package - analyzers (list): list of analyzer (keys) to run - monitor (spack.monitor.SpackMonitorClient): a monitor client - overwrite (bool): overwrite result if already exists - """ - analyzers = analyzers or list(spack.analyzers.analyzer_types.keys()) - - # Load the build environment from the spec install directory, and send - # the spec to the monitor if it's not known - if monitor: - monitor.load_build_environment(spec) - monitor.new_configuration([spec]) - - for name in analyzers: - - # Instantiate the analyzer with the spec and outdir - analyzer = spack.analyzers.get_analyzer(name)(spec, outdir) - - # Run the analyzer to get a json result - results are returned as - # a dictionary with a key corresponding to the analyzer type, so - # we can just update the data - result = analyzer.run() - - # Send the result. We do them separately because: - # 1. each analyzer might have differently organized output - # 2. the size of a result can be large - analyzer.save_result(result, overwrite) - - -def analyze(parser, args, **kwargs): - - # If the user wants to list analyzers, do so and exit - if args.analyze_command == "list-analyzers": - spack.analyzers.list_all() - sys.exit(0) - - # handle active environment, if any - env = ev.active_environment() - - # Get an disambiguate spec (we should only have one) - specs = spack.cmd.parse_specs(args.spec) - if not specs: - tty.die("You must provide one or more specs to analyze.") - spec = spack.cmd.disambiguate_spec(specs[0], env) - - # The user wants to monitor builds using github.com/spack/spack-monitor - # It is instantianted once here, and then available at spack.monitor.cli - monitor = None - if args.use_monitor: - monitor = spack.monitor.get_client( - host=args.monitor_host, - prefix=args.monitor_prefix, - ) - - # Run the analysis - analyze_spec(spec, args.analyzers, args.path, monitor, args.overwrite) diff --git a/lib/spack/spack/cmd/containerize.py b/lib/spack/spack/cmd/containerize.py index 99daa6c30e..d3b717ab47 100644 --- a/lib/spack/spack/cmd/containerize.py +++ b/lib/spack/spack/cmd/containerize.py @@ -9,7 +9,6 @@ import llnl.util.tty import spack.container import spack.container.images -import spack.monitor description = ("creates recipes to build images for different" " container runtimes") @@ -18,7 +17,6 @@ level = "long" def setup_parser(subparser): - monitor_group = spack.monitor.get_monitor_group(subparser) # noqa subparser.add_argument( '--list-os', action='store_true', default=False, help='list all the OS that can be used in the bootstrap phase and exit' @@ -46,14 +44,5 @@ def containerize(parser, args): raise ValueError(msg.format(config_file)) config = spack.container.validate(config_file) - - # If we have a monitor request, add monitor metadata to config - if args.use_monitor: - config['spack']['monitor'] = { - "host": args.monitor_host, - "keep_going": args.monitor_keep_going, - "prefix": args.monitor_prefix, - "tags": args.monitor_tags - } recipe = spack.container.recipe(config, last_phase=args.last_stage) print(recipe) diff --git a/lib/spack/spack/cmd/install.py b/lib/spack/spack/cmd/install.py index 2781deb60c..fe41316330 100644 --- a/lib/spack/spack/cmd/install.py +++ b/lib/spack/spack/cmd/install.py @@ -17,7 +17,6 @@ import spack.cmd import spack.cmd.common.arguments as arguments import spack.environment as ev import spack.fetch_strategy -import spack.monitor import spack.paths import spack.report from spack.error import SpackError @@ -105,8 +104,6 @@ the dependencies""" '--cache-only', action='store_true', dest='cache_only', default=False, help="only install package from binary mirrors") - monitor_group = spack.monitor.get_monitor_group(subparser) # noqa - subparser.add_argument( '--include-build-deps', action='store_true', dest='include_build_deps', default=False, help="""include build deps when installing from cache, @@ -292,15 +289,6 @@ environment variables: parser.print_help() return - # The user wants to monitor builds using github.com/spack/spack-monitor - if args.use_monitor: - monitor = spack.monitor.get_client( - host=args.monitor_host, - prefix=args.monitor_prefix, - tags=args.monitor_tags, - save_local=args.monitor_save_local, - ) - reporter = spack.report.collect_info( spack.package_base.PackageInstaller, '_install_task', args.log_format, args) if args.log_file: @@ -341,10 +329,6 @@ environment variables: reporter.filename = default_log_file(specs[0]) reporter.specs = specs - # Tell the monitor about the specs - if args.use_monitor and specs: - monitor.new_configuration(specs) - tty.msg("Installing environment {0}".format(env.name)) with reporter('build'): env.install_all(**kwargs) @@ -390,10 +374,6 @@ environment variables: except SpackError as e: tty.debug(e) reporter.concretization_report(e.message) - - # Tell spack monitor about it - if args.use_monitor and abstract_specs: - monitor.failed_concretization(abstract_specs) raise # 2. Concrete specs from yaml files @@ -454,17 +434,4 @@ environment variables: # overwrite all concrete explicit specs from this build kwargs['overwrite'] = [spec.dag_hash() for spec in specs] - - # Update install_args with the monitor args, needed for build task - kwargs.update({ - "monitor_keep_going": args.monitor_keep_going, - "monitor_host": args.monitor_host, - "use_monitor": args.use_monitor, - "monitor_prefix": args.monitor_prefix, - }) - - # If we are using the monitor, we send configs. and create build - # The dag_hash is the main package id - if args.use_monitor and specs: - monitor.new_configuration(specs) install_specs(args, kwargs, zip(abstract_specs, specs)) diff --git a/lib/spack/spack/cmd/monitor.py b/lib/spack/spack/cmd/monitor.py deleted file mode 100644 index 2e7731145a..0000000000 --- a/lib/spack/spack/cmd/monitor.py +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) - -import spack.monitor - -description = "interact with a monitor server" -section = "analysis" -level = "long" - - -def setup_parser(subparser): - sp = subparser.add_subparsers(metavar='SUBCOMMAND', dest='monitor_command') - - # This adds the monitor group to the subparser - spack.monitor.get_monitor_group(subparser) - - # Spack Monitor Uploads - monitor_parser = sp.add_parser('upload', description="upload to spack monitor") - monitor_parser.add_argument("upload_dir", help="directory root to upload") - - -def monitor(parser, args, **kwargs): - - if args.monitor_command == "upload": - monitor = spack.monitor.get_client( - host=args.monitor_host, - prefix=args.monitor_prefix, - ) - - # Upload the directory - monitor.upload_local_save(args.upload_dir) diff --git a/lib/spack/spack/container/writers/__init__.py b/lib/spack/spack/container/writers/__init__.py index e9541e91e7..9d0f71ccd8 100644 --- a/lib/spack/spack/container/writers/__init__.py +++ b/lib/spack/spack/container/writers/__init__.py @@ -181,26 +181,6 @@ class PathContext(tengine.Context): ) @tengine.context_property - def monitor(self): - """Enable using spack monitor during build.""" - Monitor = collections.namedtuple('Monitor', [ - 'enabled', 'host', 'prefix', 'keep_going', 'tags' - ]) - monitor = self.config.get("monitor") - - # If we don't have a monitor group, cut out early. - if not monitor: - return Monitor(False, None, None, None, None) - - return Monitor( - enabled=True, - host=monitor.get('host'), - prefix=monitor.get('prefix'), - keep_going=monitor.get("keep_going"), - tags=monitor.get('tags') - ) - - @tengine.context_property def manifest(self): """The spack.yaml file that should be used in the image""" import jsonschema @@ -208,8 +188,6 @@ class PathContext(tengine.Context): # Copy in the part of spack.yaml prescribed in the configuration file manifest = copy.deepcopy(self.config) manifest.pop('container') - if "monitor" in manifest: - manifest.pop("monitor") # Ensure that a few paths are where they need to be manifest.setdefault('config', syaml.syaml_dict()) diff --git a/lib/spack/spack/hooks/__init__.py b/lib/spack/spack/hooks/__init__.py index c9b2da5a96..de3cf2fb42 100644 --- a/lib/spack/spack/hooks/__init__.py +++ b/lib/spack/spack/hooks/__init__.py @@ -21,7 +21,6 @@ Currently the following hooks are supported: * on_phase_success(pkg, phase_name, log_file) * on_phase_error(pkg, phase_name, log_file) * on_phase_error(pkg, phase_name, log_file) - * on_analyzer_save(pkg, result) * post_env_write(env) This can be used to implement support for things like module @@ -92,8 +91,5 @@ on_install_success = _HookRunner('on_install_success') on_install_failure = _HookRunner('on_install_failure') on_install_cancel = _HookRunner('on_install_cancel') -# Analyzer hooks -on_analyzer_save = _HookRunner('on_analyzer_save') - # Environment hooks post_env_write = _HookRunner('post_env_write') diff --git a/lib/spack/spack/hooks/monitor.py b/lib/spack/spack/hooks/monitor.py deleted file mode 100644 index 9ec00c435d..0000000000 --- a/lib/spack/spack/hooks/monitor.py +++ /dev/null @@ -1,85 +0,0 @@ -# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) - -import llnl.util.tty as tty - -import spack.monitor - - -def on_install_start(spec): - """On start of an install, we want to ping the server if it exists - """ - if not spack.monitor.cli: - return - - tty.debug("Running on_install_start for %s" % spec) - build_id = spack.monitor.cli.new_build(spec) - tty.verbose("Build created with id %s" % build_id) - - -def on_install_success(spec): - """On the success of an install (after everything is complete) - """ - if not spack.monitor.cli: - return - - tty.debug("Running on_install_success for %s" % spec) - result = spack.monitor.cli.update_build(spec, status="SUCCESS") - tty.verbose(result.get('message')) - - -def on_install_failure(spec): - """Triggered on failure of an install - """ - if not spack.monitor.cli: - return - - tty.debug("Running on_install_failure for %s" % spec) - result = spack.monitor.cli.fail_task(spec) - tty.verbose(result.get('message')) - - -def on_install_cancel(spec): - """Triggered on cancel of an install - """ - if not spack.monitor.cli: - return - - tty.debug("Running on_install_cancel for %s" % spec) - result = spack.monitor.cli.cancel_task(spec) - tty.verbose(result.get('message')) - - -def on_phase_success(pkg, phase_name, log_file): - """Triggered on a phase success - """ - if not spack.monitor.cli: - return - - tty.debug("Running on_phase_success %s, phase %s" % (pkg.name, phase_name)) - result = spack.monitor.cli.send_phase(pkg, phase_name, log_file, "SUCCESS") - tty.verbose(result.get('message')) - - -def on_phase_error(pkg, phase_name, log_file): - """Triggered on a phase error - """ - if not spack.monitor.cli: - return - - tty.debug("Running on_phase_error %s, phase %s" % (pkg.name, phase_name)) - result = spack.monitor.cli.send_phase(pkg, phase_name, log_file, "ERROR") - tty.verbose(result.get('message')) - - -def on_analyzer_save(pkg, result): - """given a package and a result, if we have a spack monitor, upload - the result to it. - """ - if not spack.monitor.cli: - return - - # This hook runs after a save result - spack.monitor.cli.send_analyze_metadata(pkg, result) diff --git a/lib/spack/spack/installer.py b/lib/spack/spack/installer.py index 45ee08104b..d659c5b4b8 100644 --- a/lib/spack/spack/installer.py +++ b/lib/spack/spack/installer.py @@ -49,7 +49,6 @@ import spack.binary_distribution as binary_distribution import spack.compilers import spack.error import spack.hooks -import spack.monitor import spack.package_base import spack.package_prefs as prefs import spack.repo diff --git a/lib/spack/spack/monitor.py b/lib/spack/spack/monitor.py deleted file mode 100644 index cbaec20a48..0000000000 --- a/lib/spack/spack/monitor.py +++ /dev/null @@ -1,738 +0,0 @@ -# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) - -"""Interact with a Spack Monitor Service. Derived from -https://github.com/spack/spack-monitor/blob/main/script/spackmoncli.py -""" - -import base64 -import hashlib -import os -import re -from datetime import datetime - -try: - from urllib.error import URLError - from urllib.request import Request, urlopen -except ImportError: - from urllib2 import urlopen, Request, URLError # type: ignore # novm - -from copy import deepcopy -from glob import glob - -import llnl.util.tty as tty - -import spack -import spack.config -import spack.hash_types as ht -import spack.main -import spack.paths -import spack.store -import spack.util.path -import spack.util.spack_json as sjson -import spack.util.spack_yaml as syaml - -# A global client to instantiate once -cli = None - - -def get_client(host, prefix="ms1", allow_fail=False, tags=None, save_local=False): - """ - Get a monitor client for a particular host and prefix. - - If the client is not running, we exit early, unless allow_fail is set - to true, indicating that we should continue the build even if the - server is not present. Note that this client is defined globally as "cli" - so we can istantiate it once (checking for credentials, etc.) and then - always have access to it via spack.monitor.cli. Also note that - typically, we call the monitor by way of hooks in spack.hooks.monitor. - So if you want the monitor to have a new interaction with some part of - the codebase, it's recommended to write a hook first, and then have - the monitor use it. - """ - global cli - cli = SpackMonitorClient(host=host, prefix=prefix, allow_fail=allow_fail, - tags=tags, save_local=save_local) - - # Auth is always required unless we are saving locally - if not save_local: - cli.require_auth() - - # We will exit early if the monitoring service is not running, but - # only if we aren't doing a local save - if not save_local: - info = cli.service_info() - - # If we allow failure, the response will be done - if info: - tty.debug("%s v.%s has status %s" % ( - info['id'], - info['version'], - info['status']) - ) - return cli - - -def get_monitor_group(subparser): - """ - Retrieve the monitor group for the argument parser. - - Since the monitor group is shared between commands, we provide a common - function to generate the group for it. The user can pass the subparser, and - the group is added, and returned. - """ - # Monitoring via https://github.com/spack/spack-monitor - monitor_group = subparser.add_argument_group() - monitor_group.add_argument( - '--monitor', action='store_true', dest='use_monitor', default=False, - help="interact with a monitor server during builds.") - monitor_group.add_argument( - '--monitor-save-local', action='store_true', dest='monitor_save_local', - default=False, help="save monitor results to .spack instead of server.") - monitor_group.add_argument( - '--monitor-tags', dest='monitor_tags', default=None, - help="One or more (comma separated) tags for a build.") - monitor_group.add_argument( - '--monitor-keep-going', action='store_true', dest='monitor_keep_going', - default=False, help="continue the build if a request to monitor fails.") - monitor_group.add_argument( - '--monitor-host', dest='monitor_host', default="http://127.0.0.1", - help="If using a monitor, customize the host.") - monitor_group.add_argument( - '--monitor-prefix', dest='monitor_prefix', default="ms1", - help="The API prefix for the monitor service.") - return monitor_group - - -class SpackMonitorClient: - """Client to interact with a spack monitor server. - - We require the host url, along with the prefix to discover the - service_info endpoint. If allow_fail is set to True, we will not exit - on error with tty.die given that a request is not successful. The spack - version is one of the fields to uniquely identify a spec, so we add it - to the client on init. - """ - - def __init__(self, host=None, prefix="ms1", allow_fail=False, tags=None, - save_local=False): - # We can control setting an arbitrary version if needed - sv = spack.main.get_version() - self.spack_version = os.environ.get("SPACKMON_SPACK_VERSION") or sv - - self.host = host or "http://127.0.0.1" - self.baseurl = "%s/%s" % (self.host, prefix.strip("/")) - self.token = os.environ.get("SPACKMON_TOKEN") - self.username = os.environ.get("SPACKMON_USER") - self.headers = {} - self.allow_fail = allow_fail - self.capture_build_environment() - self.tags = tags - self.save_local = save_local - - # We key lookup of build_id by dag_hash - self.build_ids = {} - self.setup_save() - - def setup_save(self): - """Given a local save "save_local" ensure the output directory exists. - """ - if not self.save_local: - return - - save_dir = spack.util.path.canonicalize_path( - spack.config.get('config:monitor_dir', spack.paths.default_monitor_path) - ) - - # Name based on timestamp - now = datetime.now().strftime('%Y-%m-%d-%H-%M-%S-%s') - self.save_dir = os.path.join(save_dir, now) - if not os.path.exists(self.save_dir): - os.makedirs(self.save_dir) - - def save(self, obj, filename): - """ - Save a monitor json result to the save directory. - """ - filename = os.path.join(self.save_dir, filename) - write_json(obj, filename) - return {"message": "Build saved locally to %s" % filename} - - def load_build_environment(self, spec): - """ - Load a build environment from install_environment.json. - - If we are running an analyze command, we will need to load previously - used build environment metadata from install_environment.json to capture - what was done during the build. - """ - if not hasattr(spec, "package") or not spec.package: - tty.die("A spec must have a package to load the environment.") - - pkg_dir = os.path.dirname(spec.package.install_log_path) - env_file = os.path.join(pkg_dir, "install_environment.json") - build_environment = read_json(env_file) - if not build_environment: - tty.warn( - "install_environment.json not found in package folder. " - " This means that the current environment metadata will be used." - ) - else: - self.build_environment = build_environment - - def capture_build_environment(self): - """ - Capture the environment for the build. - - This uses spack.util.environment.get_host_environment_metadata to do so. - This is important because it's a unique identifier, along with the spec, - for a Build. It should look something like this: - - {'host_os': 'ubuntu20.04', - 'platform': 'linux', - 'host_target': 'skylake', - 'hostname': 'vanessa-ThinkPad-T490s', - 'spack_version': '0.16.1-1455-52d5b55b65', - 'kernel_version': '#73-Ubuntu SMP Mon Jan 18 17:25:17 UTC 2021'} - - This is saved to a package install's metadata folder as - install_environment.json, and can be loaded by the monitor for uploading - data relevant to a later analysis. - """ - from spack.util.environment import get_host_environment_metadata - self.build_environment = get_host_environment_metadata() - keys = list(self.build_environment.keys()) - - # Allow to customize any of these values via the environment - for key in keys: - envar_name = "SPACKMON_%s" % key.upper() - envar = os.environ.get(envar_name) - if envar: - self.build_environment[key] = envar - - def require_auth(self): - """ - Require authentication. - - The token and username must not be unset - """ - if not self.save_local and (not self.token or not self.username): - tty.die("You are required to export SPACKMON_TOKEN and SPACKMON_USER") - - def set_header(self, name, value): - self.headers.update({name: value}) - - def set_basic_auth(self, username, password): - """ - A wrapper to adding basic authentication to the Request - """ - auth_str = "%s:%s" % (username, password) - auth_header = base64.b64encode(auth_str.encode("utf-8")) - self.set_header("Authorization", "Basic %s" % auth_header.decode("utf-8")) - - def reset(self): - """ - Reset and prepare for a new request. - """ - if "Authorization" in self.headers: - self.headers = {"Authorization": self.headers['Authorization']} - else: - self.headers = {} - - def prepare_request(self, endpoint, data, headers): - """ - Prepare a request given an endpoint, data, and headers. - - If data is provided, urllib makes the request a POST - """ - # Always reset headers for new request. - self.reset() - - # Preserve previously used auth token - headers = headers or self.headers - - # The calling function can provide a full or partial url - if not endpoint.startswith("http"): - endpoint = "%s/%s" % (self.baseurl, endpoint) - - # If we have data, the request will be POST - if data: - if not isinstance(data, str): - data = sjson.dump(data) - data = data.encode('ascii') - - return Request(endpoint, data=data, headers=headers) - - def issue_request(self, request, retry=True): - """ - Given a prepared request, issue it. - - If we get an error, die. If - there are times when we don't want to exit on error (but instead - disable using the monitoring service) we could add that here. - """ - try: - response = urlopen(request) - except URLError as e: - - # If we have an authorization request, retry once with auth - if hasattr(e, "code") and e.code == 401 and retry: - if self.authenticate_request(e): - request = self.prepare_request( - e.url, - sjson.load(request.data.decode('utf-8')), - self.headers - ) - return self.issue_request(request, False) - - # Handle permanent re-directs! - elif hasattr(e, "code") and e.code == 308: - location = e.headers.get('Location') - - request_data = None - if request.data: - request_data = sjson.load(request.data.decode('utf-8'))[0] - - if location: - request = self.prepare_request( - location, - request_data, - self.headers - ) - return self.issue_request(request, True) - - # Otherwise, relay the message and exit on error - msg = "" - if hasattr(e, 'reason'): - msg = e.reason - elif hasattr(e, 'code'): - msg = e.code - - # If we can parse the message, try it - try: - msg += "\n%s" % e.read().decode("utf8", 'ignore') - except Exception: - pass - - if self.allow_fail: - tty.warning("Request to %s was not successful, but continuing." % e.url) - return - - tty.die(msg) - - return response - - def do_request(self, endpoint, data=None, headers=None, url=None): - """ - Do the actual request. - - If data is provided, it is POST, otherwise GET. - If an entire URL is provided, don't use the endpoint - """ - request = self.prepare_request(endpoint, data, headers) - - # If we have an authorization error, we retry with - response = self.issue_request(request) - - # A 200/201 response incidates success - if response.code in [200, 201]: - return sjson.load(response.read().decode('utf-8')) - - return response - - def authenticate_request(self, originalResponse): - """ - Authenticate the request. - - Given a response (an HTTPError 401), look for a Www-Authenticate - header to parse. We return True/False to indicate if the request - should be retried. - """ - authHeaderRaw = originalResponse.headers.get("Www-Authenticate") - if not authHeaderRaw: - return False - - # If we have a username and password, set basic auth automatically - if self.token and self.username: - self.set_basic_auth(self.username, self.token) - - headers = deepcopy(self.headers) - if "Authorization" not in headers: - tty.error( - "This endpoint requires a token. Please set " - "client.set_basic_auth(username, password) first " - "or export them to the environment." - ) - return False - - # Prepare request to retry - h = parse_auth_header(authHeaderRaw) - headers.update({ - "service": h.Service, - "Accept": "application/json", - "User-Agent": "spackmoncli"} - ) - - # Currently we don't set a scope (it defaults to build) - authResponse = self.do_request(h.Realm, headers=headers) - - # Request the token - token = authResponse.get("token") - if not token: - return False - - # Set the token to the original request and retry - self.headers.update({"Authorization": "Bearer %s" % token}) - return True - - # Functions correspond to endpoints - def service_info(self): - """ - Get the service information endpoint - """ - # Base endpoint provides service info - return self.do_request("") - - def new_configuration(self, specs): - """ - Given a list of specs, generate a new configuration for each. - - We return a lookup of specs with their package names. This assumes - that we are only installing one version of each package. We aren't - starting or creating any builds, so we don't need a build environment. - """ - configs = {} - - # There should only be one spec generally (what cases would have >1?) - for spec in specs: - # Not sure if this is needed here, but I see it elsewhere - if spec.name in spack.repo.path or spec.virtual: - spec.concretize() - - # Remove extra level of nesting - # This is the only place in Spack we still use full_hash, as `spack monitor` - # requires specs with full_hash-keyed dependencies. - as_dict = {"spec": spec.to_dict(hash=ht.full_hash)['spec'], - "spack_version": self.spack_version} - - if self.save_local: - filename = "spec-%s-%s-config.json" % (spec.name, spec.version) - self.save(as_dict, filename) - else: - response = self.do_request("specs/new/", data=sjson.dump(as_dict)) - configs[spec.package.name] = response.get('data', {}) - - return configs - - def failed_concretization(self, specs): - """ - Given a list of abstract specs, tell spack monitor concretization failed. - """ - configs = {} - - # There should only be one spec generally (what cases would have >1?) - for spec in specs: - - # update the spec to have build hash indicating that cannot be built - meta = spec.to_dict()['spec'] - nodes = [] - for node in meta.get("nodes", []): - node["full_hash"] = "FAILED_CONCRETIZATION" - nodes.append(node) - meta['nodes'] = nodes - - # We can't concretize / hash - as_dict = {"spec": meta, - "spack_version": self.spack_version} - - if self.save_local: - filename = "spec-%s-%s-config.json" % (spec.name, spec.version) - self.save(as_dict, filename) - else: - response = self.do_request("specs/new/", data=sjson.dump(as_dict)) - configs[spec.package.name] = response.get('data', {}) - - return configs - - def new_build(self, spec): - """ - Create a new build. - - This means sending the hash of the spec to be built, - along with the build environment. These two sets of data uniquely can - identify the build, and we will add objects (the binaries produced) to - it. We return the build id to the calling client. - """ - return self.get_build_id(spec, return_response=True) - - def get_build_id(self, spec, return_response=False, spec_exists=True): - """ - Retrieve a build id, either in the local cache, or query the server. - """ - dag_hash = spec.dag_hash() - if dag_hash in self.build_ids: - return self.build_ids[dag_hash] - - # Prepare build environment data (including spack version) - data = self.build_environment.copy() - data['full_hash'] = dag_hash - - # If the build should be tagged, add it - if self.tags: - data['tags'] = self.tags - - # If we allow the spec to not exist (meaning we create it) we need to - # include the full specfile here - if not spec_exists: - meta_dir = os.path.dirname(spec.package.install_log_path) - spec_file = os.path.join(meta_dir, "spec.json") - if os.path.exists(spec_file): - data['spec'] = sjson.load(read_file(spec_file)) - else: - spec_file = os.path.join(meta_dir, "spec.yaml") - data['spec'] = syaml.load(read_file(spec_file)) - - if self.save_local: - return self.get_local_build_id(data, dag_hash, return_response) - return self.get_server_build_id(data, dag_hash, return_response) - - def get_local_build_id(self, data, dag_hash, return_response): - """ - Generate a local build id based on hashing the expected data - """ - hasher = hashlib.md5() - hasher.update(str(data).encode('utf-8')) - bid = hasher.hexdigest() - filename = "build-metadata-%s.json" % bid - response = self.save(data, filename) - if return_response: - return response - return bid - - def get_server_build_id(self, data, dag_hash, return_response=False): - """ - Retrieve a build id from the spack monitor server - """ - response = self.do_request("builds/new/", data=sjson.dump(data)) - - # Add the build id to the lookup - bid = self.build_ids[dag_hash] = response['data']['build']['build_id'] - self.build_ids[dag_hash] = bid - - # If the function is called directly, the user might want output - if return_response: - return response - return bid - - def update_build(self, spec, status="SUCCESS"): - """ - Update a build with a new status. - - This typically updates the relevant package to indicate a - successful install. This endpoint can take a general status to update. - """ - data = {"build_id": self.get_build_id(spec), "status": status} - if self.save_local: - filename = "build-%s-status.json" % data['build_id'] - return self.save(data, filename) - - return self.do_request("builds/update/", data=sjson.dump(data)) - - def fail_task(self, spec): - """Given a spec, mark it as failed. This means that Spack Monitor - marks all dependencies as cancelled, unless they are already successful - """ - return self.update_build(spec, status="FAILED") - - def cancel_task(self, spec): - """Given a spec, mark it as cancelled. - """ - return self.update_build(spec, status="CANCELLED") - - def send_analyze_metadata(self, pkg, metadata): - """ - Send spack analyzer metadata to the spack monitor server. - - Given a dictionary of analyzers (with key as analyzer type, and - value as the data) upload the analyzer output to Spack Monitor. - Spack Monitor should either have a known understanding of the analyzer, - or if not (the key is not recognized), it's assumed to be a dictionary - of objects/files, each with attributes to be updated. E.g., - - {"analyzer-name": {"object-file-path": {"feature1": "value1"}}} - """ - # Prepare build environment data (including spack version) - # Since the build might not have been generated, we include the spec - data = {"build_id": self.get_build_id(pkg.spec, spec_exists=False), - "metadata": metadata} - return self.do_request("analyze/builds/", data=sjson.dump(data)) - - def send_phase(self, pkg, phase_name, phase_output_file, status): - """ - Send the result of a phase during install. - - Given a package, phase name, and status, update the monitor endpoint - to alert of the status of the stage. This includes parsing the package - metadata folder for phase output and error files - """ - data = {"build_id": self.get_build_id(pkg.spec)} - - # Send output specific to the phase (does this include error?) - data.update({"status": status, - "output": read_file(phase_output_file), - "phase_name": phase_name}) - - if self.save_local: - filename = "build-%s-phase-%s.json" % (data['build_id'], phase_name) - return self.save(data, filename) - - return self.do_request("builds/phases/update/", data=sjson.dump(data)) - - def upload_specfile(self, filename): - """ - Upload a spec file to the spack monitor server. - - Given a spec file (must be json) upload to the UploadSpec endpoint. - This function is not used in the spack to server workflow, but could - be useful is Spack Monitor is intended to send an already generated - file in some kind of separate analysis. For the environment file, we - parse out SPACK_* variables to include. - """ - # We load as json just to validate it - spec = read_json(filename) - data = {"spec": spec, "spack_verison": self.spack_version} - - if self.save_local: - filename = "spec-%s-%s.json" % (spec.name, spec.version) - return self.save(data, filename) - - return self.do_request("specs/new/", data=sjson.dump(data)) - - def iter_read(self, pattern): - """ - A helper to read json from a directory glob and return it loaded. - """ - for filename in glob(pattern): - basename = os.path.basename(filename) - tty.info("Reading %s" % basename) - yield read_json(filename) - - def upload_local_save(self, dirname): - """ - Upload results from a locally saved directory to spack monitor. - - The general workflow will first include an install with save local: - spack install --monitor --monitor-save-local - And then a request to upload the root or specific directory. - spack upload monitor ~/.spack/reports/monitor/<date>/ - """ - dirname = os.path.abspath(dirname) - if not os.path.exists(dirname): - tty.die("%s does not exist." % dirname) - - # We can't be sure the level of nesting the user has provided - # So we walk recursively through and look for build metadata - for subdir, dirs, files in os.walk(dirname): - root = os.path.join(dirname, subdir) - - # A metadata file signals a monitor export - metadata = glob("%s%sbuild-metadata*" % (root, os.sep)) - if not metadata or not files or not root or not subdir: - continue - self._upload_local_save(root) - tty.info("Upload complete") - - def _upload_local_save(self, dirname): - """ - Given a found metadata file, upload results to spack monitor. - """ - # First find all the specs - for spec in self.iter_read("%s%sspec*" % (dirname, os.sep)): - self.do_request("specs/new/", data=sjson.dump(spec)) - - # Load build metadata to generate an id - metadata = glob("%s%sbuild-metadata*" % (dirname, os.sep)) - if not metadata: - tty.die("Build metadata file(s) missing in %s" % dirname) - - # Create a build_id lookup based on hash - hashes = {} - for metafile in metadata: - data = read_json(metafile) - build = self.do_request("builds/new/", data=sjson.dump(data)) - localhash = os.path.basename(metafile).replace(".json", "") - hashes[localhash.replace('build-metadata-', "")] = build - - # Next upload build phases - for phase in self.iter_read("%s%sbuild*phase*" % (dirname, os.sep)): - build_id = hashes[phase['build_id']]['data']['build']['build_id'] - phase['build_id'] = build_id - self.do_request("builds/phases/update/", data=sjson.dump(phase)) - - # Next find the status objects - for status in self.iter_read("%s%sbuild*status*" % (dirname, os.sep)): - build_id = hashes[status['build_id']]['data']['build']['build_id'] - status['build_id'] = build_id - self.do_request("builds/update/", data=sjson.dump(status)) - - -# Helper functions - -def parse_auth_header(authHeaderRaw): - """ - Parse an authentication header into relevant pieces - """ - regex = re.compile('([a-zA-z]+)="(.+?)"') - matches = regex.findall(authHeaderRaw) - lookup = dict() - for match in matches: - lookup[match[0]] = match[1] - return authHeader(lookup) - - -class authHeader: - def __init__(self, lookup): - """Given a dictionary of values, match them to class attributes""" - for key in lookup: - if key in ["realm", "service", "scope"]: - setattr(self, key.capitalize(), lookup[key]) - - -def read_file(filename): - """ - Read a file, if it exists. Otherwise return None - """ - if not os.path.exists(filename): - return - with open(filename, 'r') as fd: - content = fd.read() - return content - - -def write_file(content, filename): - """ - Write content to file - """ - with open(filename, 'w') as fd: - fd.writelines(content) - return content - - -def write_json(obj, filename): - """ - Write a json file, if the output directory exists. - """ - if not os.path.exists(os.path.dirname(filename)): - return - return write_file(sjson.dump(obj), filename) - - -def read_json(filename): - """ - Read a file and load into json, if it exists. Otherwise return None. - """ - if not os.path.exists(filename): - return - return sjson.load(read_file(filename)) diff --git a/lib/spack/spack/test/cmd/analyze.py b/lib/spack/spack/test/cmd/analyze.py deleted file mode 100644 index 17f5331804..0000000000 --- a/lib/spack/spack/test/cmd/analyze.py +++ /dev/null @@ -1,180 +0,0 @@ -# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) - -import os -import sys - -import pytest - -import spack.cmd.install -import spack.config -import spack.package_base -import spack.util.spack_json as sjson -from spack.main import SpackCommand -from spack.spec import Spec - -install = SpackCommand('install') -analyze = SpackCommand('analyze') - -pytestmark = pytest.mark.skipif(sys.platform == 'win32', - reason="Test is unsupported on Windows") - - -def test_test_package_not_installed(mock_fetch, install_mockery_mutable_config): - # We cannot run an analysis for a package not installed - out = analyze('run', 'libdwarf', fail_on_error=False) - assert "==> Error: Spec 'libdwarf' matches no installed packages.\n" in out - - -def test_analyzer_get_install_dir(mock_fetch, install_mockery_mutable_config): - """ - Test that we cannot get an analyzer directory without a spec package. - """ - spec = Spec('libdwarf').concretized() - assert 'libdwarf' in spack.analyzers.analyzer_base.get_analyzer_dir(spec) - - # Case 1: spec is missing attribute for package - with pytest.raises(SystemExit): - spack.analyzers.analyzer_base.get_analyzer_dir(None) - - class Packageless(object): - package = None - - # Case 2: spec has package attribute, but it's None - with pytest.raises(SystemExit): - spack.analyzers.analyzer_base.get_analyzer_dir(Packageless()) - - -def test_malformed_analyzer(mock_fetch, install_mockery_mutable_config): - """ - Test that an analyzer missing needed attributes is invalid. - """ - from spack.analyzers.analyzer_base import AnalyzerBase - - # Missing attribute description - class MyAnalyzer(AnalyzerBase): - name = "my_analyzer" - outfile = "my_analyzer_output.txt" - - spec = Spec('libdwarf').concretized() - with pytest.raises(SystemExit): - MyAnalyzer(spec) - - -def test_analyze_output(tmpdir, mock_fetch, install_mockery_mutable_config): - """ - Test that an analyzer errors if requested name does not exist. - """ - install('libdwarf') - install('python@3.8') - analyzer_dir = tmpdir.join('analyzers') - - # An analyzer that doesn't exist should not work - out = analyze('run', '-a', 'pusheen', 'libdwarf', fail_on_error=False) - assert '==> Error: Analyzer pusheen does not exist\n' in out - - # We will output to this analyzer directory - analyzer_dir = tmpdir.join('analyzers') - out = analyze('run', '-a', 'install_files', '-p', str(analyzer_dir), 'libdwarf') - - # Ensure that if we run again without over write, we don't run - out = analyze('run', '-a', 'install_files', '-p', str(analyzer_dir), 'libdwarf') - assert "skipping" in out - - # With overwrite it should run - out = analyze('run', '-a', 'install_files', '-p', str(analyzer_dir), - '--overwrite', 'libdwarf') - assert "==> Writing result to" in out - - -def _run_analyzer(name, package, tmpdir): - """ - A shared function to test that an analyzer runs. - - We return the output file for further inspection. - """ - analyzer = spack.analyzers.get_analyzer(name) - analyzer_dir = tmpdir.join('analyzers') - out = analyze('run', '-a', analyzer.name, '-p', str(analyzer_dir), package) - - assert "==> Writing result to" in out - assert "/%s/%s\n" % (analyzer.name, analyzer.outfile) in out - - # The output file should exist - output_file = out.strip('\n').split(' ')[-1].strip() - assert os.path.exists(output_file) - return output_file - - -def test_installfiles_analyzer(tmpdir, mock_fetch, install_mockery_mutable_config): - """ - test the install files analyzer - """ - install('libdwarf') - output_file = _run_analyzer("install_files", "libdwarf", tmpdir) - - # Ensure it's the correct content - with open(output_file, 'r') as fd: - content = sjson.load(fd.read()) - - basenames = set() - for key, attrs in content.items(): - basenames.add(os.path.basename(key)) - - # Check for a few expected files - for key in ['.spack', 'libdwarf', 'packages', 'repo.yaml', 'repos']: - assert key in basenames - - -def test_environment_analyzer(tmpdir, mock_fetch, install_mockery_mutable_config): - """ - test the environment variables analyzer. - """ - install('libdwarf') - output_file = _run_analyzer("environment_variables", "libdwarf", tmpdir) - with open(output_file, 'r') as fd: - content = sjson.load(fd.read()) - - # Check a few expected keys - for key in ['SPACK_CC', 'SPACK_COMPILER_SPEC', 'SPACK_ENV_PATH']: - assert key in content - - # The analyzer should return no result if the output file does not exist. - spec = Spec('libdwarf').concretized() - env_file = os.path.join(spec.package.prefix, '.spack', 'spack-build-env.txt') - assert os.path.exists(env_file) - os.remove(env_file) - analyzer = spack.analyzers.get_analyzer("environment_variables") - analyzer_dir = tmpdir.join('analyzers') - result = analyzer(spec, analyzer_dir).run() - assert "environment_variables" in result - assert not result['environment_variables'] - - -def test_list_analyzers(): - """ - test that listing analyzers shows all the possible analyzers. - """ - from spack.analyzers import analyzer_types - - # all cannot be an analyzer - assert "all" not in analyzer_types - - # All types should be present! - out = analyze('list-analyzers') - for analyzer_type in analyzer_types: - assert analyzer_type in out - - -def test_configargs_analyzer(tmpdir, mock_fetch, install_mockery_mutable_config): - """ - test the config args analyzer. - - Since we don't have any, this should return an empty result. - """ - install('libdwarf') - analyzer_dir = tmpdir.join('analyzers') - out = analyze('run', '-a', 'config_args', '-p', str(analyzer_dir), 'libdwarf') - assert out == '' diff --git a/lib/spack/spack/test/monitor.py b/lib/spack/spack/test/monitor.py deleted file mode 100644 index db313e0921..0000000000 --- a/lib/spack/spack/test/monitor.py +++ /dev/null @@ -1,278 +0,0 @@ -# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) - -import os -import sys - -import pytest - -import llnl.util.tty as tty - -import spack.config -import spack.monitor -import spack.spec -from spack.main import SpackCommand -from spack.monitor import SpackMonitorClient - -install = SpackCommand('install') - - -def get_client(host, prefix="ms1", allow_fail=False, tags=None, save_local=False): - """ - We replicate this function to not generate a global client. - """ - cli = SpackMonitorClient(host=host, prefix=prefix, allow_fail=allow_fail, - tags=tags, save_local=save_local) - - # We will exit early if the monitoring service is not running, but - # only if we aren't doing a local save - if not save_local: - info = cli.service_info() - - # If we allow failure, the response will be done - if info: - tty.debug("%s v.%s has status %s" % ( - info['id'], - info['version'], - info['status']) - ) - return cli - - -@pytest.fixture -def mock_monitor_request(monkeypatch): - """ - Monitor requests that are shared across tests go here - """ - def mock_do_request(self, endpoint, *args, **kwargs): - - # monitor was originally keyed by full_hash, but now dag_hash is the full hash. - # the name of the field in monitor is still spec_full_hash, for now. - build = {"build_id": 1, - "spec_full_hash": "bpfvysmqndtmods4rmy6d6cfquwblngp", - "spec_name": "dttop"} - - # Service Info - if endpoint == "": - organization = {"name": "spack", "url": "https://github.com/spack"} - return {"id": "spackmon", "status": "running", - "name": "Spack Monitor (Spackmon)", - "description": "The best spack monitor", - "organization": organization, - "contactUrl": "https://github.com/spack/spack-monitor/issues", - "documentationUrl": "https://spack-monitor.readthedocs.io", - "createdAt": "2021-04-09T21:54:51Z", - "updatedAt": "2021-05-24T15:06:46Z", - "environment": "test", - "version": "0.0.1", - "auth_instructions_url": "url"} - - # New Build - elif endpoint == "builds/new/": - return {"message": "Build get or create was successful.", - "data": { - "build_created": True, - "build_environment_created": True, - "build": build - }, - "code": 201} - - # Update Build - elif endpoint == "builds/update/": - return {"message": "Status updated", - "data": {"build": build}, - "code": 200} - - # Send Analyze Metadata - elif endpoint == "analyze/builds/": - return {"message": "Metadata updated", - "data": {"build": build}, - "code": 200} - - # Update Build Phase - elif endpoint == "builds/phases/update/": - return {"message": "Phase autoconf was successfully updated.", - "code": 200, - "data": { - "build_phase": { - "id": 1, - "status": "SUCCESS", - "name": "autoconf" - } - }} - - # Update Phase Status - elif endpoint == "phases/update/": - return {"message": "Status updated", - "data": {"build": build}, - "code": 200} - - # New Spec - elif endpoint == "specs/new/": - return {"message": "success", - "data": { - "full_hash": "bpfvysmqndtmods4rmy6d6cfquwblngp", - "name": "dttop", - "version": "1.0", - "spack_version": "0.16.0-1379-7a5351d495", - "specs": { - "dtbuild1": "btcmljubs4njhdjqt2ebd6nrtn6vsrks", - "dtlink1": "x4z6zv6lqi7cf6l4twz4bg7hj3rkqfmk", - "dtrun1": "i6inyro74p5yqigllqk5ivvwfjfsw6qz" - } - }} - else: - pytest.fail("bad endpoint: %s" % endpoint) - monkeypatch.setattr(spack.monitor.SpackMonitorClient, "do_request", mock_do_request) - - -def test_spack_monitor_auth(mock_monitor_request): - os.environ["SPACKMON_TOKEN"] = "xxxxxxxxxxxxxxxxx" - os.environ["SPACKMON_USER"] = "spackuser" - get_client(host="http://127.0.0.1") - - -def test_spack_monitor_without_auth(mock_monitor_request): - get_client(host="hostname") - - -@pytest.mark.skipif(sys.platform == 'win32', - reason="Not supported on Windows (yet)") -def test_spack_monitor_build_env(mock_monitor_request, install_mockery_mutable_config): - monitor = get_client(host="hostname") - assert hasattr(monitor, "build_environment") - for key in ["host_os", "platform", "host_target", "hostname", "spack_version", - "kernel_version"]: - assert key in monitor.build_environment - - spec = spack.spec.Spec("dttop") - spec.concretize() - # Loads the build environment from the spec install folder - monitor.load_build_environment(spec) - - -def test_spack_monitor_basic_auth(mock_monitor_request): - monitor = get_client(host="hostname") - - # Headers should be empty - assert not monitor.headers - monitor.set_basic_auth("spackuser", "password") - assert "Authorization" in monitor.headers - assert monitor.headers['Authorization'].startswith("Basic") - - -def test_spack_monitor_new_configuration(mock_monitor_request, install_mockery): - monitor = get_client(host="hostname") - spec = spack.spec.Spec("dttop") - spec.concretize() - response = monitor.new_configuration([spec]) - - # The response is a lookup of specs - assert "dttop" in response - - -def test_spack_monitor_new_build(mock_monitor_request, install_mockery_mutable_config, - install_mockery): - monitor = get_client(host="hostname") - spec = spack.spec.Spec("dttop") - spec.concretize() - response = monitor.new_build(spec) - assert "message" in response and "data" in response and "code" in response - assert response['code'] == 201 - # We should be able to get a build id - monitor.get_build_id(spec) - - -def test_spack_monitor_update_build(mock_monitor_request, install_mockery, - install_mockery_mutable_config): - monitor = get_client(host="hostname") - spec = spack.spec.Spec("dttop") - spec.concretize() - response = monitor.update_build(spec, status="SUCCESS") - assert "message" in response and "data" in response and "code" in response - assert response['code'] == 200 - - -def test_spack_monitor_fail_task(mock_monitor_request, install_mockery, - install_mockery_mutable_config): - monitor = get_client(host="hostname") - spec = spack.spec.Spec("dttop") - spec.concretize() - response = monitor.fail_task(spec) - assert "message" in response and "data" in response and "code" in response - assert response['code'] == 200 - - -def test_spack_monitor_send_analyze_metadata(monkeypatch, mock_monitor_request, - install_mockery, - install_mockery_mutable_config): - - def buildid(*args, **kwargs): - return 1 - monkeypatch.setattr(spack.monitor.SpackMonitorClient, "get_build_id", buildid) - monitor = get_client(host="hostname") - spec = spack.spec.Spec("dttop") - spec.concretize() - response = monitor.send_analyze_metadata(spec.package, metadata={"boop": "beep"}) - assert "message" in response and "data" in response and "code" in response - assert response['code'] == 200 - - -def test_spack_monitor_send_phase(mock_monitor_request, install_mockery, - install_mockery_mutable_config): - - monitor = get_client(host="hostname") - - def get_build_id(*args, **kwargs): - return 1 - - spec = spack.spec.Spec("dttop") - spec.concretize() - response = monitor.send_phase(spec.package, "autoconf", - spec.package.install_log_path, - "SUCCESS") - assert "message" in response and "data" in response and "code" in response - assert response['code'] == 200 - - -def test_spack_monitor_info(mock_monitor_request): - os.environ["SPACKMON_TOKEN"] = "xxxxxxxxxxxxxxxxx" - os.environ["SPACKMON_USER"] = "spackuser" - monitor = get_client(host="http://127.0.0.1") - info = monitor.service_info() - - for key in ['id', 'status', 'name', 'description', 'organization', - 'contactUrl', 'documentationUrl', 'createdAt', 'updatedAt', - 'environment', 'version', 'auth_instructions_url']: - assert key in info - - -@pytest.fixture(scope='session') -def test_install_monitor_save_local(install_mockery_mutable_config, - mock_fetch, tmpdir_factory): - """ - Mock installing and saving monitor results to file. - """ - reports_dir = tmpdir_factory.mktemp('reports') - spack.config.set('config:monitor_dir', str(reports_dir)) - out = install('--monitor', '--monitor-save-local', 'dttop') - assert "Successfully installed dttop" in out - - # The reports directory should not be empty (timestamped folders) - assert os.listdir(str(reports_dir)) - - # Get the spec name - spec = spack.spec.Spec("dttop") - spec.concretize() - - # Ensure we have monitor results saved - for dirname in os.listdir(str(reports_dir)): - dated_dir = os.path.join(str(reports_dir), dirname) - build_metadata = "build-metadata-%s.json" % spec.dag_hash() - assert build_metadata in os.listdir(dated_dir) - spec_file = "spec-dttop-%s-config.json" % spec.version - assert spec_file in os.listdir(dated_dir) - - spack.config.set('config:monitor_dir', "~/.spack/reports/monitor") |