summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorMassimiliano Culpo <massimiliano.culpo@gmail.com>2019-03-29 00:56:36 +0100
committerPeter Scheibel <scheibel1@llnl.gov>2019-03-28 16:56:36 -0700
commit0a006351c8283b32c5ee5d13f8417875e534c377 (patch)
treea427e210ba8493a1119d20509b0bdff7abb4cd8b
parentb2b91a1f00bb31bb1f794a3debe83a7352b4f48e (diff)
downloadspack-0a006351c8283b32c5ee5d13f8417875e534c377.tar.gz
spack-0a006351c8283b32c5ee5d13f8417875e534c377.tar.bz2
spack-0a006351c8283b32c5ee5d13f8417875e534c377.tar.xz
spack-0a006351c8283b32c5ee5d13f8417875e534c377.zip
Spack can be extended with external commands (#8612)
This provides a mechanism to implement a new Spack command in a separate directory, and with a small configuration change point Spack to the new command. To register the command, the directory must be added to the "extensions" section of config.yaml. The command directory name must have the prefix "spack-", and have the following layout: spack-X/ pytest.ini #optional, for testing X/ cmd/ name-of-command1.py name-of-command2.py ... tests/ #optional conftest.py test_name-of-command1.py templates/ #optional jinja templates, if needed And in config.yaml: config: extensions: - /path/to/spack-X If the extension includes tests, you can run them via spack by adding the --extension option, like "spack test --extension=X"
-rw-r--r--lib/spack/llnl/util/lang.py32
-rw-r--r--lib/spack/spack/cmd/__init__.py33
-rw-r--r--lib/spack/spack/cmd/test.py16
-rw-r--r--lib/spack/spack/extensions.py123
-rw-r--r--lib/spack/spack/schema/config.py4
-rw-r--r--lib/spack/spack/tengine.py7
-rw-r--r--lib/spack/spack/test/conftest.py9
-rw-r--r--lib/spack/spack/test/llnl/util/lang.py20
8 files changed, 225 insertions, 19 deletions
diff --git a/lib/spack/llnl/util/lang.py b/lib/spack/llnl/util/lang.py
index 21eb9ee3ba..70c646bed0 100644
--- a/lib/spack/llnl/util/lang.py
+++ b/lib/spack/llnl/util/lang.py
@@ -12,6 +12,8 @@ import collections
import inspect
from datetime import datetime, timedelta
from six import string_types
+import sys
+
# Ignore emacs backups when listing modules
ignore_modules = [r'^\.#', '~$']
@@ -597,3 +599,33 @@ class LazyReference(object):
def __repr__(self):
return repr(self.ref_function())
+
+
+def load_module_from_file(module_name, module_path):
+ """Loads a python module from the path of the corresponding file.
+
+ Args:
+ module_name (str): namespace where the python module will be loaded,
+ e.g. ``foo.bar``
+ module_path (str): path of the python file containing the module
+
+ Returns:
+ A valid module object
+
+ Raises:
+ ImportError: when the module can't be loaded
+ FileNotFoundError: when module_path doesn't exist
+ """
+ if sys.version_info[0] == 3 and sys.version_info[1] >= 5:
+ import importlib.util
+ spec = importlib.util.spec_from_file_location(module_name, module_path)
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ elif sys.version_info[0] == 3 and sys.version_info[1] < 5:
+ import importlib.machinery
+ loader = importlib.machinery.SourceFileLoader(module_name, module_path)
+ module = loader.load_module()
+ elif sys.version_info[0] == 2:
+ import imp
+ module = imp.load_source(module_name, module_path)
+ return module
diff --git a/lib/spack/spack/cmd/__init__.py b/lib/spack/spack/cmd/__init__.py
index 630a6b6e63..9d003eb656 100644
--- a/lib/spack/spack/cmd/__init__.py
+++ b/lib/spack/spack/cmd/__init__.py
@@ -17,6 +17,7 @@ from llnl.util.tty.color import colorize
from llnl.util.filesystem import working_dir
import spack.config
+import spack.extensions
import spack.paths
import spack.spec
import spack.store
@@ -32,9 +33,6 @@ ignore_files = r'^\.|^__init__.py$|^#'
SETUP_PARSER = "setup_parser"
DESCRIPTION = "description"
-#: Names of all commands
-all_commands = []
-
def python_name(cmd_name):
"""Convert ``-`` to ``_`` in command name, to make a valid identifier."""
@@ -60,11 +58,16 @@ def all_commands():
global _all_commands
if _all_commands is None:
_all_commands = []
- for file in os.listdir(spack.paths.command_path):
- if file.endswith(".py") and not re.search(ignore_files, file):
- cmd = re.sub(r'.py$', '', file)
- _all_commands.append(cmd_name(cmd))
+ command_paths = [spack.paths.command_path] # Built-in commands
+ command_paths += spack.extensions.get_command_paths() # Extensions
+ for path in command_paths:
+ for file in os.listdir(path):
+ if file.endswith(".py") and not re.search(ignore_files, file):
+ cmd = re.sub(r'.py$', '', file)
+ _all_commands.append(cmd_name(cmd))
+
_all_commands.sort()
+
return _all_commands
@@ -85,10 +88,18 @@ def get_module(cmd_name):
(contains ``-``, not ``_``).
"""
pname = python_name(cmd_name)
- module_name = "%s.%s" % (__name__, pname)
- module = __import__(module_name,
- fromlist=[pname, SETUP_PARSER, DESCRIPTION],
- level=0)
+
+ try:
+ # Try to import the command from the built-in directory
+ module_name = "%s.%s" % (__name__, pname)
+ module = __import__(module_name,
+ fromlist=[pname, SETUP_PARSER, DESCRIPTION],
+ level=0)
+ tty.debug('Imported {0} from built-in commands'.format(pname))
+ except ImportError:
+ module = spack.extensions.get_module(cmd_name)
+ if not module:
+ raise
attr_setdefault(module, SETUP_PARSER, lambda *args: None) # null-op
attr_setdefault(module, DESCRIPTION, "")
diff --git a/lib/spack/spack/cmd/test.py b/lib/spack/spack/cmd/test.py
index 1c1b739a62..63e3778a6b 100644
--- a/lib/spack/spack/cmd/test.py
+++ b/lib/spack/spack/cmd/test.py
@@ -35,6 +35,10 @@ def setup_parser(subparser):
'-L', '--long-list', action='store_true', default=False,
help="list the entire hierarchy of tests")
subparser.add_argument(
+ '--extension', default=None,
+ help="run test for a given Spack extension"
+ )
+ subparser.add_argument(
'tests', nargs=argparse.REMAINDER,
help="list of tests to run (will be passed to pytest -k)")
@@ -77,8 +81,16 @@ def test(parser, args, unknown_args):
pytest.main(['-h'])
return
- # pytest.ini lives in lib/spack/spack/test
- with working_dir(spack.paths.test_path):
+ # The default is to test the core of Spack. If the option `--extension`
+ # has been used, then test that extension.
+ pytest_root = spack.paths.test_path
+ if args.extension:
+ target = args.extension
+ extensions = spack.config.get('config:extensions')
+ pytest_root = spack.extensions.path_for_extension(target, *extensions)
+
+ # pytest.ini lives in the root of the spack repository.
+ with working_dir(pytest_root):
# --list and --long-list print the test output better.
if args.list or args.long_list:
do_list(args, unknown_args)
diff --git a/lib/spack/spack/extensions.py b/lib/spack/spack/extensions.py
new file mode 100644
index 0000000000..449798e94e
--- /dev/null
+++ b/lib/spack/spack/extensions.py
@@ -0,0 +1,123 @@
+# Copyright 2013-2018 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+"""Service functions and classes to implement the hooks
+for Spack's command extensions.
+"""
+import os
+import re
+
+import llnl.util.lang
+import llnl.util.tty as tty
+
+import spack.config
+
+
+extension_regexp = re.compile(r'spack-([\w]*)')
+
+
+def extension_name(path):
+ """Returns the name of the extension in the path passed as argument.
+
+ Args:
+ path (str): path where the extension resides
+
+ Returns:
+ The extension name or None if path doesn't match the format
+ for Spack's extension.
+ """
+ regexp_match = re.search(extension_regexp, os.path.basename(path))
+ if not regexp_match:
+ msg = "[FOLDER NAMING]"
+ msg += " {0} doesn't match the format for Spack's extensions"
+ tty.warn(msg.format(path))
+ return None
+ return regexp_match.group(1)
+
+
+def load_command_extension(command, path):
+ """Loads a command extension from the path passed as argument.
+
+ Args:
+ command (str): name of the command
+ path (str): base path of the command extension
+
+ Returns:
+ A valid module object if the command is found or None
+ """
+ extension = extension_name(path)
+ if not extension:
+ return None
+
+ # Compute the absolute path of the file to be loaded, along with the
+ # name of the python module where it will be stored
+ cmd_path = os.path.join(path, extension, 'cmd', command + '.py')
+ python_name = command.replace('-', '_')
+ module_name = '{0}.{1}'.format(__name__, python_name)
+
+ try:
+ module = llnl.util.lang.load_module_from_file(module_name, cmd_path)
+ except (ImportError, IOError):
+ module = None
+
+ return module
+
+
+def get_command_paths():
+ """Return the list of paths where to search for command files."""
+ command_paths = []
+ extension_paths = spack.config.get('config:extensions') or []
+
+ for path in extension_paths:
+ extension = extension_name(path)
+ if extension:
+ command_paths.append(os.path.join(path, extension, 'cmd'))
+
+ return command_paths
+
+
+def path_for_extension(target_name, *paths):
+ """Return the test root dir for a given extension.
+
+ Args:
+ target_name (str): name of the extension to test
+ *paths: paths where the extensions reside
+
+ Returns:
+ Root directory where tests should reside or None
+ """
+ for path in paths:
+ name = extension_name(path)
+ if name == target_name:
+ return path
+ else:
+ raise IOError('extension "{0}" not found'.format(target_name))
+
+
+def get_module(cmd_name):
+ """Imports the extension module for a particular command name
+ and returns it.
+
+ Args:
+ cmd_name (str): name of the command for which to get a module
+ (contains ``-``, not ``_``).
+ """
+ # If built-in failed the import search the extension
+ # directories in order
+ extensions = spack.config.get('config:extensions') or []
+ for folder in extensions:
+ module = load_command_extension(cmd_name, folder)
+ if module:
+ return module
+ else:
+ return None
+
+
+def get_template_dirs():
+ """Returns the list of directories where to search for templates
+ in extensions.
+ """
+ extension_dirs = spack.config.get('config:extensions') or []
+ extensions = [os.path.join(x, 'templates') for x in extension_dirs]
+ return extensions
diff --git a/lib/spack/spack/schema/config.py b/lib/spack/spack/schema/config.py
index 1aa54bf48e..30c0bf0591 100644
--- a/lib/spack/spack/schema/config.py
+++ b/lib/spack/spack/schema/config.py
@@ -25,6 +25,10 @@ properties = {
{'type': 'array',
'items': {'type': 'string'}}],
},
+ 'extensions': {
+ 'type': 'array',
+ 'items': {'type': 'string'}
+ },
'template_dirs': {
'type': 'array',
'items': {'type': 'string'}
diff --git a/lib/spack/spack/tengine.py b/lib/spack/spack/tengine.py
index a48f9a27b7..6aceb391cd 100644
--- a/lib/spack/spack/tengine.py
+++ b/lib/spack/spack/tengine.py
@@ -2,7 +2,7 @@
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
-
+import itertools
import textwrap
import jinja2
@@ -72,8 +72,11 @@ def make_environment(dirs=None):
"""Returns an configured environment for template rendering."""
if dirs is None:
# Default directories where to search for templates
+ builtins = spack.config.get('config:template_dirs')
+ extensions = spack.extensions.get_template_dirs()
dirs = [canonicalize_path(d)
- for d in spack.config.get('config:template_dirs')]
+ for d in itertools.chain(builtins, extensions)]
+
# Loader for the templates
loader = jinja2.FileSystemLoader(dirs)
# Environment of the template engine
diff --git a/lib/spack/spack/test/conftest.py b/lib/spack/spack/test/conftest.py
index ff95f463bd..b2d255c340 100644
--- a/lib/spack/spack/test/conftest.py
+++ b/lib/spack/spack/test/conftest.py
@@ -24,6 +24,7 @@ import spack.caches
import spack.database
import spack.directory_layout
import spack.environment as ev
+import spack.package_prefs
import spack.paths
import spack.platforms.test
import spack.repo
@@ -118,7 +119,7 @@ def mock_stage(tmpdir_factory):
@pytest.fixture(scope='session')
-def _ignore_stage_files():
+def ignore_stage_files():
"""Session-scoped helper for check_for_leftover_stage_files.
Used to track which leftover files in the stage have been seen.
@@ -145,7 +146,7 @@ def working_env():
@pytest.fixture(scope='function', autouse=True)
-def check_for_leftover_stage_files(request, mock_stage, _ignore_stage_files):
+def check_for_leftover_stage_files(request, mock_stage, ignore_stage_files):
"""Ensure that each test leaves a clean stage when done.
This can be disabled for tests that are expected to dirty the stage
@@ -160,7 +161,7 @@ def check_for_leftover_stage_files(request, mock_stage, _ignore_stage_files):
files_in_stage = set()
if os.path.exists(spack.paths.stage_path):
files_in_stage = set(
- os.listdir(spack.paths.stage_path)) - _ignore_stage_files
+ os.listdir(spack.paths.stage_path)) - ignore_stage_files
if 'disable_clean_stage_check' in request.keywords:
# clean up after tests that are expected to be dirty
@@ -168,7 +169,7 @@ def check_for_leftover_stage_files(request, mock_stage, _ignore_stage_files):
path = os.path.join(spack.paths.stage_path, f)
remove_whatever_it_is(path)
else:
- _ignore_stage_files |= files_in_stage
+ ignore_stage_files |= files_in_stage
assert not files_in_stage
diff --git a/lib/spack/spack/test/llnl/util/lang.py b/lib/spack/spack/test/llnl/util/lang.py
index edeab72c28..cbc24b51db 100644
--- a/lib/spack/spack/test/llnl/util/lang.py
+++ b/lib/spack/spack/test/llnl/util/lang.py
@@ -5,6 +5,7 @@
import pytest
+import os.path
from datetime import datetime, timedelta
import llnl.util.lang
@@ -16,6 +17,19 @@ def now():
return datetime.now()
+@pytest.fixture()
+def module_path(tmpdir):
+ m = tmpdir.join('foo.py')
+ content = """
+import os.path
+
+value = 1
+path = os.path.join('/usr', 'bin')
+"""
+ m.write(content)
+ return str(m)
+
+
def test_pretty_date():
"""Make sure pretty_date prints the right dates."""
now = datetime.now()
@@ -110,3 +124,9 @@ def test_match_predicate():
with pytest.raises(ValueError):
matcher = match_predicate(object())
matcher('foo')
+
+
+def test_load_modules_from_file(module_path):
+ foo = llnl.util.lang.load_module_from_file('foo', module_path)
+ assert foo.value == 1
+ assert foo.path == os.path.join('/usr', 'bin')