Spack can be extended with external commands (#8612)

This provides a mechanism to implement a new Spack command in a
separate directory, and with a small configuration change point Spack
to the new command.

To register the command, the directory must be added to the
"extensions" section of config.yaml. The command directory name must
have the prefix "spack-", and have the following layout:

  spack-X/
    pytest.ini #optional, for testing
    X/
	  cmd/
	    name-of-command1.py
	    name-of-command2.py
	    ...
    tests/ #optional
      conftest.py
	  test_name-of-command1.py
    templates/ #optional jinja templates, if needed

And in config.yaml:

  config:
    extensions:
      - /path/to/spack-X

If the extension includes tests, you can run them via spack by adding
the --extension option, like "spack test --extension=X"

8 files changed, 225 insertions, 19 deletions

diff --git a/lib/spack/llnl/util/lang.py b/lib/spack/llnl/util/lang.py
index 21eb9ee3ba..70c646bed0 100644
--- a/lib/spack/llnl/util/lang.py
+++ b/lib/spack/llnl/util/lang.py
@@ -12,6 +12,8 @@ import collections
 import inspect
 from datetime import datetime, timedelta
 from six import string_types
+import sys
+
 
 # Ignore emacs backups when listing modules
 ignore_modules = [r'^\.#', '~$']
@@ -597,3 +599,33 @@ class LazyReference(object):
 
     def __repr__(self):
         return repr(self.ref_function())
+
+
+def load_module_from_file(module_name, module_path):
+    """Loads a python module from the path of the corresponding file.
+
+    Args:
+        module_name (str): namespace where the python module will be loaded,
+            e.g. ``foo.bar``
+        module_path (str): path of the python file containing the module
+
+    Returns:
+        A valid module object
+
+    Raises:
+        ImportError: when the module can't be loaded
+        FileNotFoundError: when module_path doesn't exist
+    """
+    if sys.version_info[0] == 3 and sys.version_info[1] >= 5:
+        import importlib.util
+        spec = importlib.util.spec_from_file_location(module_name, module_path)
+        module = importlib.util.module_from_spec(spec)
+        spec.loader.exec_module(module)
+    elif sys.version_info[0] == 3 and sys.version_info[1] < 5:
+        import importlib.machinery
+        loader = importlib.machinery.SourceFileLoader(module_name, module_path)
+        module = loader.load_module()
+    elif sys.version_info[0] == 2:
+        import imp
+        module = imp.load_source(module_name, module_path)
+    return module
diff --git a/lib/spack/spack/cmd/__init__.py b/lib/spack/spack/cmd/__init__.py
index 630a6b6e63..9d003eb656 100644
--- a/lib/spack/spack/cmd/__init__.py
+++ b/lib/spack/spack/cmd/__init__.py
@@ -17,6 +17,7 @@ from llnl.util.tty.color import colorize
 from llnl.util.filesystem import working_dir
 
 import spack.config
+import spack.extensions
 import spack.paths
 import spack.spec
 import spack.store
@@ -32,9 +33,6 @@ ignore_files = r'^\.|^__init__.py$|^#'
 SETUP_PARSER = "setup_parser"
 DESCRIPTION = "description"
 
-#: Names of all commands
-all_commands = []
-
 
 def python_name(cmd_name):
     """Convert ``-`` to ``_`` in command name, to make a valid identifier."""
@@ -60,11 +58,16 @@ def all_commands():
     global _all_commands
     if _all_commands is None:
         _all_commands = []
-        for file in os.listdir(spack.paths.command_path):
-            if file.endswith(".py") and not re.search(ignore_files, file):
-                cmd = re.sub(r'.py$', '', file)
-                _all_commands.append(cmd_name(cmd))
+        command_paths = [spack.paths.command_path]  # Built-in commands
+        command_paths += spack.extensions.get_command_paths()  # Extensions
+        for path in command_paths:
+            for file in os.listdir(path):
+                if file.endswith(".py") and not re.search(ignore_files, file):
+                    cmd = re.sub(r'.py$', '', file)
+                    _all_commands.append(cmd_name(cmd))
+
         _all_commands.sort()
+
     return _all_commands
 
 
@@ -85,10 +88,18 @@ def get_module(cmd_name):
             (contains ``-``, not ``_``).
     """
     pname = python_name(cmd_name)
-    module_name = "%s.%s" % (__name__, pname)
-    module = __import__(module_name,
-                        fromlist=[pname, SETUP_PARSER, DESCRIPTION],
-                        level=0)
+
+    try:
+        # Try to import the command from the built-in directory
+        module_name = "%s.%s" % (__name__, pname)
+        module = __import__(module_name,
+                            fromlist=[pname, SETUP_PARSER, DESCRIPTION],
+                            level=0)
+        tty.debug('Imported {0} from built-in commands'.format(pname))
+    except ImportError:
+        module = spack.extensions.get_module(cmd_name)
+        if not module:
+            raise
 
     attr_setdefault(module, SETUP_PARSER, lambda *args: None)  # null-op
     attr_setdefault(module, DESCRIPTION, "")
diff --git a/lib/spack/spack/cmd/test.py b/lib/spack/spack/cmd/test.py
index 1c1b739a62..63e3778a6b 100644
--- a/lib/spack/spack/cmd/test.py
+++ b/lib/spack/spack/cmd/test.py
@@ -35,6 +35,10 @@ def setup_parser(subparser):
         '-L', '--long-list', action='store_true', default=False,
         help="list the entire hierarchy of tests")
     subparser.add_argument(
+        '--extension', default=None,
+        help="run test for a given Spack extension"
+    )
+    subparser.add_argument(
         'tests', nargs=argparse.REMAINDER,
         help="list of tests to run (will be passed to pytest -k)")
 
@@ -77,8 +81,16 @@ def test(parser, args, unknown_args):
         pytest.main(['-h'])
         return
 
-    # pytest.ini lives in lib/spack/spack/test
-    with working_dir(spack.paths.test_path):
+    # The default is to test the core of Spack. If the option `--extension`
+    # has been used, then test that extension.
+    pytest_root = spack.paths.test_path
+    if args.extension:
+        target = args.extension
+        extensions = spack.config.get('config:extensions')
+        pytest_root = spack.extensions.path_for_extension(target, *extensions)
+
+    # pytest.ini lives in the root of the spack repository.
+    with working_dir(pytest_root):
         # --list and --long-list print the test output better.
         if args.list or args.long_list:
             do_list(args, unknown_args)
diff --git a/lib/spack/spack/extensions.py b/lib/spack/spack/extensions.py
new file mode 100644
index 0000000000..449798e94e
--- /dev/null
+++ b/lib/spack/spack/extensions.py
@@ -0,0 +1,123 @@
+# Copyright 2013-2018 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+"""Service functions and classes to implement the hooks
+for Spack's command extensions.
+"""
+import os
+import re
+
+import llnl.util.lang
+import llnl.util.tty as tty
+
+import spack.config
+
+
+extension_regexp = re.compile(r'spack-([\w]*)')
+
+
+def extension_name(path):
+    """Returns the name of the extension in the path passed as argument.
+
+    Args:
+        path (str): path where the extension resides
+
+    Returns:
+        The extension name or None if path doesn't match the format
+        for Spack's extension.
+    """
+    regexp_match = re.search(extension_regexp, os.path.basename(path))
+    if not regexp_match:
+        msg = "[FOLDER NAMING]"
+        msg += " {0} doesn't match the format for Spack's extensions"
+        tty.warn(msg.format(path))
+        return None
+    return regexp_match.group(1)
+
+
+def load_command_extension(command, path):
+    """Loads a command extension from the path passed as argument.
+
+    Args:
+        command (str): name of the command
+        path (str): base path of the command extension
+
+    Returns:
+        A valid module object if the command is found or None
+    """
+    extension = extension_name(path)
+    if not extension:
+        return None
+
+    # Compute the absolute path of the file to be loaded, along with the
+    # name of the python module where it will be stored
+    cmd_path = os.path.join(path, extension, 'cmd', command + '.py')
+    python_name = command.replace('-', '_')
+    module_name = '{0}.{1}'.format(__name__, python_name)
+
+    try:
+        module = llnl.util.lang.load_module_from_file(module_name, cmd_path)
+    except (ImportError, IOError):
+        module = None
+
+    return module
+
+
+def get_command_paths():
+    """Return the list of paths where to search for command files."""
+    command_paths = []
+    extension_paths = spack.config.get('config:extensions') or []
+
+    for path in extension_paths:
+        extension = extension_name(path)
+        if extension:
+            command_paths.append(os.path.join(path, extension, 'cmd'))
+
+    return command_paths
+
+
+def path_for_extension(target_name, *paths):
+    """Return the test root dir for a given extension.
+
+    Args:
+        target_name (str): name of the extension to test
+        *paths: paths where the extensions reside
+
+    Returns:
+        Root directory where tests should reside or None
+    """
+    for path in paths:
+        name = extension_name(path)
+        if name == target_name:
+            return path
+    else:
+        raise IOError('extension "{0}" not found'.format(target_name))
+
+
+def get_module(cmd_name):
+    """Imports the extension module for a particular command name
+    and returns it.
+
+    Args:
+        cmd_name (str): name of the command for which to get a module
+            (contains ``-``, not ``_``).
+    """
+    # If built-in failed the import search the extension
+    # directories in order
+    extensions = spack.config.get('config:extensions') or []
+    for folder in extensions:
+        module = load_command_extension(cmd_name, folder)
+        if module:
+            return module
+    else:
+        return None
+
+
+def get_template_dirs():
+    """Returns the list of directories where to search for templates
+    in extensions.
+    """
+    extension_dirs = spack.config.get('config:extensions') or []
+    extensions = [os.path.join(x, 'templates') for x in extension_dirs]
+    return extensions
diff --git a/lib/spack/spack/schema/config.py b/lib/spack/spack/schema/config.py
index 1aa54bf48e..30c0bf0591 100644
--- a/lib/spack/spack/schema/config.py
+++ b/lib/spack/spack/schema/config.py
@@ -25,6 +25,10 @@ properties = {
                     {'type': 'array',
                      'items': {'type': 'string'}}],
             },
+            'extensions': {
+                'type': 'array',
+                'items': {'type': 'string'}
+            },
             'template_dirs': {
                 'type': 'array',
                 'items': {'type': 'string'}
diff --git a/lib/spack/spack/tengine.py b/lib/spack/spack/tengine.py
index a48f9a27b7..6aceb391cd 100644
--- a/lib/spack/spack/tengine.py
+++ b/lib/spack/spack/tengine.py
@@ -2,7 +2,7 @@
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
-
+import itertools
 import textwrap
 
 import jinja2
@@ -72,8 +72,11 @@ def make_environment(dirs=None):
     """Returns an configured environment for template rendering."""
     if dirs is None:
         # Default directories where to search for templates
+        builtins = spack.config.get('config:template_dirs')
+        extensions = spack.extensions.get_template_dirs()
         dirs = [canonicalize_path(d)
-                for d in spack.config.get('config:template_dirs')]
+                for d in itertools.chain(builtins, extensions)]
+
     # Loader for the templates
     loader = jinja2.FileSystemLoader(dirs)
     # Environment of the template engine
diff --git a/lib/spack/spack/test/conftest.py b/lib/spack/spack/test/conftest.py
index ff95f463bd..b2d255c340 100644
--- a/lib/spack/spack/test/conftest.py
+++ b/lib/spack/spack/test/conftest.py
@@ -24,6 +24,7 @@ import spack.caches
 import spack.database
 import spack.directory_layout
 import spack.environment as ev
+import spack.package_prefs
 import spack.paths
 import spack.platforms.test
 import spack.repo
@@ -118,7 +119,7 @@ def mock_stage(tmpdir_factory):
 
 
 @pytest.fixture(scope='session')
-def _ignore_stage_files():
+def ignore_stage_files():
     """Session-scoped helper for check_for_leftover_stage_files.
 
     Used to track which leftover files in the stage have been seen.
@@ -145,7 +146,7 @@ def working_env():
 
 
 @pytest.fixture(scope='function', autouse=True)
-def check_for_leftover_stage_files(request, mock_stage, _ignore_stage_files):
+def check_for_leftover_stage_files(request, mock_stage, ignore_stage_files):
     """Ensure that each test leaves a clean stage when done.
 
     This can be disabled for tests that are expected to dirty the stage
@@ -160,7 +161,7 @@ def check_for_leftover_stage_files(request, mock_stage, _ignore_stage_files):
     files_in_stage = set()
     if os.path.exists(spack.paths.stage_path):
         files_in_stage = set(
-            os.listdir(spack.paths.stage_path)) - _ignore_stage_files
+            os.listdir(spack.paths.stage_path)) - ignore_stage_files
 
     if 'disable_clean_stage_check' in request.keywords:
         # clean up after tests that are expected to be dirty
@@ -168,7 +169,7 @@ def check_for_leftover_stage_files(request, mock_stage, _ignore_stage_files):
             path = os.path.join(spack.paths.stage_path, f)
             remove_whatever_it_is(path)
     else:
-        _ignore_stage_files |= files_in_stage
+        ignore_stage_files |= files_in_stage
         assert not files_in_stage
 
 
diff --git a/lib/spack/spack/test/llnl/util/lang.py b/lib/spack/spack/test/llnl/util/lang.py
index edeab72c28..cbc24b51db 100644
--- a/lib/spack/spack/test/llnl/util/lang.py
+++ b/lib/spack/spack/test/llnl/util/lang.py
@@ -5,6 +5,7 @@
 
 import pytest
 
+import os.path
 from datetime import datetime, timedelta
 
 import llnl.util.lang
@@ -16,6 +17,19 @@ def now():
     return datetime.now()
 
 
+@pytest.fixture()
+def module_path(tmpdir):
+    m = tmpdir.join('foo.py')
+    content = """
+import os.path
+
+value = 1
+path = os.path.join('/usr', 'bin')
+"""
+    m.write(content)
+    return str(m)
+
+
 def test_pretty_date():
     """Make sure pretty_date prints the right dates."""
     now = datetime.now()
@@ -110,3 +124,9 @@ def test_match_predicate():
     with pytest.raises(ValueError):
         matcher = match_predicate(object())
         matcher('foo')
+
+
+def test_load_modules_from_file(module_path):
+    foo = llnl.util.lang.load_module_from_file('foo', module_path)
+    assert foo.value == 1
+    assert foo.path == os.path.join('/usr', 'bin')