summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--lib/spack/docs/extensions.rst128
-rw-r--r--lib/spack/docs/index.rst1
-rw-r--r--lib/spack/spack/extensions.py41
-rw-r--r--lib/spack/spack/test/cmd_extensions.py111
4 files changed, 277 insertions, 4 deletions
diff --git a/lib/spack/docs/extensions.rst b/lib/spack/docs/extensions.rst
new file mode 100644
index 0000000000..8f4c54b435
--- /dev/null
+++ b/lib/spack/docs/extensions.rst
@@ -0,0 +1,128 @@
+.. Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+ Spack Project Developers. See the top-level COPYRIGHT file for details.
+
+ SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+.. extensions:
+
+=================
+Custom Extensions
+=================
+
+.. warning::
+
+ The support for extending Spack with custom commands is still experimental.
+ Users should expect APIs or prescribed directory structures to
+ change at any time.
+
+*Spack extensions* permit you to extend Spack capabilities by deploying your
+own custom commands or logic in an arbitrary location on your filesystem.
+This might be extremely useful e.g. to develop and maintain a command whose purpose is
+too specific to be considered for reintegration into the mainline or to
+evolve a command through its early stages before starting a discussion to merge
+it upstream.
+From Spack's point of view an extension is any path in your filesystem which
+respects a prescribed naming and layout for files:
+
+.. code-block:: console
+
+ spack-scripting/ # The top level directory must match the format 'spack-{extension_name}'
+ ├── pytest.ini # Optional file if the extension ships its own tests
+ ├── scripting # Folder that may contain modules that are needed for the extension commands
+ │   └── cmd # Folder containing extension commands
+ │   └── filter.py # A new command that will be available
+ ├── tests # Tests for this extension
+ │ ├── conftest.py
+ │ └── test_filter.py
+ └── templates # Templates that may be needed by the extension
+
+In the example above the extension named *scripting* adds an additional command (``filter``)
+and unit tests to verify its behavior. The code for this example can be
+obtained by cloning the corresponding git repository:
+
+.. TODO: write an ad-hoc "hello world" extension and make it part of the spack organization
+
+.. code-block:: console
+
+ $ pwd
+ /home/user
+ $ mkdir tmp && cd tmp
+ $ git clone https://github.com/alalazo/spack-scripting.git
+ Cloning into 'spack-scripting'...
+ remote: Counting objects: 11, done.
+ remote: Compressing objects: 100% (7/7), done.
+ remote: Total 11 (delta 0), reused 11 (delta 0), pack-reused 0
+ Receiving objects: 100% (11/11), done.
+
+As you can see by inspecting the sources, Python modules that are part of the extension
+can import any core Spack module.
+
+---------------------------------
+Configure Spack to Use Extensions
+---------------------------------
+
+To make your current Spack instance aware of extensions you should add their root
+paths to ``config.yaml``. In the case of our example this means ensuring that:
+
+.. code-block:: yaml
+
+ config:
+ extensions:
+ - /home/user/tmp/spack-scripting
+
+is part of your configuration file. Once this is setup any command that the extension provides
+will be available from the command line:
+
+.. code-block:: console
+
+ $ spack filter --help
+ usage: spack filter [-h] [--installed | --not-installed]
+ [--explicit | --implicit] [--output OUTPUT]
+ ...
+
+ filter specs based on their properties
+
+ positional arguments:
+ specs specs to be filtered
+
+ optional arguments:
+ -h, --help show this help message and exit
+ --installed select installed specs
+ --not-installed select specs that are not yet installed
+ --explicit select specs that were installed explicitly
+ --implicit select specs that are not installed or were installed implicitly
+ --output OUTPUT where to dump the result
+
+The corresponding unit tests can be run giving the appropriate options to ``spack test``:
+
+.. code-block:: console
+
+ $ spack test --extension=scripting
+
+ ============================================================== test session starts ===============================================================
+ platform linux2 -- Python 2.7.15rc1, pytest-3.2.5, py-1.4.34, pluggy-0.4.0
+ rootdir: /home/mculpo/tmp/spack-scripting, inifile: pytest.ini
+ collected 5 items
+
+ tests/test_filter.py ...XX
+ ============================================================ short test summary info =============================================================
+ XPASS tests/test_filter.py::test_filtering_specs[flags3-specs3-expected3]
+ XPASS tests/test_filter.py::test_filtering_specs[flags4-specs4-expected4]
+
+ =========================================================== slowest 20 test durations ============================================================
+ 3.74s setup tests/test_filter.py::test_filtering_specs[flags0-specs0-expected0]
+ 0.17s call tests/test_filter.py::test_filtering_specs[flags3-specs3-expected3]
+ 0.16s call tests/test_filter.py::test_filtering_specs[flags2-specs2-expected2]
+ 0.15s call tests/test_filter.py::test_filtering_specs[flags1-specs1-expected1]
+ 0.13s call tests/test_filter.py::test_filtering_specs[flags4-specs4-expected4]
+ 0.08s call tests/test_filter.py::test_filtering_specs[flags0-specs0-expected0]
+ 0.04s teardown tests/test_filter.py::test_filtering_specs[flags4-specs4-expected4]
+ 0.00s setup tests/test_filter.py::test_filtering_specs[flags4-specs4-expected4]
+ 0.00s setup tests/test_filter.py::test_filtering_specs[flags3-specs3-expected3]
+ 0.00s setup tests/test_filter.py::test_filtering_specs[flags1-specs1-expected1]
+ 0.00s setup tests/test_filter.py::test_filtering_specs[flags2-specs2-expected2]
+ 0.00s teardown tests/test_filter.py::test_filtering_specs[flags2-specs2-expected2]
+ 0.00s teardown tests/test_filter.py::test_filtering_specs[flags1-specs1-expected1]
+ 0.00s teardown tests/test_filter.py::test_filtering_specs[flags0-specs0-expected0]
+ 0.00s teardown tests/test_filter.py::test_filtering_specs[flags3-specs3-expected3]
+ ====================================================== 3 passed, 2 xpassed in 4.51 seconds =======================================================
diff --git a/lib/spack/docs/index.rst b/lib/spack/docs/index.rst
index b176835979..5ad55b6320 100644
--- a/lib/spack/docs/index.rst
+++ b/lib/spack/docs/index.rst
@@ -72,6 +72,7 @@ or refer to the full manual below.
command_index
package_list
chain
+ extensions
.. toctree::
:maxdepth: 2
diff --git a/lib/spack/spack/extensions.py b/lib/spack/spack/extensions.py
index 449798e94e..995464ea73 100644
--- a/lib/spack/spack/extensions.py
+++ b/lib/spack/spack/extensions.py
@@ -7,13 +7,13 @@ for Spack's command extensions.
"""
import os
import re
+import sys
+import types
import llnl.util.lang
import llnl.util.tty as tty
-
import spack.config
-
extension_regexp = re.compile(r'spack-([\w]*)')
@@ -50,14 +50,47 @@ def load_command_extension(command, path):
if not extension:
return None
+ # Compute the name of the module we search, exit early if already imported
+ cmd_package = '{0}.{1}.cmd'.format(__name__, extension)
+ python_name = command.replace('-', '_')
+ module_name = '{0}.{1}'.format(cmd_package, python_name)
+ if module_name in sys.modules:
+ return sys.modules[module_name]
+
+ def ensure_package_creation(name):
+ package_name = '{0}.{1}'.format(__name__, name)
+ if package_name in sys.modules:
+ return
+
+ parts = [path] + name.split('.') + ['__init__.py']
+ init_file = os.path.join(*parts)
+ if os.path.exists(init_file):
+ m = llnl.util.lang.load_module_from_file(package_name, init_file)
+ else:
+ m = types.ModuleType(package_name)
+
+ # Setting __path__ to give spack extensions the
+ # ability to import from their own tree, see:
+ #
+ # https://docs.python.org/3/reference/import.html#package-path-rules
+ #
+ m.__path__ = [os.path.dirname(init_file)]
+ sys.modules[package_name] = m
+
+ # Create a searchable package for both the root folder of the extension
+ # and the subfolder containing the commands
+ ensure_package_creation(extension)
+ ensure_package_creation(extension + '.cmd')
+
# Compute the absolute path of the file to be loaded, along with the
# name of the python module where it will be stored
cmd_path = os.path.join(path, extension, 'cmd', command + '.py')
- python_name = command.replace('-', '_')
- module_name = '{0}.{1}'.format(__name__, python_name)
try:
+ # TODO: Upon removal of support for Python 2.6 substitute the call
+ # TODO: below with importlib.import_module(module_name)
module = llnl.util.lang.load_module_from_file(module_name, cmd_path)
+ sys.modules[module_name] = module
except (ImportError, IOError):
module = None
diff --git a/lib/spack/spack/test/cmd_extensions.py b/lib/spack/spack/test/cmd_extensions.py
new file mode 100644
index 0000000000..bb018eedb5
--- /dev/null
+++ b/lib/spack/spack/test/cmd_extensions.py
@@ -0,0 +1,111 @@
+# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+import pytest
+
+import sys
+
+import spack.config
+import spack.main
+
+
+@pytest.fixture()
+def extension_root(tmpdir):
+ root = tmpdir.mkdir('spack-testcommand')
+ root.ensure('testcommand', 'cmd', dir=True)
+ return root
+
+
+@pytest.fixture()
+def hello_world_cmd(extension_root):
+ """Simple extension command with code contained in a single file."""
+ hello = extension_root.ensure('testcommand', 'cmd', 'hello.py')
+ hello.write("""
+description = "hello world extension command"
+section = "test command"
+level = "long"
+
+def setup_parser(subparser):
+ pass
+
+
+def hello(parser, args):
+ print('Hello world!')
+""")
+ list_of_modules = list(sys.modules.keys())
+ with spack.config.override('config:extensions', [str(extension_root)]):
+ yield spack.main.SpackCommand('hello')
+
+ to_be_deleted = [x for x in sys.modules if x not in list_of_modules]
+ for module_name in to_be_deleted:
+ del sys.modules[module_name]
+
+
+@pytest.fixture()
+def hello_world_with_module_in_root(extension_root):
+ """Extension command with additional code in the root folder."""
+ extension_root.ensure('testcommand', '__init__.py')
+ command_root = extension_root.join('testcommand', 'cmd')
+ hello = command_root.ensure('hello.py')
+ hello.write("""
+# Test an absolute import
+from spack.extensions.testcommand.implementation import hello_world
+
+# Test a relative import
+from ..implementation import hello_folks
+
+description = "hello world extension command"
+section = "test command"
+level = "long"
+
+# Test setting a global variable in setup_parser and retrieving
+# it in the command
+global_message = 'foo'
+
+def setup_parser(subparser):
+ sp = subparser.add_subparsers(metavar='SUBCOMMAND', dest='subcommand')
+ global global_message
+ sp.add_parser('world', help='Print Hello world!')
+ sp.add_parser('folks', help='Print Hello folks!')
+ sp.add_parser('global', help='Print Hello folks!')
+ global_message = 'bar'
+
+def hello(parser, args):
+ if args.subcommand == 'world':
+ hello_world()
+ elif args.subcommand == 'folks':
+ hello_folks()
+ elif args.subcommand == 'global':
+ print(global_message)
+""")
+ implementation = extension_root.ensure('testcommand', 'implementation.py')
+ implementation.write("""
+def hello_world():
+ print('Hello world!')
+
+def hello_folks():
+ print('Hello folks!')
+""")
+ list_of_modules = list(sys.modules.keys())
+ with spack.config.override('config:extensions', [str(extension_root)]):
+ yield spack.main.SpackCommand('hello')
+
+ to_be_deleted = [x for x in sys.modules if x not in list_of_modules]
+ for module_name in to_be_deleted:
+ del sys.modules[module_name]
+
+
+def test_simple_command_extension(hello_world_cmd):
+ output = hello_world_cmd()
+ assert 'Hello world!' in output
+
+
+def test_command_with_import(hello_world_with_module_in_root):
+ output = hello_world_with_module_in_root('world')
+ assert 'Hello world!' in output
+ output = hello_world_with_module_in_root('folks')
+ assert 'Hello folks!' in output
+ output = hello_world_with_module_in_root('global')
+ assert 'bar' in output