diff options
| -rw-r--r-- | lib/spack/docs/extensions.rst | 128 | ||||
| -rw-r--r-- | lib/spack/docs/index.rst | 1 | ||||
| -rw-r--r-- | lib/spack/spack/extensions.py | 41 | ||||
| -rw-r--r-- | lib/spack/spack/test/cmd_extensions.py | 111 |
4 files changed, 277 insertions, 4 deletions
diff --git a/lib/spack/docs/extensions.rst b/lib/spack/docs/extensions.rst new file mode 100644 index 0000000000..8f4c54b435 --- /dev/null +++ b/lib/spack/docs/extensions.rst @@ -0,0 +1,128 @@ +.. Copyright 2013-2019 Lawrence Livermore National Security, LLC and other + Spack Project Developers. See the top-level COPYRIGHT file for details. + + SPDX-License-Identifier: (Apache-2.0 OR MIT) + +.. extensions: + +================= +Custom Extensions +================= + +.. warning:: + + The support for extending Spack with custom commands is still experimental. + Users should expect APIs or prescribed directory structures to + change at any time. + +*Spack extensions* permit you to extend Spack capabilities by deploying your +own custom commands or logic in an arbitrary location on your filesystem. +This might be extremely useful e.g. to develop and maintain a command whose purpose is +too specific to be considered for reintegration into the mainline or to +evolve a command through its early stages before starting a discussion to merge +it upstream. +From Spack's point of view an extension is any path in your filesystem which +respects a prescribed naming and layout for files: + +.. code-block:: console + + spack-scripting/ # The top level directory must match the format 'spack-{extension_name}' + ├── pytest.ini # Optional file if the extension ships its own tests + ├── scripting # Folder that may contain modules that are needed for the extension commands + │ └── cmd # Folder containing extension commands + │ └── filter.py # A new command that will be available + ├── tests # Tests for this extension + │ ├── conftest.py + │ └── test_filter.py + └── templates # Templates that may be needed by the extension + +In the example above the extension named *scripting* adds an additional command (``filter``) +and unit tests to verify its behavior. The code for this example can be +obtained by cloning the corresponding git repository: + +.. TODO: write an ad-hoc "hello world" extension and make it part of the spack organization + +.. code-block:: console + + $ pwd + /home/user + $ mkdir tmp && cd tmp + $ git clone https://github.com/alalazo/spack-scripting.git + Cloning into 'spack-scripting'... + remote: Counting objects: 11, done. + remote: Compressing objects: 100% (7/7), done. + remote: Total 11 (delta 0), reused 11 (delta 0), pack-reused 0 + Receiving objects: 100% (11/11), done. + +As you can see by inspecting the sources, Python modules that are part of the extension +can import any core Spack module. + +--------------------------------- +Configure Spack to Use Extensions +--------------------------------- + +To make your current Spack instance aware of extensions you should add their root +paths to ``config.yaml``. In the case of our example this means ensuring that: + +.. code-block:: yaml + + config: + extensions: + - /home/user/tmp/spack-scripting + +is part of your configuration file. Once this is setup any command that the extension provides +will be available from the command line: + +.. code-block:: console + + $ spack filter --help + usage: spack filter [-h] [--installed | --not-installed] + [--explicit | --implicit] [--output OUTPUT] + ... + + filter specs based on their properties + + positional arguments: + specs specs to be filtered + + optional arguments: + -h, --help show this help message and exit + --installed select installed specs + --not-installed select specs that are not yet installed + --explicit select specs that were installed explicitly + --implicit select specs that are not installed or were installed implicitly + --output OUTPUT where to dump the result + +The corresponding unit tests can be run giving the appropriate options to ``spack test``: + +.. code-block:: console + + $ spack test --extension=scripting + + ============================================================== test session starts =============================================================== + platform linux2 -- Python 2.7.15rc1, pytest-3.2.5, py-1.4.34, pluggy-0.4.0 + rootdir: /home/mculpo/tmp/spack-scripting, inifile: pytest.ini + collected 5 items + + tests/test_filter.py ...XX + ============================================================ short test summary info ============================================================= + XPASS tests/test_filter.py::test_filtering_specs[flags3-specs3-expected3] + XPASS tests/test_filter.py::test_filtering_specs[flags4-specs4-expected4] + + =========================================================== slowest 20 test durations ============================================================ + 3.74s setup tests/test_filter.py::test_filtering_specs[flags0-specs0-expected0] + 0.17s call tests/test_filter.py::test_filtering_specs[flags3-specs3-expected3] + 0.16s call tests/test_filter.py::test_filtering_specs[flags2-specs2-expected2] + 0.15s call tests/test_filter.py::test_filtering_specs[flags1-specs1-expected1] + 0.13s call tests/test_filter.py::test_filtering_specs[flags4-specs4-expected4] + 0.08s call tests/test_filter.py::test_filtering_specs[flags0-specs0-expected0] + 0.04s teardown tests/test_filter.py::test_filtering_specs[flags4-specs4-expected4] + 0.00s setup tests/test_filter.py::test_filtering_specs[flags4-specs4-expected4] + 0.00s setup tests/test_filter.py::test_filtering_specs[flags3-specs3-expected3] + 0.00s setup tests/test_filter.py::test_filtering_specs[flags1-specs1-expected1] + 0.00s setup tests/test_filter.py::test_filtering_specs[flags2-specs2-expected2] + 0.00s teardown tests/test_filter.py::test_filtering_specs[flags2-specs2-expected2] + 0.00s teardown tests/test_filter.py::test_filtering_specs[flags1-specs1-expected1] + 0.00s teardown tests/test_filter.py::test_filtering_specs[flags0-specs0-expected0] + 0.00s teardown tests/test_filter.py::test_filtering_specs[flags3-specs3-expected3] + ====================================================== 3 passed, 2 xpassed in 4.51 seconds ======================================================= diff --git a/lib/spack/docs/index.rst b/lib/spack/docs/index.rst index b176835979..5ad55b6320 100644 --- a/lib/spack/docs/index.rst +++ b/lib/spack/docs/index.rst @@ -72,6 +72,7 @@ or refer to the full manual below. command_index package_list chain + extensions .. toctree:: :maxdepth: 2 diff --git a/lib/spack/spack/extensions.py b/lib/spack/spack/extensions.py index 449798e94e..995464ea73 100644 --- a/lib/spack/spack/extensions.py +++ b/lib/spack/spack/extensions.py @@ -7,13 +7,13 @@ for Spack's command extensions. """ import os import re +import sys +import types import llnl.util.lang import llnl.util.tty as tty - import spack.config - extension_regexp = re.compile(r'spack-([\w]*)') @@ -50,14 +50,47 @@ def load_command_extension(command, path): if not extension: return None + # Compute the name of the module we search, exit early if already imported + cmd_package = '{0}.{1}.cmd'.format(__name__, extension) + python_name = command.replace('-', '_') + module_name = '{0}.{1}'.format(cmd_package, python_name) + if module_name in sys.modules: + return sys.modules[module_name] + + def ensure_package_creation(name): + package_name = '{0}.{1}'.format(__name__, name) + if package_name in sys.modules: + return + + parts = [path] + name.split('.') + ['__init__.py'] + init_file = os.path.join(*parts) + if os.path.exists(init_file): + m = llnl.util.lang.load_module_from_file(package_name, init_file) + else: + m = types.ModuleType(package_name) + + # Setting __path__ to give spack extensions the + # ability to import from their own tree, see: + # + # https://docs.python.org/3/reference/import.html#package-path-rules + # + m.__path__ = [os.path.dirname(init_file)] + sys.modules[package_name] = m + + # Create a searchable package for both the root folder of the extension + # and the subfolder containing the commands + ensure_package_creation(extension) + ensure_package_creation(extension + '.cmd') + # Compute the absolute path of the file to be loaded, along with the # name of the python module where it will be stored cmd_path = os.path.join(path, extension, 'cmd', command + '.py') - python_name = command.replace('-', '_') - module_name = '{0}.{1}'.format(__name__, python_name) try: + # TODO: Upon removal of support for Python 2.6 substitute the call + # TODO: below with importlib.import_module(module_name) module = llnl.util.lang.load_module_from_file(module_name, cmd_path) + sys.modules[module_name] = module except (ImportError, IOError): module = None diff --git a/lib/spack/spack/test/cmd_extensions.py b/lib/spack/spack/test/cmd_extensions.py new file mode 100644 index 0000000000..bb018eedb5 --- /dev/null +++ b/lib/spack/spack/test/cmd_extensions.py @@ -0,0 +1,111 @@ +# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import pytest + +import sys + +import spack.config +import spack.main + + +@pytest.fixture() +def extension_root(tmpdir): + root = tmpdir.mkdir('spack-testcommand') + root.ensure('testcommand', 'cmd', dir=True) + return root + + +@pytest.fixture() +def hello_world_cmd(extension_root): + """Simple extension command with code contained in a single file.""" + hello = extension_root.ensure('testcommand', 'cmd', 'hello.py') + hello.write(""" +description = "hello world extension command" +section = "test command" +level = "long" + +def setup_parser(subparser): + pass + + +def hello(parser, args): + print('Hello world!') +""") + list_of_modules = list(sys.modules.keys()) + with spack.config.override('config:extensions', [str(extension_root)]): + yield spack.main.SpackCommand('hello') + + to_be_deleted = [x for x in sys.modules if x not in list_of_modules] + for module_name in to_be_deleted: + del sys.modules[module_name] + + +@pytest.fixture() +def hello_world_with_module_in_root(extension_root): + """Extension command with additional code in the root folder.""" + extension_root.ensure('testcommand', '__init__.py') + command_root = extension_root.join('testcommand', 'cmd') + hello = command_root.ensure('hello.py') + hello.write(""" +# Test an absolute import +from spack.extensions.testcommand.implementation import hello_world + +# Test a relative import +from ..implementation import hello_folks + +description = "hello world extension command" +section = "test command" +level = "long" + +# Test setting a global variable in setup_parser and retrieving +# it in the command +global_message = 'foo' + +def setup_parser(subparser): + sp = subparser.add_subparsers(metavar='SUBCOMMAND', dest='subcommand') + global global_message + sp.add_parser('world', help='Print Hello world!') + sp.add_parser('folks', help='Print Hello folks!') + sp.add_parser('global', help='Print Hello folks!') + global_message = 'bar' + +def hello(parser, args): + if args.subcommand == 'world': + hello_world() + elif args.subcommand == 'folks': + hello_folks() + elif args.subcommand == 'global': + print(global_message) +""") + implementation = extension_root.ensure('testcommand', 'implementation.py') + implementation.write(""" +def hello_world(): + print('Hello world!') + +def hello_folks(): + print('Hello folks!') +""") + list_of_modules = list(sys.modules.keys()) + with spack.config.override('config:extensions', [str(extension_root)]): + yield spack.main.SpackCommand('hello') + + to_be_deleted = [x for x in sys.modules if x not in list_of_modules] + for module_name in to_be_deleted: + del sys.modules[module_name] + + +def test_simple_command_extension(hello_world_cmd): + output = hello_world_cmd() + assert 'Hello world!' in output + + +def test_command_with_import(hello_world_with_module_in_root): + output = hello_world_with_module_in_root('world') + assert 'Hello world!' in output + output = hello_world_with_module_in_root('folks') + assert 'Hello folks!' in output + output = hello_world_with_module_in_root('global') + assert 'bar' in output |