diff options
Diffstat (limited to 'lib')
| -rw-r--r-- | lib/spack/llnl/util/lang.py | 32 | ||||
| -rw-r--r-- | lib/spack/spack/cmd/__init__.py | 33 | ||||
| -rw-r--r-- | lib/spack/spack/cmd/test.py | 16 | ||||
| -rw-r--r-- | lib/spack/spack/extensions.py | 123 | ||||
| -rw-r--r-- | lib/spack/spack/schema/config.py | 4 | ||||
| -rw-r--r-- | lib/spack/spack/tengine.py | 7 | ||||
| -rw-r--r-- | lib/spack/spack/test/conftest.py | 9 | ||||
| -rw-r--r-- | lib/spack/spack/test/llnl/util/lang.py | 20 |
8 files changed, 225 insertions, 19 deletions
diff --git a/lib/spack/llnl/util/lang.py b/lib/spack/llnl/util/lang.py index 21eb9ee3ba..70c646bed0 100644 --- a/lib/spack/llnl/util/lang.py +++ b/lib/spack/llnl/util/lang.py @@ -12,6 +12,8 @@ import collections import inspect from datetime import datetime, timedelta from six import string_types +import sys + # Ignore emacs backups when listing modules ignore_modules = [r'^\.#', '~$'] @@ -597,3 +599,33 @@ class LazyReference(object): def __repr__(self): return repr(self.ref_function()) + + +def load_module_from_file(module_name, module_path): + """Loads a python module from the path of the corresponding file. + + Args: + module_name (str): namespace where the python module will be loaded, + e.g. ``foo.bar`` + module_path (str): path of the python file containing the module + + Returns: + A valid module object + + Raises: + ImportError: when the module can't be loaded + FileNotFoundError: when module_path doesn't exist + """ + if sys.version_info[0] == 3 and sys.version_info[1] >= 5: + import importlib.util + spec = importlib.util.spec_from_file_location(module_name, module_path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + elif sys.version_info[0] == 3 and sys.version_info[1] < 5: + import importlib.machinery + loader = importlib.machinery.SourceFileLoader(module_name, module_path) + module = loader.load_module() + elif sys.version_info[0] == 2: + import imp + module = imp.load_source(module_name, module_path) + return module diff --git a/lib/spack/spack/cmd/__init__.py b/lib/spack/spack/cmd/__init__.py index 630a6b6e63..9d003eb656 100644 --- a/lib/spack/spack/cmd/__init__.py +++ b/lib/spack/spack/cmd/__init__.py @@ -17,6 +17,7 @@ from llnl.util.tty.color import colorize from llnl.util.filesystem import working_dir import spack.config +import spack.extensions import spack.paths import spack.spec import spack.store @@ -32,9 +33,6 @@ ignore_files = r'^\.|^__init__.py$|^#' SETUP_PARSER = "setup_parser" DESCRIPTION = "description" -#: Names of all commands -all_commands = [] - def python_name(cmd_name): """Convert ``-`` to ``_`` in command name, to make a valid identifier.""" @@ -60,11 +58,16 @@ def all_commands(): global _all_commands if _all_commands is None: _all_commands = [] - for file in os.listdir(spack.paths.command_path): - if file.endswith(".py") and not re.search(ignore_files, file): - cmd = re.sub(r'.py$', '', file) - _all_commands.append(cmd_name(cmd)) + command_paths = [spack.paths.command_path] # Built-in commands + command_paths += spack.extensions.get_command_paths() # Extensions + for path in command_paths: + for file in os.listdir(path): + if file.endswith(".py") and not re.search(ignore_files, file): + cmd = re.sub(r'.py$', '', file) + _all_commands.append(cmd_name(cmd)) + _all_commands.sort() + return _all_commands @@ -85,10 +88,18 @@ def get_module(cmd_name): (contains ``-``, not ``_``). """ pname = python_name(cmd_name) - module_name = "%s.%s" % (__name__, pname) - module = __import__(module_name, - fromlist=[pname, SETUP_PARSER, DESCRIPTION], - level=0) + + try: + # Try to import the command from the built-in directory + module_name = "%s.%s" % (__name__, pname) + module = __import__(module_name, + fromlist=[pname, SETUP_PARSER, DESCRIPTION], + level=0) + tty.debug('Imported {0} from built-in commands'.format(pname)) + except ImportError: + module = spack.extensions.get_module(cmd_name) + if not module: + raise attr_setdefault(module, SETUP_PARSER, lambda *args: None) # null-op attr_setdefault(module, DESCRIPTION, "") diff --git a/lib/spack/spack/cmd/test.py b/lib/spack/spack/cmd/test.py index 1c1b739a62..63e3778a6b 100644 --- a/lib/spack/spack/cmd/test.py +++ b/lib/spack/spack/cmd/test.py @@ -35,6 +35,10 @@ def setup_parser(subparser): '-L', '--long-list', action='store_true', default=False, help="list the entire hierarchy of tests") subparser.add_argument( + '--extension', default=None, + help="run test for a given Spack extension" + ) + subparser.add_argument( 'tests', nargs=argparse.REMAINDER, help="list of tests to run (will be passed to pytest -k)") @@ -77,8 +81,16 @@ def test(parser, args, unknown_args): pytest.main(['-h']) return - # pytest.ini lives in lib/spack/spack/test - with working_dir(spack.paths.test_path): + # The default is to test the core of Spack. If the option `--extension` + # has been used, then test that extension. + pytest_root = spack.paths.test_path + if args.extension: + target = args.extension + extensions = spack.config.get('config:extensions') + pytest_root = spack.extensions.path_for_extension(target, *extensions) + + # pytest.ini lives in the root of the spack repository. + with working_dir(pytest_root): # --list and --long-list print the test output better. if args.list or args.long_list: do_list(args, unknown_args) diff --git a/lib/spack/spack/extensions.py b/lib/spack/spack/extensions.py new file mode 100644 index 0000000000..449798e94e --- /dev/null +++ b/lib/spack/spack/extensions.py @@ -0,0 +1,123 @@ +# Copyright 2013-2018 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +"""Service functions and classes to implement the hooks +for Spack's command extensions. +""" +import os +import re + +import llnl.util.lang +import llnl.util.tty as tty + +import spack.config + + +extension_regexp = re.compile(r'spack-([\w]*)') + + +def extension_name(path): + """Returns the name of the extension in the path passed as argument. + + Args: + path (str): path where the extension resides + + Returns: + The extension name or None if path doesn't match the format + for Spack's extension. + """ + regexp_match = re.search(extension_regexp, os.path.basename(path)) + if not regexp_match: + msg = "[FOLDER NAMING]" + msg += " {0} doesn't match the format for Spack's extensions" + tty.warn(msg.format(path)) + return None + return regexp_match.group(1) + + +def load_command_extension(command, path): + """Loads a command extension from the path passed as argument. + + Args: + command (str): name of the command + path (str): base path of the command extension + + Returns: + A valid module object if the command is found or None + """ + extension = extension_name(path) + if not extension: + return None + + # Compute the absolute path of the file to be loaded, along with the + # name of the python module where it will be stored + cmd_path = os.path.join(path, extension, 'cmd', command + '.py') + python_name = command.replace('-', '_') + module_name = '{0}.{1}'.format(__name__, python_name) + + try: + module = llnl.util.lang.load_module_from_file(module_name, cmd_path) + except (ImportError, IOError): + module = None + + return module + + +def get_command_paths(): + """Return the list of paths where to search for command files.""" + command_paths = [] + extension_paths = spack.config.get('config:extensions') or [] + + for path in extension_paths: + extension = extension_name(path) + if extension: + command_paths.append(os.path.join(path, extension, 'cmd')) + + return command_paths + + +def path_for_extension(target_name, *paths): + """Return the test root dir for a given extension. + + Args: + target_name (str): name of the extension to test + *paths: paths where the extensions reside + + Returns: + Root directory where tests should reside or None + """ + for path in paths: + name = extension_name(path) + if name == target_name: + return path + else: + raise IOError('extension "{0}" not found'.format(target_name)) + + +def get_module(cmd_name): + """Imports the extension module for a particular command name + and returns it. + + Args: + cmd_name (str): name of the command for which to get a module + (contains ``-``, not ``_``). + """ + # If built-in failed the import search the extension + # directories in order + extensions = spack.config.get('config:extensions') or [] + for folder in extensions: + module = load_command_extension(cmd_name, folder) + if module: + return module + else: + return None + + +def get_template_dirs(): + """Returns the list of directories where to search for templates + in extensions. + """ + extension_dirs = spack.config.get('config:extensions') or [] + extensions = [os.path.join(x, 'templates') for x in extension_dirs] + return extensions diff --git a/lib/spack/spack/schema/config.py b/lib/spack/spack/schema/config.py index 1aa54bf48e..30c0bf0591 100644 --- a/lib/spack/spack/schema/config.py +++ b/lib/spack/spack/schema/config.py @@ -25,6 +25,10 @@ properties = { {'type': 'array', 'items': {'type': 'string'}}], }, + 'extensions': { + 'type': 'array', + 'items': {'type': 'string'} + }, 'template_dirs': { 'type': 'array', 'items': {'type': 'string'} diff --git a/lib/spack/spack/tengine.py b/lib/spack/spack/tengine.py index a48f9a27b7..6aceb391cd 100644 --- a/lib/spack/spack/tengine.py +++ b/lib/spack/spack/tengine.py @@ -2,7 +2,7 @@ # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) - +import itertools import textwrap import jinja2 @@ -72,8 +72,11 @@ def make_environment(dirs=None): """Returns an configured environment for template rendering.""" if dirs is None: # Default directories where to search for templates + builtins = spack.config.get('config:template_dirs') + extensions = spack.extensions.get_template_dirs() dirs = [canonicalize_path(d) - for d in spack.config.get('config:template_dirs')] + for d in itertools.chain(builtins, extensions)] + # Loader for the templates loader = jinja2.FileSystemLoader(dirs) # Environment of the template engine diff --git a/lib/spack/spack/test/conftest.py b/lib/spack/spack/test/conftest.py index ff95f463bd..b2d255c340 100644 --- a/lib/spack/spack/test/conftest.py +++ b/lib/spack/spack/test/conftest.py @@ -24,6 +24,7 @@ import spack.caches import spack.database import spack.directory_layout import spack.environment as ev +import spack.package_prefs import spack.paths import spack.platforms.test import spack.repo @@ -118,7 +119,7 @@ def mock_stage(tmpdir_factory): @pytest.fixture(scope='session') -def _ignore_stage_files(): +def ignore_stage_files(): """Session-scoped helper for check_for_leftover_stage_files. Used to track which leftover files in the stage have been seen. @@ -145,7 +146,7 @@ def working_env(): @pytest.fixture(scope='function', autouse=True) -def check_for_leftover_stage_files(request, mock_stage, _ignore_stage_files): +def check_for_leftover_stage_files(request, mock_stage, ignore_stage_files): """Ensure that each test leaves a clean stage when done. This can be disabled for tests that are expected to dirty the stage @@ -160,7 +161,7 @@ def check_for_leftover_stage_files(request, mock_stage, _ignore_stage_files): files_in_stage = set() if os.path.exists(spack.paths.stage_path): files_in_stage = set( - os.listdir(spack.paths.stage_path)) - _ignore_stage_files + os.listdir(spack.paths.stage_path)) - ignore_stage_files if 'disable_clean_stage_check' in request.keywords: # clean up after tests that are expected to be dirty @@ -168,7 +169,7 @@ def check_for_leftover_stage_files(request, mock_stage, _ignore_stage_files): path = os.path.join(spack.paths.stage_path, f) remove_whatever_it_is(path) else: - _ignore_stage_files |= files_in_stage + ignore_stage_files |= files_in_stage assert not files_in_stage diff --git a/lib/spack/spack/test/llnl/util/lang.py b/lib/spack/spack/test/llnl/util/lang.py index edeab72c28..cbc24b51db 100644 --- a/lib/spack/spack/test/llnl/util/lang.py +++ b/lib/spack/spack/test/llnl/util/lang.py @@ -5,6 +5,7 @@ import pytest +import os.path from datetime import datetime, timedelta import llnl.util.lang @@ -16,6 +17,19 @@ def now(): return datetime.now() +@pytest.fixture() +def module_path(tmpdir): + m = tmpdir.join('foo.py') + content = """ +import os.path + +value = 1 +path = os.path.join('/usr', 'bin') +""" + m.write(content) + return str(m) + + def test_pretty_date(): """Make sure pretty_date prints the right dates.""" now = datetime.now() @@ -110,3 +124,9 @@ def test_match_predicate(): with pytest.raises(ValueError): matcher = match_predicate(object()) matcher('foo') + + +def test_load_modules_from_file(module_path): + foo = llnl.util.lang.load_module_from_file('foo', module_path) + assert foo.value == 1 + assert foo.path == os.path.join('/usr', 'bin') |