extensions: improve docs, fix unit-tests (#41425)

3 files changed, 68 insertions, 56 deletions

diff --git a/lib/spack/docs/extensions.rst b/lib/spack/docs/extensions.rst
index bc5b3a762f..93280e9e8f 100644
--- a/lib/spack/docs/extensions.rst
+++ b/lib/spack/docs/extensions.rst
@@ -9,46 +9,42 @@
 Custom Extensions
 =================
 
-*Spack extensions* permit you to extend Spack capabilities by deploying your
+*Spack extensions* allow you to extend Spack capabilities by deploying your
 own custom commands or logic in an arbitrary location on your filesystem.
 This might be extremely useful e.g. to develop and maintain a command whose purpose is
 too specific to be considered for reintegration into the mainline or to
 evolve a command through its early stages before starting a discussion to merge
 it upstream.
+
 From Spack's point of view an extension is any path in your filesystem which
-respects a prescribed naming and layout for files:
+respects the following naming and layout for files:
 
 .. code-block:: console
 
   spack-scripting/ # The top level directory must match the format 'spack-{extension_name}'
   ├── pytest.ini # Optional file if the extension ships its own tests
   ├── scripting # Folder that may contain modules that are needed for the extension commands
-  │   └── cmd # Folder containing extension commands
-  │       └── filter.py # A new command that will be available
-  ├── tests # Tests for this extension
+  │   ├── cmd # Folder containing extension commands
+  │   │   └── filter.py # A new command that will be available
+  │   └── functions.py # Module with internal details
+  └── tests # Tests for this extension
   │   ├── conftest.py
   │   └── test_filter.py
   └── templates # Templates that may be needed by the extension
 
-In the example above the extension named *scripting* adds an additional command (``filter``)
-and unit tests to verify its behavior. The code for this example can be
-obtained by cloning the corresponding git repository:
+In the example above, the extension is named *scripting*. It adds an additional command
+(``spack filter``) and unit tests to verify its behavior.
 
-.. TODO: write an ad-hoc "hello world" extension and make it part of the spack organization
+The extension can import any core Spack module in its implementation. When loaded by
+the ``spack`` command, the extension itself is imported as a Python package in the
+``spack.extensions`` namespace. In the example above, since the extension is named
+"scripting", the corresponding Python module is ``spack.extensions.scripting``.
 
-.. code-block:: console
+The code for this example extension can be obtained by cloning the corresponding git repository:
 
-   $ cd ~/
-   $ mkdir tmp && cd tmp
-   $ git clone https://github.com/alalazo/spack-scripting.git
-   Cloning into 'spack-scripting'...
-   remote: Counting objects: 11, done.
-   remote: Compressing objects: 100% (7/7), done.
-   remote: Total 11 (delta 0), reused 11 (delta 0), pack-reused 0
-   Receiving objects: 100% (11/11), done.
+.. code-block:: console
 
-As you can see by inspecting the sources, Python modules that are part of the extension
-can import any core Spack module.
+   $ git -C /tmp clone https://github.com/spack/spack-scripting.git
 
 ---------------------------------
 Configure Spack to Use Extensions
@@ -61,7 +57,7 @@ paths to ``config.yaml``. In the case of our example this means ensuring that:
 
    config:
      extensions:
-     - ~/tmp/spack-scripting
+     - /tmp/spack-scripting
 
 is part of your configuration file. Once this is setup any command that the extension provides
 will be available from the command line:
@@ -86,37 +82,32 @@ will be available from the command line:
      --implicit       select specs that are not installed or were installed implicitly
      --output OUTPUT  where to dump the result
 
-The corresponding unit tests can be run giving the appropriate options
-to ``spack unit-test``:
+The corresponding unit tests can be run giving the appropriate options to ``spack unit-test``:
 
 .. code-block:: console
 
    $ spack unit-test --extension=scripting
-
-   ============================================================== test session starts ===============================================================
-   platform linux2 -- Python 2.7.15rc1, pytest-3.2.5, py-1.4.34, pluggy-0.4.0
-   rootdir: /home/mculpo/tmp/spack-scripting, inifile: pytest.ini
+   ========================================== test session starts ===========================================
+   platform linux -- Python 3.11.5, pytest-7.4.3, pluggy-1.3.0
+   rootdir: /home/culpo/github/spack-scripting
+   configfile: pytest.ini
+   testpaths: tests
+   plugins: xdist-3.5.0
    collected 5 items
 
-   tests/test_filter.py ...XX
-   ============================================================ short test summary info =============================================================
-   XPASS tests/test_filter.py::test_filtering_specs[flags3-specs3-expected3]
-   XPASS tests/test_filter.py::test_filtering_specs[flags4-specs4-expected4]
-
-   =========================================================== slowest 20 test durations ============================================================
-   3.74s setup    tests/test_filter.py::test_filtering_specs[flags0-specs0-expected0]
-   0.17s call     tests/test_filter.py::test_filtering_specs[flags3-specs3-expected3]
-   0.16s call     tests/test_filter.py::test_filtering_specs[flags2-specs2-expected2]
-   0.15s call     tests/test_filter.py::test_filtering_specs[flags1-specs1-expected1]
-   0.13s call     tests/test_filter.py::test_filtering_specs[flags4-specs4-expected4]
-   0.08s call     tests/test_filter.py::test_filtering_specs[flags0-specs0-expected0]
-   0.04s teardown tests/test_filter.py::test_filtering_specs[flags4-specs4-expected4]
-   0.00s setup    tests/test_filter.py::test_filtering_specs[flags4-specs4-expected4]
-   0.00s setup    tests/test_filter.py::test_filtering_specs[flags3-specs3-expected3]
-   0.00s setup    tests/test_filter.py::test_filtering_specs[flags1-specs1-expected1]
-   0.00s setup    tests/test_filter.py::test_filtering_specs[flags2-specs2-expected2]
-   0.00s teardown tests/test_filter.py::test_filtering_specs[flags2-specs2-expected2]
-   0.00s teardown tests/test_filter.py::test_filtering_specs[flags1-specs1-expected1]
-   0.00s teardown tests/test_filter.py::test_filtering_specs[flags0-specs0-expected0]
-   0.00s teardown tests/test_filter.py::test_filtering_specs[flags3-specs3-expected3]
-   ====================================================== 3 passed, 2 xpassed in 4.51 seconds =======================================================
+   tests/test_filter.py .....                                                                         [100%]
+
+   ========================================== slowest 30 durations ==========================================
+   2.31s setup    tests/test_filter.py::test_filtering_specs[kwargs0-specs0-expected0]
+   0.57s call     tests/test_filter.py::test_filtering_specs[kwargs2-specs2-expected2]
+   0.56s call     tests/test_filter.py::test_filtering_specs[kwargs4-specs4-expected4]
+   0.54s call     tests/test_filter.py::test_filtering_specs[kwargs3-specs3-expected3]
+   0.54s call     tests/test_filter.py::test_filtering_specs[kwargs1-specs1-expected1]
+   0.48s call     tests/test_filter.py::test_filtering_specs[kwargs0-specs0-expected0]
+   0.01s setup    tests/test_filter.py::test_filtering_specs[kwargs4-specs4-expected4]
+   0.01s setup    tests/test_filter.py::test_filtering_specs[kwargs2-specs2-expected2]
+   0.01s setup    tests/test_filter.py::test_filtering_specs[kwargs1-specs1-expected1]
+   0.01s setup    tests/test_filter.py::test_filtering_specs[kwargs3-specs3-expected3]
+
+   (5 durations < 0.005s hidden.  Use -vv to show these durations.)
+   =========================================== 5 passed in 5.06s ============================================
diff --git a/lib/spack/spack/cmd/unit_test.py b/lib/spack/spack/cmd/unit_test.py
index 4963ad14c6..63ae55a9d8 100644
--- a/lib/spack/spack/cmd/unit_test.py
+++ b/lib/spack/spack/cmd/unit_test.py
@@ -227,9 +227,7 @@ def unit_test(parser, args, unknown_args):
     # has been used, then test that extension.
     pytest_root = spack.paths.spack_root
     if args.extension:
-        target = args.extension
-        extensions = spack.extensions.get_extension_paths()
-        pytest_root = spack.extensions.path_for_extension(target, *extensions)
+        pytest_root = spack.extensions.load_extension(args.extension)
 
     # pytest.ini lives in the root of the spack repository.
     with llnl.util.filesystem.working_dir(pytest_root):
diff --git a/lib/spack/spack/extensions.py b/lib/spack/spack/extensions.py
index 0ee01a22a1..c6b6e0c679 100644
--- a/lib/spack/spack/extensions.py
+++ b/lib/spack/spack/extensions.py
@@ -6,11 +6,13 @@
 for Spack's command extensions.
 """
 import difflib
+import glob
 import importlib
 import os
 import re
 import sys
 import types
+from typing import List
 
 import llnl.util.lang
 
@@ -75,6 +77,15 @@ def load_command_extension(command, path):
     if not os.path.exists(cmd_path):
         return None
 
+    ensure_extension_loaded(extension, path=path)
+
+    module = importlib.import_module(module_name)
+    sys.modules[module_name] = module
+
+    return module
+
+
+def ensure_extension_loaded(extension, *, path):
     def ensure_package_creation(name):
         package_name = "{0}.{1}".format(__name__, name)
         if package_name in sys.modules:
@@ -100,10 +111,22 @@ def load_command_extension(command, path):
     ensure_package_creation(extension)
     ensure_package_creation(extension + ".cmd")
 
-    module = importlib.import_module(module_name)
-    sys.modules[module_name] = module
 
-    return module
+def load_extension(name: str) -> str:
+    """Loads a single extension into the 'spack.extensions' package.
+
+    Args:
+        name: name of the extension
+    """
+    extension_root = path_for_extension(name, paths=get_extension_paths())
+    ensure_extension_loaded(name, path=extension_root)
+    commands = glob.glob(
+        os.path.join(extension_root, extension_name(extension_root), "cmd", "*.py")
+    )
+    commands = [os.path.basename(x).rstrip(".py") for x in commands]
+    for command in commands:
+        load_command_extension(command, extension_root)
+    return extension_root
 
 
 def get_extension_paths():
@@ -125,7 +148,7 @@ def get_command_paths():
     return command_paths
 
 
-def path_for_extension(target_name, *paths):
+def path_for_extension(target_name: str, *, paths: List[str]) -> str:
     """Return the test root dir for a given extension.
 
     Args: