Fix compiler removal from command line (#38057)

* Improve lib/spack/spack/test/cmd/compiler.py

* Use "tmp_path" in the "mock_executable" fixture

* Return a pathlib.Path from mock_executable

* Fix mock_executable fixture on Windows

"mock_gcc" was very similar to mock_executable, so use the latter to reduce code duplication

* Remove wrong compiler cache, fix compiler removal

fixes #37996

_CACHE_CONFIG_FILES was both unneeded and wrong, if called
subsequently with different scopes.

Here we remove that cache, and we fix an issue with compiler
removal triggered by having the same compiler spec in multiple
scopes.

9 files changed, 200 insertions, 251 deletions

diff --git a/lib/spack/spack/cmd/compiler.py b/lib/spack/spack/cmd/compiler.py
index 563142bc8f..4bb73e628c 100644
--- a/lib/spack/spack/cmd/compiler.py
+++ b/lib/spack/spack/cmd/compiler.py
@@ -53,7 +53,7 @@ def setup_parser(subparser):
         "--scope",
         choices=scopes,
         metavar=scopes_metavar,
-        default=spack.config.default_modify_scope("compilers"),
+        default=None,
         help="configuration scope to modify",
     )
 
@@ -106,19 +106,21 @@ def compiler_find(args):
 
 
 def compiler_remove(args):
-    cspec = spack.spec.CompilerSpec(args.compiler_spec)
-    compilers = spack.compilers.compilers_for_spec(cspec, scope=args.scope)
-    if not compilers:
-        tty.die("No compilers match spec %s" % cspec)
-    elif not args.all and len(compilers) > 1:
-        tty.error("Multiple compilers match spec %s. Choose one:" % cspec)
-        colify(reversed(sorted([c.spec.display_str for c in compilers])), indent=4)
+    compiler_spec = spack.spec.CompilerSpec(args.compiler_spec)
+    candidate_compilers = spack.compilers.compilers_for_spec(compiler_spec, scope=args.scope)
+
+    if not candidate_compilers:
+        tty.die("No compilers match spec %s" % compiler_spec)
+
+    if not args.all and len(candidate_compilers) > 1:
+        tty.error(f"Multiple compilers match spec {compiler_spec}. Choose one:")
+        colify(reversed(sorted([c.spec.display_str for c in candidate_compilers])), indent=4)
         tty.msg("Or, use `spack compiler remove -a` to remove all of them.")
         sys.exit(1)
 
-    for compiler in compilers:
-        spack.compilers.remove_compiler_from_config(compiler.spec, scope=args.scope)
-        tty.msg("Removed compiler %s" % compiler.spec.display_str)
+    for current_compiler in candidate_compilers:
+        spack.compilers.remove_compiler_from_config(current_compiler.spec, scope=args.scope)
+        tty.msg(f"{current_compiler.spec.display_str} has been removed")
 
 
 def compiler_info(args):
diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py
index c867f927c0..f6064a9d3f 100644
--- a/lib/spack/spack/compilers/__init__.py
+++ b/lib/spack/spack/compilers/__init__.py
@@ -37,7 +37,6 @@ _other_instance_vars = [
     "implicit_rpaths",
     "extra_rpaths",
 ]
-_cache_config_file = []
 
 # TODO: Caches at module level make it difficult to mock configurations in
 # TODO: unit tests. It might be worth reworking their implementation.
@@ -155,52 +154,65 @@ def add_compilers_to_config(compilers, scope=None, init_config=True):
     compiler_config = get_compiler_config(scope, init_config)
     for compiler in compilers:
         compiler_config.append(_to_dict(compiler))
-    global _cache_config_file
-    _cache_config_file = compiler_config
     spack.config.set("compilers", compiler_config, scope=scope)
 
 
 @_auto_compiler_spec
 def remove_compiler_from_config(compiler_spec, scope=None):
-    """Remove compilers from the config, by spec.
+    """Remove compilers from configuration by spec.
+
+    If scope is None, all the scopes are searched for removal.
 
     Arguments:
-        compiler_specs: a list of CompilerSpec objects.
-        scope: configuration scope to modify.
+        compiler_spec: compiler to be removed
+        scope: configuration scope to modify
     """
-    # Need a better way for this
-    global _cache_config_file
+    candidate_scopes = [scope]
+    if scope is None:
+        candidate_scopes = spack.config.config.scopes.keys()
 
-    compiler_config = get_compiler_config(scope)
-    config_length = len(compiler_config)
+    removal_happened = False
+    for current_scope in candidate_scopes:
+        removal_happened |= _remove_compiler_from_scope(compiler_spec, scope=current_scope)
+
+    return removal_happened
+
+
+def _remove_compiler_from_scope(compiler_spec, scope):
+    """Removes a compiler from a specific configuration scope.
+
+    Args:
+        compiler_spec: compiler to be removed
+        scope: configuration scope under consideration
 
+    Returns:
+         True if one or more compiler entries were actually removed, False otherwise
+    """
+    assert scope is not None, "a specific scope is needed when calling this function"
+    compiler_config = get_compiler_config(scope)
     filtered_compiler_config = [
-        comp
-        for comp in compiler_config
+        compiler_entry
+        for compiler_entry in compiler_config
         if not spack.spec.parse_with_version_concrete(
-            comp["compiler"]["spec"], compiler=True
+            compiler_entry["compiler"]["spec"], compiler=True
         ).satisfies(compiler_spec)
     ]
 
-    # Update the cache for changes
-    _cache_config_file = filtered_compiler_config
-    if len(filtered_compiler_config) == config_length:  # No items removed
-        CompilerSpecInsufficientlySpecificError(compiler_spec)
-    spack.config.set("compilers", filtered_compiler_config, scope=scope)
+    if len(filtered_compiler_config) == len(compiler_config):
+        return False
+
+    # We need to preserve the YAML type for comments, hence we are copying the
+    # items in the list that has just been retrieved
+    compiler_config[:] = filtered_compiler_config
+    spack.config.set("compilers", compiler_config, scope=scope)
+    return True
 
 
 def all_compilers_config(scope=None, init_config=True):
     """Return a set of specs for all the compiler versions currently
     available to build with.  These are instances of CompilerSpec.
     """
-    # Get compilers for this architecture.
-    # Create a cache of the config file so we don't load all the time.
-    global _cache_config_file
-    if not _cache_config_file:
-        _cache_config_file = get_compiler_config(scope, init_config)
-        return _cache_config_file
-    else:
-        return _cache_config_file
+    return get_compiler_config(scope, init_config)
 
 
 def all_compiler_specs(scope=None, init_config=True):
diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py
index 2153f45589..1bc5b9b9a4 100644
--- a/lib/spack/spack/config.py
+++ b/lib/spack/spack/config.py
@@ -1353,17 +1353,11 @@ def use_configuration(*scopes_or_paths):
     configuration = _config_from(scopes_or_paths)
     config.clear_caches(), configuration.clear_caches()
 
-    # Save and clear the current compiler cache
-    saved_compiler_cache = spack.compilers._cache_config_file
-    spack.compilers._cache_config_file = []
-
     saved_config, config = config, configuration
 
     try:
         yield configuration
     finally:
-        # Restore previous config files
-        spack.compilers._cache_config_file = saved_compiler_cache
         config = saved_config
 
 
diff --git a/lib/spack/spack/test/bindist.py b/lib/spack/spack/test/bindist.py
index 2941e1896c..f16459bc7f 100644
--- a/lib/spack/spack/test/bindist.py
+++ b/lib/spack/spack/test/bindist.py
@@ -115,9 +115,6 @@ def default_config(tmpdir, config_directory, monkeypatch, install_mockery_mutabl
 
     spack.config.config, old_config = cfg, spack.config.config
     spack.config.config.set("repos", [spack.paths.mock_packages_path])
-    # This is essential, otherwise the cache will create weird side effects
-    # that will compromise subsequent tests if compilers.yaml is modified
-    monkeypatch.setattr(spack.compilers, "_cache_config_file", [])
     njobs = spack.config.get("config:build_jobs")
     if not njobs:
         spack.config.set("config:build_jobs", 4, scope="user")
diff --git a/lib/spack/spack/test/cmd/compiler.py b/lib/spack/spack/test/cmd/compiler.py
index bca913f358..87eb7e4daf 100644
--- a/lib/spack/spack/test/cmd/compiler.py
+++ b/lib/spack/spack/test/cmd/compiler.py
@@ -8,8 +8,6 @@ import sys
 
 import pytest
 
-import llnl.util.filesystem
-
 import spack.compilers
 import spack.main
 import spack.version
@@ -18,73 +16,76 @@ compiler = spack.main.SpackCommand("compiler")
 
 
 @pytest.fixture
-def mock_compiler_version():
-    return "4.5.3"
-
-
-@pytest.fixture()
-def mock_compiler_dir(tmpdir, mock_compiler_version):
-    """Return a directory containing a fake, but detectable compiler."""
+def compilers_dir(mock_executable):
+    """Create a directory with some mock compiler scripts in it.
 
-    tmpdir.ensure("bin", dir=True)
-    bin_dir = tmpdir.join("bin")
-
-    gcc_path = bin_dir.join("gcc")
-    gxx_path = bin_dir.join("g++")
-    gfortran_path = bin_dir.join("gfortran")
-
-    gcc_path.write(
-        """\
-#!/bin/sh
+    Scripts are:
+      - clang
+      - clang++
+      - gcc
+      - g++
+      - gfortran-8
 
-for arg in "$@"; do
-    if [ "$arg" = -dumpversion ]; then
-        echo '%s'
-    fi
-done
-"""
-        % mock_compiler_version
+    """
+    clang_path = mock_executable(
+        "clang",
+        output="""
+if [ "$1" = "--version" ]; then
+    echo "clang version 11.0.0 (clang-1100.0.33.16)"
+    echo "Target: x86_64-apple-darwin18.7.0"
+    echo "Thread model: posix"
+    echo "InstalledDir: /dummy"
+else
+    echo "clang: error: no input files"
+    exit 1
+fi
+""",
     )
+    shutil.copy(clang_path, clang_path.parent / "clang++")
 
-    # Create some mock compilers in the temporary directory
-    llnl.util.filesystem.set_executable(str(gcc_path))
-    gcc_path.copy(gxx_path, mode=True)
-    gcc_path.copy(gfortran_path, mode=True)
+    gcc_script = """
+if [ "$1" = "-dumpversion" ]; then
+    echo "8"
+elif [ "$1" = "-dumpfullversion" ]; then
+    echo "8.4.0"
+elif [ "$1" = "--version" ]; then
+    echo "{0} (GCC) 8.4.0 20120313 (Red Hat 8.4.0-1)"
+    echo "Copyright (C) 2010 Free Software Foundation, Inc."
+else
+    echo "{1}: fatal error: no input files"
+    echo "compilation terminated."
+    exit 1
+fi
+"""
+    mock_executable("gcc-8", output=gcc_script.format("gcc", "gcc-8"))
+    mock_executable("g++-8", output=gcc_script.format("g++", "g++-8"))
+    mock_executable("gfortran-8", output=gcc_script.format("GNU Fortran", "gfortran-8"))
 
-    return str(tmpdir)
+    return clang_path.parent
 
 
-@pytest.mark.skipif(
-    sys.platform == "win32",
-    reason="Cannot execute bash \
-                                                     script on Windows",
-)
+@pytest.mark.skipif(sys.platform == "win32", reason="Cannot execute bash script on Windows")
 @pytest.mark.regression("11678,13138")
-def test_compiler_find_without_paths(no_compilers_yaml, working_env, tmpdir):
-    with tmpdir.as_cwd():
-        with open("gcc", "w") as f:
-            f.write(
-                """\
-#!/bin/sh
-echo "0.0.0"
-"""
-            )
-        os.chmod("gcc", 0o700)
+def test_compiler_find_without_paths(no_compilers_yaml, working_env, mock_executable):
+    """Tests that 'spack compiler find' looks into PATH by default, if no specific path
+    is given.
+    """
+    gcc_path = mock_executable("gcc", output='echo "0.0.0"')
 
-    os.environ["PATH"] = str(tmpdir)
+    os.environ["PATH"] = str(gcc_path.parent)
     output = compiler("find", "--scope=site")
 
     assert "gcc" in output
 
 
 @pytest.mark.regression("17589")
-def test_compiler_find_no_apple_gcc(no_compilers_yaml, working_env, tmpdir):
-    with tmpdir.as_cwd():
-        # make a script to emulate apple gcc's version args
-        with open("gcc", "w") as f:
-            f.write(
-                """\
-#!/bin/sh
+def test_compiler_find_no_apple_gcc(no_compilers_yaml, working_env, mock_executable):
+    """Tests that Spack won't mistake Apple's GCC as a "real" GCC, since it's really
+    Clang with a few tweaks.
+    """
+    gcc_path = mock_executable(
+        "gcc",
+        output="""
 if [ "$1" = "-dumpversion" ]; then
     echo "4.2.1"
 elif [ "$1" = "--version" ]; then
@@ -96,112 +97,71 @@ elif [ "$1" = "--version" ]; then
 else
     echo "clang: error: no input files"
 fi
-"""
-            )
-        os.chmod("gcc", 0o700)
+""",
+    )
 
-    os.environ["PATH"] = str(tmpdir)
+    os.environ["PATH"] = str(gcc_path.parent)
     output = compiler("find", "--scope=site")
 
     assert "gcc" not in output
 
 
+@pytest.mark.regression("37996")
 def test_compiler_remove(mutable_config, mock_packages):
+    """Tests that we can remove a compiler from configuration."""
     assert spack.spec.CompilerSpec("gcc@=4.5.0") in spack.compilers.all_compiler_specs()
     args = spack.util.pattern.Bunch(all=True, compiler_spec="gcc@4.5.0", add_paths=[], scope=None)
     spack.cmd.compiler.compiler_remove(args)
     assert spack.spec.CompilerSpec("gcc@=4.5.0") not in spack.compilers.all_compiler_specs()
 
 
-@pytest.mark.skipif(
-    sys.platform == "win32",
-    reason="Cannot execute bash \
-                                                     script on Windows",
-)
-def test_compiler_add(mutable_config, mock_packages, mock_compiler_dir, mock_compiler_version):
-    # Compilers available by default.
-    old_compilers = set(spack.compilers.all_compiler_specs())
+@pytest.mark.regression("37996")
+def test_removing_compilers_from_multiple_scopes(mutable_config, mock_packages):
+    # Duplicate "site" scope into "user" scope
+    site_config = spack.config.get("compilers", scope="site")
+    spack.config.set("compilers", site_config, scope="user")
 
-    args = spack.util.pattern.Bunch(
-        all=None, compiler_spec=None, add_paths=[mock_compiler_dir], scope=None
-    )
-    spack.cmd.compiler.compiler_find(args)
+    assert spack.spec.CompilerSpec("gcc@=4.5.0") in spack.compilers.all_compiler_specs()
+    args = spack.util.pattern.Bunch(all=True, compiler_spec="gcc@4.5.0", add_paths=[], scope=None)
+    spack.cmd.compiler.compiler_remove(args)
+    assert spack.spec.CompilerSpec("gcc@=4.5.0") not in spack.compilers.all_compiler_specs()
 
-    # Ensure new compiler is in there
-    new_compilers = set(spack.compilers.all_compiler_specs())
-    new_compiler = new_compilers - old_compilers
-    assert any(c.version == spack.version.Version(mock_compiler_version) for c in new_compiler)
 
+@pytest.mark.skipif(sys.platform == "win32", reason="Cannot execute bash script on Windows")
+def test_compiler_add(mutable_config, mock_packages, mock_executable):
+    """Tests that we can add a compiler to configuration."""
+    expected_version = "4.5.3"
+    gcc_path = mock_executable(
+        "gcc",
+        output=f"""\
+for arg in "$@"; do
+    if [ "$arg" = -dumpversion ]; then
+        echo '{expected_version}'
+    fi
+done
+""",
+    )
+    bin_dir = gcc_path.parent
+    root_dir = bin_dir.parent
 
-@pytest.fixture
-def clangdir(tmpdir):
-    """Create a directory with some dummy compiler scripts in it.
+    compilers_before_find = set(spack.compilers.all_compiler_specs())
+    args = spack.util.pattern.Bunch(
+        all=None, compiler_spec=None, add_paths=[str(root_dir)], scope=None
+    )
+    spack.cmd.compiler.compiler_find(args)
+    compilers_after_find = set(spack.compilers.all_compiler_specs())
 
-    Scripts are:
-      - clang
-      - clang++
-      - gcc
-      - g++
-      - gfortran-8
+    compilers_added_by_find = compilers_after_find - compilers_before_find
+    assert len(compilers_added_by_find) == 1
+    new_compiler = compilers_added_by_find.pop()
+    assert new_compiler.version == spack.version.Version(expected_version)
 
-    """
-    with tmpdir.as_cwd():
-        with open("clang", "w") as f:
-            f.write(
-                """\
-#!/bin/sh
-if [ "$1" = "--version" ]; then
-    echo "clang version 11.0.0 (clang-1100.0.33.16)"
-    echo "Target: x86_64-apple-darwin18.7.0"
-    echo "Thread model: posix"
-    echo "InstalledDir: /dummy"
-else
-    echo "clang: error: no input files"
-    exit 1
-fi
-"""
-            )
-        shutil.copy("clang", "clang++")
 
-        gcc_script = """\
-#!/bin/sh
-if [ "$1" = "-dumpversion" ]; then
-    echo "8"
-elif [ "$1" = "-dumpfullversion" ]; then
-    echo "8.4.0"
-elif [ "$1" = "--version" ]; then
-    echo "{0} (GCC) 8.4.0 20120313 (Red Hat 8.4.0-1)"
-    echo "Copyright (C) 2010 Free Software Foundation, Inc."
-else
-    echo "{1}: fatal error: no input files"
-    echo "compilation terminated."
-    exit 1
-fi
-"""
-        with open("gcc-8", "w") as f:
-            f.write(gcc_script.format("gcc", "gcc-8"))
-        with open("g++-8", "w") as f:
-            f.write(gcc_script.format("g++", "g++-8"))
-        with open("gfortran-8", "w") as f:
-            f.write(gcc_script.format("GNU Fortran", "gfortran-8"))
-        os.chmod("clang", 0o700)
-        os.chmod("clang++", 0o700)
-        os.chmod("gcc-8", 0o700)
-        os.chmod("g++-8", 0o700)
-        os.chmod("gfortran-8", 0o700)
-
-    yield tmpdir
-
-
-@pytest.mark.skipif(
-    sys.platform == "win32",
-    reason="Cannot execute bash \
-                                                     script on Windows",
-)
+@pytest.mark.skipif(sys.platform == "win32", reason="Cannot execute bash script on Windows")
 @pytest.mark.regression("17590")
-def test_compiler_find_mixed_suffixes(no_compilers_yaml, working_env, clangdir):
+def test_compiler_find_mixed_suffixes(no_compilers_yaml, working_env, compilers_dir):
     """Ensure that we'll mix compilers with different suffixes when necessary."""
-    os.environ["PATH"] = str(clangdir)
+    os.environ["PATH"] = str(compilers_dir)
     output = compiler("find", "--scope=site")
 
     assert "clang@11.0.0" in output
@@ -211,39 +171,33 @@ def test_compiler_find_mixed_suffixes(no_compilers_yaml, working_env, clangdir):
     clang = next(c["compiler"] for c in config if c["compiler"]["spec"] == "clang@=11.0.0")
     gcc = next(c["compiler"] for c in config if c["compiler"]["spec"] == "gcc@=8.4.0")
 
-    gfortran_path = str(clangdir.join("gfortran-8"))
+    gfortran_path = str(compilers_dir / "gfortran-8")
 
     assert clang["paths"] == {
-        "cc": str(clangdir.join("clang")),
-        "cxx": str(clangdir.join("clang++")),
+        "cc": str(compilers_dir / "clang"),
+        "cxx": str(compilers_dir / "clang++"),
         # we only auto-detect mixed clang on macos
         "f77": gfortran_path if sys.platform == "darwin" else None,
         "fc": gfortran_path if sys.platform == "darwin" else None,
     }
 
     assert gcc["paths"] == {
-        "cc": str(clangdir.join("gcc-8")),
-        "cxx": str(clangdir.join("g++-8")),
+        "cc": str(compilers_dir / "gcc-8"),
+        "cxx": str(compilers_dir / "g++-8"),
         "f77": gfortran_path,
         "fc": gfortran_path,
     }
 
 
-@pytest.mark.skipif(
-    sys.platform == "win32",
-    reason="Cannot execute bash \
-                                                     script on Windows",
-)
+@pytest.mark.skipif(sys.platform == "win32", reason="Cannot execute bash script on Windows")
 @pytest.mark.regression("17590")
-def test_compiler_find_prefer_no_suffix(no_compilers_yaml, working_env, clangdir):
+def test_compiler_find_prefer_no_suffix(no_compilers_yaml, working_env, compilers_dir):
     """Ensure that we'll pick 'clang' over 'clang-gpu' when there is a choice."""
-    with clangdir.as_cwd():
-        shutil.copy("clang", "clang-gpu")
-        shutil.copy("clang++", "clang++-gpu")
-        os.chmod("clang-gpu", 0o700)
-        os.chmod("clang++-gpu", 0o700)
+    clang_path = compilers_dir / "clang"
+    shutil.copy(clang_path, clang_path.parent / "clang-gpu")
+    shutil.copy(clang_path, clang_path.parent / "clang++-gpu")
 
-    os.environ["PATH"] = str(clangdir)
+    os.environ["PATH"] = str(compilers_dir)
     output = compiler("find", "--scope=site")
 
     assert "clang@11.0.0" in output
@@ -252,46 +206,38 @@ def test_compiler_find_prefer_no_suffix(no_compilers_yaml, working_env, clangdir
     config = spack.compilers.get_compiler_config("site", False)
     clang = next(c["compiler"] for c in config if c["compiler"]["spec"] == "clang@=11.0.0")
 
-    assert clang["paths"]["cc"] == str(clangdir.join("clang"))
-    assert clang["paths"]["cxx"] == str(clangdir.join("clang++"))
+    assert clang["paths"]["cc"] == str(compilers_dir / "clang")
+    assert clang["paths"]["cxx"] == str(compilers_dir / "clang++")
 
 
-@pytest.mark.skipif(
-    sys.platform == "win32",
-    reason="Cannot execute bash \
-                                                     script on Windows",
-)
-def test_compiler_find_path_order(no_compilers_yaml, working_env, clangdir):
-    """Ensure that we find compilers that come first in the PATH first"""
-
-    with clangdir.as_cwd():
-        os.mkdir("first_in_path")
-        shutil.copy("gcc-8", "first_in_path/gcc-8")
-        shutil.copy("g++-8", "first_in_path/g++-8")
-        shutil.copy("gfortran-8", "first_in_path/gfortran-8")
-
-    # the first_in_path folder should be searched first
-    os.environ["PATH"] = "{0}:{1}".format(str(clangdir.join("first_in_path")), str(clangdir))
+@pytest.mark.skipif(sys.platform == "win32", reason="Cannot execute bash script on Windows")
+def test_compiler_find_path_order(no_compilers_yaml, working_env, compilers_dir):
+    """Ensure that we look for compilers in the same order as PATH, when there are duplicates"""
+    new_dir = compilers_dir / "first_in_path"
+    new_dir.mkdir()
+    for name in ("gcc-8", "g++-8", "gfortran-8"):
+        shutil.copy(compilers_dir / name, new_dir / name)
+    # Set PATH to have the new folder searched first
+    os.environ["PATH"] = "{}:{}".format(str(new_dir), str(compilers_dir))
 
     compiler("find", "--scope=site")
 
     config = spack.compilers.get_compiler_config("site", False)
-
     gcc = next(c["compiler"] for c in config if c["compiler"]["spec"] == "gcc@=8.4.0")
-
     assert gcc["paths"] == {
-        "cc": str(clangdir.join("first_in_path", "gcc-8")),
-        "cxx": str(clangdir.join("first_in_path", "g++-8")),
-        "f77": str(clangdir.join("first_in_path", "gfortran-8")),
-        "fc": str(clangdir.join("first_in_path", "gfortran-8")),
+        "cc": str(new_dir / "gcc-8"),
+        "cxx": str(new_dir / "g++-8"),
+        "f77": str(new_dir / "gfortran-8"),
+        "fc": str(new_dir / "gfortran-8"),
     }
 
 
-def test_compiler_list_empty(no_compilers_yaml, working_env, clangdir):
-    # Spack should not automatically search for compilers when listing them and none
-    # are available. And when stdout is not a tty like in tests, there should be no
-    # output and no error exit code.
-    os.environ["PATH"] = str(clangdir)
+def test_compiler_list_empty(no_compilers_yaml, working_env, compilers_dir):
+    """Spack should not automatically search for compilers when listing them and none are
+    available. And when stdout is not a tty like in tests, there should be no output and
+    no error exit code.
+    """
+    os.environ["PATH"] = str(compilers_dir)
     out = compiler("list")
     assert not out
     assert compiler.returncode == 0
diff --git a/lib/spack/spack/test/cmd/external.py b/lib/spack/spack/test/cmd/external.py
index 19c9126612..848e14315d 100644
--- a/lib/spack/spack/test/cmd/external.py
+++ b/lib/spack/spack/test/cmd/external.py
@@ -44,9 +44,8 @@ def define_plat_exe(exe):
 
 def test_find_external_single_package(mock_executable, executables_found, _platform_executables):
     pkgs_to_check = [spack.repo.path.get_pkg_class("cmake")]
-    executables_found(
-        {mock_executable("cmake", output="echo cmake version 1.foo"): define_plat_exe("cmake")}
-    )
+    cmake_path = mock_executable("cmake", output="echo cmake version 1.foo")
+    executables_found({str(cmake_path): define_plat_exe("cmake")})
 
     pkg_to_entries = spack.detection.by_executable(pkgs_to_check)
 
@@ -71,7 +70,7 @@ def test_find_external_two_instances_same_package(
         "cmake", output="echo cmake version 3.17.2", subdir=("base2", "bin")
     )
     cmake_exe = define_plat_exe("cmake")
-    executables_found({cmake_path1: cmake_exe, cmake_path2: cmake_exe})
+    executables_found({str(cmake_path1): cmake_exe, str(cmake_path2): cmake_exe})
 
     pkg_to_entries = spack.detection.by_executable(pkgs_to_check)
 
@@ -107,7 +106,7 @@ def test_get_executables(working_env, mock_executable):
     cmake_path1 = mock_executable("cmake", output="echo cmake version 1.foo")
     path_to_exe = spack.detection.executables_in_path([os.path.dirname(cmake_path1)])
     cmake_exe = define_plat_exe("cmake")
-    assert path_to_exe[cmake_path1] == cmake_exe
+    assert path_to_exe[str(cmake_path1)] == cmake_exe
 
 
 external = SpackCommand("external")
@@ -334,7 +333,7 @@ def test_packages_yaml_format(mock_executable, mutable_config, monkeypatch, _pla
     assert "extra_attributes" in external_gcc
     extra_attributes = external_gcc["extra_attributes"]
     assert "prefix" not in extra_attributes
-    assert extra_attributes["compilers"]["c"] == gcc_exe
+    assert extra_attributes["compilers"]["c"] == str(gcc_exe)
 
 
 def test_overriding_prefix(mock_executable, mutable_config, monkeypatch, _platform_executables):
diff --git a/lib/spack/spack/test/concretize.py b/lib/spack/spack/test/concretize.py
index 8ef9b558b1..f1b4ceba6a 100644
--- a/lib/spack/spack/test/concretize.py
+++ b/lib/spack/spack/test/concretize.py
@@ -337,8 +337,6 @@ class TestConcretize(object):
 
         # Get the compiler that matches the spec (
         compiler = spack.compilers.compiler_for_spec("clang@=12.2.0", spec.architecture)
-        # Clear cache for compiler config since it has its own cache mechanism outside of config
-        spack.compilers._cache_config_file = []
 
         # Configure spack to have two identical compilers with different flags
         default_dict = spack.compilers._to_dict(compiler)
@@ -2137,7 +2135,7 @@ class TestConcretize(object):
             {
                 "compiler": {
                     "spec": "gcc@foo",
-                    "paths": {"cc": gcc_path, "cxx": gcc_path, "f77": None, "fc": None},
+                    "paths": {"cc": str(gcc_path), "cxx": str(gcc_path), "f77": None, "fc": None},
                     "operating_system": "debian6",
                     "modules": [],
                 }
diff --git a/lib/spack/spack/test/conftest.py b/lib/spack/spack/test/conftest.py
index 3a6b36e85b..f0e5f7fdb2 100644
--- a/lib/spack/spack/test/conftest.py
+++ b/lib/spack/spack/test/conftest.py
@@ -1669,22 +1669,21 @@ def clear_directive_functions():
 
 
 @pytest.fixture
-def mock_executable(tmpdir):
+def mock_executable(tmp_path):
     """Factory to create a mock executable in a temporary directory that
     output a custom string when run.
     """
-    import jinja2
-
     shebang = "#!/bin/sh\n" if sys.platform != "win32" else "@ECHO OFF"
 
     def _factory(name, output, subdir=("bin",)):
-        f = tmpdir.ensure(*subdir, dir=True).join(name)
+        executable_dir = tmp_path.joinpath(*subdir)
+        executable_dir.mkdir(parents=True, exist_ok=True)
+        executable_path = executable_dir / name
         if sys.platform == "win32":
-            f += ".bat"
-        t = jinja2.Template("{{ shebang }}{{ output }}\n")
-        f.write(t.render(shebang=shebang, output=output))
-        f.chmod(0o755)
-        return str(f)
+            executable_path = executable_dir / (name + ".bat")
+        executable_path.write_text(f"{ shebang }{ output }\n")
+        executable_path.chmod(0o755)
+        return executable_path
 
     return _factory
 
diff --git a/lib/spack/spack/test/versions.py b/lib/spack/spack/test/versions.py
index 8b58bf14ee..7d329336ae 100644
--- a/lib/spack/spack/test/versions.py
+++ b/lib/spack/spack/test/versions.py
@@ -935,7 +935,7 @@ def test_inclusion_upperbound():
 
 @pytest.mark.skipif(sys.platform == "win32", reason="Not supported on Windows (yet)")
 def test_git_version_repo_attached_after_serialization(
-    mock_git_version_info, mock_packages, monkeypatch
+    mock_git_version_info, mock_packages, config, monkeypatch
 ):
     """Test that a GitVersion instance can be serialized and deserialized
     without losing its repository reference.
@@ -954,7 +954,9 @@ def test_git_version_repo_attached_after_serialization(
 
 
 @pytest.mark.skipif(sys.platform == "win32", reason="Not supported on Windows (yet)")
-def test_resolved_git_version_is_shown_in_str(mock_git_version_info, mock_packages, monkeypatch):
+def test_resolved_git_version_is_shown_in_str(
+    mock_git_version_info, mock_packages, config, monkeypatch
+):
     """Test that a GitVersion from a commit without a user supplied version is printed
     as <hash>=<version>, and not just <hash>."""
     repo_path, _, commits = mock_git_version_info
@@ -968,7 +970,7 @@ def test_resolved_git_version_is_shown_in_str(mock_git_version_info, mock_packag
     assert str(spec.version) == f"{commit}=1.0-git.1"
 
 
-def test_unresolvable_git_versions_error(mock_packages):
+def test_unresolvable_git_versions_error(config, mock_packages):
     """Test that VersionLookupError is raised when a git prop is not set on a package."""
     with pytest.raises(VersionLookupError):
         # The package exists, but does not have a git property set. When dereferencing