Factor YAML manifest manipulation out of the Environment class (#36927)

Change the signature of the Environment.__init__ method to have
a single argument, i.e. the directory where the environment manifest 
is located. Initializing that directory is now delegated to a function 
taking care of all the error handling upfront. Environment objects 
require a "spack.yaml" to be available to be constructed.

Add a class to manage the environment manifest file. The environment 
now delegates to an attribute of that class the responsibility of keeping
track of changes modifying the manifest. This allows simplifying the 
updates of the manifest file, and helps keeping in sync the spec lists in
memory with the spack.yaml on disk.

14 files changed, 978 insertions, 502 deletions

diff --git a/lib/spack/spack/ci.py b/lib/spack/spack/ci.py
index 5056cb250f..15e5b9aedd 100644
--- a/lib/spack/spack/ci.py
+++ b/lib/spack/spack/ci.py
@@ -750,7 +750,7 @@ def generate_gitlab_ci_yaml(
             env.concretize()
             env.write()
 
-    yaml_root = ev.config_dict(env.yaml)
+    yaml_root = ev.config_dict(env.manifest)
 
     # Get the joined "ci" config with all of the current scopes resolved
     ci_config = cfg.get("ci")
diff --git a/lib/spack/spack/cmd/ci.py b/lib/spack/spack/cmd/ci.py
index d96f877d9a..12bd754e6a 100644
--- a/lib/spack/spack/cmd/ci.py
+++ b/lib/spack/spack/cmd/ci.py
@@ -227,7 +227,7 @@ def ci_reindex(args):
     Use the active, gitlab-enabled environment to rebuild the buildcache
     index for the associated mirror."""
     env = spack.cmd.require_active_env(cmd_name="ci rebuild-index")
-    yaml_root = ev.config_dict(env.yaml)
+    yaml_root = ev.config_dict(env.manifest)
 
     if "mirrors" not in yaml_root or len(yaml_root["mirrors"].values()) < 1:
         tty.die("spack ci rebuild-index requires an env containing a mirror")
diff --git a/lib/spack/spack/cmd/env.py b/lib/spack/spack/cmd/env.py
index 3f06e1e4eb..29f6db412a 100644
--- a/lib/spack/spack/cmd/env.py
+++ b/lib/spack/spack/cmd/env.py
@@ -163,7 +163,7 @@ def env_activate(args):
         env = create_temp_env_directory()
         env_path = os.path.abspath(env)
         short_name = os.path.basename(env_path)
-        ev.Environment(env).write(regenerate=False)
+        ev.create_in_dir(env).write(regenerate=False)
 
     # Managed environment
     elif ev.exists(env_name_or_dir) and not args.dir:
@@ -301,16 +301,17 @@ def env_create(args):
         # object could choose to enable a view by default. False means that
         # the environment should not include a view.
         with_view = None
-    if args.envfile:
-        with open(args.envfile) as f:
-            _env_create(
-                args.create_env, f, args.dir, with_view=with_view, keep_relative=args.keep_relative
-            )
-    else:
-        _env_create(args.create_env, None, args.dir, with_view=with_view)
+
+    _env_create(
+        args.create_env,
+        init_file=args.envfile,
+        dir=args.dir,
+        with_view=with_view,
+        keep_relative=args.keep_relative,
+    )
 
 
-def _env_create(name_or_path, init_file=None, dir=False, with_view=None, keep_relative=False):
+def _env_create(name_or_path, *, init_file=None, dir=False, with_view=None, keep_relative=False):
     """Create a new environment, with an optional yaml description.
 
     Arguments:
@@ -323,18 +324,21 @@ def _env_create(name_or_path, init_file=None, dir=False, with_view=None, keep_re
             the new environment file, otherwise they may be made absolute if the
             new environment is in a different location
     """
-    if dir:
-        env = ev.Environment(name_or_path, init_file, with_view, keep_relative)
-        env.write()
-        tty.msg("Created environment in %s" % env.path)
-        tty.msg("You can activate this environment with:")
-        tty.msg("  spack env activate %s" % env.path)
-    else:
-        env = ev.create(name_or_path, init_file, with_view, keep_relative)
-        env.write()
+    if not dir:
+        env = ev.create(
+            name_or_path, init_file=init_file, with_view=with_view, keep_relative=keep_relative
+        )
         tty.msg("Created environment '%s' in %s" % (name_or_path, env.path))
         tty.msg("You can activate this environment with:")
         tty.msg("  spack env activate %s" % (name_or_path))
+        return env
+
+    env = ev.create_in_dir(
+        name_or_path, init_file=init_file, with_view=with_view, keep_relative=keep_relative
+    )
+    tty.msg("Created environment in %s" % env.path)
+    tty.msg("You can activate this environment with:")
+    tty.msg("  spack env activate %s" % env.path)
     return env
 
 
@@ -431,21 +435,22 @@ def env_view_setup_parser(subparser):
 def env_view(args):
     env = ev.active_environment()
 
-    if env:
-        if args.action == ViewAction.regenerate:
-            env.regenerate_views()
-        elif args.action == ViewAction.enable:
-            if args.view_path:
-                view_path = args.view_path
-            else:
-                view_path = env.view_path_default
-            env.update_default_view(view_path)
-            env.write()
-        elif args.action == ViewAction.disable:
-            env.update_default_view(None)
-            env.write()
-    else:
+    if not env:
         tty.msg("No active environment")
+        return
+
+    if args.action == ViewAction.regenerate:
+        env.regenerate_views()
+    elif args.action == ViewAction.enable:
+        if args.view_path:
+            view_path = args.view_path
+        else:
+            view_path = env.view_path_default
+        env.update_default_view(view_path)
+        env.write()
+    elif args.action == ViewAction.disable:
+        env.update_default_view(path_or_bool=False)
+        env.write()
 
 
 #
diff --git a/lib/spack/spack/cmd/remove.py b/lib/spack/spack/cmd/remove.py
index 3e1dde16f2..4fdb36a5e5 100644
--- a/lib/spack/spack/cmd/remove.py
+++ b/lib/spack/spack/cmd/remove.py
@@ -38,6 +38,6 @@ def remove(parser, args):
             env.clear()
         else:
             for spec in spack.cmd.parse_specs(args.specs):
-                tty.msg("Removing %s from environment %s" % (spec, env.name))
                 env.remove(spec, args.list_name, force=args.force)
+                tty.msg(f"{spec} has been removed from {env.manifest}")
         env.write()
diff --git a/lib/spack/spack/environment/__init__.py b/lib/spack/spack/environment/__init__.py
index 7a3f4fbbb1..709c386d76 100644
--- a/lib/spack/spack/environment/__init__.py
+++ b/lib/spack/spack/environment/__init__.py
@@ -340,11 +340,14 @@ from .environment import (
     all_environments,
     config_dict,
     create,
+    create_in_dir,
     deactivate,
     default_manifest_yaml,
     default_view_name,
     display_specs,
+    environment_dir_from_name,
     exists,
+    initialize_environment_dir,
     installed_specs,
     is_env_dir,
     is_latest_format,
@@ -369,11 +372,14 @@ __all__ = [
     "all_environments",
     "config_dict",
     "create",
+    "create_in_dir",
     "deactivate",
     "default_manifest_yaml",
     "default_view_name",
     "display_specs",
+    "environment_dir_from_name",
     "exists",
+    "initialize_environment_dir",
     "installed_specs",
     "is_env_dir",
     "is_latest_format",
diff --git a/lib/spack/spack/environment/environment.py b/lib/spack/spack/environment/environment.py
index 31b14c4a86..eedca8daaa 100644
--- a/lib/spack/spack/environment/environment.py
+++ b/lib/spack/spack/environment/environment.py
@@ -3,9 +3,11 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 import collections
+import collections.abc
 import contextlib
 import copy
 import os
+import pathlib
 import re
 import shutil
 import stat
@@ -14,7 +16,7 @@ import time
 import urllib.parse
 import urllib.request
 import warnings
-from typing import List, Optional
+from typing import Any, Dict, List, Optional, Union
 
 import ruamel.yaml as yaml
 
@@ -270,14 +272,87 @@ def read(name):
     return Environment(root(name))
 
 
-def create(name, init_file=None, with_view=None, keep_relative=False):
-    """Create a managed environment in Spack."""
+def create(
+    name: str,
+    init_file: Optional[Union[str, pathlib.Path]] = None,
+    with_view: Optional[Union[str, pathlib.Path, bool]] = None,
+    keep_relative: bool = False,
+) -> "Environment":
+    """Create a managed environment in Spack and returns it.
+
+    A managed environment is created in a root directory managed by this Spack instance, so that
+    Spack can keep track of them.
+
+    Args:
+        name: name of the managed environment
+        init_file: either a "spack.yaml" or a "spack.lock" file or None
+        with_view: whether a view should be maintained for the environment. If the value is a
+            string, it specifies the path to the view
+        keep_relative: if True, develop paths are copied verbatim into the new environment file,
+            otherwise they are made absolute
+    """
+    environment_dir = environment_dir_from_name(name, exists_ok=False)
+    return create_in_dir(
+        environment_dir, init_file=init_file, with_view=with_view, keep_relative=keep_relative
+    )
+
+
+def create_in_dir(
+    manifest_dir: Union[str, pathlib.Path],
+    init_file: Optional[Union[str, pathlib.Path]] = None,
+    with_view: Optional[Union[str, pathlib.Path, bool]] = None,
+    keep_relative: bool = False,
+) -> "Environment":
+    """Create an environment in the directory passed as input and returns it.
+
+    Args:
+        manifest_dir: directory where to create the environment.
+        init_file: either a "spack.yaml" or a "spack.lock" file or None
+        with_view: whether a view should be maintained for the environment. If the value is a
+            string, it specifies the path to the view
+        keep_relative: if True, develop paths are copied verbatim into the new environment file,
+            otherwise they are made absolute
+    """
+    initialize_environment_dir(manifest_dir, envfile=init_file)
+
+    if with_view is None and keep_relative:
+        return Environment(manifest_dir)
+
+    manifest = EnvironmentManifestFile(manifest_dir)
+
+    if with_view is not None:
+        manifest.set_default_view(with_view)
+
+    if not keep_relative and init_file is not None and str(init_file).endswith(manifest_name):
+        init_file = pathlib.Path(init_file)
+        manifest.absolutify_dev_paths(init_file.parent)
+
+    manifest.flush()
+
+    return Environment(manifest_dir)
+
+
+def environment_dir_from_name(name: str, exists_ok: bool = True) -> str:
+    """Returns the directory associated with a named environment.
+
+    Args:
+        name: name of the environment
+        exists_ok: if False, raise an error if the environment exists already
+
+    Raises:
+        SpackEnvironmentError: if exists_ok is False and the environment exists already
+    """
+    if not exists_ok and exists(name):
+        raise SpackEnvironmentError(f"'{name}': environment already exists at {root(name)}")
+
+    ensure_env_root_path_exists()
+    validate_env_name(name)
+    return root(name)
+
+
+def ensure_env_root_path_exists():
     if not os.path.isdir(env_root_path()):
         fs.mkdirp(env_root_path())
-    validate_env_name(name)
-    if exists(name):
-        raise SpackEnvironmentError("'%s': environment already exists at %s" % (name, root(name)))
-    return Environment(root(name), init_file, with_view, keep_relative)
 
 
 def config_dict(yaml_data):
@@ -313,7 +388,7 @@ def _read_yaml(str_or_file):
     data = syaml.load_config(str_or_file)
     filename = getattr(str_or_file, "name", None)
     default_data = spack.config.validate(data, spack.schema.env.schema, filename)
-    return (data, default_data)
+    return data, default_data
 
 
 def _write_yaml(data, str_or_file):
@@ -399,7 +474,7 @@ def _error_on_nonempty_view_dir(new_root):
     )
 
 
-class ViewDescriptor(object):
+class ViewDescriptor:
     def __init__(
         self,
         base_path,
@@ -425,6 +500,10 @@ class ViewDescriptor(object):
     def exclude_fn(self, spec):
         return not any(spec.satisfies(e) for e in self.exclude)
 
+    def update_root(self, new_path):
+        self.raw_root = new_path
+        self.root = spack.util.path.canonicalize_path(new_path, default_wd=self.base)
+
     def __eq__(self, other):
         return all(
             [
@@ -665,144 +744,77 @@ class ViewDescriptor(object):
                 tty.warn(msg)
 
 
-def _create_environment(*args, **kwargs):
-    return Environment(*args, **kwargs)
+def _create_environment(path):
+    return Environment(path)
 
 
-class Environment(object):
-    def __init__(self, path, init_file=None, with_view=None, keep_relative=False):
-        """Create a new environment.
+class Environment:
+    """A Spack environment, which bundles together configuration and a list of specs."""
 
-        The environment can be optionally initialized with either a
-        spack.yaml or spack.lock file.
+    def __init__(self, manifest_dir: Union[str, pathlib.Path]) -> None:
+        """An environment can be constructed from a directory containing a "spack.yaml" file, and
+        optionally a consistent "spack.lock" file.
 
-        Arguments:
-            path (str): path to the root directory of this environment
-            init_file (str or file object): filename or file object to
-                initialize the environment
-            with_view (str or bool): whether a view should be maintained for
-                the environment. If the value is a string, it specifies the
-                path to the view.
-            keep_relative (bool): if True, develop paths are copied verbatim
-                into the new environment file, otherwise they are made absolute
-                when the environment path is different from init_file's
-                directory.
+        Args:
+            manifest_dir: directory with the "spack.yaml" associated with the environment
         """
-        self.path = os.path.abspath(path)
-        self.init_file = init_file
-        self.with_view = with_view
-        self.keep_relative = keep_relative
+        self.path = os.path.abspath(str(manifest_dir))
 
         self.txlock = lk.Lock(self._transaction_lock_path)
 
-        # This attribute will be set properly from configuration
-        # during concretization
         self.unify = None
-        self.new_specs = []
-        self.new_installs = []
-        self.clear()
-
-        if init_file:
-            # If we are creating the environment from an init file, we don't
-            # need to lock, because there are no Spack operations that alter
-            # the init file.
-            with fs.open_if_filename(init_file) as f:
-                if hasattr(f, "name") and f.name.endswith(".lock"):
-                    self._read_manifest(default_manifest_yaml())
-                    self._read_lockfile(f)
-                    self._set_user_specs_from_lockfile()
-                else:
-                    self._read_manifest(f, raw_yaml=default_manifest_yaml())
+        self.new_specs: List[Spec] = []
+        self.new_installs: List[Spec] = []
+        self.views: Dict[str, ViewDescriptor] = {}
+
+        #: Specs from "spack.yaml"
+        self.spec_lists = {user_speclist_name: SpecList()}
+        #: Dev-build specs from "spack.yaml"
+        self.dev_specs: Dict[str, Any] = {}
+        #: User specs from the last concretization
+        self.concretized_user_specs: List[Spec] = []
+        #: Roots associated with the last concretization, in order
+        self.concretized_order: List[Spec] = []
+        #: Concretized specs by hash
+        self.specs_by_hash: Dict[str, Spec] = {}
+        #: Repository for this environment (memoized)
+        self._repo = None
+        #: Previously active environment
+        self._previous_active = None
 
-                # Rewrite relative develop paths when initializing a new
-                # environment in a different location from the spack.yaml file.
-                if not keep_relative and hasattr(f, "name") and f.name.endswith(".yaml"):
-                    init_file_dir = os.path.abspath(os.path.dirname(f.name))
-                    self._rewrite_relative_paths_on_relocation(init_file_dir)
-        else:
-            with lk.ReadTransaction(self.txlock):
-                self._read()
-
-        if with_view is False:
-            self.views = {}
-        elif with_view is True:
-            self.views = {default_view_name: ViewDescriptor(self.path, self.view_path_default)}
-        elif isinstance(with_view, str):
-            self.views = {default_view_name: ViewDescriptor(self.path, with_view)}
-        # If with_view is None, then defer to the view settings determined by
-        # the manifest file
+        with lk.ReadTransaction(self.txlock):
+            self.manifest = EnvironmentManifestFile(manifest_dir)
+            self._read()
 
     def __reduce__(self):
-        return _create_environment, (self.path, self.init_file, self.with_view, self.keep_relative)
-
-    def _rewrite_relative_paths_on_relocation(self, init_file_dir):
-        """When initializing the environment from a manifest file and we plan
-        to store the environment in a different directory, we have to rewrite
-        relative paths to absolute ones."""
-        if init_file_dir == self.path:
-            return
-
-        for name, entry in self.dev_specs.items():
-            dev_path = entry["path"]
-            expanded_path = os.path.normpath(os.path.join(init_file_dir, entry["path"]))
-
-            # Skip if the expanded path is the same (e.g. when absolute)
-            if dev_path == expanded_path:
-                continue
-
-            tty.debug("Expanding develop path for {0} to {1}".format(name, expanded_path))
-
-            self.dev_specs[name]["path"] = expanded_path
+        return _create_environment, (self.path,)
 
     def _re_read(self):
-        """Reinitialize the environment object if it has been written (this
-        may not be true if the environment was just created in this running
-        instance of Spack)."""
-        if not os.path.exists(self.manifest_path):
-            return
-
+        """Reinitialize the environment object."""
         self.clear(re_read=True)
+        self.manifest = EnvironmentManifestFile(self.path)
         self._read()
 
     def _read(self):
-        default_manifest = not os.path.exists(self.manifest_path)
-        if default_manifest:
-            # No manifest, use default yaml
-            self._read_manifest(default_manifest_yaml())
-        else:
-            with open(self.manifest_path) as f:
-                self._read_manifest(f)
+        self._construct_state_from_manifest()
 
         if os.path.exists(self.lock_path):
             with open(self.lock_path) as f:
                 read_lock_version = self._read_lockfile(f)
-            if default_manifest:
-                # No manifest, set user specs from lockfile
-                self._set_user_specs_from_lockfile()
 
             if read_lock_version == 1:
-                tty.debug(
-                    "Storing backup of old lockfile {0} at {1}".format(
-                        self.lock_path, self._lock_backup_v1_path
-                    )
-                )
+                tty.debug(f"Storing backup of {self.lock_path} at {self._lock_backup_v1_path}")
                 shutil.copy(self.lock_path, self._lock_backup_v1_path)
 
     def write_transaction(self):
         """Get a write lock context manager for use in a `with` block."""
         return lk.WriteTransaction(self.txlock, acquire=self._re_read)
 
-    def _read_manifest(self, f, raw_yaml=None):
+    def _construct_state_from_manifest(self):
         """Read manifest file and set up user specs."""
-        if raw_yaml:
-            _, self.yaml = _read_yaml(f)
-            self.raw_yaml, _ = _read_yaml(raw_yaml)
-        else:
-            self.raw_yaml, self.yaml = _read_yaml(f)
-
         self.spec_lists = collections.OrderedDict()
 
-        for item in config_dict(self.yaml).get("definitions", []):
+        for item in config_dict(self.manifest).get("definitions", []):
             entry = copy.deepcopy(item)
             when = _eval_conditional(entry.pop("when", "True"))
             assert len(entry) == 1
@@ -814,13 +826,13 @@ class Environment(object):
                 else:
                     self.spec_lists[name] = user_specs
 
-        spec_list = config_dict(self.yaml).get(user_speclist_name, [])
+        spec_list = config_dict(self.manifest).get(user_speclist_name, [])
         user_specs = SpecList(
             user_speclist_name, [s for s in spec_list if s], self.spec_lists.copy()
         )
         self.spec_lists[user_speclist_name] = user_specs
 
-        enable_view = config_dict(self.yaml).get("view")
+        enable_view = config_dict(self.manifest).get("view")
         # enable_view can be boolean, string, or None
         if enable_view is True or enable_view is None:
             self.views = {default_view_name: ViewDescriptor(self.path, self.view_path_default)}
@@ -836,13 +848,13 @@ class Environment(object):
             self.views = {}
 
         # Retrieve the current concretization strategy
-        configuration = config_dict(self.yaml)
+        configuration = config_dict(self.manifest)
 
         # Retrieve unification scheme for the concretizer
         self.unify = spack.config.get("concretizer:unify", False)
 
         # Retrieve dev-build packages:
-        self.dev_specs = configuration.get("develop", {})
+        self.dev_specs = copy.deepcopy(configuration.get("develop", {}))
         for name, entry in self.dev_specs.items():
             # spec must include a concrete version
             assert Spec(entry["spec"]).version.concrete
@@ -854,14 +866,6 @@ class Environment(object):
     def user_specs(self):
         return self.spec_lists[user_speclist_name]
 
-    def _set_user_specs_from_lockfile(self):
-        """Copy user_specs from a read-in lockfile."""
-        self.spec_lists = {
-            user_speclist_name: SpecList(
-                user_speclist_name, [str(s) for s in self.concretized_user_specs]
-            )
-        }
-
     def clear(self, re_read=False):
         """Clear the contents of the environment
 
@@ -876,7 +880,7 @@ class Environment(object):
         self.concretized_user_specs = []  # user specs from last concretize
         self.concretized_order = []  # roots of last concretize, in order
         self.specs_by_hash = {}  # concretized specs by hash
-        self._repo = None  # RepoPath for this env (memoized)
+        self.invalidate_repository_cache()
         self._previous_active = None  # previously active environment
         if not re_read:
             # things that cannot be recreated from file
@@ -970,7 +974,7 @@ class Environment(object):
 
         # load config scopes added via 'include:', in reverse so that
         # highest-precedence scopes are last.
-        includes = config_dict(self.yaml).get("include", [])
+        includes = config_dict(self.manifest).get("include", [])
         missing = []
         for i, config_path in enumerate(reversed(includes)):
             # allow paths to contain spack config/environment variables, etc.
@@ -1066,7 +1070,7 @@ class Environment(object):
             config_name,
             self.manifest_path,
             spack.schema.env.schema,
-            [spack.config.first_existing(self.raw_yaml, spack.schema.env.keys)],
+            [spack.config.first_existing(self.manifest, spack.schema.env.keys)],
         )
 
     def config_scopes(self):
@@ -1104,13 +1108,11 @@ class Environment(object):
         spec = Spec(user_spec)
 
         if list_name not in self.spec_lists:
-            raise SpackEnvironmentError(
-                "No list %s exists in environment %s" % (list_name, self.name)
-            )
+            raise SpackEnvironmentError(f"No list {list_name} exists in environment {self.name}")
 
         if list_name == user_speclist_name:
             if not spec.name:
-                raise SpackEnvironmentError("cannot add anonymous specs to an environment!")
+                raise SpackEnvironmentError("cannot add anonymous specs to an environment")
             elif not spack.repo.path.exists(spec.name):
                 virtuals = spack.repo.path.provider_index.providers.keys()
                 if spec.name not in virtuals:
@@ -1122,6 +1124,10 @@ class Environment(object):
         if not existing:
             list_to_change.add(str(spec))
             self.update_stale_references(list_name)
+            if list_name == user_speclist_name:
+                self.manifest.add_user_spec(str(user_spec))
+            else:
+                self.manifest.add_definition(str(user_spec), list_name=list_name)
 
         return bool(not existing)
 
@@ -1159,7 +1165,7 @@ class Environment(object):
                 " specify a named list that is not a matrix"
             )
 
-        matches = list(x for x in list_to_change if x.satisfies(match_spec))
+        matches = list((idx, x) for idx, x in enumerate(list_to_change) if x.satisfies(match_spec))
         if len(matches) == 0:
             raise ValueError(
                 "There are no specs named {0} in {1}".format(match_spec.name, list_name)
@@ -1167,35 +1173,42 @@ class Environment(object):
         elif len(matches) > 1 and not allow_changing_multiple_specs:
             raise ValueError("{0} matches multiple specs".format(str(match_spec)))
 
-        new_speclist = SpecList(list_name)
-        for i, spec in enumerate(list_to_change):
-            if spec.satisfies(match_spec):
-                new_speclist.add(Spec.override(spec, change_spec))
+        for idx, spec in matches:
+            override_spec = Spec.override(spec, change_spec)
+            self.spec_lists[list_name].specs[idx] = override_spec
+            if list_name == user_speclist_name:
+                self.manifest.override_user_spec(str(override_spec), idx=idx)
             else:
-                new_speclist.add(spec)
-
-        self.spec_lists[list_name] = new_speclist
-        self.update_stale_references()
+                self.manifest.override_definition(
+                    str(spec), override=str(override_spec), list_name=list_name
+                )
+        self.update_stale_references(from_list=list_name)
+        self._construct_state_from_manifest()
 
     def remove(self, query_spec, list_name=user_speclist_name, force=False):
         """Remove specs from an environment that match a query_spec"""
+        err_msg_header = (
+            f"cannot remove {query_spec} from '{list_name}' definition "
+            f"in {self.manifest.manifest_file}"
+        )
         query_spec = Spec(query_spec)
-
-        list_to_change = self.spec_lists[list_name]
-        matches = []
+        try:
+            list_to_change = self.spec_lists[list_name]
+        except KeyError as e:
+            msg = f"{err_msg_header}, since '{list_name}' does not exist"
+            raise SpackEnvironmentError(msg) from e
 
         if not query_spec.concrete:
             matches = [s for s in list_to_change if s.satisfies(query_spec)]
 
-        if not matches:
+        else:
             # concrete specs match against concrete specs in the env
             # by dag hash.
             specs_hashes = zip(self.concretized_user_specs, self.concretized_order)
-
             matches = [s for s, h in specs_hashes if query_spec.dag_hash() == h]
 
         if not matches:
-            raise SpackEnvironmentError("Not found: {0}".format(query_spec))
+            raise SpackEnvironmentError(f"{err_msg_header}, no spec matches")
 
         old_specs = set(self.user_specs)
         new_specs = set()
@@ -1208,14 +1221,20 @@ class Environment(object):
                 except spack.spec_list.SpecListError:
                     # define new specs list
                     new_specs = set(self.user_specs)
-                    msg = "Spec '%s' is part of a spec matrix and " % spec
-                    msg += "cannot be removed from list '%s'." % list_to_change
+                    msg = f"Spec '{spec}' is part of a spec matrix and "
+                    msg += f"cannot be removed from list '{list_to_change}'."
                     if force:
                         msg += " It will be removed from the concrete specs."
-                        # Mock new specs so we can remove this spec from
-                        # concrete spec lists
+                        # Mock new specs, so we can remove this spec from concrete spec lists
                         new_specs.remove(spec)
                     tty.warn(msg)
+                else:
+                    if list_name == user_speclist_name:
+                        for user_spec in matches:
+                            self.manifest.remove_user_spec(str(user_spec))
+                    else:
+                        for user_spec in matches:
+                            self.manifest.remove_definition(str(user_spec), list_name=list_name)
 
         # If force, update stale concretized specs
         for spec in old_specs - new_specs:
@@ -1280,7 +1299,9 @@ class Environment(object):
             pkg_cls(spec).stage.steal_source(abspath)
 
         # If it wasn't already in the list, append it
-        self.dev_specs[spec.name] = {"path": path, "spec": str(spec)}
+        entry = {"path": path, "spec": str(spec)}
+        self.dev_specs[spec.name] = entry
+        self.manifest.add_develop_spec(spec.name, entry=entry.copy())
         return True
 
     def undevelop(self, spec):
@@ -1290,6 +1311,7 @@ class Environment(object):
         spec = Spec(spec)  # In case it's a spec object
         if spec.name in self.dev_specs:
             del self.dev_specs[spec.name]
+            self.manifest.remove_develop_spec(spec.name)
             return True
         return False
 
@@ -1541,18 +1563,64 @@ class Environment(object):
 
         return self.views[default_view_name]
 
-    def update_default_view(self, viewpath):
-        name = default_view_name
-        if name in self.views and self.default_view.root != viewpath:
-            shutil.rmtree(self.default_view.root)
+    def update_default_view(self, path_or_bool: Union[str, bool]) -> None:
+        """Updates the path of the default view.
 
-        if viewpath:
-            if name in self.views:
-                self.default_view.root = viewpath
-            else:
-                self.views[name] = ViewDescriptor(self.path, viewpath)
+        If the argument passed as input is False the default view is deleted, if present. The
+        manifest will have an entry "view: false".
+
+        If the argument passed as input is True a default view is created, if not already present.
+        The manifest will have an entry "view: true". If a default view is already declared, it
+        will be left untouched.
+
+        If the argument passed as input is a path a default view pointing to that path is created,
+        if not present already. If a default view is already declared, only its "root" will be
+        changed.
+
+        Args:
+            path_or_bool: either True, or False or a path
+        """
+        view_path = self.view_path_default if path_or_bool is True else path_or_bool
+
+        # We don't have the view, and we want to remove it
+        if default_view_name not in self.views and path_or_bool is False:
+            return
+
+        # We want to enable the view, but we have it already
+        if default_view_name in self.views and path_or_bool is True:
+            return
+
+        # We have the view, and we want to set it to the same path
+        if default_view_name in self.views and self.default_view.root == view_path:
+            return
+
+        self.delete_default_view()
+        if path_or_bool is False:
+            self.views.pop(default_view_name, None)
+            self.manifest.remove_default_view()
+            return
+
+        # If we had a default view already just update its path,
+        # else create a new one and add it to views
+        if default_view_name in self.views:
+            self.default_view.update_root(view_path)
         else:
-            self.views.pop(name, None)
+            self.views[default_view_name] = ViewDescriptor(self.path, view_path)
+
+        self.manifest.set_default_view(self._default_view_as_yaml())
+
+    def delete_default_view(self) -> None:
+        """Deletes the default view associated with this environment."""
+        if default_view_name not in self.views:
+            return
+
+        try:
+            view = pathlib.Path(self.default_view.root)
+            shutil.rmtree(view.resolve())
+            view.unlink()
+        except FileNotFoundError as e:
+            msg = f"[ENVIRONMENT] error trying to delete the default view: {str(e)}"
+            tty.debug(msg)
 
     def regenerate_views(self):
         if not self.views:
@@ -2094,7 +2162,7 @@ class Environment(object):
         if self.specs_by_hash:
             self.ensure_env_directory_exists(dot_env=True)
             self.update_environment_repository()
-            self.update_manifest()
+            self.manifest.flush()
             # Write the lock file last. This is useful for Makefiles
             # with `spack.lock: spack.yaml` rules, where the target
             # should be newer than the prerequisite to avoid
@@ -2103,7 +2171,7 @@ class Environment(object):
         else:
             self.ensure_env_directory_exists(dot_env=False)
             with fs.safe_remove(self.lock_path):
-                self.update_manifest()
+                self.manifest.flush()
 
         if regenerate:
             self.regenerate_views()
@@ -2158,99 +2226,22 @@ class Environment(object):
             )
             warnings.warn(msg.format(self.name, self.name, ver))
 
-    def update_manifest(self):
-        """Update YAML manifest for this environment based on changes to
-        spec lists and views and write it.
-        """
-        yaml_dict = config_dict(self.yaml)
-        raw_yaml_dict = config_dict(self.raw_yaml)
-        # invalidate _repo cache
-        self._repo = None
-        # put any changes in the definitions in the YAML
-        for name, speclist in self.spec_lists.items():
-            if name == user_speclist_name:
-                # The primary list is handled differently
-                continue
+    def _default_view_as_yaml(self):
+        """This internal function assumes the default view is set"""
+        path = self.default_view.raw_root
+        if (
+            self.default_view == ViewDescriptor(self.path, self.view_path_default)
+            and len(self.views) == 1
+        ):
+            return True
 
-            active_yaml_lists = [
-                x
-                for x in yaml_dict.get("definitions", [])
-                if name in x and _eval_conditional(x.get("when", "True"))
-            ]
+        if self.default_view == ViewDescriptor(self.path, path) and len(self.views) == 1:
+            return path
 
-            # Remove any specs in yaml that are not in internal representation
-            for ayl in active_yaml_lists:
-                # If it's not a string, it's a matrix. Those can't have changed
-                # If it is a string that starts with '$', it's a reference.
-                # Those also can't have changed.
-                ayl[name][:] = [
-                    s
-                    for s in ayl.setdefault(name, [])
-                    if (not isinstance(s, str)) or s.startswith("$") or Spec(s) in speclist.specs
-                ]
-
-            # Put the new specs into the first active list from the yaml
-            new_specs = [
-                entry
-                for entry in speclist.yaml_list
-                if isinstance(entry, str)
-                and not any(entry in ayl[name] for ayl in active_yaml_lists)
-            ]
-            list_for_new_specs = active_yaml_lists[0].setdefault(name, [])
-            list_for_new_specs[:] = list_for_new_specs + new_specs
-        # put the new user specs in the YAML.
-        # This can be done directly because there can't be multiple definitions
-        # nor when clauses for `specs` list.
-        yaml_spec_list = yaml_dict.setdefault(user_speclist_name, [])
-        yaml_spec_list[:] = self.user_specs.yaml_list
-        # Construct YAML representation of view
-        default_name = default_view_name
-        if self.views and len(self.views) == 1 and default_name in self.views:
-            path = self.default_view.raw_root
-            if self.default_view == ViewDescriptor(self.path, self.view_path_default):
-                view = True
-            elif self.default_view == ViewDescriptor(self.path, path):
-                view = path
-            else:
-                view = dict((name, view.to_dict()) for name, view in self.views.items())
-        elif self.views:
-            view = dict((name, view.to_dict()) for name, view in self.views.items())
-        else:
-            view = False
-
-        yaml_dict["view"] = view
+        return self.default_view.to_dict()
 
-        if self.dev_specs:
-            # Remove entries that are mirroring defaults
-            write_dev_specs = copy.deepcopy(self.dev_specs)
-            for name, entry in write_dev_specs.items():
-                if entry["path"] == name:
-                    del entry["path"]
-            yaml_dict["develop"] = write_dev_specs
-        else:
-            yaml_dict.pop("develop", None)
-
-        # Remove yaml sections that are shadowing defaults
-        # construct garbage path to ensure we don't find a manifest by accident
-        with fs.temp_cwd() as env_dir:
-            bare_env = Environment(env_dir, with_view=self.view_path_default)
-            keys_present = list(yaml_dict.keys())
-            for key in keys_present:
-                if yaml_dict[key] == config_dict(bare_env.yaml).get(key, None):
-                    if key not in raw_yaml_dict:
-                        del yaml_dict[key]
-        # if all that worked, write out the manifest file at the top level
-        # (we used to check whether the yaml had changed and not write it out
-        # if it hadn't. We can't do that anymore because it could be the only
-        # thing that changed is the "override" attribute on a config dict,
-        # which would not show up in even a string comparison between the two
-        # keys).
-        changed = not yaml_equivalent(self.yaml, self.raw_yaml)
-        written = os.path.exists(self.manifest_path)
-        if changed or not written:
-            self.raw_yaml = copy.deepcopy(self.yaml)
-            with fs.write_tmp_and_move(os.path.realpath(self.manifest_path)) as f:
-                _write_yaml(self.yaml, f)
+    def invalidate_repository_cache(self):
+        self._repo = None
 
     def __enter__(self):
         self._previous_active = _active_environment
@@ -2500,6 +2491,360 @@ def no_active_environment():
             activate(env)
 
 
+def initialize_environment_dir(
+    environment_dir: Union[str, pathlib.Path], envfile: Optional[Union[str, pathlib.Path]]
+) -> None:
+    """Initialize an environment directory starting from an envfile.
+
+    The envfile can be either a "spack.yaml" manifest file, or a "spack.lock" file.
+
+    Args:
+        environment_dir: directory where the environment should be placed
+        envfile: manifest file or lockfile used to initialize the environment
+
+    Raises:
+        SpackEnvironmentError: if the directory can't be initialized
+    """
+    environment_dir = pathlib.Path(environment_dir)
+    target_lockfile = environment_dir / lockfile_name
+    target_manifest = environment_dir / manifest_name
+    if target_manifest.exists():
+        msg = f"cannot initialize environment, {target_manifest} already exists"
+        raise SpackEnvironmentError(msg)
+
+    if target_lockfile.exists():
+        msg = f"cannot initialize environment, {target_lockfile} already exists"
+        raise SpackEnvironmentError(msg)
+
+    def _ensure_env_dir():
+        try:
+            environment_dir.mkdir(parents=True, exist_ok=True)
+        except FileExistsError as e:
+            msg = f"cannot initialize the environment, '{environment_dir}' already exists"
+            raise SpackEnvironmentError(msg) from e
+
+    if envfile is None:
+        _ensure_env_dir()
+        target_manifest.write_text(default_manifest_yaml())
+        return
+
+    envfile = pathlib.Path(envfile)
+    if not envfile.exists() or not envfile.is_file():
+        msg = f"cannot initialize environment, {envfile} is not a valid file"
+        raise SpackEnvironmentError(msg)
+
+    if not str(envfile).endswith(manifest_name) and not str(envfile).endswith(lockfile_name):
+        msg = (
+            f"cannot initialize environment from '{envfile}', either a '{manifest_name}'"
+            f" or a '{lockfile_name}' file is needed"
+        )
+        raise SpackEnvironmentError(msg)
+
+    _ensure_env_dir()
+
+    # When we have a lockfile we should copy that and produce a consistent default manifest
+    if str(envfile).endswith(lockfile_name):
+        shutil.copy(envfile, target_lockfile)
+        # This constructor writes a spack.yaml which is consistent with the root
+        # specs in the spack.lock
+        EnvironmentManifestFile.from_lockfile(environment_dir)
+        return
+
+    shutil.copy(envfile, target_manifest)
+
+
+class EnvironmentManifestFile(collections.abc.Mapping):
+    """Manages the in-memory representation of a manifest file, and its synchronization
+    with the actual manifest on disk.
+    """
+
+    @staticmethod
+    def from_lockfile(manifest_dir: Union[pathlib.Path, str]) -> "EnvironmentManifestFile":
+        """Returns an environment manifest file compatible with the lockfile already present in
+        the environment directory.
+
+        This function also writes a spack.yaml file that is consistent with the spack.lock
+        already existing in the directory.
+
+        Args:
+             manifest_dir: directory where the lockfile is
+        """
+        manifest_dir = pathlib.Path(manifest_dir)
+        lockfile = manifest_dir / lockfile_name
+        with lockfile.open("r") as f:
+            data = sjson.load(f)
+        user_specs = data["roots"]
+
+        default_content = manifest_dir / manifest_name
+        default_content.write_text(default_manifest_yaml())
+        manifest = EnvironmentManifestFile(manifest_dir)
+        for item in user_specs:
+            manifest.add_user_spec(item["spec"])
+        manifest.flush()
+        return manifest
+
+    def __init__(self, manifest_dir: Union[pathlib.Path, str]) -> None:
+        self.manifest_dir = pathlib.Path(manifest_dir)
+        self.manifest_file = self.manifest_dir / manifest_name
+
+        if not self.manifest_file.exists():
+            msg = f"cannot find '{manifest_name}' in {self.manifest_dir}"
+            raise SpackEnvironmentError(msg)
+
+        with self.manifest_file.open() as f:
+            raw, with_defaults_added = _read_yaml(f)
+
+        #: Pristine YAML content, without defaults being added
+        self.pristine_yaml_content = raw
+        #: YAML content with defaults added by Spack, if they're missing
+        self.yaml_content = with_defaults_added
+        self.changed = False
+
+    def add_user_spec(self, user_spec: str) -> None:
+        """Appends the user spec passed as input to the list of root specs.
+
+        Args:
+            user_spec: user spec to be appended
+        """
+        config_dict(self.pristine_yaml_content)["specs"].append(user_spec)
+        config_dict(self.yaml_content)["specs"].append(user_spec)
+        self.changed = True
+
+    def remove_user_spec(self, user_spec: str) -> None:
+        """Removes the user spec passed as input from the list of root specs
+
+        Args:
+            user_spec: user spec to be removed
+
+        Raises:
+            SpackEnvironmentError: when the user spec is not in the list
+        """
+        try:
+            config_dict(self.pristine_yaml_content)["specs"].remove(user_spec)
+            config_dict(self.yaml_content)["specs"].remove(user_spec)
+        except ValueError as e:
+            msg = f"cannot remove {user_spec} from {self}, no such spec exists"
+            raise SpackEnvironmentError(msg) from e
+        self.changed = True
+
+    def override_user_spec(self, user_spec: str, idx: int) -> None:
+        """Overrides the user spec at index idx with the one passed as input.
+
+        Args:
+            user_spec: new user spec
+            idx: index of the spec to be overridden
+
+        Raises:
+            SpackEnvironmentError: when the user spec cannot be overridden
+        """
+        try:
+            config_dict(self.pristine_yaml_content)["specs"][idx] = user_spec
+            config_dict(self.yaml_content)["specs"][idx] = user_spec
+        except ValueError as e:
+            msg = f"cannot override {user_spec} from {self}"
+            raise SpackEnvironmentError(msg) from e
+        self.changed = True
+
+    def add_definition(self, user_spec: str, list_name: str) -> None:
+        """Appends a user spec to the first active definition mathing the name passed as argument.
+
+        Args:
+            user_spec: user spec to be appended
+            list_name: name of the definition where to append
+
+        Raises:
+            SpackEnvironmentError: is no valid definition exists already
+        """
+        defs = config_dict(self.pristine_yaml_content).get("definitions", [])
+        msg = f"cannot add {user_spec} to the '{list_name}' definition, no valid list exists"
+
+        for idx, item in self._iterate_on_definitions(defs, list_name=list_name, err_msg=msg):
+            item[list_name].append(user_spec)
+            break
+
+        config_dict(self.yaml_content)["definitions"][idx][list_name].append(user_spec)
+        self.changed = True
+
+    def remove_definition(self, user_spec: str, list_name: str) -> None:
+        """Removes a user spec from an active definition that matches the name passed as argument.
+
+        Args:
+            user_spec: user spec to be removed
+            list_name: name of the definition where to remove the spec from
+
+        Raises:
+            SpackEnvironmentError: if the user spec cannot be removed from the list,
+                or the list does not exist
+        """
+        defs = config_dict(self.pristine_yaml_content).get("definitions", [])
+        msg = (
+            f"cannot remove {user_spec} from the '{list_name}' definition, "
+            f"no valid list exists"
+        )
+
+        for idx, item in self._iterate_on_definitions(defs, list_name=list_name, err_msg=msg):
+            try:
+                item[list_name].remove(user_spec)
+                break
+            except ValueError:
+                pass
+
+        config_dict(self.yaml_content)["definitions"][idx][list_name].remove(user_spec)
+        self.changed = True
+
+    def override_definition(self, user_spec: str, *, override: str, list_name: str) -> None:
+        """Overrides a user spec from an active definition that matches the name passed
+        as argument.
+
+        Args:
+            user_spec: user spec to be overridden
+            override: new spec to be used
+            list_name: name of the definition where to override the spec
+
+        Raises:
+            SpackEnvironmentError: if the user spec cannot be overridden
+        """
+        defs = config_dict(self.pristine_yaml_content).get("definitions", [])
+        msg = f"cannot override {user_spec} with {override} in the '{list_name}' definition"
+
+        for idx, item in self._iterate_on_definitions(defs, list_name=list_name, err_msg=msg):
+            try:
+                sub_index = item[list_name].index(user_spec)
+                item[list_name][sub_index] = override
+                break
+            except ValueError:
+                pass
+
+        config_dict(self.yaml_content)["definitions"][idx][list_name][sub_index] = override
+        self.changed = True
+
+    def _iterate_on_definitions(self, definitions, *, list_name, err_msg):
+        """Iterates on definitions, returning the active ones matching a given name."""
+
+        def extract_name(_item):
+            names = list(x for x in _item if x != "when")
+            assert len(names) == 1, f"more than one name in {_item}"
+            return names[0]
+
+        for idx, item in enumerate(definitions):
+            name = extract_name(item)
+            if name != list_name:
+                continue
+
+            condition_str = item.get("when", "True")
+            if not _eval_conditional(condition_str):
+                continue
+
+            yield idx, item
+        else:
+            raise SpackEnvironmentError(err_msg)
+
+    def set_default_view(self, view: Union[bool, str, pathlib.Path, Dict[str, str]]) -> None:
+        """Sets the default view root in the manifest to the value passed as input.
+
+        Args:
+            view: If the value is a string or a path, it specifies the path to the view. If
+                True the default view is used for the environment, if False there's no view.
+        """
+        if isinstance(view, dict):
+            config_dict(self.pristine_yaml_content)["view"][default_view_name].update(view)
+            config_dict(self.yaml_content)["view"][default_view_name].update(view)
+            self.changed = True
+            return
+
+        if not isinstance(view, bool):
+            view = str(view)
+
+        config_dict(self.pristine_yaml_content)["view"] = view
+        config_dict(self.yaml_content)["view"] = view
+        self.changed = True
+
+    def remove_default_view(self) -> None:
+        """Removes the default view from the manifest file"""
+        view_data = config_dict(self.pristine_yaml_content).get("view")
+        if isinstance(view_data, collections.abc.Mapping):
+            config_dict(self.pristine_yaml_content)["view"].pop(default_view_name)
+            config_dict(self.yaml_content)["view"].pop(default_view_name)
+            self.changed = True
+            return
+
+        self.set_default_view(view=False)
+
+    def add_develop_spec(self, pkg_name: str, entry: Dict[str, str]) -> None:
+        """Adds a develop spec to the manifest file
+
+        Args:
+            pkg_name: name of the package to be developed
+            entry: spec and path of the developed package
+        """
+        # The environment sets the path to pkg_name is that is implicit
+        if entry["path"] == pkg_name:
+            entry.pop("path")
+
+        config_dict(self.pristine_yaml_content).setdefault("develop", {}).setdefault(
+            pkg_name, {}
+        ).update(entry)
+        config_dict(self.yaml_content).setdefault("develop", {}).setdefault(pkg_name, {}).update(
+            entry
+        )
+        self.changed = True
+
+    def remove_develop_spec(self, pkg_name: str) -> None:
+        """Removes a develop spec from the manifest file
+
+        Args:
+            pkg_name: package to be removed from development
+
+        Raises:
+            SpackEnvironmentError: if there is nothing to remove
+        """
+        try:
+            del config_dict(self.pristine_yaml_content)["develop"][pkg_name]
+        except KeyError as e:
+            msg = f"cannot remove '{pkg_name}' from develop specs in {self}, entry does not exist"
+            raise SpackEnvironmentError(msg) from e
+        del config_dict(self.yaml_content)["develop"][pkg_name]
+        self.changed = True
+
+    def absolutify_dev_paths(self, init_file_dir: Union[str, pathlib.Path]) -> None:
+        """Normalizes the dev paths in the environment with respect to the directory where the
+        initialization file resides.
+
+        Args:
+            init_file_dir: directory with the "spack.yaml" used to initialize the environment.
+        """
+        init_file_dir = pathlib.Path(init_file_dir).absolute()
+        for _, entry in config_dict(self.pristine_yaml_content).get("develop", {}).items():
+            expanded_path = os.path.normpath(str(init_file_dir / entry["path"]))
+            entry["path"] = str(expanded_path)
+
+        for _, entry in config_dict(self.yaml_content).get("develop", {}).items():
+            expanded_path = os.path.normpath(str(init_file_dir / entry["path"]))
+            entry["path"] = str(expanded_path)
+        self.changed = True
+
+    def flush(self) -> None:
+        """Synchronizes the object with the manifest file on disk."""
+        if not self.changed:
+            return
+
+        with fs.write_tmp_and_move(os.path.realpath(self.manifest_file)) as f:
+            _write_yaml(self.pristine_yaml_content, f)
+        self.changed = False
+
+    def __len__(self):
+        return len(self.yaml_content)
+
+    def __getitem__(self, key):
+        return self.yaml_content[key]
+
+    def __iter__(self):
+        return iter(self.yaml_content)
+
+    def __str__(self):
+        return str(self.manifest_file)
+
+
 class SpackEnvironmentError(spack.error.SpackError):
     """Superclass for all errors to do with Spack environments."""
 
diff --git a/lib/spack/spack/schema/ci.py b/lib/spack/spack/schema/ci.py
index 7ff6fdb0d0..2c22f071de 100644
--- a/lib/spack/spack/schema/ci.py
+++ b/lib/spack/spack/schema/ci.py
@@ -204,7 +204,7 @@ def update(data):
     # Warn if deprecated section is still in the environment
     ci_env = ev.active_environment()
     if ci_env:
-        env_config = ev.config_dict(ci_env.yaml)
+        env_config = ev.config_dict(ci_env.manifest)
         if "gitlab-ci" in env_config:
             tty.die("Error: `gitlab-ci` section detected with `ci`, these are not compatible")
 
diff --git a/lib/spack/spack/test/cmd/config.py b/lib/spack/spack/test/cmd/config.py
index 03c310ca1e..a7793d65c7 100644
--- a/lib/spack/spack/test/cmd/config.py
+++ b/lib/spack/spack/test/cmd/config.py
@@ -91,17 +91,10 @@ def test_config_edit(mutable_config, working_env):
 
 
 def test_config_get_gets_spack_yaml(mutable_mock_env_path):
-    env = ev.create("test")
-
     config("get", fail_on_error=False)
     assert config.returncode == 1
 
-    with env:
-        config("get", fail_on_error=False)
-        assert config.returncode == 1
-
-        env.write()
-
+    with ev.create("test") as env:
         assert "mpileaks" not in config("get")
 
         env.add("mpileaks")
@@ -671,4 +664,4 @@ spack:
         config("update", "-y", "config")
 
     with ev.Environment(str(tmpdir)) as e:
-        assert not e.raw_yaml["spack"]["config"]["ccache"]
+        assert not e.manifest.pristine_yaml_content["spack"]["config"]["ccache"]
diff --git a/lib/spack/spack/test/cmd/develop.py b/lib/spack/spack/test/cmd/develop.py
index 50441a6844..bea3aa3b8a 100644
--- a/lib/spack/spack/test/cmd/develop.py
+++ b/lib/spack/spack/test/cmd/develop.py
@@ -32,7 +32,7 @@ class TestDevelop(object):
         assert dev_specs_entry["spec"] == str(spec)
 
         # check yaml representation
-        yaml = ev.config_dict(env.yaml)
+        yaml = ev.config_dict(env.manifest)
         assert spec.name in yaml["develop"]
         yaml_entry = yaml["develop"][spec.name]
         assert yaml_entry["spec"] == str(spec)
diff --git a/lib/spack/spack/test/cmd/env.py b/lib/spack/spack/test/cmd/env.py
index 348c46261c..c567f6ee25 100644
--- a/lib/spack/spack/test/cmd/env.py
+++ b/lib/spack/spack/test/cmd/env.py
@@ -6,6 +6,7 @@ import filecmp
 import glob
 import io
 import os
+import pathlib
 import shutil
 import sys
 from argparse import Namespace
@@ -18,6 +19,7 @@ import llnl.util.link_tree
 import spack.cmd.env
 import spack.config
 import spack.environment as ev
+import spack.environment.environment
 import spack.environment.shell
 import spack.error
 import spack.modules
@@ -53,6 +55,18 @@ find = SpackCommand("find")
 sep = os.sep
 
 
+@pytest.fixture()
+def environment_from_manifest(tmp_path):
+    """Returns a new environment named 'test' from the content of a manifest file."""
+
+    def _create(content):
+        spack_yaml = tmp_path / ev.manifest_name
+        spack_yaml.write_text(content)
+        return _env_create("test", init_file=str(spack_yaml))
+
+    return _create
+
+
 def check_mpileaks_and_deps_in_view(viewdir):
     """Check that the expected install directories exist."""
     assert os.path.exists(str(viewdir.join(".spack", "mpileaks")))
@@ -427,11 +441,11 @@ def test_environment_status(capsys, tmpdir):
             with capsys.disabled():
                 assert "In environment test" in env("status")
 
-        with ev.Environment("local_dir"):
+        with ev.create_in_dir("local_dir"):
             with capsys.disabled():
                 assert os.path.join(os.getcwd(), "local_dir") in env("status")
 
-            e = ev.Environment("myproject")
+            e = ev.create_in_dir("myproject")
             e.write()
             with tmpdir.join("myproject").as_cwd():
                 with e:
@@ -445,21 +459,20 @@ def test_env_status_broken_view(
     mock_fetch,
     mock_custom_repository,
     install_mockery,
-    tmpdir,
+    tmp_path,
 ):
-    env_dir = str(tmpdir)
-    with ev.Environment(env_dir):
+    with ev.create_in_dir(tmp_path):
         install("--add", "trivial-install-test-package")
 
     # switch to a new repo that doesn't include the installed package
     # test that Spack detects the missing package and warns the user
     with spack.repo.use_repositories(mock_custom_repository):
-        with ev.Environment(env_dir):
+        with ev.Environment(tmp_path):
             output = env("status")
             assert "includes out of date packages or repos" in output
 
     # Test that the warning goes away when it's fixed
-    with ev.Environment(env_dir):
+    with ev.Environment(tmp_path):
         output = env("status")
         assert "includes out of date packages or repos" not in output
 
@@ -505,9 +518,9 @@ def test_env_repo():
     assert pkg_cls.namespace == "builtin.mock"
 
 
-def test_user_removed_spec():
+def test_user_removed_spec(environment_from_manifest):
     """Ensure a user can remove from any position in the spack.yaml file."""
-    initial_yaml = io.StringIO(
+    before = environment_from_manifest(
         """\
 env:
   specs:
@@ -516,8 +529,6 @@ env:
   - libelf
 """
     )
-
-    before = ev.create("test", initial_yaml)
     before.concretize()
     before.write()
 
@@ -536,17 +547,16 @@ env:
     after.concretize()
     after.write()
 
-    env_specs = after._get_environment_specs()
     read = ev.read("test")
     env_specs = read._get_environment_specs()
 
     assert not any(x.name == "hypre" for x in env_specs)
 
 
-def test_init_from_lockfile(tmpdir):
+def test_init_from_lockfile(environment_from_manifest):
     """Test that an environment can be instantiated from a lockfile."""
-    initial_yaml = io.StringIO(
-        """\
+    e1 = environment_from_manifest(
+        """
 env:
   specs:
   - mpileaks
@@ -554,11 +564,10 @@ env:
   - libelf
 """
     )
-    e1 = ev.create("test", initial_yaml)
     e1.concretize()
     e1.write()
 
-    e2 = ev.Environment(str(tmpdir), e1.lock_path)
+    e2 = _env_create("test2", init_file=e1.lock_path)
 
     for s1, s2 in zip(e1.user_specs, e2.user_specs):
         assert s1 == s2
@@ -571,10 +580,10 @@ env:
         assert s1 == s2
 
 
-def test_init_from_yaml(tmpdir):
+def test_init_from_yaml(environment_from_manifest):
     """Test that an environment can be instantiated from a lockfile."""
-    initial_yaml = io.StringIO(
-        """\
+    e1 = environment_from_manifest(
+        """
 env:
   specs:
   - mpileaks
@@ -582,11 +591,10 @@ env:
   - libelf
 """
     )
-    e1 = ev.create("test", initial_yaml)
     e1.concretize()
     e1.write()
 
-    e2 = ev.Environment(str(tmpdir), e1.manifest_path)
+    e2 = _env_create("test2", init_file=e1.manifest_path)
 
     for s1, s2 in zip(e1.user_specs, e2.user_specs):
         assert s1 == s2
@@ -597,13 +605,16 @@ env:
 
 
 @pytest.mark.usefixtures("config")
-def test_env_view_external_prefix(tmpdir_factory, mutable_database, mock_packages):
-    fake_prefix = tmpdir_factory.mktemp("a-prefix")
-    fake_bin = fake_prefix.join("bin")
-    fake_bin.ensure(dir=True)
-
-    initial_yaml = io.StringIO(
-        """\
+def test_env_view_external_prefix(tmp_path, mutable_database, mock_packages):
+    fake_prefix = tmp_path / "a-prefix"
+    fake_bin = fake_prefix / "bin"
+    fake_bin.mkdir(parents=True, exist_ok=False)
+
+    manifest_dir = tmp_path / "environment"
+    manifest_dir.mkdir(parents=True, exist_ok=False)
+    manifest_file = manifest_dir / ev.manifest_name
+    manifest_file.write_text(
+        """
 env:
   specs:
   - a
@@ -627,7 +638,7 @@ packages:
 
     test_scope = spack.config.InternalConfigScope("env-external-test", data=external_config_dict)
     with spack.config.override(test_scope):
-        e = ev.create("test", initial_yaml)
+        e = ev.create("test", manifest_file)
         e.concretize()
         # Note: normally installing specs in a test environment requires doing
         # a fake install, but not for external specs since no actions are
@@ -672,8 +683,9 @@ env:
     assert "test" not in out
 
 
-def test_env_with_config():
-    test_config = """\
+def test_env_with_config(environment_from_manifest):
+    e = environment_from_manifest(
+        """
 env:
   specs:
   - mpileaks
@@ -681,26 +693,22 @@ env:
     mpileaks:
       version: [2.2]
 """
-    _env_create("test", io.StringIO(test_config))
-
-    e = ev.read("test")
+    )
     with e:
         e.concretize()
 
     assert any(x.intersects("mpileaks@2.2") for x in e._get_environment_specs())
 
 
-def test_with_config_bad_include():
-    env_name = "test_bad_include"
-    test_config = """\
+def test_with_config_bad_include(environment_from_manifest):
+    e = environment_from_manifest(
+        """
 spack:
   include:
   - /no/such/directory
   - no/such/file.yaml
 """
-    _env_create(env_name, io.StringIO(test_config))
-
-    e = ev.read(env_name)
+    )
     with pytest.raises(spack.config.ConfigFileError) as exc:
         with e:
             e.concretize()
@@ -712,17 +720,19 @@ spack:
     assert ev.active_environment() is None
 
 
-def test_env_with_include_config_files_same_basename():
-    test_config = """\
-        env:
-            include:
-                - ./path/to/included-config.yaml
-                - ./second/path/to/include-config.yaml
-            specs:
-                [libelf, mpileaks]
-            """
+def test_env_with_include_config_files_same_basename(environment_from_manifest):
+    e = environment_from_manifest(
+        """
+env:
+  include:
+  - ./path/to/included-config.yaml
+  - ./second/path/to/include-config.yaml
+  specs:
+  - libelf
+  - mpileaks
+"""
+    )
 
-    _env_create("test", io.StringIO(test_config))
     e = ev.read("test")
 
     fs.mkdirp(os.path.join(e.path, "path", "to"))
@@ -781,14 +791,20 @@ env:
     )
 
 
-def test_env_with_included_config_file(packages_file):
+def test_env_with_included_config_file(environment_from_manifest, packages_file):
     """Test inclusion of a relative packages configuration file added to an
-    existing environment."""
+    existing environment.
+    """
     include_filename = "included-config.yaml"
-    test_config = mpileaks_env_config(os.path.join(".", include_filename))
-
-    _env_create("test", io.StringIO(test_config))
-    e = ev.read("test")
+    e = environment_from_manifest(
+        f"""\
+env:
+  include:
+  - {os.path.join(".", include_filename)}
+  specs:
+  - mpileaks
+"""
+    )
 
     included_path = os.path.join(e.path, include_filename)
     shutil.move(packages_file.strpath, included_path)
@@ -830,7 +846,7 @@ def test_env_with_included_config_missing_file(tmpdir, mutable_empty_config):
         ev.activate(env)
 
 
-def test_env_with_included_config_scope(tmpdir, packages_file):
+def test_env_with_included_config_scope(environment_from_manifest, packages_file):
     """Test inclusion of a package file from the environment's configuration
     stage directory. This test is intended to represent a case where a remote
     file has already been staged."""
@@ -838,15 +854,10 @@ def test_env_with_included_config_scope(tmpdir, packages_file):
 
     # Configure the environment to include file(s) from the environment's
     # remote configuration stage directory.
-    test_config = mpileaks_env_config(config_scope_path)
-
-    # Create the environment
-    _env_create("test", io.StringIO(test_config))
-
-    e = ev.read("test")
+    e = environment_from_manifest(mpileaks_env_config(config_scope_path))
 
     # Copy the packages.yaml file to the environment configuration
-    # directory so it is picked up during concretization. (Using
+    # directory, so it is picked up during concretization. (Using
     # copy instead of rename in case the fixture scope changes.)
     fs.mkdirp(config_scope_path)
     include_filename = os.path.basename(packages_file.strpath)
@@ -861,14 +872,11 @@ def test_env_with_included_config_scope(tmpdir, packages_file):
     assert any(x.satisfies("mpileaks@2.2") for x in e._get_environment_specs())
 
 
-def test_env_with_included_config_var_path(packages_file):
+def test_env_with_included_config_var_path(environment_from_manifest, packages_file):
     """Test inclusion of a package configuration file with path variables
     "staged" in the environment's configuration stage directory."""
     config_var_path = os.path.join("$tempdir", "included-config.yaml")
-    test_config = mpileaks_env_config(config_var_path)
-
-    _env_create("test", io.StringIO(test_config))
-    e = ev.read("test")
+    e = environment_from_manifest(mpileaks_env_config(config_var_path))
 
     config_real_path = substitute_path_variables(config_var_path)
     fs.mkdirp(os.path.dirname(config_real_path))
@@ -881,8 +889,9 @@ def test_env_with_included_config_var_path(packages_file):
     assert any(x.satisfies("mpileaks@2.2") for x in e._get_environment_specs())
 
 
-def test_env_config_precedence():
-    test_config = """\
+def test_env_config_precedence(environment_from_manifest):
+    e = environment_from_manifest(
+        """
 env:
   packages:
     libelf:
@@ -892,9 +901,7 @@ env:
   specs:
   - mpileaks
 """
-    _env_create("test", io.StringIO(test_config))
-    e = ev.read("test")
-
+    )
     with open(os.path.join(e.path, "included-config.yaml"), "w") as f:
         f.write(
             """\
@@ -916,8 +923,9 @@ packages:
     assert any(x.satisfies("libelf@0.8.12") for x in e._get_environment_specs())
 
 
-def test_included_config_precedence():
-    test_config = """\
+def test_included_config_precedence(environment_from_manifest):
+    e = environment_from_manifest(
+        """
 env:
   include:
   - ./high-config.yaml  # this one should take precedence
@@ -925,8 +933,7 @@ env:
   specs:
   - mpileaks
 """
-    _env_create("test", io.StringIO(test_config))
-    e = ev.read("test")
+    )
 
     with open(os.path.join(e.path, "high-config.yaml"), "w") as f:
         f.write(
@@ -970,7 +977,7 @@ env:
     with tmpdir.as_cwd():
         with pytest.raises(spack.config.ConfigFormatError) as e:
             env("create", "test", "./spack.yaml")
-        assert "./spack.yaml:2" in str(e)
+        assert "spack.yaml:2" in str(e)
         assert "'spacks' was unexpected" in str(e)
 
 
@@ -1255,14 +1262,17 @@ def test_env_without_view_install(tmpdir, mock_stage, mock_fetch, install_mocker
     check_mpileaks_and_deps_in_view(view_dir)
 
 
-def test_env_config_view_default(tmpdir, mock_stage, mock_fetch, install_mockery):
+def test_env_config_view_default(
+    environment_from_manifest, mock_stage, mock_fetch, install_mockery
+):
     # This config doesn't mention whether a view is enabled
-    test_config = """\
+    environment_from_manifest(
+        """
 env:
   specs:
   - mpileaks
 """
-    _env_create("test", io.StringIO(test_config))
+    )
 
     with ev.read("test"):
         install("--fake")
@@ -1879,7 +1889,9 @@ env:
 
         test = ev.read("test")
 
-        packages_lists = list(filter(lambda x: "packages" in x, test.yaml["env"]["definitions"]))
+        packages_lists = list(
+            filter(lambda x: "packages" in x, test.manifest["env"]["definitions"])
+        )
 
         assert len(packages_lists) == 2
         assert "callpath" not in packages_lists[0]["packages"]
@@ -2557,7 +2569,9 @@ spack:
     def _write_helper_raise(self):
         raise RuntimeError("some error")
 
-    monkeypatch.setattr(ev.Environment, "update_manifest", _write_helper_raise)
+    monkeypatch.setattr(
+        spack.environment.environment.EnvironmentManifestFile, "flush", _write_helper_raise
+    )
     with ev.Environment(str(tmpdir)) as e:
         e.concretize(force=True)
         with pytest.raises(RuntimeError):
@@ -2615,25 +2629,6 @@ def test_rewrite_rel_dev_path_named_env(tmpdir):
         assert e.dev_specs["mypkg2"]["path"] == sep + os.path.join("some", "other", "path")
 
 
-def test_rewrite_rel_dev_path_original_dir(tmpdir):
-    """Relative devevelop paths should not be rewritten when initializing an
-    environment with root path set to the same directory"""
-    init_env, _, _, spack_yaml = _setup_develop_packages(tmpdir)
-    with ev.Environment(str(init_env), str(spack_yaml)) as e:
-        assert e.dev_specs["mypkg1"]["path"] == "../build_folder"
-        assert e.dev_specs["mypkg2"]["path"] == "/some/other/path"
-
-
-def test_rewrite_rel_dev_path_create_original_dir(tmpdir):
-    """Relative develop paths should not be rewritten when creating an
-    environment in the original directory"""
-    init_env, _, _, spack_yaml = _setup_develop_packages(tmpdir)
-    env("create", "-d", str(init_env), str(spack_yaml))
-    with ev.Environment(str(init_env)) as e:
-        assert e.dev_specs["mypkg1"]["path"] == "../build_folder"
-        assert e.dev_specs["mypkg2"]["path"] == "/some/other/path"
-
-
 def test_does_not_rewrite_rel_dev_path_when_keep_relative_is_set(tmpdir):
     """Relative develop paths should not be rewritten when --keep-relative is
     passed to create"""
@@ -2659,8 +2654,9 @@ def test_custom_version_concretize_together(tmpdir):
     assert any("hdf5@myversion" in spec for _, spec in e.concretized_specs())
 
 
-def test_modules_relative_to_views(tmpdir, install_mockery, mock_fetch):
-    spack_yaml = """
+def test_modules_relative_to_views(environment_from_manifest, install_mockery, mock_fetch):
+    environment_from_manifest(
+        """
 spack:
   specs:
   - trivial-install-test-package
@@ -2671,7 +2667,7 @@ spack:
       roots:
         tcl: modules
 """
-    _env_create("test", io.StringIO(spack_yaml))
+    )
 
     with ev.read("test") as e:
         install()
@@ -2690,8 +2686,9 @@ spack:
     assert spec.prefix not in contents
 
 
-def test_multiple_modules_post_env_hook(tmpdir, install_mockery, mock_fetch):
-    spack_yaml = """
+def test_multiple_modules_post_env_hook(environment_from_manifest, install_mockery, mock_fetch):
+    environment_from_manifest(
+        """
 spack:
   specs:
   - trivial-install-test-package
@@ -2706,7 +2703,7 @@ spack:
       roots:
         tcl: full_modules
 """
-    _env_create("test", io.StringIO(spack_yaml))
+    )
 
     with ev.read("test") as e:
         install()
@@ -2818,17 +2815,17 @@ def test_env_view_fail_if_symlink_points_elsewhere(tmpdir, install_mockery, mock
     assert os.path.isdir(non_view_dir)
 
 
-def test_failed_view_cleanup(tmpdir, mock_stage, mock_fetch, install_mockery):
+def test_failed_view_cleanup(tmp_path, mock_stage, mock_fetch, install_mockery):
     """Tests whether Spack cleans up after itself when a view fails to create"""
-    view = str(tmpdir.join("view"))
-    with ev.create("env", with_view=view):
+    view_dir = tmp_path / "view"
+    with ev.create("env", with_view=str(view_dir)):
         add("libelf")
         install("--fake")
 
     # Save the current view directory.
-    resolved_view = os.path.realpath(view)
-    all_views = os.path.dirname(resolved_view)
-    views_before = os.listdir(all_views)
+    resolved_view = view_dir.resolve(strict=True)
+    all_views = resolved_view.parent
+    views_before = list(all_views.iterdir())
 
     # Add a spec that results in view clash when creating a view
     with ev.read("env"):
@@ -2838,9 +2835,9 @@ def test_failed_view_cleanup(tmpdir, mock_stage, mock_fetch, install_mockery):
 
     # Make sure there is no broken view in the views directory, and the current
     # view is the original view from before the failed regenerate attempt.
-    views_after = os.listdir(all_views)
+    views_after = list(all_views.iterdir())
     assert views_before == views_after
-    assert os.path.samefile(resolved_view, view)
+    assert view_dir.samefile(resolved_view), view_dir
 
 
 def test_environment_view_target_already_exists(tmpdir, mock_stage, mock_fetch, install_mockery):
@@ -2941,9 +2938,9 @@ def test_read_old_lock_and_write_new(config, tmpdir, lockfile):
                 shadowed_hash = dag_hash
 
     # make an env out of the old lockfile -- env should be able to read v1/v2/v3
-    test_lockfile_path = str(tmpdir.join("test.lock"))
+    test_lockfile_path = str(tmpdir.join("spack.lock"))
     shutil.copy(lockfile_path, test_lockfile_path)
-    _env_create("test", test_lockfile_path, with_view=False)
+    _env_create("test", init_file=test_lockfile_path, with_view=False)
 
     # re-read the old env as a new lockfile
     e = ev.read("test")
@@ -2958,24 +2955,24 @@ def test_read_old_lock_and_write_new(config, tmpdir, lockfile):
     assert old_hashes == hashes
 
 
-def test_read_v1_lock_creates_backup(config, tmpdir):
+def test_read_v1_lock_creates_backup(config, tmp_path):
     """When reading a version-1 lockfile, make sure that a backup of that file
     is created.
     """
-    # read in the JSON from a legacy v1 lockfile
-    v1_lockfile_path = os.path.join(spack.paths.test_path, "data", "legacy_env", "v1.lock")
-
-    # make an env out of the old lockfile
-    test_lockfile_path = str(tmpdir.join(ev.lockfile_name))
+    v1_lockfile_path = pathlib.Path(spack.paths.test_path) / "data" / "legacy_env" / "v1.lock"
+    test_lockfile_path = tmp_path / "init" / ev.lockfile_name
+    test_lockfile_path.parent.mkdir(parents=True, exist_ok=False)
     shutil.copy(v1_lockfile_path, test_lockfile_path)
 
-    e = ev.Environment(str(tmpdir))
+    e = ev.create_in_dir(tmp_path, init_file=test_lockfile_path)
     assert os.path.exists(e._lock_backup_v1_path)
     assert filecmp.cmp(e._lock_backup_v1_path, v1_lockfile_path)
 
 
 @pytest.mark.parametrize("lockfile", ["v1", "v2", "v3"])
-def test_read_legacy_lockfile_and_reconcretize(mock_stage, mock_fetch, install_mockery, lockfile):
+def test_read_legacy_lockfile_and_reconcretize(
+    mock_stage, mock_fetch, install_mockery, lockfile, tmp_path
+):
     # In legacy lockfiles v2 and v3 (keyed by build hash), there may be multiple
     # versions of the same spec with different build dependencies, which means
     # they will have different build hashes but the same DAG hash.
@@ -2985,9 +2982,10 @@ def test_read_legacy_lockfile_and_reconcretize(mock_stage, mock_fetch, install_m
     # After reconcretization with the *new*, finer-grained DAG hash, there should no
     # longer be conflicts, and the previously conflicting specs can coexist in the
     # same environment.
-    legacy_lockfile_path = os.path.join(
-        spack.paths.test_path, "data", "legacy_env", "%s.lock" % lockfile
-    )
+    test_path = pathlib.Path(spack.paths.test_path)
+    lockfile_content = test_path / "data" / "legacy_env" / f"{lockfile}.lock"
+    legacy_lockfile_path = tmp_path / ev.lockfile_name
+    shutil.copy(lockfile_content, legacy_lockfile_path)
 
     # The order of the root specs in this environment is:
     #     [
@@ -2997,7 +2995,7 @@ def test_read_legacy_lockfile_and_reconcretize(mock_stage, mock_fetch, install_m
     # So in v2 and v3 lockfiles we have two versions of dttop with the same DAG
     # hash but different build hashes.
 
-    env("create", "test", legacy_lockfile_path)
+    env("create", "test", str(legacy_lockfile_path))
     test = ev.read("test")
     assert len(test.specs_by_hash) == 1
 
@@ -3154,7 +3152,7 @@ def test_depfile_phony_convenience_targets(
     each package if "--make-prefix" is absent."""
     make = Executable("make")
     with fs.working_dir(str(tmpdir)):
-        with ev.Environment("."):
+        with ev.create_in_dir("."):
             add("dttop")
             concretize()
 
@@ -3277,10 +3275,11 @@ def test_env_include_packages_url(
         assert "openmpi" in cfg["all"]["providers"]["mpi"]
 
 
-def test_relative_view_path_on_command_line_is_made_absolute(tmpdir, config):
-    with fs.working_dir(str(tmpdir)):
+def test_relative_view_path_on_command_line_is_made_absolute(tmp_path, config):
+    with fs.working_dir(str(tmp_path)):
         env("create", "--with-view", "view", "--dir", "env")
         environment = ev.Environment(os.path.join(".", "env"))
+        environment.regenerate_views()
         assert os.path.samefile("view", environment.default_view.root)
 
 
diff --git a/lib/spack/spack/test/cmd/install.py b/lib/spack/spack/test/cmd/install.py
index 1eae5c6c5f..05b0ff3404 100644
--- a/lib/spack/spack/test/cmd/install.py
+++ b/lib/spack/spack/test/cmd/install.py
@@ -799,6 +799,7 @@ def test_install_no_add_in_env(tmpdir, mock_fetch, install_mockery, mutable_mock
     e.add("a")
     e.add("a ~bvv")
     e.concretize()
+    e.write()
     env_specs = e.all_specs()
 
     a_spec = None
diff --git a/lib/spack/spack/test/env.py b/lib/spack/spack/test/env.py
index 77ccd7889e..913eb7bf9e 100644
--- a/lib/spack/spack/test/env.py
+++ b/lib/spack/spack/test/env.py
@@ -3,8 +3,8 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 """Test environment internals without CLI"""
-import io
 import os
+import pickle
 import sys
 
 import pytest
@@ -13,16 +13,28 @@ import llnl.util.filesystem as fs
 
 import spack.environment as ev
 import spack.spec
+from spack.environment.environment import SpackEnvironmentViewError, _error_on_nonempty_view_dir
 
 pytestmark = pytest.mark.skipif(
     sys.platform == "win32", reason="Envs are not supported on windows"
 )
 
 
-def test_hash_change_no_rehash_concrete(tmpdir, mock_packages, config):
+class TestDirectoryInitialization:
+    def test_environment_dir_from_name(self, mutable_mock_env_path):
+        """Test the function mapping a managed environment name to its folder."""
+        env = ev.create("test")
+        environment_dir = ev.environment_dir_from_name("test")
+        assert env.path == environment_dir
+        with pytest.raises(ev.SpackEnvironmentError, match="environment already exists"):
+            ev.environment_dir_from_name("test", exists_ok=False)
+
+
+def test_hash_change_no_rehash_concrete(tmp_path, mock_packages, config):
     # create an environment
-    env_path = tmpdir.mkdir("env_dir").strpath
-    env = ev.Environment(env_path)
+    env_path = tmp_path / "env_dir"
+    env_path.mkdir(exist_ok=False)
+    env = ev.create_in_dir(env_path)
     env.write()
 
     # add a spec with a rewritten build hash
@@ -48,9 +60,10 @@ def test_hash_change_no_rehash_concrete(tmpdir, mock_packages, config):
     assert read_in.specs_by_hash[read_in.concretized_order[0]]._hash == new_hash
 
 
-def test_env_change_spec(tmpdir, mock_packages, config):
-    env_path = tmpdir.mkdir("env_dir").strpath
-    env = ev.Environment(env_path)
+def test_env_change_spec(tmp_path, mock_packages, config):
+    env_path = tmp_path / "env_dir"
+    env_path.mkdir(exist_ok=False)
+    env = ev.create_in_dir(env_path)
     env.write()
 
     spec = spack.spec.Spec("mpileaks@2.1~shared+debug")
@@ -80,9 +93,10 @@ env:
 """
 
 
-def test_env_change_spec_in_definition(tmpdir, mock_packages, config, mutable_mock_env_path):
-    initial_yaml = io.StringIO(_test_matrix_yaml)
-    e = ev.create("test", initial_yaml)
+def test_env_change_spec_in_definition(tmp_path, mock_packages, config, mutable_mock_env_path):
+    manifest_file = tmp_path / ev.manifest_name
+    manifest_file.write_text(_test_matrix_yaml)
+    e = ev.create("test", manifest_file)
     e.concretize()
     e.write()
 
@@ -96,10 +110,11 @@ def test_env_change_spec_in_definition(tmpdir, mock_packages, config, mutable_mo
 
 
 def test_env_change_spec_in_matrix_raises_error(
-    tmpdir, mock_packages, config, mutable_mock_env_path
+    tmp_path, mock_packages, config, mutable_mock_env_path
 ):
-    initial_yaml = io.StringIO(_test_matrix_yaml)
-    e = ev.create("test", initial_yaml)
+    manifest_file = tmp_path / ev.manifest_name
+    manifest_file.write_text(_test_matrix_yaml)
+    e = ev.create("test", manifest_file)
     e.concretize()
     e.write()
 
@@ -131,8 +146,8 @@ def test_user_view_path_is_not_canonicalized_in_yaml(tmpdir, config):
 
     # Serialize environment with relative view path
     with fs.working_dir(str(tmpdir)):
-        fst = ev.Environment(env_path, with_view=view)
-        fst.write()
+        fst = ev.create_in_dir(env_path, with_view=view)
+        fst.regenerate_views()
 
     # The view link should be created
     assert os.path.isdir(absolute_view)
@@ -141,7 +156,7 @@ def test_user_view_path_is_not_canonicalized_in_yaml(tmpdir, config):
     # and also check that the getter is pointing to the right dir.
     with fs.working_dir(str(tmpdir)):
         snd = ev.Environment(env_path)
-        assert snd.yaml["spack"]["view"] == view
+        assert snd.manifest["spack"]["view"] == view
         assert os.path.samefile(snd.default_view.root, absolute_view)
 
 
@@ -184,8 +199,167 @@ def test_roundtrip_spack_yaml_with_comments(original_content, mock_packages, con
     spack_yaml = tmp_path / "spack.yaml"
     spack_yaml.write_text(original_content)
 
-    e = ev.Environment(str(tmp_path))
-    e.update_manifest()
+    e = ev.Environment(tmp_path)
+    e.manifest.flush()
 
     content = spack_yaml.read_text()
     assert content == original_content
+
+
+def test_adding_anonymous_specs_to_env_fails(tmp_path):
+    """Tests that trying to add an anonymous spec to the 'specs' section of an environment
+    raises an exception
+    """
+    env = ev.create_in_dir(tmp_path)
+    with pytest.raises(ev.SpackEnvironmentError, match="cannot add anonymous"):
+        env.add("%gcc")
+
+
+def test_removing_from_non_existing_list_fails(tmp_path):
+    """Tests that trying to remove a spec from a non-existing definition fails."""
+    env = ev.create_in_dir(tmp_path)
+    with pytest.raises(ev.SpackEnvironmentError, match="'bar' does not exist"):
+        env.remove("%gcc", list_name="bar")
+
+
+@pytest.mark.parametrize(
+    "init_view,update_value",
+    [
+        (True, False),
+        (True, "./view"),
+        (False, True),
+        ("./view", True),
+        ("./view", False),
+        (True, True),
+        (False, False),
+    ],
+)
+def test_update_default_view(init_view, update_value, tmp_path, mock_packages, config):
+    """Tests updating the default view with different values."""
+    env = ev.create_in_dir(tmp_path, with_view=init_view)
+    env.update_default_view(update_value)
+    env.write(regenerate=True)
+    if not isinstance(update_value, bool):
+        assert env.default_view.raw_root == update_value
+
+    expected_value = update_value
+    if isinstance(init_view, str) and update_value is True:
+        expected_value = init_view
+
+    assert env.manifest.pristine_yaml_content["spack"]["view"] == expected_value
+
+
+@pytest.mark.parametrize(
+    "initial_content,update_value,expected_view",
+    [
+        (
+            """
+spack:
+  specs:
+  - mpileaks
+  view:
+    default:
+      root: ./view-gcc
+      select: ['%gcc']
+      link_type: symlink
+""",
+            "./another-view",
+            {"root": "./another-view", "select": ["%gcc"], "link_type": "symlink"},
+        ),
+        (
+            """
+spack:
+  specs:
+  - mpileaks
+  view:
+    default:
+      root: ./view-gcc
+      select: ['%gcc']
+      link_type: symlink
+""",
+            True,
+            {"root": "./view-gcc", "select": ["%gcc"], "link_type": "symlink"},
+        ),
+    ],
+)
+def test_update_default_complex_view(
+    initial_content, update_value, expected_view, tmp_path, mock_packages, config
+):
+    spack_yaml = tmp_path / "spack.yaml"
+    spack_yaml.write_text(initial_content)
+
+    env = ev.Environment(tmp_path)
+    env.update_default_view(update_value)
+    env.write(regenerate=True)
+
+    assert env.default_view.to_dict() == expected_view
+
+
+@pytest.mark.parametrize("filename", [ev.manifest_name, ev.lockfile_name])
+def test_cannot_initialize_in_dir_with_init_file(tmp_path, filename):
+    """Tests that initializing an environment in a directory with an already existing
+    spack.yaml or spack.lock raises an exception.
+    """
+    init_file = tmp_path / filename
+    init_file.touch()
+    with pytest.raises(ev.SpackEnvironmentError, match="cannot initialize"):
+        ev.create_in_dir(tmp_path)
+
+
+def test_cannot_initiliaze_if_dirname_exists_as_a_file(tmp_path):
+    """Tests that initializing an environment using as a location an existing file raises
+    an error.
+    """
+    dir_name = tmp_path / "dir"
+    dir_name.touch()
+    with pytest.raises(ev.SpackEnvironmentError, match="cannot initialize"):
+        ev.create_in_dir(dir_name)
+
+
+def test_cannot_initiliaze_if_init_file_does_not_exist(tmp_path):
+    """Tests that initializing an environment passing a non-existing init file raises an error."""
+    init_file = tmp_path / ev.manifest_name
+    with pytest.raises(ev.SpackEnvironmentError, match="cannot initialize"):
+        ev.create_in_dir(tmp_path, init_file=init_file)
+
+
+def test_cannot_initialize_from_random_file(tmp_path):
+    init_file = tmp_path / "foo.txt"
+    init_file.touch()
+    with pytest.raises(ev.SpackEnvironmentError, match="cannot initialize"):
+        ev.create_in_dir(tmp_path, init_file=init_file)
+
+
+def test_environment_pickle(tmp_path):
+    env1 = ev.create_in_dir(tmp_path)
+    obj = pickle.dumps(env1)
+    env2 = pickle.loads(obj)
+    assert isinstance(env2, ev.Environment)
+
+
+def test_error_on_nonempty_view_dir(tmpdir):
+    """Error when the target is not an empty dir"""
+    with tmpdir.as_cwd():
+        os.mkdir("empty_dir")
+        os.mkdir("nonempty_dir")
+        with open(os.path.join("nonempty_dir", "file"), "wb"):
+            pass
+        os.symlink("empty_dir", "symlinked_empty_dir")
+        os.symlink("does_not_exist", "broken_link")
+        os.symlink("broken_link", "file")
+
+        # This is OK.
+        _error_on_nonempty_view_dir("empty_dir")
+
+        # This is not OK.
+        with pytest.raises(SpackEnvironmentViewError):
+            _error_on_nonempty_view_dir("nonempty_dir")
+
+        with pytest.raises(SpackEnvironmentViewError):
+            _error_on_nonempty_view_dir("symlinked_empty_dir")
+
+        with pytest.raises(SpackEnvironmentViewError):
+            _error_on_nonempty_view_dir("broken_link")
+
+        with pytest.raises(SpackEnvironmentViewError):
+            _error_on_nonempty_view_dir("file")
diff --git a/lib/spack/spack/test/environment.py b/lib/spack/spack/test/environment.py
deleted file mode 100644
index 2f363dfdca..0000000000
--- a/lib/spack/spack/test/environment.py
+++ /dev/null
@@ -1,47 +0,0 @@
-# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
-# Spack Project Developers. See the top-level COPYRIGHT file for details.
-#
-# SPDX-License-Identifier: (Apache-2.0 OR MIT)
-
-import os
-import pickle
-
-import pytest
-
-from spack.environment import Environment
-from spack.environment.environment import SpackEnvironmentViewError, _error_on_nonempty_view_dir
-
-
-def test_environment_pickle(tmpdir):
-    env1 = Environment(str(tmpdir))
-    obj = pickle.dumps(env1)
-    env2 = pickle.loads(obj)
-    assert isinstance(env2, Environment)
-
-
-def test_error_on_nonempty_view_dir(tmpdir):
-    """Error when the target is not an empty dir"""
-    with tmpdir.as_cwd():
-        os.mkdir("empty_dir")
-        os.mkdir("nonempty_dir")
-        with open(os.path.join("nonempty_dir", "file"), "wb"):
-            pass
-        os.symlink("empty_dir", "symlinked_empty_dir")
-        os.symlink("does_not_exist", "broken_link")
-        os.symlink("broken_link", "file")
-
-        # This is OK.
-        _error_on_nonempty_view_dir("empty_dir")
-
-        # This is not OK.
-        with pytest.raises(SpackEnvironmentViewError):
-            _error_on_nonempty_view_dir("nonempty_dir")
-
-        with pytest.raises(SpackEnvironmentViewError):
-            _error_on_nonempty_view_dir("symlinked_empty_dir")
-
-        with pytest.raises(SpackEnvironmentViewError):
-            _error_on_nonempty_view_dir("broken_link")
-
-        with pytest.raises(SpackEnvironmentViewError):
-            _error_on_nonempty_view_dir("file")
diff --git a/lib/spack/spack/test/modules/lmod.py b/lib/spack/spack/test/modules/lmod.py
index 30e1b58905..0b3d2bf597 100644
--- a/lib/spack/spack/test/modules/lmod.py
+++ b/lib/spack/spack/test/modules/lmod.py
@@ -335,7 +335,7 @@ class TestLmod(object):
     def test_modules_relative_to_view(
         self, tmpdir, modulefile_content, module_configuration, install_mockery, mock_fetch
     ):
-        with ev.Environment(str(tmpdir), with_view=True) as e:
+        with ev.create_in_dir(str(tmpdir), with_view=True) as e:
             module_configuration("with_view")
             install("--add", "cmake")