utils: merge spack.environment into spack.util.environment

- `spack.util.environment` is the new home for routines that modify
  environment variables.

- This is to make room for `spack.environment` to contain new routines
  for dealing with spack environments

6 files changed, 628 insertions, 634 deletions

diff --git a/lib/spack/docs/tutorial_advanced_packaging.rst b/lib/spack/docs/tutorial_advanced_packaging.rst
index 9b41980a69..c7143db209 100644
--- a/lib/spack/docs/tutorial_advanced_packaging.rst
+++ b/lib/spack/docs/tutorial_advanced_packaging.rst
@@ -278,7 +278,7 @@ that spack does is not sufficient for python to import modules.
 
 To provide environment setup for a dependent, a package can implement the
 :py:func:`setup_dependent_environment <spack.package.PackageBase.setup_dependent_environment>`
-function. This function takes as a parameter a :py:class:`EnvironmentModifications <spack.environment.EnvironmentModifications>`
+function. This function takes as a parameter a :py:class:`EnvironmentModifications <spack.util.environment.EnvironmentModifications>`
 object which includes convenience methods to update the environment. For
 example, an MPI implementation can set ``MPICC`` for packages that depend on it:
 
diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py
index 3476df89d9..9ef64a6f7e 100644
--- a/lib/spack/spack/build_environment.py
+++ b/lib/spack/spack/build_environment.py
@@ -53,8 +53,8 @@ import spack.main
 import spack.paths
 import spack.store
 from spack.util.string import plural
-from spack.environment import EnvironmentModifications, validate
-from spack.environment import preserve_environment
+from spack.util.environment import EnvironmentModifications, validate
+from spack.util.environment import preserve_environment
 from spack.util.environment import env_flag, filter_system_paths, get_path
 from spack.util.environment import system_dirs
 from spack.util.executable import Executable
diff --git a/lib/spack/spack/environment.py b/lib/spack/spack/environment.py
deleted file mode 100644
index 9fb1c39af0..0000000000
--- a/lib/spack/spack/environment.py
+++ /dev/null
@@ -1,623 +0,0 @@
-# Copyright 2013-2018 Lawrence Livermore National Security, LLC and other
-# Spack Project Developers. See the top-level COPYRIGHT file for details.
-#
-# SPDX-License-Identifier: (Apache-2.0 OR MIT)
-
-import collections
-import contextlib
-import inspect
-import json
-import os
-import re
-import sys
-import os.path
-import subprocess
-
-import llnl.util.tty as tty
-
-from llnl.util.lang import dedupe
-
-
-class NameModifier(object):
-
-    def __init__(self, name, **kwargs):
-        self.name = name
-        self.args = {'name': name}
-        self.args.update(kwargs)
-
-    def update_args(self, **kwargs):
-        self.__dict__.update(kwargs)
-        self.args.update(kwargs)
-
-
-class NameValueModifier(object):
-
-    def __init__(self, name, value, **kwargs):
-        self.name = name
-        self.value = value
-        self.separator = kwargs.get('separator', ':')
-        self.args = {'name': name, 'value': value, 'separator': self.separator}
-        self.args.update(kwargs)
-
-    def update_args(self, **kwargs):
-        self.__dict__.update(kwargs)
-        self.args.update(kwargs)
-
-
-class SetEnv(NameValueModifier):
-
-    def execute(self):
-        os.environ[self.name] = str(self.value)
-
-
-class AppendFlagsEnv(NameValueModifier):
-
-    def execute(self):
-        if self.name in os.environ and os.environ[self.name]:
-            os.environ[self.name] += self.separator + str(self.value)
-        else:
-            os.environ[self.name] = str(self.value)
-
-
-class UnsetEnv(NameModifier):
-
-    def execute(self):
-        # Avoid throwing if the variable was not set
-        os.environ.pop(self.name, None)
-
-
-class SetPath(NameValueModifier):
-
-    def execute(self):
-        string_path = concatenate_paths(self.value, separator=self.separator)
-        os.environ[self.name] = string_path
-
-
-class AppendPath(NameValueModifier):
-
-    def execute(self):
-        environment_value = os.environ.get(self.name, '')
-        directories = environment_value.split(
-            self.separator) if environment_value else []
-        directories.append(os.path.normpath(self.value))
-        os.environ[self.name] = self.separator.join(directories)
-
-
-class PrependPath(NameValueModifier):
-
-    def execute(self):
-        environment_value = os.environ.get(self.name, '')
-        directories = environment_value.split(
-            self.separator) if environment_value else []
-        directories = [os.path.normpath(self.value)] + directories
-        os.environ[self.name] = self.separator.join(directories)
-
-
-class RemovePath(NameValueModifier):
-
-    def execute(self):
-        environment_value = os.environ.get(self.name, '')
-        directories = environment_value.split(
-            self.separator) if environment_value else []
-        directories = [os.path.normpath(x) for x in directories
-                       if x != os.path.normpath(self.value)]
-        os.environ[self.name] = self.separator.join(directories)
-
-
-class EnvironmentModifications(object):
-    """Keeps track of requests to modify the current environment.
-
-    Each call to a method to modify the environment stores the extra
-    information on the caller in the request:
-
-        * 'filename' : filename of the module where the caller is defined
-        * 'lineno': line number where the request occurred
-        * 'context' : line of code that issued the request that failed
-    """
-
-    def __init__(self, other=None):
-        """Initializes a new instance, copying commands from 'other'
-        if it is not None.
-
-        Args:
-            other (EnvironmentModifications): list of environment modifications
-                to be extended (optional)
-        """
-        self.env_modifications = []
-        if other is not None:
-            self.extend(other)
-
-    def __iter__(self):
-        return iter(self.env_modifications)
-
-    def __len__(self):
-        return len(self.env_modifications)
-
-    def extend(self, other):
-        self._check_other(other)
-        self.env_modifications.extend(other.env_modifications)
-
-    @staticmethod
-    def _check_other(other):
-        if not isinstance(other, EnvironmentModifications):
-            raise TypeError(
-                'other must be an instance of EnvironmentModifications')
-
-    def _get_outside_caller_attributes(self):
-        stack = inspect.stack()
-        try:
-            _, filename, lineno, _, context, index = stack[2]
-            context = context[index].strip()
-        except Exception:
-            filename = 'unknown file'
-            lineno = 'unknown line'
-            context = 'unknown context'
-        args = {'filename': filename, 'lineno': lineno, 'context': context}
-        return args
-
-    def set(self, name, value, **kwargs):
-        """Stores a request to set an environment variable.
-
-        Args:
-            name: name of the environment variable to be set
-            value: value of the environment variable
-        """
-        kwargs.update(self._get_outside_caller_attributes())
-        item = SetEnv(name, value, **kwargs)
-        self.env_modifications.append(item)
-
-    def append_flags(self, name, value, sep=' ', **kwargs):
-        """
-        Stores in the current object a request to append to an env variable
-
-        Args:
-            name: name of the environment variable to be appended to
-            value: value to append to the environment variable
-        Appends with spaces separating different additions to the variable
-        """
-        kwargs.update(self._get_outside_caller_attributes())
-        kwargs.update({'separator': sep})
-        item = AppendFlagsEnv(name, value, **kwargs)
-        self.env_modifications.append(item)
-
-    def unset(self, name, **kwargs):
-        """Stores a request to unset an environment variable.
-
-        Args:
-            name: name of the environment variable to be set
-        """
-        kwargs.update(self._get_outside_caller_attributes())
-        item = UnsetEnv(name, **kwargs)
-        self.env_modifications.append(item)
-
-    def set_path(self, name, elements, **kwargs):
-        """Stores a request to set a path generated from a list.
-
-        Args:
-            name: name o the environment variable to be set.
-            elements: elements of the path to set.
-        """
-        kwargs.update(self._get_outside_caller_attributes())
-        item = SetPath(name, elements, **kwargs)
-        self.env_modifications.append(item)
-
-    def append_path(self, name, path, **kwargs):
-        """Stores a request to append a path to a path list.
-
-        Args:
-            name: name of the path list in the environment
-            path: path to be appended
-        """
-        kwargs.update(self._get_outside_caller_attributes())
-        item = AppendPath(name, path, **kwargs)
-        self.env_modifications.append(item)
-
-    def prepend_path(self, name, path, **kwargs):
-        """Same as `append_path`, but the path is pre-pended.
-
-        Args:
-            name: name of the path list in the environment
-            path: path to be pre-pended
-        """
-        kwargs.update(self._get_outside_caller_attributes())
-        item = PrependPath(name, path, **kwargs)
-        self.env_modifications.append(item)
-
-    def remove_path(self, name, path, **kwargs):
-        """Stores a request to remove a path from a path list.
-
-        Args:
-            name: name of the path list in the environment
-            path: path to be removed
-        """
-        kwargs.update(self._get_outside_caller_attributes())
-        item = RemovePath(name, path, **kwargs)
-        self.env_modifications.append(item)
-
-    def group_by_name(self):
-        """Returns a dict of the modifications grouped by variable name.
-
-        Returns:
-            dict mapping the environment variable name to the modifications to
-            be done on it
-        """
-        modifications = collections.defaultdict(list)
-        for item in self:
-            modifications[item.name].append(item)
-        return modifications
-
-    def clear(self):
-        """
-        Clears the current list of modifications
-        """
-        self.env_modifications.clear()
-
-    def apply_modifications(self):
-        """Applies the modifications and clears the list."""
-        modifications = self.group_by_name()
-        # Apply modifications one variable at a time
-        for name, actions in sorted(modifications.items()):
-            for x in actions:
-                x.execute()
-
-    @staticmethod
-    def from_sourcing_file(filename, *args, **kwargs):
-        """Returns modifications that would be made by sourcing a file.
-
-        Parameters:
-            filename (str): The file to source
-            *args (list of str): Arguments to pass on the command line
-
-        Keyword Arguments:
-            shell (str): The shell to use (default: ``bash``)
-            shell_options (str): Options passed to the shell (default: ``-c``)
-            source_command (str): The command to run (default: ``source``)
-            suppress_output (str): Redirect used to suppress output of command
-                (default: ``&> /dev/null``)
-            concatenate_on_success (str): Operator used to execute a command
-                only when the previous command succeeds (default: ``&&``)
-            blacklist ([str or re]): Ignore any modifications of these
-                variables (default: [])
-            whitelist ([str or re]): Always respect modifications of these
-                variables (default: []). Has precedence over blacklist.
-            clean (bool): In addition to removing empty entries,
-                also remove duplicate entries (default: False).
-
-        Returns:
-            EnvironmentModifications: an object that, if executed, has
-                the same effect on the environment as sourcing the file
-        """
-        # Check if the file actually exists
-        if not os.path.isfile(filename):
-            msg = 'Trying to source non-existing file: {0}'.format(filename)
-            raise RuntimeError(msg)
-
-        # Kwargs parsing and default values
-        shell                  = kwargs.get('shell', '/bin/bash')
-        shell_options          = kwargs.get('shell_options', '-c')
-        source_command         = kwargs.get('source_command', 'source')
-        suppress_output        = kwargs.get('suppress_output', '&> /dev/null')
-        concatenate_on_success = kwargs.get('concatenate_on_success', '&&')
-        blacklist              = kwargs.get('blacklist', [])
-        whitelist              = kwargs.get('whitelist', [])
-        clean                  = kwargs.get('clean', False)
-
-        source_file = [source_command, filename]
-        source_file.extend(args)
-        source_file = ' '.join(source_file)
-
-        dump_cmd = 'import os, json; print(json.dumps(dict(os.environ)))'
-        dump_environment = 'python -c "{0}"'.format(dump_cmd)
-
-        # Construct the command that will be executed
-        command = [
-            shell,
-            shell_options,
-            ' '.join([
-                source_file, suppress_output,
-                concatenate_on_success, dump_environment,
-            ]),
-        ]
-
-        # Try to source the file
-        proc = subprocess.Popen(
-            command, stdout=subprocess.PIPE, env=os.environ)
-        proc.wait()
-
-        if proc.returncode != 0:
-            msg = 'Sourcing file {0} returned a non-zero exit code'.format(
-                filename)
-            raise RuntimeError(msg)
-
-        output = ''.join([line.decode('utf-8') for line in proc.stdout])
-
-        # Construct dictionaries of the environment before and after
-        # sourcing the file, so that we can diff them.
-        env_before = dict(os.environ)
-        env_after = json.loads(output)
-
-        # If we're in python2, convert to str objects instead of unicode
-        # like json gives us.  We can't put unicode in os.environ anyway.
-        if sys.version_info[0] < 3:
-            env_after = dict((k.encode('utf-8'), v.encode('utf-8'))
-                             for k, v in env_after.items())
-
-        # Other variables unrelated to sourcing a file
-        blacklist.extend(['SHLVL', '_', 'PWD', 'OLDPWD', 'PS2'])
-
-        def set_intersection(fullset, *args):
-            # A set intersection using string literals and regexs
-            meta = '[' + re.escape('[$()*?[]^{|}') + ']'
-            subset = fullset & set(args)   # As literal
-            for name in args:
-                if re.search(meta, name):
-                    pattern = re.compile(name)
-                    for k in fullset:
-                        if re.match(pattern, k):
-                            subset.add(k)
-            return subset
-
-        for d in env_after, env_before:
-            # Retain (whitelist) has priority over prune (blacklist)
-            prune = set_intersection(set(d), *blacklist)
-            prune -= set_intersection(prune, *whitelist)
-            for k in prune:
-                d.pop(k, None)
-
-        # Fill the EnvironmentModifications instance
-        env = EnvironmentModifications()
-
-        # New variables
-        new_variables = list(set(env_after) - set(env_before))
-        # Variables that have been unset
-        unset_variables = list(set(env_before) - set(env_after))
-        # Variables that have been modified
-        common_variables = set(env_before).intersection(set(env_after))
-
-        modified_variables = [x for x in common_variables
-                              if env_before[x] != env_after[x]]
-
-        # Consistent output order - looks nicer, easier comparison...
-        new_variables.sort()
-        unset_variables.sort()
-        modified_variables.sort()
-
-        def return_separator_if_any(*args):
-            separators = ':', ';'
-            for separator in separators:
-                for arg in args:
-                    if separator in arg:
-                        return separator
-            return None
-
-        # Add variables to env.
-        # Assume that variables with 'PATH' in the name or that contain
-        # separators like ':' or ';' are more likely to be paths
-        for x in new_variables:
-            sep = return_separator_if_any(env_after[x])
-            if sep:
-                env.prepend_path(x, env_after[x], separator=sep)
-            elif 'PATH' in x:
-                env.prepend_path(x, env_after[x])
-            else:
-                # We just need to set the variable to the new value
-                env.set(x, env_after[x])
-
-        for x in unset_variables:
-            env.unset(x)
-
-        for x in modified_variables:
-            before = env_before[x]
-            after = env_after[x]
-            sep = return_separator_if_any(before, after)
-            if sep:
-                before_list = before.split(sep)
-                after_list = after.split(sep)
-
-                # Filter out empty strings
-                before_list = list(filter(None, before_list))
-                after_list = list(filter(None, after_list))
-
-                # Remove duplicate entries (worse matching, bloats env)
-                if clean:
-                    before_list = list(dedupe(before_list))
-                    after_list = list(dedupe(after_list))
-                    # The reassembled cleaned entries
-                    before = sep.join(before_list)
-                    after = sep.join(after_list)
-
-                # Paths that have been removed
-                remove_list = [
-                    ii for ii in before_list if ii not in after_list]
-                # Check that nothing has been added in the middle of
-                # before_list
-                remaining_list = [
-                    ii for ii in before_list if ii in after_list]
-                try:
-                    start = after_list.index(remaining_list[0])
-                    end = after_list.index(remaining_list[-1])
-                    search = sep.join(after_list[start:end + 1])
-                except IndexError:
-                    env.prepend_path(x, after)
-
-                if search not in before:
-                    # We just need to set the variable to the new value
-                    env.prepend_path(x, after)
-                else:
-                    try:
-                        prepend_list = after_list[:start]
-                        prepend_list.reverse()  # Preserve order after prepend
-                    except KeyError:
-                        prepend_list = []
-                    try:
-                        append_list = after_list[end + 1:]
-                    except KeyError:
-                        append_list = []
-
-                    for item in remove_list:
-                        env.remove_path(x, item)
-                    for item in append_list:
-                        env.append_path(x, item)
-                    for item in prepend_list:
-                        env.prepend_path(x, item)
-            else:
-                # We just need to set the variable to the new value
-                env.set(x, after)
-
-        return env
-
-
-def concatenate_paths(paths, separator=':'):
-    """Concatenates an iterable of paths into a string of paths separated by
-    separator, defaulting to colon.
-
-    Args:
-        paths: iterable of paths
-        separator: the separator to use, default ':'
-
-    Returns:
-        string
-    """
-    return separator.join(str(item) for item in paths)
-
-
-def set_or_unset_not_first(variable, changes, errstream):
-    """Check if we are going to set or unset something after other
-    modifications have already been requested.
-    """
-    indexes = [ii for ii, item in enumerate(changes)
-               if ii != 0 and
-               not item.args.get('force', False) and
-               type(item) in [SetEnv, UnsetEnv]]
-    if indexes:
-        good = '\t    \t{context} at {filename}:{lineno}'
-        nogood = '\t--->\t{context} at {filename}:{lineno}'
-        message = "Suspicious requests to set or unset '{var}' found"
-        errstream(message.format(var=variable))
-        for ii, item in enumerate(changes):
-            print_format = nogood if ii in indexes else good
-            errstream(print_format.format(**item.args))
-
-
-def validate(env, errstream):
-    """Validates the environment modifications to check for the presence of
-    suspicious patterns. Prompts a warning for everything that was found.
-
-    Current checks:
-    - set or unset variables after other changes on the same variable
-
-    Args:
-        env: list of environment modifications
-    """
-    modifications = env.group_by_name()
-    for variable, list_of_changes in sorted(modifications.items()):
-        set_or_unset_not_first(variable, list_of_changes, errstream)
-
-
-def filter_environment_blacklist(env, variables):
-    """Generator that filters out any change to environment variables present in
-    the input list.
-
-    Args:
-        env: list of environment modifications
-        variables: list of variable names to be filtered
-
-    Returns:
-        items in env if they are not in variables
-    """
-    for item in env:
-        if item.name not in variables:
-            yield item
-
-
-def inspect_path(root, inspections, exclude=None):
-    """Inspects ``root`` to search for the subdirectories in ``inspections``.
-    Adds every path found to a list of prepend-path commands and returns it.
-
-    Args:
-        root (str): absolute path where to search for subdirectories
-
-        inspections (dict): maps relative paths to a list of environment
-            variables that will be modified if the path exists. The
-            modifications are not performed immediately, but stored in a
-            command object that is returned to client
-
-        exclude (callable): optional callable. If present it must accept an
-            absolute path and return True if it should be excluded from the
-            inspection
-
-    Examples:
-
-    The following lines execute an inspection in ``/usr`` to search for
-    ``/usr/include`` and ``/usr/lib64``. If found we want to prepend
-    ``/usr/include`` to ``CPATH`` and ``/usr/lib64`` to ``MY_LIB64_PATH``.
-
-        .. code-block:: python
-
-            # Set up the dictionary containing the inspection
-            inspections = {
-                'include': ['CPATH'],
-                'lib64': ['MY_LIB64_PATH']
-            }
-
-            # Get back the list of command needed to modify the environment
-            env = inspect_path('/usr', inspections)
-
-            # Eventually execute the commands
-            env.apply_modifications()
-
-    Returns:
-        instance of EnvironmentModifications containing the requested
-        modifications
-    """
-    if exclude is None:
-        exclude = lambda x: False
-
-    env = EnvironmentModifications()
-    # Inspect the prefix to check for the existence of common directories
-    for relative_path, variables in inspections.items():
-        expected = os.path.join(root, relative_path)
-
-        if os.path.isdir(expected) and not exclude(expected):
-            for variable in variables:
-                env.prepend_path(variable, expected)
-
-    return env
-
-
-@contextlib.contextmanager
-def preserve_environment(*variables):
-    """Ensures that the value of the environment variables passed as
-    arguments is the same before entering to the context manager and after
-    exiting it.
-
-    Variables that are unset before entering the context manager will be
-    explicitly unset on exit.
-
-    Args:
-        variables (list of str): list of environment variables to be preserved
-    """
-    cache = {}
-    for var in variables:
-        # The environment variable to be preserved might not be there.
-        # In that case store None as a placeholder.
-        cache[var] = os.environ.get(var, None)
-
-    yield
-
-    for var in variables:
-        value = cache[var]
-        msg = '[PRESERVE_ENVIRONMENT]'
-        if value is not None:
-            # Print a debug statement if the value changed
-            if var not in os.environ:
-                msg += ' {0} was unset, will be reset to "{1}"'
-                tty.debug(msg.format(var, value))
-            elif os.environ[var] != value:
-                msg += ' {0} was set to "{1}", will be reset to "{2}"'
-                tty.debug(msg.format(var, os.environ[var], value))
-            os.environ[var] = value
-        elif var in os.environ:
-            msg += ' {0} was set to "{1}", will be unset'
-            tty.debug(msg.format(var, os.environ[var]))
-            del os.environ[var]
diff --git a/lib/spack/spack/modules/common.py b/lib/spack/spack/modules/common.py
index 6e0d062156..3acc2b0dcf 100644
--- a/lib/spack/spack/modules/common.py
+++ b/lib/spack/spack/modules/common.py
@@ -40,7 +40,7 @@ import llnl.util.tty as tty
 
 import spack.paths
 import spack.build_environment as build_environment
-import spack.environment
+import spack.util.environment
 import spack.tengine as tengine
 import spack.util.path
 import spack.util.environment
@@ -256,7 +256,7 @@ class BaseConfiguration(object):
         """List of environment modifications that should be done in the
         module.
         """
-        env_mods = spack.environment.EnvironmentModifications()
+        env_mods = spack.util.environment.EnvironmentModifications()
         actions = self.conf.get('environment', {})
 
         def process_arglist(arglist):
@@ -500,14 +500,14 @@ class BaseContext(tengine.Context):
     def environment_modifications(self):
         """List of environment modifications to be processed."""
         # Modifications guessed inspecting the spec prefix
-        env = spack.environment.inspect_path(
+        env = spack.util.environment.inspect_path(
             self.spec.prefix,
             prefix_inspections,
             exclude=spack.util.environment.is_system_path
         )
 
         # Modifications that are coded at package level
-        _ = spack.environment.EnvironmentModifications()
+        _ = spack.util.environment.EnvironmentModifications()
         # TODO : the code down below is quite similar to
         # TODO : build_environment.setup_package and needs to be factored out
         # TODO : to a single place
diff --git a/lib/spack/spack/test/environment_modifications.py b/lib/spack/spack/test/environment_modifications.py
index 0451f00b5d..a7941a30cb 100644
--- a/lib/spack/spack/test/environment_modifications.py
+++ b/lib/spack/spack/test/environment_modifications.py
@@ -6,11 +6,11 @@
 import os
 
 import pytest
-import spack.environment as environment
+import spack.util.environment as environment
 from spack.paths import spack_root
-from spack.environment import EnvironmentModifications
-from spack.environment import RemovePath, PrependPath, AppendPath
-from spack.environment import SetEnv, UnsetEnv
+from spack.util.environment import EnvironmentModifications
+from spack.util.environment import RemovePath, PrependPath, AppendPath
+from spack.util.environment import SetEnv, UnsetEnv
 from spack.util.environment import filter_system_paths, is_system_path
 
 
diff --git a/lib/spack/spack/util/environment.py b/lib/spack/spack/util/environment.py
index f8f5e23144..fc6d416e7e 100644
--- a/lib/spack/spack/util/environment.py
+++ b/lib/spack/spack/util/environment.py
@@ -3,8 +3,20 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
+"""Utilities for setting and modifying environment variables."""
+import collections
 import contextlib
+import inspect
+import json
 import os
+import re
+import sys
+import os.path
+import subprocess
+
+import llnl.util.tty as tty
+
+from llnl.util.lang import dedupe
 
 
 system_paths = ['/', '/usr', '/usr/local']
@@ -96,3 +108,608 @@ def set_env(**kwargs):
         else:
             if var in os.environ:
                 del os.environ[var]
+
+
+class NameModifier(object):
+
+    def __init__(self, name, **kwargs):
+        self.name = name
+        self.args = {'name': name}
+        self.args.update(kwargs)
+
+    def update_args(self, **kwargs):
+        self.__dict__.update(kwargs)
+        self.args.update(kwargs)
+
+
+class NameValueModifier(object):
+
+    def __init__(self, name, value, **kwargs):
+        self.name = name
+        self.value = value
+        self.separator = kwargs.get('separator', ':')
+        self.args = {'name': name, 'value': value, 'separator': self.separator}
+        self.args.update(kwargs)
+
+    def update_args(self, **kwargs):
+        self.__dict__.update(kwargs)
+        self.args.update(kwargs)
+
+
+class SetEnv(NameValueModifier):
+
+    def execute(self):
+        os.environ[self.name] = str(self.value)
+
+
+class AppendFlagsEnv(NameValueModifier):
+
+    def execute(self):
+        if self.name in os.environ and os.environ[self.name]:
+            os.environ[self.name] += self.separator + str(self.value)
+        else:
+            os.environ[self.name] = str(self.value)
+
+
+class UnsetEnv(NameModifier):
+
+    def execute(self):
+        # Avoid throwing if the variable was not set
+        os.environ.pop(self.name, None)
+
+
+class SetPath(NameValueModifier):
+
+    def execute(self):
+        string_path = concatenate_paths(self.value, separator=self.separator)
+        os.environ[self.name] = string_path
+
+
+class AppendPath(NameValueModifier):
+
+    def execute(self):
+        environment_value = os.environ.get(self.name, '')
+        directories = environment_value.split(
+            self.separator) if environment_value else []
+        directories.append(os.path.normpath(self.value))
+        os.environ[self.name] = self.separator.join(directories)
+
+
+class PrependPath(NameValueModifier):
+
+    def execute(self):
+        environment_value = os.environ.get(self.name, '')
+        directories = environment_value.split(
+            self.separator) if environment_value else []
+        directories = [os.path.normpath(self.value)] + directories
+        os.environ[self.name] = self.separator.join(directories)
+
+
+class RemovePath(NameValueModifier):
+
+    def execute(self):
+        environment_value = os.environ.get(self.name, '')
+        directories = environment_value.split(
+            self.separator) if environment_value else []
+        directories = [os.path.normpath(x) for x in directories
+                       if x != os.path.normpath(self.value)]
+        os.environ[self.name] = self.separator.join(directories)
+
+
+class EnvironmentModifications(object):
+    """Keeps track of requests to modify the current environment.
+
+    Each call to a method to modify the environment stores the extra
+    information on the caller in the request:
+
+        * 'filename' : filename of the module where the caller is defined
+        * 'lineno': line number where the request occurred
+        * 'context' : line of code that issued the request that failed
+    """
+
+    def __init__(self, other=None):
+        """Initializes a new instance, copying commands from 'other'
+        if it is not None.
+
+        Args:
+            other (EnvironmentModifications): list of environment modifications
+                to be extended (optional)
+        """
+        self.env_modifications = []
+        if other is not None:
+            self.extend(other)
+
+    def __iter__(self):
+        return iter(self.env_modifications)
+
+    def __len__(self):
+        return len(self.env_modifications)
+
+    def extend(self, other):
+        self._check_other(other)
+        self.env_modifications.extend(other.env_modifications)
+
+    @staticmethod
+    def _check_other(other):
+        if not isinstance(other, EnvironmentModifications):
+            raise TypeError(
+                'other must be an instance of EnvironmentModifications')
+
+    def _get_outside_caller_attributes(self):
+        stack = inspect.stack()
+        try:
+            _, filename, lineno, _, context, index = stack[2]
+            context = context[index].strip()
+        except Exception:
+            filename = 'unknown file'
+            lineno = 'unknown line'
+            context = 'unknown context'
+        args = {'filename': filename, 'lineno': lineno, 'context': context}
+        return args
+
+    def set(self, name, value, **kwargs):
+        """Stores a request to set an environment variable.
+
+        Args:
+            name: name of the environment variable to be set
+            value: value of the environment variable
+        """
+        kwargs.update(self._get_outside_caller_attributes())
+        item = SetEnv(name, value, **kwargs)
+        self.env_modifications.append(item)
+
+    def append_flags(self, name, value, sep=' ', **kwargs):
+        """
+        Stores in the current object a request to append to an env variable
+
+        Args:
+            name: name of the environment variable to be appended to
+            value: value to append to the environment variable
+        Appends with spaces separating different additions to the variable
+        """
+        kwargs.update(self._get_outside_caller_attributes())
+        kwargs.update({'separator': sep})
+        item = AppendFlagsEnv(name, value, **kwargs)
+        self.env_modifications.append(item)
+
+    def unset(self, name, **kwargs):
+        """Stores a request to unset an environment variable.
+
+        Args:
+            name: name of the environment variable to be set
+        """
+        kwargs.update(self._get_outside_caller_attributes())
+        item = UnsetEnv(name, **kwargs)
+        self.env_modifications.append(item)
+
+    def set_path(self, name, elements, **kwargs):
+        """Stores a request to set a path generated from a list.
+
+        Args:
+            name: name o the environment variable to be set.
+            elements: elements of the path to set.
+        """
+        kwargs.update(self._get_outside_caller_attributes())
+        item = SetPath(name, elements, **kwargs)
+        self.env_modifications.append(item)
+
+    def append_path(self, name, path, **kwargs):
+        """Stores a request to append a path to a path list.
+
+        Args:
+            name: name of the path list in the environment
+            path: path to be appended
+        """
+        kwargs.update(self._get_outside_caller_attributes())
+        item = AppendPath(name, path, **kwargs)
+        self.env_modifications.append(item)
+
+    def prepend_path(self, name, path, **kwargs):
+        """Same as `append_path`, but the path is pre-pended.
+
+        Args:
+            name: name of the path list in the environment
+            path: path to be pre-pended
+        """
+        kwargs.update(self._get_outside_caller_attributes())
+        item = PrependPath(name, path, **kwargs)
+        self.env_modifications.append(item)
+
+    def remove_path(self, name, path, **kwargs):
+        """Stores a request to remove a path from a path list.
+
+        Args:
+            name: name of the path list in the environment
+            path: path to be removed
+        """
+        kwargs.update(self._get_outside_caller_attributes())
+        item = RemovePath(name, path, **kwargs)
+        self.env_modifications.append(item)
+
+    def group_by_name(self):
+        """Returns a dict of the modifications grouped by variable name.
+
+        Returns:
+            dict mapping the environment variable name to the modifications to
+            be done on it
+        """
+        modifications = collections.defaultdict(list)
+        for item in self:
+            modifications[item.name].append(item)
+        return modifications
+
+    def clear(self):
+        """
+        Clears the current list of modifications
+        """
+        self.env_modifications.clear()
+
+    def apply_modifications(self):
+        """Applies the modifications and clears the list."""
+        modifications = self.group_by_name()
+        # Apply modifications one variable at a time
+        for name, actions in sorted(modifications.items()):
+            for x in actions:
+                x.execute()
+
+    @staticmethod
+    def from_sourcing_file(filename, *args, **kwargs):
+        """Returns modifications that would be made by sourcing a file.
+
+        Parameters:
+            filename (str): The file to source
+            *args (list of str): Arguments to pass on the command line
+
+        Keyword Arguments:
+            shell (str): The shell to use (default: ``bash``)
+            shell_options (str): Options passed to the shell (default: ``-c``)
+            source_command (str): The command to run (default: ``source``)
+            suppress_output (str): Redirect used to suppress output of command
+                (default: ``&> /dev/null``)
+            concatenate_on_success (str): Operator used to execute a command
+                only when the previous command succeeds (default: ``&&``)
+            blacklist ([str or re]): Ignore any modifications of these
+                variables (default: [])
+            whitelist ([str or re]): Always respect modifications of these
+                variables (default: []). Has precedence over blacklist.
+            clean (bool): In addition to removing empty entries,
+                also remove duplicate entries (default: False).
+
+        Returns:
+            EnvironmentModifications: an object that, if executed, has
+                the same effect on the environment as sourcing the file
+        """
+        # Check if the file actually exists
+        if not os.path.isfile(filename):
+            msg = 'Trying to source non-existing file: {0}'.format(filename)
+            raise RuntimeError(msg)
+
+        # Kwargs parsing and default values
+        shell                  = kwargs.get('shell', '/bin/bash')
+        shell_options          = kwargs.get('shell_options', '-c')
+        source_command         = kwargs.get('source_command', 'source')
+        suppress_output        = kwargs.get('suppress_output', '&> /dev/null')
+        concatenate_on_success = kwargs.get('concatenate_on_success', '&&')
+        blacklist              = kwargs.get('blacklist', [])
+        whitelist              = kwargs.get('whitelist', [])
+        clean                  = kwargs.get('clean', False)
+
+        source_file = [source_command, filename]
+        source_file.extend(args)
+        source_file = ' '.join(source_file)
+
+        dump_cmd = 'import os, json; print(json.dumps(dict(os.environ)))'
+        dump_environment = 'python -c "{0}"'.format(dump_cmd)
+
+        # Construct the command that will be executed
+        command = [
+            shell,
+            shell_options,
+            ' '.join([
+                source_file, suppress_output,
+                concatenate_on_success, dump_environment,
+            ]),
+        ]
+
+        # Try to source the file
+        proc = subprocess.Popen(
+            command, stdout=subprocess.PIPE, env=os.environ)
+        proc.wait()
+
+        if proc.returncode != 0:
+            msg = 'Sourcing file {0} returned a non-zero exit code'.format(
+                filename)
+            raise RuntimeError(msg)
+
+        output = ''.join([line.decode('utf-8') for line in proc.stdout])
+
+        # Construct dictionaries of the environment before and after
+        # sourcing the file, so that we can diff them.
+        env_before = dict(os.environ)
+        env_after = json.loads(output)
+
+        # If we're in python2, convert to str objects instead of unicode
+        # like json gives us.  We can't put unicode in os.environ anyway.
+        if sys.version_info[0] < 3:
+            env_after = dict((k.encode('utf-8'), v.encode('utf-8'))
+                             for k, v in env_after.items())
+
+        # Other variables unrelated to sourcing a file
+        blacklist.extend(['SHLVL', '_', 'PWD', 'OLDPWD', 'PS2'])
+
+        def set_intersection(fullset, *args):
+            # A set intersection using string literals and regexs
+            meta = '[' + re.escape('[$()*?[]^{|}') + ']'
+            subset = fullset & set(args)   # As literal
+            for name in args:
+                if re.search(meta, name):
+                    pattern = re.compile(name)
+                    for k in fullset:
+                        if re.match(pattern, k):
+                            subset.add(k)
+            return subset
+
+        for d in env_after, env_before:
+            # Retain (whitelist) has priority over prune (blacklist)
+            prune = set_intersection(set(d), *blacklist)
+            prune -= set_intersection(prune, *whitelist)
+            for k in prune:
+                d.pop(k, None)
+
+        # Fill the EnvironmentModifications instance
+        env = EnvironmentModifications()
+
+        # New variables
+        new_variables = list(set(env_after) - set(env_before))
+        # Variables that have been unset
+        unset_variables = list(set(env_before) - set(env_after))
+        # Variables that have been modified
+        common_variables = set(env_before).intersection(set(env_after))
+
+        modified_variables = [x for x in common_variables
+                              if env_before[x] != env_after[x]]
+
+        # Consistent output order - looks nicer, easier comparison...
+        new_variables.sort()
+        unset_variables.sort()
+        modified_variables.sort()
+
+        def return_separator_if_any(*args):
+            separators = ':', ';'
+            for separator in separators:
+                for arg in args:
+                    if separator in arg:
+                        return separator
+            return None
+
+        # Add variables to env.
+        # Assume that variables with 'PATH' in the name or that contain
+        # separators like ':' or ';' are more likely to be paths
+        for x in new_variables:
+            sep = return_separator_if_any(env_after[x])
+            if sep:
+                env.prepend_path(x, env_after[x], separator=sep)
+            elif 'PATH' in x:
+                env.prepend_path(x, env_after[x])
+            else:
+                # We just need to set the variable to the new value
+                env.set(x, env_after[x])
+
+        for x in unset_variables:
+            env.unset(x)
+
+        for x in modified_variables:
+            before = env_before[x]
+            after = env_after[x]
+            sep = return_separator_if_any(before, after)
+            if sep:
+                before_list = before.split(sep)
+                after_list = after.split(sep)
+
+                # Filter out empty strings
+                before_list = list(filter(None, before_list))
+                after_list = list(filter(None, after_list))
+
+                # Remove duplicate entries (worse matching, bloats env)
+                if clean:
+                    before_list = list(dedupe(before_list))
+                    after_list = list(dedupe(after_list))
+                    # The reassembled cleaned entries
+                    before = sep.join(before_list)
+                    after = sep.join(after_list)
+
+                # Paths that have been removed
+                remove_list = [
+                    ii for ii in before_list if ii not in after_list]
+                # Check that nothing has been added in the middle of
+                # before_list
+                remaining_list = [
+                    ii for ii in before_list if ii in after_list]
+                try:
+                    start = after_list.index(remaining_list[0])
+                    end = after_list.index(remaining_list[-1])
+                    search = sep.join(after_list[start:end + 1])
+                except IndexError:
+                    env.prepend_path(x, after)
+
+                if search not in before:
+                    # We just need to set the variable to the new value
+                    env.prepend_path(x, after)
+                else:
+                    try:
+                        prepend_list = after_list[:start]
+                        prepend_list.reverse()  # Preserve order after prepend
+                    except KeyError:
+                        prepend_list = []
+                    try:
+                        append_list = after_list[end + 1:]
+                    except KeyError:
+                        append_list = []
+
+                    for item in remove_list:
+                        env.remove_path(x, item)
+                    for item in append_list:
+                        env.append_path(x, item)
+                    for item in prepend_list:
+                        env.prepend_path(x, item)
+            else:
+                # We just need to set the variable to the new value
+                env.set(x, after)
+
+        return env
+
+
+def concatenate_paths(paths, separator=':'):
+    """Concatenates an iterable of paths into a string of paths separated by
+    separator, defaulting to colon.
+
+    Args:
+        paths: iterable of paths
+        separator: the separator to use, default ':'
+
+    Returns:
+        string
+    """
+    return separator.join(str(item) for item in paths)
+
+
+def set_or_unset_not_first(variable, changes, errstream):
+    """Check if we are going to set or unset something after other
+    modifications have already been requested.
+    """
+    indexes = [ii for ii, item in enumerate(changes)
+               if ii != 0 and
+               not item.args.get('force', False) and
+               type(item) in [SetEnv, UnsetEnv]]
+    if indexes:
+        good = '\t    \t{context} at {filename}:{lineno}'
+        nogood = '\t--->\t{context} at {filename}:{lineno}'
+        message = "Suspicious requests to set or unset '{var}' found"
+        errstream(message.format(var=variable))
+        for ii, item in enumerate(changes):
+            print_format = nogood if ii in indexes else good
+            errstream(print_format.format(**item.args))
+
+
+def validate(env, errstream):
+    """Validates the environment modifications to check for the presence of
+    suspicious patterns. Prompts a warning for everything that was found.
+
+    Current checks:
+    - set or unset variables after other changes on the same variable
+
+    Args:
+        env: list of environment modifications
+    """
+    modifications = env.group_by_name()
+    for variable, list_of_changes in sorted(modifications.items()):
+        set_or_unset_not_first(variable, list_of_changes, errstream)
+
+
+def filter_environment_blacklist(env, variables):
+    """Generator that filters out any change to environment variables present in
+    the input list.
+
+    Args:
+        env: list of environment modifications
+        variables: list of variable names to be filtered
+
+    Returns:
+        items in env if they are not in variables
+    """
+    for item in env:
+        if item.name not in variables:
+            yield item
+
+
+def inspect_path(root, inspections, exclude=None):
+    """Inspects ``root`` to search for the subdirectories in ``inspections``.
+    Adds every path found to a list of prepend-path commands and returns it.
+
+    Args:
+        root (str): absolute path where to search for subdirectories
+
+        inspections (dict): maps relative paths to a list of environment
+            variables that will be modified if the path exists. The
+            modifications are not performed immediately, but stored in a
+            command object that is returned to client
+
+        exclude (callable): optional callable. If present it must accept an
+            absolute path and return True if it should be excluded from the
+            inspection
+
+    Examples:
+
+    The following lines execute an inspection in ``/usr`` to search for
+    ``/usr/include`` and ``/usr/lib64``. If found we want to prepend
+    ``/usr/include`` to ``CPATH`` and ``/usr/lib64`` to ``MY_LIB64_PATH``.
+
+        .. code-block:: python
+
+            # Set up the dictionary containing the inspection
+            inspections = {
+                'include': ['CPATH'],
+                'lib64': ['MY_LIB64_PATH']
+            }
+
+            # Get back the list of command needed to modify the environment
+            env = inspect_path('/usr', inspections)
+
+            # Eventually execute the commands
+            env.apply_modifications()
+
+    Returns:
+        instance of EnvironmentModifications containing the requested
+        modifications
+    """
+    if exclude is None:
+        exclude = lambda x: False
+
+    env = EnvironmentModifications()
+    # Inspect the prefix to check for the existence of common directories
+    for relative_path, variables in inspections.items():
+        expected = os.path.join(root, relative_path)
+
+        if os.path.isdir(expected) and not exclude(expected):
+            for variable in variables:
+                env.prepend_path(variable, expected)
+
+    return env
+
+
+@contextlib.contextmanager
+def preserve_environment(*variables):
+    """Ensures that the value of the environment variables passed as
+    arguments is the same before entering to the context manager and after
+    exiting it.
+
+    Variables that are unset before entering the context manager will be
+    explicitly unset on exit.
+
+    Args:
+        variables (list of str): list of environment variables to be preserved
+    """
+    cache = {}
+    for var in variables:
+        # The environment variable to be preserved might not be there.
+        # In that case store None as a placeholder.
+        cache[var] = os.environ.get(var, None)
+
+    yield
+
+    for var in variables:
+        value = cache[var]
+        msg = '[PRESERVE_ENVIRONMENT]'
+        if value is not None:
+            # Print a debug statement if the value changed
+            if var not in os.environ:
+                msg += ' {0} was unset, will be reset to "{1}"'
+                tty.debug(msg.format(var, value))
+            elif os.environ[var] != value:
+                msg += ' {0} was set to "{1}", will be reset to "{2}"'
+                tty.debug(msg.format(var, os.environ[var], value))
+            os.environ[var] = value
+        elif var in os.environ:
+            msg += ' {0} was set to "{1}", will be unset'
+            tty.debug(msg.format(var, os.environ[var]))
+            del os.environ[var]