summaryrefslogtreecommitdiff
path: root/lib
diff options
context:
space:
mode:
Diffstat (limited to 'lib')
-rw-r--r--lib/spack/docs/tutorial_advanced_packaging.rst2
-rw-r--r--lib/spack/spack/build_environment.py4
-rw-r--r--lib/spack/spack/environment.py623
-rw-r--r--lib/spack/spack/modules/common.py8
-rw-r--r--lib/spack/spack/test/environment_modifications.py8
-rw-r--r--lib/spack/spack/util/environment.py617
6 files changed, 628 insertions, 634 deletions
diff --git a/lib/spack/docs/tutorial_advanced_packaging.rst b/lib/spack/docs/tutorial_advanced_packaging.rst
index 9b41980a69..c7143db209 100644
--- a/lib/spack/docs/tutorial_advanced_packaging.rst
+++ b/lib/spack/docs/tutorial_advanced_packaging.rst
@@ -278,7 +278,7 @@ that spack does is not sufficient for python to import modules.
To provide environment setup for a dependent, a package can implement the
:py:func:`setup_dependent_environment <spack.package.PackageBase.setup_dependent_environment>`
-function. This function takes as a parameter a :py:class:`EnvironmentModifications <spack.environment.EnvironmentModifications>`
+function. This function takes as a parameter a :py:class:`EnvironmentModifications <spack.util.environment.EnvironmentModifications>`
object which includes convenience methods to update the environment. For
example, an MPI implementation can set ``MPICC`` for packages that depend on it:
diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py
index 3476df89d9..9ef64a6f7e 100644
--- a/lib/spack/spack/build_environment.py
+++ b/lib/spack/spack/build_environment.py
@@ -53,8 +53,8 @@ import spack.main
import spack.paths
import spack.store
from spack.util.string import plural
-from spack.environment import EnvironmentModifications, validate
-from spack.environment import preserve_environment
+from spack.util.environment import EnvironmentModifications, validate
+from spack.util.environment import preserve_environment
from spack.util.environment import env_flag, filter_system_paths, get_path
from spack.util.environment import system_dirs
from spack.util.executable import Executable
diff --git a/lib/spack/spack/environment.py b/lib/spack/spack/environment.py
deleted file mode 100644
index 9fb1c39af0..0000000000
--- a/lib/spack/spack/environment.py
+++ /dev/null
@@ -1,623 +0,0 @@
-# Copyright 2013-2018 Lawrence Livermore National Security, LLC and other
-# Spack Project Developers. See the top-level COPYRIGHT file for details.
-#
-# SPDX-License-Identifier: (Apache-2.0 OR MIT)
-
-import collections
-import contextlib
-import inspect
-import json
-import os
-import re
-import sys
-import os.path
-import subprocess
-
-import llnl.util.tty as tty
-
-from llnl.util.lang import dedupe
-
-
-class NameModifier(object):
-
- def __init__(self, name, **kwargs):
- self.name = name
- self.args = {'name': name}
- self.args.update(kwargs)
-
- def update_args(self, **kwargs):
- self.__dict__.update(kwargs)
- self.args.update(kwargs)
-
-
-class NameValueModifier(object):
-
- def __init__(self, name, value, **kwargs):
- self.name = name
- self.value = value
- self.separator = kwargs.get('separator', ':')
- self.args = {'name': name, 'value': value, 'separator': self.separator}
- self.args.update(kwargs)
-
- def update_args(self, **kwargs):
- self.__dict__.update(kwargs)
- self.args.update(kwargs)
-
-
-class SetEnv(NameValueModifier):
-
- def execute(self):
- os.environ[self.name] = str(self.value)
-
-
-class AppendFlagsEnv(NameValueModifier):
-
- def execute(self):
- if self.name in os.environ and os.environ[self.name]:
- os.environ[self.name] += self.separator + str(self.value)
- else:
- os.environ[self.name] = str(self.value)
-
-
-class UnsetEnv(NameModifier):
-
- def execute(self):
- # Avoid throwing if the variable was not set
- os.environ.pop(self.name, None)
-
-
-class SetPath(NameValueModifier):
-
- def execute(self):
- string_path = concatenate_paths(self.value, separator=self.separator)
- os.environ[self.name] = string_path
-
-
-class AppendPath(NameValueModifier):
-
- def execute(self):
- environment_value = os.environ.get(self.name, '')
- directories = environment_value.split(
- self.separator) if environment_value else []
- directories.append(os.path.normpath(self.value))
- os.environ[self.name] = self.separator.join(directories)
-
-
-class PrependPath(NameValueModifier):
-
- def execute(self):
- environment_value = os.environ.get(self.name, '')
- directories = environment_value.split(
- self.separator) if environment_value else []
- directories = [os.path.normpath(self.value)] + directories
- os.environ[self.name] = self.separator.join(directories)
-
-
-class RemovePath(NameValueModifier):
-
- def execute(self):
- environment_value = os.environ.get(self.name, '')
- directories = environment_value.split(
- self.separator) if environment_value else []
- directories = [os.path.normpath(x) for x in directories
- if x != os.path.normpath(self.value)]
- os.environ[self.name] = self.separator.join(directories)
-
-
-class EnvironmentModifications(object):
- """Keeps track of requests to modify the current environment.
-
- Each call to a method to modify the environment stores the extra
- information on the caller in the request:
-
- * 'filename' : filename of the module where the caller is defined
- * 'lineno': line number where the request occurred
- * 'context' : line of code that issued the request that failed
- """
-
- def __init__(self, other=None):
- """Initializes a new instance, copying commands from 'other'
- if it is not None.
-
- Args:
- other (EnvironmentModifications): list of environment modifications
- to be extended (optional)
- """
- self.env_modifications = []
- if other is not None:
- self.extend(other)
-
- def __iter__(self):
- return iter(self.env_modifications)
-
- def __len__(self):
- return len(self.env_modifications)
-
- def extend(self, other):
- self._check_other(other)
- self.env_modifications.extend(other.env_modifications)
-
- @staticmethod
- def _check_other(other):
- if not isinstance(other, EnvironmentModifications):
- raise TypeError(
- 'other must be an instance of EnvironmentModifications')
-
- def _get_outside_caller_attributes(self):
- stack = inspect.stack()
- try:
- _, filename, lineno, _, context, index = stack[2]
- context = context[index].strip()
- except Exception:
- filename = 'unknown file'
- lineno = 'unknown line'
- context = 'unknown context'
- args = {'filename': filename, 'lineno': lineno, 'context': context}
- return args
-
- def set(self, name, value, **kwargs):
- """Stores a request to set an environment variable.
-
- Args:
- name: name of the environment variable to be set
- value: value of the environment variable
- """
- kwargs.update(self._get_outside_caller_attributes())
- item = SetEnv(name, value, **kwargs)
- self.env_modifications.append(item)
-
- def append_flags(self, name, value, sep=' ', **kwargs):
- """
- Stores in the current object a request to append to an env variable
-
- Args:
- name: name of the environment variable to be appended to
- value: value to append to the environment variable
- Appends with spaces separating different additions to the variable
- """
- kwargs.update(self._get_outside_caller_attributes())
- kwargs.update({'separator': sep})
- item = AppendFlagsEnv(name, value, **kwargs)
- self.env_modifications.append(item)
-
- def unset(self, name, **kwargs):
- """Stores a request to unset an environment variable.
-
- Args:
- name: name of the environment variable to be set
- """
- kwargs.update(self._get_outside_caller_attributes())
- item = UnsetEnv(name, **kwargs)
- self.env_modifications.append(item)
-
- def set_path(self, name, elements, **kwargs):
- """Stores a request to set a path generated from a list.
-
- Args:
- name: name o the environment variable to be set.
- elements: elements of the path to set.
- """
- kwargs.update(self._get_outside_caller_attributes())
- item = SetPath(name, elements, **kwargs)
- self.env_modifications.append(item)
-
- def append_path(self, name, path, **kwargs):
- """Stores a request to append a path to a path list.
-
- Args:
- name: name of the path list in the environment
- path: path to be appended
- """
- kwargs.update(self._get_outside_caller_attributes())
- item = AppendPath(name, path, **kwargs)
- self.env_modifications.append(item)
-
- def prepend_path(self, name, path, **kwargs):
- """Same as `append_path`, but the path is pre-pended.
-
- Args:
- name: name of the path list in the environment
- path: path to be pre-pended
- """
- kwargs.update(self._get_outside_caller_attributes())
- item = PrependPath(name, path, **kwargs)
- self.env_modifications.append(item)
-
- def remove_path(self, name, path, **kwargs):
- """Stores a request to remove a path from a path list.
-
- Args:
- name: name of the path list in the environment
- path: path to be removed
- """
- kwargs.update(self._get_outside_caller_attributes())
- item = RemovePath(name, path, **kwargs)
- self.env_modifications.append(item)
-
- def group_by_name(self):
- """Returns a dict of the modifications grouped by variable name.
-
- Returns:
- dict mapping the environment variable name to the modifications to
- be done on it
- """
- modifications = collections.defaultdict(list)
- for item in self:
- modifications[item.name].append(item)
- return modifications
-
- def clear(self):
- """
- Clears the current list of modifications
- """
- self.env_modifications.clear()
-
- def apply_modifications(self):
- """Applies the modifications and clears the list."""
- modifications = self.group_by_name()
- # Apply modifications one variable at a time
- for name, actions in sorted(modifications.items()):
- for x in actions:
- x.execute()
-
- @staticmethod
- def from_sourcing_file(filename, *args, **kwargs):
- """Returns modifications that would be made by sourcing a file.
-
- Parameters:
- filename (str): The file to source
- *args (list of str): Arguments to pass on the command line
-
- Keyword Arguments:
- shell (str): The shell to use (default: ``bash``)
- shell_options (str): Options passed to the shell (default: ``-c``)
- source_command (str): The command to run (default: ``source``)
- suppress_output (str): Redirect used to suppress output of command
- (default: ``&> /dev/null``)
- concatenate_on_success (str): Operator used to execute a command
- only when the previous command succeeds (default: ``&&``)
- blacklist ([str or re]): Ignore any modifications of these
- variables (default: [])
- whitelist ([str or re]): Always respect modifications of these
- variables (default: []). Has precedence over blacklist.
- clean (bool): In addition to removing empty entries,
- also remove duplicate entries (default: False).
-
- Returns:
- EnvironmentModifications: an object that, if executed, has
- the same effect on the environment as sourcing the file
- """
- # Check if the file actually exists
- if not os.path.isfile(filename):
- msg = 'Trying to source non-existing file: {0}'.format(filename)
- raise RuntimeError(msg)
-
- # Kwargs parsing and default values
- shell = kwargs.get('shell', '/bin/bash')
- shell_options = kwargs.get('shell_options', '-c')
- source_command = kwargs.get('source_command', 'source')
- suppress_output = kwargs.get('suppress_output', '&> /dev/null')
- concatenate_on_success = kwargs.get('concatenate_on_success', '&&')
- blacklist = kwargs.get('blacklist', [])
- whitelist = kwargs.get('whitelist', [])
- clean = kwargs.get('clean', False)
-
- source_file = [source_command, filename]
- source_file.extend(args)
- source_file = ' '.join(source_file)
-
- dump_cmd = 'import os, json; print(json.dumps(dict(os.environ)))'
- dump_environment = 'python -c "{0}"'.format(dump_cmd)
-
- # Construct the command that will be executed
- command = [
- shell,
- shell_options,
- ' '.join([
- source_file, suppress_output,
- concatenate_on_success, dump_environment,
- ]),
- ]
-
- # Try to source the file
- proc = subprocess.Popen(
- command, stdout=subprocess.PIPE, env=os.environ)
- proc.wait()
-
- if proc.returncode != 0:
- msg = 'Sourcing file {0} returned a non-zero exit code'.format(
- filename)
- raise RuntimeError(msg)
-
- output = ''.join([line.decode('utf-8') for line in proc.stdout])
-
- # Construct dictionaries of the environment before and after
- # sourcing the file, so that we can diff them.
- env_before = dict(os.environ)
- env_after = json.loads(output)
-
- # If we're in python2, convert to str objects instead of unicode
- # like json gives us. We can't put unicode in os.environ anyway.
- if sys.version_info[0] < 3:
- env_after = dict((k.encode('utf-8'), v.encode('utf-8'))
- for k, v in env_after.items())
-
- # Other variables unrelated to sourcing a file
- blacklist.extend(['SHLVL', '_', 'PWD', 'OLDPWD', 'PS2'])
-
- def set_intersection(fullset, *args):
- # A set intersection using string literals and regexs
- meta = '[' + re.escape('[$()*?[]^{|}') + ']'
- subset = fullset & set(args) # As literal
- for name in args:
- if re.search(meta, name):
- pattern = re.compile(name)
- for k in fullset:
- if re.match(pattern, k):
- subset.add(k)
- return subset
-
- for d in env_after, env_before:
- # Retain (whitelist) has priority over prune (blacklist)
- prune = set_intersection(set(d), *blacklist)
- prune -= set_intersection(prune, *whitelist)
- for k in prune:
- d.pop(k, None)
-
- # Fill the EnvironmentModifications instance
- env = EnvironmentModifications()
-
- # New variables
- new_variables = list(set(env_after) - set(env_before))
- # Variables that have been unset
- unset_variables = list(set(env_before) - set(env_after))
- # Variables that have been modified
- common_variables = set(env_before).intersection(set(env_after))
-
- modified_variables = [x for x in common_variables
- if env_before[x] != env_after[x]]
-
- # Consistent output order - looks nicer, easier comparison...
- new_variables.sort()
- unset_variables.sort()
- modified_variables.sort()
-
- def return_separator_if_any(*args):
- separators = ':', ';'
- for separator in separators:
- for arg in args:
- if separator in arg:
- return separator
- return None
-
- # Add variables to env.
- # Assume that variables with 'PATH' in the name or that contain
- # separators like ':' or ';' are more likely to be paths
- for x in new_variables:
- sep = return_separator_if_any(env_after[x])
- if sep:
- env.prepend_path(x, env_after[x], separator=sep)
- elif 'PATH' in x:
- env.prepend_path(x, env_after[x])
- else:
- # We just need to set the variable to the new value
- env.set(x, env_after[x])
-
- for x in unset_variables:
- env.unset(x)
-
- for x in modified_variables:
- before = env_before[x]
- after = env_after[x]
- sep = return_separator_if_any(before, after)
- if sep:
- before_list = before.split(sep)
- after_list = after.split(sep)
-
- # Filter out empty strings
- before_list = list(filter(None, before_list))
- after_list = list(filter(None, after_list))
-
- # Remove duplicate entries (worse matching, bloats env)
- if clean:
- before_list = list(dedupe(before_list))
- after_list = list(dedupe(after_list))
- # The reassembled cleaned entries
- before = sep.join(before_list)
- after = sep.join(after_list)
-
- # Paths that have been removed
- remove_list = [
- ii for ii in before_list if ii not in after_list]
- # Check that nothing has been added in the middle of
- # before_list
- remaining_list = [
- ii for ii in before_list if ii in after_list]
- try:
- start = after_list.index(remaining_list[0])
- end = after_list.index(remaining_list[-1])
- search = sep.join(after_list[start:end + 1])
- except IndexError:
- env.prepend_path(x, after)
-
- if search not in before:
- # We just need to set the variable to the new value
- env.prepend_path(x, after)
- else:
- try:
- prepend_list = after_list[:start]
- prepend_list.reverse() # Preserve order after prepend
- except KeyError:
- prepend_list = []
- try:
- append_list = after_list[end + 1:]
- except KeyError:
- append_list = []
-
- for item in remove_list:
- env.remove_path(x, item)
- for item in append_list:
- env.append_path(x, item)
- for item in prepend_list:
- env.prepend_path(x, item)
- else:
- # We just need to set the variable to the new value
- env.set(x, after)
-
- return env
-
-
-def concatenate_paths(paths, separator=':'):
- """Concatenates an iterable of paths into a string of paths separated by
- separator, defaulting to colon.
-
- Args:
- paths: iterable of paths
- separator: the separator to use, default ':'
-
- Returns:
- string
- """
- return separator.join(str(item) for item in paths)
-
-
-def set_or_unset_not_first(variable, changes, errstream):
- """Check if we are going to set or unset something after other
- modifications have already been requested.
- """
- indexes = [ii for ii, item in enumerate(changes)
- if ii != 0 and
- not item.args.get('force', False) and
- type(item) in [SetEnv, UnsetEnv]]
- if indexes:
- good = '\t \t{context} at {filename}:{lineno}'
- nogood = '\t--->\t{context} at {filename}:{lineno}'
- message = "Suspicious requests to set or unset '{var}' found"
- errstream(message.format(var=variable))
- for ii, item in enumerate(changes):
- print_format = nogood if ii in indexes else good
- errstream(print_format.format(**item.args))
-
-
-def validate(env, errstream):
- """Validates the environment modifications to check for the presence of
- suspicious patterns. Prompts a warning for everything that was found.
-
- Current checks:
- - set or unset variables after other changes on the same variable
-
- Args:
- env: list of environment modifications
- """
- modifications = env.group_by_name()
- for variable, list_of_changes in sorted(modifications.items()):
- set_or_unset_not_first(variable, list_of_changes, errstream)
-
-
-def filter_environment_blacklist(env, variables):
- """Generator that filters out any change to environment variables present in
- the input list.
-
- Args:
- env: list of environment modifications
- variables: list of variable names to be filtered
-
- Returns:
- items in env if they are not in variables
- """
- for item in env:
- if item.name not in variables:
- yield item
-
-
-def inspect_path(root, inspections, exclude=None):
- """Inspects ``root`` to search for the subdirectories in ``inspections``.
- Adds every path found to a list of prepend-path commands and returns it.
-
- Args:
- root (str): absolute path where to search for subdirectories
-
- inspections (dict): maps relative paths to a list of environment
- variables that will be modified if the path exists. The
- modifications are not performed immediately, but stored in a
- command object that is returned to client
-
- exclude (callable): optional callable. If present it must accept an
- absolute path and return True if it should be excluded from the
- inspection
-
- Examples:
-
- The following lines execute an inspection in ``/usr`` to search for
- ``/usr/include`` and ``/usr/lib64``. If found we want to prepend
- ``/usr/include`` to ``CPATH`` and ``/usr/lib64`` to ``MY_LIB64_PATH``.
-
- .. code-block:: python
-
- # Set up the dictionary containing the inspection
- inspections = {
- 'include': ['CPATH'],
- 'lib64': ['MY_LIB64_PATH']
- }
-
- # Get back the list of command needed to modify the environment
- env = inspect_path('/usr', inspections)
-
- # Eventually execute the commands
- env.apply_modifications()
-
- Returns:
- instance of EnvironmentModifications containing the requested
- modifications
- """
- if exclude is None:
- exclude = lambda x: False
-
- env = EnvironmentModifications()
- # Inspect the prefix to check for the existence of common directories
- for relative_path, variables in inspections.items():
- expected = os.path.join(root, relative_path)
-
- if os.path.isdir(expected) and not exclude(expected):
- for variable in variables:
- env.prepend_path(variable, expected)
-
- return env
-
-
-@contextlib.contextmanager
-def preserve_environment(*variables):
- """Ensures that the value of the environment variables passed as
- arguments is the same before entering to the context manager and after
- exiting it.
-
- Variables that are unset before entering the context manager will be
- explicitly unset on exit.
-
- Args:
- variables (list of str): list of environment variables to be preserved
- """
- cache = {}
- for var in variables:
- # The environment variable to be preserved might not be there.
- # In that case store None as a placeholder.
- cache[var] = os.environ.get(var, None)
-
- yield
-
- for var in variables:
- value = cache[var]
- msg = '[PRESERVE_ENVIRONMENT]'
- if value is not None:
- # Print a debug statement if the value changed
- if var not in os.environ:
- msg += ' {0} was unset, will be reset to "{1}"'
- tty.debug(msg.format(var, value))
- elif os.environ[var] != value:
- msg += ' {0} was set to "{1}", will be reset to "{2}"'
- tty.debug(msg.format(var, os.environ[var], value))
- os.environ[var] = value
- elif var in os.environ:
- msg += ' {0} was set to "{1}", will be unset'
- tty.debug(msg.format(var, os.environ[var]))
- del os.environ[var]
diff --git a/lib/spack/spack/modules/common.py b/lib/spack/spack/modules/common.py
index 6e0d062156..3acc2b0dcf 100644
--- a/lib/spack/spack/modules/common.py
+++ b/lib/spack/spack/modules/common.py
@@ -40,7 +40,7 @@ import llnl.util.tty as tty
import spack.paths
import spack.build_environment as build_environment
-import spack.environment
+import spack.util.environment
import spack.tengine as tengine
import spack.util.path
import spack.util.environment
@@ -256,7 +256,7 @@ class BaseConfiguration(object):
"""List of environment modifications that should be done in the
module.
"""
- env_mods = spack.environment.EnvironmentModifications()
+ env_mods = spack.util.environment.EnvironmentModifications()
actions = self.conf.get('environment', {})
def process_arglist(arglist):
@@ -500,14 +500,14 @@ class BaseContext(tengine.Context):
def environment_modifications(self):
"""List of environment modifications to be processed."""
# Modifications guessed inspecting the spec prefix
- env = spack.environment.inspect_path(
+ env = spack.util.environment.inspect_path(
self.spec.prefix,
prefix_inspections,
exclude=spack.util.environment.is_system_path
)
# Modifications that are coded at package level
- _ = spack.environment.EnvironmentModifications()
+ _ = spack.util.environment.EnvironmentModifications()
# TODO : the code down below is quite similar to
# TODO : build_environment.setup_package and needs to be factored out
# TODO : to a single place
diff --git a/lib/spack/spack/test/environment_modifications.py b/lib/spack/spack/test/environment_modifications.py
index 0451f00b5d..a7941a30cb 100644
--- a/lib/spack/spack/test/environment_modifications.py
+++ b/lib/spack/spack/test/environment_modifications.py
@@ -6,11 +6,11 @@
import os
import pytest
-import spack.environment as environment
+import spack.util.environment as environment
from spack.paths import spack_root
-from spack.environment import EnvironmentModifications
-from spack.environment import RemovePath, PrependPath, AppendPath
-from spack.environment import SetEnv, UnsetEnv
+from spack.util.environment import EnvironmentModifications
+from spack.util.environment import RemovePath, PrependPath, AppendPath
+from spack.util.environment import SetEnv, UnsetEnv
from spack.util.environment import filter_system_paths, is_system_path
diff --git a/lib/spack/spack/util/environment.py b/lib/spack/spack/util/environment.py
index f8f5e23144..fc6d416e7e 100644
--- a/lib/spack/spack/util/environment.py
+++ b/lib/spack/spack/util/environment.py
@@ -3,8 +3,20 @@
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+"""Utilities for setting and modifying environment variables."""
+import collections
import contextlib
+import inspect
+import json
import os
+import re
+import sys
+import os.path
+import subprocess
+
+import llnl.util.tty as tty
+
+from llnl.util.lang import dedupe
system_paths = ['/', '/usr', '/usr/local']
@@ -96,3 +108,608 @@ def set_env(**kwargs):
else:
if var in os.environ:
del os.environ[var]
+
+
+class NameModifier(object):
+
+ def __init__(self, name, **kwargs):
+ self.name = name
+ self.args = {'name': name}
+ self.args.update(kwargs)
+
+ def update_args(self, **kwargs):
+ self.__dict__.update(kwargs)
+ self.args.update(kwargs)
+
+
+class NameValueModifier(object):
+
+ def __init__(self, name, value, **kwargs):
+ self.name = name
+ self.value = value
+ self.separator = kwargs.get('separator', ':')
+ self.args = {'name': name, 'value': value, 'separator': self.separator}
+ self.args.update(kwargs)
+
+ def update_args(self, **kwargs):
+ self.__dict__.update(kwargs)
+ self.args.update(kwargs)
+
+
+class SetEnv(NameValueModifier):
+
+ def execute(self):
+ os.environ[self.name] = str(self.value)
+
+
+class AppendFlagsEnv(NameValueModifier):
+
+ def execute(self):
+ if self.name in os.environ and os.environ[self.name]:
+ os.environ[self.name] += self.separator + str(self.value)
+ else:
+ os.environ[self.name] = str(self.value)
+
+
+class UnsetEnv(NameModifier):
+
+ def execute(self):
+ # Avoid throwing if the variable was not set
+ os.environ.pop(self.name, None)
+
+
+class SetPath(NameValueModifier):
+
+ def execute(self):
+ string_path = concatenate_paths(self.value, separator=self.separator)
+ os.environ[self.name] = string_path
+
+
+class AppendPath(NameValueModifier):
+
+ def execute(self):
+ environment_value = os.environ.get(self.name, '')
+ directories = environment_value.split(
+ self.separator) if environment_value else []
+ directories.append(os.path.normpath(self.value))
+ os.environ[self.name] = self.separator.join(directories)
+
+
+class PrependPath(NameValueModifier):
+
+ def execute(self):
+ environment_value = os.environ.get(self.name, '')
+ directories = environment_value.split(
+ self.separator) if environment_value else []
+ directories = [os.path.normpath(self.value)] + directories
+ os.environ[self.name] = self.separator.join(directories)
+
+
+class RemovePath(NameValueModifier):
+
+ def execute(self):
+ environment_value = os.environ.get(self.name, '')
+ directories = environment_value.split(
+ self.separator) if environment_value else []
+ directories = [os.path.normpath(x) for x in directories
+ if x != os.path.normpath(self.value)]
+ os.environ[self.name] = self.separator.join(directories)
+
+
+class EnvironmentModifications(object):
+ """Keeps track of requests to modify the current environment.
+
+ Each call to a method to modify the environment stores the extra
+ information on the caller in the request:
+
+ * 'filename' : filename of the module where the caller is defined
+ * 'lineno': line number where the request occurred
+ * 'context' : line of code that issued the request that failed
+ """
+
+ def __init__(self, other=None):
+ """Initializes a new instance, copying commands from 'other'
+ if it is not None.
+
+ Args:
+ other (EnvironmentModifications): list of environment modifications
+ to be extended (optional)
+ """
+ self.env_modifications = []
+ if other is not None:
+ self.extend(other)
+
+ def __iter__(self):
+ return iter(self.env_modifications)
+
+ def __len__(self):
+ return len(self.env_modifications)
+
+ def extend(self, other):
+ self._check_other(other)
+ self.env_modifications.extend(other.env_modifications)
+
+ @staticmethod
+ def _check_other(other):
+ if not isinstance(other, EnvironmentModifications):
+ raise TypeError(
+ 'other must be an instance of EnvironmentModifications')
+
+ def _get_outside_caller_attributes(self):
+ stack = inspect.stack()
+ try:
+ _, filename, lineno, _, context, index = stack[2]
+ context = context[index].strip()
+ except Exception:
+ filename = 'unknown file'
+ lineno = 'unknown line'
+ context = 'unknown context'
+ args = {'filename': filename, 'lineno': lineno, 'context': context}
+ return args
+
+ def set(self, name, value, **kwargs):
+ """Stores a request to set an environment variable.
+
+ Args:
+ name: name of the environment variable to be set
+ value: value of the environment variable
+ """
+ kwargs.update(self._get_outside_caller_attributes())
+ item = SetEnv(name, value, **kwargs)
+ self.env_modifications.append(item)
+
+ def append_flags(self, name, value, sep=' ', **kwargs):
+ """
+ Stores in the current object a request to append to an env variable
+
+ Args:
+ name: name of the environment variable to be appended to
+ value: value to append to the environment variable
+ Appends with spaces separating different additions to the variable
+ """
+ kwargs.update(self._get_outside_caller_attributes())
+ kwargs.update({'separator': sep})
+ item = AppendFlagsEnv(name, value, **kwargs)
+ self.env_modifications.append(item)
+
+ def unset(self, name, **kwargs):
+ """Stores a request to unset an environment variable.
+
+ Args:
+ name: name of the environment variable to be set
+ """
+ kwargs.update(self._get_outside_caller_attributes())
+ item = UnsetEnv(name, **kwargs)
+ self.env_modifications.append(item)
+
+ def set_path(self, name, elements, **kwargs):
+ """Stores a request to set a path generated from a list.
+
+ Args:
+ name: name o the environment variable to be set.
+ elements: elements of the path to set.
+ """
+ kwargs.update(self._get_outside_caller_attributes())
+ item = SetPath(name, elements, **kwargs)
+ self.env_modifications.append(item)
+
+ def append_path(self, name, path, **kwargs):
+ """Stores a request to append a path to a path list.
+
+ Args:
+ name: name of the path list in the environment
+ path: path to be appended
+ """
+ kwargs.update(self._get_outside_caller_attributes())
+ item = AppendPath(name, path, **kwargs)
+ self.env_modifications.append(item)
+
+ def prepend_path(self, name, path, **kwargs):
+ """Same as `append_path`, but the path is pre-pended.
+
+ Args:
+ name: name of the path list in the environment
+ path: path to be pre-pended
+ """
+ kwargs.update(self._get_outside_caller_attributes())
+ item = PrependPath(name, path, **kwargs)
+ self.env_modifications.append(item)
+
+ def remove_path(self, name, path, **kwargs):
+ """Stores a request to remove a path from a path list.
+
+ Args:
+ name: name of the path list in the environment
+ path: path to be removed
+ """
+ kwargs.update(self._get_outside_caller_attributes())
+ item = RemovePath(name, path, **kwargs)
+ self.env_modifications.append(item)
+
+ def group_by_name(self):
+ """Returns a dict of the modifications grouped by variable name.
+
+ Returns:
+ dict mapping the environment variable name to the modifications to
+ be done on it
+ """
+ modifications = collections.defaultdict(list)
+ for item in self:
+ modifications[item.name].append(item)
+ return modifications
+
+ def clear(self):
+ """
+ Clears the current list of modifications
+ """
+ self.env_modifications.clear()
+
+ def apply_modifications(self):
+ """Applies the modifications and clears the list."""
+ modifications = self.group_by_name()
+ # Apply modifications one variable at a time
+ for name, actions in sorted(modifications.items()):
+ for x in actions:
+ x.execute()
+
+ @staticmethod
+ def from_sourcing_file(filename, *args, **kwargs):
+ """Returns modifications that would be made by sourcing a file.
+
+ Parameters:
+ filename (str): The file to source
+ *args (list of str): Arguments to pass on the command line
+
+ Keyword Arguments:
+ shell (str): The shell to use (default: ``bash``)
+ shell_options (str): Options passed to the shell (default: ``-c``)
+ source_command (str): The command to run (default: ``source``)
+ suppress_output (str): Redirect used to suppress output of command
+ (default: ``&> /dev/null``)
+ concatenate_on_success (str): Operator used to execute a command
+ only when the previous command succeeds (default: ``&&``)
+ blacklist ([str or re]): Ignore any modifications of these
+ variables (default: [])
+ whitelist ([str or re]): Always respect modifications of these
+ variables (default: []). Has precedence over blacklist.
+ clean (bool): In addition to removing empty entries,
+ also remove duplicate entries (default: False).
+
+ Returns:
+ EnvironmentModifications: an object that, if executed, has
+ the same effect on the environment as sourcing the file
+ """
+ # Check if the file actually exists
+ if not os.path.isfile(filename):
+ msg = 'Trying to source non-existing file: {0}'.format(filename)
+ raise RuntimeError(msg)
+
+ # Kwargs parsing and default values
+ shell = kwargs.get('shell', '/bin/bash')
+ shell_options = kwargs.get('shell_options', '-c')
+ source_command = kwargs.get('source_command', 'source')
+ suppress_output = kwargs.get('suppress_output', '&> /dev/null')
+ concatenate_on_success = kwargs.get('concatenate_on_success', '&&')
+ blacklist = kwargs.get('blacklist', [])
+ whitelist = kwargs.get('whitelist', [])
+ clean = kwargs.get('clean', False)
+
+ source_file = [source_command, filename]
+ source_file.extend(args)
+ source_file = ' '.join(source_file)
+
+ dump_cmd = 'import os, json; print(json.dumps(dict(os.environ)))'
+ dump_environment = 'python -c "{0}"'.format(dump_cmd)
+
+ # Construct the command that will be executed
+ command = [
+ shell,
+ shell_options,
+ ' '.join([
+ source_file, suppress_output,
+ concatenate_on_success, dump_environment,
+ ]),
+ ]
+
+ # Try to source the file
+ proc = subprocess.Popen(
+ command, stdout=subprocess.PIPE, env=os.environ)
+ proc.wait()
+
+ if proc.returncode != 0:
+ msg = 'Sourcing file {0} returned a non-zero exit code'.format(
+ filename)
+ raise RuntimeError(msg)
+
+ output = ''.join([line.decode('utf-8') for line in proc.stdout])
+
+ # Construct dictionaries of the environment before and after
+ # sourcing the file, so that we can diff them.
+ env_before = dict(os.environ)
+ env_after = json.loads(output)
+
+ # If we're in python2, convert to str objects instead of unicode
+ # like json gives us. We can't put unicode in os.environ anyway.
+ if sys.version_info[0] < 3:
+ env_after = dict((k.encode('utf-8'), v.encode('utf-8'))
+ for k, v in env_after.items())
+
+ # Other variables unrelated to sourcing a file
+ blacklist.extend(['SHLVL', '_', 'PWD', 'OLDPWD', 'PS2'])
+
+ def set_intersection(fullset, *args):
+ # A set intersection using string literals and regexs
+ meta = '[' + re.escape('[$()*?[]^{|}') + ']'
+ subset = fullset & set(args) # As literal
+ for name in args:
+ if re.search(meta, name):
+ pattern = re.compile(name)
+ for k in fullset:
+ if re.match(pattern, k):
+ subset.add(k)
+ return subset
+
+ for d in env_after, env_before:
+ # Retain (whitelist) has priority over prune (blacklist)
+ prune = set_intersection(set(d), *blacklist)
+ prune -= set_intersection(prune, *whitelist)
+ for k in prune:
+ d.pop(k, None)
+
+ # Fill the EnvironmentModifications instance
+ env = EnvironmentModifications()
+
+ # New variables
+ new_variables = list(set(env_after) - set(env_before))
+ # Variables that have been unset
+ unset_variables = list(set(env_before) - set(env_after))
+ # Variables that have been modified
+ common_variables = set(env_before).intersection(set(env_after))
+
+ modified_variables = [x for x in common_variables
+ if env_before[x] != env_after[x]]
+
+ # Consistent output order - looks nicer, easier comparison...
+ new_variables.sort()
+ unset_variables.sort()
+ modified_variables.sort()
+
+ def return_separator_if_any(*args):
+ separators = ':', ';'
+ for separator in separators:
+ for arg in args:
+ if separator in arg:
+ return separator
+ return None
+
+ # Add variables to env.
+ # Assume that variables with 'PATH' in the name or that contain
+ # separators like ':' or ';' are more likely to be paths
+ for x in new_variables:
+ sep = return_separator_if_any(env_after[x])
+ if sep:
+ env.prepend_path(x, env_after[x], separator=sep)
+ elif 'PATH' in x:
+ env.prepend_path(x, env_after[x])
+ else:
+ # We just need to set the variable to the new value
+ env.set(x, env_after[x])
+
+ for x in unset_variables:
+ env.unset(x)
+
+ for x in modified_variables:
+ before = env_before[x]
+ after = env_after[x]
+ sep = return_separator_if_any(before, after)
+ if sep:
+ before_list = before.split(sep)
+ after_list = after.split(sep)
+
+ # Filter out empty strings
+ before_list = list(filter(None, before_list))
+ after_list = list(filter(None, after_list))
+
+ # Remove duplicate entries (worse matching, bloats env)
+ if clean:
+ before_list = list(dedupe(before_list))
+ after_list = list(dedupe(after_list))
+ # The reassembled cleaned entries
+ before = sep.join(before_list)
+ after = sep.join(after_list)
+
+ # Paths that have been removed
+ remove_list = [
+ ii for ii in before_list if ii not in after_list]
+ # Check that nothing has been added in the middle of
+ # before_list
+ remaining_list = [
+ ii for ii in before_list if ii in after_list]
+ try:
+ start = after_list.index(remaining_list[0])
+ end = after_list.index(remaining_list[-1])
+ search = sep.join(after_list[start:end + 1])
+ except IndexError:
+ env.prepend_path(x, after)
+
+ if search not in before:
+ # We just need to set the variable to the new value
+ env.prepend_path(x, after)
+ else:
+ try:
+ prepend_list = after_list[:start]
+ prepend_list.reverse() # Preserve order after prepend
+ except KeyError:
+ prepend_list = []
+ try:
+ append_list = after_list[end + 1:]
+ except KeyError:
+ append_list = []
+
+ for item in remove_list:
+ env.remove_path(x, item)
+ for item in append_list:
+ env.append_path(x, item)
+ for item in prepend_list:
+ env.prepend_path(x, item)
+ else:
+ # We just need to set the variable to the new value
+ env.set(x, after)
+
+ return env
+
+
+def concatenate_paths(paths, separator=':'):
+ """Concatenates an iterable of paths into a string of paths separated by
+ separator, defaulting to colon.
+
+ Args:
+ paths: iterable of paths
+ separator: the separator to use, default ':'
+
+ Returns:
+ string
+ """
+ return separator.join(str(item) for item in paths)
+
+
+def set_or_unset_not_first(variable, changes, errstream):
+ """Check if we are going to set or unset something after other
+ modifications have already been requested.
+ """
+ indexes = [ii for ii, item in enumerate(changes)
+ if ii != 0 and
+ not item.args.get('force', False) and
+ type(item) in [SetEnv, UnsetEnv]]
+ if indexes:
+ good = '\t \t{context} at {filename}:{lineno}'
+ nogood = '\t--->\t{context} at {filename}:{lineno}'
+ message = "Suspicious requests to set or unset '{var}' found"
+ errstream(message.format(var=variable))
+ for ii, item in enumerate(changes):
+ print_format = nogood if ii in indexes else good
+ errstream(print_format.format(**item.args))
+
+
+def validate(env, errstream):
+ """Validates the environment modifications to check for the presence of
+ suspicious patterns. Prompts a warning for everything that was found.
+
+ Current checks:
+ - set or unset variables after other changes on the same variable
+
+ Args:
+ env: list of environment modifications
+ """
+ modifications = env.group_by_name()
+ for variable, list_of_changes in sorted(modifications.items()):
+ set_or_unset_not_first(variable, list_of_changes, errstream)
+
+
+def filter_environment_blacklist(env, variables):
+ """Generator that filters out any change to environment variables present in
+ the input list.
+
+ Args:
+ env: list of environment modifications
+ variables: list of variable names to be filtered
+
+ Returns:
+ items in env if they are not in variables
+ """
+ for item in env:
+ if item.name not in variables:
+ yield item
+
+
+def inspect_path(root, inspections, exclude=None):
+ """Inspects ``root`` to search for the subdirectories in ``inspections``.
+ Adds every path found to a list of prepend-path commands and returns it.
+
+ Args:
+ root (str): absolute path where to search for subdirectories
+
+ inspections (dict): maps relative paths to a list of environment
+ variables that will be modified if the path exists. The
+ modifications are not performed immediately, but stored in a
+ command object that is returned to client
+
+ exclude (callable): optional callable. If present it must accept an
+ absolute path and return True if it should be excluded from the
+ inspection
+
+ Examples:
+
+ The following lines execute an inspection in ``/usr`` to search for
+ ``/usr/include`` and ``/usr/lib64``. If found we want to prepend
+ ``/usr/include`` to ``CPATH`` and ``/usr/lib64`` to ``MY_LIB64_PATH``.
+
+ .. code-block:: python
+
+ # Set up the dictionary containing the inspection
+ inspections = {
+ 'include': ['CPATH'],
+ 'lib64': ['MY_LIB64_PATH']
+ }
+
+ # Get back the list of command needed to modify the environment
+ env = inspect_path('/usr', inspections)
+
+ # Eventually execute the commands
+ env.apply_modifications()
+
+ Returns:
+ instance of EnvironmentModifications containing the requested
+ modifications
+ """
+ if exclude is None:
+ exclude = lambda x: False
+
+ env = EnvironmentModifications()
+ # Inspect the prefix to check for the existence of common directories
+ for relative_path, variables in inspections.items():
+ expected = os.path.join(root, relative_path)
+
+ if os.path.isdir(expected) and not exclude(expected):
+ for variable in variables:
+ env.prepend_path(variable, expected)
+
+ return env
+
+
+@contextlib.contextmanager
+def preserve_environment(*variables):
+ """Ensures that the value of the environment variables passed as
+ arguments is the same before entering to the context manager and after
+ exiting it.
+
+ Variables that are unset before entering the context manager will be
+ explicitly unset on exit.
+
+ Args:
+ variables (list of str): list of environment variables to be preserved
+ """
+ cache = {}
+ for var in variables:
+ # The environment variable to be preserved might not be there.
+ # In that case store None as a placeholder.
+ cache[var] = os.environ.get(var, None)
+
+ yield
+
+ for var in variables:
+ value = cache[var]
+ msg = '[PRESERVE_ENVIRONMENT]'
+ if value is not None:
+ # Print a debug statement if the value changed
+ if var not in os.environ:
+ msg += ' {0} was unset, will be reset to "{1}"'
+ tty.debug(msg.format(var, value))
+ elif os.environ[var] != value:
+ msg += ' {0} was set to "{1}", will be reset to "{2}"'
+ tty.debug(msg.format(var, os.environ[var], value))
+ os.environ[var] = value
+ elif var in os.environ:
+ msg += ' {0} was set to "{1}", will be unset'
+ tty.debug(msg.format(var, os.environ[var]))
+ del os.environ[var]