From a6681f2d7f0c3ff1a1ec359751710e98d2d52b63 Mon Sep 17 00:00:00 2001 From: alalazo Date: Sun, 12 Jun 2016 15:06:17 +0200 Subject: environment modules : added function to construct them from source files --- lib/spack/spack/environment.py | 76 +++++++++++++++++++++++++++- lib/spack/spack/test/data/sourceme_first.sh | 3 ++ lib/spack/spack/test/data/sourceme_second.sh | 3 ++ lib/spack/spack/test/environment.py | 39 +++++++++++++- 4 files changed, 119 insertions(+), 2 deletions(-) create mode 100644 lib/spack/spack/test/data/sourceme_first.sh create mode 100644 lib/spack/spack/test/data/sourceme_second.sh (limited to 'lib') diff --git a/lib/spack/spack/environment.py b/lib/spack/spack/environment.py index af642dcc9b..87759b36e1 100644 --- a/lib/spack/spack/environment.py +++ b/lib/spack/spack/environment.py @@ -26,7 +26,9 @@ import collections import inspect import os import os.path - +import subprocess +import shlex +import json class NameModifier(object): def __init__(self, name, **kwargs): @@ -240,6 +242,78 @@ class EnvironmentModifications(object): for x in actions: x.execute() + @staticmethod + def from_sourcing_files(*args, **kwargs): + """ + Creates an instance of EnvironmentModifications that, if executed, + has the same effect on the environment as sourcing the files passed as + parameters + + Args: + *args: list of files to be sourced + + Returns: + instance of EnvironmentModifications + """ + env = EnvironmentModifications() + # Check if the files are actually there + if not all(os.path.isfile(file) for file in args): + raise RuntimeError('trying to source non-existing files') + # Relevant kwd parameters and formats + info = dict(kwargs) + info.setdefault('shell', '/bin/bash') + info.setdefault('shell_options', '-c') + info.setdefault('source_command', 'source') + info.setdefault('suppress_output', '&> /dev/null') + info.setdefault('concatenate_on_success', '&&') + + shell = '{shell}'.format(**info) + shell_options = '{shell_options}'.format(**info) + source_file = '{source_command} {file} {concatenate_on_success}' + dump_environment = 'python -c "import os, json; print json.dumps(dict(os.environ))"' + # Construct the command that will be executed + command = [source_file.format(file=file, **info) for file in args] + command.append(dump_environment) + command = ' '.join(command) + command = [ + shell, + shell_options, + command + ] + + # Try to source all the files, + proc = subprocess.Popen(command, stdout=subprocess.PIPE, env=os.environ) + proc.wait() + if proc.returncode != 0: + raise RuntimeError('sourcing files returned a non-zero exit code') + output = ''.join([line for line in proc.stdout]) + # Construct a dictionary with all the variables in the environment + after_source_env = dict(json.loads(output)) + + # Filter variables that are due to how we source + after_source_env.pop('SHLVL') + after_source_env.pop('_') + after_source_env.pop('PWD') + + # Fill the EnvironmentModifications instance + this_environment = dict(os.environ) + # New variables + new_variables = set(after_source_env) - set(this_environment) + for x in new_variables: + env.set(x, after_source_env[x]) + # Variables that have been unset + unset_variables = set(this_environment) - set(after_source_env) + for x in unset_variables: + env.unset(x) + # Variables that have been modified + common_variables = set(this_environment).intersection(set(after_source_env)) + modified_variables = [x for x in common_variables if this_environment[x] != after_source_env[x]] + for x in modified_variables: + # TODO : we may be smarter here, and try to parse if we could compose append_path + # TODO : and prepend_path to modify the value + env.set(x, after_source_env[x]) + return env + def concatenate_paths(paths, separator=':'): """ diff --git a/lib/spack/spack/test/data/sourceme_first.sh b/lib/spack/spack/test/data/sourceme_first.sh new file mode 100644 index 0000000000..800f639ac8 --- /dev/null +++ b/lib/spack/spack/test/data/sourceme_first.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +export NEW_VAR='new' +export UNSET_ME='overridden' diff --git a/lib/spack/spack/test/data/sourceme_second.sh b/lib/spack/spack/test/data/sourceme_second.sh new file mode 100644 index 0000000000..db88b8334a --- /dev/null +++ b/lib/spack/spack/test/data/sourceme_second.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +export PATH_LIST='/path/first:/path/second:/path/third:/path/fourth' +unset EMPTY_PATH_LIST \ No newline at end of file diff --git a/lib/spack/spack/test/environment.py b/lib/spack/spack/test/environment.py index ded1539e18..34b4485f8e 100644 --- a/lib/spack/spack/test/environment.py +++ b/lib/spack/spack/test/environment.py @@ -24,7 +24,10 @@ ############################################################################## import unittest import os -from spack.environment import EnvironmentModifications + +from spack import spack_root +from llnl.util.filesystem import join_path +from spack.environment import EnvironmentModifications, SetEnv, UnsetEnv class EnvironmentTest(unittest.TestCase): @@ -95,3 +98,37 @@ class EnvironmentTest(unittest.TestCase): self.assertEqual(len(copy_construct), 2) for x, y in zip(env, copy_construct): assert x is y + + def test_source_files(self): + datadir = join_path(spack_root, 'lib', 'spack', 'spack', 'test', 'data') + files = [ + join_path(datadir, 'sourceme_first.sh'), + join_path(datadir, 'sourceme_second.sh') + ] + env = EnvironmentModifications.from_sourcing_files(*files) + modifications = env.group_by_name() + + self.assertEqual(len(modifications), 4) + # Set new variables + self.assertEqual(len(modifications['NEW_VAR']), 1) + self.assertTrue(isinstance(modifications['NEW_VAR'][0], SetEnv)) + self.assertEqual(modifications['NEW_VAR'][0].value, 'new') + # Unset variables + self.assertEqual(len(modifications['EMPTY_PATH_LIST']), 1) + self.assertTrue(isinstance(modifications['EMPTY_PATH_LIST'][0], UnsetEnv)) + # Modified variables + self.assertEqual(len(modifications['UNSET_ME']), 1) + self.assertTrue(isinstance(modifications['UNSET_ME'][0], SetEnv)) + self.assertEqual(modifications['UNSET_ME'][0].value, 'overridden') + + self.assertEqual(len(modifications['PATH_LIST']), 1) + self.assertTrue(isinstance(modifications['PATH_LIST'][0], SetEnv)) + self.assertEqual(modifications['PATH_LIST'][0].value, '/path/first:/path/second:/path/third:/path/fourth') + + # TODO : with reference to the TODO in spack/environment.py + # TODO : remove the above and insert + # self.assertEqual(len(modifications['PATH_LIST']), 2) + # self.assertTrue(isinstance(modifications['PATH_LIST'][0], PrependPath)) + # self.assertEqual(modifications['PATH_LIST'][0].value, '/path/first') + # self.assertTrue(isinstance(modifications['PATH_LIST'][1], AppendPath)) + # self.assertEqual(modifications['PATH_LIST'][1].value, '/path/fourth') -- cgit v1.2.3-60-g2f50 From 5300ffac7f1e4f77fcb1ba52083d4a7ba7183467 Mon Sep 17 00:00:00 2001 From: alalazo Date: Sun, 12 Jun 2016 15:11:26 +0200 Subject: qa : fixed flak8 checks --- lib/spack/spack/environment.py | 25 ++++++++++++++++++------- lib/spack/spack/test/environment.py | 37 ++++++++++++++++++++++++++++--------- 2 files changed, 46 insertions(+), 16 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/environment.py b/lib/spack/spack/environment.py index 87759b36e1..623bfa6ed2 100644 --- a/lib/spack/spack/environment.py +++ b/lib/spack/spack/environment.py @@ -24,13 +24,14 @@ ############################################################################## import collections import inspect +import json import os import os.path import subprocess -import shlex -import json + class NameModifier(object): + def __init__(self, name, **kwargs): self.name = name self.args = {'name': name} @@ -38,6 +39,7 @@ class NameModifier(object): class NameValueModifier(object): + def __init__(self, name, value, **kwargs): self.name = name self.value = value @@ -47,23 +49,27 @@ class NameValueModifier(object): class SetEnv(NameValueModifier): + def execute(self): os.environ[self.name] = str(self.value) class UnsetEnv(NameModifier): + def execute(self): # Avoid throwing if the variable was not set os.environ.pop(self.name, None) class SetPath(NameValueModifier): + def execute(self): string_path = concatenate_paths(self.value, separator=self.separator) os.environ[self.name] = string_path class AppendPath(NameValueModifier): + def execute(self): environment_value = os.environ.get(self.name, '') directories = environment_value.split( @@ -73,6 +79,7 @@ class AppendPath(NameValueModifier): class PrependPath(NameValueModifier): + def execute(self): environment_value = os.environ.get(self.name, '') directories = environment_value.split( @@ -82,6 +89,7 @@ class PrependPath(NameValueModifier): class RemovePath(NameValueModifier): + def execute(self): environment_value = os.environ.get(self.name, '') directories = environment_value.split( @@ -270,7 +278,7 @@ class EnvironmentModifications(object): shell = '{shell}'.format(**info) shell_options = '{shell_options}'.format(**info) source_file = '{source_command} {file} {concatenate_on_success}' - dump_environment = 'python -c "import os, json; print json.dumps(dict(os.environ))"' + dump_environment = 'python -c "import os, json; print json.dumps(dict(os.environ))"' # NOQA: ignore=E501 # Construct the command that will be executed command = [source_file.format(file=file, **info) for file in args] command.append(dump_environment) @@ -282,7 +290,8 @@ class EnvironmentModifications(object): ] # Try to source all the files, - proc = subprocess.Popen(command, stdout=subprocess.PIPE, env=os.environ) + proc = subprocess.Popen( + command, stdout=subprocess.PIPE, env=os.environ) proc.wait() if proc.returncode != 0: raise RuntimeError('sourcing files returned a non-zero exit code') @@ -306,10 +315,12 @@ class EnvironmentModifications(object): for x in unset_variables: env.unset(x) # Variables that have been modified - common_variables = set(this_environment).intersection(set(after_source_env)) - modified_variables = [x for x in common_variables if this_environment[x] != after_source_env[x]] + common_variables = set(this_environment).intersection( + set(after_source_env)) + modified_variables = [x for x in common_variables if this_environment[x] != after_source_env[x]] # NOQA: ignore=E501 for x in modified_variables: - # TODO : we may be smarter here, and try to parse if we could compose append_path + # TODO : we may be smarter here, and try to parse + # TODO : if we could compose append_path # TODO : and prepend_path to modify the value env.set(x, after_source_env[x]) return env diff --git a/lib/spack/spack/test/environment.py b/lib/spack/spack/test/environment.py index 34b4485f8e..31a85f9ee3 100644 --- a/lib/spack/spack/test/environment.py +++ b/lib/spack/spack/test/environment.py @@ -31,12 +31,13 @@ from spack.environment import EnvironmentModifications, SetEnv, UnsetEnv class EnvironmentTest(unittest.TestCase): + def setUp(self): os.environ.clear() os.environ['UNSET_ME'] = 'foo' os.environ['EMPTY_PATH_LIST'] = '' os.environ['PATH_LIST'] = '/path/second:/path/third' - os.environ['REMOVE_PATH_LIST'] = '/a/b:/duplicate:/a/c:/remove/this:/a/d:/duplicate/:/f/g' + os.environ['REMOVE_PATH_LIST'] = '/a/b:/duplicate:/a/c:/remove/this:/a/d:/duplicate/:/f/g' # NOQA: ignore=E501 def test_set(self): env = EnvironmentModifications() @@ -77,9 +78,18 @@ class EnvironmentTest(unittest.TestCase): env.remove_path('REMOVE_PATH_LIST', '/duplicate/') env.apply_modifications() - self.assertEqual('/path/first:/path/second:/path/third:/path/last', os.environ['PATH_LIST']) - self.assertEqual('/path/first:/path/middle:/path/last', os.environ['EMPTY_PATH_LIST']) - self.assertEqual('/path/first:/path/middle:/path/last', os.environ['NEWLY_CREATED_PATH_LIST']) + self.assertEqual( + '/path/first:/path/second:/path/third:/path/last', + os.environ['PATH_LIST'] + ) + self.assertEqual( + '/path/first:/path/middle:/path/last', + os.environ['EMPTY_PATH_LIST'] + ) + self.assertEqual( + '/path/first:/path/middle:/path/last', + os.environ['NEWLY_CREATED_PATH_LIST'] + ) self.assertEqual('/a/b:/a/c:/a/d:/f/g', os.environ['REMOVE_PATH_LIST']) def test_extra_arguments(self): @@ -100,7 +110,8 @@ class EnvironmentTest(unittest.TestCase): assert x is y def test_source_files(self): - datadir = join_path(spack_root, 'lib', 'spack', 'spack', 'test', 'data') + datadir = join_path(spack_root, 'lib', 'spack', + 'spack', 'test', 'data') files = [ join_path(datadir, 'sourceme_first.sh'), join_path(datadir, 'sourceme_second.sh') @@ -115,7 +126,8 @@ class EnvironmentTest(unittest.TestCase): self.assertEqual(modifications['NEW_VAR'][0].value, 'new') # Unset variables self.assertEqual(len(modifications['EMPTY_PATH_LIST']), 1) - self.assertTrue(isinstance(modifications['EMPTY_PATH_LIST'][0], UnsetEnv)) + self.assertTrue(isinstance( + modifications['EMPTY_PATH_LIST'][0], UnsetEnv)) # Modified variables self.assertEqual(len(modifications['UNSET_ME']), 1) self.assertTrue(isinstance(modifications['UNSET_ME'][0], SetEnv)) @@ -123,12 +135,19 @@ class EnvironmentTest(unittest.TestCase): self.assertEqual(len(modifications['PATH_LIST']), 1) self.assertTrue(isinstance(modifications['PATH_LIST'][0], SetEnv)) - self.assertEqual(modifications['PATH_LIST'][0].value, '/path/first:/path/second:/path/third:/path/fourth') + self.assertEqual( + modifications['PATH_LIST'][0].value, + '/path/first:/path/second:/path/third:/path/fourth' + ) # TODO : with reference to the TODO in spack/environment.py # TODO : remove the above and insert # self.assertEqual(len(modifications['PATH_LIST']), 2) - # self.assertTrue(isinstance(modifications['PATH_LIST'][0], PrependPath)) + # self.assertTrue( + # isinstance(modifications['PATH_LIST'][0], PrependPath) + # ) # self.assertEqual(modifications['PATH_LIST'][0].value, '/path/first') - # self.assertTrue(isinstance(modifications['PATH_LIST'][1], AppendPath)) + # self.assertTrue( + # isinstance(modifications['PATH_LIST'][1], AppendPath) + # ) # self.assertEqual(modifications['PATH_LIST'][1].value, '/path/fourth') -- cgit v1.2.3-60-g2f50 From 30c9d976f6e7b74b6713215e8f7c7924a9f43f43 Mon Sep 17 00:00:00 2001 From: Massimiliano Culpo Date: Sun, 12 Jun 2016 18:49:12 +0200 Subject: environment : added more logic to treat append and prepend path --- lib/spack/spack/environment.py | 60 ++++++++++++++++++++++++---- lib/spack/spack/test/data/sourceme_second.sh | 2 +- lib/spack/spack/test/environment.py | 33 +++++++-------- 3 files changed, 68 insertions(+), 27 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/environment.py b/lib/spack/spack/environment.py index 623bfa6ed2..cfba060459 100644 --- a/lib/spack/spack/environment.py +++ b/lib/spack/spack/environment.py @@ -1,4 +1,4 @@ -############################################################################## +# # Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory. # @@ -21,7 +21,7 @@ # You should have received a copy of the GNU Lesser General Public # License along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -############################################################################## +# import collections import inspect import json @@ -100,6 +100,7 @@ class RemovePath(NameValueModifier): class EnvironmentModifications(object): + """ Keeps track of requests to modify the current environment. @@ -315,14 +316,57 @@ class EnvironmentModifications(object): for x in unset_variables: env.unset(x) # Variables that have been modified - common_variables = set(this_environment).intersection( - set(after_source_env)) + common_variables = set(this_environment).intersection(set(after_source_env)) # NOQA: ignore=E501 modified_variables = [x for x in common_variables if this_environment[x] != after_source_env[x]] # NOQA: ignore=E501 + + def return_separator_if_any(first_value, second_value): + separators = ':', ';' + for separator in separators: + if separator in first_value and separator in second_value: + return separator + return None + for x in modified_variables: - # TODO : we may be smarter here, and try to parse - # TODO : if we could compose append_path - # TODO : and prepend_path to modify the value - env.set(x, after_source_env[x]) + current = this_environment[x] + modified = after_source_env[x] + sep = return_separator_if_any(current, modified) + if sep is None: + # We just need to set the variable to the new value + env.set(x, after_source_env[x]) + else: + current_list = current.split(sep) + modified_list = modified.split(sep) + # Paths that have been removed + remove_list = [ + ii for ii in current_list if ii not in modified_list] + # Check that nothing has been added in the middle of vurrent + # list + remaining_list = [ + ii for ii in current_list if ii in modified_list] + start = modified_list.index(remaining_list[0]) + end = modified_list.index(remaining_list[-1]) + search = sep.join(modified_list[start:end + 1]) + if search not in current: + # We just need to set the variable to the new value + env.set(x, after_source_env[x]) + break + else: + try: + prepend_list = modified_list[:start] + except KeyError: + prepend_list = [] + try: + append_list = modified_list[end + 1:] + except KeyError: + append_list = [] + + for item in remove_list: + env.remove_path(x, item) + for item in append_list: + env.append_path(x, item) + for item in prepend_list: + env.prepend_path(x, item) + return env diff --git a/lib/spack/spack/test/data/sourceme_second.sh b/lib/spack/spack/test/data/sourceme_second.sh index db88b8334a..9955a0e6d6 100644 --- a/lib/spack/spack/test/data/sourceme_second.sh +++ b/lib/spack/spack/test/data/sourceme_second.sh @@ -1,3 +1,3 @@ #!/usr/bin/env bash -export PATH_LIST='/path/first:/path/second:/path/third:/path/fourth' +export PATH_LIST='/path/first:/path/second:/path/fourth' unset EMPTY_PATH_LIST \ No newline at end of file diff --git a/lib/spack/spack/test/environment.py b/lib/spack/spack/test/environment.py index 31a85f9ee3..219c68e5a8 100644 --- a/lib/spack/spack/test/environment.py +++ b/lib/spack/spack/test/environment.py @@ -27,7 +27,9 @@ import os from spack import spack_root from llnl.util.filesystem import join_path -from spack.environment import EnvironmentModifications, SetEnv, UnsetEnv +from spack.environment import EnvironmentModifications +from spack.environment import SetEnv, UnsetEnv +from spack.environment import RemovePath, PrependPath, AppendPath class EnvironmentTest(unittest.TestCase): @@ -133,21 +135,16 @@ class EnvironmentTest(unittest.TestCase): self.assertTrue(isinstance(modifications['UNSET_ME'][0], SetEnv)) self.assertEqual(modifications['UNSET_ME'][0].value, 'overridden') - self.assertEqual(len(modifications['PATH_LIST']), 1) - self.assertTrue(isinstance(modifications['PATH_LIST'][0], SetEnv)) - self.assertEqual( - modifications['PATH_LIST'][0].value, - '/path/first:/path/second:/path/third:/path/fourth' + self.assertEqual(len(modifications['PATH_LIST']), 3) + self.assertTrue( + isinstance(modifications['PATH_LIST'][0], RemovePath) ) - - # TODO : with reference to the TODO in spack/environment.py - # TODO : remove the above and insert - # self.assertEqual(len(modifications['PATH_LIST']), 2) - # self.assertTrue( - # isinstance(modifications['PATH_LIST'][0], PrependPath) - # ) - # self.assertEqual(modifications['PATH_LIST'][0].value, '/path/first') - # self.assertTrue( - # isinstance(modifications['PATH_LIST'][1], AppendPath) - # ) - # self.assertEqual(modifications['PATH_LIST'][1].value, '/path/fourth') + self.assertEqual(modifications['PATH_LIST'][0].value, '/path/third') + self.assertTrue( + isinstance(modifications['PATH_LIST'][1], AppendPath) + ) + self.assertEqual(modifications['PATH_LIST'][1].value, '/path/fourth') + self.assertTrue( + isinstance(modifications['PATH_LIST'][2], PrependPath) + ) + self.assertEqual(modifications['PATH_LIST'][2].value, '/path/first') -- cgit v1.2.3-60-g2f50 From 9e0c20c794cea3d54fc16ea7470035384985f1c6 Mon Sep 17 00:00:00 2001 From: alalazo Date: Sat, 18 Jun 2016 13:39:08 +0200 Subject: environment : filter the current environment Previously only the environment obtained after sourcing the file was filtered. This caused the appeareance of spurious unset commands in the list. --- lib/spack/spack/environment.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/environment.py b/lib/spack/spack/environment.py index cfba060459..30c6228ca4 100644 --- a/lib/spack/spack/environment.py +++ b/lib/spack/spack/environment.py @@ -297,16 +297,18 @@ class EnvironmentModifications(object): if proc.returncode != 0: raise RuntimeError('sourcing files returned a non-zero exit code') output = ''.join([line for line in proc.stdout]) - # Construct a dictionary with all the variables in the environment + # Construct a dictionary with all the variables in the new environment after_source_env = dict(json.loads(output)) + this_environment = dict(os.environ) - # Filter variables that are due to how we source - after_source_env.pop('SHLVL') - after_source_env.pop('_') - after_source_env.pop('PWD') + # Filter variables that are not related to sourcing a file + to_be_filtered = 'SHLVL', '_', 'PWD', 'OLDPWD' + for d in after_source_env, this_environment: + for name in to_be_filtered: + d.pop(name, None) # Fill the EnvironmentModifications instance - this_environment = dict(os.environ) + # New variables new_variables = set(after_source_env) - set(this_environment) for x in new_variables: -- cgit v1.2.3-60-g2f50