From 1297e47463cd1ecaad6637e9f86a92550f3d9a63 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sat, 18 Mar 2017 22:48:20 -0700 Subject: Use byte-encoded UTF-8 for sourced environment in Python 2 (#3489) - Fixes recurring errors on develop with unicode commit characters. - still Python3-proof: python3 will use str instead of bytestrings. --- lib/spack/spack/environment.py | 21 ++++++++++++----- lib/spack/spack/test/data/sourceme_unicode.sh | 34 +++++++++++++++++++++++++++ lib/spack/spack/test/environment.py | 6 ++--- 3 files changed, 52 insertions(+), 9 deletions(-) create mode 100644 lib/spack/spack/test/data/sourceme_unicode.sh (limited to 'lib') diff --git a/lib/spack/spack/environment.py b/lib/spack/spack/environment.py index da69979ae1..1333054518 100644 --- a/lib/spack/spack/environment.py +++ b/lib/spack/spack/environment.py @@ -26,6 +26,7 @@ import collections import inspect import json import os +import sys import os.path import subprocess @@ -268,8 +269,8 @@ class EnvironmentModifications(object): :param \*args: list of files to be sourced :rtype: instance of EnvironmentModifications """ - env = EnvironmentModifications() + # Check if the files are actually there files = [line.split(' ')[0] for line in args] non_existing = [file for file in files if not os.path.isfile(file)] @@ -277,6 +278,7 @@ class EnvironmentModifications(object): message = 'trying to source non-existing files\n' message += '\n'.join(non_existing) raise RuntimeError(message) + # Relevant kwd parameters and formats info = dict(kwargs) info.setdefault('shell', '/bin/bash') @@ -309,11 +311,17 @@ class EnvironmentModifications(object): if proc.returncode != 0: raise RuntimeError('sourcing files returned a non-zero exit code') output = ''.join([line for line in proc.stdout]) - # Construct a dictionary with all the variables in the new environment - after_source_env = dict( - (k, v.decode('utf8')) for k, v in json.loads(output).items()) - this_environment = dict( - (k, v.decode('utf8')) for k, v in os.environ.items()) + + # Construct a dictionaries of the environment before and after + # sourcing the files, so that we can diff them. + this_environment = dict(os.environ) + after_source_env = json.loads(output) + + # If we're in python2, convert to str objects instead of unicode + # like json gives us. We can't put unicode in os.environ anyway. + if sys.version_info[0] < 3: + after_source_env = dict((k.encode('utf-8'), v.encode('utf-8')) + for k, v in after_source_env.items()) # Filter variables that are not related to sourcing a file to_be_filtered = 'SHLVL', '_', 'PWD', 'OLDPWD' @@ -364,6 +372,7 @@ class EnvironmentModifications(object): start = modified_list.index(remaining_list[0]) end = modified_list.index(remaining_list[-1]) search = sep.join(modified_list[start:end + 1]) + if search not in current: # We just need to set the variable to the new value env.set(x, after_source_env[x]) diff --git a/lib/spack/spack/test/data/sourceme_unicode.sh b/lib/spack/spack/test/data/sourceme_unicode.sh new file mode 100644 index 0000000000..f01f6a4927 --- /dev/null +++ b/lib/spack/spack/test/data/sourceme_unicode.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash + +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## + +# Set an environment variable with some unicode in it to ensure that +# Spack can decode it. +# +# This has caused squashed commits on develop to break, as some +# committers use unicode in their messages, and Travis sets the +# current commit message in an environment variable. +export UNICODE_VAR='don\xe2\x80\x99t' diff --git a/lib/spack/spack/test/environment.py b/lib/spack/spack/test/environment.py index 0da137c192..dbad2c5e1f 100644 --- a/lib/spack/spack/test/environment.py +++ b/lib/spack/spack/test/environment.py @@ -96,7 +96,8 @@ def files_to_be_sourced(): files = [ os.path.join(datadir, 'sourceme_first.sh'), os.path.join(datadir, 'sourceme_second.sh'), - os.path.join(datadir, 'sourceme_parameters.sh intel64') + os.path.join(datadir, 'sourceme_parameters.sh intel64'), + os.path.join(datadir, 'sourceme_unicode.sh') ] return files @@ -230,7 +231,6 @@ def test_source_files(files_to_be_sourced): """Tests the construction of a list of environment modifications that are the result of sourcing a file. """ - env = EnvironmentModifications.from_sourcing_files(*files_to_be_sourced) modifications = env.group_by_name() @@ -238,7 +238,7 @@ def test_source_files(files_to_be_sourced): # spurious entries for things like PS1 # # TODO: figure out how to make a bit more robust. - assert len(modifications) >= 4 + assert len(modifications) >= 5 # Set new variables assert len(modifications['NEW_VAR']) == 1 -- cgit v1.2.3-60-g2f50