Build process output: handle UTF-8 for python 3.x to 3.7 (#22888)

We set LC_ALL=C to encourage a build process to generate ASCII
output (so our logger daemon can decode it). Most packages
respect this but it appears that intel-oneapi-compilers does
not in some cases (see #22813). This reads the output of the build
process as UTF-8, which still works if the build process respects
LC_ALL=C but also works if the process generates UTF-8 output.

For Python >= 3.7 all files are opened with UTF-8 encoding by
default. Python 2 does not support the encoding argument on
'open', so to support Python 2 the files would have to be
opened in byte mode and explicitly decoded (as a side note,
this would be the only way to handle other encodings without
being informed of them in advance).

1 files changed, 9 insertions, 2 deletions

diff --git a/lib/spack/llnl/util/tty/log.py b/lib/spack/llnl/util/tty/log.py
index ef14baeacf..337eb0ac68 100644
--- a/lib/spack/llnl/util/tty/log.py
+++ b/lib/spack/llnl/util/tty/log.py
@@ -321,7 +321,10 @@ class FileWrapper(object):
     def unwrap(self):
         if self.open:
             if self.file_like:
-                self.file = open(self.file_like, 'w')
+                if sys.version_info < (3,):
+                    self.file = open(self.file_like, 'w')
+                else:
+                    self.file = open(self.file_like, 'w', encoding='utf-8')
             else:
                 self.file = StringIO()
             return self.file
@@ -722,7 +725,11 @@ def _writer_daemon(stdin_multiprocess_fd, read_multiprocess_fd, write_fd, echo,
 
     # Use line buffering (3rd param = 1) since Python 3 has a bug
     # that prevents unbuffered text I/O.
-    in_pipe = os.fdopen(read_multiprocess_fd.fd, 'r', 1)
+    if sys.version_info < (3,):
+        in_pipe = os.fdopen(read_multiprocess_fd.fd, 'r', 1)
+    else:
+        # Python 3.x before 3.7 does not open with UTF-8 encoding by default
+        in_pipe = os.fdopen(read_multiprocess_fd.fd, 'r', 1, encoding='utf-8')
 
     if stdin_multiprocess_fd:
         stdin = os.fdopen(stdin_multiprocess_fd.fd)