diff options
Diffstat (limited to 'lib/spack')
-rw-r--r-- | lib/spack/llnl/util/lang.py | 18 | ||||
-rw-r--r-- | lib/spack/llnl/util/tty/log.py | 3 | ||||
-rw-r--r-- | lib/spack/llnl/util/tty/pty.py | 3 | ||||
-rw-r--r-- | lib/spack/spack/build_environment.py | 4 | ||||
-rw-r--r-- | lib/spack/spack/test/database.py | 4 | ||||
-rw-r--r-- | lib/spack/spack/test/llnl/util/lock.py | 5 |
6 files changed, 29 insertions, 8 deletions
diff --git a/lib/spack/llnl/util/lang.py b/lib/spack/llnl/util/lang.py index e746ce096c..0bcbebba72 100644 --- a/lib/spack/llnl/util/lang.py +++ b/lib/spack/llnl/util/lang.py @@ -5,6 +5,7 @@ from __future__ import division +import multiprocessing import os import re import functools @@ -19,6 +20,23 @@ import sys ignore_modules = [r'^\.#', '~$'] +# On macOS, Python 3.8 multiprocessing now defaults to the 'spawn' start +# method. Spack cannot currently handle this, so force the process to start +# using the 'fork' start method. +# +# TODO: This solution is not ideal, as the 'fork' start method can lead to +# crashes of the subprocess. Figure out how to make 'spawn' work. +# +# See: +# * https://github.com/spack/spack/pull/18124 +# * https://docs.python.org/3/library/multiprocessing.html#contexts-and-start-methods # noqa: E501 +# * https://bugs.python.org/issue33725 +if sys.version_info >= (3,): # novm + fork_context = multiprocessing.get_context('fork') +else: + fork_context = multiprocessing + + def index_by(objects, *funcs): """Create a hierarchy of dictionaries by splitting the supplied set of objects on unique values of the supplied functions. diff --git a/lib/spack/llnl/util/tty/log.py b/lib/spack/llnl/util/tty/log.py index de5ffa8eec..97fba0592c 100644 --- a/lib/spack/llnl/util/tty/log.py +++ b/lib/spack/llnl/util/tty/log.py @@ -21,6 +21,7 @@ from six import string_types from six import StringIO import llnl.util.tty as tty +from llnl.util.lang import fork_context try: import termios @@ -430,7 +431,7 @@ class log_output(object): except BaseException: input_stream = None # just don't forward input if this fails - self.process = multiprocessing.Process( + self.process = fork_context.Process( target=_writer_daemon, args=( input_stream, read_fd, write_fd, self.echo, self.log_file, diff --git a/lib/spack/llnl/util/tty/pty.py b/lib/spack/llnl/util/tty/pty.py index 84c272a6e2..79f4ce20cf 100644 --- a/lib/spack/llnl/util/tty/pty.py +++ b/lib/spack/llnl/util/tty/pty.py @@ -24,6 +24,7 @@ import time import traceback import llnl.util.tty.log as log +from llnl.util.lang import fork_context from spack.util.executable import which @@ -233,7 +234,7 @@ class PseudoShell(object): ``minion_function``. """ - self.proc = multiprocessing.Process( + self.proc = fork_context.Process( target=PseudoShell._set_up_and_run_controller_function, args=(self.controller_function, self.minion_function, self.controller_timeout, self.sleep_time), diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 4a57dde77b..be6b2cc1ab 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -45,7 +45,7 @@ from six import StringIO import llnl.util.tty as tty from llnl.util.tty.color import cescape, colorize from llnl.util.filesystem import mkdirp, install, install_tree -from llnl.util.lang import dedupe +from llnl.util.lang import dedupe, fork_context import spack.build_systems.cmake import spack.build_systems.meson @@ -886,7 +886,7 @@ def fork(pkg, function, dirty, fake): if sys.stdin.isatty() and hasattr(sys.stdin, 'fileno'): input_stream = os.fdopen(os.dup(sys.stdin.fileno())) - p = multiprocessing.Process( + p = fork_context.Process( target=child_process, args=(child_pipe, input_stream)) p.start() diff --git a/lib/spack/spack/test/database.py b/lib/spack/spack/test/database.py index a161a22908..a97243ae9c 100644 --- a/lib/spack/spack/test/database.py +++ b/lib/spack/spack/test/database.py @@ -9,7 +9,6 @@ both in memory and in its file """ import datetime import functools -import multiprocessing import os import pytest import json @@ -24,6 +23,7 @@ from jsonschema import validate import llnl.util.lock as lk from llnl.util.tty.colify import colify +from llnl.util.lang import fork_context import spack.repo import spack.store @@ -524,7 +524,7 @@ def test_030_db_sanity_from_another_process(mutable_database): with mutable_database.write_transaction(): _mock_remove('mpileaks ^zmpi') - p = multiprocessing.Process(target=read_and_modify, args=()) + p = fork_context.Process(target=read_and_modify, args=()) p.start() p.join() diff --git a/lib/spack/spack/test/llnl/util/lock.py b/lib/spack/spack/test/llnl/util/lock.py index b2b7cf85ac..99c1eba588 100644 --- a/lib/spack/spack/test/llnl/util/lock.py +++ b/lib/spack/spack/test/llnl/util/lock.py @@ -51,13 +51,14 @@ import traceback import glob import getpass from contextlib import contextmanager -from multiprocessing import Process, Queue +from multiprocessing import Queue import pytest import llnl.util.lock as lk import llnl.util.multiproc as mp from llnl.util.filesystem import touch +from llnl.util.lang import fork_context # @@ -214,7 +215,7 @@ def local_multiproc_test(*functions, **kwargs): b = mp.Barrier(len(functions), timeout=barrier_timeout) args = (b,) + tuple(kwargs.get('extra_args', ())) - procs = [Process(target=f, args=args, name=f.__name__) + procs = [fork_context.Process(target=f, args=args, name=f.__name__) for f in functions] for p in procs: |