summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--.github/workflows/macos_python.yml8
-rw-r--r--.github/workflows/macos_unit_tests.yaml2
-rw-r--r--.github/workflows/style_and_docs.yaml2
-rw-r--r--lib/spack/llnl/util/lang.py18
-rw-r--r--lib/spack/llnl/util/tty/log.py3
-rw-r--r--lib/spack/llnl/util/tty/pty.py3
-rw-r--r--lib/spack/spack/build_environment.py4
-rw-r--r--lib/spack/spack/test/database.py4
-rw-r--r--lib/spack/spack/test/llnl/util/lock.py5
9 files changed, 35 insertions, 14 deletions
diff --git a/.github/workflows/macos_python.yml b/.github/workflows/macos_python.yml
index feb9dcef04..856d850cc8 100644
--- a/.github/workflows/macos_python.yml
+++ b/.github/workflows/macos_python.yml
@@ -27,7 +27,7 @@ jobs:
- uses: actions/checkout@v2
- uses: actions/setup-python@v2
with:
- python-version: 3.7
+ python-version: 3.8
- name: spack install
run: |
. .github/workflows/install_spack.sh
@@ -42,7 +42,7 @@ jobs:
- uses: actions/checkout@v2
- uses: actions/setup-python@v2
with:
- python-version: 3.7
+ python-version: 3.8
- name: spack install
run: |
. .github/workflows/install_spack.sh
@@ -56,7 +56,7 @@ jobs:
- uses: actions/checkout@v2
- uses: actions/setup-python@v2
with:
- python-version: 3.7
+ python-version: 3.8
- name: spack install
run: |
. .github/workflows/install_spack.sh
@@ -71,7 +71,7 @@ jobs:
- uses: actions/checkout@v2
- uses: actions/setup-python@v2
with:
- python-version: 3.7
+ python-version: 3.8
- name: spack install
run: |
. .github/workflows/install_spack.sh
diff --git a/.github/workflows/macos_unit_tests.yaml b/.github/workflows/macos_unit_tests.yaml
index 1e60f76918..24affa8327 100644
--- a/.github/workflows/macos_unit_tests.yaml
+++ b/.github/workflows/macos_unit_tests.yaml
@@ -18,7 +18,7 @@ jobs:
fetch-depth: 0
- uses: actions/setup-python@v2
with:
- python-version: 3.7
+ python-version: 3.8
- name: Install Python packages
run: |
pip install --upgrade pip six setuptools
diff --git a/.github/workflows/style_and_docs.yaml b/.github/workflows/style_and_docs.yaml
index 2277d5598f..84eb0efeab 100644
--- a/.github/workflows/style_and_docs.yaml
+++ b/.github/workflows/style_and_docs.yaml
@@ -16,7 +16,7 @@ jobs:
- uses: actions/checkout@v2
- uses: actions/setup-python@v2
with:
- python-version: 3.7
+ python-version: 3.8
- name: Install Python Packages
run: |
pip install --upgrade pip
diff --git a/lib/spack/llnl/util/lang.py b/lib/spack/llnl/util/lang.py
index e746ce096c..0bcbebba72 100644
--- a/lib/spack/llnl/util/lang.py
+++ b/lib/spack/llnl/util/lang.py
@@ -5,6 +5,7 @@
from __future__ import division
+import multiprocessing
import os
import re
import functools
@@ -19,6 +20,23 @@ import sys
ignore_modules = [r'^\.#', '~$']
+# On macOS, Python 3.8 multiprocessing now defaults to the 'spawn' start
+# method. Spack cannot currently handle this, so force the process to start
+# using the 'fork' start method.
+#
+# TODO: This solution is not ideal, as the 'fork' start method can lead to
+# crashes of the subprocess. Figure out how to make 'spawn' work.
+#
+# See:
+# * https://github.com/spack/spack/pull/18124
+# * https://docs.python.org/3/library/multiprocessing.html#contexts-and-start-methods # noqa: E501
+# * https://bugs.python.org/issue33725
+if sys.version_info >= (3,): # novm
+ fork_context = multiprocessing.get_context('fork')
+else:
+ fork_context = multiprocessing
+
+
def index_by(objects, *funcs):
"""Create a hierarchy of dictionaries by splitting the supplied
set of objects on unique values of the supplied functions.
diff --git a/lib/spack/llnl/util/tty/log.py b/lib/spack/llnl/util/tty/log.py
index de5ffa8eec..97fba0592c 100644
--- a/lib/spack/llnl/util/tty/log.py
+++ b/lib/spack/llnl/util/tty/log.py
@@ -21,6 +21,7 @@ from six import string_types
from six import StringIO
import llnl.util.tty as tty
+from llnl.util.lang import fork_context
try:
import termios
@@ -430,7 +431,7 @@ class log_output(object):
except BaseException:
input_stream = None # just don't forward input if this fails
- self.process = multiprocessing.Process(
+ self.process = fork_context.Process(
target=_writer_daemon,
args=(
input_stream, read_fd, write_fd, self.echo, self.log_file,
diff --git a/lib/spack/llnl/util/tty/pty.py b/lib/spack/llnl/util/tty/pty.py
index 84c272a6e2..79f4ce20cf 100644
--- a/lib/spack/llnl/util/tty/pty.py
+++ b/lib/spack/llnl/util/tty/pty.py
@@ -24,6 +24,7 @@ import time
import traceback
import llnl.util.tty.log as log
+from llnl.util.lang import fork_context
from spack.util.executable import which
@@ -233,7 +234,7 @@ class PseudoShell(object):
``minion_function``.
"""
- self.proc = multiprocessing.Process(
+ self.proc = fork_context.Process(
target=PseudoShell._set_up_and_run_controller_function,
args=(self.controller_function, self.minion_function,
self.controller_timeout, self.sleep_time),
diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py
index 4a57dde77b..be6b2cc1ab 100644
--- a/lib/spack/spack/build_environment.py
+++ b/lib/spack/spack/build_environment.py
@@ -45,7 +45,7 @@ from six import StringIO
import llnl.util.tty as tty
from llnl.util.tty.color import cescape, colorize
from llnl.util.filesystem import mkdirp, install, install_tree
-from llnl.util.lang import dedupe
+from llnl.util.lang import dedupe, fork_context
import spack.build_systems.cmake
import spack.build_systems.meson
@@ -886,7 +886,7 @@ def fork(pkg, function, dirty, fake):
if sys.stdin.isatty() and hasattr(sys.stdin, 'fileno'):
input_stream = os.fdopen(os.dup(sys.stdin.fileno()))
- p = multiprocessing.Process(
+ p = fork_context.Process(
target=child_process, args=(child_pipe, input_stream))
p.start()
diff --git a/lib/spack/spack/test/database.py b/lib/spack/spack/test/database.py
index a161a22908..a97243ae9c 100644
--- a/lib/spack/spack/test/database.py
+++ b/lib/spack/spack/test/database.py
@@ -9,7 +9,6 @@ both in memory and in its file
"""
import datetime
import functools
-import multiprocessing
import os
import pytest
import json
@@ -24,6 +23,7 @@ from jsonschema import validate
import llnl.util.lock as lk
from llnl.util.tty.colify import colify
+from llnl.util.lang import fork_context
import spack.repo
import spack.store
@@ -524,7 +524,7 @@ def test_030_db_sanity_from_another_process(mutable_database):
with mutable_database.write_transaction():
_mock_remove('mpileaks ^zmpi')
- p = multiprocessing.Process(target=read_and_modify, args=())
+ p = fork_context.Process(target=read_and_modify, args=())
p.start()
p.join()
diff --git a/lib/spack/spack/test/llnl/util/lock.py b/lib/spack/spack/test/llnl/util/lock.py
index b2b7cf85ac..99c1eba588 100644
--- a/lib/spack/spack/test/llnl/util/lock.py
+++ b/lib/spack/spack/test/llnl/util/lock.py
@@ -51,13 +51,14 @@ import traceback
import glob
import getpass
from contextlib import contextmanager
-from multiprocessing import Process, Queue
+from multiprocessing import Queue
import pytest
import llnl.util.lock as lk
import llnl.util.multiproc as mp
from llnl.util.filesystem import touch
+from llnl.util.lang import fork_context
#
@@ -214,7 +215,7 @@ def local_multiproc_test(*functions, **kwargs):
b = mp.Barrier(len(functions), timeout=barrier_timeout)
args = (b,) + tuple(kwargs.get('extra_args', ()))
- procs = [Process(target=f, args=args, name=f.__name__)
+ procs = [fork_context.Process(target=f, args=args, name=f.__name__)
for f in functions]
for p in procs:
generated by cgit v1.2.3-60-g2f50 (git 2.42.0) at 2024-07-02 03:25:36 +0000