summaryrefslogtreecommitdiff
path: root/lib
diff options
context:
space:
mode:
authorHarmen Stoppels <me@harmenstoppels.nl>2024-01-18 17:28:50 +0100
committerGitHub <noreply@github.com>2024-01-18 17:28:50 +0100
commitcb685b049d38c8cdbea6d06999254f48909e7c48 (patch)
tree38cb289e79c05a4e2f4ab361553c5f8a7108daea /lib
parent51c02be9095c407f55c8d95d8d05a7895f578914 (diff)
downloadspack-cb685b049d38c8cdbea6d06999254f48909e7c48.tar.gz
spack-cb685b049d38c8cdbea6d06999254f48909e7c48.tar.bz2
spack-cb685b049d38c8cdbea6d06999254f48909e7c48.tar.xz
spack-cb685b049d38c8cdbea6d06999254f48909e7c48.zip
oci: only push in parallel when forking (#42143)
Diffstat (limited to 'lib')
-rw-r--r--lib/spack/spack/cmd/buildcache.py35
-rw-r--r--lib/spack/spack/test/conftest.py17
2 files changed, 30 insertions, 22 deletions
diff --git a/lib/spack/spack/cmd/buildcache.py b/lib/spack/spack/cmd/buildcache.py
index f2d88c07bf..9203832a8e 100644
--- a/lib/spack/spack/cmd/buildcache.py
+++ b/lib/spack/spack/cmd/buildcache.py
@@ -7,13 +7,14 @@ import copy
import glob
import hashlib
import json
+import multiprocessing
import multiprocessing.pool
import os
import shutil
import sys
import tempfile
import urllib.request
-from typing import Dict, List, Optional, Tuple
+from typing import Dict, List, Optional, Tuple, Union
import llnl.util.tty as tty
from llnl.string import plural
@@ -326,8 +327,30 @@ def _progress(i: int, total: int):
return ""
-def _make_pool():
- return multiprocessing.pool.Pool(determine_number_of_jobs(parallel=True))
+class NoPool:
+ def map(self, func, args):
+ return [func(a) for a in args]
+
+ def starmap(self, func, args):
+ return [func(*a) for a in args]
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, *args):
+ pass
+
+
+MaybePool = Union[multiprocessing.pool.Pool, NoPool]
+
+
+def _make_pool() -> MaybePool:
+ """Can't use threading because it's unsafe, and can't use spawned processes because of globals.
+ That leaves only forking"""
+ if multiprocessing.get_start_method() == "fork":
+ return multiprocessing.pool.Pool(determine_number_of_jobs(parallel=True))
+ else:
+ return NoPool()
def push_fn(args):
@@ -663,7 +686,7 @@ def _push_oci(
base_image: Optional[ImageReference],
installed_specs_with_deps: List[Spec],
tmpdir: str,
- pool: multiprocessing.pool.Pool,
+ pool: MaybePool,
force: bool = False,
) -> Tuple[List[str], Dict[str, Tuple[dict, dict]], Dict[str, spack.oci.oci.Blob]]:
"""Push specs to an OCI registry
@@ -779,9 +802,7 @@ def _config_from_tag(image_ref: ImageReference, tag: str) -> Optional[dict]:
return config if "spec" in config else None
-def _update_index_oci(
- image_ref: ImageReference, tmpdir: str, pool: multiprocessing.pool.Pool
-) -> None:
+def _update_index_oci(image_ref: ImageReference, tmpdir: str, pool: MaybePool) -> None:
response = spack.oci.opener.urlopen(urllib.request.Request(url=image_ref.tags_url()))
spack.oci.opener.ensure_status(response, 200)
tags = json.load(response)["tags"]
diff --git a/lib/spack/spack/test/conftest.py b/lib/spack/spack/test/conftest.py
index 22f2e95a7c..d5c05e2bfe 100644
--- a/lib/spack/spack/test/conftest.py
+++ b/lib/spack/spack/test/conftest.py
@@ -1950,23 +1950,10 @@ def pytest_runtest_setup(item):
pytest.skip(*not_on_windows_marker.args)
-class MockPool:
- def map(self, func, args):
- return [func(a) for a in args]
-
- def starmap(self, func, args):
- return [func(*a) for a in args]
-
- def __enter__(self):
- return self
-
- def __exit__(self, *args):
- pass
-
-
@pytest.fixture(scope="function")
def disable_parallel_buildcache_push(monkeypatch):
- monkeypatch.setattr(spack.cmd.buildcache, "_make_pool", MockPool)
+ """Disable process pools in tests."""
+ monkeypatch.setattr(spack.cmd.buildcache, "_make_pool", spack.cmd.buildcache.NoPool)
def _root_path(x, y, *, path):