diff options
author | Harmen Stoppels <me@harmenstoppels.nl> | 2024-01-18 17:28:50 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2024-01-18 17:28:50 +0100 |
commit | cb685b049d38c8cdbea6d06999254f48909e7c48 (patch) | |
tree | 38cb289e79c05a4e2f4ab361553c5f8a7108daea /lib | |
parent | 51c02be9095c407f55c8d95d8d05a7895f578914 (diff) | |
download | spack-cb685b049d38c8cdbea6d06999254f48909e7c48.tar.gz spack-cb685b049d38c8cdbea6d06999254f48909e7c48.tar.bz2 spack-cb685b049d38c8cdbea6d06999254f48909e7c48.tar.xz spack-cb685b049d38c8cdbea6d06999254f48909e7c48.zip |
oci: only push in parallel when forking (#42143)
Diffstat (limited to 'lib')
-rw-r--r-- | lib/spack/spack/cmd/buildcache.py | 35 | ||||
-rw-r--r-- | lib/spack/spack/test/conftest.py | 17 |
2 files changed, 30 insertions, 22 deletions
diff --git a/lib/spack/spack/cmd/buildcache.py b/lib/spack/spack/cmd/buildcache.py index f2d88c07bf..9203832a8e 100644 --- a/lib/spack/spack/cmd/buildcache.py +++ b/lib/spack/spack/cmd/buildcache.py @@ -7,13 +7,14 @@ import copy import glob import hashlib import json +import multiprocessing import multiprocessing.pool import os import shutil import sys import tempfile import urllib.request -from typing import Dict, List, Optional, Tuple +from typing import Dict, List, Optional, Tuple, Union import llnl.util.tty as tty from llnl.string import plural @@ -326,8 +327,30 @@ def _progress(i: int, total: int): return "" -def _make_pool(): - return multiprocessing.pool.Pool(determine_number_of_jobs(parallel=True)) +class NoPool: + def map(self, func, args): + return [func(a) for a in args] + + def starmap(self, func, args): + return [func(*a) for a in args] + + def __enter__(self): + return self + + def __exit__(self, *args): + pass + + +MaybePool = Union[multiprocessing.pool.Pool, NoPool] + + +def _make_pool() -> MaybePool: + """Can't use threading because it's unsafe, and can't use spawned processes because of globals. + That leaves only forking""" + if multiprocessing.get_start_method() == "fork": + return multiprocessing.pool.Pool(determine_number_of_jobs(parallel=True)) + else: + return NoPool() def push_fn(args): @@ -663,7 +686,7 @@ def _push_oci( base_image: Optional[ImageReference], installed_specs_with_deps: List[Spec], tmpdir: str, - pool: multiprocessing.pool.Pool, + pool: MaybePool, force: bool = False, ) -> Tuple[List[str], Dict[str, Tuple[dict, dict]], Dict[str, spack.oci.oci.Blob]]: """Push specs to an OCI registry @@ -779,9 +802,7 @@ def _config_from_tag(image_ref: ImageReference, tag: str) -> Optional[dict]: return config if "spec" in config else None -def _update_index_oci( - image_ref: ImageReference, tmpdir: str, pool: multiprocessing.pool.Pool -) -> None: +def _update_index_oci(image_ref: ImageReference, tmpdir: str, pool: MaybePool) -> None: response = spack.oci.opener.urlopen(urllib.request.Request(url=image_ref.tags_url())) spack.oci.opener.ensure_status(response, 200) tags = json.load(response)["tags"] diff --git a/lib/spack/spack/test/conftest.py b/lib/spack/spack/test/conftest.py index 22f2e95a7c..d5c05e2bfe 100644 --- a/lib/spack/spack/test/conftest.py +++ b/lib/spack/spack/test/conftest.py @@ -1950,23 +1950,10 @@ def pytest_runtest_setup(item): pytest.skip(*not_on_windows_marker.args) -class MockPool: - def map(self, func, args): - return [func(a) for a in args] - - def starmap(self, func, args): - return [func(*a) for a in args] - - def __enter__(self): - return self - - def __exit__(self, *args): - pass - - @pytest.fixture(scope="function") def disable_parallel_buildcache_push(monkeypatch): - monkeypatch.setattr(spack.cmd.buildcache, "_make_pool", MockPool) + """Disable process pools in tests.""" + monkeypatch.setattr(spack.cmd.buildcache, "_make_pool", spack.cmd.buildcache.NoPool) def _root_path(x, y, *, path): |