summaryrefslogtreecommitdiff
path: root/lib
diff options
context:
space:
mode:
Diffstat (limited to 'lib')
-rw-r--r--lib/spack/llnl/util/lang.py8
-rw-r--r--lib/spack/llnl/util/tty/log.py215
-rw-r--r--lib/spack/llnl/util/tty/pty.py3
-rw-r--r--lib/spack/spack/build_environment.py147
-rw-r--r--lib/spack/spack/fetch_strategy.py11
-rw-r--r--lib/spack/spack/hooks/sbang.py10
-rw-r--r--lib/spack/spack/installer.py233
-rw-r--r--lib/spack/spack/package.py10
-rw-r--r--lib/spack/spack/repo.py4
-rw-r--r--lib/spack/spack/spec.py11
-rw-r--r--lib/spack/spack/stage.py12
-rw-r--r--lib/spack/spack/store.py19
-rw-r--r--lib/spack/spack/subprocess_context.py149
-rw-r--r--lib/spack/spack/test/bindist.py26
-rw-r--r--lib/spack/spack/test/cmd/dev_build.py24
-rw-r--r--lib/spack/spack/test/cmd/install.py10
-rw-r--r--lib/spack/spack/test/conftest.py200
-rw-r--r--lib/spack/spack/test/database.py19
-rw-r--r--lib/spack/spack/test/install.py84
-rw-r--r--lib/spack/spack/test/installer.py2
-rw-r--r--lib/spack/spack/test/llnl/util/lock.py334
-rw-r--r--lib/spack/spack/test/llnl/util/tty/log.py5
-rw-r--r--lib/spack/spack/test/sbang.py35
-rw-r--r--lib/spack/spack/util/pattern.py21
-rw-r--r--lib/spack/spack/util/prefix.py6
25 files changed, 1041 insertions, 557 deletions
diff --git a/lib/spack/llnl/util/lang.py b/lib/spack/llnl/util/lang.py
index 0bcbebba72..b9b034d4d4 100644
--- a/lib/spack/llnl/util/lang.py
+++ b/lib/spack/llnl/util/lang.py
@@ -568,6 +568,12 @@ class Singleton(object):
return self._instance
def __getattr__(self, name):
+ # When unpickling Singleton objects, the 'instance' attribute may be
+ # requested but not yet set. The final 'getattr' line here requires
+ # 'instance'/'_instance' to be defined or it will enter an infinite
+ # loop, so protect against that here.
+ if name in ['_instance', 'instance']:
+ raise AttributeError()
return getattr(self.instance, name)
def __getitem__(self, name):
@@ -596,6 +602,8 @@ class LazyReference(object):
self.ref_function = ref_function
def __getattr__(self, name):
+ if name == 'ref_function':
+ raise AttributeError()
return getattr(self.ref_function(), name)
def __getitem__(self, name):
diff --git a/lib/spack/llnl/util/tty/log.py b/lib/spack/llnl/util/tty/log.py
index 97fba0592c..658de5c596 100644
--- a/lib/spack/llnl/util/tty/log.py
+++ b/lib/spack/llnl/util/tty/log.py
@@ -21,7 +21,6 @@ from six import string_types
from six import StringIO
import llnl.util.tty as tty
-from llnl.util.lang import fork_context
try:
import termios
@@ -237,6 +236,8 @@ class keyboard_input(object):
"""If termios was available, restore old settings."""
if self.old_cfg:
self._restore_default_terminal_settings()
+ if sys.version_info >= (3,):
+ atexit.unregister(self._restore_default_terminal_settings)
# restore SIGSTP and SIGCONT handlers
if self.old_handlers:
@@ -288,6 +289,109 @@ def _file_descriptors_work(*streams):
return False
+class FileWrapper(object):
+ """Represents a file. Can be an open stream, a path to a file (not opened
+ yet), or neither. When unwrapped, it returns an open file (or file-like)
+ object.
+ """
+ def __init__(self, file_like):
+ # This records whether the file-like object returned by "unwrap" is
+ # purely in-memory. In that case a subprocess will need to explicitly
+ # transmit the contents to the parent.
+ self.write_in_parent = False
+
+ self.file_like = file_like
+
+ if isinstance(file_like, string_types):
+ self.open = True
+ elif _file_descriptors_work(file_like):
+ self.open = False
+ else:
+ self.file_like = None
+ self.open = True
+ self.write_in_parent = True
+
+ self.file = None
+
+ def unwrap(self):
+ if self.open:
+ if self.file_like:
+ self.file = open(self.file_like, 'w')
+ else:
+ self.file = StringIO()
+ return self.file
+ else:
+ # We were handed an already-open file object. In this case we also
+ # will not actually close the object when requested to.
+ return self.file_like
+
+ def close(self):
+ if self.file:
+ self.file.close()
+
+
+class MultiProcessFd(object):
+ """Return an object which stores a file descriptor and can be passed as an
+ argument to a function run with ``multiprocessing.Process``, such that
+ the file descriptor is available in the subprocess."""
+ def __init__(self, fd):
+ self._connection = None
+ self._fd = None
+ if sys.version_info >= (3, 8):
+ self._connection = multiprocessing.connection.Connection(fd)
+ else:
+ self._fd = fd
+
+ @property
+ def fd(self):
+ if self._connection:
+ return self._connection._handle
+ else:
+ return self._fd
+
+ def close(self):
+ if self._connection:
+ self._connection.close()
+ else:
+ os.close(self._fd)
+
+
+def close_connection_and_file(multiprocess_fd, file):
+ # MultiprocessFd is intended to transmit a FD
+ # to a child process, this FD is then opened to a Python File object
+ # (using fdopen). In >= 3.8, MultiprocessFd encapsulates a
+ # multiprocessing.connection.Connection; Connection closes the FD
+ # when it is deleted, and prints a warning about duplicate closure if
+ # it is not explicitly closed. In < 3.8, MultiprocessFd encapsulates a
+ # simple FD; closing the FD here appears to conflict with
+ # closure of the File object (in < 3.8 that is). Therefore this needs
+ # to choose whether to close the File or the Connection.
+ if sys.version_info >= (3, 8):
+ multiprocess_fd.close()
+ else:
+ file.close()
+
+
+@contextmanager
+def replace_environment(env):
+ """Replace the current environment (`os.environ`) with `env`.
+
+ If `env` is empty (or None), this unsets all current environment
+ variables.
+ """
+ env = env or {}
+ old_env = os.environ.copy()
+ try:
+ os.environ.clear()
+ for name, val in env.items():
+ os.environ[name] = val
+ yield
+ finally:
+ os.environ.clear()
+ for name, val in old_env.items():
+ os.environ[name] = val
+
+
class log_output(object):
"""Context manager that logs its output to a file.
@@ -324,7 +428,8 @@ class log_output(object):
work within test frameworks like nose and pytest.
"""
- def __init__(self, file_like=None, echo=False, debug=0, buffer=False):
+ def __init__(self, file_like=None, echo=False, debug=0, buffer=False,
+ env=None):
"""Create a new output log context manager.
Args:
@@ -352,6 +457,7 @@ class log_output(object):
self.echo = echo
self.debug = debug
self.buffer = buffer
+ self.env = env # the environment to use for _writer_daemon
self._active = False # used to prevent re-entry
@@ -393,18 +499,7 @@ class log_output(object):
"file argument must be set by either __init__ or __call__")
# set up a stream for the daemon to write to
- self.close_log_in_parent = True
- self.write_log_in_parent = False
- if isinstance(self.file_like, string_types):
- self.log_file = open(self.file_like, 'w')
-
- elif _file_descriptors_work(self.file_like):
- self.log_file = self.file_like
- self.close_log_in_parent = False
-
- else:
- self.log_file = StringIO()
- self.write_log_in_parent = True
+ self.log_file = FileWrapper(self.file_like)
# record parent color settings before redirecting. We do this
# because color output depends on whether the *original* stdout
@@ -419,6 +514,8 @@ class log_output(object):
# OS-level pipe for redirecting output to logger
read_fd, write_fd = os.pipe()
+ read_multiprocess_fd = MultiProcessFd(read_fd)
+
# Multiprocessing pipe for communication back from the daemon
# Currently only used to save echo value between uses
self.parent_pipe, child_pipe = multiprocessing.Pipe()
@@ -427,24 +524,28 @@ class log_output(object):
try:
# need to pass this b/c multiprocessing closes stdin in child.
try:
- input_stream = os.fdopen(os.dup(sys.stdin.fileno()))
+ input_multiprocess_fd = MultiProcessFd(
+ os.dup(sys.stdin.fileno())
+ )
except BaseException:
- input_stream = None # just don't forward input if this fails
-
- self.process = fork_context.Process(
- target=_writer_daemon,
- args=(
- input_stream, read_fd, write_fd, self.echo, self.log_file,
- child_pipe
+ # just don't forward input if this fails
+ input_multiprocess_fd = None
+
+ with replace_environment(self.env):
+ self.process = multiprocessing.Process(
+ target=_writer_daemon,
+ args=(
+ input_multiprocess_fd, read_multiprocess_fd, write_fd,
+ self.echo, self.log_file, child_pipe
+ )
)
- )
- self.process.daemon = True # must set before start()
- self.process.start()
- os.close(read_fd) # close in the parent process
+ self.process.daemon = True # must set before start()
+ self.process.start()
finally:
- if input_stream:
- input_stream.close()
+ if input_multiprocess_fd:
+ input_multiprocess_fd.close()
+ read_multiprocess_fd.close()
# Flush immediately before redirecting so that anything buffered
# goes to the original stream
@@ -515,18 +616,21 @@ class log_output(object):
sys.stderr = self._saved_stderr
# print log contents in parent if needed.
- if self.write_log_in_parent:
+ if self.log_file.write_in_parent:
string = self.parent_pipe.recv()
self.file_like.write(string)
- if self.close_log_in_parent:
- self.log_file.close()
-
# recover and store echo settings from the child before it dies
- self.echo = self.parent_pipe.recv()
-
- # join the daemon process. The daemon will quit automatically
- # when the write pipe is closed; we just wait for it here.
+ try:
+ self.echo = self.parent_pipe.recv()
+ except EOFError:
+ # This may occur if some exception prematurely terminates the
+ # _writer_daemon. An exception will have already been generated.
+ pass
+
+ # now that the write pipe is closed (in this __exit__, when we restore
+ # stdout with dup2), the logger daemon process loop will terminate. We
+ # wait for that here.
self.process.join()
# restore old color and debug settings
@@ -555,7 +659,8 @@ class log_output(object):
sys.stdout.flush()
-def _writer_daemon(stdin, read_fd, write_fd, echo, log_file, control_pipe):
+def _writer_daemon(stdin_multiprocess_fd, read_multiprocess_fd, write_fd, echo,
+ log_file_wrapper, control_pipe):
"""Daemon used by ``log_output`` to write to a log file and to ``stdout``.
The daemon receives output from the parent process and writes it both
@@ -592,26 +697,39 @@ def _writer_daemon(stdin, read_fd, write_fd, echo, log_file, control_pipe):
``StringIO`` in the parent. This is mainly for testing.
Arguments:
- stdin (stream): input from the terminal
- read_fd (int): pipe for reading from parent's redirected stdout
- write_fd (int): parent's end of the pipe will write to (will be
- immediately closed by the writer daemon)
+ stdin_multiprocess_fd (int): input from the terminal
+ read_multiprocess_fd (int): pipe for reading from parent's redirected
+ stdout
echo (bool): initial echo setting -- controlled by user and
preserved across multiple writer daemons
- log_file (file-like): file to log all output
+ log_file_wrapper (FileWrapper): file to log all output
control_pipe (Pipe): multiprocessing pipe on which to send control
information to the parent
"""
+ # If this process was forked, then it will inherit file descriptors from
+ # the parent process. This process depends on closing all instances of
+ # write_fd to terminate the reading loop, so we close the file descriptor
+ # here. Forking is the process spawning method everywhere except Mac OS
+ # for Python >= 3.8 and on Windows
+ if sys.version_info < (3, 8) or sys.platform != 'darwin':
+ os.close(write_fd)
+
# Use line buffering (3rd param = 1) since Python 3 has a bug
# that prevents unbuffered text I/O.
- in_pipe = os.fdopen(read_fd, 'r', 1)
- os.close(write_fd)
+ in_pipe = os.fdopen(read_multiprocess_fd.fd, 'r', 1)
+
+ if stdin_multiprocess_fd:
+ stdin = os.fdopen(stdin_multiprocess_fd.fd)
+ else:
+ stdin = None
# list of streams to select from
istreams = [in_pipe, stdin] if stdin else [in_pipe]
force_echo = False # parent can force echo for certain output
+ log_file = log_file_wrapper.unwrap()
+
try:
with keyboard_input(stdin) as kb:
while True:
@@ -672,10 +790,13 @@ def _writer_daemon(stdin, read_fd, write_fd, echo, log_file, control_pipe):
# send written data back to parent if we used a StringIO
if isinstance(log_file, StringIO):
control_pipe.send(log_file.getvalue())
- log_file.close()
+ log_file_wrapper.close()
+ close_connection_and_file(read_multiprocess_fd, in_pipe)
+ if stdin_multiprocess_fd:
+ close_connection_and_file(stdin_multiprocess_fd, stdin)
- # send echo value back to the parent so it can be preserved.
- control_pipe.send(echo)
+ # send echo value back to the parent so it can be preserved.
+ control_pipe.send(echo)
def _retry(function):
diff --git a/lib/spack/llnl/util/tty/pty.py b/lib/spack/llnl/util/tty/pty.py
index 79f4ce20cf..84c272a6e2 100644
--- a/lib/spack/llnl/util/tty/pty.py
+++ b/lib/spack/llnl/util/tty/pty.py
@@ -24,7 +24,6 @@ import time
import traceback
import llnl.util.tty.log as log
-from llnl.util.lang import fork_context
from spack.util.executable import which
@@ -234,7 +233,7 @@ class PseudoShell(object):
``minion_function``.
"""
- self.proc = fork_context.Process(
+ self.proc = multiprocessing.Process(
target=PseudoShell._set_up_and_run_controller_function,
args=(self.controller_function, self.minion_function,
self.controller_timeout, self.sleep_time),
diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py
index be6b2cc1ab..a1501034d4 100644
--- a/lib/spack/spack/build_environment.py
+++ b/lib/spack/spack/build_environment.py
@@ -45,7 +45,8 @@ from six import StringIO
import llnl.util.tty as tty
from llnl.util.tty.color import cescape, colorize
from llnl.util.filesystem import mkdirp, install, install_tree
-from llnl.util.lang import dedupe, fork_context
+from llnl.util.lang import dedupe
+from llnl.util.tty.log import MultiProcessFd
import spack.build_systems.cmake
import spack.build_systems.meson
@@ -54,6 +55,7 @@ import spack.main
import spack.paths
import spack.schema.environment
import spack.store
+import spack.subprocess_context
import spack.architecture as arch
from spack.util.string import plural
from spack.util.environment import (
@@ -803,13 +805,72 @@ def modifications_from_dependencies(spec, context):
return env
-def fork(pkg, function, dirty, fake):
- """Fork a child process to do part of a spack build.
+def _setup_pkg_and_run(serialized_pkg, function, kwargs, child_pipe,
+ input_multiprocess_fd):
+
+ try:
+ # We are in the child process. Python sets sys.stdin to
+ # open(os.devnull) to prevent our process and its parent from
+ # simultaneously reading from the original stdin. But, we assume
+ # that the parent process is not going to read from it till we
+ # are done with the child, so we undo Python's precaution.
+ if input_multiprocess_fd is not None:
+ sys.stdin = os.fdopen(input_multiprocess_fd.fd)
+
+ pkg = serialized_pkg.restore()
+
+ if not kwargs.get('fake', False):
+ kwargs['unmodified_env'] = os.environ.copy()
+ setup_package(pkg, dirty=kwargs.get('dirty', False))
+ return_value = function(pkg, kwargs)
+ child_pipe.send(return_value)
+
+ except StopPhase as e:
+ # Do not create a full ChildError from this, it's not an error
+ # it's a control statement.
+ child_pipe.send(e)
+ except BaseException:
+ # catch ANYTHING that goes wrong in the child process
+ exc_type, exc, tb = sys.exc_info()
+
+ # Need to unwind the traceback in the child because traceback
+ # objects can't be sent to the parent.
+ tb_string = traceback.format_exc()
+
+ # build up some context from the offending package so we can
+ # show that, too.
+ package_context = get_package_context(tb)
+
+ build_log = None
+ try:
+ if hasattr(pkg, 'log_path'):
+ build_log = pkg.log_path
+ except NameError:
+ # 'pkg' is not defined yet
+ pass
+
+ # make a pickleable exception to send to parent.
+ msg = "%s: %s" % (exc_type.__name__, str(exc))
+
+ ce = ChildError(msg,
+ exc_type.__module__,
+ exc_type.__name__,
+ tb_string, build_log, package_context)
+ child_pipe.send(ce)
+
+ finally:
+ child_pipe.close()
+ if input_multiprocess_fd is not None:
+ input_multiprocess_fd.close()
+
+
+def start_build_process(pkg, function, kwargs):
+ """Create a child process to do part of a spack build.
Args:
pkg (PackageBase): package whose environment we should set up the
- forked process for.
+ child process for.
function (callable): argless function to run in the child
process.
dirty (bool): If True, do NOT clean the environment before
@@ -820,9 +881,9 @@ def fork(pkg, function, dirty, fake):
def child_fun():
# do stuff
- build_env.fork(pkg, child_fun)
+ build_env.start_build_process(pkg, child_fun)
- Forked processes are run with the build environment set up by
+ The child process is run with the build environment set up by
spack.build_environment. This allows package authors to have full
control over the environment, etc. without affecting other builds
that might be executed in the same spack call.
@@ -830,64 +891,36 @@ def fork(pkg, function, dirty, fake):
If something goes wrong, the child process catches the error and
passes it to the parent wrapped in a ChildError. The parent is
expected to handle (or re-raise) the ChildError.
- """
-
- def child_process(child_pipe, input_stream):
- # We are in the child process. Python sets sys.stdin to
- # open(os.devnull) to prevent our process and its parent from
- # simultaneously reading from the original stdin. But, we assume
- # that the parent process is not going to read from it till we
- # are done with the child, so we undo Python's precaution.
- if input_stream is not None:
- sys.stdin = input_stream
- try:
- if not fake:
- setup_package(pkg, dirty=dirty)
- return_value = function()
- child_pipe.send(return_value)
-
- except StopPhase as e:
- # Do not create a full ChildError from this, it's not an error
- # it's a control statement.
- child_pipe.send(e)
- except BaseException:
- # catch ANYTHING that goes wrong in the child process
- exc_type, exc, tb = sys.exc_info()
-
- # Need to unwind the traceback in the child because traceback
- # objects can't be sent to the parent.
- tb_string = traceback.format_exc()
-
- # build up some context from the offending package so we can
- # show that, too.
- package_context = get_package_context(tb)
-
- build_log = None
- if hasattr(pkg, 'log_path'):
- build_log = pkg.log_path
+ This uses `multiprocessing.Process` to create the child process. The
+ mechanism used to create the process differs on different operating
+ systems and for different versions of Python. In some cases "fork"
+ is used (i.e. the "fork" system call) and some cases it starts an
+ entirely new Python interpreter process (in the docs this is referred
+ to as the "spawn" start method). Breaking it down by OS:
- # make a pickleable exception to send to parent.
- msg = "%s: %s" % (exc_type.__name__, str(exc))
+ - Linux always uses fork.
+ - Mac OS uses fork before Python 3.8 and "spawn" for 3.8 and after.
+ - Windows always uses the "spawn" start method.
- ce = ChildError(msg,
- exc_type.__module__,
- exc_type.__name__,
- tb_string, build_log, package_context)
- child_pipe.send(ce)
+ For more information on `multiprocessing` child process creation
+ mechanisms, see https://docs.python.org/3/library/multiprocessing.html#contexts-and-start-methods
+ """
+ parent_pipe, child_pipe = multiprocessing.Pipe()
+ input_multiprocess_fd = None
- finally:
- child_pipe.close()
+ serialized_pkg = spack.subprocess_context.PackageInstallContext(pkg)
- parent_pipe, child_pipe = multiprocessing.Pipe()
- input_stream = None
try:
# Forward sys.stdin when appropriate, to allow toggling verbosity
if sys.stdin.isatty() and hasattr(sys.stdin, 'fileno'):
- input_stream = os.fdopen(os.dup(sys.stdin.fileno()))
+ input_fd = os.dup(sys.stdin.fileno())
+ input_multiprocess_fd = MultiProcessFd(input_fd)
- p = fork_context.Process(
- target=child_process, args=(child_pipe, input_stream))
+ p = multiprocessing.Process(
+ target=_setup_pkg_and_run,
+ args=(serialized_pkg, function, kwargs, child_pipe,
+ input_multiprocess_fd))
p.start()
except InstallError as e:
@@ -896,8 +929,8 @@ def fork(pkg, function, dirty, fake):
finally:
# Close the input stream in the parent process
- if input_stream is not None:
- input_stream.close()
+ if input_multiprocess_fd is not None:
+ input_multiprocess_fd.close()
child_result = parent_pipe.recv()
p.join()
diff --git a/lib/spack/spack/fetch_strategy.py b/lib/spack/spack/fetch_strategy.py
index c407f6908e..f075b893ae 100644
--- a/lib/spack/spack/fetch_strategy.py
+++ b/lib/spack/spack/fetch_strategy.py
@@ -214,14 +214,17 @@ class BundleFetchStrategy(FetchStrategy):
"""BundlePackages don't have a mirror id."""
-@pattern.composite(interface=FetchStrategy)
-class FetchStrategyComposite(object):
+class FetchStrategyComposite(pattern.Composite):
"""Composite for a FetchStrategy object.
-
- Implements the GoF composite pattern.
"""
matches = FetchStrategy.matches
+ def __init__(self):
+ super(FetchStrategyComposite, self).__init__([
+ 'fetch', 'check', 'expand', 'reset', 'archive', 'cachable',
+ 'mirror_id'
+ ])
+
def source_id(self):
component_ids = tuple(i.source_id() for i in self)
if all(component_ids):
diff --git a/lib/spack/spack/hooks/sbang.py b/lib/spack/spack/hooks/sbang.py
index 12a2cbf6ef..d3a6174948 100644
--- a/lib/spack/spack/hooks/sbang.py
+++ b/lib/spack/spack/hooks/sbang.py
@@ -17,9 +17,13 @@ import spack.paths
import spack.store
-#: Character limit for shebang line. Using Linux's 127 characters
-#: here, as it is the shortest I could find on a modern OS.
-shebang_limit = 127
+#: OS-imposed character limit for shebang line: 127 for Linux; 511 for Mac.
+#: Different Linux distributions have different limits, but 127 is the
+#: smallest among all modern versions.
+if sys.platform == 'darwin':
+ shebang_limit = 511
+else:
+ shebang_limit = 127
def sbang_install_path():
diff --git a/lib/spack/spack/installer.py b/lib/spack/spack/installer.py
index 6494f6d922..51919f3729 100644
--- a/lib/spack/spack/installer.py
+++ b/lib/spack/spack/installer.py
@@ -1056,15 +1056,9 @@ class PackageInstaller(object):
task (BuildTask): the installation build task for a package"""
cache_only = kwargs.get('cache_only', False)
- dirty = kwargs.get('dirty', False)
- fake = kwargs.get('fake', False)
- install_source = kwargs.get('install_source', False)
- keep_stage = kwargs.get('keep_stage', False)
- skip_patch = kwargs.get('skip_patch', False)
tests = kwargs.get('tests', False)
unsigned = kwargs.get('unsigned', False)
use_cache = kwargs.get('use_cache', True)
- verbose = kwargs.get('verbose', False)
full_hash_match = kwargs.get('full_hash_match', False)
pkg = task.pkg
@@ -1087,110 +1081,6 @@ class PackageInstaller(object):
pkg.run_tests = (tests is True or tests and pkg.name in tests)
- pid = '{0}: '.format(self.pid) if tty.show_pid() else ''
- pre = '{0}{1}:'.format(pid, pkg.name)
-
- def build_process():
- """
- This function implements the process forked for each build.
-
- It has its own process and python module space set up by
- build_environment.fork().
-
- This function's return value is returned to the parent process.
- """
- start_time = time.time()
- if not fake:
- if not skip_patch:
- pkg.do_patch()
- else:
- pkg.do_stage()
-
- pkg_id = package_id(pkg)
- tty.debug('{0} Building {1} [{2}]'
- .format(pre, pkg_id, pkg.build_system_class))
-
- # get verbosity from do_install() parameter or saved value
- echo = verbose
- if spack.package.PackageBase._verbose is not None:
- echo = spack.package.PackageBase._verbose
-
- pkg.stage.keep = keep_stage
-
- # parent process already has a prefix write lock
- with pkg.stage:
- # Run the pre-install hook in the child process after
- # the directory is created.
- spack.hooks.pre_install(pkg.spec)
- if fake:
- _do_fake_install(pkg)
- else:
- source_path = pkg.stage.source_path
- if install_source and os.path.isdir(source_path):
- src_target = os.path.join(pkg.spec.prefix, 'share',
- pkg.name, 'src')
- tty.debug('{0} Copying source to {1}'
- .format(pre, src_target))
- fs.install_tree(pkg.stage.source_path, src_target)
-
- # Do the real install in the source directory.
- with fs.working_dir(pkg.stage.source_path):
- # Save the build environment in a file before building.
- dump_environment(pkg.env_path)
-
- for attr in ('configure_args', 'cmake_args'):
- try:
- configure_args = getattr(pkg, attr)()
- configure_args = ' '.join(configure_args)
-
- with open(pkg.configure_args_path, 'w') as \
- args_file:
- args_file.write(configure_args)
-
- break
- except Exception:
- pass
-
- # cache debug settings
- debug_level = tty.debug_level()
-
- # Spawn a daemon that reads from a pipe and redirects
- # everything to log_path
- with log_output(pkg.log_path, echo, True) as logger:
- for phase_name, phase_attr in zip(
- pkg.phases, pkg._InstallPhase_phases):
-
- with logger.force_echo():
- inner_debug_level = tty.debug_level()
- tty.set_debug(debug_level)
- tty.msg("{0} Executing phase: '{1}'"
- .format(pre, phase_name))
- tty.set_debug(inner_debug_level)
-
- # Redirect stdout and stderr to daemon pipe
- phase = getattr(pkg, phase_attr)
- phase(pkg.spec, pkg.prefix)
-
- echo = logger.echo
- log(pkg)
-
- # Run post install hooks before build stage is removed.
- spack.hooks.post_install(pkg.spec)
-
- # Stop the timer
- pkg._total_time = time.time() - start_time
- build_time = pkg._total_time - pkg._fetch_time
-
- tty.debug('{0} Successfully installed {1}'
- .format(pre, pkg_id),
- 'Fetch: {0}. Build: {1}. Total: {2}.'
- .format(_hms(pkg._fetch_time), _hms(build_time),
- _hms(pkg._total_time)))
- _print_installed_pkg(pkg.prefix)
-
- # preserve verbosity across runs
- return echo
-
# hook that allows tests to inspect the Package before installation
# see unit_test_check() docs.
if not pkg.unit_test_check():
@@ -1199,10 +1089,12 @@ class PackageInstaller(object):
try:
self._setup_install_dir(pkg)
- # Fork a child to do the actual installation.
+ # Create a child process to do the actual installation.
# Preserve verbosity settings across installs.
- spack.package.PackageBase._verbose = spack.build_environment.fork(
- pkg, build_process, dirty=dirty, fake=fake)
+ spack.package.PackageBase._verbose = (
+ spack.build_environment.start_build_process(
+ pkg, build_process, kwargs)
+ )
# Note: PARENT of the build process adds the new package to
# the database, so that we don't need to re-read from file.
@@ -1216,9 +1108,9 @@ class PackageInstaller(object):
except spack.build_environment.StopPhase as e:
# A StopPhase exception means that do_install was asked to
# stop early from clients, and is not an error at this point
- pre = '{0}'.format(self.pid) if tty.show_pid() else ''
+ pid = '{0}: '.format(pkg.pid) if tty.show_pid() else ''
tty.debug('{0}{1}'.format(pid, str(e)))
- tty.debug('Package stage directory : {0}'
+ tty.debug('Package stage directory: {0}'
.format(pkg.stage.source_path))
_install_task.__doc__ += install_args_docstring
@@ -1666,6 +1558,117 @@ class PackageInstaller(object):
return self.pkg.spec
+def build_process(pkg, kwargs):
+ """Perform the installation/build of the package.
+
+ This runs in a separate child process, and has its own process and
+ python module space set up by build_environment.start_build_process().
+
+ This function's return value is returned to the parent process.
+ """
+ keep_stage = kwargs.get('keep_stage', False)
+ install_source = kwargs.get('install_source', False)
+ skip_patch = kwargs.get('skip_patch', False)
+ verbose = kwargs.get('verbose', False)
+ fake = kwargs.get('fake', False)
+ unmodified_env = kwargs.get('unmodified_env', {})
+
+ start_time = time.time()
+ if not fake:
+ if not skip_patch:
+ pkg.do_patch()
+ else:
+ pkg.do_stage()
+
+ pid = '{0}: '.format(pkg.pid) if tty.show_pid() else ''
+ pre = '{0}{1}:'.format(pid, pkg.name)
+ pkg_id = package_id(pkg)
+
+ tty.debug('{0} Building {1} [{2}]'
+ .format(pre, pkg_id, pkg.build_system_class))
+
+ # get verbosity from do_install() parameter or saved value
+ echo = verbose
+ if spack.package.PackageBase._verbose is not None:
+ echo = spack.package.PackageBase._verbose
+
+ pkg.stage.keep = keep_stage
+ with pkg.stage:
+ # Run the pre-install hook in the child process after
+ # the directory is created.
+ spack.hooks.pre_install(pkg.spec)
+ if fake:
+ _do_fake_install(pkg)
+ else:
+ source_path = pkg.stage.source_path
+ if install_source and os.path.isdir(source_path):
+ src_target = os.path.join(pkg.spec.prefix, 'share',
+ pkg.name, 'src')
+ tty.debug('{0} Copying source to {1}'
+ .format(pre, src_target))
+ fs.install_tree(pkg.stage.source_path, src_target)
+
+ # Do the real install in the source directory.
+ with fs.working_dir(pkg.stage.source_path):
+ # Save the build environment in a file before building.
+ dump_environment(pkg.env_path)
+
+ for attr in ('configure_args', 'cmake_args'):
+ try:
+ configure_args = getattr(pkg, attr)()
+ configure_args = ' '.join(configure_args)
+
+ with open(pkg.configure_args_path, 'w') as \
+ args_file:
+ args_file.write(configure_args)
+
+ break
+ except Exception:
+ pass
+
+ # cache debug settings
+ debug_level = tty.debug_level()
+
+ # Spawn a daemon that reads from a pipe and redirects
+ # everything to log_path
+ with log_output(pkg.log_path, echo, True,
+ env=unmodified_env) as logger:
+
+ for phase_name, phase_attr in zip(
+ pkg.phases, pkg._InstallPhase_phases):
+
+ with logger.force_echo():
+ inner_debug_level = tty.debug_level()
+ tty.set_debug(debug_level)
+ tty.msg("{0} Executing phase: '{1}'"
+ .format(pre, phase_name))
+ tty.set_debug(inner_debug_level)
+
+ # Redirect stdout and stderr to daemon pipe
+ phase = getattr(pkg, phase_attr)
+ phase(pkg.spec, pkg.prefix)
+
+ echo = logger.echo
+ log(pkg)
+
+ # Run post install hooks before build stage is removed.
+ spack.hooks.post_install(pkg.spec)
+
+ # Stop the timer
+ pkg._total_time = time.time() - start_time
+ build_time = pkg._total_time - pkg._fetch_time
+
+ tty.debug('{0} Successfully installed {1}'
+ .format(pre, pkg_id),
+ 'Fetch: {0}. Build: {1}. Total: {2}.'
+ .format(_hms(pkg._fetch_time), _hms(build_time),
+ _hms(pkg._total_time)))
+ _print_installed_pkg(pkg.prefix)
+
+ # preserve verbosity across runs
+ return echo
+
+
class BuildTask(object):
"""Class for representing the build task for a package."""
diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py
index a4e91fc186..a692d04052 100644
--- a/lib/spack/spack/package.py
+++ b/lib/spack/spack/package.py
@@ -901,6 +901,10 @@ class PackageBase(with_metaclass(PackageMeta, PackageViewMixin, object)):
path=self.path)
return stage
+ def _download_search(self):
+ dynamic_fetcher = fs.from_list_url(self)
+ return [dynamic_fetcher] if dynamic_fetcher else []
+
def _make_root_stage(self, fetcher):
# Construct a mirror path (TODO: get this out of package.py)
mirror_paths = spack.mirror.mirror_archive_paths(
@@ -912,12 +916,8 @@ class PackageBase(with_metaclass(PackageMeta, PackageViewMixin, object)):
stage_name = "{0}{1}-{2}-{3}".format(stage_prefix, s.name, s.version,
s.dag_hash())
- def download_search():
- dynamic_fetcher = fs.from_list_url(self)
- return [dynamic_fetcher] if dynamic_fetcher else []
-
stage = Stage(fetcher, mirror_paths=mirror_paths, name=stage_name,
- path=self.path, search_fn=download_search)
+ path=self.path, search_fn=self._download_search)
return stage
def _make_stage(self):
diff --git a/lib/spack/spack/repo.py b/lib/spack/spack/repo.py
index 8d31f2cf49..d9dddce6b6 100644
--- a/lib/spack/spack/repo.py
+++ b/lib/spack/spack/repo.py
@@ -1164,14 +1164,14 @@ def create_or_construct(path, namespace=None):
return Repo(path)
-def _path():
+def _path(repo_dirs=None):
"""Get the singleton RepoPath instance for Spack.
Create a RepoPath, add it to sys.meta_path, and return it.
TODO: consider not making this a singleton.
"""
- repo_dirs = spack.config.get('repos')
+ repo_dirs = repo_dirs or spack.config.get('repos')
if not repo_dirs:
raise NoRepoConfiguredError(
"Spack configuration contains no package repositories.")
diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py
index 256e3edc83..9a13a7db46 100644
--- a/lib/spack/spack/spec.py
+++ b/lib/spack/spack/spec.py
@@ -919,6 +919,12 @@ class ForwardQueryToPackage(object):
raise AttributeError(msg.format(cls_name, self.attribute_name))
+# Represents a query state in a BuildInterface object
+QueryState = collections.namedtuple(
+ 'QueryState', ['name', 'extra_parameters', 'isvirtual']
+)
+
+
class SpecBuildInterface(lang.ObjectWrapper):
command = ForwardQueryToPackage(
'command',
@@ -938,11 +944,6 @@ class SpecBuildInterface(lang.ObjectWrapper):
def __init__(self, spec, name, query_parameters):
super(SpecBuildInterface, self).__init__(spec)
- # Represents a query state in a BuildInterface object
- QueryState = collections.namedtuple(
- 'QueryState', ['name', 'extra_parameters', 'isvirtual']
- )
-
is_virtual = Spec.is_virtual(name)
self.last_query = QueryState(
name=name,
diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py
index f254b60fd5..719d408d84 100644
--- a/lib/spack/spack/stage.py
+++ b/lib/spack/spack/stage.py
@@ -690,11 +690,7 @@ class ResourceStage(Stage):
install(src, destination_path)
-@pattern.composite(method_list=[
- 'fetch', 'create', 'created', 'check', 'expand_archive', 'restage',
- 'destroy', 'cache_local', 'cache_mirror', 'steal_source',
- 'managed_by_spack'])
-class StageComposite:
+class StageComposite(pattern.Composite):
"""Composite for Stage type objects. The first item in this composite is
considered to be the root package, and operations that return a value are
forwarded to it."""
@@ -702,6 +698,12 @@ class StageComposite:
# __enter__ and __exit__ delegate to all stages in the composite.
#
+ def __init__(self):
+ super(StageComposite, self).__init__([
+ 'fetch', 'create', 'created', 'check', 'expand_archive', 'restage',
+ 'destroy', 'cache_local', 'cache_mirror', 'steal_source',
+ 'managed_by_spack'])
+
def __enter__(self):
for item in self:
item.__enter__()
diff --git a/lib/spack/spack/store.py b/lib/spack/spack/store.py
index 756a0a2e03..75c3969362 100644
--- a/lib/spack/spack/store.py
+++ b/lib/spack/spack/store.py
@@ -103,10 +103,23 @@ def _store():
#: Singleton store instance
store = llnl.util.lang.Singleton(_store)
+
+def _store_root():
+ return store.root
+
+
+def _store_db():
+ return store.db
+
+
+def _store_layout():
+ return store.layout
+
+
# convenience accessors for parts of the singleton store
-root = llnl.util.lang.LazyReference(lambda: store.root)
-db = llnl.util.lang.LazyReference(lambda: store.db)
-layout = llnl.util.lang.LazyReference(lambda: store.layout)
+root = llnl.util.lang.LazyReference(_store_root)
+db = llnl.util.lang.LazyReference(_store_db)
+layout = llnl.util.lang.LazyReference(_store_layout)
def retrieve_upstream_dbs():
diff --git a/lib/spack/spack/subprocess_context.py b/lib/spack/spack/subprocess_context.py
new file mode 100644
index 0000000000..b8e157740f
--- /dev/null
+++ b/lib/spack/spack/subprocess_context.py
@@ -0,0 +1,149 @@
+# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+"""
+This module handles transmission of Spack state to child processes started
+using the 'spawn' start method. Notably, installations are performed in a
+subprocess and require transmitting the Package object (in such a way
+that the repository is available for importing when it is deserialized);
+installations performed in Spack unit tests may include additional
+modifications to global state in memory that must be replicated in the
+child process.
+"""
+
+from types import ModuleType
+
+import pickle
+import pydoc
+import io
+import sys
+import multiprocessing
+
+import spack.architecture
+import spack.config
+
+
+_serialize = sys.version_info >= (3, 8) and sys.platform == 'darwin'
+
+
+patches = None
+
+
+def append_patch(patch):
+ global patches
+ if not patches:
+ patches = list()
+ patches.append(patch)
+
+
+def serialize(obj):
+ serialized_obj = io.BytesIO()
+ pickle.dump(obj, serialized_obj)
+ serialized_obj.seek(0)
+ return serialized_obj
+
+
+class SpackTestProcess(object):
+ def __init__(self, fn):
+ self.fn = fn
+
+ def _restore_and_run(self, fn, test_state):
+ test_state.restore()
+ fn()
+
+ def create(self):
+ test_state = TestState()
+ return multiprocessing.Process(
+ target=self._restore_and_run,
+ args=(self.fn, test_state))
+
+
+class PackageInstallContext(object):
+ """Captures the in-memory process state of a package installation that
+ needs to be transmitted to a child process.
+ """
+ def __init__(self, pkg):
+ if _serialize:
+ self.serialized_pkg = serialize(pkg)
+ else:
+ self.pkg = pkg
+ self.test_state = TestState()
+
+ def restore(self):
+ self.test_state.restore()
+ if _serialize:
+ return pickle.load(self.serialized_pkg)
+ else:
+ return self.pkg
+
+
+class TestState(object):
+ """Spack tests may modify state that is normally read from disk in memory;
+ this object is responsible for properly serializing that state to be
+ applied to a subprocess. This isn't needed outside of a testing environment
+ but this logic is designed to behave the same inside or outside of tests.
+ """
+ def __init__(self):
+ if _serialize:
+ self.repo_dirs = list(r.root for r in spack.repo.path.repos)
+ self.config = spack.config.config
+ self.platform = spack.architecture.platform
+ self.test_patches = store_patches()
+
+ # TODO: transfer spack.store.store? note that you should not
+ # transfer spack.store.store and spack.store.db: 'db' is a
+ # shortcut that accesses the store (so transferring both can
+ # create an inconsistency). Some tests set 'db' directly, and
+ # others set 'store'
+
+ def restore(self):
+ if _serialize:
+ spack.repo.path = spack.repo._path(self.repo_dirs)
+ spack.config.config = self.config
+ spack.architecture.platform = self.platform
+ self.test_patches.restore()
+
+
+class TestPatches(object):
+ def __init__(self, module_patches, class_patches):
+ self.module_patches = list(
+ (x, y, serialize(z)) for (x, y, z) in module_patches)
+ self.class_patches = list(
+ (x, y, serialize(z)) for (x, y, z) in class_patches)
+
+ def restore(self):
+ for module_name, attr_name, value in self.module_patches:
+ value = pickle.load(value)
+ module = __import__(module_name)
+ setattr(module, attr_name, value)
+ for class_fqn, attr_name, value in self.class_patches:
+ value = pickle.load(value)
+ cls = pydoc.locate(class_fqn)
+ setattr(cls, attr_name, value)
+
+
+def store_patches():
+ global patches
+ module_patches = list()
+ class_patches = list()
+ if not patches:
+ return TestPatches(list(), list())
+ for patch in patches:
+ for target, name, _ in patches:
+ if isinstance(target, ModuleType):
+ new_val = getattr(target, name)
+ module_name = target.__name__
+ module_patches.append((module_name, name, new_val))
+ elif isinstance(target, type):
+ new_val = getattr(target, name)
+ class_fqn = '.'.join([target.__module__, target.__name__])
+ class_patches.append((class_fqn, name, new_val))
+
+ return TestPatches(module_patches, class_patches)
+
+
+def clear_patches():
+ global patches
+ patches = None
diff --git a/lib/spack/spack/test/bindist.py b/lib/spack/spack/test/bindist.py
index 3949df117e..ef54e658f0 100644
--- a/lib/spack/spack/test/bindist.py
+++ b/lib/spack/spack/test/bindist.py
@@ -149,9 +149,11 @@ def install_dir_default_layout(tmpdir):
spack.store.store = spack.store.Store(str(tmpdir.join('opt')))
spack.store.layout = YamlDirectoryLayout(str(tmpdir.join('opt')),
path_scheme=def_install_path_scheme) # noqa: E501
- yield spack.store
- spack.store.store = real_store
- spack.store.layout = real_layout
+ try:
+ yield spack.store
+ finally:
+ spack.store.store = real_store
+ spack.store.layout = real_layout
@pytest.fixture(scope='function')
@@ -162,9 +164,11 @@ def install_dir_non_default_layout(tmpdir):
spack.store.store = spack.store.Store(str(tmpdir.join('opt')))
spack.store.layout = YamlDirectoryLayout(str(tmpdir.join('opt')),
path_scheme=ndef_install_path_scheme) # noqa: E501
- yield spack.store
- spack.store.store = real_store
- spack.store.layout = real_layout
+ try:
+ yield spack.store
+ finally:
+ spack.store.store = real_store
+ spack.store.layout = real_layout
args = ['strings', 'file']
@@ -582,6 +586,13 @@ def test_built_spec_cache(tmpdir,
mirror.mirror(mparser, margs)
+def fake_full_hash(spec):
+ # Generate an arbitrary hash that is intended to be different than
+ # whatever a Spec reported before (to test actions that trigger when
+ # the hash changes)
+ return 'tal4c7h4z0gqmixb1eqa92mjoybxn5l6'
+
+
def test_spec_needs_rebuild(install_mockery_mutable_config, mock_packages,
mock_fetch, monkeypatch, tmpdir):
"""Make sure needs_rebuild properly compares remote full_hash
@@ -606,9 +617,6 @@ def test_spec_needs_rebuild(install_mockery_mutable_config, mock_packages,
assert not rebuild
# Now monkey patch Spec to change the full hash on the package
- def fake_full_hash(spec):
- print('fake_full_hash')
- return 'tal4c7h4z0gqmixb1eqa92mjoybxn5l6'
monkeypatch.setattr(spack.spec.Spec, 'full_hash', fake_full_hash)
rebuild = bindist.needs_rebuild(s, mirror_url, rebuild_on_errors=True)
diff --git a/lib/spack/spack/test/cmd/dev_build.py b/lib/spack/spack/test/cmd/dev_build.py
index 111e500dcd..f11025b7b1 100644
--- a/lib/spack/spack/test/cmd/dev_build.py
+++ b/lib/spack/spack/test/cmd/dev_build.py
@@ -108,20 +108,24 @@ def test_dev_build_before_until(tmpdir, mock_packages, install_mockery):
'dev-build-test-install@0.0.0')
+def print_spack_cc(*args):
+ # Eat arguments and print environment variable to test
+ print(os.environ.get('CC', ''))
+
+
+# `module unload cray-libsci` in test environment causes failure
+# It does not fail for actual installs
+# build_environment.py imports module directly, so we monkeypatch it there
+# rather than in module_cmd
+def mock_module_noop(*args):
+ pass
+
+
def test_dev_build_drop_in(tmpdir, mock_packages, monkeypatch,
install_mockery):
- def print_spack_cc(*args):
- # Eat arguments and print environment variable to test
- print(os.environ.get('CC', ''))
monkeypatch.setattr(os, 'execvp', print_spack_cc)
- # `module unload cray-libsci` in test environment causes failure
- # It does not fail for actual installs
- # build_environment.py imports module directly, so we monkeypatch it there
- # rather than in module_cmd
- def module(*args):
- pass
- monkeypatch.setattr(spack.build_environment, 'module', module)
+ monkeypatch.setattr(spack.build_environment, 'module', mock_module_noop)
with tmpdir.as_cwd():
output = dev_build('-b', 'edit', '--drop-in', 'sh',
diff --git a/lib/spack/spack/test/cmd/install.py b/lib/spack/spack/test/cmd/install.py
index 4af3e8a339..2d7571ef4b 100644
--- a/lib/spack/spack/test/cmd/install.py
+++ b/lib/spack/spack/test/cmd/install.py
@@ -811,6 +811,13 @@ def test_install_fails_no_args_suggests_env_activation(tmpdir):
assert 'using the `spack.yaml` in this directory' in output
+def fake_full_hash(spec):
+ # Generate an arbitrary hash that is intended to be different than
+ # whatever a Spec reported before (to test actions that trigger when
+ # the hash changes)
+ return 'tal4c7h4z0gqmixb1eqa92mjoybxn5l6'
+
+
def test_cache_install_full_hash_match(
install_mockery_mutable_config, mock_packages, mock_fetch,
mock_archive, mutable_config, monkeypatch, tmpdir):
@@ -843,9 +850,6 @@ def test_cache_install_full_hash_match(
uninstall('-y', s.name)
# Now monkey patch Spec to change the full hash on the package
- def fake_full_hash(spec):
- print('fake_full_hash')
- return 'tal4c7h4z0gqmixb1eqa92mjoybxn5l6'
monkeypatch.setattr(spack.spec.Spec, 'full_hash', fake_full_hash)
# Check that even if the full hash changes, we install from binary when
diff --git a/lib/spack/spack/test/conftest.py b/lib/spack/spack/test/conftest.py
index f43d6933ba..af77684329 100644
--- a/lib/spack/spack/test/conftest.py
+++ b/lib/spack/spack/test/conftest.py
@@ -34,6 +34,7 @@ import spack.repo
import spack.stage
import spack.util.executable
import spack.util.gpg
+import spack.subprocess_context
import spack.util.spack_yaml as syaml
from spack.util.pattern import Bunch
@@ -41,11 +42,34 @@ from spack.fetch_strategy import FetchStrategyComposite, URLFetchStrategy
from spack.fetch_strategy import FetchError
+@pytest.fixture(autouse=True)
+def clear_recorded_monkeypatches():
+ yield
+ spack.subprocess_context.clear_patches()
+
+
+@pytest.fixture(scope='session', autouse=True)
+def record_monkeypatch_setattr():
+ import _pytest
+ saved_setattr = _pytest.monkeypatch.MonkeyPatch.setattr
+
+ def record_setattr(cls, target, name, value, *args, **kwargs):
+ spack.subprocess_context.append_patch((target, name, value))
+ saved_setattr(cls, target, name, value, *args, **kwargs)
+
+ _pytest.monkeypatch.MonkeyPatch.setattr = record_setattr
+ try:
+ yield
+ finally:
+ _pytest.monkeypatch.MonkeyPatch.setattr = saved_setattr
+
+
+def _can_access(path, perms):
+ return False
+
+
@pytest.fixture
def no_path_access(monkeypatch):
- def _can_access(path, perms):
- return False
-
monkeypatch.setattr(os, 'access', _can_access)
@@ -70,6 +94,10 @@ def clean_user_environment():
ev.activate(active)
+def _verify_executables_noop(*args):
+ return None
+
+
#
# Disable checks on compiler executable existence
#
@@ -86,7 +114,7 @@ def mock_compiler_executable_verification(request, monkeypatch):
if 'enable_compiler_verification' not in request.keywords:
monkeypatch.setattr(spack.compiler.Compiler,
'verify_executables',
- lambda x: None)
+ _verify_executables_noop)
# Hooks to add command line options or set other custom behaviors.
@@ -245,25 +273,27 @@ def check_for_leftover_stage_files(request, mock_stage, ignore_stage_files):
assert not files_in_stage
+class MockCache(object):
+ def store(self, copy_cmd, relative_dest):
+ pass
+
+ def fetcher(self, target_path, digest, **kwargs):
+ return MockCacheFetcher()
+
+
+class MockCacheFetcher(object):
+ def fetch(self):
+ raise FetchError('Mock cache always fails for tests')
+
+ def __str__(self):
+ return "[mock fetch cache]"
+
+
@pytest.fixture(autouse=True)
def mock_fetch_cache(monkeypatch):
"""Substitutes spack.paths.fetch_cache with a mock object that does nothing
and raises on fetch.
"""
- class MockCache(object):
- def store(self, copy_cmd, relative_dest):
- pass
-
- def fetcher(self, target_path, digest, **kwargs):
- return MockCacheFetcher()
-
- class MockCacheFetcher(object):
- def fetch(self):
- raise FetchError('Mock cache always fails for tests')
-
- def __str__(self):
- return "[mock fetch cache]"
-
monkeypatch.setattr(spack.caches, 'fetch_cache', MockCache())
@@ -286,7 +316,15 @@ def _skip_if_missing_executables(request):
# FIXME: session-scope. Anyhow doing it is not easy, as it seems
# FIXME: there's some weird interaction with compilers during concretization.
spack.architecture.real_platform = spack.architecture.platform
-spack.architecture.platform = lambda: spack.platforms.test.Test()
+
+
+def test_platform():
+ return spack.platforms.test.Test()
+
+
+spack.architecture.platform = test_platform
+
+
# FIXME: Since we change the architecture above, we have to (re)initialize
# FIXME: the config singleton. If it gets initialized too early with the
# FIXME: actual architecture, tests will fail.
@@ -356,14 +394,15 @@ def mock_repo_path():
yield spack.repo.RepoPath(spack.paths.mock_packages_path)
+def _pkg_install_fn(pkg, spec, prefix):
+ # sanity_check_prefix requires something in the install directory
+ mkdirp(prefix.bin)
+
+
@pytest.fixture
def mock_pkg_install(monkeypatch):
- def _pkg_install_fn(pkg, spec, prefix):
- # sanity_check_prefix requires something in the install directory
- mkdirp(prefix.bin)
-
- monkeypatch.setattr(spack.package.PackageBase, 'install', _pkg_install_fn,
- raising=False)
+ monkeypatch.setattr(spack.package.PackageBase, 'install',
+ _pkg_install_fn, raising=False)
@pytest.fixture(scope='function')
@@ -580,10 +619,10 @@ def mock_store(tmpdir_factory, mock_repo_path, mock_configuration,
@pytest.fixture(scope='function')
-def database(mock_store, mock_packages, config):
+def database(mock_store, mock_packages, config, monkeypatch):
"""This activates the mock store, packages, AND config."""
- with use_store(mock_store):
- yield mock_store.db
+ monkeypatch.setattr(spack.store, 'store', mock_store)
+ yield mock_store.db
# Force reading the database again between tests
mock_store.db.last_seen_verifier = ''
@@ -631,6 +670,10 @@ def dirs_with_libfiles(tmpdir_factory):
yield lib_to_dirs, all_dirs
+def _compiler_link_paths_noop(*args):
+ return []
+
+
@pytest.fixture(scope='function', autouse=True)
def disable_compiler_execution(monkeypatch, request):
"""
@@ -641,31 +684,27 @@ def disable_compiler_execution(monkeypatch, request):
If a test is marked in that way this is a no-op."""
if 'enable_compiler_link_paths' not in request.keywords:
- def noop(*args):
- return []
-
# Compiler.determine_implicit_rpaths actually runs the compiler. So
# replace that function with a noop that simulates finding no implicit
# RPATHs
monkeypatch.setattr(
spack.compiler.Compiler,
'_get_compiler_link_paths',
- noop
+ _compiler_link_paths_noop
)
@pytest.fixture(scope='function')
def install_mockery(tmpdir, config, mock_packages, monkeypatch):
"""Hooks a fake install directory, DB, and stage directory into Spack."""
- real_store = spack.store.store
- spack.store.store = spack.store.Store(str(tmpdir.join('opt')))
+ monkeypatch.setattr(
+ spack.store, 'store', spack.store.Store(str(tmpdir.join('opt'))))
# We use a fake package, so temporarily disable checksumming
with spack.config.override('config:checksum', False):
yield
tmpdir.join('opt').remove()
- spack.store.store = real_store
# Also wipe out any cached prefix failure locks (associated with
# the session-scoped mock archive).
@@ -687,31 +726,24 @@ def install_mockery_mutable_config(
also need to modify configuration (and hence would want to use
'mutable config'): 'install_mockery' does not support this.
"""
- real_store = spack.store.store
- spack.store.store = spack.store.Store(str(tmpdir.join('opt')))
+ monkeypatch.setattr(
+ spack.store, 'store', spack.store.Store(str(tmpdir.join('opt'))))
# We use a fake package, so temporarily disable checksumming
with spack.config.override('config:checksum', False):
yield
tmpdir.join('opt').remove()
- spack.store.store = real_store
@pytest.fixture()
-def mock_fetch(mock_archive):
+def mock_fetch(mock_archive, monkeypatch):
"""Fake the URL for a package so it downloads from a file."""
- fetcher = FetchStrategyComposite()
- fetcher.append(URLFetchStrategy(mock_archive.url))
+ mock_fetcher = FetchStrategyComposite()
+ mock_fetcher.append(URLFetchStrategy(mock_archive.url))
- @property
- def fake_fn(self):
- return fetcher
-
- orig_fn = spack.package.PackageBase.fetcher
- spack.package.PackageBase.fetcher = fake_fn
- yield
- spack.package.PackageBase.fetcher = orig_fn
+ monkeypatch.setattr(
+ spack.package.PackageBase, 'fetcher', mock_fetcher)
class MockLayout(object):
@@ -737,6 +769,47 @@ def gen_mock_layout(tmpdir):
yield create_layout
+class MockConfig(object):
+ def __init__(self, configuration, writer_key):
+ self._configuration = configuration
+ self.writer_key = writer_key
+
+ def configuration(self):
+ return self._configuration
+
+ def writer_configuration(self):
+ return self.configuration()[self.writer_key]
+
+
+class ConfigUpdate(object):
+ def __init__(self, root_for_conf, writer_mod, writer_key, monkeypatch):
+ self.root_for_conf = root_for_conf
+ self.writer_mod = writer_mod
+ self.writer_key = writer_key
+ self.monkeypatch = monkeypatch
+
+ def __call__(self, filename):
+ file = os.path.join(self.root_for_conf, filename + '.yaml')
+ with open(file) as f:
+ mock_config = MockConfig(syaml.load_config(f), self.writer_key)
+
+ self.monkeypatch.setattr(
+ spack.modules.common,
+ 'configuration',
+ mock_config.configuration
+ )
+ self.monkeypatch.setattr(
+ self.writer_mod,
+ 'configuration',
+ mock_config.writer_configuration
+ )
+ self.monkeypatch.setattr(
+ self.writer_mod,
+ 'configuration_registry',
+ {}
+ )
+
+
@pytest.fixture()
def module_configuration(monkeypatch, request):
"""Reads the module configuration file from the mock ones prepared
@@ -753,34 +826,7 @@ def module_configuration(monkeypatch, request):
spack.paths.test_path, 'data', 'modules', writer_key
)
- def _impl(filename):
-
- file = os.path.join(root_for_conf, filename + '.yaml')
- with open(file) as f:
- configuration = syaml.load_config(f)
-
- def mock_config_function():
- return configuration
-
- def writer_key_function():
- return mock_config_function()[writer_key]
-
- monkeypatch.setattr(
- spack.modules.common,
- 'configuration',
- mock_config_function
- )
- monkeypatch.setattr(
- writer_mod,
- 'configuration',
- writer_key_function
- )
- monkeypatch.setattr(
- writer_mod,
- 'configuration_registry',
- {}
- )
- return _impl
+ return ConfigUpdate(root_for_conf, writer_mod, writer_key, monkeypatch)
@pytest.fixture()
diff --git a/lib/spack/spack/test/database.py b/lib/spack/spack/test/database.py
index a97243ae9c..473573fa2e 100644
--- a/lib/spack/spack/test/database.py
+++ b/lib/spack/spack/test/database.py
@@ -23,7 +23,6 @@ from jsonschema import validate
import llnl.util.lock as lk
from llnl.util.tty.colify import colify
-from llnl.util.lang import fork_context
import spack.repo
import spack.store
@@ -319,7 +318,7 @@ def _check_merkleiness():
def _check_db_sanity(database):
- """Utiilty function to check db against install layout."""
+ """Utility function to check db against install layout."""
pkg_in_layout = sorted(spack.store.layout.all_specs())
actual = sorted(database.query())
@@ -517,14 +516,20 @@ def test_026_reindex_after_deprecate(mutable_database):
_check_db_sanity(mutable_database)
-def test_030_db_sanity_from_another_process(mutable_database):
- def read_and_modify():
+class ReadModify(object):
+ """Provide a function which can execute in a separate process that removes
+ a spec from the database.
+ """
+ def __call__(self):
# check that other process can read DB
- _check_db_sanity(mutable_database)
- with mutable_database.write_transaction():
+ _check_db_sanity(spack.store.db)
+ with spack.store.db.write_transaction():
_mock_remove('mpileaks ^zmpi')
- p = fork_context.Process(target=read_and_modify, args=())
+
+def test_030_db_sanity_from_another_process(mutable_database):
+ spack_process = spack.subprocess_context.SpackTestProcess(ReadModify())
+ p = spack_process.create()
p.start()
p.join()
diff --git a/lib/spack/spack/test/install.py b/lib/spack/spack/test/install.py
index 47ec06bbc4..61475e4dc6 100644
--- a/lib/spack/spack/test/install.py
+++ b/lib/spack/spack/test/install.py
@@ -19,6 +19,11 @@ from spack.package import (_spack_build_envfile, _spack_build_logfile,
_spack_configure_argsfile)
+def find_nothing(*args):
+ raise spack.repo.UnknownPackageError(
+ 'Repo package access is disabled for test')
+
+
def test_install_and_uninstall(install_mockery, mock_fetch, monkeypatch):
# Get a basic concrete spec for the trivial install package.
spec = Spec('trivial-install-test-package')
@@ -28,10 +33,6 @@ def test_install_and_uninstall(install_mockery, mock_fetch, monkeypatch):
# Get the package
pkg = spec.package
- def find_nothing(*args):
- raise spack.repo.UnknownPackageError(
- 'Repo package access is disabled for test')
-
try:
pkg.do_install()
@@ -83,23 +84,25 @@ class MockStage(object):
self.wrapped_stage.create()
def __getattr__(self, attr):
+ if attr == 'wrapped_stage':
+ # This attribute may not be defined at some point during unpickling
+ raise AttributeError()
return getattr(self.wrapped_stage, attr)
def test_partial_install_delete_prefix_and_stage(install_mockery, mock_fetch):
spec = Spec('canfail').concretized()
pkg = spack.repo.get(spec)
- remove_prefix = spack.package.Package.remove_prefix
instance_rm_prefix = pkg.remove_prefix
try:
pkg.succeed = False
- spack.package.Package.remove_prefix = mock_remove_prefix
+ pkg.remove_prefix = mock_remove_prefix
with pytest.raises(MockInstallError):
pkg.do_install()
assert os.path.isdir(pkg.prefix)
rm_prefix_checker = RemovePrefixChecker(instance_rm_prefix)
- spack.package.Package.remove_prefix = rm_prefix_checker.remove_prefix
+ pkg.remove_prefix = rm_prefix_checker.remove_prefix
# must clear failure markings for the package before re-installing it
spack.store.db.clear_failure(spec, True)
@@ -113,7 +116,7 @@ def test_partial_install_delete_prefix_and_stage(install_mockery, mock_fetch):
assert pkg.installed
finally:
- spack.package.Package.remove_prefix = remove_prefix
+ pkg.remove_prefix = instance_rm_prefix
def test_dont_add_patches_to_installed_package(install_mockery, mock_fetch):
@@ -181,7 +184,8 @@ def test_flatten_deps(
def test_installed_upstream_external(
- tmpdir_factory, install_mockery, mock_fetch, gen_mock_layout):
+ tmpdir_factory, install_mockery, mock_fetch, gen_mock_layout,
+ monkeypatch):
"""Check that when a dependency package is recorded as installed in
an upstream database that it is not reinstalled.
"""
@@ -194,29 +198,26 @@ def test_installed_upstream_external(
dependency.concretize()
prepared_db.add(dependency, upstream_layout)
- try:
- original_db = spack.store.db
- downstream_db_root = str(
- tmpdir_factory.mktemp('mock_downstream_db_root'))
- spack.store.db = spack.database.Database(
- downstream_db_root, upstream_dbs=[prepared_db])
- dependent = spack.spec.Spec('externaltest')
- dependent.concretize()
+ downstream_db_root = str(
+ tmpdir_factory.mktemp('mock_downstream_db_root'))
+ db_for_test = spack.database.Database(
+ downstream_db_root, upstream_dbs=[prepared_db])
+ monkeypatch.setattr(spack.store, 'db', db_for_test)
+ dependent = spack.spec.Spec('externaltest')
+ dependent.concretize()
- new_dependency = dependent['externaltool']
- assert new_dependency.external
- assert new_dependency.prefix == '/path/to/external_tool'
+ new_dependency = dependent['externaltool']
+ assert new_dependency.external
+ assert new_dependency.prefix == '/path/to/external_tool'
- dependent.package.do_install()
+ dependent.package.do_install()
- assert not os.path.exists(new_dependency.prefix)
- assert os.path.exists(dependent.prefix)
- finally:
- spack.store.db = original_db
+ assert not os.path.exists(new_dependency.prefix)
+ assert os.path.exists(dependent.prefix)
def test_installed_upstream(tmpdir_factory, install_mockery, mock_fetch,
- gen_mock_layout):
+ gen_mock_layout, monkeypatch):
"""Check that when a dependency package is recorded as installed in
an upstream database that it is not reinstalled.
"""
@@ -229,26 +230,23 @@ def test_installed_upstream(tmpdir_factory, install_mockery, mock_fetch,
dependency.concretize()
prepared_db.add(dependency, upstream_layout)
- try:
- original_db = spack.store.db
- downstream_db_root = str(
- tmpdir_factory.mktemp('mock_downstream_db_root'))
- spack.store.db = spack.database.Database(
- downstream_db_root, upstream_dbs=[prepared_db])
- dependent = spack.spec.Spec('dependent-install')
- dependent.concretize()
+ downstream_db_root = str(
+ tmpdir_factory.mktemp('mock_downstream_db_root'))
+ db_for_test = spack.database.Database(
+ downstream_db_root, upstream_dbs=[prepared_db])
+ monkeypatch.setattr(spack.store, 'db', db_for_test)
+ dependent = spack.spec.Spec('dependent-install')
+ dependent.concretize()
- new_dependency = dependent['dependency-install']
- assert new_dependency.package.installed_upstream
- assert (new_dependency.prefix ==
- upstream_layout.path_for_spec(dependency))
+ new_dependency = dependent['dependency-install']
+ assert new_dependency.package.installed_upstream
+ assert (new_dependency.prefix ==
+ upstream_layout.path_for_spec(dependency))
- dependent.package.do_install()
+ dependent.package.do_install()
- assert not os.path.exists(new_dependency.prefix)
- assert os.path.exists(dependent.prefix)
- finally:
- spack.store.db = original_db
+ assert not os.path.exists(new_dependency.prefix)
+ assert os.path.exists(dependent.prefix)
@pytest.mark.disable_clean_stage_check
diff --git a/lib/spack/spack/test/installer.py b/lib/spack/spack/test/installer.py
index 09ba78f58d..99f0af709c 100644
--- a/lib/spack/spack/test/installer.py
+++ b/lib/spack/spack/test/installer.py
@@ -626,7 +626,7 @@ def test_install_task_add_compiler(install_mockery, monkeypatch, capfd):
# Preclude any meaningful side-effects
monkeypatch.setattr(spack.package.PackageBase, 'unit_test_check', _true)
monkeypatch.setattr(inst.PackageInstaller, '_setup_install_dir', _noop)
- monkeypatch.setattr(spack.build_environment, 'fork', _noop)
+ monkeypatch.setattr(spack.build_environment, 'start_build_process', _noop)
monkeypatch.setattr(spack.database.Database, 'add', _noop)
monkeypatch.setattr(spack.compilers, 'add_compilers_to_config', _add)
diff --git a/lib/spack/spack/test/llnl/util/lock.py b/lib/spack/spack/test/llnl/util/lock.py
index 596a801179..800c9d63ad 100644
--- a/lib/spack/spack/test/llnl/util/lock.py
+++ b/lib/spack/spack/test/llnl/util/lock.py
@@ -53,14 +53,13 @@ import traceback
import glob
import getpass
from contextlib import contextmanager
-from multiprocessing import Queue
+from multiprocessing import Process, Queue
import pytest
import llnl.util.lock as lk
import llnl.util.multiproc as mp
from llnl.util.filesystem import touch
-from llnl.util.lang import fork_context
#
@@ -217,7 +216,7 @@ def local_multiproc_test(*functions, **kwargs):
b = mp.Barrier(len(functions), timeout=barrier_timeout)
args = (b,) + tuple(kwargs.get('extra_args', ()))
- procs = [fork_context.Process(target=f, args=args, name=f.__name__)
+ procs = [Process(target=f, args=args, name=f.__name__)
for f in functions]
for p in procs:
@@ -278,42 +277,74 @@ multiproc_test = mpi_multiproc_test if mpi else local_multiproc_test
#
# Process snippets below can be composed into tests.
#
-def acquire_write(lock_path, start=0, length=0):
- def fn(barrier):
- lock = lk.Lock(lock_path, start, length)
+class AcquireWrite(object):
+ def __init__(self, lock_path, start=0, length=0):
+ self.lock_path = lock_path
+ self.start = start
+ self.length = length
+
+ @property
+ def __name__(self):
+ return self.__class__.__name__
+
+ def __call__(self, barrier):
+ lock = lk.Lock(self.lock_path, self.start, self.length)
lock.acquire_write() # grab exclusive lock
barrier.wait()
barrier.wait() # hold the lock until timeout in other procs.
- return fn
-def acquire_read(lock_path, start=0, length=0):
- def fn(barrier):
- lock = lk.Lock(lock_path, start, length)
+class AcquireRead(object):
+ def __init__(self, lock_path, start=0, length=0):
+ self.lock_path = lock_path
+ self.start = start
+ self.length = length
+
+ @property
+ def __name__(self):
+ return self.__class__.__name__
+
+ def __call__(self, barrier):
+ lock = lk.Lock(self.lock_path, self.start, self.length)
lock.acquire_read() # grab shared lock
barrier.wait()
barrier.wait() # hold the lock until timeout in other procs.
- return fn
-def timeout_write(lock_path, start=0, length=0):
- def fn(barrier):
- lock = lk.Lock(lock_path, start, length)
+class TimeoutWrite(object):
+ def __init__(self, lock_path, start=0, length=0):
+ self.lock_path = lock_path
+ self.start = start
+ self.length = length
+
+ @property
+ def __name__(self):
+ return self.__class__.__name__
+
+ def __call__(self, barrier):
+ lock = lk.Lock(self.lock_path, self.start, self.length)
barrier.wait() # wait for lock acquire in first process
with pytest.raises(lk.LockTimeoutError):
lock.acquire_write(lock_fail_timeout)
barrier.wait()
- return fn
-def timeout_read(lock_path, start=0, length=0):
- def fn(barrier):
- lock = lk.Lock(lock_path, start, length)
+class TimeoutRead(object):
+ def __init__(self, lock_path, start=0, length=0):
+ self.lock_path = lock_path
+ self.start = start
+ self.length = length
+
+ @property
+ def __name__(self):
+ return self.__class__.__name__
+
+ def __call__(self, barrier):
+ lock = lk.Lock(self.lock_path, self.start, self.length)
barrier.wait() # wait for lock acquire in first process
with pytest.raises(lk.LockTimeoutError):
lock.acquire_read(lock_fail_timeout)
barrier.wait()
- return fn
#
@@ -322,57 +353,57 @@ def timeout_read(lock_path, start=0, length=0):
#
def test_write_lock_timeout_on_write(lock_path):
multiproc_test(
- acquire_write(lock_path),
- timeout_write(lock_path))
+ AcquireWrite(lock_path),
+ TimeoutWrite(lock_path))
def test_write_lock_timeout_on_write_2(lock_path):
multiproc_test(
- acquire_write(lock_path),
- timeout_write(lock_path),
- timeout_write(lock_path))
+ AcquireWrite(lock_path),
+ TimeoutWrite(lock_path),
+ TimeoutWrite(lock_path))
def test_write_lock_timeout_on_write_3(lock_path):
multiproc_test(
- acquire_write(lock_path),
- timeout_write(lock_path),
- timeout_write(lock_path),
- timeout_write(lock_path))
+ AcquireWrite(lock_path),
+ TimeoutWrite(lock_path),
+ TimeoutWrite(lock_path),
+ TimeoutWrite(lock_path))
def test_write_lock_timeout_on_write_ranges(lock_path):
multiproc_test(
- acquire_write(lock_path, 0, 1),
- timeout_write(lock_path, 0, 1))
+ AcquireWrite(lock_path, 0, 1),
+ TimeoutWrite(lock_path, 0, 1))
def test_write_lock_timeout_on_write_ranges_2(lock_path):
multiproc_test(
- acquire_write(lock_path, 0, 64),
- acquire_write(lock_path, 65, 1),
- timeout_write(lock_path, 0, 1),
- timeout_write(lock_path, 63, 1))
+ AcquireWrite(lock_path, 0, 64),
+ AcquireWrite(lock_path, 65, 1),
+ TimeoutWrite(lock_path, 0, 1),
+ TimeoutWrite(lock_path, 63, 1))
def test_write_lock_timeout_on_write_ranges_3(lock_path):
multiproc_test(
- acquire_write(lock_path, 0, 1),
- acquire_write(lock_path, 1, 1),
- timeout_write(lock_path),
- timeout_write(lock_path),
- timeout_write(lock_path))
+ AcquireWrite(lock_path, 0, 1),
+ AcquireWrite(lock_path, 1, 1),
+ TimeoutWrite(lock_path),
+ TimeoutWrite(lock_path),
+ TimeoutWrite(lock_path))
def test_write_lock_timeout_on_write_ranges_4(lock_path):
multiproc_test(
- acquire_write(lock_path, 0, 1),
- acquire_write(lock_path, 1, 1),
- acquire_write(lock_path, 2, 456),
- acquire_write(lock_path, 500, 64),
- timeout_write(lock_path),
- timeout_write(lock_path),
- timeout_write(lock_path))
+ AcquireWrite(lock_path, 0, 1),
+ AcquireWrite(lock_path, 1, 1),
+ AcquireWrite(lock_path, 2, 456),
+ AcquireWrite(lock_path, 500, 64),
+ TimeoutWrite(lock_path),
+ TimeoutWrite(lock_path),
+ TimeoutWrite(lock_path))
#
@@ -381,46 +412,46 @@ def test_write_lock_timeout_on_write_ranges_4(lock_path):
#
def test_read_lock_timeout_on_write(lock_path):
multiproc_test(
- acquire_write(lock_path),
- timeout_read(lock_path))
+ AcquireWrite(lock_path),
+ TimeoutRead(lock_path))
def test_read_lock_timeout_on_write_2(lock_path):
multiproc_test(
- acquire_write(lock_path),
- timeout_read(lock_path),
- timeout_read(lock_path))
+ AcquireWrite(lock_path),
+ TimeoutRead(lock_path),
+ TimeoutRead(lock_path))
def test_read_lock_timeout_on_write_3(lock_path):
multiproc_test(
- acquire_write(lock_path),
- timeout_read(lock_path),
- timeout_read(lock_path),
- timeout_read(lock_path))
+ AcquireWrite(lock_path),
+ TimeoutRead(lock_path),
+ TimeoutRead(lock_path),
+ TimeoutRead(lock_path))
def test_read_lock_timeout_on_write_ranges(lock_path):
"""small write lock, read whole file."""
multiproc_test(
- acquire_write(lock_path, 0, 1),
- timeout_read(lock_path))
+ AcquireWrite(lock_path, 0, 1),
+ TimeoutRead(lock_path))
def test_read_lock_timeout_on_write_ranges_2(lock_path):
"""small write lock, small read lock"""
multiproc_test(
- acquire_write(lock_path, 0, 1),
- timeout_read(lock_path, 0, 1))
+ AcquireWrite(lock_path, 0, 1),
+ TimeoutRead(lock_path, 0, 1))
def test_read_lock_timeout_on_write_ranges_3(lock_path):
"""two write locks, overlapping read locks"""
multiproc_test(
- acquire_write(lock_path, 0, 1),
- acquire_write(lock_path, 64, 128),
- timeout_read(lock_path, 0, 1),
- timeout_read(lock_path, 128, 256))
+ AcquireWrite(lock_path, 0, 1),
+ AcquireWrite(lock_path, 64, 128),
+ TimeoutRead(lock_path, 0, 1),
+ TimeoutRead(lock_path, 128, 256))
#
@@ -428,58 +459,58 @@ def test_read_lock_timeout_on_write_ranges_3(lock_path):
#
def test_write_lock_timeout_on_read(lock_path):
multiproc_test(
- acquire_read(lock_path),
- timeout_write(lock_path))
+ AcquireRead(lock_path),
+ TimeoutWrite(lock_path))
def test_write_lock_timeout_on_read_2(lock_path):
multiproc_test(
- acquire_read(lock_path),
- timeout_write(lock_path),
- timeout_write(lock_path))
+ AcquireRead(lock_path),
+ TimeoutWrite(lock_path),
+ TimeoutWrite(lock_path))
def test_write_lock_timeout_on_read_3(lock_path):
multiproc_test(
- acquire_read(lock_path),
- timeout_write(lock_path),
- timeout_write(lock_path),
- timeout_write(lock_path))
+ AcquireRead(lock_path),
+ TimeoutWrite(lock_path),
+ TimeoutWrite(lock_path),
+ TimeoutWrite(lock_path))
def test_write_lock_timeout_on_read_ranges(lock_path):
multiproc_test(
- acquire_read(lock_path, 0, 1),
- timeout_write(lock_path))
+ AcquireRead(lock_path, 0, 1),
+ TimeoutWrite(lock_path))
def test_write_lock_timeout_on_read_ranges_2(lock_path):
multiproc_test(
- acquire_read(lock_path, 0, 1),
- timeout_write(lock_path, 0, 1))
+ AcquireRead(lock_path, 0, 1),
+ TimeoutWrite(lock_path, 0, 1))
def test_write_lock_timeout_on_read_ranges_3(lock_path):
multiproc_test(
- acquire_read(lock_path, 0, 1),
- acquire_read(lock_path, 10, 1),
- timeout_write(lock_path, 0, 1),
- timeout_write(lock_path, 10, 1))
+ AcquireRead(lock_path, 0, 1),
+ AcquireRead(lock_path, 10, 1),
+ TimeoutWrite(lock_path, 0, 1),
+ TimeoutWrite(lock_path, 10, 1))
def test_write_lock_timeout_on_read_ranges_4(lock_path):
multiproc_test(
- acquire_read(lock_path, 0, 64),
- timeout_write(lock_path, 10, 1),
- timeout_write(lock_path, 32, 1))
+ AcquireRead(lock_path, 0, 64),
+ TimeoutWrite(lock_path, 10, 1),
+ TimeoutWrite(lock_path, 32, 1))
def test_write_lock_timeout_on_read_ranges_5(lock_path):
multiproc_test(
- acquire_read(lock_path, 64, 128),
- timeout_write(lock_path, 65, 1),
- timeout_write(lock_path, 127, 1),
- timeout_write(lock_path, 90, 10))
+ AcquireRead(lock_path, 64, 128),
+ TimeoutWrite(lock_path, 65, 1),
+ TimeoutWrite(lock_path, 127, 1),
+ TimeoutWrite(lock_path, 90, 10))
#
@@ -487,67 +518,67 @@ def test_write_lock_timeout_on_read_ranges_5(lock_path):
#
def test_write_lock_timeout_with_multiple_readers_2_1(lock_path):
multiproc_test(
- acquire_read(lock_path),
- acquire_read(lock_path),
- timeout_write(lock_path))
+ AcquireRead(lock_path),
+ AcquireRead(lock_path),
+ TimeoutWrite(lock_path))
def test_write_lock_timeout_with_multiple_readers_2_2(lock_path):
multiproc_test(
- acquire_read(lock_path),
- acquire_read(lock_path),
- timeout_write(lock_path),
- timeout_write(lock_path))
+ AcquireRead(lock_path),
+ AcquireRead(lock_path),
+ TimeoutWrite(lock_path),
+ TimeoutWrite(lock_path))
def test_write_lock_timeout_with_multiple_readers_3_1(lock_path):
multiproc_test(
- acquire_read(lock_path),
- acquire_read(lock_path),
- acquire_read(lock_path),
- timeout_write(lock_path))
+ AcquireRead(lock_path),
+ AcquireRead(lock_path),
+ AcquireRead(lock_path),
+ TimeoutWrite(lock_path))
def test_write_lock_timeout_with_multiple_readers_3_2(lock_path):
multiproc_test(
- acquire_read(lock_path),
- acquire_read(lock_path),
- acquire_read(lock_path),
- timeout_write(lock_path),
- timeout_write(lock_path))
+ AcquireRead(lock_path),
+ AcquireRead(lock_path),
+ AcquireRead(lock_path),
+ TimeoutWrite(lock_path),
+ TimeoutWrite(lock_path))
def test_write_lock_timeout_with_multiple_readers_2_1_ranges(lock_path):
multiproc_test(
- acquire_read(lock_path, 0, 10),
- acquire_read(lock_path, 0.5, 10),
- timeout_write(lock_path, 5, 5))
+ AcquireRead(lock_path, 0, 10),
+ AcquireRead(lock_path, 0.5, 10),
+ TimeoutWrite(lock_path, 5, 5))
def test_write_lock_timeout_with_multiple_readers_2_3_ranges(lock_path):
multiproc_test(
- acquire_read(lock_path, 0, 10),
- acquire_read(lock_path, 5, 15),
- timeout_write(lock_path, 0, 1),
- timeout_write(lock_path, 11, 3),
- timeout_write(lock_path, 7, 1))
+ AcquireRead(lock_path, 0, 10),
+ AcquireRead(lock_path, 5, 15),
+ TimeoutWrite(lock_path, 0, 1),
+ TimeoutWrite(lock_path, 11, 3),
+ TimeoutWrite(lock_path, 7, 1))
def test_write_lock_timeout_with_multiple_readers_3_1_ranges(lock_path):
multiproc_test(
- acquire_read(lock_path, 0, 5),
- acquire_read(lock_path, 5, 5),
- acquire_read(lock_path, 10, 5),
- timeout_write(lock_path, 0, 15))
+ AcquireRead(lock_path, 0, 5),
+ AcquireRead(lock_path, 5, 5),
+ AcquireRead(lock_path, 10, 5),
+ TimeoutWrite(lock_path, 0, 15))
def test_write_lock_timeout_with_multiple_readers_3_2_ranges(lock_path):
multiproc_test(
- acquire_read(lock_path, 0, 5),
- acquire_read(lock_path, 5, 5),
- acquire_read(lock_path, 10, 5),
- timeout_write(lock_path, 3, 10),
- timeout_write(lock_path, 5, 1))
+ AcquireRead(lock_path, 0, 5),
+ AcquireRead(lock_path, 5, 5),
+ AcquireRead(lock_path, 10, 5),
+ TimeoutWrite(lock_path, 3, 10),
+ TimeoutWrite(lock_path, 5, 1))
@pytest.mark.skipif(os.getuid() == 0, reason='user is root')
@@ -651,13 +682,12 @@ def test_upgrade_read_to_write_fails_with_readonly_file(private_lock_path):
lock.acquire_write()
-#
-# Longer test case that ensures locks are reusable. Ordering is
-# enforced by barriers throughout -- steps are shown with numbers.
-#
-def test_complex_acquire_and_release_chain(lock_path):
- def p1(barrier):
- lock = lk.Lock(lock_path)
+class ComplexAcquireAndRelease(object):
+ def __init__(self, lock_path):
+ self.lock_path = lock_path
+
+ def p1(self, barrier):
+ lock = lk.Lock(self.lock_path)
lock.acquire_write()
barrier.wait() # ---------------------------------------- 1
@@ -697,8 +727,8 @@ def test_complex_acquire_and_release_chain(lock_path):
barrier.wait() # ---------------------------------------- 13
lock.release_read()
- def p2(barrier):
- lock = lk.Lock(lock_path)
+ def p2(self, barrier):
+ lock = lk.Lock(self.lock_path)
# p1 acquires write
barrier.wait() # ---------------------------------------- 1
@@ -737,8 +767,8 @@ def test_complex_acquire_and_release_chain(lock_path):
barrier.wait() # ---------------------------------------- 13
lock.release_read()
- def p3(barrier):
- lock = lk.Lock(lock_path)
+ def p3(self, barrier):
+ lock = lk.Lock(self.lock_path)
# p1 acquires write
barrier.wait() # ---------------------------------------- 1
@@ -777,7 +807,16 @@ def test_complex_acquire_and_release_chain(lock_path):
barrier.wait() # ---------------------------------------- 13
lock.release_read()
- multiproc_test(p1, p2, p3)
+
+#
+# Longer test case that ensures locks are reusable. Ordering is
+# enforced by barriers throughout -- steps are shown with numbers.
+#
+def test_complex_acquire_and_release_chain(lock_path):
+ test_chain = ComplexAcquireAndRelease(lock_path)
+ multiproc_test(test_chain.p1,
+ test_chain.p2,
+ test_chain.p3)
class AssertLock(lk.Lock):
@@ -1146,24 +1185,26 @@ def test_nested_reads(lock_path):
assert vals['read'] == 1
-def test_lock_debug_output(lock_path):
- host = socket.getfqdn()
+class LockDebugOutput(object):
+ def __init__(self, lock_path):
+ self.lock_path = lock_path
+ self.host = socket.getfqdn()
- def p1(barrier, q1, q2):
+ def p1(self, barrier, q1, q2):
# exchange pids
p1_pid = os.getpid()
q1.put(p1_pid)
p2_pid = q2.get()
# set up lock
- lock = lk.Lock(lock_path, debug=True)
+ lock = lk.Lock(self.lock_path, debug=True)
with lk.WriteTransaction(lock):
# p1 takes write lock and writes pid/host to file
barrier.wait() # ------------------------------------ 1
assert lock.pid == p1_pid
- assert lock.host == host
+ assert lock.host == self.host
# wait for p2 to verify contents of file
barrier.wait() # ---------------------------------------- 2
@@ -1174,21 +1215,21 @@ def test_lock_debug_output(lock_path):
# verify pid/host info again
with lk.ReadTransaction(lock):
assert lock.old_pid == p1_pid
- assert lock.old_host == host
+ assert lock.old_host == self.host
assert lock.pid == p2_pid
- assert lock.host == host
+ assert lock.host == self.host
barrier.wait() # ---------------------------------------- 4
- def p2(barrier, q1, q2):
+ def p2(self, barrier, q1, q2):
# exchange pids
p2_pid = os.getpid()
p1_pid = q1.get()
q2.put(p2_pid)
# set up lock
- lock = lk.Lock(lock_path, debug=True)
+ lock = lk.Lock(self.lock_path, debug=True)
# p1 takes write lock and writes pid/host to file
barrier.wait() # ---------------------------------------- 1
@@ -1196,25 +1237,28 @@ def test_lock_debug_output(lock_path):
# verify that p1 wrote information to lock file
with lk.ReadTransaction(lock):
assert lock.pid == p1_pid
- assert lock.host == host
+ assert lock.host == self.host
barrier.wait() # ---------------------------------------- 2
# take a write lock on the file and verify pid/host info
with lk.WriteTransaction(lock):
assert lock.old_pid == p1_pid
- assert lock.old_host == host
+ assert lock.old_host == self.host
assert lock.pid == p2_pid
- assert lock.host == host
+ assert lock.host == self.host
barrier.wait() # ------------------------------------ 3
# wait for p1 to verify pid/host info
barrier.wait() # ---------------------------------------- 4
+
+def test_lock_debug_output(lock_path):
+ test_debug = LockDebugOutput(lock_path)
q1, q2 = Queue(), Queue()
- local_multiproc_test(p2, p1, extra_args=(q1, q2))
+ local_multiproc_test(test_debug.p2, test_debug.p1, extra_args=(q1, q2))
def test_lock_with_no_parent_directory(tmpdir):
diff --git a/lib/spack/spack/test/llnl/util/tty/log.py b/lib/spack/spack/test/llnl/util/tty/log.py
index c6cef00977..811b6c6642 100644
--- a/lib/spack/spack/test/llnl/util/tty/log.py
+++ b/lib/spack/spack/test/llnl/util/tty/log.py
@@ -396,6 +396,11 @@ def mock_shell_v_v_no_termios(proc, ctl, **kwargs):
def test_foreground_background_output(
test_fn, capfd, termios_on_or_off, tmpdir):
"""Tests hitting 'v' toggles output, and that force_echo works."""
+ if (sys.version_info >= (3, 8) and sys.platform == 'darwin'
+ and termios_on_or_off == no_termios):
+
+ return
+
shell = PseudoShell(test_fn, synchronized_logger)
log_path = str(tmpdir.join("log.txt"))
diff --git a/lib/spack/spack/test/sbang.py b/lib/spack/spack/test/sbang.py
index 18bbe9f345..ca7d2a6fe6 100644
--- a/lib/spack/spack/test/sbang.py
+++ b/lib/spack/spack/test/sbang.py
@@ -21,29 +21,36 @@ import spack.hooks.sbang as sbang
from spack.util.executable import which
+too_long = sbang.shebang_limit + 1
+
+
short_line = "#!/this/is/short/bin/bash\n"
-long_line = "#!/this/" + ('x' * 200) + "/is/long\n"
+long_line = "#!/this/" + ('x' * too_long) + "/is/long\n"
-lua_line = "#!/this/" + ('x' * 200) + "/is/lua\n"
+lua_line = "#!/this/" + ('x' * too_long) + "/is/lua\n"
lua_in_text = ("line\n") * 100 + "lua\n" + ("line\n" * 100)
-lua_line_patched = "--!/this/" + ('x' * 200) + "/is/lua\n"
+lua_line_patched = "--!/this/" + ('x' * too_long) + "/is/lua\n"
-node_line = "#!/this/" + ('x' * 200) + "/is/node\n"
+node_line = "#!/this/" + ('x' * too_long) + "/is/node\n"
node_in_text = ("line\n") * 100 + "lua\n" + ("line\n" * 100)
-node_line_patched = "//!/this/" + ('x' * 200) + "/is/node\n"
+node_line_patched = "//!/this/" + ('x' * too_long) + "/is/node\n"
-php_line = "#!/this/" + ('x' * 200) + "/is/php\n"
+php_line = "#!/this/" + ('x' * too_long) + "/is/php\n"
php_in_text = ("line\n") * 100 + "php\n" + ("line\n" * 100)
-php_line_patched = "<?php #!/this/" + ('x' * 200) + "/is/php\n"
+php_line_patched = "<?php #!/this/" + ('x' * too_long) + "/is/php\n"
php_line_patched2 = "?>\n"
-sbang_line = '#!/bin/sh %s/bin/sbang\n' % spack.store.layout.root
last_line = "last!\n"
+@pytest.fixture
+def sbang_line():
+ yield '#!/bin/sh %s/bin/sbang\n' % spack.store.layout.root
+
+
class ScriptDirectory(object):
"""Directory full of test scripts to run sbang instrumentation on."""
- def __init__(self):
+ def __init__(self, sbang_line):
self.tempdir = tempfile.mkdtemp()
self.directory = os.path.join(self.tempdir, 'dir')
@@ -117,13 +124,13 @@ class ScriptDirectory(object):
@pytest.fixture
-def script_dir():
- sdir = ScriptDirectory()
+def script_dir(sbang_line):
+ sdir = ScriptDirectory(sbang_line)
yield sdir
sdir.destroy()
-def test_shebang_handling(script_dir):
+def test_shebang_handling(script_dir, sbang_line):
assert sbang.shebang_too_long(script_dir.lua_shebang)
assert sbang.shebang_too_long(script_dir.long_shebang)
@@ -169,13 +176,13 @@ def test_shebang_handling(script_dir):
assert f.readline() == last_line
-def test_shebang_handles_non_writable_files(script_dir):
+def test_shebang_handles_non_writable_files(script_dir, sbang_line):
# make a file non-writable
st = os.stat(script_dir.long_shebang)
not_writable_mode = st.st_mode & ~stat.S_IWRITE
os.chmod(script_dir.long_shebang, not_writable_mode)
- test_shebang_handling(script_dir)
+ test_shebang_handling(script_dir, sbang_line)
st = os.stat(script_dir.long_shebang)
assert oct(not_writable_mode) == oct(st.st_mode)
diff --git a/lib/spack/spack/util/pattern.py b/lib/spack/spack/util/pattern.py
index f467e2d18e..fcbb4e6baa 100644
--- a/lib/spack/spack/util/pattern.py
+++ b/lib/spack/spack/util/pattern.py
@@ -8,6 +8,27 @@ import collections
import functools
+class Delegate(object):
+ def __init__(self, name, container):
+ self.name = name
+ self.container = container
+
+ def __call__(self, *args, **kwargs):
+ return [getattr(item, self.name)(*args, **kwargs)
+ for item in self.container]
+
+
+class Composite(list):
+ def __init__(self, fns_to_delegate):
+ self.fns_to_delegate = fns_to_delegate
+
+ def __getattr__(self, name):
+ if name != 'fns_to_delegate' and name in self.fns_to_delegate:
+ return Delegate(name, self)
+ else:
+ return self.__getattribute__(name)
+
+
def composite(interface=None, method_list=None, container=list):
"""Decorator implementing the GoF composite pattern.
diff --git a/lib/spack/spack/util/prefix.py b/lib/spack/spack/util/prefix.py
index 804790c1d3..10ec50d30c 100644
--- a/lib/spack/spack/util/prefix.py
+++ b/lib/spack/spack/util/prefix.py
@@ -49,3 +49,9 @@ class Prefix(str):
Prefix: the newly created installation prefix
"""
return Prefix(os.path.join(self, string))
+
+ def __getstate__(self):
+ return self.__dict__
+
+ def __setstate__(self, d):
+ self.__dict__.update(d)