diff options
Diffstat (limited to 'lib/spack/llnl')
-rw-r--r-- | lib/spack/llnl/util/filesystem.py | 13 | ||||
-rw-r--r-- | lib/spack/llnl/util/lock.py | 59 | ||||
-rw-r--r-- | lib/spack/llnl/util/symlink.py | 41 | ||||
-rw-r--r-- | lib/spack/llnl/util/tty/__init__.py | 5 | ||||
-rw-r--r-- | lib/spack/llnl/util/tty/log.py | 3 | ||||
-rw-r--r-- | lib/spack/llnl/util/tty/pty.py | 1 |
6 files changed, 59 insertions, 63 deletions
diff --git a/lib/spack/llnl/util/filesystem.py b/lib/spack/llnl/util/filesystem.py index 6fd6c140c8..9f46b275b2 100644 --- a/lib/spack/llnl/util/filesystem.py +++ b/lib/spack/llnl/util/filesystem.py @@ -36,17 +36,6 @@ else: import win32security -is_windows = _platform == 'win32' - -if not is_windows: - import grp - import pwd - -if sys.version_info >= (3, 3): - from collections.abc import Sequence # novm -else: - from collections import Sequence - __all__ = [ 'FileFilter', 'FileList', @@ -908,7 +897,7 @@ def ancestor(dir, n=1): parent = os.path.abspath(dir) for i in range(n): parent = os.path.dirname(parent) - return parent.replace("\\", "/") + return parent @system_path_filter diff --git a/lib/spack/llnl/util/lock.py b/lib/spack/llnl/util/lock.py index 3b93303297..37989bac35 100644 --- a/lib/spack/llnl/util/lock.py +++ b/lib/spack/llnl/util/lock.py @@ -4,9 +4,9 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) import errno -import fcntl import os import socket +import sys import time from datetime import datetime from typing import Dict, Tuple # novm @@ -15,6 +15,10 @@ import llnl.util.tty as tty import spack.util.string +if sys.platform != 'win32': + import fcntl + + __all__ = [ 'Lock', 'LockDowngradeError', @@ -29,8 +33,6 @@ __all__ = [ 'CantCreateLockError' ] -#: Mapping of supported locks to description -lock_type = {fcntl.LOCK_SH: 'read', fcntl.LOCK_EX: 'write'} #: A useful replacement for functions that should return True when not provided #: for example. @@ -166,6 +168,29 @@ def _attempts_str(wait_time, nattempts): return ' after {0:0.2f}s and {1}'.format(wait_time, attempts) +class LockType(object): + READ = 0 + WRITE = 1 + + @staticmethod + def to_str(tid): + ret = "READ" + if tid == LockType.WRITE: + ret = "WRITE" + return ret + + @staticmethod + def to_module(tid): + lock = fcntl.LOCK_SH + if tid == LockType.WRITE: + lock = fcntl.LOCK_EX + return lock + + @staticmethod + def is_valid(op): + return op == LockType.READ \ + or op == LockType.WRITE + class Lock(object): """This is an implementation of a filesystem lock using Python's lockf. @@ -276,9 +301,10 @@ class Lock(object): successfully acquired, the total wait time and the number of attempts is returned. """ - assert op in lock_type + assert LockType.is_valid(op) + op_str = LockType.to_str(op) - self._log_acquiring('{0} LOCK'.format(lock_type[op].upper())) + self._log_acquiring('{0} LOCK'.format(op_str)) timeout = timeout or self.default_timeout # Create file and parent directories if they don't exist. @@ -286,13 +312,13 @@ class Lock(object): self._ensure_parent_directory() self._file = file_tracker.get_fh(self.path) - if op == fcntl.LOCK_EX and self._file.mode == 'r': + if LockType.to_module(op) == fcntl.LOCK_EX and self._file.mode == 'r': # Attempt to upgrade to write lock w/a read-only file. # If the file were writable, we'd have opened it 'r+' raise LockROFileError(self.path) self._log_debug("{0} locking [{1}:{2}]: timeout {3} sec" - .format(lock_type[op], self._start, self._length, + .format(op_str.lower(), self._start, self._length, timeout)) poll_intervals = iter(Lock._poll_interval_generator()) @@ -313,17 +339,16 @@ class Lock(object): return total_wait_time, num_attempts raise LockTimeoutError("Timed out waiting for a {0} lock." - .format(lock_type[op])) + .format(op_str.lower())) def _poll_lock(self, op): """Attempt to acquire the lock in a non-blocking manner. Return whether the locking attempt succeeds """ - assert op in lock_type - + module_op = LockType.to_module(op) try: # Try to get the lock (will raise if not available.) - fcntl.lockf(self._file, op | fcntl.LOCK_NB, + fcntl.lockf(self._file, module_op | fcntl.LOCK_NB, self._length, self._start, os.SEEK_SET) # help for debugging distributed locking @@ -331,11 +356,11 @@ class Lock(object): # All locks read the owner PID and host self._read_log_debug_data() self._log_debug('{0} locked {1} [{2}:{3}] (owner={4})' - .format(lock_type[op], self.path, + .format(LockType.to_str(op), self.path, self._start, self._length, self.pid)) # Exclusive locks write their PID/host - if op == fcntl.LOCK_EX: + if module_op == fcntl.LOCK_EX: self._write_log_debug_data() return True @@ -420,7 +445,7 @@ class Lock(object): if self._reads == 0 and self._writes == 0: # can raise LockError. - wait_time, nattempts = self._lock(fcntl.LOCK_SH, timeout=timeout) + wait_time, nattempts = self._lock(LockType.READ, timeout=timeout) self._reads += 1 # Log if acquired, which includes counts when verbose self._log_acquired('READ LOCK', wait_time, nattempts) @@ -445,7 +470,7 @@ class Lock(object): if self._writes == 0: # can raise LockError. - wait_time, nattempts = self._lock(fcntl.LOCK_EX, timeout=timeout) + wait_time, nattempts = self._lock(LockType.WRITE, timeout=timeout) self._writes += 1 # Log if acquired, which includes counts when verbose self._log_acquired('WRITE LOCK', wait_time, nattempts) @@ -489,7 +514,7 @@ class Lock(object): if self._writes == 1 and self._reads == 0: self._log_downgrading() # can raise LockError. - wait_time, nattempts = self._lock(fcntl.LOCK_SH, timeout=timeout) + wait_time, nattempts = self._lock(LockType.READ, timeout=timeout) self._reads = 1 self._writes = 0 self._log_downgraded(wait_time, nattempts) @@ -508,7 +533,7 @@ class Lock(object): if self._reads == 1 and self._writes == 0: self._log_upgrading() # can raise LockError. - wait_time, nattempts = self._lock(fcntl.LOCK_EX, timeout=timeout) + wait_time, nattempts = self._lock(LockType.WRITE, timeout=timeout) self._reads = 0 self._writes = 1 self._log_upgraded(wait_time, nattempts) diff --git a/lib/spack/llnl/util/symlink.py b/lib/spack/llnl/util/symlink.py index 3e5f0d4868..d6e4651557 100644 --- a/lib/spack/llnl/util/symlink.py +++ b/lib/spack/llnl/util/symlink.py @@ -9,9 +9,11 @@ import tempfile from os.path import exists, join from sys import platform as _platform -is_windows = _platform == 'win32' +from llnl.util import lang + +from spack.util.executable import Executable -__win32_can_symlink__ = None +is_windows = _platform == 'win32' def symlink(real_path, link_path): @@ -51,21 +53,20 @@ def _win32_junction(path, link): path = os.path.join(parent, path) path = os.path.abspath(path) + command = "mklink" + default_args = [link, path] if os.path.isdir(path): # try using a junction - command = 'mklink /J "%s" "%s"' % (link, path) + default_args.insert(0, '/J') else: # try using a hard link - command = 'mklink /H "%s" "%s"' % (link, path) + default_args.insert(0, '/H') - _cmd(command) + Executable(command)(*default_args) +@lang.memoized def _win32_can_symlink(): - global __win32_can_symlink__ - if __win32_can_symlink__ is not None: - return __win32_can_symlink__ - tempdir = tempfile.mkdtemp() dpath = join(tempdir, 'dpath') @@ -92,9 +93,7 @@ def _win32_can_symlink(): # Cleanup the test directory shutil.rmtree(tempdir) - __win32_can_symlink__ = can_symlink_directories and can_symlink_files - - return __win32_can_symlink__ + return can_symlink_directories and can_symlink_files def _win32_is_junction(path): @@ -119,21 +118,3 @@ def _win32_is_junction(path): bool(res & FILE_ATTRIBUTE_REPARSE_POINT) return False - - -# Based on https://github.com/Erotemic/ubelt/blob/master/ubelt/util_cmd.py -def _cmd(command): - import subprocess - - # Create a new process to execute the command - def make_proc(): - # delay the creation of the process until we validate all args - proc = subprocess.Popen(command, stdout=subprocess.PIPE, - stderr=subprocess.PIPE, shell=True, - universal_newlines=True, cwd=None, env=None) - return proc - - proc = make_proc() - (out, err) = proc.communicate() - if proc.wait() != 0: - raise OSError(str(err)) diff --git a/lib/spack/llnl/util/tty/__init__.py b/lib/spack/llnl/util/tty/__init__.py index 5b8729d7ec..ae7634afea 100644 --- a/lib/spack/llnl/util/tty/__init__.py +++ b/lib/spack/llnl/util/tty/__init__.py @@ -394,7 +394,8 @@ def terminal_size(): return int(rc[0]), int(rc[1]) else: - # return shutil.get_terminal_size() - # TODO: find python 2 compatible module to get terminal size + if sys.version_info[0] < 3: + raise RuntimeError("""Terminal size not obtainable on Windows with a + Python version older than 3""") rc = (os.environ.get('LINES', 25), os.environ.get('COLUMNS', 80)) return int(rc[0]), int(rc[1]) diff --git a/lib/spack/llnl/util/tty/log.py b/lib/spack/llnl/util/tty/log.py index c16982241e..0df3cf7829 100644 --- a/lib/spack/llnl/util/tty/log.py +++ b/lib/spack/llnl/util/tty/log.py @@ -938,8 +938,7 @@ def _writer_daemon(stdin_multiprocess_fd, read_multiprocess_fd, write_fd, echo, # write_fd to terminate the reading loop, so we close the file descriptor # here. Forking is the process spawning method everywhere except Mac OS # for Python >= 3.8 and on Windows - if sys.version_info < (3, 8) \ - or sys.platform not in ['darwin', 'cygwin']: + if sys.version_info < (3, 8) or sys.platform != 'darwin': os.close(write_fd) # Use line buffering (3rd param = 1) since Python 3 has a bug diff --git a/lib/spack/llnl/util/tty/pty.py b/lib/spack/llnl/util/tty/pty.py index 0475667825..1a5731a5c7 100644 --- a/lib/spack/llnl/util/tty/pty.py +++ b/lib/spack/llnl/util/tty/pty.py @@ -11,6 +11,7 @@ If this is used outside a testing environment, we will want to reconsider things like timeouts in ``ProcessController.wait()``, which are set to get tests done quickly, not to avoid high CPU usage. +Note: The functionality in this module is unsupported on Windows """ from __future__ import print_function |