From 8648e2cda55cbc84cac4cc6316f3f51f9b38ba7d Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Thu, 28 Sep 2017 11:27:40 -0700 Subject: Add testing for spack blame; refactor llnl.util tests --- lib/spack/llnl/util/lang.py | 29 +- lib/spack/spack/test/cmd/blame.py | 4 +- lib/spack/spack/test/file_list.py | 277 --------- lib/spack/spack/test/link_tree.py | 152 ----- lib/spack/spack/test/llnl/util/file_list.py | 277 +++++++++ lib/spack/spack/test/llnl/util/lang.py | 96 +++ lib/spack/spack/test/llnl/util/link_tree.py | 152 +++++ lib/spack/spack/test/llnl/util/lock.py | 902 ++++++++++++++++++++++++++++ lib/spack/spack/test/llnl/util/log.py | 95 +++ lib/spack/spack/test/lock.py | 902 ---------------------------- lib/spack/spack/test/log.py | 95 --- 11 files changed, 1532 insertions(+), 1449 deletions(-) delete mode 100644 lib/spack/spack/test/file_list.py delete mode 100644 lib/spack/spack/test/link_tree.py create mode 100644 lib/spack/spack/test/llnl/util/file_list.py create mode 100644 lib/spack/spack/test/llnl/util/lang.py create mode 100644 lib/spack/spack/test/llnl/util/link_tree.py create mode 100644 lib/spack/spack/test/llnl/util/lock.py create mode 100644 lib/spack/spack/test/llnl/util/log.py delete mode 100644 lib/spack/spack/test/lock.py delete mode 100644 lib/spack/spack/test/log.py (limited to 'lib') diff --git a/lib/spack/llnl/util/lang.py b/lib/spack/llnl/util/lang.py index 07338b21ef..00e6c10b3f 100644 --- a/lib/spack/llnl/util/lang.py +++ b/lib/spack/llnl/util/lang.py @@ -22,6 +22,8 @@ # License along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## +from __future__ import division + import os import re import functools @@ -104,21 +106,6 @@ def index_by(objects, *funcs): return result -def partition_list(elements, predicate): - """Partition a list into two lists, the first containing elements - for which the predicate evaluates to true, the second containing - those for which it is false. - """ - trues = [] - falses = [] - for elt in elements: - if predicate(elt): - trues.append(elt) - else: - falses.append(elt) - return trues, falses - - def caller_locals(): """This will return the locals of the *parent* of the caller. This allows a function to insert variables into its caller's @@ -425,30 +412,30 @@ def pretty_date(time, now=None): if second_diff < 120: return "a minute ago" if second_diff < 3600: - return str(second_diff / 60) + " minutes ago" + return str(second_diff // 60) + " minutes ago" if second_diff < 7200: return "an hour ago" if second_diff < 86400: - return str(second_diff / 3600) + " hours ago" + return str(second_diff // 3600) + " hours ago" if day_diff == 1: return "yesterday" if day_diff < 7: return str(day_diff) + " days ago" if day_diff < 28: - weeks = day_diff / 7 + weeks = day_diff // 7 if weeks == 1: return "a week ago" else: - return str(day_diff / 7) + " weeks ago" + return str(day_diff // 7) + " weeks ago" if day_diff < 365: - months = day_diff / 30 + months = day_diff // 30 if months == 1: return "a month ago" elif months == 12: months -= 1 return str(months) + " months ago" - diff = day_diff / 365 + diff = day_diff // 365 if diff == 1: return "a year ago" else: diff --git a/lib/spack/spack/test/cmd/blame.py b/lib/spack/spack/test/cmd/blame.py index 0031052b24..9342ff1f7c 100644 --- a/lib/spack/spack/test/cmd/blame.py +++ b/lib/spack/spack/test/cmd/blame.py @@ -55,5 +55,5 @@ def test_blame_by_git(builtin_mock, capfd): """Sanity check the blame command to make sure it works.""" with capfd.disabled(): out = blame('--git', 'mpich') - assert 'Mpich' in out - assert 'mock_packages' in out + assert 'class Mpich' in out + assert ' homepage = "http://www.mpich.org"' in out diff --git a/lib/spack/spack/test/file_list.py b/lib/spack/spack/test/file_list.py deleted file mode 100644 index 4b71881313..0000000000 --- a/lib/spack/spack/test/file_list.py +++ /dev/null @@ -1,277 +0,0 @@ -############################################################################## -# Copyright (c) 2013-2017, Lawrence Livermore National Security, LLC. -# Produced at the Lawrence Livermore National Laboratory. -# -# This file is part of Spack. -# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. -# LLNL-CODE-647188 -# -# For details, see https://github.com/llnl/spack -# Please also see the NOTICE and LICENSE files for our notice and the LGPL. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU Lesser General Public License (as -# published by the Free Software Foundation) version 2.1, February 1999. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and -# conditions of the GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -############################################################################## - -import fnmatch -import os - -import pytest -import six -import spack -from llnl.util.filesystem import LibraryList, HeaderList -from llnl.util.filesystem import find_libraries, find_headers - - -@pytest.fixture() -def library_list(): - """Returns an instance of LibraryList.""" - # Test all valid extensions: ['.a', '.dylib', '.so'] - l = [ - '/dir1/liblapack.a', - '/dir2/libpython3.6.dylib', # name may contain periods - '/dir1/libblas.a', - '/dir3/libz.so', - 'libmpi.so.20.10.1', # shared object libraries may be versioned - ] - - return LibraryList(l) - - -@pytest.fixture() -def header_list(): - """Returns an instance of header list""" - # Test all valid extensions: ['.h', '.hpp', '.hh', '.cuh'] - h = [ - '/dir1/Python.h', - '/dir2/date.time.h', - '/dir1/pyconfig.hpp', - '/dir3/core.hh', - 'pymem.cuh', - ] - h = HeaderList(h) - h.add_macro('-DBOOST_LIB_NAME=boost_regex') - h.add_macro('-DBOOST_DYN_LINK') - return h - - -class TestLibraryList(object): - - def test_repr(self, library_list): - x = eval(repr(library_list)) - assert library_list == x - - def test_joined_and_str(self, library_list): - - s1 = library_list.joined() - expected = '/dir1/liblapack.a /dir2/libpython3.6.dylib /dir1/libblas.a /dir3/libz.so libmpi.so.20.10.1' # noqa: E501 - assert s1 == expected - - s2 = str(library_list) - assert s1 == s2 - - s3 = library_list.joined(';') - expected = '/dir1/liblapack.a;/dir2/libpython3.6.dylib;/dir1/libblas.a;/dir3/libz.so;libmpi.so.20.10.1' # noqa: E501 - assert s3 == expected - - def test_flags(self, library_list): - - search_flags = library_list.search_flags - assert '-L/dir1' in search_flags - assert '-L/dir2' in search_flags - assert '-L/dir3' in search_flags - assert isinstance(search_flags, str) - assert search_flags == '-L/dir1 -L/dir2 -L/dir3' - - link_flags = library_list.link_flags - assert '-llapack' in link_flags - assert '-lpython3.6' in link_flags - assert '-lblas' in link_flags - assert '-lz' in link_flags - assert '-lmpi' in link_flags - assert isinstance(link_flags, str) - assert link_flags == '-llapack -lpython3.6 -lblas -lz -lmpi' - - ld_flags = library_list.ld_flags - assert isinstance(ld_flags, str) - assert ld_flags == search_flags + ' ' + link_flags - - def test_paths_manipulation(self, library_list): - names = library_list.names - assert names == ['lapack', 'python3.6', 'blas', 'z', 'mpi'] - - directories = library_list.directories - assert directories == ['/dir1', '/dir2', '/dir3'] - - def test_get_item(self, library_list): - a = library_list[0] - assert a == '/dir1/liblapack.a' - - b = library_list[:] - assert type(b) == type(library_list) - assert library_list == b - assert library_list is not b - - def test_add(self, library_list): - pylist = [ - '/dir1/liblapack.a', # removed from the final list - '/dir2/libmpi.so', - '/dir4/libnew.a' - ] - another = LibraryList(pylist) - l = library_list + another - assert len(l) == 7 - - # Invariant : l == l + l - assert l == l + l - - # Always produce an instance of LibraryList - assert type(library_list + pylist) == type(library_list) - assert type(pylist + library_list) == type(library_list) - - -class TestHeaderList(object): - - def test_repr(self, header_list): - x = eval(repr(header_list)) - assert header_list == x - - def test_joined_and_str(self, header_list): - s1 = header_list.joined() - expected = '/dir1/Python.h /dir2/date.time.h /dir1/pyconfig.hpp /dir3/core.hh pymem.cuh' # noqa: E501 - assert s1 == expected - - s2 = str(header_list) - assert s1 == s2 - - s3 = header_list.joined(';') - expected = '/dir1/Python.h;/dir2/date.time.h;/dir1/pyconfig.hpp;/dir3/core.hh;pymem.cuh' # noqa: E501 - assert s3 == expected - - def test_flags(self, header_list): - include_flags = header_list.include_flags - assert '-I/dir1' in include_flags - assert '-I/dir2' in include_flags - assert '-I/dir3' in include_flags - assert isinstance(include_flags, str) - assert include_flags == '-I/dir1 -I/dir2 -I/dir3' - - macros = header_list.macro_definitions - assert '-DBOOST_LIB_NAME=boost_regex' in macros - assert '-DBOOST_DYN_LINK' in macros - assert isinstance(macros, str) - assert macros == '-DBOOST_LIB_NAME=boost_regex -DBOOST_DYN_LINK' - - cpp_flags = header_list.cpp_flags - assert isinstance(cpp_flags, str) - assert cpp_flags == include_flags + ' ' + macros - - def test_paths_manipulation(self, header_list): - names = header_list.names - assert names == ['Python', 'date.time', 'pyconfig', 'core', 'pymem'] - - directories = header_list.directories - assert directories == ['/dir1', '/dir2', '/dir3'] - - def test_get_item(self, header_list): - a = header_list[0] - assert a == '/dir1/Python.h' - - b = header_list[:] - assert type(b) == type(header_list) - assert header_list == b - assert header_list is not b - - def test_add(self, header_list): - pylist = [ - '/dir1/Python.h', # removed from the final list - '/dir2/pyconfig.hpp', - '/dir4/date.time.h' - ] - another = HeaderList(pylist) - h = header_list + another - assert len(h) == 7 - - # Invariant : l == l + l - assert h == h + h - - # Always produce an instance of HeaderList - assert type(header_list + pylist) == type(header_list) - assert type(pylist + header_list) == type(header_list) - - -#: Directory where the data for the test below is stored -search_dir = os.path.join(spack.test_path, 'data', 'directory_search') - - -@pytest.mark.parametrize('search_fn,search_list,root,kwargs', [ - (find_libraries, 'liba', search_dir, {'recurse': True}), - (find_libraries, ['liba'], search_dir, {'recurse': True}), - (find_libraries, 'libb', search_dir, {'recurse': True}), - (find_libraries, ['libc'], search_dir, {'recurse': True}), - (find_libraries, ['libc', 'liba'], search_dir, {'recurse': True}), - (find_libraries, ['liba', 'libc'], search_dir, {'recurse': True}), - (find_libraries, ['libc', 'libb', 'liba'], search_dir, {'recurse': True}), - (find_libraries, ['liba', 'libc'], search_dir, {'recurse': True}), - (find_libraries, - ['libc', 'libb', 'liba'], - search_dir, - {'recurse': True, 'shared': False} - ), - (find_headers, 'a', search_dir, {'recurse': True}), - (find_headers, ['a'], search_dir, {'recurse': True}), - (find_headers, 'b', search_dir, {'recurse': True}), - (find_headers, ['c'], search_dir, {'recurse': True}), - (find_headers, ['c', 'a'], search_dir, {'recurse': True}), - (find_headers, ['a', 'c'], search_dir, {'recurse': True}), - (find_headers, ['c', 'b', 'a'], search_dir, {'recurse': True}), - (find_headers, ['a', 'c'], search_dir, {'recurse': True}), - (find_libraries, - ['liba', 'libd'], - os.path.join(search_dir, 'b'), - {'recurse': False} - ), - (find_headers, - ['b', 'd'], - os.path.join(search_dir, 'b'), - {'recurse': False} - ), -]) -def test_searching_order(search_fn, search_list, root, kwargs): - - # Test search - result = search_fn(search_list, root, **kwargs) - - # The tests are set-up so that something is always found - assert len(result) != 0 - - # Now reverse the result and start discarding things - # as soon as you have matches. In the end the list should - # be emptied. - l = list(reversed(result)) - - # At this point make sure the search list is a sequence - if isinstance(search_list, six.string_types): - search_list = [search_list] - - # Discard entries in the order they appear in search list - for x in search_list: - try: - while fnmatch.fnmatch(l[-1], x) or x in l[-1]: - l.pop() - except IndexError: - # List is empty - pass - - # List should be empty here - assert len(l) == 0 diff --git a/lib/spack/spack/test/link_tree.py b/lib/spack/spack/test/link_tree.py deleted file mode 100644 index 64836a86f3..0000000000 --- a/lib/spack/spack/test/link_tree.py +++ /dev/null @@ -1,152 +0,0 @@ -############################################################################## -# Copyright (c) 2013-2017, Lawrence Livermore National Security, LLC. -# Produced at the Lawrence Livermore National Laboratory. -# -# This file is part of Spack. -# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. -# LLNL-CODE-647188 -# -# For details, see https://github.com/llnl/spack -# Please also see the NOTICE and LICENSE files for our notice and the LGPL. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU Lesser General Public License (as -# published by the Free Software Foundation) version 2.1, February 1999. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and -# conditions of the GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -############################################################################## -import os - -import pytest -from llnl.util.filesystem import * -from llnl.util.link_tree import LinkTree -from spack.stage import Stage - - -@pytest.fixture() -def stage(): - """Creates a stage with the directory structure for the tests.""" - s = Stage('link-tree-test') - s.create() - - with working_dir(s.path): - touchp('source/1') - touchp('source/a/b/2') - touchp('source/a/b/3') - touchp('source/c/4') - touchp('source/c/d/5') - touchp('source/c/d/6') - touchp('source/c/d/e/7') - - yield s - - s.destroy() - - -@pytest.fixture() -def link_tree(stage): - """Return a properly initialized LinkTree instance.""" - source_path = os.path.join(stage.path, 'source') - return LinkTree(source_path) - - -def check_file_link(filename): - assert os.path.isfile(filename) - assert os.path.islink(filename) - - -def check_dir(filename): - assert os.path.isdir(filename) - - -def test_merge_to_new_directory(stage, link_tree): - with working_dir(stage.path): - link_tree.merge('dest') - - check_file_link('dest/1') - check_file_link('dest/a/b/2') - check_file_link('dest/a/b/3') - check_file_link('dest/c/4') - check_file_link('dest/c/d/5') - check_file_link('dest/c/d/6') - check_file_link('dest/c/d/e/7') - - link_tree.unmerge('dest') - - assert not os.path.exists('dest') - - -def test_merge_to_existing_directory(stage, link_tree): - with working_dir(stage.path): - - touchp('dest/x') - touchp('dest/a/b/y') - - link_tree.merge('dest') - - check_file_link('dest/1') - check_file_link('dest/a/b/2') - check_file_link('dest/a/b/3') - check_file_link('dest/c/4') - check_file_link('dest/c/d/5') - check_file_link('dest/c/d/6') - check_file_link('dest/c/d/e/7') - - assert os.path.isfile('dest/x') - assert os.path.isfile('dest/a/b/y') - - link_tree.unmerge('dest') - - assert os.path.isfile('dest/x') - assert os.path.isfile('dest/a/b/y') - - assert not os.path.isfile('dest/1') - assert not os.path.isfile('dest/a/b/2') - assert not os.path.isfile('dest/a/b/3') - assert not os.path.isfile('dest/c/4') - assert not os.path.isfile('dest/c/d/5') - assert not os.path.isfile('dest/c/d/6') - assert not os.path.isfile('dest/c/d/e/7') - - -def test_merge_with_empty_directories(stage, link_tree): - with working_dir(stage.path): - mkdirp('dest/f/g') - mkdirp('dest/a/b/h') - - link_tree.merge('dest') - link_tree.unmerge('dest') - - assert not os.path.exists('dest/1') - assert not os.path.exists('dest/a/b/2') - assert not os.path.exists('dest/a/b/3') - assert not os.path.exists('dest/c/4') - assert not os.path.exists('dest/c/d/5') - assert not os.path.exists('dest/c/d/6') - assert not os.path.exists('dest/c/d/e/7') - - assert os.path.isdir('dest/a/b/h') - assert os.path.isdir('dest/f/g') - - -def test_ignore(stage, link_tree): - with working_dir(stage.path): - touchp('source/.spec') - touchp('dest/.spec') - - link_tree.merge('dest', ignore=lambda x: x == '.spec') - link_tree.unmerge('dest', ignore=lambda x: x == '.spec') - - assert not os.path.exists('dest/1') - assert not os.path.exists('dest/a') - assert not os.path.exists('dest/c') - - assert os.path.isfile('source/.spec') - assert os.path.isfile('dest/.spec') diff --git a/lib/spack/spack/test/llnl/util/file_list.py b/lib/spack/spack/test/llnl/util/file_list.py new file mode 100644 index 0000000000..4b71881313 --- /dev/null +++ b/lib/spack/spack/test/llnl/util/file_list.py @@ -0,0 +1,277 @@ +############################################################################## +# Copyright (c) 2013-2017, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the NOTICE and LICENSE files for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## + +import fnmatch +import os + +import pytest +import six +import spack +from llnl.util.filesystem import LibraryList, HeaderList +from llnl.util.filesystem import find_libraries, find_headers + + +@pytest.fixture() +def library_list(): + """Returns an instance of LibraryList.""" + # Test all valid extensions: ['.a', '.dylib', '.so'] + l = [ + '/dir1/liblapack.a', + '/dir2/libpython3.6.dylib', # name may contain periods + '/dir1/libblas.a', + '/dir3/libz.so', + 'libmpi.so.20.10.1', # shared object libraries may be versioned + ] + + return LibraryList(l) + + +@pytest.fixture() +def header_list(): + """Returns an instance of header list""" + # Test all valid extensions: ['.h', '.hpp', '.hh', '.cuh'] + h = [ + '/dir1/Python.h', + '/dir2/date.time.h', + '/dir1/pyconfig.hpp', + '/dir3/core.hh', + 'pymem.cuh', + ] + h = HeaderList(h) + h.add_macro('-DBOOST_LIB_NAME=boost_regex') + h.add_macro('-DBOOST_DYN_LINK') + return h + + +class TestLibraryList(object): + + def test_repr(self, library_list): + x = eval(repr(library_list)) + assert library_list == x + + def test_joined_and_str(self, library_list): + + s1 = library_list.joined() + expected = '/dir1/liblapack.a /dir2/libpython3.6.dylib /dir1/libblas.a /dir3/libz.so libmpi.so.20.10.1' # noqa: E501 + assert s1 == expected + + s2 = str(library_list) + assert s1 == s2 + + s3 = library_list.joined(';') + expected = '/dir1/liblapack.a;/dir2/libpython3.6.dylib;/dir1/libblas.a;/dir3/libz.so;libmpi.so.20.10.1' # noqa: E501 + assert s3 == expected + + def test_flags(self, library_list): + + search_flags = library_list.search_flags + assert '-L/dir1' in search_flags + assert '-L/dir2' in search_flags + assert '-L/dir3' in search_flags + assert isinstance(search_flags, str) + assert search_flags == '-L/dir1 -L/dir2 -L/dir3' + + link_flags = library_list.link_flags + assert '-llapack' in link_flags + assert '-lpython3.6' in link_flags + assert '-lblas' in link_flags + assert '-lz' in link_flags + assert '-lmpi' in link_flags + assert isinstance(link_flags, str) + assert link_flags == '-llapack -lpython3.6 -lblas -lz -lmpi' + + ld_flags = library_list.ld_flags + assert isinstance(ld_flags, str) + assert ld_flags == search_flags + ' ' + link_flags + + def test_paths_manipulation(self, library_list): + names = library_list.names + assert names == ['lapack', 'python3.6', 'blas', 'z', 'mpi'] + + directories = library_list.directories + assert directories == ['/dir1', '/dir2', '/dir3'] + + def test_get_item(self, library_list): + a = library_list[0] + assert a == '/dir1/liblapack.a' + + b = library_list[:] + assert type(b) == type(library_list) + assert library_list == b + assert library_list is not b + + def test_add(self, library_list): + pylist = [ + '/dir1/liblapack.a', # removed from the final list + '/dir2/libmpi.so', + '/dir4/libnew.a' + ] + another = LibraryList(pylist) + l = library_list + another + assert len(l) == 7 + + # Invariant : l == l + l + assert l == l + l + + # Always produce an instance of LibraryList + assert type(library_list + pylist) == type(library_list) + assert type(pylist + library_list) == type(library_list) + + +class TestHeaderList(object): + + def test_repr(self, header_list): + x = eval(repr(header_list)) + assert header_list == x + + def test_joined_and_str(self, header_list): + s1 = header_list.joined() + expected = '/dir1/Python.h /dir2/date.time.h /dir1/pyconfig.hpp /dir3/core.hh pymem.cuh' # noqa: E501 + assert s1 == expected + + s2 = str(header_list) + assert s1 == s2 + + s3 = header_list.joined(';') + expected = '/dir1/Python.h;/dir2/date.time.h;/dir1/pyconfig.hpp;/dir3/core.hh;pymem.cuh' # noqa: E501 + assert s3 == expected + + def test_flags(self, header_list): + include_flags = header_list.include_flags + assert '-I/dir1' in include_flags + assert '-I/dir2' in include_flags + assert '-I/dir3' in include_flags + assert isinstance(include_flags, str) + assert include_flags == '-I/dir1 -I/dir2 -I/dir3' + + macros = header_list.macro_definitions + assert '-DBOOST_LIB_NAME=boost_regex' in macros + assert '-DBOOST_DYN_LINK' in macros + assert isinstance(macros, str) + assert macros == '-DBOOST_LIB_NAME=boost_regex -DBOOST_DYN_LINK' + + cpp_flags = header_list.cpp_flags + assert isinstance(cpp_flags, str) + assert cpp_flags == include_flags + ' ' + macros + + def test_paths_manipulation(self, header_list): + names = header_list.names + assert names == ['Python', 'date.time', 'pyconfig', 'core', 'pymem'] + + directories = header_list.directories + assert directories == ['/dir1', '/dir2', '/dir3'] + + def test_get_item(self, header_list): + a = header_list[0] + assert a == '/dir1/Python.h' + + b = header_list[:] + assert type(b) == type(header_list) + assert header_list == b + assert header_list is not b + + def test_add(self, header_list): + pylist = [ + '/dir1/Python.h', # removed from the final list + '/dir2/pyconfig.hpp', + '/dir4/date.time.h' + ] + another = HeaderList(pylist) + h = header_list + another + assert len(h) == 7 + + # Invariant : l == l + l + assert h == h + h + + # Always produce an instance of HeaderList + assert type(header_list + pylist) == type(header_list) + assert type(pylist + header_list) == type(header_list) + + +#: Directory where the data for the test below is stored +search_dir = os.path.join(spack.test_path, 'data', 'directory_search') + + +@pytest.mark.parametrize('search_fn,search_list,root,kwargs', [ + (find_libraries, 'liba', search_dir, {'recurse': True}), + (find_libraries, ['liba'], search_dir, {'recurse': True}), + (find_libraries, 'libb', search_dir, {'recurse': True}), + (find_libraries, ['libc'], search_dir, {'recurse': True}), + (find_libraries, ['libc', 'liba'], search_dir, {'recurse': True}), + (find_libraries, ['liba', 'libc'], search_dir, {'recurse': True}), + (find_libraries, ['libc', 'libb', 'liba'], search_dir, {'recurse': True}), + (find_libraries, ['liba', 'libc'], search_dir, {'recurse': True}), + (find_libraries, + ['libc', 'libb', 'liba'], + search_dir, + {'recurse': True, 'shared': False} + ), + (find_headers, 'a', search_dir, {'recurse': True}), + (find_headers, ['a'], search_dir, {'recurse': True}), + (find_headers, 'b', search_dir, {'recurse': True}), + (find_headers, ['c'], search_dir, {'recurse': True}), + (find_headers, ['c', 'a'], search_dir, {'recurse': True}), + (find_headers, ['a', 'c'], search_dir, {'recurse': True}), + (find_headers, ['c', 'b', 'a'], search_dir, {'recurse': True}), + (find_headers, ['a', 'c'], search_dir, {'recurse': True}), + (find_libraries, + ['liba', 'libd'], + os.path.join(search_dir, 'b'), + {'recurse': False} + ), + (find_headers, + ['b', 'd'], + os.path.join(search_dir, 'b'), + {'recurse': False} + ), +]) +def test_searching_order(search_fn, search_list, root, kwargs): + + # Test search + result = search_fn(search_list, root, **kwargs) + + # The tests are set-up so that something is always found + assert len(result) != 0 + + # Now reverse the result and start discarding things + # as soon as you have matches. In the end the list should + # be emptied. + l = list(reversed(result)) + + # At this point make sure the search list is a sequence + if isinstance(search_list, six.string_types): + search_list = [search_list] + + # Discard entries in the order they appear in search list + for x in search_list: + try: + while fnmatch.fnmatch(l[-1], x) or x in l[-1]: + l.pop() + except IndexError: + # List is empty + pass + + # List should be empty here + assert len(l) == 0 diff --git a/lib/spack/spack/test/llnl/util/lang.py b/lib/spack/spack/test/llnl/util/lang.py new file mode 100644 index 0000000000..133ef6cc6c --- /dev/null +++ b/lib/spack/spack/test/llnl/util/lang.py @@ -0,0 +1,96 @@ +############################################################################## +# Copyright (c) 2013-2017, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the NOTICE and LICENSE files for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +import pytest +from datetime import datetime, timedelta + +from llnl.util.lang import * + + +def test_pretty_date(): + """Make sure pretty_date prints the right dates.""" + now = datetime.now() + + just_now = now - timedelta(seconds=5) + assert pretty_date(just_now, now) == "just now" + + seconds = now - timedelta(seconds=30) + assert pretty_date(seconds, now) == "30 seconds ago" + + a_minute = now - timedelta(seconds=60) + assert pretty_date(a_minute, now) == "a minute ago" + + minutes = now - timedelta(seconds=1800) + assert pretty_date(minutes, now) == "30 minutes ago" + + an_hour = now - timedelta(hours=1) + assert pretty_date(an_hour, now) == "an hour ago" + + hours = now - timedelta(hours=2) + assert pretty_date(hours, now) == "2 hours ago" + + yesterday = now - timedelta(days=1) + assert pretty_date(yesterday, now) == "yesterday" + + days = now - timedelta(days=3) + assert pretty_date(days, now) == "3 days ago" + + a_week = now - timedelta(weeks=1) + assert pretty_date(a_week, now) == "a week ago" + + weeks = now - timedelta(weeks=2) + assert pretty_date(weeks, now) == "2 weeks ago" + + a_month = now - timedelta(days=30) + assert pretty_date(a_month, now) == "a month ago" + + months = now - timedelta(days=60) + assert pretty_date(months, now) == "2 months ago" + + a_year = now - timedelta(days=365) + assert pretty_date(a_year, now) == "a year ago" + + years = now - timedelta(days=365 * 2) + assert pretty_date(years, now) == "2 years ago" + + +def test_match_predicate(): + matcher = match_predicate(lambda x: True) + assert matcher('foo') + assert matcher('bar') + assert matcher('baz') + + matcher = match_predicate(['foo', 'bar']) + assert matcher('foo') + assert matcher('bar') + assert not matcher('baz') + + matcher = match_predicate(r'^(foo|bar)$') + assert matcher('foo') + assert matcher('bar') + assert not matcher('baz') + + with pytest.raises(ValueError): + matcher = match_predicate(object()) + matcher('foo') diff --git a/lib/spack/spack/test/llnl/util/link_tree.py b/lib/spack/spack/test/llnl/util/link_tree.py new file mode 100644 index 0000000000..64836a86f3 --- /dev/null +++ b/lib/spack/spack/test/llnl/util/link_tree.py @@ -0,0 +1,152 @@ +############################################################################## +# Copyright (c) 2013-2017, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the NOTICE and LICENSE files for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +import os + +import pytest +from llnl.util.filesystem import * +from llnl.util.link_tree import LinkTree +from spack.stage import Stage + + +@pytest.fixture() +def stage(): + """Creates a stage with the directory structure for the tests.""" + s = Stage('link-tree-test') + s.create() + + with working_dir(s.path): + touchp('source/1') + touchp('source/a/b/2') + touchp('source/a/b/3') + touchp('source/c/4') + touchp('source/c/d/5') + touchp('source/c/d/6') + touchp('source/c/d/e/7') + + yield s + + s.destroy() + + +@pytest.fixture() +def link_tree(stage): + """Return a properly initialized LinkTree instance.""" + source_path = os.path.join(stage.path, 'source') + return LinkTree(source_path) + + +def check_file_link(filename): + assert os.path.isfile(filename) + assert os.path.islink(filename) + + +def check_dir(filename): + assert os.path.isdir(filename) + + +def test_merge_to_new_directory(stage, link_tree): + with working_dir(stage.path): + link_tree.merge('dest') + + check_file_link('dest/1') + check_file_link('dest/a/b/2') + check_file_link('dest/a/b/3') + check_file_link('dest/c/4') + check_file_link('dest/c/d/5') + check_file_link('dest/c/d/6') + check_file_link('dest/c/d/e/7') + + link_tree.unmerge('dest') + + assert not os.path.exists('dest') + + +def test_merge_to_existing_directory(stage, link_tree): + with working_dir(stage.path): + + touchp('dest/x') + touchp('dest/a/b/y') + + link_tree.merge('dest') + + check_file_link('dest/1') + check_file_link('dest/a/b/2') + check_file_link('dest/a/b/3') + check_file_link('dest/c/4') + check_file_link('dest/c/d/5') + check_file_link('dest/c/d/6') + check_file_link('dest/c/d/e/7') + + assert os.path.isfile('dest/x') + assert os.path.isfile('dest/a/b/y') + + link_tree.unmerge('dest') + + assert os.path.isfile('dest/x') + assert os.path.isfile('dest/a/b/y') + + assert not os.path.isfile('dest/1') + assert not os.path.isfile('dest/a/b/2') + assert not os.path.isfile('dest/a/b/3') + assert not os.path.isfile('dest/c/4') + assert not os.path.isfile('dest/c/d/5') + assert not os.path.isfile('dest/c/d/6') + assert not os.path.isfile('dest/c/d/e/7') + + +def test_merge_with_empty_directories(stage, link_tree): + with working_dir(stage.path): + mkdirp('dest/f/g') + mkdirp('dest/a/b/h') + + link_tree.merge('dest') + link_tree.unmerge('dest') + + assert not os.path.exists('dest/1') + assert not os.path.exists('dest/a/b/2') + assert not os.path.exists('dest/a/b/3') + assert not os.path.exists('dest/c/4') + assert not os.path.exists('dest/c/d/5') + assert not os.path.exists('dest/c/d/6') + assert not os.path.exists('dest/c/d/e/7') + + assert os.path.isdir('dest/a/b/h') + assert os.path.isdir('dest/f/g') + + +def test_ignore(stage, link_tree): + with working_dir(stage.path): + touchp('source/.spec') + touchp('dest/.spec') + + link_tree.merge('dest', ignore=lambda x: x == '.spec') + link_tree.unmerge('dest', ignore=lambda x: x == '.spec') + + assert not os.path.exists('dest/1') + assert not os.path.exists('dest/a') + assert not os.path.exists('dest/c') + + assert os.path.isfile('source/.spec') + assert os.path.isfile('dest/.spec') diff --git a/lib/spack/spack/test/llnl/util/lock.py b/lib/spack/spack/test/llnl/util/lock.py new file mode 100644 index 0000000000..208777ae51 --- /dev/null +++ b/lib/spack/spack/test/llnl/util/lock.py @@ -0,0 +1,902 @@ +############################################################################## +# Copyright (c) 2013-2017, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the NOTICE and LICENSE files for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +"""These tests ensure that our lock works correctly. + +This can be run in two ways. + +First, it can be run as a node-local test, with a typical invocation like +this:: + + spack test lock + +You can *also* run it as an MPI program, which allows you to test locks +across nodes. So, e.g., you can run the test like this:: + + mpirun -n 7 spack test lock + +And it will test locking correctness among MPI processes. Ideally, you +want the MPI processes to span across multiple nodes, so, e.g., for SLURM +you might do this:: + + srun -N 7 -n 7 -m cyclic spack test lock + +You can use this to test whether your shared filesystem properly supports +POSIX reader-writer locking with byte ranges through fcntl. + +If you want to test on multiple filesystems, you can modify the +``locations`` list below. By default it looks like this:: + + locations = [ + tempfile.gettempdir(), # standard tmp directory (potentially local) + '/nfs/tmp2/%u', # NFS tmp mount + '/p/lscratch*/%u' # Lustre scratch mount + ] + +Add names and paths for your preferred filesystem mounts to test on them; +the tests are parametrized to run on all the filesystems listed in this +dict. Note that 'tmp' will be skipped for MPI testing, as it is often a +node-local filesystem, and multi-node tests will fail if the locks aren't +actually on a shared filesystem. + +""" +import os +import shutil +import tempfile +import traceback +import glob +import getpass +from contextlib import contextmanager +from multiprocessing import Process + +import pytest + +from llnl.util.filesystem import join_path, touch +from llnl.util.lock import * +from spack.util.multiproc import Barrier + + +# +# This test can be run with MPI. MPI is "enabled" if we can import +# mpi4py and the number of total MPI processes is greater than 1. +# Otherwise it just runs as a node-local test. +# +# NOTE: MPI mode is different from node-local mode in that node-local +# mode will spawn its own test processes, while MPI mode assumes you've +# run this script as a SPMD application. In MPI mode, no additional +# processes are spawned, and you need to ensure that you mpirun the +# script with enough processes for all the multiproc_test cases below. +# +# If you don't run with enough processes, tests that require more +# processes than you currently have will be skipped. +# +mpi = False +comm = None +try: + from mpi4py import MPI + comm = MPI.COMM_WORLD + if comm.size > 1: + mpi = True +except: + pass + + +"""This is a list of filesystem locations to test locks in. Paths are +expanded so that %u is replaced with the current username. '~' is also +legal and will be expanded to the user's home directory. + +Tests are skipped for directories that don't exist, so you'll need to +update this with the locations of NFS, Lustre, and other mounts on your +system. +""" +locations = [ + tempfile.gettempdir(), + os.path.join('/nfs/tmp2/', getpass.getuser()), + os.path.join('/p/lscratch*/', getpass.getuser()), +] + +"""This is the longest a failed multiproc test will take. +Barriers will time out and raise an exception after this interval. +In MPI mode, barriers don't time out (they hang). See mpi_multiproc_test. +""" +barrier_timeout = 5 + +"""This is the lock timeout for expected failures. +This may need to be higher for some filesystems.""" +lock_fail_timeout = 0.1 + + +@contextmanager +def read_only(path): + orginal_mode = os.stat(path).st_mode + os.chmod(path, 0o444) + yield + os.chmod(path, orginal_mode) + + +@pytest.fixture(scope='session', params=locations) +def lock_test_directory(request): + """This fixture causes tests to be executed for many different mounts. + + See the ``locations`` dict above for details. + """ + return request.param + + +@pytest.fixture(scope='session') +def lock_dir(lock_test_directory): + parent = next((p for p in glob.glob(lock_test_directory) + if os.path.exists(p) and os.access(p, os.W_OK)), None) + if not parent: + # Skip filesystems that don't exist or aren't writable + pytest.skip("requires filesystem: '%s'" % lock_test_directory) + elif mpi and parent == tempfile.gettempdir(): + # Skip local tmp test for MPI runs + pytest.skip("skipping local tmp directory for MPI test.") + + tempdir = None + if not mpi or comm.rank == 0: + tempdir = tempfile.mkdtemp(dir=parent) + if mpi: + tempdir = comm.bcast(tempdir) + + yield tempdir + + if mpi: + # rank 0 may get here before others, in which case it'll try to + # remove the directory while other processes try to re-create the + # lock. This will give errno 39: directory not empty. Use a + # barrier to ensure everyone is done first. + comm.barrier() + + if not mpi or comm.rank == 0: + shutil.rmtree(tempdir) + + +@pytest.fixture +def private_lock_path(lock_dir): + """In MPI mode, this is a private lock for each rank in a multiproc test. + + For other modes, it is the same as a shared lock. + """ + lock_file = join_path(lock_dir, 'lockfile') + if mpi: + lock_file += '.%s' % comm.rank + yield lock_file + + +@pytest.fixture +def lock_path(lock_dir): + """This lock is shared among all processes in a multiproc test.""" + lock_file = join_path(lock_dir, 'lockfile') + yield lock_file + + +def local_multiproc_test(*functions): + """Order some processes using simple barrier synchronization.""" + b = Barrier(len(functions), timeout=barrier_timeout) + procs = [Process(target=f, args=(b,)) for f in functions] + + for p in procs: + p.start() + + for p in procs: + p.join() + assert p.exitcode == 0 + + +def mpi_multiproc_test(*functions): + """SPMD version of multiproc test. + + This needs to be run like so: + + srun spack test lock + + Each process executes its corresponding function. This is different + from ``multiproc_test`` above, which spawns the processes. This will + skip tests if there are too few processes to run them. + """ + procs = len(functions) + if procs > comm.size: + pytest.skip("requires at least %d MPI processes" % procs) + + comm.Barrier() # barrier before each MPI test + + include = comm.rank < len(functions) + subcomm = comm.Split(include) + + class subcomm_barrier(object): + """Stand-in for multiproc barrier for MPI-parallel jobs.""" + def wait(self): + subcomm.Barrier() + + if include: + try: + functions[subcomm.rank](subcomm_barrier()) + except: + # aborting is the best we can do for MPI tests without + # hanging, since we're using MPI barriers. This will fail + # early and it loses the nice pytest output, but at least it + # gets use a stacktrace on the processes that failed. + traceback.print_exc() + comm.Abort() + subcomm.Free() + + comm.Barrier() # barrier after each MPI test. + + +"""``multiproc_test()`` should be called by tests below. +``multiproc_test()`` will work for either MPI runs or for local runs. +""" +multiproc_test = mpi_multiproc_test if mpi else local_multiproc_test + + +# +# Process snippets below can be composed into tests. +# +def acquire_write(lock_path, start=0, length=0): + def fn(barrier): + lock = Lock(lock_path, start, length) + lock.acquire_write() # grab exclusive lock + barrier.wait() + barrier.wait() # hold the lock until timeout in other procs. + return fn + + +def acquire_read(lock_path, start=0, length=0): + def fn(barrier): + lock = Lock(lock_path, start, length) + lock.acquire_read() # grab shared lock + barrier.wait() + barrier.wait() # hold the lock until timeout in other procs. + return fn + + +def timeout_write(lock_path, start=0, length=0): + def fn(barrier): + lock = Lock(lock_path, start, length) + barrier.wait() # wait for lock acquire in first process + with pytest.raises(LockError): + lock.acquire_write(lock_fail_timeout) + barrier.wait() + return fn + + +def timeout_read(lock_path, start=0, length=0): + def fn(barrier): + lock = Lock(lock_path, start, length) + barrier.wait() # wait for lock acquire in first process + with pytest.raises(LockError): + lock.acquire_read(lock_fail_timeout) + barrier.wait() + return fn + + +# +# Test that exclusive locks on other processes time out when an +# exclusive lock is held. +# +def test_write_lock_timeout_on_write(lock_path): + multiproc_test( + acquire_write(lock_path), + timeout_write(lock_path)) + + +def test_write_lock_timeout_on_write_2(lock_path): + multiproc_test( + acquire_write(lock_path), + timeout_write(lock_path), + timeout_write(lock_path)) + + +def test_write_lock_timeout_on_write_3(lock_path): + multiproc_test( + acquire_write(lock_path), + timeout_write(lock_path), + timeout_write(lock_path), + timeout_write(lock_path)) + + +def test_write_lock_timeout_on_write_ranges(lock_path): + multiproc_test( + acquire_write(lock_path, 0, 1), + timeout_write(lock_path, 0, 1)) + + +def test_write_lock_timeout_on_write_ranges_2(lock_path): + multiproc_test( + acquire_write(lock_path, 0, 64), + acquire_write(lock_path, 65, 1), + timeout_write(lock_path, 0, 1), + timeout_write(lock_path, 63, 1)) + + +def test_write_lock_timeout_on_write_ranges_3(lock_path): + multiproc_test( + acquire_write(lock_path, 0, 1), + acquire_write(lock_path, 1, 1), + timeout_write(lock_path), + timeout_write(lock_path), + timeout_write(lock_path)) + + +def test_write_lock_timeout_on_write_ranges_4(lock_path): + multiproc_test( + acquire_write(lock_path, 0, 1), + acquire_write(lock_path, 1, 1), + acquire_write(lock_path, 2, 456), + acquire_write(lock_path, 500, 64), + timeout_write(lock_path), + timeout_write(lock_path), + timeout_write(lock_path)) + + +# +# Test that shared locks on other processes time out when an +# exclusive lock is held. +# +def test_read_lock_timeout_on_write(lock_path): + multiproc_test( + acquire_write(lock_path), + timeout_read(lock_path)) + + +def test_read_lock_timeout_on_write_2(lock_path): + multiproc_test( + acquire_write(lock_path), + timeout_read(lock_path), + timeout_read(lock_path)) + + +def test_read_lock_timeout_on_write_3(lock_path): + multiproc_test( + acquire_write(lock_path), + timeout_read(lock_path), + timeout_read(lock_path), + timeout_read(lock_path)) + + +def test_read_lock_timeout_on_write_ranges(lock_path): + """small write lock, read whole file.""" + multiproc_test( + acquire_write(lock_path, 0, 1), + timeout_read(lock_path)) + + +def test_read_lock_timeout_on_write_ranges_2(lock_path): + """small write lock, small read lock""" + multiproc_test( + acquire_write(lock_path, 0, 1), + timeout_read(lock_path, 0, 1)) + + +def test_read_lock_timeout_on_write_ranges_3(lock_path): + """two write locks, overlapping read locks""" + multiproc_test( + acquire_write(lock_path, 0, 1), + acquire_write(lock_path, 64, 128), + timeout_read(lock_path, 0, 1), + timeout_read(lock_path, 128, 256)) + + +# +# Test that exclusive locks time out when shared locks are held. +# +def test_write_lock_timeout_on_read(lock_path): + multiproc_test( + acquire_read(lock_path), + timeout_write(lock_path)) + + +def test_write_lock_timeout_on_read_2(lock_path): + multiproc_test( + acquire_read(lock_path), + timeout_write(lock_path), + timeout_write(lock_path)) + + +def test_write_lock_timeout_on_read_3(lock_path): + multiproc_test( + acquire_read(lock_path), + timeout_write(lock_path), + timeout_write(lock_path), + timeout_write(lock_path)) + + +def test_write_lock_timeout_on_read_ranges(lock_path): + multiproc_test( + acquire_read(lock_path, 0, 1), + timeout_write(lock_path)) + + +def test_write_lock_timeout_on_read_ranges_2(lock_path): + multiproc_test( + acquire_read(lock_path, 0, 1), + timeout_write(lock_path, 0, 1)) + + +def test_write_lock_timeout_on_read_ranges_3(lock_path): + multiproc_test( + acquire_read(lock_path, 0, 1), + acquire_read(lock_path, 10, 1), + timeout_write(lock_path, 0, 1), + timeout_write(lock_path, 10, 1)) + + +def test_write_lock_timeout_on_read_ranges_4(lock_path): + multiproc_test( + acquire_read(lock_path, 0, 64), + timeout_write(lock_path, 10, 1), + timeout_write(lock_path, 32, 1)) + + +def test_write_lock_timeout_on_read_ranges_5(lock_path): + multiproc_test( + acquire_read(lock_path, 64, 128), + timeout_write(lock_path, 65, 1), + timeout_write(lock_path, 127, 1), + timeout_write(lock_path, 90, 10)) + + +# +# Test that exclusive locks time while lots of shared locks are held. +# +def test_write_lock_timeout_with_multiple_readers_2_1(lock_path): + multiproc_test( + acquire_read(lock_path), + acquire_read(lock_path), + timeout_write(lock_path)) + + +def test_write_lock_timeout_with_multiple_readers_2_2(lock_path): + multiproc_test( + acquire_read(lock_path), + acquire_read(lock_path), + timeout_write(lock_path), + timeout_write(lock_path)) + + +def test_write_lock_timeout_with_multiple_readers_3_1(lock_path): + multiproc_test( + acquire_read(lock_path), + acquire_read(lock_path), + acquire_read(lock_path), + timeout_write(lock_path)) + + +def test_write_lock_timeout_with_multiple_readers_3_2(lock_path): + multiproc_test( + acquire_read(lock_path), + acquire_read(lock_path), + acquire_read(lock_path), + timeout_write(lock_path), + timeout_write(lock_path)) + + +def test_write_lock_timeout_with_multiple_readers_2_1_ranges(lock_path): + multiproc_test( + acquire_read(lock_path, 0, 10), + acquire_read(lock_path, 0.5, 10), + timeout_write(lock_path, 5, 5)) + + +def test_write_lock_timeout_with_multiple_readers_2_3_ranges(lock_path): + multiproc_test( + acquire_read(lock_path, 0, 10), + acquire_read(lock_path, 5, 15), + timeout_write(lock_path, 0, 1), + timeout_write(lock_path, 11, 3), + timeout_write(lock_path, 7, 1)) + + +def test_write_lock_timeout_with_multiple_readers_3_1_ranges(lock_path): + multiproc_test( + acquire_read(lock_path, 0, 5), + acquire_read(lock_path, 5, 5), + acquire_read(lock_path, 10, 5), + timeout_write(lock_path, 0, 15)) + + +def test_write_lock_timeout_with_multiple_readers_3_2_ranges(lock_path): + multiproc_test( + acquire_read(lock_path, 0, 5), + acquire_read(lock_path, 5, 5), + acquire_read(lock_path, 10, 5), + timeout_write(lock_path, 3, 10), + timeout_write(lock_path, 5, 1)) + + +# +# Test that read can be upgraded to write. +# +def test_upgrade_read_to_write(private_lock_path): + """Test that a read lock can be upgraded to a write lock. + + Note that to upgrade a read lock to a write lock, you have the be the + only holder of a read lock. Client code needs to coordinate that for + shared locks. For this test, we use a private lock just to test that an + upgrade is possible. + """ + # ensure lock file exists the first time, so we open it read-only + # to begin wtih. + touch(private_lock_path) + + lock = Lock(private_lock_path) + assert lock._reads == 0 + assert lock._writes == 0 + + lock.acquire_read() + assert lock._reads == 1 + assert lock._writes == 0 + assert lock._file.mode == 'r+' + + lock.acquire_write() + assert lock._reads == 1 + assert lock._writes == 1 + assert lock._file.mode == 'r+' + + lock.release_write() + assert lock._reads == 1 + assert lock._writes == 0 + assert lock._file.mode == 'r+' + + lock.release_read() + assert lock._reads == 0 + assert lock._writes == 0 + assert lock._file is None + + +# +# Test that read-only file can be read-locked but not write-locked. +# +def test_upgrade_read_to_write_fails_with_readonly_file(private_lock_path): + # ensure lock file exists the first time, so we open it read-only + # to begin wtih. + touch(private_lock_path) + + with read_only(private_lock_path): + lock = Lock(private_lock_path) + assert lock._reads == 0 + assert lock._writes == 0 + + lock.acquire_read() + assert lock._reads == 1 + assert lock._writes == 0 + assert lock._file.mode == 'r' + + with pytest.raises(LockError): + lock.acquire_write() + + +# +# Longer test case that ensures locks are reusable. Ordering is +# enforced by barriers throughout -- steps are shown with numbers. +# +def test_complex_acquire_and_release_chain(lock_path): + def p1(barrier): + lock = Lock(lock_path) + + lock.acquire_write() + barrier.wait() # ---------------------------------------- 1 + # others test timeout + barrier.wait() # ---------------------------------------- 2 + lock.release_write() # release and others acquire read + barrier.wait() # ---------------------------------------- 3 + with pytest.raises(LockError): + lock.acquire_write(lock_fail_timeout) + lock.acquire_read() + barrier.wait() # ---------------------------------------- 4 + lock.release_read() + barrier.wait() # ---------------------------------------- 5 + + # p2 upgrades read to write + barrier.wait() # ---------------------------------------- 6 + with pytest.raises(LockError): + lock.acquire_write(lock_fail_timeout) + with pytest.raises(LockError): + lock.acquire_read(lock_fail_timeout) + barrier.wait() # ---------------------------------------- 7 + # p2 releases write and read + barrier.wait() # ---------------------------------------- 8 + + # p3 acquires read + barrier.wait() # ---------------------------------------- 9 + # p3 upgrades read to write + barrier.wait() # ---------------------------------------- 10 + with pytest.raises(LockError): + lock.acquire_write(lock_fail_timeout) + with pytest.raises(LockError): + lock.acquire_read(lock_fail_timeout) + barrier.wait() # ---------------------------------------- 11 + # p3 releases locks + barrier.wait() # ---------------------------------------- 12 + lock.acquire_read() + barrier.wait() # ---------------------------------------- 13 + lock.release_read() + + def p2(barrier): + lock = Lock(lock_path) + + # p1 acquires write + barrier.wait() # ---------------------------------------- 1 + with pytest.raises(LockError): + lock.acquire_write(lock_fail_timeout) + with pytest.raises(LockError): + lock.acquire_read(lock_fail_timeout) + barrier.wait() # ---------------------------------------- 2 + lock.acquire_read() + barrier.wait() # ---------------------------------------- 3 + # p1 tests shared read + barrier.wait() # ---------------------------------------- 4 + # others release reads + barrier.wait() # ---------------------------------------- 5 + + lock.acquire_write() # upgrade read to write + barrier.wait() # ---------------------------------------- 6 + # others test timeout + barrier.wait() # ---------------------------------------- 7 + lock.release_write() # release read AND write (need both) + lock.release_read() + barrier.wait() # ---------------------------------------- 8 + + # p3 acquires read + barrier.wait() # ---------------------------------------- 9 + # p3 upgrades read to write + barrier.wait() # ---------------------------------------- 10 + with pytest.raises(LockError): + lock.acquire_write(lock_fail_timeout) + with pytest.raises(LockError): + lock.acquire_read(lock_fail_timeout) + barrier.wait() # ---------------------------------------- 11 + # p3 releases locks + barrier.wait() # ---------------------------------------- 12 + lock.acquire_read() + barrier.wait() # ---------------------------------------- 13 + lock.release_read() + + def p3(barrier): + lock = Lock(lock_path) + + # p1 acquires write + barrier.wait() # ---------------------------------------- 1 + with pytest.raises(LockError): + lock.acquire_write(lock_fail_timeout) + with pytest.raises(LockError): + lock.acquire_read(lock_fail_timeout) + barrier.wait() # ---------------------------------------- 2 + lock.acquire_read() + barrier.wait() # ---------------------------------------- 3 + # p1 tests shared read + barrier.wait() # ---------------------------------------- 4 + lock.release_read() + barrier.wait() # ---------------------------------------- 5 + + # p2 upgrades read to write + barrier.wait() # ---------------------------------------- 6 + with pytest.raises(LockError): + lock.acquire_write(lock_fail_timeout) + with pytest.raises(LockError): + lock.acquire_read(lock_fail_timeout) + barrier.wait() # ---------------------------------------- 7 + # p2 releases write & read + barrier.wait() # ---------------------------------------- 8 + + lock.acquire_read() + barrier.wait() # ---------------------------------------- 9 + lock.acquire_write() + barrier.wait() # ---------------------------------------- 10 + # others test timeout + barrier.wait() # ---------------------------------------- 11 + lock.release_read() # release read AND write in opposite + lock.release_write() # order from before on p2 + barrier.wait() # ---------------------------------------- 12 + lock.acquire_read() + barrier.wait() # ---------------------------------------- 13 + lock.release_read() + + multiproc_test(p1, p2, p3) + + +def test_transaction(lock_path): + def enter_fn(): + vals['entered'] = True + + def exit_fn(t, v, tb): + vals['exited'] = True + vals['exception'] = (t or v or tb) + + lock = Lock(lock_path) + vals = {'entered': False, 'exited': False, 'exception': False} + with ReadTransaction(lock, enter_fn, exit_fn): + pass + + assert vals['entered'] + assert vals['exited'] + assert not vals['exception'] + + vals = {'entered': False, 'exited': False, 'exception': False} + with WriteTransaction(lock, enter_fn, exit_fn): + pass + + assert vals['entered'] + assert vals['exited'] + assert not vals['exception'] + + +def test_transaction_with_exception(lock_path): + def enter_fn(): + vals['entered'] = True + + def exit_fn(t, v, tb): + vals['exited'] = True + vals['exception'] = (t or v or tb) + + lock = Lock(lock_path) + + def do_read_with_exception(): + with ReadTransaction(lock, enter_fn, exit_fn): + raise Exception() + + def do_write_with_exception(): + with WriteTransaction(lock, enter_fn, exit_fn): + raise Exception() + + vals = {'entered': False, 'exited': False, 'exception': False} + with pytest.raises(Exception): + do_read_with_exception() + assert vals['entered'] + assert vals['exited'] + assert vals['exception'] + + vals = {'entered': False, 'exited': False, 'exception': False} + with pytest.raises(Exception): + do_write_with_exception() + assert vals['entered'] + assert vals['exited'] + assert vals['exception'] + + +def test_transaction_with_context_manager(lock_path): + class TestContextManager(object): + + def __enter__(self): + vals['entered'] = True + + def __exit__(self, t, v, tb): + vals['exited'] = True + vals['exception'] = (t or v or tb) + + def exit_fn(t, v, tb): + vals['exited_fn'] = True + vals['exception_fn'] = (t or v or tb) + + lock = Lock(lock_path) + + vals = {'entered': False, 'exited': False, 'exited_fn': False, + 'exception': False, 'exception_fn': False} + with ReadTransaction(lock, TestContextManager, exit_fn): + pass + + assert vals['entered'] + assert vals['exited'] + assert not vals['exception'] + assert vals['exited_fn'] + assert not vals['exception_fn'] + + vals = {'entered': False, 'exited': False, 'exited_fn': False, + 'exception': False, 'exception_fn': False} + with ReadTransaction(lock, TestContextManager): + pass + + assert vals['entered'] + assert vals['exited'] + assert not vals['exception'] + assert not vals['exited_fn'] + assert not vals['exception_fn'] + + vals = {'entered': False, 'exited': False, 'exited_fn': False, + 'exception': False, 'exception_fn': False} + with WriteTransaction(lock, TestContextManager, exit_fn): + pass + + assert vals['entered'] + assert vals['exited'] + assert not vals['exception'] + assert vals['exited_fn'] + assert not vals['exception_fn'] + + vals = {'entered': False, 'exited': False, 'exited_fn': False, + 'exception': False, 'exception_fn': False} + with WriteTransaction(lock, TestContextManager): + pass + + assert vals['entered'] + assert vals['exited'] + assert not vals['exception'] + assert not vals['exited_fn'] + assert not vals['exception_fn'] + + +def test_transaction_with_context_manager_and_exception(lock_path): + class TestContextManager(object): + def __enter__(self): + vals['entered'] = True + + def __exit__(self, t, v, tb): + vals['exited'] = True + vals['exception'] = (t or v or tb) + + def exit_fn(t, v, tb): + vals['exited_fn'] = True + vals['exception_fn'] = (t or v or tb) + + lock = Lock(lock_path) + + def do_read_with_exception(exit_fn): + with ReadTransaction(lock, TestContextManager, exit_fn): + raise Exception() + + def do_write_with_exception(exit_fn): + with WriteTransaction(lock, TestContextManager, exit_fn): + raise Exception() + + vals = {'entered': False, 'exited': False, 'exited_fn': False, + 'exception': False, 'exception_fn': False} + with pytest.raises(Exception): + do_read_with_exception(exit_fn) + assert vals['entered'] + assert vals['exited'] + assert vals['exception'] + assert vals['exited_fn'] + assert vals['exception_fn'] + + vals = {'entered': False, 'exited': False, 'exited_fn': False, + 'exception': False, 'exception_fn': False} + with pytest.raises(Exception): + do_read_with_exception(None) + assert vals['entered'] + assert vals['exited'] + assert vals['exception'] + assert not vals['exited_fn'] + assert not vals['exception_fn'] + + vals = {'entered': False, 'exited': False, 'exited_fn': False, + 'exception': False, 'exception_fn': False} + with pytest.raises(Exception): + do_write_with_exception(exit_fn) + assert vals['entered'] + assert vals['exited'] + assert vals['exception'] + assert vals['exited_fn'] + assert vals['exception_fn'] + + vals = {'entered': False, 'exited': False, 'exited_fn': False, + 'exception': False, 'exception_fn': False} + with pytest.raises(Exception): + do_write_with_exception(None) + assert vals['entered'] + assert vals['exited'] + assert vals['exception'] + assert not vals['exited_fn'] + assert not vals['exception_fn'] diff --git a/lib/spack/spack/test/llnl/util/log.py b/lib/spack/spack/test/llnl/util/log.py new file mode 100644 index 0000000000..4a43350f99 --- /dev/null +++ b/lib/spack/spack/test/llnl/util/log.py @@ -0,0 +1,95 @@ +############################################################################## +# Copyright (c) 2013-2017, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the NOTICE and LICENSE files for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from __future__ import print_function +import pytest + +from llnl.util.tty.log import log_output +from spack.util.executable import which + + +def test_log_python_output_with_python_stream(capsys, tmpdir): + # pytest's DontReadFromInput object does not like what we do here, so + # disable capsys or things hang. + with capsys.disabled(): + with log_output('foo.txt'): + print('logged') + + with open('foo.txt') as f: + assert f.read() == 'logged\n' + + assert capsys.readouterr() == ('', '') + + +def test_log_python_output_with_fd_stream(capfd, tmpdir): + with log_output('foo.txt'): + print('logged') + + with open('foo.txt') as f: + assert f.read() == 'logged\n' + + assert capfd.readouterr() == ('', '') + + +def test_log_python_output_and_echo_output(capfd, tmpdir): + with log_output('foo.txt') as logger: + with logger.force_echo(): + print('echo') + print('logged') + + assert capfd.readouterr() == ('echo\n', '') + + with open('foo.txt') as f: + assert f.read() == 'echo\nlogged\n' + + +@pytest.mark.skipif(not which('echo'), reason="needs echo command") +def test_log_subproc_output(capsys, tmpdir): + echo = which('echo') + + # pytest seems to interfere here, so we need to use capsys.disabled() + # TODO: figure out why this is and whether it means we're doing + # sometihng wrong with OUR redirects. Seems like it should work even + # with capsys enabled. + with capsys.disabled(): + with log_output('foo.txt'): + echo('logged') + + with open('foo.txt') as f: + assert f.read() == 'logged\n' + + +@pytest.mark.skipif(not which('echo'), reason="needs echo command") +def test_log_subproc_and_echo_output(capfd, tmpdir): + echo = which('echo') + + with log_output('foo.txt') as logger: + with logger.force_echo(): + echo('echo') + print('logged') + + assert capfd.readouterr() == ('echo\n', '') + + with open('foo.txt') as f: + assert f.read() == 'logged\n' diff --git a/lib/spack/spack/test/lock.py b/lib/spack/spack/test/lock.py deleted file mode 100644 index 208777ae51..0000000000 --- a/lib/spack/spack/test/lock.py +++ /dev/null @@ -1,902 +0,0 @@ -############################################################################## -# Copyright (c) 2013-2017, Lawrence Livermore National Security, LLC. -# Produced at the Lawrence Livermore National Laboratory. -# -# This file is part of Spack. -# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. -# LLNL-CODE-647188 -# -# For details, see https://github.com/llnl/spack -# Please also see the NOTICE and LICENSE files for our notice and the LGPL. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU Lesser General Public License (as -# published by the Free Software Foundation) version 2.1, February 1999. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and -# conditions of the GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -############################################################################## -"""These tests ensure that our lock works correctly. - -This can be run in two ways. - -First, it can be run as a node-local test, with a typical invocation like -this:: - - spack test lock - -You can *also* run it as an MPI program, which allows you to test locks -across nodes. So, e.g., you can run the test like this:: - - mpirun -n 7 spack test lock - -And it will test locking correctness among MPI processes. Ideally, you -want the MPI processes to span across multiple nodes, so, e.g., for SLURM -you might do this:: - - srun -N 7 -n 7 -m cyclic spack test lock - -You can use this to test whether your shared filesystem properly supports -POSIX reader-writer locking with byte ranges through fcntl. - -If you want to test on multiple filesystems, you can modify the -``locations`` list below. By default it looks like this:: - - locations = [ - tempfile.gettempdir(), # standard tmp directory (potentially local) - '/nfs/tmp2/%u', # NFS tmp mount - '/p/lscratch*/%u' # Lustre scratch mount - ] - -Add names and paths for your preferred filesystem mounts to test on them; -the tests are parametrized to run on all the filesystems listed in this -dict. Note that 'tmp' will be skipped for MPI testing, as it is often a -node-local filesystem, and multi-node tests will fail if the locks aren't -actually on a shared filesystem. - -""" -import os -import shutil -import tempfile -import traceback -import glob -import getpass -from contextlib import contextmanager -from multiprocessing import Process - -import pytest - -from llnl.util.filesystem import join_path, touch -from llnl.util.lock import * -from spack.util.multiproc import Barrier - - -# -# This test can be run with MPI. MPI is "enabled" if we can import -# mpi4py and the number of total MPI processes is greater than 1. -# Otherwise it just runs as a node-local test. -# -# NOTE: MPI mode is different from node-local mode in that node-local -# mode will spawn its own test processes, while MPI mode assumes you've -# run this script as a SPMD application. In MPI mode, no additional -# processes are spawned, and you need to ensure that you mpirun the -# script with enough processes for all the multiproc_test cases below. -# -# If you don't run with enough processes, tests that require more -# processes than you currently have will be skipped. -# -mpi = False -comm = None -try: - from mpi4py import MPI - comm = MPI.COMM_WORLD - if comm.size > 1: - mpi = True -except: - pass - - -"""This is a list of filesystem locations to test locks in. Paths are -expanded so that %u is replaced with the current username. '~' is also -legal and will be expanded to the user's home directory. - -Tests are skipped for directories that don't exist, so you'll need to -update this with the locations of NFS, Lustre, and other mounts on your -system. -""" -locations = [ - tempfile.gettempdir(), - os.path.join('/nfs/tmp2/', getpass.getuser()), - os.path.join('/p/lscratch*/', getpass.getuser()), -] - -"""This is the longest a failed multiproc test will take. -Barriers will time out and raise an exception after this interval. -In MPI mode, barriers don't time out (they hang). See mpi_multiproc_test. -""" -barrier_timeout = 5 - -"""This is the lock timeout for expected failures. -This may need to be higher for some filesystems.""" -lock_fail_timeout = 0.1 - - -@contextmanager -def read_only(path): - orginal_mode = os.stat(path).st_mode - os.chmod(path, 0o444) - yield - os.chmod(path, orginal_mode) - - -@pytest.fixture(scope='session', params=locations) -def lock_test_directory(request): - """This fixture causes tests to be executed for many different mounts. - - See the ``locations`` dict above for details. - """ - return request.param - - -@pytest.fixture(scope='session') -def lock_dir(lock_test_directory): - parent = next((p for p in glob.glob(lock_test_directory) - if os.path.exists(p) and os.access(p, os.W_OK)), None) - if not parent: - # Skip filesystems that don't exist or aren't writable - pytest.skip("requires filesystem: '%s'" % lock_test_directory) - elif mpi and parent == tempfile.gettempdir(): - # Skip local tmp test for MPI runs - pytest.skip("skipping local tmp directory for MPI test.") - - tempdir = None - if not mpi or comm.rank == 0: - tempdir = tempfile.mkdtemp(dir=parent) - if mpi: - tempdir = comm.bcast(tempdir) - - yield tempdir - - if mpi: - # rank 0 may get here before others, in which case it'll try to - # remove the directory while other processes try to re-create the - # lock. This will give errno 39: directory not empty. Use a - # barrier to ensure everyone is done first. - comm.barrier() - - if not mpi or comm.rank == 0: - shutil.rmtree(tempdir) - - -@pytest.fixture -def private_lock_path(lock_dir): - """In MPI mode, this is a private lock for each rank in a multiproc test. - - For other modes, it is the same as a shared lock. - """ - lock_file = join_path(lock_dir, 'lockfile') - if mpi: - lock_file += '.%s' % comm.rank - yield lock_file - - -@pytest.fixture -def lock_path(lock_dir): - """This lock is shared among all processes in a multiproc test.""" - lock_file = join_path(lock_dir, 'lockfile') - yield lock_file - - -def local_multiproc_test(*functions): - """Order some processes using simple barrier synchronization.""" - b = Barrier(len(functions), timeout=barrier_timeout) - procs = [Process(target=f, args=(b,)) for f in functions] - - for p in procs: - p.start() - - for p in procs: - p.join() - assert p.exitcode == 0 - - -def mpi_multiproc_test(*functions): - """SPMD version of multiproc test. - - This needs to be run like so: - - srun spack test lock - - Each process executes its corresponding function. This is different - from ``multiproc_test`` above, which spawns the processes. This will - skip tests if there are too few processes to run them. - """ - procs = len(functions) - if procs > comm.size: - pytest.skip("requires at least %d MPI processes" % procs) - - comm.Barrier() # barrier before each MPI test - - include = comm.rank < len(functions) - subcomm = comm.Split(include) - - class subcomm_barrier(object): - """Stand-in for multiproc barrier for MPI-parallel jobs.""" - def wait(self): - subcomm.Barrier() - - if include: - try: - functions[subcomm.rank](subcomm_barrier()) - except: - # aborting is the best we can do for MPI tests without - # hanging, since we're using MPI barriers. This will fail - # early and it loses the nice pytest output, but at least it - # gets use a stacktrace on the processes that failed. - traceback.print_exc() - comm.Abort() - subcomm.Free() - - comm.Barrier() # barrier after each MPI test. - - -"""``multiproc_test()`` should be called by tests below. -``multiproc_test()`` will work for either MPI runs or for local runs. -""" -multiproc_test = mpi_multiproc_test if mpi else local_multiproc_test - - -# -# Process snippets below can be composed into tests. -# -def acquire_write(lock_path, start=0, length=0): - def fn(barrier): - lock = Lock(lock_path, start, length) - lock.acquire_write() # grab exclusive lock - barrier.wait() - barrier.wait() # hold the lock until timeout in other procs. - return fn - - -def acquire_read(lock_path, start=0, length=0): - def fn(barrier): - lock = Lock(lock_path, start, length) - lock.acquire_read() # grab shared lock - barrier.wait() - barrier.wait() # hold the lock until timeout in other procs. - return fn - - -def timeout_write(lock_path, start=0, length=0): - def fn(barrier): - lock = Lock(lock_path, start, length) - barrier.wait() # wait for lock acquire in first process - with pytest.raises(LockError): - lock.acquire_write(lock_fail_timeout) - barrier.wait() - return fn - - -def timeout_read(lock_path, start=0, length=0): - def fn(barrier): - lock = Lock(lock_path, start, length) - barrier.wait() # wait for lock acquire in first process - with pytest.raises(LockError): - lock.acquire_read(lock_fail_timeout) - barrier.wait() - return fn - - -# -# Test that exclusive locks on other processes time out when an -# exclusive lock is held. -# -def test_write_lock_timeout_on_write(lock_path): - multiproc_test( - acquire_write(lock_path), - timeout_write(lock_path)) - - -def test_write_lock_timeout_on_write_2(lock_path): - multiproc_test( - acquire_write(lock_path), - timeout_write(lock_path), - timeout_write(lock_path)) - - -def test_write_lock_timeout_on_write_3(lock_path): - multiproc_test( - acquire_write(lock_path), - timeout_write(lock_path), - timeout_write(lock_path), - timeout_write(lock_path)) - - -def test_write_lock_timeout_on_write_ranges(lock_path): - multiproc_test( - acquire_write(lock_path, 0, 1), - timeout_write(lock_path, 0, 1)) - - -def test_write_lock_timeout_on_write_ranges_2(lock_path): - multiproc_test( - acquire_write(lock_path, 0, 64), - acquire_write(lock_path, 65, 1), - timeout_write(lock_path, 0, 1), - timeout_write(lock_path, 63, 1)) - - -def test_write_lock_timeout_on_write_ranges_3(lock_path): - multiproc_test( - acquire_write(lock_path, 0, 1), - acquire_write(lock_path, 1, 1), - timeout_write(lock_path), - timeout_write(lock_path), - timeout_write(lock_path)) - - -def test_write_lock_timeout_on_write_ranges_4(lock_path): - multiproc_test( - acquire_write(lock_path, 0, 1), - acquire_write(lock_path, 1, 1), - acquire_write(lock_path, 2, 456), - acquire_write(lock_path, 500, 64), - timeout_write(lock_path), - timeout_write(lock_path), - timeout_write(lock_path)) - - -# -# Test that shared locks on other processes time out when an -# exclusive lock is held. -# -def test_read_lock_timeout_on_write(lock_path): - multiproc_test( - acquire_write(lock_path), - timeout_read(lock_path)) - - -def test_read_lock_timeout_on_write_2(lock_path): - multiproc_test( - acquire_write(lock_path), - timeout_read(lock_path), - timeout_read(lock_path)) - - -def test_read_lock_timeout_on_write_3(lock_path): - multiproc_test( - acquire_write(lock_path), - timeout_read(lock_path), - timeout_read(lock_path), - timeout_read(lock_path)) - - -def test_read_lock_timeout_on_write_ranges(lock_path): - """small write lock, read whole file.""" - multiproc_test( - acquire_write(lock_path, 0, 1), - timeout_read(lock_path)) - - -def test_read_lock_timeout_on_write_ranges_2(lock_path): - """small write lock, small read lock""" - multiproc_test( - acquire_write(lock_path, 0, 1), - timeout_read(lock_path, 0, 1)) - - -def test_read_lock_timeout_on_write_ranges_3(lock_path): - """two write locks, overlapping read locks""" - multiproc_test( - acquire_write(lock_path, 0, 1), - acquire_write(lock_path, 64, 128), - timeout_read(lock_path, 0, 1), - timeout_read(lock_path, 128, 256)) - - -# -# Test that exclusive locks time out when shared locks are held. -# -def test_write_lock_timeout_on_read(lock_path): - multiproc_test( - acquire_read(lock_path), - timeout_write(lock_path)) - - -def test_write_lock_timeout_on_read_2(lock_path): - multiproc_test( - acquire_read(lock_path), - timeout_write(lock_path), - timeout_write(lock_path)) - - -def test_write_lock_timeout_on_read_3(lock_path): - multiproc_test( - acquire_read(lock_path), - timeout_write(lock_path), - timeout_write(lock_path), - timeout_write(lock_path)) - - -def test_write_lock_timeout_on_read_ranges(lock_path): - multiproc_test( - acquire_read(lock_path, 0, 1), - timeout_write(lock_path)) - - -def test_write_lock_timeout_on_read_ranges_2(lock_path): - multiproc_test( - acquire_read(lock_path, 0, 1), - timeout_write(lock_path, 0, 1)) - - -def test_write_lock_timeout_on_read_ranges_3(lock_path): - multiproc_test( - acquire_read(lock_path, 0, 1), - acquire_read(lock_path, 10, 1), - timeout_write(lock_path, 0, 1), - timeout_write(lock_path, 10, 1)) - - -def test_write_lock_timeout_on_read_ranges_4(lock_path): - multiproc_test( - acquire_read(lock_path, 0, 64), - timeout_write(lock_path, 10, 1), - timeout_write(lock_path, 32, 1)) - - -def test_write_lock_timeout_on_read_ranges_5(lock_path): - multiproc_test( - acquire_read(lock_path, 64, 128), - timeout_write(lock_path, 65, 1), - timeout_write(lock_path, 127, 1), - timeout_write(lock_path, 90, 10)) - - -# -# Test that exclusive locks time while lots of shared locks are held. -# -def test_write_lock_timeout_with_multiple_readers_2_1(lock_path): - multiproc_test( - acquire_read(lock_path), - acquire_read(lock_path), - timeout_write(lock_path)) - - -def test_write_lock_timeout_with_multiple_readers_2_2(lock_path): - multiproc_test( - acquire_read(lock_path), - acquire_read(lock_path), - timeout_write(lock_path), - timeout_write(lock_path)) - - -def test_write_lock_timeout_with_multiple_readers_3_1(lock_path): - multiproc_test( - acquire_read(lock_path), - acquire_read(lock_path), - acquire_read(lock_path), - timeout_write(lock_path)) - - -def test_write_lock_timeout_with_multiple_readers_3_2(lock_path): - multiproc_test( - acquire_read(lock_path), - acquire_read(lock_path), - acquire_read(lock_path), - timeout_write(lock_path), - timeout_write(lock_path)) - - -def test_write_lock_timeout_with_multiple_readers_2_1_ranges(lock_path): - multiproc_test( - acquire_read(lock_path, 0, 10), - acquire_read(lock_path, 0.5, 10), - timeout_write(lock_path, 5, 5)) - - -def test_write_lock_timeout_with_multiple_readers_2_3_ranges(lock_path): - multiproc_test( - acquire_read(lock_path, 0, 10), - acquire_read(lock_path, 5, 15), - timeout_write(lock_path, 0, 1), - timeout_write(lock_path, 11, 3), - timeout_write(lock_path, 7, 1)) - - -def test_write_lock_timeout_with_multiple_readers_3_1_ranges(lock_path): - multiproc_test( - acquire_read(lock_path, 0, 5), - acquire_read(lock_path, 5, 5), - acquire_read(lock_path, 10, 5), - timeout_write(lock_path, 0, 15)) - - -def test_write_lock_timeout_with_multiple_readers_3_2_ranges(lock_path): - multiproc_test( - acquire_read(lock_path, 0, 5), - acquire_read(lock_path, 5, 5), - acquire_read(lock_path, 10, 5), - timeout_write(lock_path, 3, 10), - timeout_write(lock_path, 5, 1)) - - -# -# Test that read can be upgraded to write. -# -def test_upgrade_read_to_write(private_lock_path): - """Test that a read lock can be upgraded to a write lock. - - Note that to upgrade a read lock to a write lock, you have the be the - only holder of a read lock. Client code needs to coordinate that for - shared locks. For this test, we use a private lock just to test that an - upgrade is possible. - """ - # ensure lock file exists the first time, so we open it read-only - # to begin wtih. - touch(private_lock_path) - - lock = Lock(private_lock_path) - assert lock._reads == 0 - assert lock._writes == 0 - - lock.acquire_read() - assert lock._reads == 1 - assert lock._writes == 0 - assert lock._file.mode == 'r+' - - lock.acquire_write() - assert lock._reads == 1 - assert lock._writes == 1 - assert lock._file.mode == 'r+' - - lock.release_write() - assert lock._reads == 1 - assert lock._writes == 0 - assert lock._file.mode == 'r+' - - lock.release_read() - assert lock._reads == 0 - assert lock._writes == 0 - assert lock._file is None - - -# -# Test that read-only file can be read-locked but not write-locked. -# -def test_upgrade_read_to_write_fails_with_readonly_file(private_lock_path): - # ensure lock file exists the first time, so we open it read-only - # to begin wtih. - touch(private_lock_path) - - with read_only(private_lock_path): - lock = Lock(private_lock_path) - assert lock._reads == 0 - assert lock._writes == 0 - - lock.acquire_read() - assert lock._reads == 1 - assert lock._writes == 0 - assert lock._file.mode == 'r' - - with pytest.raises(LockError): - lock.acquire_write() - - -# -# Longer test case that ensures locks are reusable. Ordering is -# enforced by barriers throughout -- steps are shown with numbers. -# -def test_complex_acquire_and_release_chain(lock_path): - def p1(barrier): - lock = Lock(lock_path) - - lock.acquire_write() - barrier.wait() # ---------------------------------------- 1 - # others test timeout - barrier.wait() # ---------------------------------------- 2 - lock.release_write() # release and others acquire read - barrier.wait() # ---------------------------------------- 3 - with pytest.raises(LockError): - lock.acquire_write(lock_fail_timeout) - lock.acquire_read() - barrier.wait() # ---------------------------------------- 4 - lock.release_read() - barrier.wait() # ---------------------------------------- 5 - - # p2 upgrades read to write - barrier.wait() # ---------------------------------------- 6 - with pytest.raises(LockError): - lock.acquire_write(lock_fail_timeout) - with pytest.raises(LockError): - lock.acquire_read(lock_fail_timeout) - barrier.wait() # ---------------------------------------- 7 - # p2 releases write and read - barrier.wait() # ---------------------------------------- 8 - - # p3 acquires read - barrier.wait() # ---------------------------------------- 9 - # p3 upgrades read to write - barrier.wait() # ---------------------------------------- 10 - with pytest.raises(LockError): - lock.acquire_write(lock_fail_timeout) - with pytest.raises(LockError): - lock.acquire_read(lock_fail_timeout) - barrier.wait() # ---------------------------------------- 11 - # p3 releases locks - barrier.wait() # ---------------------------------------- 12 - lock.acquire_read() - barrier.wait() # ---------------------------------------- 13 - lock.release_read() - - def p2(barrier): - lock = Lock(lock_path) - - # p1 acquires write - barrier.wait() # ---------------------------------------- 1 - with pytest.raises(LockError): - lock.acquire_write(lock_fail_timeout) - with pytest.raises(LockError): - lock.acquire_read(lock_fail_timeout) - barrier.wait() # ---------------------------------------- 2 - lock.acquire_read() - barrier.wait() # ---------------------------------------- 3 - # p1 tests shared read - barrier.wait() # ---------------------------------------- 4 - # others release reads - barrier.wait() # ---------------------------------------- 5 - - lock.acquire_write() # upgrade read to write - barrier.wait() # ---------------------------------------- 6 - # others test timeout - barrier.wait() # ---------------------------------------- 7 - lock.release_write() # release read AND write (need both) - lock.release_read() - barrier.wait() # ---------------------------------------- 8 - - # p3 acquires read - barrier.wait() # ---------------------------------------- 9 - # p3 upgrades read to write - barrier.wait() # ---------------------------------------- 10 - with pytest.raises(LockError): - lock.acquire_write(lock_fail_timeout) - with pytest.raises(LockError): - lock.acquire_read(lock_fail_timeout) - barrier.wait() # ---------------------------------------- 11 - # p3 releases locks - barrier.wait() # ---------------------------------------- 12 - lock.acquire_read() - barrier.wait() # ---------------------------------------- 13 - lock.release_read() - - def p3(barrier): - lock = Lock(lock_path) - - # p1 acquires write - barrier.wait() # ---------------------------------------- 1 - with pytest.raises(LockError): - lock.acquire_write(lock_fail_timeout) - with pytest.raises(LockError): - lock.acquire_read(lock_fail_timeout) - barrier.wait() # ---------------------------------------- 2 - lock.acquire_read() - barrier.wait() # ---------------------------------------- 3 - # p1 tests shared read - barrier.wait() # ---------------------------------------- 4 - lock.release_read() - barrier.wait() # ---------------------------------------- 5 - - # p2 upgrades read to write - barrier.wait() # ---------------------------------------- 6 - with pytest.raises(LockError): - lock.acquire_write(lock_fail_timeout) - with pytest.raises(LockError): - lock.acquire_read(lock_fail_timeout) - barrier.wait() # ---------------------------------------- 7 - # p2 releases write & read - barrier.wait() # ---------------------------------------- 8 - - lock.acquire_read() - barrier.wait() # ---------------------------------------- 9 - lock.acquire_write() - barrier.wait() # ---------------------------------------- 10 - # others test timeout - barrier.wait() # ---------------------------------------- 11 - lock.release_read() # release read AND write in opposite - lock.release_write() # order from before on p2 - barrier.wait() # ---------------------------------------- 12 - lock.acquire_read() - barrier.wait() # ---------------------------------------- 13 - lock.release_read() - - multiproc_test(p1, p2, p3) - - -def test_transaction(lock_path): - def enter_fn(): - vals['entered'] = True - - def exit_fn(t, v, tb): - vals['exited'] = True - vals['exception'] = (t or v or tb) - - lock = Lock(lock_path) - vals = {'entered': False, 'exited': False, 'exception': False} - with ReadTransaction(lock, enter_fn, exit_fn): - pass - - assert vals['entered'] - assert vals['exited'] - assert not vals['exception'] - - vals = {'entered': False, 'exited': False, 'exception': False} - with WriteTransaction(lock, enter_fn, exit_fn): - pass - - assert vals['entered'] - assert vals['exited'] - assert not vals['exception'] - - -def test_transaction_with_exception(lock_path): - def enter_fn(): - vals['entered'] = True - - def exit_fn(t, v, tb): - vals['exited'] = True - vals['exception'] = (t or v or tb) - - lock = Lock(lock_path) - - def do_read_with_exception(): - with ReadTransaction(lock, enter_fn, exit_fn): - raise Exception() - - def do_write_with_exception(): - with WriteTransaction(lock, enter_fn, exit_fn): - raise Exception() - - vals = {'entered': False, 'exited': False, 'exception': False} - with pytest.raises(Exception): - do_read_with_exception() - assert vals['entered'] - assert vals['exited'] - assert vals['exception'] - - vals = {'entered': False, 'exited': False, 'exception': False} - with pytest.raises(Exception): - do_write_with_exception() - assert vals['entered'] - assert vals['exited'] - assert vals['exception'] - - -def test_transaction_with_context_manager(lock_path): - class TestContextManager(object): - - def __enter__(self): - vals['entered'] = True - - def __exit__(self, t, v, tb): - vals['exited'] = True - vals['exception'] = (t or v or tb) - - def exit_fn(t, v, tb): - vals['exited_fn'] = True - vals['exception_fn'] = (t or v or tb) - - lock = Lock(lock_path) - - vals = {'entered': False, 'exited': False, 'exited_fn': False, - 'exception': False, 'exception_fn': False} - with ReadTransaction(lock, TestContextManager, exit_fn): - pass - - assert vals['entered'] - assert vals['exited'] - assert not vals['exception'] - assert vals['exited_fn'] - assert not vals['exception_fn'] - - vals = {'entered': False, 'exited': False, 'exited_fn': False, - 'exception': False, 'exception_fn': False} - with ReadTransaction(lock, TestContextManager): - pass - - assert vals['entered'] - assert vals['exited'] - assert not vals['exception'] - assert not vals['exited_fn'] - assert not vals['exception_fn'] - - vals = {'entered': False, 'exited': False, 'exited_fn': False, - 'exception': False, 'exception_fn': False} - with WriteTransaction(lock, TestContextManager, exit_fn): - pass - - assert vals['entered'] - assert vals['exited'] - assert not vals['exception'] - assert vals['exited_fn'] - assert not vals['exception_fn'] - - vals = {'entered': False, 'exited': False, 'exited_fn': False, - 'exception': False, 'exception_fn': False} - with WriteTransaction(lock, TestContextManager): - pass - - assert vals['entered'] - assert vals['exited'] - assert not vals['exception'] - assert not vals['exited_fn'] - assert not vals['exception_fn'] - - -def test_transaction_with_context_manager_and_exception(lock_path): - class TestContextManager(object): - def __enter__(self): - vals['entered'] = True - - def __exit__(self, t, v, tb): - vals['exited'] = True - vals['exception'] = (t or v or tb) - - def exit_fn(t, v, tb): - vals['exited_fn'] = True - vals['exception_fn'] = (t or v or tb) - - lock = Lock(lock_path) - - def do_read_with_exception(exit_fn): - with ReadTransaction(lock, TestContextManager, exit_fn): - raise Exception() - - def do_write_with_exception(exit_fn): - with WriteTransaction(lock, TestContextManager, exit_fn): - raise Exception() - - vals = {'entered': False, 'exited': False, 'exited_fn': False, - 'exception': False, 'exception_fn': False} - with pytest.raises(Exception): - do_read_with_exception(exit_fn) - assert vals['entered'] - assert vals['exited'] - assert vals['exception'] - assert vals['exited_fn'] - assert vals['exception_fn'] - - vals = {'entered': False, 'exited': False, 'exited_fn': False, - 'exception': False, 'exception_fn': False} - with pytest.raises(Exception): - do_read_with_exception(None) - assert vals['entered'] - assert vals['exited'] - assert vals['exception'] - assert not vals['exited_fn'] - assert not vals['exception_fn'] - - vals = {'entered': False, 'exited': False, 'exited_fn': False, - 'exception': False, 'exception_fn': False} - with pytest.raises(Exception): - do_write_with_exception(exit_fn) - assert vals['entered'] - assert vals['exited'] - assert vals['exception'] - assert vals['exited_fn'] - assert vals['exception_fn'] - - vals = {'entered': False, 'exited': False, 'exited_fn': False, - 'exception': False, 'exception_fn': False} - with pytest.raises(Exception): - do_write_with_exception(None) - assert vals['entered'] - assert vals['exited'] - assert vals['exception'] - assert not vals['exited_fn'] - assert not vals['exception_fn'] diff --git a/lib/spack/spack/test/log.py b/lib/spack/spack/test/log.py deleted file mode 100644 index 4a43350f99..0000000000 --- a/lib/spack/spack/test/log.py +++ /dev/null @@ -1,95 +0,0 @@ -############################################################################## -# Copyright (c) 2013-2017, Lawrence Livermore National Security, LLC. -# Produced at the Lawrence Livermore National Laboratory. -# -# This file is part of Spack. -# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. -# LLNL-CODE-647188 -# -# For details, see https://github.com/llnl/spack -# Please also see the NOTICE and LICENSE files for our notice and the LGPL. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU Lesser General Public License (as -# published by the Free Software Foundation) version 2.1, February 1999. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and -# conditions of the GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -############################################################################## -from __future__ import print_function -import pytest - -from llnl.util.tty.log import log_output -from spack.util.executable import which - - -def test_log_python_output_with_python_stream(capsys, tmpdir): - # pytest's DontReadFromInput object does not like what we do here, so - # disable capsys or things hang. - with capsys.disabled(): - with log_output('foo.txt'): - print('logged') - - with open('foo.txt') as f: - assert f.read() == 'logged\n' - - assert capsys.readouterr() == ('', '') - - -def test_log_python_output_with_fd_stream(capfd, tmpdir): - with log_output('foo.txt'): - print('logged') - - with open('foo.txt') as f: - assert f.read() == 'logged\n' - - assert capfd.readouterr() == ('', '') - - -def test_log_python_output_and_echo_output(capfd, tmpdir): - with log_output('foo.txt') as logger: - with logger.force_echo(): - print('echo') - print('logged') - - assert capfd.readouterr() == ('echo\n', '') - - with open('foo.txt') as f: - assert f.read() == 'echo\nlogged\n' - - -@pytest.mark.skipif(not which('echo'), reason="needs echo command") -def test_log_subproc_output(capsys, tmpdir): - echo = which('echo') - - # pytest seems to interfere here, so we need to use capsys.disabled() - # TODO: figure out why this is and whether it means we're doing - # sometihng wrong with OUR redirects. Seems like it should work even - # with capsys enabled. - with capsys.disabled(): - with log_output('foo.txt'): - echo('logged') - - with open('foo.txt') as f: - assert f.read() == 'logged\n' - - -@pytest.mark.skipif(not which('echo'), reason="needs echo command") -def test_log_subproc_and_echo_output(capfd, tmpdir): - echo = which('echo') - - with log_output('foo.txt') as logger: - with logger.force_echo(): - echo('echo') - print('logged') - - assert capfd.readouterr() == ('echo\n', '') - - with open('foo.txt') as f: - assert f.read() == 'logged\n' -- cgit v1.2.3-60-g2f50