diff options
Diffstat (limited to 'lib')
-rw-r--r-- | lib/spack/llnl/util/filesystem.py | 181 | ||||
-rw-r--r-- | lib/spack/spack/installer.py | 14 | ||||
-rw-r--r-- | lib/spack/spack/package_base.py | 35 | ||||
-rw-r--r-- | lib/spack/spack/test/conftest.py | 6 | ||||
-rw-r--r-- | lib/spack/spack/test/data/directory_search/a/libc.dll | 0 | ||||
-rw-r--r-- | lib/spack/spack/test/data/directory_search/a/libc.lib | 0 | ||||
-rw-r--r-- | lib/spack/spack/test/data/directory_search/b/liba.dll | 0 | ||||
-rw-r--r-- | lib/spack/spack/test/data/directory_search/b/liba.lib | 0 | ||||
-rw-r--r-- | lib/spack/spack/test/data/directory_search/b/libd.dll | 0 | ||||
-rw-r--r-- | lib/spack/spack/test/data/directory_search/b/libd.lib | 0 | ||||
-rw-r--r-- | lib/spack/spack/test/data/directory_search/c/libb.dll | 0 | ||||
-rw-r--r-- | lib/spack/spack/test/data/directory_search/c/libb.lib | 0 | ||||
-rw-r--r-- | lib/spack/spack/test/install.py | 10 | ||||
-rw-r--r-- | lib/spack/spack/test/llnl/util/file_list.py | 65 |
14 files changed, 271 insertions, 40 deletions
diff --git a/lib/spack/llnl/util/filesystem.py b/lib/spack/llnl/util/filesystem.py index 8a420756ef..a161a64d2a 100644 --- a/lib/spack/llnl/util/filesystem.py +++ b/lib/spack/llnl/util/filesystem.py @@ -22,7 +22,7 @@ import six from llnl.util import tty from llnl.util.compat import Sequence from llnl.util.lang import dedupe, memoized -from llnl.util.symlink import symlink +from llnl.util.symlink import islink, symlink from spack.util.executable import Executable from spack.util.path import path_to_os_path, system_path_filter @@ -637,7 +637,11 @@ def copy_tree(src, dest, symlinks=True, ignore=None, _permissions=False): if symlinks: target = os.readlink(s) if os.path.isabs(target): - new_target = re.sub(abs_src, abs_dest, target) + + def escaped_path(path): + return path.replace("\\", r"\\") + + new_target = re.sub(escaped_path(abs_src), escaped_path(abs_dest), target) if new_target != target: tty.debug("Redirecting link {0} to {1}".format(target, new_target)) target = new_target @@ -1903,7 +1907,11 @@ class LibraryList(FileList): name = x[3:] # Valid extensions include: ['.dylib', '.so', '.a'] - for ext in [".dylib", ".so", ".a"]: + # on non Windows platform + # Windows valid library extensions are: + # ['.dll', '.lib'] + valid_exts = [".dll", ".lib"] if is_windows else [".dylib", ".so", ".a"] + for ext in valid_exts: i = name.rfind(ext) if i != -1: names.append(name[:i]) @@ -2046,15 +2054,23 @@ def find_libraries(libraries, root, shared=True, recursive=False): message = message.format(find_libraries.__name__, type(libraries)) raise TypeError(message) + if is_windows: + static = "lib" + shared = "dll" + else: + # Used on both Linux and macOS + static = "a" + shared = "so" + # Construct the right suffix for the library if shared: # Used on both Linux and macOS - suffixes = ["so"] + suffixes = [shared] if sys.platform == "darwin": # Only used on macOS suffixes.append("dylib") else: - suffixes = ["a"] + suffixes = [static] # List of libraries we are searching with suffixes libraries = ["{0}.{1}".format(lib, suffix) for lib in libraries for suffix in suffixes] @@ -2067,7 +2083,11 @@ def find_libraries(libraries, root, shared=True, recursive=False): # perform first non-recursive search in root/lib then in root/lib64 and # finally search all of root recursively. The search stops when the first # match is found. - for subdir in ("lib", "lib64"): + common_lib_dirs = ["lib", "lib64"] + if is_windows: + common_lib_dirs.extend(["bin", "Lib"]) + + for subdir in common_lib_dirs: dirname = join_path(root, subdir) if not os.path.isdir(dirname): continue @@ -2080,6 +2100,155 @@ def find_libraries(libraries, root, shared=True, recursive=False): return LibraryList(found_libs) +def find_all_shared_libraries(root, recursive=False): + """Convenience function that returns the list of all shared libraries found + in the directory passed as argument. + + See documentation for `llnl.util.filesystem.find_libraries` for more information + """ + return find_libraries("*", root=root, shared=True, recursive=recursive) + + +def find_all_static_libraries(root, recursive=False): + """Convenience function that returns the list of all static libraries found + in the directory passed as argument. + + See documentation for `llnl.util.filesystem.find_libraries` for more information + """ + return find_libraries("*", root=root, shared=False, recursive=recursive) + + +def find_all_libraries(root, recursive=False): + """Convenience function that returns the list of all libraries found + in the directory passed as argument. + + See documentation for `llnl.util.filesystem.find_libraries` for more information + """ + + return find_all_shared_libraries(root, recursive=recursive) + find_all_static_libraries( + root, recursive=recursive + ) + + +class WindowsSimulatedRPath(object): + """Class representing Windows filesystem rpath analog + + One instance of this class is associated with a package (only on Windows) + For each lib/binary directory in an associated package, this class introduces + a symlink to any/all dependent libraries/binaries. This includes the packages + own bin/lib directories, meaning the libraries are linked to the bianry directory + and vis versa. + """ + + def __init__(self, package, link_install_prefix=True): + """ + Args: + package (spack.package_base.PackageBase): Package requiring links + link_install_prefix (bool): Link against package's own install or stage root. + Packages that run their own executables during build and require rpaths to + the build directory during build time require this option. Default: install + root + """ + self.pkg = package + self._addl_rpaths = set() + self.link_install_prefix = link_install_prefix + self._internal_links = set() + + @property + def link_dest(self): + """ + Set of directories where package binaries/libraries are located. + """ + if hasattr(self.pkg, "libs") and self.pkg.libs: + pkg_libs = set(self.pkg.libs.directories) + else: + pkg_libs = set((self.pkg.prefix.lib, self.pkg.prefix.lib64)) + + return pkg_libs | set([self.pkg.prefix.bin]) | self.internal_links + + @property + def internal_links(self): + """ + linking that would need to be established within the package itself. Useful for links + against extension modules/build time executables/internal linkage + """ + return self._internal_links + + def add_internal_links(self, *dest): + """ + Incorporate additional paths into the rpath (sym)linking scheme. + + Paths provided to this method are linked against by a package's libraries + and libraries found at these paths are linked against a package's binaries. + (i.e. /site-packages -> /bin and /bin -> /site-packages) + + Specified paths should be outside of a package's lib, lib64, and bin + directories. + """ + self._internal_links = self._internal_links | set(*dest) + + @property + def link_targets(self): + """ + Set of libraries this package needs to link against during runtime + These packages will each be symlinked into the packages lib and binary dir + """ + + dependent_libs = [] + for path in self.pkg.rpath: + dependent_libs.extend(list(find_all_shared_libraries(path, recursive=True))) + for extra_path in self._addl_rpaths: + dependent_libs.extend(list(find_all_shared_libraries(extra_path, recursive=True))) + return set(dependent_libs) + + def include_additional_link_paths(self, *paths): + """ + Add libraries found at the root of provided paths to runtime linking + + These are libraries found outside of the typical scope of rpath linking + that require manual inclusion in a runtime linking scheme + + Args: + *paths (str): arbitrary number of paths to be added to runtime linking + """ + self._addl_rpaths = self._addl_rpaths | set(paths) + + def establish_link(self): + """ + (sym)link packages to runtime dependencies based on RPath configuration for + Windows heuristics + """ + # from build_environment.py:463 + # The top-level package is always RPATHed. It hasn't been installed yet + # so the RPATHs are added unconditionally + + # for each binary install dir in self.pkg (i.e. pkg.prefix.bin, pkg.prefix.lib) + # install a symlink to each dependent library + for library, lib_dir in itertools.product(self.link_targets, self.link_dest): + if not path_contains_subdirectory(library, lib_dir): + file_name = os.path.basename(library) + dest_file = os.path.join(lib_dir, file_name) + if os.path.exists(lib_dir): + try: + symlink(library, dest_file) + # For py2 compatibility, we have to catch the specific Windows error code + # associate with trying to create a file that already exists (winerror 183) + except OSError as e: + if e.winerror == 183: + # We have either already symlinked or we are encoutering a naming clash + # either way, we don't want to overwrite existing libraries + already_linked = islink(dest_file) + tty.debug( + "Linking library %s to %s failed, " % (library, dest_file) + + "already linked." + if already_linked + else "library with name %s already exists." % file_name + ) + pass + else: + raise e + + @system_path_filter @memoized def can_access_dir(path): diff --git a/lib/spack/spack/installer.py b/lib/spack/spack/installer.py index d97db35535..2f0415b0c1 100644 --- a/lib/spack/spack/installer.py +++ b/lib/spack/spack/installer.py @@ -84,6 +84,9 @@ STATUS_DEQUEUED = "dequeued" #: queue invariants). STATUS_REMOVED = "removed" +is_windows = sys.platform == "win32" +is_osx = sys.platform == "darwin" + class InstallAction(object): #: Don't perform an install @@ -165,7 +168,9 @@ def _do_fake_install(pkg): if not pkg.name.startswith("lib"): library = "lib" + library - dso_suffix = ".dylib" if sys.platform == "darwin" else ".so" + plat_shared = ".dll" if is_windows else ".so" + plat_static = ".lib" if is_windows else ".a" + dso_suffix = ".dylib" if is_osx else plat_shared # Install fake command fs.mkdirp(pkg.prefix.bin) @@ -180,7 +185,7 @@ def _do_fake_install(pkg): # Install fake shared and static libraries fs.mkdirp(pkg.prefix.lib) - for suffix in [dso_suffix, ".a"]: + for suffix in [dso_suffix, plat_static]: fs.touch(os.path.join(pkg.prefix.lib, library + suffix)) # Install fake man page @@ -1214,7 +1219,10 @@ class PackageInstaller(object): spack.package_base.PackageBase._verbose = spack.build_environment.start_build_process( pkg, build_process, install_args ) - + # Currently this is how RPATH-like behavior is achieved on Windows, after install + # establish runtime linkage via Windows Runtime link object + # Note: this is a no-op on non Windows platforms + pkg.windows_establish_runtime_linkage() # Note: PARENT of the build process adds the new package to # the database, so that we don't need to re-read from file. spack.store.db.add(pkg.spec, spack.store.layout, explicit=explicit) diff --git a/lib/spack/spack/package_base.py b/lib/spack/spack/package_base.py index e8f7ccd92a..f4573e6f19 100644 --- a/lib/spack/spack/package_base.py +++ b/lib/spack/spack/package_base.py @@ -97,6 +97,9 @@ _spack_times_log = "install_times.json" _spack_configure_argsfile = "spack-configure-args.txt" +is_windows = sys.platform == "win32" + + def preferred_version(pkg): """ Returns a sorted list of the preferred versions of the package. @@ -182,6 +185,30 @@ class InstallPhase(object): return other +class WindowsRPathMeta(object): + """Collection of functionality surrounding Windows RPATH specific features + + This is essentially meaningless for all other platforms + due to their use of RPATH. All methods within this class are no-ops on + non Windows. Packages can customize and manipulate this class as + they would a genuine RPATH, i.e. adding directories that contain + runtime library dependencies""" + + def add_search_paths(self, *path): + """Add additional rpaths that are not implicitly included in the search + scheme + """ + self.win_rpath.include_additional_link_paths(*path) + + def windows_establish_runtime_linkage(self): + """Establish RPATH on Windows + + Performs symlinking to incorporate rpath dependencies to Windows runtime search paths + """ + if is_windows: + self.win_rpath.establish_link() + + #: Registers which are the detectable packages, by repo and package name #: Need a pass of package repositories to be filled. detectable_packages = collections.defaultdict(list) @@ -221,7 +248,7 @@ class DetectablePackageMeta(object): plat_exe = [] if hasattr(cls, "executables"): for exe in cls.executables: - if sys.platform == "win32": + if is_windows: exe = to_windows_exe(exe) plat_exe.append(exe) return plat_exe @@ -513,7 +540,7 @@ def test_log_pathname(test_stage, spec): return os.path.join(test_stage, "test-{0}-out.txt".format(TestSuite.test_pkg_id(spec))) -class PackageBase(six.with_metaclass(PackageMeta, PackageViewMixin, object)): +class PackageBase(six.with_metaclass(PackageMeta, WindowsRPathMeta, PackageViewMixin, object)): """This is the superclass for all spack packages. ***The Package class*** @@ -753,6 +780,8 @@ class PackageBase(six.with_metaclass(PackageMeta, PackageViewMixin, object)): # Set up timing variables self._fetch_time = 0.0 + self.win_rpath = fsys.WindowsSimulatedRPath(self) + if self.is_extension: pkg_cls = spack.repo.path.get_pkg_class(self.extendee_spec.name) pkg_cls(self.extendee_spec)._check_extendable() @@ -2754,6 +2783,8 @@ class PackageBase(six.with_metaclass(PackageMeta, PackageViewMixin, object)): deps = self.spec.dependencies(deptype="link") rpaths.extend(d.prefix.lib for d in deps if os.path.isdir(d.prefix.lib)) rpaths.extend(d.prefix.lib64 for d in deps if os.path.isdir(d.prefix.lib64)) + if is_windows: + rpaths.extend(d.prefix.bin for d in deps if os.path.isdir(d.prefix.bin)) return rpaths @property diff --git a/lib/spack/spack/test/conftest.py b/lib/spack/spack/test/conftest.py index ed8dc0f9bd..97a6b9de7b 100644 --- a/lib/spack/spack/test/conftest.py +++ b/lib/spack/spack/test/conftest.py @@ -27,7 +27,7 @@ import archspec.cpu.schema import llnl.util.lang import llnl.util.tty as tty -from llnl.util.filesystem import mkdirp, remove_linked_tree, working_dir +from llnl.util.filesystem import copy_tree, mkdirp, remove_linked_tree, working_dir import spack.binary_distribution import spack.caches @@ -803,7 +803,7 @@ def mock_store(tmpdir_factory, mock_repo_path, mock_configuration_scopes, _store with spack.store.use_store(str(store_path)) as store: with spack.repo.use_repositories(mock_repo_path): _populate(store.db) - store_path.copy(store_cache, mode=True, stat=True) + copy_tree(str(store_path), str(store_cache)) # Make the DB filesystem read-only to ensure we can't modify entries store_path.join(".spack-db").chmod(mode=0o555, rec=1) @@ -844,7 +844,7 @@ def mutable_database(database_mutable_config, _store_dir_and_cache): # Restore the initial state by copying the content of the cache back into # the store and making the database read-only store_path.remove(rec=1) - store_cache.copy(store_path, mode=True, stat=True) + copy_tree(str(store_cache), str(store_path)) store_path.join(".spack-db").chmod(mode=0o555, rec=1) diff --git a/lib/spack/spack/test/data/directory_search/a/libc.dll b/lib/spack/spack/test/data/directory_search/a/libc.dll new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/lib/spack/spack/test/data/directory_search/a/libc.dll diff --git a/lib/spack/spack/test/data/directory_search/a/libc.lib b/lib/spack/spack/test/data/directory_search/a/libc.lib new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/lib/spack/spack/test/data/directory_search/a/libc.lib diff --git a/lib/spack/spack/test/data/directory_search/b/liba.dll b/lib/spack/spack/test/data/directory_search/b/liba.dll new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/lib/spack/spack/test/data/directory_search/b/liba.dll diff --git a/lib/spack/spack/test/data/directory_search/b/liba.lib b/lib/spack/spack/test/data/directory_search/b/liba.lib new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/lib/spack/spack/test/data/directory_search/b/liba.lib diff --git a/lib/spack/spack/test/data/directory_search/b/libd.dll b/lib/spack/spack/test/data/directory_search/b/libd.dll new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/lib/spack/spack/test/data/directory_search/b/libd.dll diff --git a/lib/spack/spack/test/data/directory_search/b/libd.lib b/lib/spack/spack/test/data/directory_search/b/libd.lib new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/lib/spack/spack/test/data/directory_search/b/libd.lib diff --git a/lib/spack/spack/test/data/directory_search/c/libb.dll b/lib/spack/spack/test/data/directory_search/c/libb.dll new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/lib/spack/spack/test/data/directory_search/c/libb.dll diff --git a/lib/spack/spack/test/data/directory_search/c/libb.lib b/lib/spack/spack/test/data/directory_search/c/libb.lib new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/lib/spack/spack/test/data/directory_search/c/libb.lib diff --git a/lib/spack/spack/test/install.py b/lib/spack/spack/test/install.py index 079ff8851c..9093593bed 100644 --- a/lib/spack/spack/test/install.py +++ b/lib/spack/spack/test/install.py @@ -5,6 +5,7 @@ import os import shutil +import sys import pytest @@ -85,12 +86,11 @@ def test_pkg_attributes(install_mockery, mock_fetch, monkeypatch): # assert baz_headers.basenames == ['baz.h'] assert baz_headers.directories == [spec["baz"].home.include] - if "platform=windows" in spec: - lib_suffix = ".lib" - elif "platform=darwin" in spec: + lib_suffix = ".so" + if sys.platform == "win32": + lib_suffix = ".dll" + elif sys.platform == "darwin": lib_suffix = ".dylib" - else: - lib_suffix = ".so" foo_libs = spec[foo].libs assert foo_libs.basenames == ["libFoo" + lib_suffix] diff --git a/lib/spack/spack/test/llnl/util/file_list.py b/lib/spack/spack/test/llnl/util/file_list.py index 9ae33a90b5..0fe7572406 100644 --- a/lib/spack/spack/test/llnl/util/file_list.py +++ b/lib/spack/spack/test/llnl/util/file_list.py @@ -5,6 +5,7 @@ import fnmatch import os.path +import sys import pytest import six @@ -19,18 +20,30 @@ from llnl.util.filesystem import ( import spack.paths +is_windows = sys.platform == "win32" + @pytest.fixture() def library_list(): """Returns an instance of LibraryList.""" # Test all valid extensions: ['.a', '.dylib', '.so'] - libs = [ - "/dir1/liblapack.a", - "/dir2/libpython3.6.dylib", # name may contain periods - "/dir1/libblas.a", - "/dir3/libz.so", - "libmpi.so.20.10.1", # shared object libraries may be versioned - ] + libs = ( + [ + "/dir1/liblapack.a", + "/dir2/libpython3.6.dylib", # name may contain periods + "/dir1/libblas.a", + "/dir3/libz.so", + "libmpi.so.20.10.1", # shared object libraries may be versioned + ] + if not is_windows + else [ + "/dir1/liblapack.lib", + "/dir2/libpython3.6.dll", + "/dir1/libblas.lib", + "/dir3/libz.dll", + "libmpi.dll.20.10.1", + ] + ) return LibraryList(libs) @@ -52,6 +65,16 @@ def header_list(): return h +# TODO: Remove below when llnl.util.filesystem.find_libraries becomes spec aware +plat_static_ext = "lib" if is_windows else "a" + + +plat_shared_ext = "dll" if is_windows else "so" + + +plat_apple_shared_ext = "dll" if is_windows else "dylib" + + class TestLibraryList(object): def test_repr(self, library_list): x = eval(repr(library_list)) @@ -62,11 +85,11 @@ class TestLibraryList(object): s1 = library_list.joined() expected = " ".join( [ - "/dir1/liblapack.a", - "/dir2/libpython3.6.dylib", - "/dir1/libblas.a", - "/dir3/libz.so", - "libmpi.so.20.10.1", + "/dir1/liblapack.%s" % plat_static_ext, + "/dir2/libpython3.6.%s" % plat_apple_shared_ext, + "/dir1/libblas.%s" % plat_static_ext, + "/dir3/libz.%s" % plat_shared_ext, + "libmpi.%s.20.10.1" % plat_shared_ext, ] ) assert s1 == expected @@ -77,11 +100,11 @@ class TestLibraryList(object): s3 = library_list.joined(";") expected = ";".join( [ - "/dir1/liblapack.a", - "/dir2/libpython3.6.dylib", - "/dir1/libblas.a", - "/dir3/libz.so", - "libmpi.so.20.10.1", + "/dir1/liblapack.%s" % plat_static_ext, + "/dir2/libpython3.6.%s" % plat_apple_shared_ext, + "/dir1/libblas.%s" % plat_static_ext, + "/dir3/libz.%s" % plat_shared_ext, + "libmpi.%s.20.10.1" % plat_shared_ext, ] ) assert s3 == expected @@ -117,7 +140,7 @@ class TestLibraryList(object): def test_get_item(self, library_list): a = library_list[0] - assert a == "/dir1/liblapack.a" + assert a == "/dir1/liblapack.%s" % plat_static_ext b = library_list[:] assert type(b) == type(library_list) @@ -126,9 +149,9 @@ class TestLibraryList(object): def test_add(self, library_list): pylist = [ - "/dir1/liblapack.a", # removed from the final list - "/dir2/libmpi.so", - "/dir4/libnew.a", + "/dir1/liblapack.%s" % plat_static_ext, # removed from the final list + "/dir2/libmpi.%s" % plat_shared_ext, + "/dir4/libnew.%s" % plat_static_ext, ] another = LibraryList(pylist) both = library_list + another |