summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--lib/spack/llnl/util/filesystem.py181
-rw-r--r--lib/spack/spack/installer.py14
-rw-r--r--lib/spack/spack/package_base.py35
-rw-r--r--lib/spack/spack/test/conftest.py6
-rw-r--r--lib/spack/spack/test/data/directory_search/a/libc.dll0
-rw-r--r--lib/spack/spack/test/data/directory_search/a/libc.lib0
-rw-r--r--lib/spack/spack/test/data/directory_search/b/liba.dll0
-rw-r--r--lib/spack/spack/test/data/directory_search/b/liba.lib0
-rw-r--r--lib/spack/spack/test/data/directory_search/b/libd.dll0
-rw-r--r--lib/spack/spack/test/data/directory_search/b/libd.lib0
-rw-r--r--lib/spack/spack/test/data/directory_search/c/libb.dll0
-rw-r--r--lib/spack/spack/test/data/directory_search/c/libb.lib0
-rw-r--r--lib/spack/spack/test/install.py10
-rw-r--r--lib/spack/spack/test/llnl/util/file_list.py65
-rw-r--r--var/spack/repos/builtin.mock/packages/attributes-foo/package.py12
15 files changed, 277 insertions, 46 deletions
diff --git a/lib/spack/llnl/util/filesystem.py b/lib/spack/llnl/util/filesystem.py
index 8a420756ef..a161a64d2a 100644
--- a/lib/spack/llnl/util/filesystem.py
+++ b/lib/spack/llnl/util/filesystem.py
@@ -22,7 +22,7 @@ import six
from llnl.util import tty
from llnl.util.compat import Sequence
from llnl.util.lang import dedupe, memoized
-from llnl.util.symlink import symlink
+from llnl.util.symlink import islink, symlink
from spack.util.executable import Executable
from spack.util.path import path_to_os_path, system_path_filter
@@ -637,7 +637,11 @@ def copy_tree(src, dest, symlinks=True, ignore=None, _permissions=False):
if symlinks:
target = os.readlink(s)
if os.path.isabs(target):
- new_target = re.sub(abs_src, abs_dest, target)
+
+ def escaped_path(path):
+ return path.replace("\\", r"\\")
+
+ new_target = re.sub(escaped_path(abs_src), escaped_path(abs_dest), target)
if new_target != target:
tty.debug("Redirecting link {0} to {1}".format(target, new_target))
target = new_target
@@ -1903,7 +1907,11 @@ class LibraryList(FileList):
name = x[3:]
# Valid extensions include: ['.dylib', '.so', '.a']
- for ext in [".dylib", ".so", ".a"]:
+ # on non Windows platform
+ # Windows valid library extensions are:
+ # ['.dll', '.lib']
+ valid_exts = [".dll", ".lib"] if is_windows else [".dylib", ".so", ".a"]
+ for ext in valid_exts:
i = name.rfind(ext)
if i != -1:
names.append(name[:i])
@@ -2046,15 +2054,23 @@ def find_libraries(libraries, root, shared=True, recursive=False):
message = message.format(find_libraries.__name__, type(libraries))
raise TypeError(message)
+ if is_windows:
+ static = "lib"
+ shared = "dll"
+ else:
+ # Used on both Linux and macOS
+ static = "a"
+ shared = "so"
+
# Construct the right suffix for the library
if shared:
# Used on both Linux and macOS
- suffixes = ["so"]
+ suffixes = [shared]
if sys.platform == "darwin":
# Only used on macOS
suffixes.append("dylib")
else:
- suffixes = ["a"]
+ suffixes = [static]
# List of libraries we are searching with suffixes
libraries = ["{0}.{1}".format(lib, suffix) for lib in libraries for suffix in suffixes]
@@ -2067,7 +2083,11 @@ def find_libraries(libraries, root, shared=True, recursive=False):
# perform first non-recursive search in root/lib then in root/lib64 and
# finally search all of root recursively. The search stops when the first
# match is found.
- for subdir in ("lib", "lib64"):
+ common_lib_dirs = ["lib", "lib64"]
+ if is_windows:
+ common_lib_dirs.extend(["bin", "Lib"])
+
+ for subdir in common_lib_dirs:
dirname = join_path(root, subdir)
if not os.path.isdir(dirname):
continue
@@ -2080,6 +2100,155 @@ def find_libraries(libraries, root, shared=True, recursive=False):
return LibraryList(found_libs)
+def find_all_shared_libraries(root, recursive=False):
+ """Convenience function that returns the list of all shared libraries found
+ in the directory passed as argument.
+
+ See documentation for `llnl.util.filesystem.find_libraries` for more information
+ """
+ return find_libraries("*", root=root, shared=True, recursive=recursive)
+
+
+def find_all_static_libraries(root, recursive=False):
+ """Convenience function that returns the list of all static libraries found
+ in the directory passed as argument.
+
+ See documentation for `llnl.util.filesystem.find_libraries` for more information
+ """
+ return find_libraries("*", root=root, shared=False, recursive=recursive)
+
+
+def find_all_libraries(root, recursive=False):
+ """Convenience function that returns the list of all libraries found
+ in the directory passed as argument.
+
+ See documentation for `llnl.util.filesystem.find_libraries` for more information
+ """
+
+ return find_all_shared_libraries(root, recursive=recursive) + find_all_static_libraries(
+ root, recursive=recursive
+ )
+
+
+class WindowsSimulatedRPath(object):
+ """Class representing Windows filesystem rpath analog
+
+ One instance of this class is associated with a package (only on Windows)
+ For each lib/binary directory in an associated package, this class introduces
+ a symlink to any/all dependent libraries/binaries. This includes the packages
+ own bin/lib directories, meaning the libraries are linked to the bianry directory
+ and vis versa.
+ """
+
+ def __init__(self, package, link_install_prefix=True):
+ """
+ Args:
+ package (spack.package_base.PackageBase): Package requiring links
+ link_install_prefix (bool): Link against package's own install or stage root.
+ Packages that run their own executables during build and require rpaths to
+ the build directory during build time require this option. Default: install
+ root
+ """
+ self.pkg = package
+ self._addl_rpaths = set()
+ self.link_install_prefix = link_install_prefix
+ self._internal_links = set()
+
+ @property
+ def link_dest(self):
+ """
+ Set of directories where package binaries/libraries are located.
+ """
+ if hasattr(self.pkg, "libs") and self.pkg.libs:
+ pkg_libs = set(self.pkg.libs.directories)
+ else:
+ pkg_libs = set((self.pkg.prefix.lib, self.pkg.prefix.lib64))
+
+ return pkg_libs | set([self.pkg.prefix.bin]) | self.internal_links
+
+ @property
+ def internal_links(self):
+ """
+ linking that would need to be established within the package itself. Useful for links
+ against extension modules/build time executables/internal linkage
+ """
+ return self._internal_links
+
+ def add_internal_links(self, *dest):
+ """
+ Incorporate additional paths into the rpath (sym)linking scheme.
+
+ Paths provided to this method are linked against by a package's libraries
+ and libraries found at these paths are linked against a package's binaries.
+ (i.e. /site-packages -> /bin and /bin -> /site-packages)
+
+ Specified paths should be outside of a package's lib, lib64, and bin
+ directories.
+ """
+ self._internal_links = self._internal_links | set(*dest)
+
+ @property
+ def link_targets(self):
+ """
+ Set of libraries this package needs to link against during runtime
+ These packages will each be symlinked into the packages lib and binary dir
+ """
+
+ dependent_libs = []
+ for path in self.pkg.rpath:
+ dependent_libs.extend(list(find_all_shared_libraries(path, recursive=True)))
+ for extra_path in self._addl_rpaths:
+ dependent_libs.extend(list(find_all_shared_libraries(extra_path, recursive=True)))
+ return set(dependent_libs)
+
+ def include_additional_link_paths(self, *paths):
+ """
+ Add libraries found at the root of provided paths to runtime linking
+
+ These are libraries found outside of the typical scope of rpath linking
+ that require manual inclusion in a runtime linking scheme
+
+ Args:
+ *paths (str): arbitrary number of paths to be added to runtime linking
+ """
+ self._addl_rpaths = self._addl_rpaths | set(paths)
+
+ def establish_link(self):
+ """
+ (sym)link packages to runtime dependencies based on RPath configuration for
+ Windows heuristics
+ """
+ # from build_environment.py:463
+ # The top-level package is always RPATHed. It hasn't been installed yet
+ # so the RPATHs are added unconditionally
+
+ # for each binary install dir in self.pkg (i.e. pkg.prefix.bin, pkg.prefix.lib)
+ # install a symlink to each dependent library
+ for library, lib_dir in itertools.product(self.link_targets, self.link_dest):
+ if not path_contains_subdirectory(library, lib_dir):
+ file_name = os.path.basename(library)
+ dest_file = os.path.join(lib_dir, file_name)
+ if os.path.exists(lib_dir):
+ try:
+ symlink(library, dest_file)
+ # For py2 compatibility, we have to catch the specific Windows error code
+ # associate with trying to create a file that already exists (winerror 183)
+ except OSError as e:
+ if e.winerror == 183:
+ # We have either already symlinked or we are encoutering a naming clash
+ # either way, we don't want to overwrite existing libraries
+ already_linked = islink(dest_file)
+ tty.debug(
+ "Linking library %s to %s failed, " % (library, dest_file)
+ + "already linked."
+ if already_linked
+ else "library with name %s already exists." % file_name
+ )
+ pass
+ else:
+ raise e
+
+
@system_path_filter
@memoized
def can_access_dir(path):
diff --git a/lib/spack/spack/installer.py b/lib/spack/spack/installer.py
index d97db35535..2f0415b0c1 100644
--- a/lib/spack/spack/installer.py
+++ b/lib/spack/spack/installer.py
@@ -84,6 +84,9 @@ STATUS_DEQUEUED = "dequeued"
#: queue invariants).
STATUS_REMOVED = "removed"
+is_windows = sys.platform == "win32"
+is_osx = sys.platform == "darwin"
+
class InstallAction(object):
#: Don't perform an install
@@ -165,7 +168,9 @@ def _do_fake_install(pkg):
if not pkg.name.startswith("lib"):
library = "lib" + library
- dso_suffix = ".dylib" if sys.platform == "darwin" else ".so"
+ plat_shared = ".dll" if is_windows else ".so"
+ plat_static = ".lib" if is_windows else ".a"
+ dso_suffix = ".dylib" if is_osx else plat_shared
# Install fake command
fs.mkdirp(pkg.prefix.bin)
@@ -180,7 +185,7 @@ def _do_fake_install(pkg):
# Install fake shared and static libraries
fs.mkdirp(pkg.prefix.lib)
- for suffix in [dso_suffix, ".a"]:
+ for suffix in [dso_suffix, plat_static]:
fs.touch(os.path.join(pkg.prefix.lib, library + suffix))
# Install fake man page
@@ -1214,7 +1219,10 @@ class PackageInstaller(object):
spack.package_base.PackageBase._verbose = spack.build_environment.start_build_process(
pkg, build_process, install_args
)
-
+ # Currently this is how RPATH-like behavior is achieved on Windows, after install
+ # establish runtime linkage via Windows Runtime link object
+ # Note: this is a no-op on non Windows platforms
+ pkg.windows_establish_runtime_linkage()
# Note: PARENT of the build process adds the new package to
# the database, so that we don't need to re-read from file.
spack.store.db.add(pkg.spec, spack.store.layout, explicit=explicit)
diff --git a/lib/spack/spack/package_base.py b/lib/spack/spack/package_base.py
index e8f7ccd92a..f4573e6f19 100644
--- a/lib/spack/spack/package_base.py
+++ b/lib/spack/spack/package_base.py
@@ -97,6 +97,9 @@ _spack_times_log = "install_times.json"
_spack_configure_argsfile = "spack-configure-args.txt"
+is_windows = sys.platform == "win32"
+
+
def preferred_version(pkg):
"""
Returns a sorted list of the preferred versions of the package.
@@ -182,6 +185,30 @@ class InstallPhase(object):
return other
+class WindowsRPathMeta(object):
+ """Collection of functionality surrounding Windows RPATH specific features
+
+ This is essentially meaningless for all other platforms
+ due to their use of RPATH. All methods within this class are no-ops on
+ non Windows. Packages can customize and manipulate this class as
+ they would a genuine RPATH, i.e. adding directories that contain
+ runtime library dependencies"""
+
+ def add_search_paths(self, *path):
+ """Add additional rpaths that are not implicitly included in the search
+ scheme
+ """
+ self.win_rpath.include_additional_link_paths(*path)
+
+ def windows_establish_runtime_linkage(self):
+ """Establish RPATH on Windows
+
+ Performs symlinking to incorporate rpath dependencies to Windows runtime search paths
+ """
+ if is_windows:
+ self.win_rpath.establish_link()
+
+
#: Registers which are the detectable packages, by repo and package name
#: Need a pass of package repositories to be filled.
detectable_packages = collections.defaultdict(list)
@@ -221,7 +248,7 @@ class DetectablePackageMeta(object):
plat_exe = []
if hasattr(cls, "executables"):
for exe in cls.executables:
- if sys.platform == "win32":
+ if is_windows:
exe = to_windows_exe(exe)
plat_exe.append(exe)
return plat_exe
@@ -513,7 +540,7 @@ def test_log_pathname(test_stage, spec):
return os.path.join(test_stage, "test-{0}-out.txt".format(TestSuite.test_pkg_id(spec)))
-class PackageBase(six.with_metaclass(PackageMeta, PackageViewMixin, object)):
+class PackageBase(six.with_metaclass(PackageMeta, WindowsRPathMeta, PackageViewMixin, object)):
"""This is the superclass for all spack packages.
***The Package class***
@@ -753,6 +780,8 @@ class PackageBase(six.with_metaclass(PackageMeta, PackageViewMixin, object)):
# Set up timing variables
self._fetch_time = 0.0
+ self.win_rpath = fsys.WindowsSimulatedRPath(self)
+
if self.is_extension:
pkg_cls = spack.repo.path.get_pkg_class(self.extendee_spec.name)
pkg_cls(self.extendee_spec)._check_extendable()
@@ -2754,6 +2783,8 @@ class PackageBase(six.with_metaclass(PackageMeta, PackageViewMixin, object)):
deps = self.spec.dependencies(deptype="link")
rpaths.extend(d.prefix.lib for d in deps if os.path.isdir(d.prefix.lib))
rpaths.extend(d.prefix.lib64 for d in deps if os.path.isdir(d.prefix.lib64))
+ if is_windows:
+ rpaths.extend(d.prefix.bin for d in deps if os.path.isdir(d.prefix.bin))
return rpaths
@property
diff --git a/lib/spack/spack/test/conftest.py b/lib/spack/spack/test/conftest.py
index ed8dc0f9bd..97a6b9de7b 100644
--- a/lib/spack/spack/test/conftest.py
+++ b/lib/spack/spack/test/conftest.py
@@ -27,7 +27,7 @@ import archspec.cpu.schema
import llnl.util.lang
import llnl.util.tty as tty
-from llnl.util.filesystem import mkdirp, remove_linked_tree, working_dir
+from llnl.util.filesystem import copy_tree, mkdirp, remove_linked_tree, working_dir
import spack.binary_distribution
import spack.caches
@@ -803,7 +803,7 @@ def mock_store(tmpdir_factory, mock_repo_path, mock_configuration_scopes, _store
with spack.store.use_store(str(store_path)) as store:
with spack.repo.use_repositories(mock_repo_path):
_populate(store.db)
- store_path.copy(store_cache, mode=True, stat=True)
+ copy_tree(str(store_path), str(store_cache))
# Make the DB filesystem read-only to ensure we can't modify entries
store_path.join(".spack-db").chmod(mode=0o555, rec=1)
@@ -844,7 +844,7 @@ def mutable_database(database_mutable_config, _store_dir_and_cache):
# Restore the initial state by copying the content of the cache back into
# the store and making the database read-only
store_path.remove(rec=1)
- store_cache.copy(store_path, mode=True, stat=True)
+ copy_tree(str(store_cache), str(store_path))
store_path.join(".spack-db").chmod(mode=0o555, rec=1)
diff --git a/lib/spack/spack/test/data/directory_search/a/libc.dll b/lib/spack/spack/test/data/directory_search/a/libc.dll
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/lib/spack/spack/test/data/directory_search/a/libc.dll
diff --git a/lib/spack/spack/test/data/directory_search/a/libc.lib b/lib/spack/spack/test/data/directory_search/a/libc.lib
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/lib/spack/spack/test/data/directory_search/a/libc.lib
diff --git a/lib/spack/spack/test/data/directory_search/b/liba.dll b/lib/spack/spack/test/data/directory_search/b/liba.dll
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/lib/spack/spack/test/data/directory_search/b/liba.dll
diff --git a/lib/spack/spack/test/data/directory_search/b/liba.lib b/lib/spack/spack/test/data/directory_search/b/liba.lib
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/lib/spack/spack/test/data/directory_search/b/liba.lib
diff --git a/lib/spack/spack/test/data/directory_search/b/libd.dll b/lib/spack/spack/test/data/directory_search/b/libd.dll
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/lib/spack/spack/test/data/directory_search/b/libd.dll
diff --git a/lib/spack/spack/test/data/directory_search/b/libd.lib b/lib/spack/spack/test/data/directory_search/b/libd.lib
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/lib/spack/spack/test/data/directory_search/b/libd.lib
diff --git a/lib/spack/spack/test/data/directory_search/c/libb.dll b/lib/spack/spack/test/data/directory_search/c/libb.dll
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/lib/spack/spack/test/data/directory_search/c/libb.dll
diff --git a/lib/spack/spack/test/data/directory_search/c/libb.lib b/lib/spack/spack/test/data/directory_search/c/libb.lib
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/lib/spack/spack/test/data/directory_search/c/libb.lib
diff --git a/lib/spack/spack/test/install.py b/lib/spack/spack/test/install.py
index 079ff8851c..9093593bed 100644
--- a/lib/spack/spack/test/install.py
+++ b/lib/spack/spack/test/install.py
@@ -5,6 +5,7 @@
import os
import shutil
+import sys
import pytest
@@ -85,12 +86,11 @@ def test_pkg_attributes(install_mockery, mock_fetch, monkeypatch):
# assert baz_headers.basenames == ['baz.h']
assert baz_headers.directories == [spec["baz"].home.include]
- if "platform=windows" in spec:
- lib_suffix = ".lib"
- elif "platform=darwin" in spec:
+ lib_suffix = ".so"
+ if sys.platform == "win32":
+ lib_suffix = ".dll"
+ elif sys.platform == "darwin":
lib_suffix = ".dylib"
- else:
- lib_suffix = ".so"
foo_libs = spec[foo].libs
assert foo_libs.basenames == ["libFoo" + lib_suffix]
diff --git a/lib/spack/spack/test/llnl/util/file_list.py b/lib/spack/spack/test/llnl/util/file_list.py
index 9ae33a90b5..0fe7572406 100644
--- a/lib/spack/spack/test/llnl/util/file_list.py
+++ b/lib/spack/spack/test/llnl/util/file_list.py
@@ -5,6 +5,7 @@
import fnmatch
import os.path
+import sys
import pytest
import six
@@ -19,18 +20,30 @@ from llnl.util.filesystem import (
import spack.paths
+is_windows = sys.platform == "win32"
+
@pytest.fixture()
def library_list():
"""Returns an instance of LibraryList."""
# Test all valid extensions: ['.a', '.dylib', '.so']
- libs = [
- "/dir1/liblapack.a",
- "/dir2/libpython3.6.dylib", # name may contain periods
- "/dir1/libblas.a",
- "/dir3/libz.so",
- "libmpi.so.20.10.1", # shared object libraries may be versioned
- ]
+ libs = (
+ [
+ "/dir1/liblapack.a",
+ "/dir2/libpython3.6.dylib", # name may contain periods
+ "/dir1/libblas.a",
+ "/dir3/libz.so",
+ "libmpi.so.20.10.1", # shared object libraries may be versioned
+ ]
+ if not is_windows
+ else [
+ "/dir1/liblapack.lib",
+ "/dir2/libpython3.6.dll",
+ "/dir1/libblas.lib",
+ "/dir3/libz.dll",
+ "libmpi.dll.20.10.1",
+ ]
+ )
return LibraryList(libs)
@@ -52,6 +65,16 @@ def header_list():
return h
+# TODO: Remove below when llnl.util.filesystem.find_libraries becomes spec aware
+plat_static_ext = "lib" if is_windows else "a"
+
+
+plat_shared_ext = "dll" if is_windows else "so"
+
+
+plat_apple_shared_ext = "dll" if is_windows else "dylib"
+
+
class TestLibraryList(object):
def test_repr(self, library_list):
x = eval(repr(library_list))
@@ -62,11 +85,11 @@ class TestLibraryList(object):
s1 = library_list.joined()
expected = " ".join(
[
- "/dir1/liblapack.a",
- "/dir2/libpython3.6.dylib",
- "/dir1/libblas.a",
- "/dir3/libz.so",
- "libmpi.so.20.10.1",
+ "/dir1/liblapack.%s" % plat_static_ext,
+ "/dir2/libpython3.6.%s" % plat_apple_shared_ext,
+ "/dir1/libblas.%s" % plat_static_ext,
+ "/dir3/libz.%s" % plat_shared_ext,
+ "libmpi.%s.20.10.1" % plat_shared_ext,
]
)
assert s1 == expected
@@ -77,11 +100,11 @@ class TestLibraryList(object):
s3 = library_list.joined(";")
expected = ";".join(
[
- "/dir1/liblapack.a",
- "/dir2/libpython3.6.dylib",
- "/dir1/libblas.a",
- "/dir3/libz.so",
- "libmpi.so.20.10.1",
+ "/dir1/liblapack.%s" % plat_static_ext,
+ "/dir2/libpython3.6.%s" % plat_apple_shared_ext,
+ "/dir1/libblas.%s" % plat_static_ext,
+ "/dir3/libz.%s" % plat_shared_ext,
+ "libmpi.%s.20.10.1" % plat_shared_ext,
]
)
assert s3 == expected
@@ -117,7 +140,7 @@ class TestLibraryList(object):
def test_get_item(self, library_list):
a = library_list[0]
- assert a == "/dir1/liblapack.a"
+ assert a == "/dir1/liblapack.%s" % plat_static_ext
b = library_list[:]
assert type(b) == type(library_list)
@@ -126,9 +149,9 @@ class TestLibraryList(object):
def test_add(self, library_list):
pylist = [
- "/dir1/liblapack.a", # removed from the final list
- "/dir2/libmpi.so",
- "/dir4/libnew.a",
+ "/dir1/liblapack.%s" % plat_static_ext, # removed from the final list
+ "/dir2/libmpi.%s" % plat_shared_ext,
+ "/dir4/libnew.%s" % plat_static_ext,
]
another = LibraryList(pylist)
both = library_list + another
diff --git a/var/spack/repos/builtin.mock/packages/attributes-foo/package.py b/var/spack/repos/builtin.mock/packages/attributes-foo/package.py
index f4bb9059a4..9010c52958 100644
--- a/var/spack/repos/builtin.mock/packages/attributes-foo/package.py
+++ b/var/spack/repos/builtin.mock/packages/attributes-foo/package.py
@@ -2,6 +2,7 @@
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+import sys
from spack.package import *
@@ -14,13 +15,12 @@ class AttributesFoo(BundlePackage):
provides("baz")
def install(self, spec, prefix):
- if "platform=windows" in spec:
- lib_suffix = ".lib"
- elif "platform=darwin" in spec:
- lib_suffix = ".dylib"
- else:
- lib_suffix = ".so"
+ lib_suffix = ".so"
+ if sys.platform == "win32":
+ lib_suffix = ".dll"
+ elif sys.platform == "darwin":
+ lib_suffix = ".dylib"
mkdirp(prefix.include)
touch(prefix.include.join("foo.h"))
mkdirp(prefix.include.bar)
generated by cgit v1.2.3-60-g2f50 (git 2.42.0) at 2024-06-29 04:18:34 +0000