From c22a14534411e48a34879688b3b6ef17cf109ed3 Mon Sep 17 00:00:00 2001 From: Ben Boeckel Date: Sat, 24 Aug 2019 09:21:45 -0400 Subject: Add implicit rpaths to compiler detection (#7153) Uses code from CMake to detect implicit link paths from compilers System paths are filtered out of implicit link paths Implicit link paths added to compiler config and object under `implicit_rpaths` Implicit link paths added as rpaths to compile line through env/cc wrapper Authored by: "Ben Boeckel " Co-authored by: "Peter Scheibel " Co-authored by: "Gregory Becker " --- lib/spack/env/cc | 6 + lib/spack/spack/build_environment.py | 4 + lib/spack/spack/compiler.py | 147 ++++++++++++++++++++- lib/spack/spack/compilers/__init__.py | 18 ++- lib/spack/spack/compilers/arm.py | 4 + lib/spack/spack/compilers/cce.py | 4 + lib/spack/spack/compilers/clang.py | 4 + lib/spack/spack/compilers/gcc.py | 4 + lib/spack/spack/compilers/intel.py | 4 + lib/spack/spack/compilers/pgi.py | 4 + lib/spack/spack/compilers/xl.py | 4 + lib/spack/spack/schema/compilers.py | 4 + .../data/compiler_verbose_output/cce-8.6.5.txt | 10 ++ .../data/compiler_verbose_output/clang-4.0.1.txt | 20 +++ .../clang-9.0.0-apple-ld.txt | 7 + .../data/compiler_verbose_output/gcc-7.3.1.txt | 36 +++++ .../data/compiler_verbose_output/icc-16.0.3.txt | 4 + .../obscure-parsing-rules.txt | 3 + .../data/compiler_verbose_output/pgcc-16.3.txt | 11 ++ .../data/compiler_verbose_output/xl-13.1.5.txt | 5 + lib/spack/spack/test/link_paths.py | 89 +++++++++++++ 21 files changed, 384 insertions(+), 8 deletions(-) create mode 100644 lib/spack/spack/test/data/compiler_verbose_output/cce-8.6.5.txt create mode 100644 lib/spack/spack/test/data/compiler_verbose_output/clang-4.0.1.txt create mode 100644 lib/spack/spack/test/data/compiler_verbose_output/clang-9.0.0-apple-ld.txt create mode 100644 lib/spack/spack/test/data/compiler_verbose_output/gcc-7.3.1.txt create mode 100644 lib/spack/spack/test/data/compiler_verbose_output/icc-16.0.3.txt create mode 100644 lib/spack/spack/test/data/compiler_verbose_output/obscure-parsing-rules.txt create mode 100644 lib/spack/spack/test/data/compiler_verbose_output/pgcc-16.3.txt create mode 100644 lib/spack/spack/test/data/compiler_verbose_output/xl-13.1.5.txt create mode 100644 lib/spack/spack/test/link_paths.py (limited to 'lib') diff --git a/lib/spack/env/cc b/lib/spack/env/cc index f5a71d87d1..c7ea8c793b 100755 --- a/lib/spack/env/cc +++ b/lib/spack/env/cc @@ -426,6 +426,12 @@ case "$mode" in rpaths+=("${extra_rpaths[@]}") fi + # Set implicit RPATHs + IFS=':' read -ra implicit_rpaths <<< "$SPACK_COMPILER_IMPLICIT_RPATHS" + if [[ "$add_rpaths" != "false" ]] ; then + rpaths+=("${implicit_rpaths[@]}") + fi + # Add SPACK_LDLIBS to args for lib in "${SPACK_LDLIBS[@]}"; do libs+=("${lib#-l}") diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index ff7d41a39a..d37e139e3c 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -348,6 +348,10 @@ def set_build_environment_variables(pkg, env, dirty): extra_rpaths = ':'.join(compiler.extra_rpaths) env.set('SPACK_COMPILER_EXTRA_RPATHS', extra_rpaths) + if compiler.implicit_rpaths: + implicit_rpaths = ':'.join(compiler.implicit_rpaths) + env.set('SPACK_COMPILER_IMPLICIT_RPATHS', implicit_rpaths) + # Add bin directories from dependencies to the PATH for the build. for prefix in build_prefixes: for dirname in ['bin', 'bin64']: diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index 1136485ba9..6fd3dd0457 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -6,13 +6,18 @@ import os import re import itertools +import shutil +import tempfile import llnl.util.filesystem +import llnl.util.tty as tty import spack.error import spack.spec import spack.architecture import spack.util.executable +import spack.compilers +from spack.util.environment import filter_system_paths __all__ = ['Compiler'] @@ -58,6 +63,97 @@ def tokenize_flags(flags_str): return flags +#: regex for parsing linker lines +_LINKER_LINE = re.compile( + r'^( *|.*[/\\])' + r'(link|ld|([^/\\]+-)?ld|collect2)' + r'[^/\\]*( |$)') + +#: components of linker lines to ignore +_LINKER_LINE_IGNORE = re.compile(r'(collect2 version|^[A-Za-z0-9_]+=|/ldfe )') + +#: regex to match linker search paths +_LINK_DIR_ARG = re.compile(r'^-L(.:)?(?P[/\\].*)') + +#: regex to match linker library path arguments +_LIBPATH_ARG = re.compile(r'^[-/](LIBPATH|libpath):(?P.*)') + + +def is_subdirectory(path, prefix): + path = os.path.abspath(path) + prefix = os.path.abspath(prefix) + os.path.sep + return path.startswith(prefix) + + +def _parse_implicit_rpaths(string): + """Parse implicit link paths from compiler debug output. + + This gives the compiler runtime library paths that we need to add to + the RPATH of generated binaries and libraries. It allows us to + ensure, e.g., that codes load the right libstdc++ for their compiler. + """ + lib_search_paths = False + raw_link_dirs = [] + tty.debug('parsing implicit link info') + for line in string.splitlines(): + if lib_search_paths: + if line.startswith('\t'): + raw_link_dirs.append(line[1:]) + continue + else: + lib_search_paths = False + elif line.startswith('Library search paths:'): + lib_search_paths = True + + if not _LINKER_LINE.match(line): + continue + if _LINKER_LINE_IGNORE.match(line): + continue + tty.debug('linker line: %s' % line) + + next_arg = False + for arg in line.split(): + if arg in ('-L', '-Y'): + next_arg = True + continue + + if next_arg: + raw_link_dirs.append(arg) + next_arg = False + continue + + link_dir_arg = _LINK_DIR_ARG.match(arg) + if link_dir_arg: + link_dir = link_dir_arg.group('dir') + tty.debug('linkdir: %s' % link_dir) + raw_link_dirs.append(link_dir) + + link_dir_arg = _LIBPATH_ARG.match(arg) + if link_dir_arg: + link_dir = link_dir_arg.group('dir') + tty.debug('libpath: %s', link_dir) + raw_link_dirs.append(link_dir) + tty.debug('found raw link dirs: %s' % ', '.join(raw_link_dirs)) + + implicit_link_dirs = list() + visited = set() + for link_dir in raw_link_dirs: + normalized_path = os.path.abspath(link_dir) + if normalized_path not in visited: + implicit_link_dirs.append(normalized_path) + visited.add(normalized_path) + implicit_link_dirs = filter_system_paths(implicit_link_dirs) + + # Additional filtering: we also want to exclude paths that are + # subdirectories of /usr/lib/ and /lib/ + implicit_link_dirs = list( + path for path in implicit_link_dirs + if not any(is_subdirectory(path, d) for d in ['/lib/', '/usr/lib/'])) + + tty.debug('found link dirs: %s' % ', '.join(implicit_link_dirs)) + return implicit_link_dirs + + class Compiler(object): """This class encapsulates a Spack "compiler", which includes C, C++, and Fortran compilers. Subclasses should implement @@ -114,12 +210,15 @@ class Compiler(object): def __init__(self, cspec, operating_system, target, paths, modules=[], alias=None, environment=None, - extra_rpaths=None, **kwargs): + extra_rpaths=None, implicit_rpaths=None, + **kwargs): self.spec = cspec self.operating_system = str(operating_system) self.target = target self.modules = modules self.alias = alias + self.extra_rpaths = extra_rpaths + self.implicit_rpaths = implicit_rpaths def check(exe): if exe is None: @@ -152,6 +251,52 @@ class Compiler(object): def version(self): return self.spec.version + @classmethod + def verbose_flag(cls): + """ + This property should be overridden in the compiler subclass if a + verbose flag is available. + + If it is not overridden, it is assumed to not be supported. + """ + + @classmethod + def parse_implicit_rpaths(cls, string): + """Parses link paths out of compiler debug output. + + Args: + string (str): compiler debug output as a string + + Returns: + (list of str): implicit link paths parsed from the compiler output + + Subclasses can override this to customize. + """ + return _parse_implicit_rpaths(string) + + @classmethod + def determine_implicit_rpaths(cls, paths): + first_compiler = next((c for c in paths if c), None) + if not first_compiler: + return [] + + try: + tmpdir = tempfile.mkdtemp(prefix='spack-implicit-link-info') + fout = os.path.join(tmpdir, 'output') + fin = os.path.join(tmpdir, 'main.c') + + with open(fin, 'w+') as csource: + csource.write( + 'int main(int argc, char* argv[]) { ' + '(void)argc; (void)argv; return 0; }\n') + compiler_exe = spack.util.executable.Executable(first_compiler) + output = str(compiler_exe(cls.verbose_flag(), fin, '-o', fout, + output=str, error=str)) # str for py2 + + return cls.parse_implicit_rpaths(output) + finally: + shutil.rmtree(tmpdir, ignore_errors=True) + # This property should be overridden in the compiler subclass if # OpenMP is supported by that compiler @property diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index 20f40eef09..b4c548365e 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -10,7 +10,6 @@ import collections import itertools import multiprocessing.pool import os - import platform as py_platform import six @@ -31,7 +30,7 @@ _imported_compilers_module = 'spack.compilers' _path_instance_vars = ['cc', 'cxx', 'f77', 'fc'] _flags_instance_vars = ['cflags', 'cppflags', 'cxxflags', 'fflags'] _other_instance_vars = ['modules', 'operating_system', 'environment', - 'extra_rpaths'] + 'extra_rpaths', 'implicit_rpaths'] _cache_config_file = [] # TODO: Caches at module level make it difficult to mock configurations in @@ -71,9 +70,10 @@ def _to_dict(compiler): if hasattr(compiler, attr))) d['operating_system'] = str(compiler.operating_system) d['target'] = str(compiler.target) - d['modules'] = compiler.modules if compiler.modules else [] - d['environment'] = compiler.environment if compiler.environment else {} - d['extra_rpaths'] = compiler.extra_rpaths if compiler.extra_rpaths else [] + d['modules'] = compiler.modules or [] + d['environment'] = compiler.environment or {} + d['extra_rpaths'] = compiler.extra_rpaths or [] + d['implicit_rpaths'] = compiler.implicit_rpaths or [] if compiler.alias: d['alias'] = compiler.alias @@ -350,9 +350,11 @@ def compiler_from_dict(items): compiler_flags = items.get('flags', {}) environment = items.get('environment', {}) extra_rpaths = items.get('extra_rpaths', []) + implicit_rpaths = items.get('implicit_rpaths') return cls(cspec, os, target, compiler_paths, mods, alias, - environment, extra_rpaths, **compiler_flags) + environment, extra_rpaths, implicit_rpaths, + **compiler_flags) def _compiler_from_config_entry(items): @@ -635,8 +637,10 @@ def make_compiler_list(detected_versions): compiler_cls = spack.compilers.class_for_compiler_name(compiler_name) spec = spack.spec.CompilerSpec(compiler_cls.name, version) paths = [paths.get(l, None) for l in ('cc', 'cxx', 'f77', 'fc')] + implicit_rpaths = compiler_cls.determine_implicit_rpaths(paths) compiler = compiler_cls( - spec, operating_system, py_platform.machine(), paths + spec, operating_system, py_platform.machine(), paths, + implicit_rpaths=implicit_rpaths ) return [compiler] diff --git a/lib/spack/spack/compilers/arm.py b/lib/spack/spack/compilers/arm.py index 4892c5a63d..2f9ee002c5 100644 --- a/lib/spack/spack/compilers/arm.py +++ b/lib/spack/spack/compilers/arm.py @@ -37,6 +37,10 @@ class Arm(spack.compiler.Compiler): version_argument = '--version' version_regex = r'Arm C\/C\+\+\/Fortran Compiler version ([^ )]+)' + @classmethod + def verbose_flag(cls): + return "-v" + @property def openmp_flag(self): return "-fopenmp" diff --git a/lib/spack/spack/compilers/cce.py b/lib/spack/spack/compilers/cce.py index 32ccd3f5e2..50b9694188 100644 --- a/lib/spack/spack/compilers/cce.py +++ b/lib/spack/spack/compilers/cce.py @@ -35,6 +35,10 @@ class Cce(Compiler): version_argument = '-V' version_regex = r'[Vv]ersion.*?(\d+(\.\d+)+)' + @classmethod + def verbose_flag(cls): + return "-v" + @property def openmp_flag(self): return "-h omp" diff --git a/lib/spack/spack/compilers/clang.py b/lib/spack/spack/compilers/clang.py index e53332363b..79353b0d1d 100644 --- a/lib/spack/spack/compilers/clang.py +++ b/lib/spack/spack/compilers/clang.py @@ -81,6 +81,10 @@ class Clang(Compiler): ver_string = str(self.version) return ver_string.endswith('-apple') + @classmethod + def verbose_flag(cls): + return "-v" + @property def openmp_flag(self): if self.is_apple: diff --git a/lib/spack/spack/compilers/gcc.py b/lib/spack/spack/compilers/gcc.py index 4d3c319c85..fd1d3446fb 100644 --- a/lib/spack/spack/compilers/gcc.py +++ b/lib/spack/spack/compilers/gcc.py @@ -38,6 +38,10 @@ class Gcc(Compiler): PrgEnv = 'PrgEnv-gnu' PrgEnv_compiler = 'gcc' + @classmethod + def verbose_flag(cls): + return "-v" + @property def openmp_flag(self): return "-fopenmp" diff --git a/lib/spack/spack/compilers/intel.py b/lib/spack/spack/compilers/intel.py index c0fb5ebe51..c3cbea4751 100644 --- a/lib/spack/spack/compilers/intel.py +++ b/lib/spack/spack/compilers/intel.py @@ -32,6 +32,10 @@ class Intel(Compiler): version_argument = '--version' version_regex = r'\((?:IFORT|ICC)\) ([^ ]+)' + @classmethod + def verbose_flag(cls): + return "-v" + @property def openmp_flag(self): if self.version < ver('16.0'): diff --git a/lib/spack/spack/compilers/pgi.py b/lib/spack/spack/compilers/pgi.py index 1b382fe075..0f7585bc40 100644 --- a/lib/spack/spack/compilers/pgi.py +++ b/lib/spack/spack/compilers/pgi.py @@ -32,6 +32,10 @@ class Pgi(Compiler): version_argument = '-V' version_regex = r'pg[^ ]* ([0-9.]+)-[0-9]+ (LLVM )?[^ ]+ target on ' + @classmethod + def verbose_flag(cls): + return "-v" + @property def openmp_flag(self): return "-mp" diff --git a/lib/spack/spack/compilers/xl.py b/lib/spack/spack/compilers/xl.py index f82ae31b59..fcfaa29d12 100644 --- a/lib/spack/spack/compilers/xl.py +++ b/lib/spack/spack/compilers/xl.py @@ -29,6 +29,10 @@ class Xl(Compiler): version_argument = '-qversion' version_regex = r'([0-9]?[0-9]\.[0-9])' + @classmethod + def verbose_flag(cls): + return "-V" + @property def openmp_flag(self): return "-qsmp=omp" diff --git a/lib/spack/spack/schema/compilers.py b/lib/spack/spack/schema/compilers.py index 69659917ff..934c3cad3d 100644 --- a/lib/spack/spack/schema/compilers.py +++ b/lib/spack/spack/schema/compilers.py @@ -61,6 +61,10 @@ properties = { 'modules': {'anyOf': [{'type': 'string'}, {'type': 'null'}, {'type': 'array'}]}, + 'implicit_rpaths': { + 'type': 'array', + 'items': {'type': 'string'} + }, 'environment': { 'type': 'object', 'default': {}, diff --git a/lib/spack/spack/test/data/compiler_verbose_output/cce-8.6.5.txt b/lib/spack/spack/test/data/compiler_verbose_output/cce-8.6.5.txt new file mode 100644 index 0000000000..f1098f4b54 --- /dev/null +++ b/lib/spack/spack/test/data/compiler_verbose_output/cce-8.6.5.txt @@ -0,0 +1,10 @@ +rm foo + +/opt/cray/pe/cce/8.6.5/binutils/x86_64/x86_64-pc-linux-gnu/bin/ld /usr/lib64//crt1.o /usr/lib64//crti.o /opt/gcc/6.1.0/snos/lib/gcc/x86_64-suse-linux/6.1.0//crtbeginT.o /opt/gcc/6.1.0/snos/lib/gcc/x86_64-suse-linux/6.1.0//crtfastmath.o /opt/cray/pe/cce/8.6.5/cce/x86_64/lib/no_mmap.o foo.o -Bstatic -rpath=/opt/cray/pe/cce/8.6.5/cce/x86_64/lib -L /opt/gcc/6.1.0/snos/lib64 -rpath=/opt/cray/pe/gcc-libs -L /usr/lib64 -L /lib64 -L /opt/cray/dmapp/default/lib64 -L /opt/cray/pe/mpt/7.7.0/gni/mpich-cray/8.6/lib -L /opt/cray/dmapp/default/lib64 -L /opt/cray/pe/mpt/7.7.0/gni/mpich-cray/8.6/lib -L /opt/cray/pe/libsci/17.12.1/CRAY/8.6/x86_64/lib -L /opt/cray/rca/2.2.16-6.0.5.0_15.34__g5e09e6d.ari/lib64 -L /opt/cray/pe/pmi/5.0.13/lib64 -L /opt/cray/xpmem/2.2.4-6.0.5.0_4.8__g35d5e73.ari/lib64 -L /opt/cray/dmapp/7.1.1-6.0.5.0_49.8__g1125556.ari/lib64 -L /opt/cray/ugni/6.0.14-6.0.5.0_16.9__g19583bb.ari/lib64 -L /opt/cray/udreg/2.3.2-6.0.5.0_13.12__ga14955a.ari/lib64 -L /opt/cray/alps/6.5.28-6.0.5.0_18.6__g13a91b6.ari/lib64 -L /opt/cray/pe/atp/2.1.1/libApp -L /opt/cray/pe/cce/8.6.5/cce/x86_64/lib/pkgconfig/../ -L /opt/cray/wlm_detect/1.3.2-6.0.5.0_3.1__g388ccd5.ari/lib64 --no-as-needed -lAtpSigHandler -lAtpSigHCommData --undefined=_ATP_Data_Globals --undefined=__atpHandlerInstall -lpthread -lmpichcxx_cray -lrt -lpthread -lugni -lpmi -lsci_cray_mpi_mp -lm -lf -lsci_cray_mp -lmpich_cray -lrt -lpthread -lugni -lpmi -lsci_cray_mp -lcraymp -lm -lpthread -lf -lhugetlbfs -lpgas-dmapp -lfi -lu -lrt --undefined=dmapp_get_flag_nbi -ldmapp -lugni -ludreg -lpthread -lm -lcray-c++-rts -lstdc++ -lxpmem -ldmapp -lpthread -lpmi -lpthread -lalpslli -lpthread -lwlm_detect -lugni -lpthread -lalpsutil -lpthread -lrca -ludreg -lquadmath -lm -lomp -lcraymp -lpthread -lrt -ldl -lcray-c++-rts -lstdc++ -lm -lmodules -lm -lfi -lm -lquadmath -lcraymath -lm -lgfortran -lquadmath -lf -lm -lpthread -lu -lrt -ldl -lcray-c++-rts -lstdc++ -lm -lcsup --as-needed -latomic --no-as-needed -lcray-c++-rts -lstdc++ -lsupc++ -lstdc++ -lpthread --start-group -lc -lcsup -lgcc_eh -lm -lgcc --end-group -T/opt/cray/pe/cce/8.6.5/cce/x86_64/lib/2.23.1.cce.ld -L /opt/gcc/6.1.0/snos/lib/gcc/x86_64-suse-linux/6.1.0 -L /opt/cray/pe/cce/8.6.5/binutils/x86_64/x86_64-pc-linux-gnu/..//x86_64-unknown-linux-gnu/lib -EL -o foo --undefined=__pthread_initialize_minimal /opt/gcc/6.1.0/snos/lib/gcc/x86_64-suse-linux/6.1.0//crtend.o /usr/lib64//crtn.o + +/opt/cray/pe/cce/8.6.5/binutils/x86_64/x86_64-pc-linux-gnu/bin/objcopy --remove-section=.note.ftn_module_data foo +rm /tmp/pe_27645//pldir/PL_path +rm /tmp/pe_27645//pldir/PL_module_list +rm /tmp/pe_27645//pldir/PL_global_data +rmdir /tmp/pe_27645//pldir +rmdir /tmp/pe_27645/ diff --git a/lib/spack/spack/test/data/compiler_verbose_output/clang-4.0.1.txt b/lib/spack/spack/test/data/compiler_verbose_output/clang-4.0.1.txt new file mode 100644 index 0000000000..c55d3a68e7 --- /dev/null +++ b/lib/spack/spack/test/data/compiler_verbose_output/clang-4.0.1.txt @@ -0,0 +1,20 @@ +clang version 4.0.1 (tags/RELEASE_401/final) +Target: x86_64-unknown-linux-gnu +Thread model: posix +InstalledDir: /usr/bin +Found candidate GCC installation: /usr/bin/../lib/gcc/x86_64-redhat-linux/7 +Found candidate GCC installation: /usr/lib/gcc/x86_64-redhat-linux/7 +Selected GCC installation: /usr/bin/../lib/gcc/x86_64-redhat-linux/7 +Candidate multilib: .;@m64 +Candidate multilib: 32;@m32 +Selected multilib: .;@m64 + "/usr/bin/clang-4.0" -cc1 -triple x86_64-unknown-linux-gnu -emit-obj -mrelax-all -disable-free -disable-llvm-verifier -discard-value-names -main-file-name main.c -mrelocation-model static -mthread-model posix -mdisable-fp-elim -fmath-errno -masm-verbose -mconstructor-aliases -munwind-tables -fuse-init-array -target-cpu x86-64 -v -dwarf-column-info -debugger-tuning=gdb -resource-dir /usr/bin/../lib64/clang/4.0.1 -internal-isystem /usr/local/include -internal-isystem /usr/bin/../lib64/clang/4.0.1/include -internal-externc-isystem /include -internal-externc-isystem /usr/include -fdebug-compilation-dir /tmp/spack-test -ferror-limit 19 -fmessage-length 0 -fobjc-runtime=gcc -fdiagnostics-show-option -o /tmp/main-bf64f0.o -x c main.c +clang -cc1 version 4.0.1 based upon LLVM 4.0.1 default target x86_64-unknown-linux-gnu +ignoring nonexistent directory "/include" +#include "..." search starts here: +#include <...> search starts here: + /usr/local/include + /usr/bin/../lib64/clang/4.0.1/include + /usr/include +End of search list. + "/usr/bin/ld" --hash-style=gnu --no-add-needed --build-id --eh-frame-hdr -m elf_x86_64 -dynamic-linker /lib64/ld-linux-x86-64.so.2 -o output /usr/bin/../lib/gcc/x86_64-redhat-linux/7/../../../../lib64/crt1.o /usr/bin/../lib/gcc/x86_64-redhat-linux/7/../../../../lib64/crti.o /usr/bin/../lib/gcc/x86_64-redhat-linux/7/crtbegin.o -L/usr/bin/../lib/gcc/x86_64-redhat-linux/7 -L/usr/bin/../lib/gcc/x86_64-redhat-linux/7/../../../../lib64 -L/usr/bin/../lib64 -L/lib/../lib64 -L/usr/lib/../lib64 -L/usr/bin/../lib/gcc/x86_64-redhat-linux/7/../../.. -L/usr/bin/../lib -L/lib -L/usr/lib /tmp/main-bf64f0.o -lgcc --as-needed -lgcc_s --no-as-needed -lc -lgcc --as-needed -lgcc_s --no-as-needed /usr/bin/../lib/gcc/x86_64-redhat-linux/7/crtend.o /usr/bin/../lib/gcc/x86_64-redhat-linux/7/../../../../lib64/crtn.o diff --git a/lib/spack/spack/test/data/compiler_verbose_output/clang-9.0.0-apple-ld.txt b/lib/spack/spack/test/data/compiler_verbose_output/clang-9.0.0-apple-ld.txt new file mode 100644 index 0000000000..7592329045 --- /dev/null +++ b/lib/spack/spack/test/data/compiler_verbose_output/clang-9.0.0-apple-ld.txt @@ -0,0 +1,7 @@ +@(#)PROGRAM:ld PROJECT:ld64-305 +configured to support archs: armv6 armv7 armv7s arm64 i386 x86_64 x86_64h armv6m armv7k armv7m armv7em (tvOS) +Library search paths: + /usr/local/lib + /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX10.13.sdk/usr/lib +Framework search paths: + /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX10.13.sdk/System/Library/Frameworks/ diff --git a/lib/spack/spack/test/data/compiler_verbose_output/gcc-7.3.1.txt b/lib/spack/spack/test/data/compiler_verbose_output/gcc-7.3.1.txt new file mode 100644 index 0000000000..76a4537a59 --- /dev/null +++ b/lib/spack/spack/test/data/compiler_verbose_output/gcc-7.3.1.txt @@ -0,0 +1,36 @@ +Using built-in specs. +COLLECT_GCC=/usr/bin/gcc +COLLECT_LTO_WRAPPER=/usr/libexec/gcc/x86_64-redhat-linux/7/lto-wrapper +OFFLOAD_TARGET_NAMES=nvptx-none +OFFLOAD_TARGET_DEFAULT=1 +Target: x86_64-redhat-linux +Configured with: ../configure --enable-bootstrap --enable-languages=c,c++,objc,obj-c++,fortran,ada,go,lto --prefix=/usr --mandir=/usr/share/man --infodir=/usr/share/info --with-bugurl=http://bugzilla.redhat.com/bugzilla --enable-shared --enable-threads=posix --enable-checking=release --enable-multilib --with-system-zlib --enable-__cxa_atexit --disable-libunwind-exceptions --enable-gnu-unique-object --enable-linker-build-id --with-gcc-major-version-only --with-linker-hash-style=gnu --enable-plugin --enable-initfini-array --with-isl --enable-libmpx --enable-offload-targets=nvptx-none --without-cuda-driver --enable-gnu-indirect-function --with-tune=generic --with-arch_32=i686 --build=x86_64-redhat-linux +Thread model: posix +gcc version 7.3.1 20180130 (Red Hat 7.3.1-2) (GCC) +COLLECT_GCC_OPTIONS='-v' '-o' 'output' '-mtune=generic' '-march=x86-64' + /usr/libexec/gcc/x86_64-redhat-linux/7/cc1 -quiet -v main.c -quiet -dumpbase main.c -mtune=generic -march=x86-64 -auxbase main -version -o /tmp/ccM76aqK.s +GNU C11 (GCC) version 7.3.1 20180130 (Red Hat 7.3.1-2) (x86_64-redhat-linux) + compiled by GNU C version 7.3.1 20180130 (Red Hat 7.3.1-2), GMP version 6.1.2, MPFR version 3.1.5, MPC version 1.0.2, isl version isl-0.16.1-GMP + +GGC heuristics: --param ggc-min-expand=100 --param ggc-min-heapsize=131072 +ignoring nonexistent directory "/usr/lib/gcc/x86_64-redhat-linux/7/include-fixed" +ignoring nonexistent directory "/usr/lib/gcc/x86_64-redhat-linux/7/../../../../x86_64-redhat-linux/include" +#include "..." search starts here: +#include <...> search starts here: + /usr/lib/gcc/x86_64-redhat-linux/7/include + /usr/local/include + /usr/include +End of search list. +GNU C11 (GCC) version 7.3.1 20180130 (Red Hat 7.3.1-2) (x86_64-redhat-linux) + compiled by GNU C version 7.3.1 20180130 (Red Hat 7.3.1-2), GMP version 6.1.2, MPFR version 3.1.5, MPC version 1.0.2, isl version isl-0.16.1-GMP + +GGC heuristics: --param ggc-min-expand=100 --param ggc-min-heapsize=131072 +Compiler executable checksum: ad7c3a488cf591743af375264d348c5c +COLLECT_GCC_OPTIONS='-v' '-o' 'output' '-mtune=generic' '-march=x86-64' + as -v --64 -o /tmp/ccYFphwj.o /tmp/ccM76aqK.s +GNU assembler version 2.27 (x86_64-redhat-linux) using BFD version version 2.27-28.fc26 +COMPILER_PATH=/usr/libexec/gcc/x86_64-redhat-linux/7/:/usr/libexec/gcc/x86_64-redhat-linux/7/:/usr/libexec/gcc/x86_64-redhat-linux/:/usr/lib/gcc/x86_64-redhat-linux/7/:/usr/lib/gcc/x86_64-redhat-linux/ +LIBRARY_PATH=/usr/lib/gcc/x86_64-redhat-linux/7/:/usr/lib/gcc/x86_64-redhat-linux/7/../../../../lib64/:/lib/../lib64/:/usr/lib/../lib64/:/usr/lib/gcc/x86_64-redhat-linux/7/../../../:/lib/:/usr/lib/ +COLLECT_GCC_OPTIONS='-v' '-o' 'output' '-mtune=generic' '-march=x86-64' + /usr/libexec/gcc/x86_64-redhat-linux/7/collect2 -plugin /usr/libexec/gcc/x86_64-redhat-linux/7/liblto_plugin.so -plugin-opt=/usr/libexec/gcc/x86_64-redhat-linux/7/lto-wrapper -plugin-opt=-fresolution=/tmp/ccw0b6CS.res -plugin-opt=-pass-through=-lgcc -plugin-opt=-pass-through=-lgcc_s -plugin-opt=-pass-through=-lc -plugin-opt=-pass-through=-lgcc -plugin-opt=-pass-through=-lgcc_s --build-id --no-add-needed --eh-frame-hdr --hash-style=gnu -m elf_x86_64 -dynamic-linker /lib64/ld-linux-x86-64.so.2 -o output /usr/lib/gcc/x86_64-redhat-linux/7/../../../../lib64/crt1.o /usr/lib/gcc/x86_64-redhat-linux/7/../../../../lib64/crti.o /usr/lib/gcc/x86_64-redhat-linux/7/crtbegin.o -L/usr/lib/gcc/x86_64-redhat-linux/7 -L/usr/lib/gcc/x86_64-redhat-linux/7/../../../../lib64 -L/lib/../lib64 -L/usr/lib/../lib64 -L/usr/lib/gcc/x86_64-redhat-linux/7/../../.. /tmp/ccYFphwj.o -lgcc --as-needed -lgcc_s --no-as-needed -lc -lgcc --as-needed -lgcc_s --no-as-needed /usr/lib/gcc/x86_64-redhat-linux/7/crtend.o /usr/lib/gcc/x86_64-redhat-linux/7/../../../../lib64/crtn.o +COLLECT_GCC_OPTIONS='-v' '-o' 'output' '-mtune=generic' '-march=x86-64' diff --git a/lib/spack/spack/test/data/compiler_verbose_output/icc-16.0.3.txt b/lib/spack/spack/test/data/compiler_verbose_output/icc-16.0.3.txt new file mode 100644 index 0000000000..cdbe4455b9 --- /dev/null +++ b/lib/spack/spack/test/data/compiler_verbose_output/icc-16.0.3.txt @@ -0,0 +1,4 @@ +icc.orig version 16.0.3 (gcc version 4.9.3 compatibility) +ld /lib/../lib64/crt1.o /lib/../lib64/crti.o /usr/tce/packages/gcc/gcc-4.9.3/lib64/gcc/x86_64-unknown-linux-gnu/4.9.3/crtbegin.o --eh-frame-hdr --build-id -dynamic-linker /lib64/ld-linux-x86-64.so.2 -m elf_x86_64 -o blah -L/usr/tce/packages/intel/intel-16.0.3/compilers_and_libraries_2016.3.210/linux/compiler/lib/intel64_lin -L/usr/tce/packages/gcc/gcc-4.9.3/lib64/gcc/x86_64-unknown-linux-gnu/4.9.3/ -L/usr/tce/packages/gcc/gcc-4.9.3/lib64/gcc/x86_64-unknown-linux-gnu/4.9.3/../../../../lib64 -L/usr/tce/packages/gcc/gcc-4.9.3/lib64/gcc/x86_64-unknown-linux-gnu/4.9.3/../../../../lib64/ -L/lib/../lib64 -L/lib/../lib64/ -L/usr/lib/../lib64 -L/usr/lib/../lib64/ -L/usr/tce/packages/gcc/gcc-4.9.3/lib64/gcc/x86_64-unknown-linux-gnu/4.9.3/../../../ -L/lib64 -L/lib/ -L/usr/lib64 -L/usr/lib -rpath /usr/tce/packages/intel/intel-16.0.3/lib/intel64 -rpath=/usr/tce/packages/gcc/default/lib64 -Bdynamic -Bstatic -limf -lsvml -lirng -Bdynamic -lm -Bstatic -lipgo -ldecimal --as-needed -Bdynamic -lcilkrts -lstdc++ --no-as-needed -lgcc -lgcc_s -Bstatic -lirc -lsvml -Bdynamic -lc -lgcc -lgcc_s -Bstatic -lirc_s -Bdynamic -ldl -lc /usr/tce/packages/gcc/gcc-4.9.3/lib64/gcc/x86_64-unknown-linux-gnu/4.9.3/crtend.o /lib/../lib64/crtn.o +/lib/../lib64/crt1.o: In function `_start': +(.text+0x20): undefined reference to `main' diff --git a/lib/spack/spack/test/data/compiler_verbose_output/obscure-parsing-rules.txt b/lib/spack/spack/test/data/compiler_verbose_output/obscure-parsing-rules.txt new file mode 100644 index 0000000000..7cf090a846 --- /dev/null +++ b/lib/spack/spack/test/data/compiler_verbose_output/obscure-parsing-rules.txt @@ -0,0 +1,3 @@ +This is synthetic data to test parsing cases for which I could not find compiler output +ld -LIBPATH:/first/path /LIBPATH:/second/path -libpath:/third/path +collect2 version ld -LIBPATH:/skip/path -L/skip/this/too diff --git a/lib/spack/spack/test/data/compiler_verbose_output/pgcc-16.3.txt b/lib/spack/spack/test/data/compiler_verbose_output/pgcc-16.3.txt new file mode 100644 index 0000000000..9041322350 --- /dev/null +++ b/lib/spack/spack/test/data/compiler_verbose_output/pgcc-16.3.txt @@ -0,0 +1,11 @@ +Export PGI=/usr/tce/packages/pgi/pgi-16.3 + +/usr/tce/packages/pgi/pgi-16.3/linux86-64/16.3/bin/pgc test.c -opt 1 -x 119 0xa10000 -x 122 0x40 -x 123 0x1000 -x 127 4 -x 127 17 -x 19 0x400000 -x 28 0x40000 -x 120 0x10000000 -x 70 0x8000 -x 122 1 -x 125 0x20000 -quad -x 59 4 -tp haswell -x 120 0x1000 -astype 0 -stdinc /usr/tce/packages/pgi/pgi-16.3/linux86-64/16.3/include-gcc48:/usr/tce/packages/pgi/pgi-16.3/linux86-64/16.3/include:/usr/lib/gcc/x86_64-redhat-linux/4.8.5/include:/usr/local/include:/usr/include -def unix -def __unix -def __unix__ -def linux -def __linux -def __linux__ -def __NO_MATH_INLINES -def __LP64__ -def __x86_64 -def __x86_64__ -def __LONG_MAX__=9223372036854775807L -def '__SIZE_TYPE__=unsigned long int' -def '__PTRDIFF_TYPE__=long int' -def __THROW= -def __extension__= -def __amd_64__amd64__ -def __k8 -def __k8__ -def __SSE__ -def __MMX__ -def __SSE2__ -def __SSE3__ -def __SSSE3__ -def __STDC_HOSTED__ -predicate '#machine(x86_64) #lint(off) #system(posix) #cpu(x86_64)' -cmdline '+pgcc test.c -v -o test.o' -x 123 0x80000000 -x 123 4 -x 2 0x400 -x 119 0x20 -def __pgnu_vsn=40805 -x 120 0x200000 -x 70 0x40000000 -y 163 0xc0000000 -x 189 0x10 -y 189 0x4000000 -asm /var/tmp/gamblin2/pgccL0MCVCOQsq6l.s +PGC/x86-64 Linux 16.3-0: compilation successful + +/usr/bin/as /var/tmp/gamblin2/pgccL0MCVCOQsq6l.s -o /var/tmp/gamblin2/pgcc10MCFxmYXjgo.o + +/usr/tce/bin/ld /usr/lib64/crt1.o /usr/lib64/crti.o /usr/tce/packages/pgi/pgi-16.3/linux86-64/16.3/lib/trace_init.o /usr/lib/gcc/x86_64-redhat-linux/4.8.5/crtbegin.o /usr/tce/packages/pgi/pgi-16.3/linux86-64/16.3/lib/initmp.o --eh-frame-hdr -m elf_x86_64 -dynamic-linker /lib64/ld-linux-x86-64.so.2 /usr/tce/packages/pgi/pgi-16.3/linux86-64/16.3/lib/pgi.ld -L/usr/tce/packages/pgi/pgi-16.3/linux86-64/16.3/lib -L/usr/lib64 -L/usr/lib/gcc/x86_64-redhat-linux/4.8.5 /var/tmp/gamblin2/pgcc10MCFxmYXjgo.o -rpath /usr/tce/packages/pgi/pgi-16.3/linux86-64/16.3/lib -o test.o -lpgmp -lnuma -lpthread -lnspgc -lpgc -lm -lgcc -lc -lgcc /usr/lib/gcc/x86_64-redhat-linux/4.8.5/crtend.o /usr/lib64/crtn.o +Unlinking /var/tmp/gamblin2/pgccL0MCVCOQsq6l.s +Unlinking /var/tmp/gamblin2/pgccn0MCNcmgIbh8.ll +Unlinking /var/tmp/gamblin2/pgcc10MCFxmYXjgo.o diff --git a/lib/spack/spack/test/data/compiler_verbose_output/xl-13.1.5.txt b/lib/spack/spack/test/data/compiler_verbose_output/xl-13.1.5.txt new file mode 100644 index 0000000000..2ff243a4d9 --- /dev/null +++ b/lib/spack/spack/test/data/compiler_verbose_output/xl-13.1.5.txt @@ -0,0 +1,5 @@ +export XL_CONFIG=/opt/ibm/xlC/13.1.5/etc/xlc.cfg.centos.7.gcc.4.8.5:xlc +/usr/bin/ld --eh-frame-hdr -Qy -melf64lppc /usr/lib/gcc/ppc64le-redhat-linux/4.8.5/../../../../lib64/crt1.o /usr/lib/gcc/ppc64le-redhat-linux/4.8.5/../../../../lib64/crti.o /usr/lib/gcc/ppc64le-redhat-linux/4.8.5/crtbegin.o -L/opt/ibm/xlsmp/4.1.5/lib -L/opt/ibm/xlmass/8.1.5/lib -L/opt/ibm/xlC/13.1.5/lib -R/opt/ibm/lib -L/usr/lib/gcc/ppc64le-redhat-linux/4.8.5 -L/usr/lib/gcc/ppc64le-redhat-linux/4.8.5/../../../../lib64 -L/lib/../lib64 -L/usr/lib/../lib64 -L/usr/lib/gcc/ppc64le-redhat-linux/4.8.5/../../.. --no-toc-optimize -o foo foo.o -dynamic-linker /lib64/ld64.so.2 --enable-new-dtags -lxlopt -lxl --as-needed -ldl --no-as-needed -lgcc_s --as-needed -lpthread --no-as-needed -lgcc -lm -lc -lgcc_s -lgcc /usr/lib/gcc/ppc64le-redhat-linux/4.8.5/crtend.o /usr/lib/gcc/ppc64le-redhat-linux/4.8.5/../../../../lib64/crtn.o +rm /tmp/xlcW0iQ4uI8 +rm /tmp/xlcW1aPLBFY +rm /tmp/xlcW2ALFICO diff --git a/lib/spack/spack/test/link_paths.py b/lib/spack/spack/test/link_paths.py new file mode 100644 index 0000000000..33ac68cdc0 --- /dev/null +++ b/lib/spack/spack/test/link_paths.py @@ -0,0 +1,89 @@ +# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import os + +import spack.paths +from spack.compiler import _parse_implicit_rpaths + +#: directory with sample compiler data +datadir = os.path.join(spack.paths.test_path, 'data', + 'compiler_verbose_output') + + +def check_link_paths(filename, paths): + with open(os.path.join(datadir, filename)) as file: + output = file.read() + detected_paths = _parse_implicit_rpaths(output) + + actual = detected_paths + expected = paths + + missing_paths = list(x for x in expected if x not in actual) + assert not missing_paths + + extra_paths = list(x for x in actual if x not in expected) + assert not extra_paths + + assert actual == expected + + +def test_icc16_link_paths(): + check_link_paths('icc-16.0.3.txt', [ + '/usr/tce/packages/intel/intel-16.0.3/compilers_and_libraries_2016.3.210/linux/compiler/lib/intel64_lin', # noqa + '/usr/tce/packages/gcc/gcc-4.9.3/lib64/gcc/x86_64-unknown-linux-gnu/4.9.3', # noqa + '/usr/tce/packages/gcc/gcc-4.9.3/lib64']) + + +def test_pgi_link_paths(): + check_link_paths('pgcc-16.3.txt', [ + '/usr/tce/packages/pgi/pgi-16.3/linux86-64/16.3/lib']) + + +def test_gcc7_link_paths(): + check_link_paths('gcc-7.3.1.txt', []) + + +def test_clang4_link_paths(): + check_link_paths('clang-4.0.1.txt', []) + + +def test_xl_link_paths(): + check_link_paths('xl-13.1.5.txt', [ + '/opt/ibm/xlsmp/4.1.5/lib', + '/opt/ibm/xlmass/8.1.5/lib', + '/opt/ibm/xlC/13.1.5/lib']) + + +def test_cce_link_paths(): + check_link_paths('cce-8.6.5.txt', [ + '/opt/gcc/6.1.0/snos/lib64', + '/opt/cray/dmapp/default/lib64', + '/opt/cray/pe/mpt/7.7.0/gni/mpich-cray/8.6/lib', + '/opt/cray/pe/libsci/17.12.1/CRAY/8.6/x86_64/lib', + '/opt/cray/rca/2.2.16-6.0.5.0_15.34__g5e09e6d.ari/lib64', + '/opt/cray/pe/pmi/5.0.13/lib64', + '/opt/cray/xpmem/2.2.4-6.0.5.0_4.8__g35d5e73.ari/lib64', + '/opt/cray/dmapp/7.1.1-6.0.5.0_49.8__g1125556.ari/lib64', + '/opt/cray/ugni/6.0.14-6.0.5.0_16.9__g19583bb.ari/lib64', + '/opt/cray/udreg/2.3.2-6.0.5.0_13.12__ga14955a.ari/lib64', + '/opt/cray/alps/6.5.28-6.0.5.0_18.6__g13a91b6.ari/lib64', + '/opt/cray/pe/atp/2.1.1/libApp', + '/opt/cray/pe/cce/8.6.5/cce/x86_64/lib', + '/opt/cray/wlm_detect/1.3.2-6.0.5.0_3.1__g388ccd5.ari/lib64', + '/opt/gcc/6.1.0/snos/lib/gcc/x86_64-suse-linux/6.1.0', + '/opt/cray/pe/cce/8.6.5/binutils/x86_64/x86_64-unknown-linux-gnu/lib']) + + +def test_clang_apple_ld_link_paths(): + check_link_paths('clang-9.0.0-apple-ld.txt', [ + '/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX10.13.sdk/usr/lib']) # noqa + + +def test_obscure_parsing_rules(): + check_link_paths('obscure-parsing-rules.txt', [ + '/first/path', + '/second/path', + '/third/path']) -- cgit v1.2.3-60-g2f50