Add implicit rpaths to compiler detection (#7153)

Uses code from CMake to detect implicit link paths from compilers
System paths are filtered out of implicit link paths
Implicit link paths added to compiler config and object under `implicit_rpaths`
Implicit link paths added as rpaths to compile line through env/cc wrapper

Authored by: "Ben Boeckel <ben.boeckel@kitware.com>"
Co-authored by: "Peter Scheibel <scheibel1@llnl.gov>"
Co-authored by: "Gregory Becker <becker33@llnl.gov>"

21 files changed, 384 insertions, 8 deletions

diff --git a/lib/spack/env/cc b/lib/spack/env/cc
index f5a71d87d1..c7ea8c793b 100755
--- a/lib/spack/env/cc
+++ b/lib/spack/env/cc
@@ -426,6 +426,12 @@ case "$mode" in
             rpaths+=("${extra_rpaths[@]}")
         fi
 
+        # Set implicit RPATHs
+        IFS=':' read -ra implicit_rpaths <<< "$SPACK_COMPILER_IMPLICIT_RPATHS"
+        if [[ "$add_rpaths" != "false" ]] ; then
+            rpaths+=("${implicit_rpaths[@]}")
+        fi
+
         # Add SPACK_LDLIBS to args
         for lib in "${SPACK_LDLIBS[@]}"; do
             libs+=("${lib#-l}")
diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py
index ff7d41a39a..d37e139e3c 100644
--- a/lib/spack/spack/build_environment.py
+++ b/lib/spack/spack/build_environment.py
@@ -348,6 +348,10 @@ def set_build_environment_variables(pkg, env, dirty):
         extra_rpaths = ':'.join(compiler.extra_rpaths)
         env.set('SPACK_COMPILER_EXTRA_RPATHS', extra_rpaths)
 
+    if compiler.implicit_rpaths:
+        implicit_rpaths = ':'.join(compiler.implicit_rpaths)
+        env.set('SPACK_COMPILER_IMPLICIT_RPATHS', implicit_rpaths)
+
     # Add bin directories from dependencies to the PATH for the build.
     for prefix in build_prefixes:
         for dirname in ['bin', 'bin64']:
diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py
index 1136485ba9..6fd3dd0457 100644
--- a/lib/spack/spack/compiler.py
+++ b/lib/spack/spack/compiler.py
@@ -6,13 +6,18 @@
 import os
 import re
 import itertools
+import shutil
+import tempfile
 
 import llnl.util.filesystem
+import llnl.util.tty as tty
 
 import spack.error
 import spack.spec
 import spack.architecture
 import spack.util.executable
+import spack.compilers
+from spack.util.environment import filter_system_paths
 
 __all__ = ['Compiler']
 
@@ -58,6 +63,97 @@ def tokenize_flags(flags_str):
     return flags
 
 
+#: regex for parsing linker lines
+_LINKER_LINE = re.compile(
+    r'^( *|.*[/\\])'
+    r'(link|ld|([^/\\]+-)?ld|collect2)'
+    r'[^/\\]*( |$)')
+
+#: components of linker lines to ignore
+_LINKER_LINE_IGNORE = re.compile(r'(collect2 version|^[A-Za-z0-9_]+=|/ldfe )')
+
+#: regex to match linker search paths
+_LINK_DIR_ARG = re.compile(r'^-L(.:)?(?P<dir>[/\\].*)')
+
+#: regex to match linker library path arguments
+_LIBPATH_ARG = re.compile(r'^[-/](LIBPATH|libpath):(?P<dir>.*)')
+
+
+def is_subdirectory(path, prefix):
+    path = os.path.abspath(path)
+    prefix = os.path.abspath(prefix) + os.path.sep
+    return path.startswith(prefix)
+
+
+def _parse_implicit_rpaths(string):
+    """Parse implicit link paths from compiler debug output.
+
+    This gives the compiler runtime library paths that we need to add to
+    the RPATH of generated binaries and libraries.  It allows us to
+    ensure, e.g., that codes load the right libstdc++ for their compiler.
+    """
+    lib_search_paths = False
+    raw_link_dirs = []
+    tty.debug('parsing implicit link info')
+    for line in string.splitlines():
+        if lib_search_paths:
+            if line.startswith('\t'):
+                raw_link_dirs.append(line[1:])
+                continue
+            else:
+                lib_search_paths = False
+        elif line.startswith('Library search paths:'):
+            lib_search_paths = True
+
+        if not _LINKER_LINE.match(line):
+            continue
+        if _LINKER_LINE_IGNORE.match(line):
+            continue
+        tty.debug('linker line: %s' % line)
+
+        next_arg = False
+        for arg in line.split():
+            if arg in ('-L', '-Y'):
+                next_arg = True
+                continue
+
+            if next_arg:
+                raw_link_dirs.append(arg)
+                next_arg = False
+                continue
+
+            link_dir_arg = _LINK_DIR_ARG.match(arg)
+            if link_dir_arg:
+                link_dir = link_dir_arg.group('dir')
+                tty.debug('linkdir: %s' % link_dir)
+                raw_link_dirs.append(link_dir)
+
+            link_dir_arg = _LIBPATH_ARG.match(arg)
+            if link_dir_arg:
+                link_dir = link_dir_arg.group('dir')
+                tty.debug('libpath: %s', link_dir)
+                raw_link_dirs.append(link_dir)
+    tty.debug('found raw link dirs: %s' % ', '.join(raw_link_dirs))
+
+    implicit_link_dirs = list()
+    visited = set()
+    for link_dir in raw_link_dirs:
+        normalized_path = os.path.abspath(link_dir)
+        if normalized_path not in visited:
+            implicit_link_dirs.append(normalized_path)
+            visited.add(normalized_path)
+    implicit_link_dirs = filter_system_paths(implicit_link_dirs)
+
+    # Additional filtering: we also want to exclude paths that are
+    # subdirectories of /usr/lib/ and /lib/
+    implicit_link_dirs = list(
+        path for path in implicit_link_dirs
+        if not any(is_subdirectory(path, d) for d in ['/lib/', '/usr/lib/']))
+
+    tty.debug('found link dirs: %s' % ', '.join(implicit_link_dirs))
+    return implicit_link_dirs
+
+
 class Compiler(object):
     """This class encapsulates a Spack "compiler", which includes C,
        C++, and Fortran compilers.  Subclasses should implement
@@ -114,12 +210,15 @@ class Compiler(object):
 
     def __init__(self, cspec, operating_system, target,
                  paths, modules=[], alias=None, environment=None,
-                 extra_rpaths=None, **kwargs):
+                 extra_rpaths=None, implicit_rpaths=None,
+                 **kwargs):
         self.spec = cspec
         self.operating_system = str(operating_system)
         self.target = target
         self.modules = modules
         self.alias = alias
+        self.extra_rpaths = extra_rpaths
+        self.implicit_rpaths = implicit_rpaths
 
         def check(exe):
             if exe is None:
@@ -152,6 +251,52 @@ class Compiler(object):
     def version(self):
         return self.spec.version
 
+    @classmethod
+    def verbose_flag(cls):
+        """
+        This property should be overridden in the compiler subclass if a
+        verbose flag is available.
+
+        If it is not overridden, it is assumed to not be supported.
+        """
+
+    @classmethod
+    def parse_implicit_rpaths(cls, string):
+        """Parses link paths out of compiler debug output.
+
+        Args:
+            string (str): compiler debug output as a string
+
+        Returns:
+            (list of str): implicit link paths parsed from the compiler output
+
+        Subclasses can override this to customize.
+        """
+        return _parse_implicit_rpaths(string)
+
+    @classmethod
+    def determine_implicit_rpaths(cls, paths):
+        first_compiler = next((c for c in paths if c), None)
+        if not first_compiler:
+            return []
+
+        try:
+            tmpdir = tempfile.mkdtemp(prefix='spack-implicit-link-info')
+            fout = os.path.join(tmpdir, 'output')
+            fin = os.path.join(tmpdir, 'main.c')
+
+            with open(fin, 'w+') as csource:
+                csource.write(
+                    'int main(int argc, char* argv[]) { '
+                    '(void)argc; (void)argv; return 0; }\n')
+            compiler_exe = spack.util.executable.Executable(first_compiler)
+            output = str(compiler_exe(cls.verbose_flag(), fin, '-o', fout,
+                                      output=str, error=str))  # str for py2
+
+            return cls.parse_implicit_rpaths(output)
+        finally:
+            shutil.rmtree(tmpdir, ignore_errors=True)
+
     # This property should be overridden in the compiler subclass if
     # OpenMP is supported by that compiler
     @property
diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py
index 20f40eef09..b4c548365e 100644
--- a/lib/spack/spack/compilers/__init__.py
+++ b/lib/spack/spack/compilers/__init__.py
@@ -10,7 +10,6 @@ import collections
 import itertools
 import multiprocessing.pool
 import os
-
 import platform as py_platform
 import six
 
@@ -31,7 +30,7 @@ _imported_compilers_module = 'spack.compilers'
 _path_instance_vars = ['cc', 'cxx', 'f77', 'fc']
 _flags_instance_vars = ['cflags', 'cppflags', 'cxxflags', 'fflags']
 _other_instance_vars = ['modules', 'operating_system', 'environment',
-                        'extra_rpaths']
+                        'extra_rpaths', 'implicit_rpaths']
 _cache_config_file = []
 
 # TODO: Caches at module level make it difficult to mock configurations in
@@ -71,9 +70,10 @@ def _to_dict(compiler):
                            if hasattr(compiler, attr)))
     d['operating_system'] = str(compiler.operating_system)
     d['target'] = str(compiler.target)
-    d['modules'] = compiler.modules if compiler.modules else []
-    d['environment'] = compiler.environment if compiler.environment else {}
-    d['extra_rpaths'] = compiler.extra_rpaths if compiler.extra_rpaths else []
+    d['modules'] = compiler.modules or []
+    d['environment'] = compiler.environment or {}
+    d['extra_rpaths'] = compiler.extra_rpaths or []
+    d['implicit_rpaths'] = compiler.implicit_rpaths or []
 
     if compiler.alias:
         d['alias'] = compiler.alias
@@ -350,9 +350,11 @@ def compiler_from_dict(items):
     compiler_flags = items.get('flags', {})
     environment = items.get('environment', {})
     extra_rpaths = items.get('extra_rpaths', [])
+    implicit_rpaths = items.get('implicit_rpaths')
 
     return cls(cspec, os, target, compiler_paths, mods, alias,
-               environment, extra_rpaths, **compiler_flags)
+               environment, extra_rpaths, implicit_rpaths,
+               **compiler_flags)
 
 
 def _compiler_from_config_entry(items):
@@ -635,8 +637,10 @@ def make_compiler_list(detected_versions):
         compiler_cls = spack.compilers.class_for_compiler_name(compiler_name)
         spec = spack.spec.CompilerSpec(compiler_cls.name, version)
         paths = [paths.get(l, None) for l in ('cc', 'cxx', 'f77', 'fc')]
+        implicit_rpaths = compiler_cls.determine_implicit_rpaths(paths)
         compiler = compiler_cls(
-            spec, operating_system, py_platform.machine(), paths
+            spec, operating_system, py_platform.machine(), paths,
+            implicit_rpaths=implicit_rpaths
         )
         return [compiler]
 
diff --git a/lib/spack/spack/compilers/arm.py b/lib/spack/spack/compilers/arm.py
index 4892c5a63d..2f9ee002c5 100644
--- a/lib/spack/spack/compilers/arm.py
+++ b/lib/spack/spack/compilers/arm.py
@@ -37,6 +37,10 @@ class Arm(spack.compiler.Compiler):
     version_argument = '--version'
     version_regex = r'Arm C\/C\+\+\/Fortran Compiler version ([^ )]+)'
 
+    @classmethod
+    def verbose_flag(cls):
+        return "-v"
+
     @property
     def openmp_flag(self):
         return "-fopenmp"
diff --git a/lib/spack/spack/compilers/cce.py b/lib/spack/spack/compilers/cce.py
index 32ccd3f5e2..50b9694188 100644
--- a/lib/spack/spack/compilers/cce.py
+++ b/lib/spack/spack/compilers/cce.py
@@ -35,6 +35,10 @@ class Cce(Compiler):
     version_argument = '-V'
     version_regex = r'[Vv]ersion.*?(\d+(\.\d+)+)'
 
+    @classmethod
+    def verbose_flag(cls):
+        return "-v"
+
     @property
     def openmp_flag(self):
         return "-h omp"
diff --git a/lib/spack/spack/compilers/clang.py b/lib/spack/spack/compilers/clang.py
index e53332363b..79353b0d1d 100644
--- a/lib/spack/spack/compilers/clang.py
+++ b/lib/spack/spack/compilers/clang.py
@@ -81,6 +81,10 @@ class Clang(Compiler):
         ver_string = str(self.version)
         return ver_string.endswith('-apple')
 
+    @classmethod
+    def verbose_flag(cls):
+        return "-v"
+
     @property
     def openmp_flag(self):
         if self.is_apple:
diff --git a/lib/spack/spack/compilers/gcc.py b/lib/spack/spack/compilers/gcc.py
index 4d3c319c85..fd1d3446fb 100644
--- a/lib/spack/spack/compilers/gcc.py
+++ b/lib/spack/spack/compilers/gcc.py
@@ -38,6 +38,10 @@ class Gcc(Compiler):
     PrgEnv = 'PrgEnv-gnu'
     PrgEnv_compiler = 'gcc'
 
+    @classmethod
+    def verbose_flag(cls):
+        return "-v"
+
     @property
     def openmp_flag(self):
         return "-fopenmp"
diff --git a/lib/spack/spack/compilers/intel.py b/lib/spack/spack/compilers/intel.py
index c0fb5ebe51..c3cbea4751 100644
--- a/lib/spack/spack/compilers/intel.py
+++ b/lib/spack/spack/compilers/intel.py
@@ -32,6 +32,10 @@ class Intel(Compiler):
     version_argument = '--version'
     version_regex = r'\((?:IFORT|ICC)\) ([^ ]+)'
 
+    @classmethod
+    def verbose_flag(cls):
+        return "-v"
+
     @property
     def openmp_flag(self):
         if self.version < ver('16.0'):
diff --git a/lib/spack/spack/compilers/pgi.py b/lib/spack/spack/compilers/pgi.py
index 1b382fe075..0f7585bc40 100644
--- a/lib/spack/spack/compilers/pgi.py
+++ b/lib/spack/spack/compilers/pgi.py
@@ -32,6 +32,10 @@ class Pgi(Compiler):
     version_argument = '-V'
     version_regex = r'pg[^ ]* ([0-9.]+)-[0-9]+ (LLVM )?[^ ]+ target on '
 
+    @classmethod
+    def verbose_flag(cls):
+        return "-v"
+
     @property
     def openmp_flag(self):
         return "-mp"
diff --git a/lib/spack/spack/compilers/xl.py b/lib/spack/spack/compilers/xl.py
index f82ae31b59..fcfaa29d12 100644
--- a/lib/spack/spack/compilers/xl.py
+++ b/lib/spack/spack/compilers/xl.py
@@ -29,6 +29,10 @@ class Xl(Compiler):
     version_argument = '-qversion'
     version_regex = r'([0-9]?[0-9]\.[0-9])'
 
+    @classmethod
+    def verbose_flag(cls):
+        return "-V"
+
     @property
     def openmp_flag(self):
         return "-qsmp=omp"
diff --git a/lib/spack/spack/schema/compilers.py b/lib/spack/spack/schema/compilers.py
index 69659917ff..934c3cad3d 100644
--- a/lib/spack/spack/schema/compilers.py
+++ b/lib/spack/spack/schema/compilers.py
@@ -61,6 +61,10 @@ properties = {
                         'modules': {'anyOf': [{'type': 'string'},
                                               {'type': 'null'},
                                               {'type': 'array'}]},
+                        'implicit_rpaths': {
+                            'type': 'array',
+                            'items': {'type': 'string'}
+                        },
                         'environment': {
                             'type': 'object',
                             'default': {},
diff --git a/lib/spack/spack/test/data/compiler_verbose_output/cce-8.6.5.txt b/lib/spack/spack/test/data/compiler_verbose_output/cce-8.6.5.txt
new file mode 100644
index 0000000000..f1098f4b54
--- /dev/null
+++ b/lib/spack/spack/test/data/compiler_verbose_output/cce-8.6.5.txt
@@ -0,0 +1,10 @@
+rm foo
+
+/opt/cray/pe/cce/8.6.5/binutils/x86_64/x86_64-pc-linux-gnu/bin/ld /usr/lib64//crt1.o /usr/lib64//crti.o /opt/gcc/6.1.0/snos/lib/gcc/x86_64-suse-linux/6.1.0//crtbeginT.o /opt/gcc/6.1.0/snos/lib/gcc/x86_64-suse-linux/6.1.0//crtfastmath.o /opt/cray/pe/cce/8.6.5/cce/x86_64/lib/no_mmap.o foo.o -Bstatic -rpath=/opt/cray/pe/cce/8.6.5/cce/x86_64/lib -L /opt/gcc/6.1.0/snos/lib64 -rpath=/opt/cray/pe/gcc-libs -L /usr/lib64 -L /lib64 -L /opt/cray/dmapp/default/lib64 -L /opt/cray/pe/mpt/7.7.0/gni/mpich-cray/8.6/lib -L /opt/cray/dmapp/default/lib64 -L /opt/cray/pe/mpt/7.7.0/gni/mpich-cray/8.6/lib -L /opt/cray/pe/libsci/17.12.1/CRAY/8.6/x86_64/lib -L /opt/cray/rca/2.2.16-6.0.5.0_15.34__g5e09e6d.ari/lib64 -L /opt/cray/pe/pmi/5.0.13/lib64 -L /opt/cray/xpmem/2.2.4-6.0.5.0_4.8__g35d5e73.ari/lib64 -L /opt/cray/dmapp/7.1.1-6.0.5.0_49.8__g1125556.ari/lib64 -L /opt/cray/ugni/6.0.14-6.0.5.0_16.9__g19583bb.ari/lib64 -L /opt/cray/udreg/2.3.2-6.0.5.0_13.12__ga14955a.ari/lib64 -L /opt/cray/alps/6.5.28-6.0.5.0_18.6__g13a91b6.ari/lib64 -L /opt/cray/pe/atp/2.1.1/libApp -L /opt/cray/pe/cce/8.6.5/cce/x86_64/lib/pkgconfig/../ -L /opt/cray/wlm_detect/1.3.2-6.0.5.0_3.1__g388ccd5.ari/lib64 --no-as-needed -lAtpSigHandler -lAtpSigHCommData --undefined=_ATP_Data_Globals --undefined=__atpHandlerInstall -lpthread -lmpichcxx_cray -lrt -lpthread -lugni -lpmi -lsci_cray_mpi_mp -lm -lf -lsci_cray_mp -lmpich_cray -lrt -lpthread -lugni -lpmi -lsci_cray_mp -lcraymp -lm -lpthread -lf -lhugetlbfs -lpgas-dmapp -lfi -lu -lrt --undefined=dmapp_get_flag_nbi -ldmapp -lugni -ludreg -lpthread -lm -lcray-c++-rts -lstdc++ -lxpmem -ldmapp -lpthread -lpmi -lpthread -lalpslli -lpthread -lwlm_detect -lugni -lpthread -lalpsutil -lpthread -lrca -ludreg -lquadmath -lm -lomp -lcraymp -lpthread -lrt -ldl -lcray-c++-rts -lstdc++ -lm -lmodules -lm -lfi -lm -lquadmath -lcraymath -lm -lgfortran -lquadmath -lf -lm -lpthread -lu -lrt -ldl -lcray-c++-rts -lstdc++ -lm -lcsup --as-needed -latomic --no-as-needed -lcray-c++-rts -lstdc++ -lsupc++ -lstdc++ -lpthread --start-group -lc -lcsup -lgcc_eh -lm -lgcc --end-group -T/opt/cray/pe/cce/8.6.5/cce/x86_64/lib/2.23.1.cce.ld -L /opt/gcc/6.1.0/snos/lib/gcc/x86_64-suse-linux/6.1.0 -L /opt/cray/pe/cce/8.6.5/binutils/x86_64/x86_64-pc-linux-gnu/..//x86_64-unknown-linux-gnu/lib -EL -o foo --undefined=__pthread_initialize_minimal /opt/gcc/6.1.0/snos/lib/gcc/x86_64-suse-linux/6.1.0//crtend.o /usr/lib64//crtn.o
+
+/opt/cray/pe/cce/8.6.5/binutils/x86_64/x86_64-pc-linux-gnu/bin/objcopy --remove-section=.note.ftn_module_data foo
+rm /tmp/pe_27645//pldir/PL_path
+rm /tmp/pe_27645//pldir/PL_module_list
+rm /tmp/pe_27645//pldir/PL_global_data
+rmdir /tmp/pe_27645//pldir
+rmdir /tmp/pe_27645/
diff --git a/lib/spack/spack/test/data/compiler_verbose_output/clang-4.0.1.txt b/lib/spack/spack/test/data/compiler_verbose_output/clang-4.0.1.txt
new file mode 100644
index 0000000000..c55d3a68e7
--- /dev/null
+++ b/lib/spack/spack/test/data/compiler_verbose_output/clang-4.0.1.txt
@@ -0,0 +1,20 @@
+clang version 4.0.1 (tags/RELEASE_401/final)
+Target: x86_64-unknown-linux-gnu
+Thread model: posix
+InstalledDir: /usr/bin
+Found candidate GCC installation: /usr/bin/../lib/gcc/x86_64-redhat-linux/7
+Found candidate GCC installation: /usr/lib/gcc/x86_64-redhat-linux/7
+Selected GCC installation: /usr/bin/../lib/gcc/x86_64-redhat-linux/7
+Candidate multilib: .;@m64
+Candidate multilib: 32;@m32
+Selected multilib: .;@m64
+ "/usr/bin/clang-4.0" -cc1 -triple x86_64-unknown-linux-gnu -emit-obj -mrelax-all -disable-free -disable-llvm-verifier -discard-value-names -main-file-name main.c -mrelocation-model static -mthread-model posix -mdisable-fp-elim -fmath-errno -masm-verbose -mconstructor-aliases -munwind-tables -fuse-init-array -target-cpu x86-64 -v -dwarf-column-info -debugger-tuning=gdb -resource-dir /usr/bin/../lib64/clang/4.0.1 -internal-isystem /usr/local/include -internal-isystem /usr/bin/../lib64/clang/4.0.1/include -internal-externc-isystem /include -internal-externc-isystem /usr/include -fdebug-compilation-dir /tmp/spack-test -ferror-limit 19 -fmessage-length 0 -fobjc-runtime=gcc -fdiagnostics-show-option -o /tmp/main-bf64f0.o -x c main.c
+clang -cc1 version 4.0.1 based upon LLVM 4.0.1 default target x86_64-unknown-linux-gnu
+ignoring nonexistent directory "/include"
+#include "..." search starts here:
+#include <...> search starts here:
+ /usr/local/include
+ /usr/bin/../lib64/clang/4.0.1/include
+ /usr/include
+End of search list.
+ "/usr/bin/ld" --hash-style=gnu --no-add-needed --build-id --eh-frame-hdr -m elf_x86_64 -dynamic-linker /lib64/ld-linux-x86-64.so.2 -o output /usr/bin/../lib/gcc/x86_64-redhat-linux/7/../../../../lib64/crt1.o /usr/bin/../lib/gcc/x86_64-redhat-linux/7/../../../../lib64/crti.o /usr/bin/../lib/gcc/x86_64-redhat-linux/7/crtbegin.o -L/usr/bin/../lib/gcc/x86_64-redhat-linux/7 -L/usr/bin/../lib/gcc/x86_64-redhat-linux/7/../../../../lib64 -L/usr/bin/../lib64 -L/lib/../lib64 -L/usr/lib/../lib64 -L/usr/bin/../lib/gcc/x86_64-redhat-linux/7/../../.. -L/usr/bin/../lib -L/lib -L/usr/lib /tmp/main-bf64f0.o -lgcc --as-needed -lgcc_s --no-as-needed -lc -lgcc --as-needed -lgcc_s --no-as-needed /usr/bin/../lib/gcc/x86_64-redhat-linux/7/crtend.o /usr/bin/../lib/gcc/x86_64-redhat-linux/7/../../../../lib64/crtn.o
diff --git a/lib/spack/spack/test/data/compiler_verbose_output/clang-9.0.0-apple-ld.txt b/lib/spack/spack/test/data/compiler_verbose_output/clang-9.0.0-apple-ld.txt
new file mode 100644
index 0000000000..7592329045
--- /dev/null
+++ b/lib/spack/spack/test/data/compiler_verbose_output/clang-9.0.0-apple-ld.txt
@@ -0,0 +1,7 @@
+@(#)PROGRAM:ld  PROJECT:ld64-305
+configured to support archs: armv6 armv7 armv7s arm64 i386 x86_64 x86_64h armv6m armv7k armv7m armv7em (tvOS)
+Library search paths:
+	/usr/local/lib
+	/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX10.13.sdk/usr/lib
+Framework search paths:
+	  /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX10.13.sdk/System/Library/Frameworks/
diff --git a/lib/spack/spack/test/data/compiler_verbose_output/gcc-7.3.1.txt b/lib/spack/spack/test/data/compiler_verbose_output/gcc-7.3.1.txt
new file mode 100644
index 0000000000..76a4537a59
--- /dev/null
+++ b/lib/spack/spack/test/data/compiler_verbose_output/gcc-7.3.1.txt
@@ -0,0 +1,36 @@
+Using built-in specs.
+COLLECT_GCC=/usr/bin/gcc
+COLLECT_LTO_WRAPPER=/usr/libexec/gcc/x86_64-redhat-linux/7/lto-wrapper
+OFFLOAD_TARGET_NAMES=nvptx-none
+OFFLOAD_TARGET_DEFAULT=1
+Target: x86_64-redhat-linux
+Configured with: ../configure --enable-bootstrap --enable-languages=c,c++,objc,obj-c++,fortran,ada,go,lto --prefix=/usr --mandir=/usr/share/man --infodir=/usr/share/info --with-bugurl=http://bugzilla.redhat.com/bugzilla --enable-shared --enable-threads=posix --enable-checking=release --enable-multilib --with-system-zlib --enable-__cxa_atexit --disable-libunwind-exceptions --enable-gnu-unique-object --enable-linker-build-id --with-gcc-major-version-only --with-linker-hash-style=gnu --enable-plugin --enable-initfini-array --with-isl --enable-libmpx --enable-offload-targets=nvptx-none --without-cuda-driver --enable-gnu-indirect-function --with-tune=generic --with-arch_32=i686 --build=x86_64-redhat-linux
+Thread model: posix
+gcc version 7.3.1 20180130 (Red Hat 7.3.1-2) (GCC) 
+COLLECT_GCC_OPTIONS='-v' '-o' 'output' '-mtune=generic' '-march=x86-64'
+ /usr/libexec/gcc/x86_64-redhat-linux/7/cc1 -quiet -v main.c -quiet -dumpbase main.c -mtune=generic -march=x86-64 -auxbase main -version -o /tmp/ccM76aqK.s
+GNU C11 (GCC) version 7.3.1 20180130 (Red Hat 7.3.1-2) (x86_64-redhat-linux)
+	compiled by GNU C version 7.3.1 20180130 (Red Hat 7.3.1-2), GMP version 6.1.2, MPFR version 3.1.5, MPC version 1.0.2, isl version isl-0.16.1-GMP
+
+GGC heuristics: --param ggc-min-expand=100 --param ggc-min-heapsize=131072
+ignoring nonexistent directory "/usr/lib/gcc/x86_64-redhat-linux/7/include-fixed"
+ignoring nonexistent directory "/usr/lib/gcc/x86_64-redhat-linux/7/../../../../x86_64-redhat-linux/include"
+#include "..." search starts here:
+#include <...> search starts here:
+ /usr/lib/gcc/x86_64-redhat-linux/7/include
+ /usr/local/include
+ /usr/include
+End of search list.
+GNU C11 (GCC) version 7.3.1 20180130 (Red Hat 7.3.1-2) (x86_64-redhat-linux)
+	compiled by GNU C version 7.3.1 20180130 (Red Hat 7.3.1-2), GMP version 6.1.2, MPFR version 3.1.5, MPC version 1.0.2, isl version isl-0.16.1-GMP
+
+GGC heuristics: --param ggc-min-expand=100 --param ggc-min-heapsize=131072
+Compiler executable checksum: ad7c3a488cf591743af375264d348c5c
+COLLECT_GCC_OPTIONS='-v' '-o' 'output' '-mtune=generic' '-march=x86-64'
+ as -v --64 -o /tmp/ccYFphwj.o /tmp/ccM76aqK.s
+GNU assembler version 2.27 (x86_64-redhat-linux) using BFD version version 2.27-28.fc26
+COMPILER_PATH=/usr/libexec/gcc/x86_64-redhat-linux/7/:/usr/libexec/gcc/x86_64-redhat-linux/7/:/usr/libexec/gcc/x86_64-redhat-linux/:/usr/lib/gcc/x86_64-redhat-linux/7/:/usr/lib/gcc/x86_64-redhat-linux/
+LIBRARY_PATH=/usr/lib/gcc/x86_64-redhat-linux/7/:/usr/lib/gcc/x86_64-redhat-linux/7/../../../../lib64/:/lib/../lib64/:/usr/lib/../lib64/:/usr/lib/gcc/x86_64-redhat-linux/7/../../../:/lib/:/usr/lib/
+COLLECT_GCC_OPTIONS='-v' '-o' 'output' '-mtune=generic' '-march=x86-64'
+ /usr/libexec/gcc/x86_64-redhat-linux/7/collect2 -plugin /usr/libexec/gcc/x86_64-redhat-linux/7/liblto_plugin.so -plugin-opt=/usr/libexec/gcc/x86_64-redhat-linux/7/lto-wrapper -plugin-opt=-fresolution=/tmp/ccw0b6CS.res -plugin-opt=-pass-through=-lgcc -plugin-opt=-pass-through=-lgcc_s -plugin-opt=-pass-through=-lc -plugin-opt=-pass-through=-lgcc -plugin-opt=-pass-through=-lgcc_s --build-id --no-add-needed --eh-frame-hdr --hash-style=gnu -m elf_x86_64 -dynamic-linker /lib64/ld-linux-x86-64.so.2 -o output /usr/lib/gcc/x86_64-redhat-linux/7/../../../../lib64/crt1.o /usr/lib/gcc/x86_64-redhat-linux/7/../../../../lib64/crti.o /usr/lib/gcc/x86_64-redhat-linux/7/crtbegin.o -L/usr/lib/gcc/x86_64-redhat-linux/7 -L/usr/lib/gcc/x86_64-redhat-linux/7/../../../../lib64 -L/lib/../lib64 -L/usr/lib/../lib64 -L/usr/lib/gcc/x86_64-redhat-linux/7/../../.. /tmp/ccYFphwj.o -lgcc --as-needed -lgcc_s --no-as-needed -lc -lgcc --as-needed -lgcc_s --no-as-needed /usr/lib/gcc/x86_64-redhat-linux/7/crtend.o /usr/lib/gcc/x86_64-redhat-linux/7/../../../../lib64/crtn.o
+COLLECT_GCC_OPTIONS='-v' '-o' 'output' '-mtune=generic' '-march=x86-64'
diff --git a/lib/spack/spack/test/data/compiler_verbose_output/icc-16.0.3.txt b/lib/spack/spack/test/data/compiler_verbose_output/icc-16.0.3.txt
new file mode 100644
index 0000000000..cdbe4455b9
--- /dev/null
+++ b/lib/spack/spack/test/data/compiler_verbose_output/icc-16.0.3.txt
@@ -0,0 +1,4 @@
+icc.orig version 16.0.3 (gcc version 4.9.3 compatibility)
+ld    /lib/../lib64/crt1.o /lib/../lib64/crti.o /usr/tce/packages/gcc/gcc-4.9.3/lib64/gcc/x86_64-unknown-linux-gnu/4.9.3/crtbegin.o --eh-frame-hdr --build-id -dynamic-linker /lib64/ld-linux-x86-64.so.2 -m elf_x86_64 -o blah -L/usr/tce/packages/intel/intel-16.0.3/compilers_and_libraries_2016.3.210/linux/compiler/lib/intel64_lin -L/usr/tce/packages/gcc/gcc-4.9.3/lib64/gcc/x86_64-unknown-linux-gnu/4.9.3/ -L/usr/tce/packages/gcc/gcc-4.9.3/lib64/gcc/x86_64-unknown-linux-gnu/4.9.3/../../../../lib64 -L/usr/tce/packages/gcc/gcc-4.9.3/lib64/gcc/x86_64-unknown-linux-gnu/4.9.3/../../../../lib64/ -L/lib/../lib64 -L/lib/../lib64/ -L/usr/lib/../lib64 -L/usr/lib/../lib64/ -L/usr/tce/packages/gcc/gcc-4.9.3/lib64/gcc/x86_64-unknown-linux-gnu/4.9.3/../../../ -L/lib64 -L/lib/ -L/usr/lib64 -L/usr/lib -rpath /usr/tce/packages/intel/intel-16.0.3/lib/intel64 -rpath=/usr/tce/packages/gcc/default/lib64 -Bdynamic -Bstatic -limf -lsvml -lirng -Bdynamic -lm -Bstatic -lipgo -ldecimal --as-needed -Bdynamic -lcilkrts -lstdc++ --no-as-needed -lgcc -lgcc_s -Bstatic -lirc -lsvml -Bdynamic -lc -lgcc -lgcc_s -Bstatic -lirc_s -Bdynamic -ldl -lc /usr/tce/packages/gcc/gcc-4.9.3/lib64/gcc/x86_64-unknown-linux-gnu/4.9.3/crtend.o /lib/../lib64/crtn.o
+/lib/../lib64/crt1.o: In function `_start':
+(.text+0x20): undefined reference to `main'
diff --git a/lib/spack/spack/test/data/compiler_verbose_output/obscure-parsing-rules.txt b/lib/spack/spack/test/data/compiler_verbose_output/obscure-parsing-rules.txt
new file mode 100644
index 0000000000..7cf090a846
--- /dev/null
+++ b/lib/spack/spack/test/data/compiler_verbose_output/obscure-parsing-rules.txt
@@ -0,0 +1,3 @@
+This is synthetic data to test parsing cases for which I could not find compiler output
+ld -LIBPATH:/first/path /LIBPATH:/second/path -libpath:/third/path
+collect2 version ld -LIBPATH:/skip/path -L/skip/this/too
diff --git a/lib/spack/spack/test/data/compiler_verbose_output/pgcc-16.3.txt b/lib/spack/spack/test/data/compiler_verbose_output/pgcc-16.3.txt
new file mode 100644
index 0000000000..9041322350
--- /dev/null
+++ b/lib/spack/spack/test/data/compiler_verbose_output/pgcc-16.3.txt
@@ -0,0 +1,11 @@
+Export PGI=/usr/tce/packages/pgi/pgi-16.3
+
+/usr/tce/packages/pgi/pgi-16.3/linux86-64/16.3/bin/pgc test.c -opt 1 -x 119 0xa10000 -x 122 0x40 -x 123 0x1000 -x 127 4 -x 127 17 -x 19 0x400000 -x 28 0x40000 -x 120 0x10000000 -x 70 0x8000 -x 122 1 -x 125 0x20000 -quad -x 59 4 -tp haswell -x 120 0x1000 -astype 0 -stdinc /usr/tce/packages/pgi/pgi-16.3/linux86-64/16.3/include-gcc48:/usr/tce/packages/pgi/pgi-16.3/linux86-64/16.3/include:/usr/lib/gcc/x86_64-redhat-linux/4.8.5/include:/usr/local/include:/usr/include -def unix -def __unix -def __unix__ -def linux -def __linux -def __linux__ -def __NO_MATH_INLINES -def __LP64__ -def __x86_64 -def __x86_64__ -def __LONG_MAX__=9223372036854775807L -def '__SIZE_TYPE__=unsigned long int' -def '__PTRDIFF_TYPE__=long int' -def __THROW= -def __extension__= -def __amd_64__amd64__ -def __k8 -def __k8__ -def __SSE__ -def __MMX__ -def __SSE2__ -def __SSE3__ -def __SSSE3__ -def __STDC_HOSTED__ -predicate '#machine(x86_64) #lint(off) #system(posix) #cpu(x86_64)' -cmdline '+pgcc test.c -v -o test.o' -x 123 0x80000000 -x 123 4 -x 2 0x400 -x 119 0x20 -def __pgnu_vsn=40805 -x 120 0x200000 -x 70 0x40000000 -y 163 0xc0000000 -x 189 0x10 -y 189 0x4000000 -asm /var/tmp/gamblin2/pgccL0MCVCOQsq6l.s
+PGC/x86-64 Linux 16.3-0: compilation successful
+
+/usr/bin/as /var/tmp/gamblin2/pgccL0MCVCOQsq6l.s -o /var/tmp/gamblin2/pgcc10MCFxmYXjgo.o
+
+/usr/tce/bin/ld /usr/lib64/crt1.o /usr/lib64/crti.o /usr/tce/packages/pgi/pgi-16.3/linux86-64/16.3/lib/trace_init.o /usr/lib/gcc/x86_64-redhat-linux/4.8.5/crtbegin.o /usr/tce/packages/pgi/pgi-16.3/linux86-64/16.3/lib/initmp.o --eh-frame-hdr -m elf_x86_64 -dynamic-linker /lib64/ld-linux-x86-64.so.2 /usr/tce/packages/pgi/pgi-16.3/linux86-64/16.3/lib/pgi.ld -L/usr/tce/packages/pgi/pgi-16.3/linux86-64/16.3/lib -L/usr/lib64 -L/usr/lib/gcc/x86_64-redhat-linux/4.8.5 /var/tmp/gamblin2/pgcc10MCFxmYXjgo.o -rpath /usr/tce/packages/pgi/pgi-16.3/linux86-64/16.3/lib -o test.o -lpgmp -lnuma -lpthread -lnspgc -lpgc -lm -lgcc -lc -lgcc /usr/lib/gcc/x86_64-redhat-linux/4.8.5/crtend.o /usr/lib64/crtn.o
+Unlinking /var/tmp/gamblin2/pgccL0MCVCOQsq6l.s
+Unlinking /var/tmp/gamblin2/pgccn0MCNcmgIbh8.ll
+Unlinking /var/tmp/gamblin2/pgcc10MCFxmYXjgo.o
diff --git a/lib/spack/spack/test/data/compiler_verbose_output/xl-13.1.5.txt b/lib/spack/spack/test/data/compiler_verbose_output/xl-13.1.5.txt
new file mode 100644
index 0000000000..2ff243a4d9
--- /dev/null
+++ b/lib/spack/spack/test/data/compiler_verbose_output/xl-13.1.5.txt
@@ -0,0 +1,5 @@
+export XL_CONFIG=/opt/ibm/xlC/13.1.5/etc/xlc.cfg.centos.7.gcc.4.8.5:xlc 
+/usr/bin/ld --eh-frame-hdr -Qy -melf64lppc /usr/lib/gcc/ppc64le-redhat-linux/4.8.5/../../../../lib64/crt1.o /usr/lib/gcc/ppc64le-redhat-linux/4.8.5/../../../../lib64/crti.o /usr/lib/gcc/ppc64le-redhat-linux/4.8.5/crtbegin.o -L/opt/ibm/xlsmp/4.1.5/lib -L/opt/ibm/xlmass/8.1.5/lib -L/opt/ibm/xlC/13.1.5/lib -R/opt/ibm/lib -L/usr/lib/gcc/ppc64le-redhat-linux/4.8.5 -L/usr/lib/gcc/ppc64le-redhat-linux/4.8.5/../../../../lib64 -L/lib/../lib64 -L/usr/lib/../lib64 -L/usr/lib/gcc/ppc64le-redhat-linux/4.8.5/../../.. --no-toc-optimize -o foo foo.o -dynamic-linker /lib64/ld64.so.2 --enable-new-dtags -lxlopt -lxl --as-needed -ldl --no-as-needed -lgcc_s --as-needed -lpthread --no-as-needed -lgcc -lm -lc -lgcc_s -lgcc /usr/lib/gcc/ppc64le-redhat-linux/4.8.5/crtend.o /usr/lib/gcc/ppc64le-redhat-linux/4.8.5/../../../../lib64/crtn.o 
+rm /tmp/xlcW0iQ4uI8
+rm /tmp/xlcW1aPLBFY
+rm /tmp/xlcW2ALFICO
diff --git a/lib/spack/spack/test/link_paths.py b/lib/spack/spack/test/link_paths.py
new file mode 100644
index 0000000000..33ac68cdc0
--- /dev/null
+++ b/lib/spack/spack/test/link_paths.py
@@ -0,0 +1,89 @@
+# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+import os
+
+import spack.paths
+from spack.compiler import _parse_implicit_rpaths
+
+#: directory with sample compiler data
+datadir = os.path.join(spack.paths.test_path, 'data',
+                       'compiler_verbose_output')
+
+
+def check_link_paths(filename, paths):
+    with open(os.path.join(datadir, filename)) as file:
+        output = file.read()
+    detected_paths = _parse_implicit_rpaths(output)
+
+    actual = detected_paths
+    expected = paths
+
+    missing_paths = list(x for x in expected if x not in actual)
+    assert not missing_paths
+
+    extra_paths = list(x for x in actual if x not in expected)
+    assert not extra_paths
+
+    assert actual == expected
+
+
+def test_icc16_link_paths():
+    check_link_paths('icc-16.0.3.txt', [
+        '/usr/tce/packages/intel/intel-16.0.3/compilers_and_libraries_2016.3.210/linux/compiler/lib/intel64_lin', # noqa
+        '/usr/tce/packages/gcc/gcc-4.9.3/lib64/gcc/x86_64-unknown-linux-gnu/4.9.3', # noqa
+        '/usr/tce/packages/gcc/gcc-4.9.3/lib64'])
+
+
+def test_pgi_link_paths():
+    check_link_paths('pgcc-16.3.txt', [
+        '/usr/tce/packages/pgi/pgi-16.3/linux86-64/16.3/lib'])
+
+
+def test_gcc7_link_paths():
+    check_link_paths('gcc-7.3.1.txt', [])
+
+
+def test_clang4_link_paths():
+    check_link_paths('clang-4.0.1.txt', [])
+
+
+def test_xl_link_paths():
+    check_link_paths('xl-13.1.5.txt', [
+        '/opt/ibm/xlsmp/4.1.5/lib',
+        '/opt/ibm/xlmass/8.1.5/lib',
+        '/opt/ibm/xlC/13.1.5/lib'])
+
+
+def test_cce_link_paths():
+    check_link_paths('cce-8.6.5.txt', [
+        '/opt/gcc/6.1.0/snos/lib64',
+        '/opt/cray/dmapp/default/lib64',
+        '/opt/cray/pe/mpt/7.7.0/gni/mpich-cray/8.6/lib',
+        '/opt/cray/pe/libsci/17.12.1/CRAY/8.6/x86_64/lib',
+        '/opt/cray/rca/2.2.16-6.0.5.0_15.34__g5e09e6d.ari/lib64',
+        '/opt/cray/pe/pmi/5.0.13/lib64',
+        '/opt/cray/xpmem/2.2.4-6.0.5.0_4.8__g35d5e73.ari/lib64',
+        '/opt/cray/dmapp/7.1.1-6.0.5.0_49.8__g1125556.ari/lib64',
+        '/opt/cray/ugni/6.0.14-6.0.5.0_16.9__g19583bb.ari/lib64',
+        '/opt/cray/udreg/2.3.2-6.0.5.0_13.12__ga14955a.ari/lib64',
+        '/opt/cray/alps/6.5.28-6.0.5.0_18.6__g13a91b6.ari/lib64',
+        '/opt/cray/pe/atp/2.1.1/libApp',
+        '/opt/cray/pe/cce/8.6.5/cce/x86_64/lib',
+        '/opt/cray/wlm_detect/1.3.2-6.0.5.0_3.1__g388ccd5.ari/lib64',
+        '/opt/gcc/6.1.0/snos/lib/gcc/x86_64-suse-linux/6.1.0',
+        '/opt/cray/pe/cce/8.6.5/binutils/x86_64/x86_64-unknown-linux-gnu/lib'])
+
+
+def test_clang_apple_ld_link_paths():
+    check_link_paths('clang-9.0.0-apple-ld.txt', [
+        '/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX10.13.sdk/usr/lib'])  # noqa
+
+
+def test_obscure_parsing_rules():
+    check_link_paths('obscure-parsing-rules.txt', [
+        '/first/path',
+        '/second/path',
+        '/third/path'])