diff options
| -rw-r--r-- | lib/spack/spack/compiler.py | 64 | ||||
| -rw-r--r-- | lib/spack/spack/compilers/nag.py | 30 | ||||
| -rw-r--r-- | lib/spack/spack/test/compilers.py | 41 | ||||
| -rw-r--r-- | lib/spack/spack/test/data/compiler_verbose_output/collect2-6.3.0-gnu-ld.txt | 3 | ||||
| -rw-r--r-- | lib/spack/spack/test/data/compiler_verbose_output/nag-6.2-gcc-6.5.0.txt | 13 | ||||
| -rw-r--r-- | lib/spack/spack/test/link_paths.py | 26 |
6 files changed, 139 insertions, 38 deletions
diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index 410188fff4..784fd5544e 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -3,6 +3,7 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) +import contextlib import os import platform import re @@ -354,10 +355,11 @@ class Compiler(object): for flag_type in flags: for flag in self.flags.get(flag_type, []): compiler_exe.add_default_arg(flag) - output = str(compiler_exe(self.verbose_flag, fin, '-o', fout, - output=str, error=str)) # str for py2 - - return _parse_non_system_link_dirs(output) + with self._compiler_environment(): + output = str(compiler_exe( + self.verbose_flag, fin, '-o', fout, + output=str, error=str)) # str for py2 + return _parse_non_system_link_dirs(output) except spack.util.executable.ProcessError as pe: tty.debug('ProcessError: Command exited with non-zero status: ' + pe.long_message) @@ -468,32 +470,12 @@ class Compiler(object): Use the runtime environment of the compiler (modules and environment modifications) to enable the compiler to run properly on any platform. """ - # store environment to replace later - backup_env = os.environ.copy() - - # load modules and set env variables - for module in self.modules: - # On cray, mic-knl module cannot be loaded without cce module - # See: https://github.com/spack/spack/issues/3153 - if os.environ.get("CRAY_CPU_TARGET") == 'mic-knl': - spack.util.module_cmd.load_module('cce') - spack.util.module_cmd.load_module(module) - - # apply other compiler environment changes - env = spack.util.environment.EnvironmentModifications() - env.extend(spack.schema.environment.parse(self.environment)) - env.apply_modifications() - cc = spack.util.executable.Executable(self.cc) - output = cc(self.version_argument, - output=str, error=str, - ignore_errors=tuple(self.ignore_version_errors)) - - # Restore environment - os.environ.clear() - os.environ.update(backup_env) - - return self.extract_version_from_output(output) + with self._compiler_environment(): + output = cc(self.version_argument, + output=str, error=str, + ignore_errors=tuple(self.ignore_version_errors)) + return self.extract_version_from_output(output) # # Compiler classes have methods for querying the version of @@ -562,6 +544,30 @@ class Compiler(object): self.cc, self.cxx, self.f77, self.fc, self.modules, str(self.operating_system))))) + @contextlib.contextmanager + def _compiler_environment(self): + # store environment to replace later + backup_env = os.environ.copy() + + # load modules and set env variables + for module in self.modules: + # On cray, mic-knl module cannot be loaded without cce module + # See: https://github.com/spack/spack/issues/3153 + if os.environ.get("CRAY_CPU_TARGET") == 'mic-knl': + spack.util.module_cmd.load_module('cce') + spack.util.module_cmd.load_module(module) + + # apply other compiler environment changes + env = spack.util.environment.EnvironmentModifications() + env.extend(spack.schema.environment.parse(self.environment)) + env.apply_modifications() + + yield + + # Restore environment + os.environ.clear() + os.environ.update(backup_env) + class CompilerAccessError(spack.error.SpackError): diff --git a/lib/spack/spack/compilers/nag.py b/lib/spack/spack/compilers/nag.py index 4b2de06c07..503a31e404 100644 --- a/lib/spack/spack/compilers/nag.py +++ b/lib/spack/spack/compilers/nag.py @@ -31,6 +31,36 @@ class Nag(spack.compiler.Compiler): version_regex = r'NAG Fortran Compiler Release ([0-9.]+)' @property + def verbose_flag(self): + # NAG does not support a flag that would enable verbose output and + # compilation/linking at the same time (with either '-#' or '-dryrun' + # the compiler only prints the commands but does not run them). + # Therefore, the only thing we can do is to pass the '-v' argument to + # the underlying GCC. In order to get verbose output from the latter + # at both compile and linking stages, we need to call NAG with two + # additional flags: '-Wc,-v' and '-Wl,-v'. However, we return only + # '-Wl,-v' for the following reasons: + # 1) the interface of this method does not support multiple flags in + # the return value and, at least currently, verbose output at the + # linking stage has a higher priority for us; + # 2) NAG is usually mixed with GCC compiler, which also accepts + # '-Wl,-v' and produces meaningful result with it: '-v' is passed + # to the linker and the latter produces verbose output for the + # linking stage ('-Wc,-v', however, would break the compilation + # with a message from GCC that the flag is not recognized). + # + # This way, we at least enable the implicit rpath detection, which is + # based on compilation of a C file (see method + # spack.compiler._get_compiler_link_paths): in the case of a mixed + # NAG/GCC toolchain, the flag will be passed to g++ (e.g. + # 'g++ -Wl,-v ./main.c'), otherwise, the flag will be passed to nagfor + # (e.g. 'nagfor -Wl,-v ./main.c' - note that nagfor recognizes '.c' + # extension and treats the file accordingly). The list of detected + # rpaths will contain only GCC-related directories and rpaths to + # NAG-related directories are injected by nagfor anyway. + return "-Wl,-v" + + @property def openmp_flag(self): return "-openmp" diff --git a/lib/spack/spack/test/compilers.py b/lib/spack/spack/test/compilers.py index 7c79e9027f..4598605fe4 100644 --- a/lib/spack/spack/test/compilers.py +++ b/lib/spack/spack/test/compilers.py @@ -157,13 +157,14 @@ default_compiler_entry = { class MockCompiler(Compiler): def __init__(self): super(MockCompiler, self).__init__( - "badcompiler@1.0.0", - default_compiler_entry['operating_system'], - None, - [default_compiler_entry['paths']['cc'], - default_compiler_entry['paths']['cxx'], - default_compiler_entry['paths']['fc'], - default_compiler_entry['paths']['f77']]) + cspec="badcompiler@1.0.0", + operating_system=default_compiler_entry['operating_system'], + target=None, + paths=[default_compiler_entry['paths']['cc'], + default_compiler_entry['paths']['cxx'], + default_compiler_entry['paths']['fc'], + default_compiler_entry['paths']['f77']], + environment={}) _get_compiler_link_paths = Compiler._get_compiler_link_paths @@ -267,6 +268,32 @@ def test_get_compiler_link_paths_no_verbose_flag(): assert dirs == [] +def test_get_compiler_link_paths_load_env(working_env, monkeypatch, tmpdir): + gcc = str(tmpdir.join('gcc')) + with open(gcc, 'w') as f: + f.write("""#!/bin/bash +if [[ $ENV_SET == "1" && $MODULE_LOADED == "1" ]]; then + echo '""" + no_flag_output + """' +fi +""") + fs.set_executable(gcc) + + # Set module load to turn compiler on + def module(*args): + if args[0] == 'show': + return '' + elif args[0] == 'load': + os.environ['MODULE_LOADED'] = "1" + monkeypatch.setattr(spack.util.module_cmd, 'module', module) + + compiler = MockCompiler() + compiler.environment = {'set': {'ENV_SET': '1'}} + compiler.modules = ['turn_on'] + + dirs = compiler._get_compiler_link_paths([gcc]) + assert dirs == no_flag_dirs + + # Get the desired flag from the specified compiler spec. def flag_value(flag, spec): compiler = None diff --git a/lib/spack/spack/test/data/compiler_verbose_output/collect2-6.3.0-gnu-ld.txt b/lib/spack/spack/test/data/compiler_verbose_output/collect2-6.3.0-gnu-ld.txt new file mode 100644 index 0000000000..cceff7fc65 --- /dev/null +++ b/lib/spack/spack/test/data/compiler_verbose_output/collect2-6.3.0-gnu-ld.txt @@ -0,0 +1,3 @@ +collect2 version 6.5.0 +/usr/bin/ld -plugin /scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/libexec/gcc/x86_64-pc-linux-gnu/6.5.0/liblto_plugin.so -plugin-opt=/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/libexec/gcc/x86_64-pc-linux-gnu/6.5.0/lto-wrapper -plugin-opt=-fresolution=/tmp/ccbFmewQ.res -plugin-opt=-pass-through=-lgcc_s -plugin-opt=-pass-through=-lgcc -plugin-opt=-pass-through=-lc -plugin-opt=-pass-through=-lgcc_s -plugin-opt=-pass-through=-lgcc -rpath /scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/lib:/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/lib64 --eh-frame-hdr -m elf_x86_64 -dynamic-linker /lib64/ld-linux-x86-64.so.2 -o output /usr/lib/x86_64-linux-gnu/crt1.o /usr/lib/x86_64-linux-gnu/crti.o /scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/lib/gcc/x86_64-pc-linux-gnu/6.5.0/crtbegin.o -L/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/lib/gcc/x86_64-pc-linux-gnu/6.5.0 -L/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/lib/gcc/x86_64-pc-linux-gnu/6.5.0/../../../../lib64 -L/lib/x86_64-linux-gnu -L/lib/../lib64 -L/usr/lib/x86_64-linux-gnu -L/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/lib/gcc/x86_64-pc-linux-gnu/6.5.0/../../.. -v /tmp/ccxz6i1I.o -lstdc++ -lm -lgcc_s -lgcc -lc -lgcc_s -lgcc /scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/lib/gcc/x86_64-pc-linux-gnu/6.5.0/crtend.o /usr/lib/x86_64-linux-gnu/crtn.o +GNU ld (GNU Binutils for Debian) 2.28 diff --git a/lib/spack/spack/test/data/compiler_verbose_output/nag-6.2-gcc-6.5.0.txt b/lib/spack/spack/test/data/compiler_verbose_output/nag-6.2-gcc-6.5.0.txt new file mode 100644 index 0000000000..9fa3dc69ba --- /dev/null +++ b/lib/spack/spack/test/data/compiler_verbose_output/nag-6.2-gcc-6.5.0.txt @@ -0,0 +1,13 @@ +NAG Fortran Compiler Release 6.2(Chiyoda) Build 6223 +Reading specs from /scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/lib/gcc/x86_64-pc-linux-gnu/6.5.0/specs +COLLECT_GCC=/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/bin/gcc +COLLECT_LTO_WRAPPER=/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/libexec/gcc/x86_64-pc-linux-gnu/6.5.0/lto-wrapper +Target: x86_64-pc-linux-gnu +Configured with: /tmp/m300488/spack-stage/spack-stage-gcc-6.5.0-4sdjgrsboy3lowtq3t7pmp7rx3ogkqtz/spack-src/configure --prefix=/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs --with-pkgversion='Spack GCC' --with-bugurl=https://github.com/spack/spack/issues --disable-multilib --enable-languages=c,c++,fortran --disable-nls --with-mpfr=/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/mpfr-3.1.6-w63rspk --with-gmp=/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gmp-6.1.2-et64cuj --with-system-zlib --with-mpc=/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/mpc-1.1.0-en66k4t --with-isl=/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/isl-0.18-62v4uyg +Thread model: posix +gcc version 6.5.0 (Spack GCC) +COMPILER_PATH=/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/libexec/gcc/x86_64-pc-linux-gnu/6.5.0/:/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/libexec/gcc/x86_64-pc-linux-gnu/6.5.0/:/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/libexec/gcc/x86_64-pc-linux-gnu/:/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/lib/gcc/x86_64-pc-linux-gnu/6.5.0/:/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/lib/gcc/x86_64-pc-linux-gnu/ +LIBRARY_PATH=/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/lib/gcc/x86_64-pc-linux-gnu/6.5.0/:/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/lib/gcc/x86_64-pc-linux-gnu/6.5.0/../../../../lib64/:/lib/x86_64-linux-gnu/:/lib/../lib64/:/usr/lib/x86_64-linux-gnu/:/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/lib/gcc/x86_64-pc-linux-gnu/6.5.0/../../../:/lib/:/usr/lib/ +COLLECT_GCC_OPTIONS='-m64' '-o' 'output' '-v' '-mtune=generic' '-march=x86-64' + /scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/libexec/gcc/x86_64-pc-linux-gnu/6.5.0/collect2 -plugin /scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/libexec/gcc/x86_64-pc-linux-gnu/6.5.0/liblto_plugin.so -plugin-opt=/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/libexec/gcc/x86_64-pc-linux-gnu/6.5.0/lto-wrapper -plugin-opt=-fresolution=/tmp/ccBpU203.res -plugin-opt=-pass-through=-lgcc -plugin-opt=-pass-through=-lgcc_s -plugin-opt=-pass-through=-lc -plugin-opt=-pass-through=-lgcc -plugin-opt=-pass-through=-lgcc_s -rpath /scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/lib:/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/lib64 --eh-frame-hdr -m elf_x86_64 -dynamic-linker /lib64/ld-linux-x86-64.so.2 -o output /usr/lib/x86_64-linux-gnu/crt1.o /usr/lib/x86_64-linux-gnu/crti.o /scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/lib/gcc/x86_64-pc-linux-gnu/6.5.0/crtbegin.o -L/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/lib/gcc/x86_64-pc-linux-gnu/6.5.0 -L/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/lib/gcc/x86_64-pc-linux-gnu/6.5.0/../../../../lib64 -L/lib/x86_64-linux-gnu -L/lib/../lib64 -L/usr/lib/x86_64-linux-gnu -L/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/lib/gcc/x86_64-pc-linux-gnu/6.5.0/../../.. /sw/stretch-x64/nag/nag-6.2/lib/NAG_Fortran/f62init.o /sw/stretch-x64/nag/nag-6.2/lib/NAG_Fortran/quickfit.o /tmp/main.000786.o -rpath /sw/stretch-x64/nag/nag-6.2/lib/NAG_Fortran /sw/stretch-x64/nag/nag-6.2/lib/NAG_Fortran/libf62rts.so /sw/stretch-x64/nag/nag-6.2/lib/NAG_Fortran/libf62rts.a -lm -lgcc --as-needed -lgcc_s --no-as-needed -lc -lgcc --as-needed -lgcc_s --no-as-needed /scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/lib/gcc/x86_64-pc-linux-gnu/6.5.0/crtend.o /usr/lib/x86_64-linux-gnu/crtn.o +COLLECT_GCC_OPTIONS='-m64' '-o' 'output' '-v' '-mtune=generic' '-march=x86-64' diff --git a/lib/spack/spack/test/link_paths.py b/lib/spack/spack/test/link_paths.py index 27e42d2194..4ae0a35cf8 100644 --- a/lib/spack/spack/test/link_paths.py +++ b/lib/spack/spack/test/link_paths.py @@ -32,8 +32,8 @@ def check_link_paths(filename, paths): def test_icc16_link_paths(): check_link_paths('icc-16.0.3.txt', [ - '/usr/tce/packages/intel/intel-16.0.3/compilers_and_libraries_2016.3.210/linux/compiler/lib/intel64_lin', # noqa - '/usr/tce/packages/gcc/gcc-4.9.3/lib64/gcc/x86_64-unknown-linux-gnu/4.9.3', # noqa + '/usr/tce/packages/intel/intel-16.0.3/compilers_and_libraries_2016.3.210/linux/compiler/lib/intel64_lin', # noqa + '/usr/tce/packages/gcc/gcc-4.9.3/lib64/gcc/x86_64-unknown-linux-gnu/4.9.3', # noqa '/usr/tce/packages/gcc/gcc-4.9.3/lib64']) @@ -82,6 +82,28 @@ def test_clang_apple_ld_link_paths(): '/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX10.13.sdk/usr/lib']) # noqa +def test_nag_mixed_gcc_gnu_ld_link_paths(): + # This is a test of a mixed NAG/GCC toolchain, i.e. 'cxx' is set to g++ and + # is used for the rpath detection. The reference compiler output is a + # result of + # '/path/to/gcc/bin/g++ -Wl,-v ./main.c'. + check_link_paths('collect2-6.3.0-gnu-ld.txt', [ + '/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/lib/gcc/x86_64-pc-linux-gnu/6.5.0', # noqa + '/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/lib64', # noqa + '/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/lib']) # noqa + + +def test_nag_link_paths(): + # This is a test of a NAG-only toolchain, i.e. 'cc' and 'cxx' are empty, + # and therefore 'fc' is used for the rpath detection). The reference + # compiler output is a result of + # 'nagfor -Wc=/path/to/gcc/bin/gcc -Wl,-v ./main.c'. + check_link_paths('nag-6.2-gcc-6.5.0.txt', [ + '/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/lib/gcc/x86_64-pc-linux-gnu/6.5.0', # noqa + '/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/lib64', # noqa + '/scratch/local1/spack/opt/spack/gcc-6.3.0-haswell/gcc-6.5.0-4sdjgrs/lib']) # noqa + + def test_obscure_parsing_rules(): check_link_paths('obscure-parsing-rules.txt', [ '/first/path', |