From bcf8ebff4ff3ca25c17e690187b1e7cf46657ee5 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Mon, 8 Jun 2020 18:44:07 -0500 Subject: Cray: fix Blue Waters support and user-built MPIs on Cray (#16593) * Cray: fix Blue Waters support * pkg-config env vars needed on Blue Waters * cray platform: fix support for user-build MPI on cray machines * reintroduce cray environment cleaning behind cnl version guard * cray platform: fix support for user-build MPI on cray machines Co-authored-by: Gregory --- lib/spack/spack/build_environment.py | 24 ++++++++++++++-------- lib/spack/spack/platforms/cray.py | 6 ++++-- var/spack/repos/builtin/packages/mpich/package.py | 9 +++++--- .../repos/builtin/packages/mvapich2/package.py | 9 +++++--- 4 files changed, 31 insertions(+), 17 deletions(-) diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index f57ecd1e58..435de8594e 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -53,6 +53,7 @@ import spack.main import spack.paths import spack.schema.environment import spack.store +import spack.architecture as arch from spack.util.string import plural from spack.util.environment import ( env_flag, filter_system_paths, get_path, is_system_path, @@ -141,17 +142,21 @@ def clean_environment(): # can affect how some packages find libraries. We want to make # sure that builds never pull in unintended external dependencies. env.unset('LD_LIBRARY_PATH') - env.unset('CRAY_LD_LIBRARY_PATH') env.unset('LIBRARY_PATH') env.unset('CPATH') env.unset('LD_RUN_PATH') env.unset('DYLD_LIBRARY_PATH') env.unset('DYLD_FALLBACK_LIBRARY_PATH') - # Remove all pkgconfig stuff from craype - for varname in os.environ.keys(): - if 'PKGCONF' in varname: - env.unset(varname) + # On Cray systems newer than CNL5, unset CRAY_LD_LIBRARY_PATH to avoid + # interference with Spack dependencies. CNL5 (e.g. Blue Waters) requires + # these variables to be set. + hostarch = arch.Arch(arch.platform(), 'default_os', 'default_target') + if str(hostarch.platform) == 'cray' and str(hostarch.os) != 'cnl5': + env.unset('CRAY_LD_LIBRARY_PATH') + for varname in os.environ.keys(): + if 'PKGCONF' in varname: + env.unset(varname) build_lang = spack.config.get('config:build_language') if build_lang: @@ -355,10 +360,6 @@ def set_build_environment_variables(pkg, env, dirty): extra_rpaths = ':'.join(compiler.extra_rpaths) env.set('SPACK_COMPILER_EXTRA_RPATHS', extra_rpaths) - implicit_rpaths = compiler.implicit_rpaths() - if implicit_rpaths: - env.set('SPACK_COMPILER_IMPLICIT_RPATHS', ':'.join(implicit_rpaths)) - # Add bin directories from dependencies to the PATH for the build. for prefix in build_prefixes: for dirname in ['bin', 'bin64']: @@ -733,6 +734,11 @@ def setup_package(pkg, dirty): load_external_modules(pkg) + implicit_rpaths = pkg.compiler.implicit_rpaths() + if implicit_rpaths: + build_env.set('SPACK_COMPILER_IMPLICIT_RPATHS', + ':'.join(implicit_rpaths)) + # Make sure nothing's strange about the Spack environment. validate(build_env, tty.warn) build_env.apply_modifications() diff --git a/lib/spack/spack/platforms/cray.py b/lib/spack/spack/platforms/cray.py index 2fccf2fe55..9c8770c368 100644 --- a/lib/spack/spack/platforms/cray.py +++ b/lib/spack/spack/platforms/cray.py @@ -22,7 +22,9 @@ _craype_name_to_target_name = { 'x86-naples': 'zen', 'x86-rome': 'zen', # Cheating because we have the wrong modules on rzcrayz 'x86-skylake': 'skylake_avx512', - 'mic-knl': 'mic_knl' + 'mic-knl': 'mic_knl', + 'interlagos': 'bulldozer', + 'abudhabi': 'piledriver', } @@ -176,7 +178,7 @@ class Cray(Platform): craype_default_path = '/opt/cray/pe/craype/default/modulefiles' if os.path.isdir(craype_default_path): return os.listdir(craype_default_path) - return None + return [] if getattr(self, '_craype_targets', None) is None: strategies = [ diff --git a/var/spack/repos/builtin/packages/mpich/package.py b/var/spack/repos/builtin/packages/mpich/package.py index 7b8c807cce..042c2c5b12 100644 --- a/var/spack/repos/builtin/packages/mpich/package.py +++ b/var/spack/repos/builtin/packages/mpich/package.py @@ -161,8 +161,9 @@ spack package at this time.''', env.set('FFLAGS', '-fallow-argument-mismatch') def setup_dependent_build_environment(self, env, dependent_spec): - # On Cray, the regular compiler wrappers *are* the MPI wrappers. - if 'platform=cray' in self.spec: + # For Cray MPIs, the regular compiler wrappers *are* the MPI wrappers. + # Cray MPIs always have cray in the module name, e.g. "cray-mpich" + if self.spec.external_module and 'cray' in self.spec.external_module: env.set('MPICC', spack_cc) env.set('MPICXX', spack_cxx) env.set('MPIF77', spack_fc) @@ -180,7 +181,9 @@ spack package at this time.''', env.set('MPICH_FC', spack_fc) def setup_dependent_package(self, module, dependent_spec): - if 'platform=cray' in self.spec: + # For Cray MPIs, the regular compiler wrappers *are* the MPI wrappers. + # Cray MPIs always have cray in the module name, e.g. "cray-mpich" + if self.spec.external_module and 'cray' in self.spec.external_module: self.spec.mpicc = spack_cc self.spec.mpicxx = spack_cxx self.spec.mpifc = spack_fc diff --git a/var/spack/repos/builtin/packages/mvapich2/package.py b/var/spack/repos/builtin/packages/mvapich2/package.py index 04205b6e24..0fb3c36b28 100644 --- a/var/spack/repos/builtin/packages/mvapich2/package.py +++ b/var/spack/repos/builtin/packages/mvapich2/package.py @@ -209,8 +209,9 @@ class Mvapich2(AutotoolsPackage): env.set('SLURM_MPI_TYPE', 'pmi2') def setup_dependent_build_environment(self, env, dependent_spec): - # On Cray, the regular compiler wrappers *are* the MPI wrappers. - if 'platform=cray' in self.spec: + # For Cray MPIs, the regular compiler wrappers *are* the MPI wrappers. + # Cray MPIs always have cray in the module name, e.g. "cray-mvapich" + if self.spec.external_module and 'cray' in self.spec.external_module: env.set('MPICC', spack_cc) env.set('MPICXX', spack_cxx) env.set('MPIF77', spack_fc) @@ -228,7 +229,9 @@ class Mvapich2(AutotoolsPackage): env.set('MPICH_FC', spack_fc) def setup_dependent_package(self, module, dependent_spec): - if 'platform=cray' in self.spec: + # For Cray MPIs, the regular compiler wrappers *are* the MPI wrappers. + # Cray MPIs always have cray in the module name, e.g. "cray-mvapich" + if self.spec.external_module and 'cray' in self.spec.external_module: self.spec.mpicc = spack_cc self.spec.mpicxx = spack_cxx self.spec.mpifc = spack_fc -- cgit v1.2.3-70-g09d2