From b59d4f243bc012f6673a53be75d197c16e5e7a06 Mon Sep 17 00:00:00 2001 From: alalazo Date: Wed, 15 Jun 2016 13:16:29 +0200 Subject: openmpi, mvapich2, mpich : filtered out "--Wl,--enable-new-dtag" in wrappers, as it turns RPATH into RUNPATH Apart from that turned `os.path.join` into `join_path` and `os.environ` into `env` in the packages --- var/spack/repos/builtin/packages/mpich/package.py | 28 +++++---- .../repos/builtin/packages/mvapich2/package.py | 24 +++++--- .../repos/builtin/packages/openmpi/package.py | 67 ++++++++++------------ 3 files changed, 62 insertions(+), 57 deletions(-) (limited to 'var') diff --git a/var/spack/repos/builtin/packages/mpich/package.py b/var/spack/repos/builtin/packages/mpich/package.py index 164d9db541..e7ae63ef70 100644 --- a/var/spack/repos/builtin/packages/mpich/package.py +++ b/var/spack/repos/builtin/packages/mpich/package.py @@ -23,7 +23,6 @@ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## from spack import * -import os class Mpich(Package): @@ -102,18 +101,25 @@ class Mpich(Package): be bound to whatever compiler they were built with. """ bin = self.prefix.bin - mpicc = os.path.join(bin, 'mpicc') - mpicxx = os.path.join(bin, 'mpicxx') - mpif77 = os.path.join(bin, 'mpif77') - mpif90 = os.path.join(bin, 'mpif90') - - spack_cc = os.environ['CC'] - spack_cxx = os.environ['CXX'] - spack_f77 = os.environ['F77'] - spack_fc = os.environ['FC'] - + mpicc = join_path(bin, 'mpicc') + mpicxx = join_path(bin, 'mpicxx') + mpif77 = join_path(bin, 'mpif77') + mpif90 = join_path(bin, 'mpif90') + + spack_cc = env['CC'] + spack_cxx = env['CXX'] + spack_f77 = env['F77'] + spack_fc = env['FC'] + + # Substitute Spack compile wrappers for the real + # underlying compiler kwargs = { 'ignore_absent' : True, 'backup' : False, 'string' : True } filter_file('CC="%s"' % spack_cc , 'CC="%s"' % self.compiler.cc, mpicc, **kwargs) filter_file('CXX="%s"'% spack_cxx, 'CXX="%s"' % self.compiler.cxx, mpicxx, **kwargs) filter_file('F77="%s"'% spack_f77, 'F77="%s"' % self.compiler.f77, mpif77, **kwargs) filter_file('FC="%s"' % spack_fc , 'FC="%s"' % self.compiler.fc, mpif90, **kwargs) + + # Remove this linking flag if present + # (it turns RPATH into RUNPATH) + for wrapper in (mpicc, mpicxx, mpif77, mpif90): + filter_file('-Wl,--enable-new-dtags', '', wrapper, **kwargs) diff --git a/var/spack/repos/builtin/packages/mvapich2/package.py b/var/spack/repos/builtin/packages/mvapich2/package.py index d1944023d1..131d430cf2 100644 --- a/var/spack/repos/builtin/packages/mvapich2/package.py +++ b/var/spack/repos/builtin/packages/mvapich2/package.py @@ -23,7 +23,6 @@ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## from spack import * -import os class Mvapich2(Package): @@ -245,15 +244,15 @@ class Mvapich2(Package): be bound to whatever compiler they were built with. """ bin = self.prefix.bin - mpicc = os.path.join(bin, 'mpicc') - mpicxx = os.path.join(bin, 'mpicxx') - mpif77 = os.path.join(bin, 'mpif77') - mpif90 = os.path.join(bin, 'mpif90') + mpicc = join_path(bin, 'mpicc') + mpicxx = join_path(bin, 'mpicxx') + mpif77 = join_path(bin, 'mpif77') + mpif90 = join_path(bin, 'mpif90') - spack_cc = os.environ['CC'] - spack_cxx = os.environ['CXX'] - spack_f77 = os.environ['F77'] - spack_fc = os.environ['FC'] + spack_cc = env['CC'] + spack_cxx = env['CXX'] + spack_f77 = env['F77'] + spack_fc = env['FC'] kwargs = { 'ignore_absent': True, @@ -261,6 +260,8 @@ class Mvapich2(Package): 'string': True } + # Substitute Spack compile wrappers for the real + # underlying compiler filter_file('CC="%s"' % spack_cc, 'CC="%s"' % self.compiler.cc, mpicc, **kwargs) filter_file('CXX="%s"' % spack_cxx, @@ -269,3 +270,8 @@ class Mvapich2(Package): 'F77="%s"' % self.compiler.f77, mpif77, **kwargs) filter_file('FC="%s"' % spack_fc, 'FC="%s"' % self.compiler.fc, mpif90, **kwargs) + + # Remove this linking flag if present + # (it turns RPATH into RUNPATH) + for wrapper in (mpicc, mpicxx, mpif77, mpif90): + filter_file('-Wl,--enable-new-dtags', '', wrapper, **kwargs) diff --git a/var/spack/repos/builtin/packages/openmpi/package.py b/var/spack/repos/builtin/packages/openmpi/package.py index 4e465e1784..e3d9a28b4a 100644 --- a/var/spack/repos/builtin/packages/openmpi/package.py +++ b/var/spack/repos/builtin/packages/openmpi/package.py @@ -173,40 +173,33 @@ class Openmpi(Package): be bound to whatever compiler they were built with. """ kwargs = {'ignore_absent': True, 'backup': False, 'string': False} - dir = os.path.join(self.prefix, 'share/openmpi/') - - cc_wrappers = ['mpicc-vt-wrapper-data.txt', 'mpicc-wrapper-data.txt', - 'ortecc-wrapper-data.txt', 'shmemcc-wrapper-data.txt'] - - cxx_wrappers = ['mpic++-vt-wrapper-data.txt', 'mpic++-wrapper-data.txt', - 'ortec++-wrapper-data.txt'] - - fc_wrappers = ['mpifort-vt-wrapper-data.txt', - 'mpifort-wrapper-data.txt', 'shmemfort-wrapper-data.txt'] - - for wrapper in cc_wrappers: - filter_file('compiler=.*', 'compiler=%s' % self.compiler.cc, - os.path.join(dir, wrapper), **kwargs) - - for wrapper in cxx_wrappers: - filter_file('compiler=.*', 'compiler=%s' % self.compiler.cxx, - os.path.join(dir, wrapper), **kwargs) - - for wrapper in fc_wrappers: - filter_file('compiler=.*', 'compiler=%s' % self.compiler.fc, - os.path.join(dir, wrapper), **kwargs) - - # These are symlinks in newer versions, so check that here - f77_wrappers = ['mpif77-vt-wrapper-data.txt', 'mpif77-wrapper-data.txt'] - f90_wrappers = ['mpif90-vt-wrapper-data.txt', 'mpif90-wrapper-data.txt'] - - for wrapper in f77_wrappers: - path = os.path.join(dir, wrapper) - if not os.path.islink(path): - filter_file('compiler=.*', 'compiler=%s' % self.compiler.f77, - path, **kwargs) - for wrapper in f90_wrappers: - path = os.path.join(dir, wrapper) - if not os.path.islink(path): - filter_file('compiler=.*', 'compiler=%s' % self.compiler.fc, - path, **kwargs) + wrapper_basepath = join_path(self.prefix, 'share', 'openmpi') + + wrappers = [ + ('mpicc-vt-wrapper-data.txt', self.compiler.cc), + ('mpicc-wrapper-data.txt', self.compiler.cc), + ('ortecc-wrapper-data.txt', self.compiler.cc), + ('shmemcc-wrapper-data.txt', self.compiler.cc), + ('mpic++-vt-wrapper-data.txt', self.compiler.cxx), + ('mpic++-wrapper-data.txt', self.compiler.cxx), + ('ortec++-wrapper-data.txt', self.compiler.cxx), + ('mpifort-vt-wrapper-data.txt', self.compiler.fc), + ('mpifort-wrapper-data.txt', self.compiler.fc), + ('shmemfort-wrapper-data.txt', self.compiler.fc), + ('mpif90-vt-wrapper-data.txt', self.compiler.fc), + ('mpif90-wrapper-data.txt', self.compiler.fc), + ('mpif77-vt-wrapper-data.txt', self.compiler.f77), + ('mpif77-wrapper-data.txt', self.compiler.f77) + ] + + for wrapper_name, compiler in wrappers: + wrapper = join_path(wrapper_basepath, wrapper_name) + if not os.path.islink(wrapper): + # Substitute Spack compile wrappers for the real + # underlying compiler + match = 'compiler=.*' + substitute = 'compiler={compiler}'.format(compiler=compiler) + filter_file(match, substitute, wrapper, **kwargs) + # Remove this linking flag if present + # (it turns RPATH into RUNPATH) + filter_file('-Wl,--enable-new-dtags', '', wrapper, **kwargs) -- cgit v1.2.3-70-g09d2 From b09bee81584aa38e78596f3f7f8f4cc09a4418ca Mon Sep 17 00:00:00 2001 From: alalazo Date: Wed, 15 Jun 2016 13:29:30 +0200 Subject: qa : fixed flake8 issues --- var/spack/repos/builtin/packages/mpich/package.py | 32 +++++++++++--------- .../repos/builtin/packages/openmpi/package.py | 35 +++++++++++++--------- 2 files changed, 39 insertions(+), 28 deletions(-) (limited to 'var') diff --git a/var/spack/repos/builtin/packages/mpich/package.py b/var/spack/repos/builtin/packages/mpich/package.py index e7ae63ef70..511beafbbd 100644 --- a/var/spack/repos/builtin/packages/mpich/package.py +++ b/var/spack/repos/builtin/packages/mpich/package.py @@ -29,8 +29,8 @@ class Mpich(Package): """MPICH is a high performance and widely portable implementation of the Message Passing Interface (MPI) standard.""" homepage = "http://www.mpich.org" - url = "http://www.mpich.org/static/downloads/3.0.4/mpich-3.0.4.tar.gz" - list_url = "http://www.mpich.org/static/downloads/" + url = "http://www.mpich.org/static/downloads/3.0.4/mpich-3.0.4.tar.gz" + list_url = "http://www.mpich.org/static/downloads/" list_depth = 2 version('3.2', 'f414cfa77099cd1fa1a5ae4e22db508a') @@ -41,7 +41,8 @@ class Mpich(Package): version('3.1', '5643dd176499bfb7d25079aaff25f2ec') version('3.0.4', '9c5d5d4fe1e17dd12153f40bc5b6dbc0') - variant('verbs', default=False, description='Build support for OpenFabrics verbs.') + variant('verbs', default=False, + description='Build support for OpenFabrics verbs.') variant('pmi', default=True, description='Build with PMI support') variant('hydra', default=True, description='Build the hydra process manager') @@ -56,9 +57,9 @@ class Mpich(Package): spack_env.set('MPICH_FC', spack_fc) def setup_dependent_package(self, module, dep_spec): - self.spec.mpicc = join_path(self.prefix.bin, 'mpicc') + self.spec.mpicc = join_path(self.prefix.bin, 'mpicc') self.spec.mpicxx = join_path(self.prefix.bin, 'mpic++') - self.spec.mpifc = join_path(self.prefix.bin, 'mpif90') + self.spec.mpifc = join_path(self.prefix.bin, 'mpif90') self.spec.mpif77 = join_path(self.prefix.bin, 'mpif77') def install(self, spec, prefix): @@ -91,7 +92,6 @@ class Mpich(Package): self.filter_compilers() - def filter_compilers(self): """Run after install to make the MPI compilers use the compilers that Spack built the package with. @@ -101,23 +101,27 @@ class Mpich(Package): be bound to whatever compiler they were built with. """ bin = self.prefix.bin - mpicc = join_path(bin, 'mpicc') + mpicc = join_path(bin, 'mpicc') mpicxx = join_path(bin, 'mpicxx') mpif77 = join_path(bin, 'mpif77') mpif90 = join_path(bin, 'mpif90') - spack_cc = env['CC'] + spack_cc = env['CC'] spack_cxx = env['CXX'] spack_f77 = env['F77'] - spack_fc = env['FC'] + spack_fc = env['FC'] # Substitute Spack compile wrappers for the real # underlying compiler - kwargs = { 'ignore_absent' : True, 'backup' : False, 'string' : True } - filter_file('CC="%s"' % spack_cc , 'CC="%s"' % self.compiler.cc, mpicc, **kwargs) - filter_file('CXX="%s"'% spack_cxx, 'CXX="%s"' % self.compiler.cxx, mpicxx, **kwargs) - filter_file('F77="%s"'% spack_f77, 'F77="%s"' % self.compiler.f77, mpif77, **kwargs) - filter_file('FC="%s"' % spack_fc , 'FC="%s"' % self.compiler.fc, mpif90, **kwargs) + kwargs = {'ignore_absent': True, 'backup': False, 'string': True} + filter_file('CC="%s"' % spack_cc, + 'CC="%s"' % self.compiler.cc, mpicc, **kwargs) + filter_file('CXX="%s"' % spack_cxx, + 'CXX="%s"' % self.compiler.cxx, mpicxx, **kwargs) + filter_file('F77="%s"' % spack_f77, + 'F77="%s"' % self.compiler.f77, mpif77, **kwargs) + filter_file('FC="%s"' % spack_fc, + 'FC="%s"' % self.compiler.fc, mpif90, **kwargs) # Remove this linking flag if present # (it turns RPATH into RUNPATH) diff --git a/var/spack/repos/builtin/packages/openmpi/package.py b/var/spack/repos/builtin/packages/openmpi/package.py index e3d9a28b4a..ab71b4b503 100644 --- a/var/spack/repos/builtin/packages/openmpi/package.py +++ b/var/spack/repos/builtin/packages/openmpi/package.py @@ -72,20 +72,27 @@ class Openmpi(Package): patch('configure.patch', when="@1.10.0:1.10.1") variant('psm', default=False, description='Build support for the PSM library.') - variant('psm2', default=False, description='Build support for the Intel PSM2 library.') - variant('pmi', default=False, description='Build support for PMI-based launchers') - variant('verbs', default=_verbs_dir() is not None, description='Build support for OpenFabrics verbs.') + variant('psm2', default=False, + description='Build support for the Intel PSM2 library.') + variant('pmi', default=False, + description='Build support for PMI-based launchers') + variant('verbs', default=_verbs_dir() is not None, + description='Build support for OpenFabrics verbs.') variant('mxm', default=False, description='Build Mellanox Messaging support') - variant('thread_multiple', default=False, description='Enable MPI_THREAD_MULTIPLE support') + variant('thread_multiple', default=False, + description='Enable MPI_THREAD_MULTIPLE support') # TODO : variant support for alps, loadleveler is missing - variant('tm', default=False, description='Build TM (Torque, PBSPro, and compatible) support') - variant('slurm', default=False, description='Build SLURM scheduler component') + variant('tm', default=False, + description='Build TM (Torque, PBSPro, and compatible) support') + variant('slurm', default=False, + description='Build SLURM scheduler component') variant('sqlite3', default=False, description='Build sqlite3 support') - variant('vt', default=True, description='Build support for contributed package vt') + variant('vt', default=True, + description='Build support for contributed package vt') # TODO : support for CUDA is missing @@ -96,8 +103,7 @@ class Openmpi(Package): depends_on('sqlite', when='+sqlite3') def url_for_version(self, version): - return "http://www.open-mpi.org/software/ompi/v%s/downloads/openmpi-%s.tar.bz2" % (version.up_to(2), version) - + return "http://www.open-mpi.org/software/ompi/v%s/downloads/openmpi-%s.tar.bz2" % (version.up_to(2), version) # NOQA: ignore=E501 def setup_dependent_environment(self, spack_env, run_env, dependent_spec): spack_env.set('OMPI_CC', spack_cc) @@ -106,14 +112,15 @@ class Openmpi(Package): spack_env.set('OMPI_F77', spack_f77) def setup_dependent_package(self, module, dep_spec): - self.spec.mpicc = join_path(self.prefix.bin, 'mpicc') + self.spec.mpicc = join_path(self.prefix.bin, 'mpicc') self.spec.mpicxx = join_path(self.prefix.bin, 'mpic++') - self.spec.mpifc = join_path(self.prefix.bin, 'mpif90') + self.spec.mpifc = join_path(self.prefix.bin, 'mpif90') self.spec.mpif77 = join_path(self.prefix.bin, 'mpif77') @property def verbs(self): - # Up through version 1.6, this option was previously named --with-openib + # Up through version 1.6, this option was previously named + # --with-openib if self.spec.satisfies('@:1.6'): return 'openib' # In version 1.7, it was renamed to be --with-verbs @@ -135,7 +142,7 @@ class Openmpi(Package): '--with-psm2' if '+psm2' in spec else '--without-psm2', '--with-mxm' if '+mxm' in spec else '--without-mxm', # Other options - '--enable-mpi-thread-multiple' if '+thread_multiple' in spec else '--disable-mpi-thread-multiple', + '--enable-mpi-thread-multiple' if '+thread_multiple' in spec else '--disable-mpi-thread-multiple', # NOQA: ignore=E501 '--with-pmi' if '+pmi' in spec else '--without-pmi', '--with-sqlite3' if '+sqlite3' in spec else '--without-sqlite3', '--enable-vt' if '+vt' in spec else '--disable-vt' @@ -153,7 +160,7 @@ class Openmpi(Package): # use this for LANL builds, but for LLNL builds, we need: # "--with-platform=contrib/platform/llnl/optimized" if self.version == ver("1.6.5") and '+lanl' in spec: - config_args.append("--with-platform=contrib/platform/lanl/tlcc2/optimized-nopanasas") + config_args.append("--with-platform=contrib/platform/lanl/tlcc2/optimized-nopanasas") # NOQA: ignore=E501 if not self.compiler.f77 and not self.compiler.fc: config_args.append("--enable-mpi-fortran=no") -- cgit v1.2.3-70-g09d2 From daa46105a71f99946b26e1fa9836f0bb27897ceb Mon Sep 17 00:00:00 2001 From: Massimiliano Culpo Date: Wed, 15 Jun 2016 21:37:12 +0200 Subject: mpich based providers : shortened filter_compilers --- var/spack/repos/builtin/packages/mpich/package.py | 17 ++++----------- .../repos/builtin/packages/mvapich2/package.py | 24 +++++----------------- 2 files changed, 9 insertions(+), 32 deletions(-) (limited to 'var') diff --git a/var/spack/repos/builtin/packages/mpich/package.py b/var/spack/repos/builtin/packages/mpich/package.py index 511beafbbd..ea1844aea0 100644 --- a/var/spack/repos/builtin/packages/mpich/package.py +++ b/var/spack/repos/builtin/packages/mpich/package.py @@ -106,22 +106,13 @@ class Mpich(Package): mpif77 = join_path(bin, 'mpif77') mpif90 = join_path(bin, 'mpif90') - spack_cc = env['CC'] - spack_cxx = env['CXX'] - spack_f77 = env['F77'] - spack_fc = env['FC'] - # Substitute Spack compile wrappers for the real # underlying compiler kwargs = {'ignore_absent': True, 'backup': False, 'string': True} - filter_file('CC="%s"' % spack_cc, - 'CC="%s"' % self.compiler.cc, mpicc, **kwargs) - filter_file('CXX="%s"' % spack_cxx, - 'CXX="%s"' % self.compiler.cxx, mpicxx, **kwargs) - filter_file('F77="%s"' % spack_f77, - 'F77="%s"' % self.compiler.f77, mpif77, **kwargs) - filter_file('FC="%s"' % spack_fc, - 'FC="%s"' % self.compiler.fc, mpif90, **kwargs) + filter_file(env['CC'], self.compiler.cc, mpicc, **kwargs) + filter_file(env['CXX'], self.compiler.cxx, mpicxx, **kwargs) + filter_file(env['F77'], self.compiler.f77, mpif77, **kwargs) + filter_file(env['FC'], self.compiler.fc, mpif90, **kwargs) # Remove this linking flag if present # (it turns RPATH into RUNPATH) diff --git a/var/spack/repos/builtin/packages/mvapich2/package.py b/var/spack/repos/builtin/packages/mvapich2/package.py index 131d430cf2..34bace10d6 100644 --- a/var/spack/repos/builtin/packages/mvapich2/package.py +++ b/var/spack/repos/builtin/packages/mvapich2/package.py @@ -249,27 +249,13 @@ class Mvapich2(Package): mpif77 = join_path(bin, 'mpif77') mpif90 = join_path(bin, 'mpif90') - spack_cc = env['CC'] - spack_cxx = env['CXX'] - spack_f77 = env['F77'] - spack_fc = env['FC'] - - kwargs = { - 'ignore_absent': True, - 'backup': False, - 'string': True - } - # Substitute Spack compile wrappers for the real # underlying compiler - filter_file('CC="%s"' % spack_cc, - 'CC="%s"' % self.compiler.cc, mpicc, **kwargs) - filter_file('CXX="%s"' % spack_cxx, - 'CXX="%s"' % self.compiler.cxx, mpicxx, **kwargs) - filter_file('F77="%s"' % spack_f77, - 'F77="%s"' % self.compiler.f77, mpif77, **kwargs) - filter_file('FC="%s"' % spack_fc, - 'FC="%s"' % self.compiler.fc, mpif90, **kwargs) + kwargs = {'ignore_absent': True, 'backup': False, 'string': True} + filter_file(env['CC'], self.compiler.cc, mpicc, **kwargs) + filter_file(env['CXX'], self.compiler.cxx, mpicxx, **kwargs) + filter_file(env['F77'], self.compiler.f77, mpif77, **kwargs) + filter_file(env['FC'], self.compiler.fc, mpif90, **kwargs) # Remove this linking flag if present # (it turns RPATH into RUNPATH) -- cgit v1.2.3-70-g09d2