Charm++: fix build prefix; add ucx/pmi support (#15666)

* Add pmi support (required by ucx, ofi, and gni backends)
* Add support for ucx backend
* Add dependency on MPI for pmi=simplepmi, slurmpmi, or slurmpmi2
* Remove charmpp as an MPI provider since the changes in this PR can
  add MPI as a dependency (mentioned previously)
* Install into transport_protocol-OS-arch subdirectory to match
  default charmpp installation behavior (which helps dependents find it)

2 files changed, 124 insertions, 42 deletions

diff --git a/var/spack/repos/builtin/packages/charmpp/package.py b/var/spack/repos/builtin/packages/charmpp/package.py
index 158dd59937..e9a133604c 100644
--- a/var/spack/repos/builtin/packages/charmpp/package.py
+++ b/var/spack/repos/builtin/packages/charmpp/package.py
@@ -58,10 +58,18 @@ class Charmpp(Package):
         "backend",
         default="netlrts",
         values=("mpi", "multicore", "netlrts", "verbs", "gni",
-                "ofi", "pami", "pamilrts"),
+                "ucx", "ofi", "pami", "pamilrts"),
         description="Set the backend to use"
     )
 
+    # Process management interface
+    variant(
+        "pmi",
+        default="none",
+        values=("none", "simplepmi", "slurmpmi", "slurmpmi2", "pmix"),
+        description="The ucx/ofi/gni backends need PMI to run!"
+    )
+
     # Other options
     variant("papi", default=False, description="Enable PAPI integration")
     variant("syncft", default=False, description="Compile with Charm++ fault tolerance support")
@@ -78,29 +86,23 @@ class Charmpp(Package):
     variant("production", default=True, description="Build charm++ with all optimizations")
     variant("tracing", default=False, description="Enable tracing modules")
 
-    # FIXME: backend=mpi also provides mpi, but spack does not support
-    # depends_on("mpi") and provides("mpi") in the same package currently.
-    for b in ['multicore', 'netlrts', 'verbs', 'gni', 'ofi', 'pami',
-              'pamilrts']:
-        provides('mpi@2', when='@6.7.1: build-target=AMPI backend={0}'.format(b))
-        provides('mpi@2', when='@6.7.1: build-target=LIBS backend={0}'.format(b))
-
-    def setup_dependent_build_environment(self, env, dependent_spec):
-        env.set('MPICC',  self.prefix.bin.ampicc)
-        env.set('MPICXX', self.prefix.bin.ampicxx)
-        env.set('MPIF77', self.prefix.bin.ampif77)
-        env.set('MPIF90', self.prefix.bin.ampif90)
-
-    def setup_dependent_package(self, module, dependent_spec):
-        self.spec.mpicc  = self.prefix.bin.ampicc
-        self.spec.mpicxx = self.prefix.bin.ampicxx
-        self.spec.mpifc  = self.prefix.bin.ampif90
-        self.spec.mpif77 = self.prefix.bin.ampif77
-
     depends_on("mpi", when="backend=mpi")
     depends_on("papi", when="+papi")
     depends_on("cuda", when="+cuda")
 
+    depends_on("ucx", when="backend=ucx")
+    depends_on("slurm@:17-11-9-2", when="pmi=slurmpmi")
+    depends_on("slurm@17-11-9-2:", when="pmi=slurmpmi2")
+
+    # FIXME : As of now spack's OpenMPI recipe does not have a PMIx variant
+    # But if users have external installs of OpenMPI with PMIx support, this
+    # will allow them to build charm++ with it.
+    depends_on("openmpi", when="pmi=pmix")
+
+    depends_on("mpi", when="pmi=simplepmi")
+    depends_on("mpi", when="pmi=slurmpmi")
+    depends_on("mpi", when="pmi=slurmpmi2")
+
     # Git versions of Charm++ require automake and autoconf
     depends_on("automake", when="@develop")
     depends_on("autoconf", when="@develop")
@@ -108,11 +110,12 @@ class Charmpp(Package):
     conflicts("~tracing", "+papi")
 
     conflicts("backend=multicore", "+smp")
+    conflicts("backend=ucx", when="@:6.9.99")
 
-    def install(self, spec, prefix):
-        target = spec.variants["build-target"].value
-
+    @property
+    def charmarch(self):
         plat = sys.platform
+
         if plat.startswith("linux"):
             plat = "linux"
         elif plat.startswith("win"):
@@ -123,12 +126,13 @@ class Charmpp(Package):
             plat = "cnk"
 
         mach = platform.machine()
+
         if mach.startswith("ppc"):
             mach = "ppc"
         elif mach.startswith("arm"):
             mach = "arm"
 
-        comm = spec.variants['backend'].value
+        comm = self.spec.variants['backend'].value
 
         # Define Charm++ version names for various (plat, mach, comm)
         # combinations. Note that not all combinations are supported.
@@ -145,6 +149,7 @@ class Charmpp(Package):
             ("linux",   "x86_64",   "netlrts"):     "netlrts-linux-x86_64",
             ("linux",   "x86_64",   "verbs"):       "verbs-linux-x86_64",
             ("linux",   "x86_64",   "ofi"):         "ofi-linux-x86_64",
+            ("linux",   "x86_64",   "ucx"):         "ucx-linux-x86_64",
             ("linux",   "x86_64",   "uth"):         "uth-linux-x86_64",
             ("linux",   "ppc",      "mpi"):         "mpi-linux-ppc",
             ("linux",   "ppc",      "multicore"):   "multicore-linux-ppc",
@@ -167,7 +172,47 @@ class Charmpp(Package):
                 "The communication mechanism %s is not supported "
                 "on a %s platform with a %s CPU" %
                 (comm, plat, mach))
-        version = versions[(plat, mach, comm)]
+
+        return versions[(plat, mach, comm)]
+
+    # FIXME: backend=mpi also provides mpi, but spack does not support
+    # depends_on("mpi") and provides("mpi") in the same package currently.
+    # for b in ['multicore', 'netlrts', 'verbs', 'gni', 'ofi', 'pami',
+    #          'pamilrts']:
+    #    provides('mpi@2', when='@6.7.1:
+    #            build-target=AMPI backend={0}'.format(b))
+    #    provides('mpi@2', when='@6.7.1:
+    #            build-target=LIBS backend={0}'.format(b))
+
+    def install(self, spec, prefix):
+
+        if not("backend=mpi" in self.spec) or \
+           not("backend=netlrts" in self.spec):
+            if ("+pthreads" in self.spec):
+                raise InstallError("The pthreads option is only\
+                                    available on the Netlrts and MPI \
+                                    network layers.")
+
+        if ("backend=ucx" in self.spec) or \
+           ("backend=ofi" in self.spec) or \
+           ("backend=gni" in self.spec):
+            if ("pmi=none" in self.spec):
+                raise InstallError("The UCX/OFI/GNI backends need \
+                                    PMI to run. Please add pmi=... \
+                                    Note that PMIx is the preferred \
+                                    option.")
+
+        if ("pmi=simplepmi" in self.spec) or \
+           ("pmi=slurmpmi" in self.spec) or \
+           ("pmi=slurmpmi2" in self.spec):
+            if ("^openmpi" in self.spec):
+                raise InstallError("To use any process management \
+                                    interface other than PMIx, \
+                                    a non OpenMPI based MPI must be \
+                                    present on the system")
+
+        target = spec.variants["build-target"].value
+        builddir = prefix + "/" + str(self.charmarch)
 
         # We assume that Spack's compiler wrappers make this work. If
         # not, then we need to query the compiler vendor from Spack
@@ -176,9 +221,17 @@ class Charmpp(Package):
             os.path.basename(self.compiler.cc),
             os.path.basename(self.compiler.fc),
             "-j%d" % make_jobs,
-            "--destination=%s" % prefix,
+            "--destination=%s" % builddir,
         ]
 
+        if "pmi=slurmpmi" in spec:
+            options.append("slurmpmi")
+        if "pmi=slurmpmi2" in spec:
+            options.append("slurmpmi2")
+        if "pmi=pmix" in spec:
+            options.append("ompipmix")
+            options.extend(["--basedir=%s" % spec["openmpi"].prefix])
+
         if 'backend=mpi' in spec:
             # in intelmpi <prefix>/include and <prefix>/lib fails so --basedir
             # cannot be used
@@ -190,6 +243,9 @@ class Charmpp(Package):
                 '--libdir={0}'.format(libdir)
                 for libdir in spec["mpi"].libs.directories
             ])
+
+        if "backend=ucx" in spec:
+            options.extend(["--basedir=%s" % spec["ucx"].prefix])
         if "+papi" in spec:
             options.extend(["papi", "--basedir=%s" % spec["papi"].prefix])
         if "+smp" in spec:
@@ -223,11 +279,11 @@ class Charmpp(Package):
         # could dissect the build script; the build instructions say
         # this wouldn't be difficult.
         build = Executable(join_path(".", "build"))
-        build(target, version, *options)
+        build(target, self.charmarch, *options)
 
         # Charm++'s install script does not copy files, it only creates
         # symbolic links. Fix this.
-        for dirpath, dirnames, filenames in os.walk(prefix):
+        for dirpath, dirnames, filenames in os.walk(builddir):
             for filename in filenames:
                 filepath = join_path(dirpath, filename)
                 if os.path.islink(filepath):
@@ -239,18 +295,32 @@ class Charmpp(Package):
                         os.rename(tmppath, filepath)
                     except (IOError, OSError):
                         pass
-        shutil.rmtree(join_path(prefix, "tmp"))
+        shutil.rmtree(join_path(builddir, "tmp"))
 
-        # A broken 'doc' link in the prefix can break the build.
-        # Remove it and replace it if it is broken.
-        try:
-            os.stat(prefix.doc)
-        except OSError:
-            os.remove(prefix.doc)
-            mkdirp(prefix.doc)
+        if self.spec.satisfies('@6.9.99'):
+            # A broken 'doc' link in the prefix can break the build.
+            # Remove it and replace it if it is broken.
+            try:
+                os.stat(prefix.doc)
+            except OSError:
+                os.remove(prefix.doc)
+                mkdirp(prefix.doc)
 
     @run_after('install')
     @on_package_attributes(run_tests=True)
     def check_build(self):
         make('-C', join_path(self.stage.source_path, 'charm/tests'),
              'test', parallel=False)
+
+    def setup_dependent_build_environment(self, env, dependent_spec):
+        env.set('MPICC',  self.prefix.bin.ampicc)
+        env.set('MPICXX', self.prefix.bin.ampicxx)
+        env.set('MPIF77', self.prefix.bin.ampif77)
+        env.set('MPIF90', self.prefix.bin.ampif90)
+
+    def setup_dependent_package(self, module, dependent_spec):
+        self.spec.mpicc     = self.prefix.bin.ampicc
+        self.spec.mpicxx    = self.prefix.bin.ampicxx
+        self.spec.mpifc     = self.prefix.bin.ampif90
+        self.spec.mpif77    = self.prefix.bin.ampif77
+        self.spec.charmarch = self.charmarch
diff --git a/var/spack/repos/builtin/packages/namd/package.py b/var/spack/repos/builtin/packages/namd/package.py
index dcd80e94a8..d6f5b11b4a 100644
--- a/var/spack/repos/builtin/packages/namd/package.py
+++ b/var/spack/repos/builtin/packages/namd/package.py
@@ -15,8 +15,11 @@ class Namd(MakefilePackage):
 
     homepage = "http://www.ks.uiuc.edu/Research/namd/"
     url      = "file://{0}/NAMD_2.12_Source.tar.gz".format(os.getcwd())
+    git      = "https://charm.cs.illinois.edu/gerrit/namd.git"
     manual_download = True
 
+    version("develop", branch="master")
+    version('2.14b1', sha256='9407e54f5271b3d3039a5a9d2eae63c7e108ce31b7481e2197c19e1125b43919')
     version('2.13', '9e3323ed856e36e34d5c17a7b0341e38')
     version('2.12', '2a1191909b1ab03bf0205971ad4d8ee9')
 
@@ -26,7 +29,9 @@ class Namd(MakefilePackage):
     variant('interface', default='none', values=('none', 'tcl', 'python'),
             description='Enables TCL and/or python interface')
 
-    depends_on('charmpp')
+    depends_on('charmpp@6.10.1:', when="@2.14b1:")
+    depends_on('charmpp@6.8.2', when="@2.13")
+    depends_on('charmpp@6.7.1', when="@2.12")
 
     depends_on('fftw@:2.99', when="fftw=2")
     depends_on('fftw@3:', when="fftw=3")
@@ -73,17 +78,24 @@ class Namd(MakefilePackage):
             with open('{0}.arch'.format(self.build_directory), 'w') as fh:
                 # this options are take from the default provided
                 # configuration files
-                optims_opts = {
-                    'gcc': '-m64 -O3 -fexpensive-optimizations -ffast-math',
-                    'intel': '-O2 -ip'
-                }
+                # https://github.com/UIUC-PPL/charm/pull/2778
+                if self.spec.satisfies('^charmpp@:6.10.1'):
+                    optims_opts = {
+                        'gcc': '-m64 -O3 -fexpensive-optimizations \
+                                -ffast-math -lpthread',
+                        'intel': '-O2 -ip'}
+                else:
+                    optims_opts = {
+                        'gcc': '-m64 -O3 -fexpensive-optimizations \
+                                -ffast-math',
+                        'intel': '-O2 -ip'}
 
                 optim_opts = optims_opts[self.compiler.name] \
                     if self.compiler.name in optims_opts else ''
 
                 fh.write('\n'.join([
                     'NAMD_ARCH = {0}'.format(self.arch),
-                    'CHARMARCH = ',
+                    'CHARMARCH = {0}'.format(self.spec['charmpp'].charmarch),
                     'CXX = {0.cxx} {0.cxx11_flag}'.format(
                         self.compiler),
                     'CXXOPTS = {0}'.format(optim_opts),