Axom: bring in changes from axom repo (#32643)

* bring in changes from axom repo

Co-authored-by: white238 <white238@users.noreply.github.com>
Co-authored-by: Tamara Dahlgren <35777542+tldahlgren@users.noreply.github.com>

1 files changed, 139 insertions, 61 deletions

diff --git a/var/spack/repos/builtin/packages/axom/package.py b/var/spack/repos/builtin/packages/axom/package.py
index a63118e2c7..d1cb7f1069 100644
--- a/var/spack/repos/builtin/packages/axom/package.py
+++ b/var/spack/repos/builtin/packages/axom/package.py
@@ -28,7 +28,7 @@ def get_spec_path(spec, package_name, path_replacements={}, use_bin=False):
     return path
 
 
-class Axom(CachedCMakePackage, CudaPackage):
+class Axom(CachedCMakePackage, CudaPackage, ROCmPackage):
     """Axom provides a robust, flexible software infrastructure for the development
     of multi-physics applications and computational tools."""
 
@@ -40,6 +40,7 @@ class Axom(CachedCMakePackage, CudaPackage):
 
     version("main", branch="main")
     version("develop", branch="develop")
+    version("0.7.0", tag="v0.7.0")
     version("0.6.1", tag="v0.6.1")
     version("0.6.0", tag="v0.6.0")
     version("0.5.0", tag="v0.5.0")
@@ -50,10 +51,6 @@ class Axom(CachedCMakePackage, CudaPackage):
     version("0.3.0", tag="v0.3.0")
     version("0.2.9", tag="v0.2.9")
 
-    @property
-    def submodules(self):
-        return True
-
     # https://github.com/spack/spack/issues/31829
     patch("examples-oneapi.patch", when="@0.6.1 +examples %oneapi")
 
@@ -65,11 +62,11 @@ class Axom(CachedCMakePackage, CudaPackage):
     # Variants
     # -----------------------------------------------------------------------
     variant("shared", default=True, description="Enable build of shared libraries")
-    variant("debug", default=False, description="Build debug instead of optimized version")
 
     variant("examples", default=True, description="Build examples")
     variant("tools", default=True, description="Build tools")
 
+    # Hard requirement after Axom 0.6.1
     variant("cpp14", default=True, description="Build with C++14 support")
 
     variant("fortran", default=True, description="Build with Fortran support")
@@ -95,36 +92,48 @@ class Axom(CachedCMakePackage, CudaPackage):
     # -----------------------------------------------------------------------
     # Basics
     depends_on("cmake@3.8.2:", type="build")
+    depends_on("cmake@3.16.8:", type="build", when="+rocm")
+
+    depends_on("blt", type="build")
+    depends_on("blt@0.5.1:", type="build", when="@0.6.2:")
+
     depends_on("mpi", when="+mpi")
 
     # Libraries
-    depends_on("conduit+python", when="+python")
-    depends_on("conduit~python", when="~python")
-    depends_on("conduit+hdf5", when="+hdf5")
-    depends_on("conduit~hdf5", when="~hdf5")
+    # Forward variants to Conduit
+    for _var in ["fortran", "hdf5", "mpi", "python"]:
+        depends_on("conduit+{0}".format(_var), when="+{0}".format(_var))
+        depends_on("conduit~{0}".format(_var), when="~{0}".format(_var))
+
+    depends_on("hdf5", when="+hdf5")
 
     depends_on("lua", when="+lua")
 
     depends_on("scr", when="+scr")
-    depends_on("kvtree@main", when="+scr")
-    depends_on("dtcmp", when="+scr")
+    depends_on("scr~fortran", when="+scr~fortran")
 
     with when("+umpire"):
-        depends_on("umpire@6.0.0:", when="@0.6.0:")
+        depends_on("umpire@2022.03.0:", when="@0.7.0:")
+        depends_on("umpire@6.0.0", when="@0.6.0")
         depends_on("umpire@5:5.0.1", when="@:0.5.0")
         depends_on("umpire +openmp", when="+openmp")
         depends_on("umpire +cuda", when="+cuda")
 
     with when("+raja"):
-        depends_on("raja@0.14.0:", when="@0.6.0:")
+        depends_on("raja@2022.03.0:", when="@0.7.0:")
+        depends_on("raja@0.14.0", when="@0.6.0")
         depends_on("raja@:0.13.0", when="@:0.5.0")
         depends_on("raja~openmp", when="~openmp")
         depends_on("raja+openmp", when="+openmp")
         depends_on("raja+cuda", when="+cuda")
 
-    for sm_ in CudaPackage.cuda_arch_values:
-        depends_on("raja cuda_arch={0}".format(sm_), when="+raja cuda_arch={0}".format(sm_))
-        depends_on("umpire cuda_arch={0}".format(sm_), when="+umpire cuda_arch={0}".format(sm_))
+    for val in CudaPackage.cuda_arch_values:
+        depends_on("raja cuda_arch={0}".format(val), when="+raja cuda_arch={0}".format(val))
+        depends_on("umpire cuda_arch={0}".format(val), when="+umpire cuda_arch={0}".format(val))
+
+    for val in ROCmPackage.amdgpu_targets:
+        depends_on("raja amdgpu_target={0}".format(val), when="amdgpu_target={0}".format(val))
+        depends_on("umpire amdgpu_target={0}".format(val), when="amdgpu_target={0}".format(val))
 
     depends_on("mfem", when="+mfem")
     depends_on("mfem~mpi", when="+mfem~mpi")
@@ -140,12 +149,18 @@ class Axom(CachedCMakePackage, CudaPackage):
     depends_on("py-shroud", when="+devtools")
     depends_on("llvm+clang@10.0.0", when="+devtools", type="build")
 
+    # Hard requirement after Axom 0.6.1
+    conflicts("~cpp14", when="@0.6.2:")
+
     # Conduit's cmake config files moved and < 0.4.0 can't find it
     conflicts("^conduit@0.7.2:", when="@:0.4.0")
 
     # Sidre requires conduit_blueprint_mpi.hpp
     conflicts("^conduit@:0.6.0", when="@0.5.0:")
 
+    conflicts("+openmp", when="+rocm")
+    conflicts("+cuda", when="+rocm")
+
     def flag_handler(self, name, flags):
         if self.spec.satisfies("%cce") and name == "fflags":
             flags.append("-ef")
@@ -161,43 +176,51 @@ class Axom(CachedCMakePackage, CudaPackage):
             sys_type = env["SYS_TYPE"]
         return sys_type
 
+    def is_fortran_compiler(self, compiler):
+        if self.compiler.fc is not None and compiler in self.compiler.fc:
+            return True
+        return False
+
     @property
     def cache_name(self):
         hostname = socket.gethostname()
         if "SYS_TYPE" in env:
             # Are we on a LLNL system then strip node number
             hostname = hostname.rstrip("1234567890")
-        return "{0}-{1}-{2}@{3}.cmake".format(
+        special_case = ""
+        if "+cuda" in self.spec:
+            special_case += "_cuda"
+        if "~fortran" in self.spec:
+            special_case += "_nofortran"
+        if "+rocm" in self.spec:
+            special_case += "_hip"
+        return "{0}-{1}-{2}@{3}{4}.cmake".format(
             hostname,
             self._get_sys_type(self.spec),
             self.spec.compiler.name,
             self.spec.compiler.version,
+            special_case,
         )
 
     def initconfig_compiler_entries(self):
         spec = self.spec
         entries = super(Axom, self).initconfig_compiler_entries()
 
-        if "+fortran" in spec or self.compiler.fc is not None:
+        if "+fortran" in spec:
             entries.append(cmake_cache_option("ENABLE_FORTRAN", True))
+            if self.is_fortran_compiler("gfortran") and "clang" in self.compiler.cxx:
+                libdir = pjoin(os.path.dirname(os.path.dirname(self.compiler.cxx)), "lib")
+                flags = ""
+                for _libpath in [libdir, libdir + "64"]:
+                    if os.path.exists(_libpath):
+                        flags += " -Wl,-rpath,{0}".format(_libpath)
+                description = "Adds a missing libstdc++ rpath"
+                if flags:
+                    entries.append(cmake_cache_string("BLT_EXE_LINKER_FLAGS", flags, description))
         else:
             entries.append(cmake_cache_option("ENABLE_FORTRAN", False))
 
-        if (
-            (self.compiler.fc is not None)
-            and ("gfortran" in self.compiler.fc)
-            and ("clang" in self.compiler.cxx)
-        ):
-            libdir = pjoin(os.path.dirname(os.path.dirname(self.compiler.cxx)), "lib")
-            flags = ""
-            for _libpath in [libdir, libdir + "64"]:
-                if os.path.exists(_libpath):
-                    flags += " -Wl,-rpath,{0}".format(_libpath)
-            description = "Adds a missing libstdc++ rpath"
-            if flags:
-                entries.append(cmake_cache_string("BLT_EXE_LINKER_FLAGS", flags, description))
-
-        if "+cpp14" in spec:
+        if "+cpp14" in spec and spec.satisfies("@:0.6.1"):
             entries.append(cmake_cache_string("BLT_CXX_STD", "c++14", ""))
 
         return entries
@@ -222,15 +245,50 @@ class Axom(CachedCMakePackage, CudaPackage):
             else:
                 entries.append("# cuda_arch could not be determined\n\n")
 
-            if "+cpp14" in spec:
-                cudaflags += " -std=c++14"
-            else:
-                cudaflags += " -std=c++11"
+            if spec.satisfies("^blt@:0.6.1"):
+                # This is handled internally by BLT now
+                if "+cpp14" in spec:
+                    cudaflags += " -std=c++14"
+                else:
+                    cudaflags += " -std=c++11"
             entries.append(cmake_cache_string("CMAKE_CUDA_FLAGS", cudaflags))
 
             entries.append("# nvcc does not like gtest's 'pthreads' flag\n")
             entries.append(cmake_cache_option("gtest_disable_pthreads", True))
 
+        if "+rocm" in spec:
+            entries.append("#------------------{0}\n".format("-" * 60))
+            entries.append("# HIP\n")
+            entries.append("#------------------{0}\n\n".format("-" * 60))
+
+            entries.append(cmake_cache_option("ENABLE_HIP", True))
+
+            hip_root = spec["hip"].prefix
+            rocm_root = hip_root + "/.."
+
+            entries.append(cmake_cache_string("HIP_ROOT_DIR", hip_root))
+            entries.append(cmake_cache_string("HIP_CLANG_PATH", rocm_root + "/llvm/bin"))
+
+            archs = self.spec.variants["amdgpu_target"].value
+            if archs != "none":
+                arch_str = ",".join(archs)
+                entries.append(cmake_cache_string("CMAKE_HIP_ARCHITECTURES", arch_str))
+
+            # Fixes for mpi for rocm until wrapper paths are fixed
+            # These flags are already part of the wrapped compilers on TOSS4 systems
+            hip_link_flags = ""
+            if "+fortran" in spec and self.is_fortran_compiler("amdflang"):
+                hip_link_flags += "-Wl,--disable-new-dtags "
+                hip_link_flags += "-L{0}/../llvm/lib -L{0}/lib ".format(hip_root)
+                hip_link_flags += "-Wl,-rpath,{0}/../llvm/lib:{0}/lib ".format(hip_root)
+                hip_link_flags += "-lpgmath -lflang -lflangrti -lompstub -lamdhip64 "
+
+            # Additional libraries for TOSS4
+            hip_link_flags += " -L{0}/../lib64 -Wl,-rpath,{0}/../lib64 ".format(hip_root)
+            hip_link_flags += "-lhsakmt -lamd_comgr "
+
+            entries.append(cmake_cache_string("CMAKE_EXE_LINKER_FLAGS", hip_link_flags))
+
         entries.append("#------------------{0}".format("-" * 30))
         entries.append("# Hardware Specifics")
         entries.append("#------------------{0}\n".format("-" * 30))
@@ -245,7 +303,7 @@ class Axom(CachedCMakePackage, CudaPackage):
             )
         )
 
-        if (self.compiler.fc is not None) and ("xlf" in self.compiler.fc):
+        if "+fortran" in spec and self.is_fortran_compiler("xlf"):
             # Grab lib directory for the current fortran compiler
             libdir = pjoin(os.path.dirname(os.path.dirname(self.compiler.fc)), "lib")
             description = (
@@ -269,25 +327,25 @@ class Axom(CachedCMakePackage, CudaPackage):
                 )
             )
 
-            if spec.satisfies("target=ppc64le:"):
-                # Fix for working around CMake adding implicit link directories
-                # returned by the BlueOS compilers to link executables with
-                # non-system default stdlib
-                _roots = ["/usr/tce/packages/gcc/gcc-4.9.3", "/usr/tce/packages/gcc/gcc-4.9.3/gnu"]
-                _subdirs = ["lib64", "lib64/gcc/powerpc64le-unknown-linux-gnu/4.9.3"]
-                _existing_paths = []
-                for root in _roots:
-                    for subdir in _subdirs:
-                        _curr_path = pjoin(root, subdir)
-                        if os.path.exists(_curr_path):
-                            _existing_paths.append(_curr_path)
-                if _existing_paths:
-                    entries.append(
-                        cmake_cache_string(
-                            "BLT_CMAKE_IMPLICIT_LINK_DIRECTORIES_EXCLUDE",
-                            ";".join(_existing_paths),
-                        )
+        if spec.satisfies("target=ppc64le:"):
+            # Fix for working around CMake adding implicit link directories
+            # returned by the BlueOS compilers to link executables with
+            # non-system default stdlib
+            _roots = ["/usr/tce/packages/gcc/gcc-4.9.3", "/usr/tce/packages/gcc/gcc-4.9.3/gnu"]
+            _subdirs = ["lib64", "lib64/gcc/powerpc64le-unknown-linux-gnu/4.9.3"]
+            _existing_paths = []
+            for root in _roots:
+                for subdir in _subdirs:
+                    _curr_path = pjoin(root, subdir)
+                    if os.path.exists(_curr_path):
+                        _existing_paths.append(_curr_path)
+            if _existing_paths:
+                entries.append(
+                    cmake_cache_string(
+                        "BLT_CMAKE_IMPLICIT_LINK_DIRECTORIES_EXCLUDE",
+                        ";".join(_existing_paths),
                     )
+                )
 
         return entries
 
@@ -336,12 +394,29 @@ class Axom(CachedCMakePackage, CudaPackage):
             else:
                 entries.append("# %s not built\n" % dep.upper())
 
+        if "+umpire" in spec and spec.satisfies("^camp"):
+            dep_dir = get_spec_path(spec, "camp", path_replacements)
+            entries.append(cmake_cache_path("CAMP_DIR", dep_dir))
+
+        # SCR does not export it's targets so we need to pull in its dependencies
         if "+scr" in spec:
             dep_dir = get_spec_path(spec, "scr", path_replacements)
             entries.append(cmake_cache_path("SCR_DIR", dep_dir))
 
             # scr's dependencies
-            for dep in ("kvtree", "dtcmp"):
+            scr_deps = (
+                "kvtree",
+                "dtcmp",
+                "spath",
+                "axl",
+                "lwgrp",
+                "er",
+                "rankstr",
+                "redset",
+                "shuffile",
+                "libyogrt",
+            )
+            for dep in scr_deps:
                 if spec.satisfies("^{0}".format(dep)):
                     dep_dir = get_spec_path(spec, dep, path_replacements)
                     entries.append(cmake_cache_path("%s_DIR" % dep.upper(), dep_dir))
@@ -366,7 +441,8 @@ class Axom(CachedCMakePackage, CudaPackage):
             entries.append("# Root directory for generated developer tools\n")
             entries.append(cmake_cache_path("DEVTOOLS_ROOT", devtools_root))
 
-            # Only turn on clangformat support if devtools is on
+        if "+devtools" in spec and "toss_4" not in self._get_sys_type(spec):
+            # Only turn on clangformat support if devtools is on and not TOSS4
             clang_fmt_path = spec["llvm"].prefix.bin.join("clang-format")
             entries.append(cmake_cache_path("CLANGFORMAT_EXECUTABLE", clang_fmt_path))
         else:
@@ -383,9 +459,9 @@ class Axom(CachedCMakePackage, CudaPackage):
         entries.append(cmake_cache_option("ENABLE_DOCS", enable_docs))
 
         if spec.satisfies("^py-sphinx"):
-            python_bin_dir = get_spec_path(spec, "python", path_replacements, use_bin=True)
+            sphinx_bin_dir = get_spec_path(spec, "py-sphinx", path_replacements, use_bin=True)
             entries.append(
-                cmake_cache_path("SPHINX_EXECUTABLE", pjoin(python_bin_dir, "sphinx-build"))
+                cmake_cache_path("SPHINX_EXECUTABLE", pjoin(sphinx_bin_dir, "sphinx-build"))
             )
 
         if spec.satisfies("^py-shroud"):
@@ -404,6 +480,8 @@ class Axom(CachedCMakePackage, CudaPackage):
     def cmake_args(self):
         options = []
 
+        options.append("-DBLT_SOURCE_DIR:PATH={0}".format(self.spec["blt"].prefix))
+
         if self.run_tests is False:
             options.append("-DENABLE_TESTS=OFF")
         else: