ExaGO: add v1.4.0 and v1.4.1 (#29712)

* ExaGO: Handling of cuda architectures and amdgpu targets changed
  to effectively handle multiple targets. See #28441.
* Add ROCm support to ExaGO and update ROCm support in HiOp
* ExaGO+rocm requires HiOp+rocm
* Newer versions of CMake may set HIP_CLANG_INCLUDE_PATH incorrectly:
  add comments to the ExaGO/HiOp packages explaining how to address
  this problem if it occurs.

1 files changed, 44 insertions, 16 deletions

diff --git a/var/spack/repos/builtin/packages/exago/package.py b/var/spack/repos/builtin/packages/exago/package.py
index 9046748573..037c26bf07 100644
--- a/var/spack/repos/builtin/packages/exago/package.py
+++ b/var/spack/repos/builtin/packages/exago/package.py
@@ -6,7 +6,7 @@
 from spack import *
 
 
-class Exago(CMakePackage, CudaPackage):
+class Exago(CMakePackage, CudaPackage, ROCmPackage):
     """ExaGO is a package for solving large-scale power grid optimization
     problems on parallel and distributed architectures, particularly targeted
     for exascale machines."""
@@ -15,7 +15,9 @@ class Exago(CMakePackage, CudaPackage):
     git = 'https://gitlab.pnnl.gov/exasgd/frameworks/exago.git'
     maintainers = ['ashermancinelli', 'CameronRutherford']
 
-    version('1.3.0', commit='58b039d746a6eac8e84b0afc01354cd58caec485', submodules=True, preferred=True)
+    version('1.4.1', commit='ea607c685444b5f345bfdc9a59c345f0f30adde2', submodules=True, preferred=True)
+    version('1.4.0', commit='4f4c3fdb40b52ace2d6ba000e7f24b340ec8e886', submodules=True)
+    version('1.3.0', commit='58b039d746a6eac8e84b0afc01354cd58caec485', submodules=True)
     version('1.2.0', commit='255a214e', submodules=True)
     version('1.1.2', commit='db3bb16e', submodules=True)
     version('1.1.1', commit='0e0a3f27', submodules=True)
@@ -30,6 +32,7 @@ class Exago(CMakePackage, CudaPackage):
     variant('mpi', default=True, description='Enable/Disable MPI')
     variant('raja', default=False, description='Enable/Disable RAJA')
     variant('python', default=True, description='Enable/Disable Python bindings')
+    conflicts('+python', when='@:1.3.0', msg='Python bindings require ExaGO 1.4')
 
     # Solver options
     variant('hiop', default=False, description='Enable/Disable HiOp')
@@ -44,6 +47,7 @@ class Exago(CMakePackage, CudaPackage):
     depends_on('raja', when='+raja')
 
     depends_on('raja+cuda', when='+raja+cuda')
+    depends_on('raja+rocm', when='+raja+rocm')
     depends_on('raja@0.14.0:', when='@1.1.0: +raja')
     depends_on('umpire', when='+raja')
     depends_on('umpire@6.0.0:', when='@1.1.0: +raja')
@@ -65,37 +69,61 @@ class Exago(CMakePackage, CudaPackage):
     depends_on('hiop@0.5.3:', when='@1.3.0:+hiop')
 
     depends_on('hiop+cuda', when='+hiop+cuda')
+    depends_on('hiop+rocm', when='+hiop+rocm')
     depends_on('hiop~mpi', when='+hiop~mpi')
     depends_on('hiop+mpi', when='+hiop+mpi')
 
     depends_on('petsc@3.13:3.14', when='@:1.2.99')
-    depends_on('petsc@3.16.0', when='@1.3.0:')
+    depends_on('petsc@3.16.0:', when='@1.3.0:')
     depends_on('petsc~mpi', when='~mpi')
 
     depends_on('ipopt', when='+ipopt')
 
+    depends_on('py-mpi4py', when='@1.3.0:+mpi+python')
+
     flag_handler = build_system_flags
 
     def cmake_args(self):
         args = []
         spec = self.spec
 
-        args.append("-DEXAGO_RUN_TESTS=ON")
-
-        args.append(self.define_from_variant('EXAGO_ENABLE_MPI', 'mpi'))
-        args.append(self.define_from_variant('EXAGO_ENABLE_RAJA', 'raja'))
-        args.append(self.define_from_variant('EXAGO_ENABLE_HIOP', 'hiop'))
-        args.append(self.define_from_variant('EXAGO_ENABLE_IPOPT', 'ipopt'))
-        args.append(self.define_from_variant('EXAGO_ENABLE_GPU', 'cuda'))
-        args.append(self.define_from_variant('EXAGO_ENABLE_CUDA', 'cuda'))
-        args.append(self.define_from_variant('EXAGO_ENABLE_PYTHON', 'python'))
-        args.append("-DPETSC_DIR='{0}'".format(spec['petsc'].prefix))
+        # NOTE: If building with spack develop on a cluster, you may want to
+        # change the ctest launch command to use your job scheduler like so:
+        #
+        # args.append(
+        #     self.define('EXAGO_CTEST_LAUNCH_COMMAND', 'srun -t 10:00'))
+
+        args.extend([
+            self.define('EXAGO_ENABLE_GPU', '+cuda' in spec or '+rocm' in spec),
+            self.define_from_variant('EXAGO_ENABLE_CUDA', 'cuda'),
+            self.define_from_variant('EXAGO_ENABLE_HIP', 'rocm'),
+            self.define('PETSC_DIR', spec['petsc'].prefix),
+            self.define('EXAGO_RUN_TESTS', True),
+            self.define_from_variant('EXAGO_ENABLE_MPI', 'mpi'),
+            self.define_from_variant('EXAGO_ENABLE_RAJA', 'raja'),
+            self.define_from_variant('EXAGO_ENABLE_HIOP', 'hiop'),
+            self.define_from_variant('EXAGO_ENABLE_IPOPT', 'ipopt'),
+            self.define_from_variant('EXAGO_ENABLE_PYTHON', 'python'),
+        ])
 
         if '+cuda' in spec:
             cuda_arch_list = spec.variants['cuda_arch'].value
-            cuda_arch = cuda_arch_list[0]
-            if cuda_arch != 'none':
+            if cuda_arch_list[0] != 'none':
                 args.append(
-                    "-DCMAKE_CUDA_ARCHITECTURES={0}".format(cuda_arch))
+                    self.define('CMAKE_CUDA_ARCHITECTURES', cuda_arch_list))
+
+        # NOTE: if +rocm, some HIP CMake variables may not be set correctly.
+        # Namely, HIP_CLANG_INCLUDE_PATH. If the configure phase fails due to
+        # this variable being undefined, adding the following line typically
+        # resolves this issue:
+        #
+        # args.append(
+        #     self.define('HIP_CLANG_INCLUDE_PATH',
+        #         '/opt/rocm-X.Y.Z/llvm/lib/clang/14.0.0/include/'))
+        if '+rocm' in spec:
+            rocm_arch_list = spec.variants['amdgpu_target'].value
+            if rocm_arch_list[0] != 'none':
+                args.append(self.define('GPU_TARGETS', rocm_arch_list))
+                args.append(self.define('AMDGPU_TARGETS', rocm_arch_list))
 
         return args