# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os from spack.package import * class Amrex(CMakePackage, CudaPackage, ROCmPackage): """AMReX is a publicly available software framework designed for building massively parallel block- structured adaptive mesh refinement (AMR) applications.""" homepage = "https://amrex-codes.github.io/amrex/" url = "https://github.com/AMReX-Codes/amrex/releases/download/23.05/amrex-23.05.tar.gz" git = "https://github.com/AMReX-Codes/amrex.git" test_requires_compiler = True tags = ["ecp", "e4s"] maintainers("WeiqunZhang", "asalmgren", "atmyers") license("BSD-3-Clause") version("develop", branch="development") version("24.01", sha256="83dbd4dad6dc51fa4a80aad0347b15ee5a6d816cf4abcd87f7b0e2987d8131b7") version("23.12", sha256="90e00410833d7a82bf6d9e71a70ce85d2bfb89770da7e34d0dda940f2bf5384a") version("23.11", sha256="49b9fea10cd2a2b6cb0fedf7eac8f7889eacc68a05ae5ac7c5702bc0eb1b3848") version("23.10", sha256="3c85aa0ad5f96303e797960a6e0aa37c427f6483f39cdd61dbc2f7ca16357714") version("23.09", sha256="1a539c2628041b17ad910afd9270332060251c8e346b1482764fdb87a4f25053") version("23.08", sha256="a83b7249d65ad8b6ac1881377e5f814b6db8ed8410ea5562b8ae9d4ed1f37c29") version("23.07", sha256="4edb991da51bcaad040f852e42c82834d8605301aa7eeb01cd1512d389a58d90") version("23.06", sha256="3bddcb07cce3e65e06cac35005c30820d311ce47ae54b46e4af333fa272b236b") version("23.05", sha256="a4bf5ad5322e706b9fae46ff52043e2cca5ddba81479647816251e9ab21c0027") version("23.04", sha256="b070949611abd2156208e675e40e5e73ed405bf83e3b1e8ba70fbb451a9e7dd7") version("23.03", sha256="e17c721b1aba4f66e467723f61b59e56c02cf1b72cab5a2680b13ff6e79ef903") version("23.02", sha256="f443c5eb4b89f4a74bf0e1b8a5943da18ab81cdc76aff12e8282ca43ffd06412") version("23.01", sha256="3b1770653a7c6d3e6167bc3cce98cbf838962102c510d1f872ab08f1115933b7") version("22.12", sha256="7b11e547e70bdd6f4b36682708a755d173eaecd8738536306d4217df4dd1be3d") version("22.11", sha256="8be9d5c6934d73b98c71c9c67ca7113f18794268f257333591d9b2449d7410c4") version("22.10", sha256="458da410d7f43e428726bfc905123e85d05786080f892ebaa26f94c5f8e79b07") version("22.09", sha256="24601fbb9d554f7b66d7db89b14ff95dadb18d51db893af7ee6c70d4b7dd4be6") version("22.08", sha256="d89167c4567fa246b06478a5b160010a0117dc58be9e879beb15be53cb08b6e9") version("22.07", sha256="7df433c780ab8429362df8d6d995c95d87a7c3f31ab81d5b0f416203dece086d") version("22.06", sha256="d8aa58e72c86a3da9a7be5a5947294fd3eaac6b233f563366f9e000d833726db") version("22.05", sha256="a760c7ca12915ca56b60d1f3c44103185db21ec2b8c01bc7b6762ff9c84e3f53") version("22.04", sha256="c33f5bdbc1ca21d8dd34b494a9c6c67a7eda4f42403cec3a7c13963f9140ebcf") version("22.03", sha256="2a67233e55f20b937e2da97f1ed3ab0666e12ef283b4d14c9456ebf21f36b77c") version("22.02", sha256="5d8dd3fa3c416b04e70188e06b7e8fc2838d78b43a2cf33a285184c77f0c1e1e") version("22.01", sha256="857df5b2fa8e3010b8856b81879a5be32ba7cc2e575474256eae7ef815b8354d") version("21.12", sha256="439f9ebf2b440fc739a7976f3ade188ec3e1de5f51a0b151e6b8dda36fa67278") version("21.11", sha256="2edb72d7cf7e86340fcaceb325368560957bcd952fd34cd501bfdf038e1338a4") version("21.10", sha256="a11954c03b1ec26c26b676460dc5de5195469e813b70fbcea6dfdefeafaf5407") version("21.09", sha256="983b41d93bf9417c032080fd2ec7c04d0d2b820e613a076bd07566aa5a8aa4bd") version("21.08", sha256="34fb6c72735c74820b27db1138e5bc9fe698ffbd8344aae10a5fbdace479b57f") version("21.07", sha256="9630b8c0c7ffbf3f5ea4d973a3fdb40b9b10fec0f8df33b9e24d76d2c1d15771") version("21.06", sha256="6982c22837d7c0bc4583065d9da55e0aebcf07b54386e4b90a779391fe73fd53") version("21.05", sha256="eb6d21e48279ad67278413c77b29a1754c18ffe741aa6b3a9f3f01eeac13177f") version("21.04", sha256="1c610e4b0800b16f7f1da74193ff11af0abfb12198b36a7e565a6a7f793087fa") version("21.03", sha256="6307bf75c80c2076bf5bd1cff4d12483280a32b5175fe117f32eed9c89cd1ac5") version("21.02", sha256="4a7ef997c43f9f03f1b06dd1aafa01218773a3265a5c1811f77eb4521b5e75b3") version("21.01", sha256="59de3ed429347ee6a7ad4f09c0c431248f2e081f59c301db37cacb36993622f4") version("20.12", sha256="a8ba1d605780250da77619939582ce44b33cd286f2dbcc0dfd5cdbaf209140a5") version("20.11", sha256="b86f4f2ebf414cec050e562d4ab81545944bda581b496d69767b4bf6a3060855") version("20.10", sha256="92def480d1f0bcb5bcb9dfae2ddc8997060414386a1d71ccbfdad785fa2e46fa") version("20.09", sha256="3ae203f18656117d8201da16e899a6144ec217817a2a5d9b7649e2eef9cacdf9") version("20.08", sha256="a202430cd8dbef2de29b20fe9b5881cc58ee762326556ec3c0ad9c3f85ddfc2f") version("20.07", sha256="c386f566f4c57ee56b5630f79ce2c6117d5a612a4aab69b7b26e48d577251165") version("20.06", sha256="be2f2a5107111fcb8b3928b76024b370c7cb01a9e5dd79484cf7fcf59d0b4858") version("20.05", sha256="97d753bb75e845a0a959ec1a044a48e6adb86dd008b5e29ce7a01d49ed276338") version("20.04", sha256="a7ece54d5d89cc00fd555551902a0d4d0fb50db15d2600f441353eed0dddd83b") version("20.03", sha256="9728f20c0d7297c935fe5cbc63c1ee60f983b833a735c797340ee2765d626165") version("20.02", sha256="2eda858b43e7455718ccb96c18f678da1778ec61031e90effdcb9c3e7e6f9bb5") version("20.01", sha256="957e7a7fe90a0a9f4ae10bf9e46dba68d72448d0bec69a4a4e66a544930caca3") version("19.10", sha256="9f30a2b3ec13711dfc6a1b59af59bd7df78449b5846ac6457b5dbbdecb20c576") version("19.08", sha256="94b1e9a9dcfb8c5b52aef91a2ed373aef504d766dd7d0aba6731ceb94e48e940") version("18.10.1", sha256="e648465c9c3b7ff4c696dfa8b6d079b4f61c80d96c51e27af210951c9367c201") version("18.10", sha256="298eba03ef03d617c346079433af1089d38076d6fab2c34476c687740c1f4234") version("18.09.1", sha256="a065ee4d1d98324b6c492ae20ea63ba12a4a4e23432bf5b3fe9788d44aa4398e") # Config options variant( "dimensions", default="3", values=("1", "2", "3"), multi=False, description="Dimensionality", when="@:23.05", ) variant( "dimensions", default="1,2,3", values=("1", "2", "3"), multi=True, description="Dimensionality", when="@23.06:", ) variant("shared", default=False, description="Build shared library") variant("mpi", default=True, description="Build with MPI support") variant("openmp", default=False, description="Build with OpenMP support") variant( "precision", default="double", description="Real precision (double/single)", values=("single", "double"), ) variant("eb", default=False, description="Build Embedded Boundary classes") variant("fortran", default=False, description="Build Fortran API") variant("linear_solvers", default=True, description="Build linear solvers") variant("amrdata", default=False, description="Build data services") variant("particles", default=False, description="Build particle classes") variant("plotfile_tools", default=False, description="Build plotfile_tools like fcompare") variant("tiny_profile", default=False, description="Enable tiny profiling") variant("hdf5", default=False, description="Enable HDF5-based I/O") variant("hypre", default=False, description="Enable Hypre interfaces") variant("petsc", default=False, description="Enable PETSc interfaces") variant("sundials", default=False, description="Enable SUNDIALS interfaces") variant("pic", default=False, description="Enable PIC") variant("sycl", default=False, description="Enable SYCL backend") # Build dependencies depends_on("mpi", when="+mpi") depends_on("sundials@4.0.0:4.1.0 +ARKODE +CVODE", when="@19.08:20.11 +sundials") depends_on("sundials@5.7.0: +ARKODE +CVODE", when="@21.07:22.04 +sundials") depends_on("sundials@6.0.0: +ARKODE +CVODE", when="@22.05: +sundials") for arch in CudaPackage.cuda_arch_values: depends_on( "sundials@5.7.0: +ARKODE +CVODE +cuda cuda_arch=%s" % arch, when="@21.07:22.04 +sundials +cuda cuda_arch=%s" % arch, ) depends_on( "sundials@6.0.0: +ARKODE +CVODE +cuda cuda_arch=%s" % arch, when="@22.05: +sundials +cuda cuda_arch=%s" % arch, ) for tgt in ROCmPackage.amdgpu_targets: depends_on( "sundials@5.7.0: +ARKODE +CVODE +rocm amdgpu_target=%s" % tgt, when="@21.07:22.04 +sundials +rocm amdgpu_target=%s" % tgt, ) depends_on( "sundials@6.0.0: +ARKODE +CVODE +rocm amdgpu_target=%s" % tgt, when="@22.05: +sundials +rocm amdgpu_target=%s" % tgt, ) depends_on("cuda@9.0.0:", when="@:22.04 +cuda") depends_on("cuda@10.0.0:", when="@22.05: +cuda") depends_on("cuda@11.0.0:", when="@22.12: +cuda") depends_on("python@2.7:", type="build", when="@:20.04") depends_on("cmake@3.5:", type="build", when="@:18.10") depends_on("cmake@3.13:", type="build", when="@18.11:19.03") depends_on("cmake@3.14:", type="build", when="@19.04:22.05") depends_on("cmake@3.17:", type="build", when="@22.06:23.01") depends_on("cmake@3.18:", type="build", when="@23.02:") # cmake @3.17: is necessary to handle cuda @11: correctly depends_on("cmake@3.17:", type="build", when="^cuda @11:") depends_on("cmake@3.20:", type="build", when="+rocm") depends_on("cmake@3.22:", type="build", when="+sycl") depends_on("hdf5@1.10.4: +mpi", when="+hdf5") depends_on("rocrand", type="build", when="+rocm") depends_on("hiprand", type="build", when="+rocm") depends_on("rocprim", type="build", when="@21.05: +rocm") depends_on("hypre@2.18.2:", type="link", when="@:21.02 +hypre") depends_on("hypre@2.19.0:", type="link", when="@21.03: ~cuda +hypre") depends_on("hypre@2.20.0:", type="link", when="@21.03: +cuda +hypre") depends_on("petsc", type="link", when="+petsc") depends_on("intel-oneapi-mkl", type=("build", "link"), when="+sycl") # these versions of gcc have lambda function issues # see https://github.com/spack/spack/issues/22310 conflicts("%gcc@8.1.0:8.3.0", when="@21.03") conflicts("%gcc@8.1.0:8.2.0", when="@21.01:21.02") # Check options compatibility conflicts( "+sundials", when="@19.08:20.11 ~fortran", msg="AMReX SUNDIALS support needs AMReX Fortran API (+fortran)", ) conflicts( "+sundials", when="@20.12:21.06", msg="AMReX 20.12 -- 21.06 does not support SUNDIALS interfaces", ) conflicts( "+hdf5", when="@:20.06", msg="AMReX HDF5 support needs AMReX newer than version 20.06" ) conflicts( "+hypre", when="@:20.06", msg="AMReX Hypre support needs AMReX newer than version 20.06" ) conflicts( "+hypre", when="@:20.07 ~fortran", msg="AMReX < 20.08 needs the Fortran API (+fortran) for Hypre support", ) conflicts( "+hypre", when="~linear_solvers", msg="AMReX Hypre support needs variant +linear_solvers" ) conflicts("+petsc", when="@:20.06", msg="PETSc support needs AMReX newer than version 20.06") conflicts( "+petsc", when="@:20.07 ~fortran", msg="AMReX < 20.08 needs the Fortran API (+fortran) for PETSc support", ) conflicts( "+petsc", when="~linear_solvers", msg="AMReX PETSc support needs variant +linear_solvers" ) conflicts( "+cuda", when="@:19.08", msg="AMReX CUDA support needs AMReX newer than version 19.08" ) conflicts("cuda_arch=10", when="+cuda", msg="AMReX only supports compute capabilities >= 3.5") conflicts("cuda_arch=11", when="+cuda", msg="AMReX only supports compute capabilities >= 3.5") conflicts("cuda_arch=12", when="+cuda", msg="AMReX only supports compute capabilities >= 3.5") conflicts("cuda_arch=13", when="+cuda", msg="AMReX only supports compute capabilities >= 3.5") conflicts("cuda_arch=20", when="+cuda", msg="AMReX only supports compute capabilities >= 3.5") conflicts("cuda_arch=21", when="+cuda", msg="AMReX only supports compute capabilities >= 3.5") conflicts("cuda_arch=30", when="+cuda", msg="AMReX only supports compute capabilities >= 3.5") conflicts("cuda_arch=32", when="+cuda", msg="AMReX only supports compute capabilities >= 3.5") conflicts( "+rocm", when="@:20.11", msg="AMReX HIP support needs AMReX newer than version 20.11" ) conflicts( "%rocm@4.2.0:4.2", when="+rocm", msg="AMReX does not support rocm-4.2 due to a compiler bug", ) # GPU vendor support is mutually exclusive conflicts("+cuda", when="+rocm", msg="CUDA and HIP support are exclusive") conflicts("+cuda", when="+sycl", msg="CUDA and SYCL support are exclusive") conflicts("+rocm", when="+sycl", msg="HIP and SYCL support are exclusive") conflicts( "+sycl", when="@:21.05", msg="For SYCL support, AMReX version 21.06 and newer suggested." ) def url_for_version(self, version): if version >= Version("20.05"): url = "https://github.com/AMReX-Codes/amrex/releases/download/{0}/amrex-{0}.tar.gz" else: url = "https://github.com/AMReX-Codes/amrex/archive/{0}.tar.gz" return url.format(version.dotted) def get_cuda_arch_string(self, values): if "none" in values: return "Auto" else: # Use format x.y instead of CudaPackage xy format vf = tuple(float(x) / 10.0 for x in values) return ";".join(str(x) for x in vf) # # For versions > 20.11 # @when("@20.12:,develop") def cmake_args(self): if self.spec.satisfies("@23.01: +sycl") and not self.spec.satisfies("%oneapi@2023.0.0:"): raise InstallError("amrex +sycl requires %oneapi@2023.0.0:") args = [ "-DUSE_XSDK_DEFAULTS=ON", self.define_from_variant("AMReX_SPACEDIM", "dimensions"), self.define_from_variant("BUILD_SHARED_LIBS", "shared"), self.define_from_variant("AMReX_MPI", "mpi"), self.define_from_variant("AMReX_OMP", "openmp"), "-DXSDK_PRECISION:STRING=%s" % self.spec.variants["precision"].value.upper(), self.define_from_variant("XSDK_ENABLE_Fortran", "fortran"), self.define_from_variant("AMReX_FORTRAN_INTERFACES", "fortran"), self.define_from_variant("AMReX_EB", "eb"), self.define_from_variant("AMReX_LINEAR_SOLVERS", "linear_solvers"), self.define_from_variant("AMReX_AMRDATA", "amrdata"), self.define_from_variant("AMReX_PARTICLES", "particles"), self.define_from_variant("AMReX_PLOTFILE_TOOLS", "plotfile_tools"), self.define_from_variant("AMReX_TINY_PROFILE", "tiny_profile"), self.define_from_variant("AMReX_HDF5", "hdf5"), self.define_from_variant("AMReX_HYPRE", "hypre"), self.define_from_variant("AMReX_PETSC", "petsc"), self.define_from_variant("AMReX_SUNDIALS", "sundials"), self.define_from_variant("AMReX_PIC", "pic"), ] if self.spec.satisfies("%fj"): args.append("-DCMAKE_Fortran_MODDIR_FLAG=-M") if "+cuda" in self.spec: args.append("-DAMReX_GPU_BACKEND=CUDA") args.append("-DAMReX_CUDA_ERROR_CAPTURE_THIS=ON") args.append("-DAMReX_CUDA_ERROR_CROSS_EXECUTION_SPACE_CALL=ON") cuda_arch = self.spec.variants["cuda_arch"].value args.append("-DAMReX_CUDA_ARCH=" + self.get_cuda_arch_string(cuda_arch)) if "+rocm" in self.spec: args.append("-DCMAKE_CXX_COMPILER={0}".format(self.spec["hip"].hipcc)) args.append("-DAMReX_GPU_BACKEND=HIP") targets = self.spec.variants["amdgpu_target"].value args.append("-DAMReX_AMD_ARCH=" + ";".join(str(x) for x in targets)) if "+sycl" in self.spec: args.append("-DAMReX_GPU_BACKEND=SYCL") # SYCL GPU backend only supported with Intel's oneAPI or DPC++ compilers sycl_compatible_compilers = ["dpcpp", "icpx"] if not (os.path.basename(self.compiler.cxx) in sycl_compatible_compilers): raise InstallError( "AMReX's SYCL GPU Backend requires DPC++ (dpcpp)" + " or the oneAPI CXX (icpx) compiler." ) return args # # For versions <= 20.11 # @when("@:20.11") def cmake_args(self): args = [ "-DUSE_XSDK_DEFAULTS=ON", self.define_from_variant("DIM", "dimensions"), self.define_from_variant("BUILD_SHARED_LIBS", "shared"), self.define_from_variant("ENABLE_MPI", "mpi"), self.define_from_variant("ENABLE_OMP", "openmp"), "-DXSDK_PRECISION:STRING=%s" % self.spec.variants["precision"].value.upper(), self.define_from_variant("XSDK_ENABLE_Fortran", "fortran"), self.define_from_variant("ENABLE_FORTRAN_INTERFACES", "fortran"), self.define_from_variant("ENABLE_EB", "eb"), self.define_from_variant("ENABLE_LINEAR_SOLVERS", "linear_solvers"), self.define_from_variant("ENABLE_AMRDATA", "amrdata"), self.define_from_variant("ENABLE_PARTICLES", "particles"), self.define_from_variant("ENABLE_SUNDIALS", "sundials"), self.define_from_variant("ENABLE_HDF5", "hdf5"), self.define_from_variant("ENABLE_HYPRE", "hypre"), self.define_from_variant("ENABLE_PETSC", "petsc"), self.define_from_variant("ENABLE_CUDA", "cuda"), ] if self.spec.satisfies("%fj"): args.append("-DCMAKE_Fortran_MODDIR_FLAG=-M") if "+cuda" in self.spec: cuda_arch = self.spec.variants["cuda_arch"].value args.append("-DCUDA_ARCH=" + self.get_cuda_arch_string(cuda_arch)) return args @run_after("build") def setup_standalone_test(self): """Setup stand-alonetests for AMReX versions from 21.12 on.""" if self.spec.satisfies("@:21.11"): return self.cache_extra_test_sources(["Tests"]) def test_run_install_test(self): """build and run AmrCore test""" if self.spec.satisfies("@:21.11"): raise SkipTest("Test is not supported for versions @:21.11") args = ["-S{0}".format(join_path(".", "cache", "amrex", "Tests", "SpackSmokeTest"))] args.append("-DAMReX_ROOT=" + self.prefix) if "+mpi" in self.spec: args.append("-DMPI_C_COMPILER=" + self.spec["mpi"].mpicc) args.append("-DMPI_CXX_COMPILER=" + self.spec["mpi"].mpicxx) if "+cuda" in self.spec: args.append("-DCMAKE_CUDA_COMPILER=" + join_path(self.spec["cuda"].prefix.bin, "nvcc")) args.extend(self.cmake_args()) cmake = which(self.spec["cmake"].prefix.bin.cmake) cmake(*args) make = which("make") make() install_test = which("install_test") inputs_path = join_path( ".", "cache", "amrex", "Tests", "Amr", "Advection_AmrCore", "Exec", "inputs-ci" ) out = install_test(inputs_path, output=str.split, error=str.split) assert "finalized" in out