AOCC support for WRFv3.9.1.1 (#20568)

* AOCC support for WRFv3.9.1.1

* r' as prefix for string literal

5 files changed, 236 insertions, 1 deletions

diff --git a/var/spack/repos/builtin/packages/hdf5/package.py b/var/spack/repos/builtin/packages/hdf5/package.py
index 6ed37d1f70..2877d33133 100644
--- a/var/spack/repos/builtin/packages/hdf5/package.py
+++ b/var/spack/repos/builtin/packages/hdf5/package.py
@@ -324,6 +324,15 @@ class Hdf5(AutotoolsPackage):
                     arg for arg in m.group(1).split(' ') if arg != '-l'),
                 'libtool')
 
+    @run_after('configure')
+    def patch_libtool(self):
+        """AOCC support for HDF5"""
+        if '%aocc' in self.spec:
+            filter_file(
+                r'\$wl-soname \$wl\$soname',
+                r'-fuse-ld=ld -Wl,-soname,\$soname',
+                'libtool', string=True)
+
     @run_after('install')
     @on_package_attributes(run_tests=True)
     def check_install(self):
diff --git a/var/spack/repos/builtin/packages/netcdf-fortran/package.py b/var/spack/repos/builtin/packages/netcdf-fortran/package.py
index 7f119b4925..3552101abf 100644
--- a/var/spack/repos/builtin/packages/netcdf-fortran/package.py
+++ b/var/spack/repos/builtin/packages/netcdf-fortran/package.py
@@ -136,6 +136,15 @@ class NetcdfFortran(AutotoolsPackage):
 
         return config_args
 
+    @run_after('configure')
+    def patch_libtool(self):
+        """AOCC support for NETCDF-F"""
+        if '%aocc' in self.spec:
+            filter_file(
+                r'\${wl}-soname \$wl\$soname',
+                r'-fuse-ld=ld -Wl,-soname,\$soname',
+                'libtool', string=True)
+
     @when('@:4.4.5')
     def check(self):
         with working_dir(self.build_directory):
diff --git a/var/spack/repos/builtin/packages/wrf/package.py b/var/spack/repos/builtin/packages/wrf/package.py
index d638192297..196c5efc97 100644
--- a/var/spack/repos/builtin/packages/wrf/package.py
+++ b/var/spack/repos/builtin/packages/wrf/package.py
@@ -105,6 +105,7 @@ class Wrf(Package):
     patch("patches/3.9/netcdf_backport.patch", when="@3.9.1.1")
     patch("patches/3.9/tirpc_detect.patch", when="@3.9.1.1")
     patch("patches/3.9/add_aarch64.patch", when="@3.9.1.1")
+    patch("patches/3.9/configure_aocc.patch", when="@3.9.1.1 %aocc@:3.0")
 
     # These patches deal with netcdf & netcdf-fortran being two diff things
     # Patches are based on:
@@ -144,6 +145,7 @@ class Wrf(Package):
     depends_on("libpng")
     depends_on("zlib")
     depends_on("perl")
+    depends_on("jemalloc", when="%aocc")
     # not sure if +fortran is required, but seems like a good idea
     depends_on("hdf5+fortran+hl+mpi")
     # build script use csh
@@ -170,6 +172,11 @@ class Wrf(Package):
             env.set("FCFLAGS", args)
             env.set("FFLAGS", args)
 
+        if self.spec.satisfies("%aocc"):
+            env.set("WRFIO_NCD_LARGE_FILE_SUPPORT", 1)
+            env.set("HDF5", self.spec["hdf5"].prefix)
+            env.prepend_path('PATH', ancestor(self.compiler.cc))
+
     def patch(self):
         # Let's not assume csh is intalled in bin
         files = glob.glob("*.csh")
@@ -232,12 +239,31 @@ class Wrf(Package):
                             )
                         ofh.write(line)
 
+        if self.spec.satisfies("@3.9.1.1 %aocc"):
+            rename(
+                "./arch/configure_new.defaults",
+                "./arch/configure_new.defaults.bak",
+            )
+            with open("./arch/configure_new.defaults.bak", "rt") as ifh:
+                with open("./arch/configure_new.defaults", "wt") as ofh:
+                    for line in ifh:
+                        if line.startswith("DM_"):
+                            line = line.replace(
+                                "mpif90 -DMPI2_SUPPORT",
+                                self.spec['mpi'].mpifc + " -DMPI2_SUPPORT"
+                            )
+                            line = line.replace(
+                                "mpicc -DMPI2_SUPPORT",
+                                self.spec['mpi'].mpicc + " -DMPI2_SUPPORT"
+                            )
+                        ofh.write(line)
+
     def configure(self, spec, prefix):
 
         # Remove broken default options...
         self.do_configure_fixup()
 
-        if self.spec.compiler.name not in ["intel", "gcc"]:
+        if self.spec.compiler.name not in ["intel", "gcc", "aocc"]:
             raise InstallError(
                 "Compiler %s not currently supported for WRF build."
                 % self.spec.compiler.name
@@ -284,6 +310,12 @@ class Wrf(Package):
         if returncode != 0:
             raise InstallError("Configure failed - unknown error")
 
+    @run_after("configure")
+    def patch_for_libmvec(self):
+        if self.spec.satisfies("@3.9.1.1 %aocc@:3.0"):
+            fp = self.package_dir + "/patches/3.9/aocc_lmvec.patch"
+            which('patch')('-s', '-p1', '-i', '{0}'.format(fp), '-d', '.')
+
     def run_compile_script(self):
         csh_bin = self.spec["tcsh"].prefix.bin.csh
         csh = Executable(csh_bin)
diff --git a/var/spack/repos/builtin/packages/wrf/patches/3.9/aocc_lmvec.patch b/var/spack/repos/builtin/packages/wrf/patches/3.9/aocc_lmvec.patch
new file mode 100644
index 0000000000..6d67b04612
--- /dev/null
+++ b/var/spack/repos/builtin/packages/wrf/patches/3.9/aocc_lmvec.patch
@@ -0,0 +1,130 @@
+diff --git a/configure.wrf b/configure_libm.wrf
+index b0e306c..41a6cba 100644
+--- a/configure.wrf
++++ b/configure_libm.wrf
+@@ -263,56 +263,56 @@ wrfio_nf :
+ 	( cd $(WRF_SRC_ROOT_DIR)/external/io_netcdf ; \
+           make $(J) NETCDFPATH="$(NETCDFPATH)" NETCDFFPATH="$(NETCDFFPATH)" RANLIB="$(RANLIB)" CPP="$(CPP)" \
+           CC="$(SCC)" CFLAGS="$(CFLAGS)" \
+-          FC="$(SFC) $(PROMOTION) $(OMP) $(FCFLAGS)" TRADFLAG="$(TRADFLAG)" AR="$(AR)" ARFLAGS="$(ARFLAGS)" )
++          FC="$(SFC) $(PROMOTION) $(OMP) $(LIBMVEC) $(FCFLAGS)" TRADFLAG="$(TRADFLAG)" AR="$(AR)" ARFLAGS="$(ARFLAGS)" )
+ 
+ wrfio_pnf : 
+ 	( cd $(WRF_SRC_ROOT_DIR)/external/io_pnetcdf ; \
+           make $(J) NETCDFPATH="$(PNETCDFPATH)" RANLIB="$(RANLIB)" CPP="$(CPP) $(ARCHFLAGS)" \
+-          FC="$(FC) $(PROMOTION) $(OMP) $(FCFLAGS)" TRADFLAG="$(TRADFLAG)" AR="$(AR)" ARFLAGS="$(ARFLAGS)" )
++          FC="$(FC) $(PROMOTION) $(OMP) $(LIBMVEC) $(FCFLAGS)" TRADFLAG="$(TRADFLAG)" AR="$(AR)" ARFLAGS="$(ARFLAGS)" )
+ 
+ wrfio_grib_share :
+ 	( cd $(WRF_SRC_ROOT_DIR)/external/io_grib_share ; \
+           make $(J) CC="$(SCC)" CFLAGS="$(CFLAGS)" RM="$(RM)" RANLIB="$(RANLIB)" CPP="$(CPP)" \
+-          FC="$(SFC) $(PROMOTION) -I. $(FCDEBUG) $(FCBASEOPTS) $(FCSUFFIX)" TRADFLAG="$(TRADFLAG)" AR="$(AR)" ARFLAGS="$(ARFLAGS)" archive) 
++          FC="$(SFC) $(PROMOTION) -I. $(LIBMVEC) $(FCDEBUG) $(FCBASEOPTS) $(FCSUFFIX)" TRADFLAG="$(TRADFLAG)" AR="$(AR)" ARFLAGS="$(ARFLAGS)" archive) 
+ 
+ wrfio_grib1 :
+ 	( cd $(WRF_SRC_ROOT_DIR)/external/io_grib1 ; \
+           make $(J) CC="$(SCC)" CFLAGS="$(CFLAGS)" RM="$(RM)" RANLIB="$(RANLIB)" CPP="$(CPP)" \
+-          FC="$(SFC) $(PROMOTION) -I. $(FCDEBUG) $(FCBASEOPTS) $(FCSUFFIX)" TRADFLAG="$(TRADFLAG)" AR="$(AR)" ARFLAGS="$(ARFLAGS)" archive)
++          FC="$(SFC) $(PROMOTION) -I. $(LIBMVEC) $(FCDEBUG) $(FCBASEOPTS) $(FCSUFFIX)" TRADFLAG="$(TRADFLAG)" AR="$(AR)" ARFLAGS="$(ARFLAGS)" archive)
+ 
+ wrfio_grib2 :
+ 	( cd $(WRF_SRC_ROOT_DIR)/external/io_grib2 ; \
+           make $(J) CC="$(SCC)" CFLAGS="$(CFLAGS) " RM="$(RM)" RANLIB="$(RANLIB)" \
+           CPP="$(CPP)" \
+-          FC="$(SFC) $(PROMOTION) -I. $(FCDEBUG) $(FCBASEOPTS) $(FCSUFFIX)" TRADFLAG="-traditional" AR="$(AR)" ARFLAGS="$(ARFLAGS)" \
++          FC="$(SFC) $(PROMOTION) -I. $(LIBMVEC) $(FCDEBUG) $(FCBASEOPTS) $(FCSUFFIX)" TRADFLAG="-traditional" AR="$(AR)" ARFLAGS="$(ARFLAGS)" \
+           FIXED="$(FORMAT_FIXED)" archive)
+ 
+ wrfio_int : 
+ 	( cd $(WRF_SRC_ROOT_DIR)/external/io_int ; \
+           make $(J) CC="$(CC)" CFLAGS_LOCAL="$(CFLAGS_LOCAL)" RM="$(RM)" RANLIB="$(RANLIB)" CPP="$(CPP)" \
+-          FC="$(FC) $(PROMOTION) $(FCDEBUG) $(FCBASEOPTS) $(OMP)" FGREP="$(FGREP)" \
++          FC="$(FC) $(PROMOTION) $(LIBMVEC) $(FCDEBUG) $(FCBASEOPTS) $(OMP)" FGREP="$(FGREP)" \
+           TRADFLAG="$(TRADFLAG)" AR="$(AR)" ARFLAGS="$(ARFLAGS)" ARCHFLAGS="$(ARCHFLAGS)" all )
+ 
+ esmf_time : 
+ 	( cd $(WRF_SRC_ROOT_DIR)/external/esmf_time_f90 ; \
+-          make $(J) FC="$(SFC) $(PROMOTION) $(FCDEBUG) $(FCBASEOPTS)" RANLIB="$(RANLIB)" \
++          make $(J) FC="$(SFC) $(PROMOTION) $(LIBMVEC) $(FCDEBUG) $(FCBASEOPTS)" RANLIB="$(RANLIB)" \
+           CPP="$(CPP) -I$(WRF_SRC_ROOT_DIR)/inc -I. $(ARCHFLAGS) $(TRADFLAG)" AR="$(AR)" ARFLAGS="$(ARFLAGS)" )
+ 
+ fftpack :
+ 	( cd $(WRF_SRC_ROOT_DIR)/external/fftpack/fftpack5 ; \
+-          make $(J) FC="$(SFC)" FFLAGS="$(PROMOTION) $(FCDEBUG) $(FCBASEOPTS)" RANLIB="$(RANLIB)" AR="$(AR)" \
++          make $(J) FC="$(SFC)" FFLAGS="$(PROMOTION) $(LIBMVEC) $(FCDEBUG) $(FCBASEOPTS)" RANLIB="$(RANLIB)" AR="$(AR)" \
+           ARFLAGS="$(ARFLAGS)" CPP="$(CPP)" CPPFLAGS="$(CPPFLAGS)" RM="$(RM)" )
+ 
+ atm_ocn :
+ 	( cd $(WRF_SRC_ROOT_DIR)/external/atm_ocn ; \
+           make $(J) CC="$(SCC)" CFLAGS="$(CFLAGS) " RM="$(RM)" RANLIB="$(RANLIB)" \
+           CPP="$(CPP)" CPPFLAGS="$(CPPFLAGS)" \
+-          FC="$(DM_FC) $(PROMOTION) -I. $(FCDEBUG) $(FCBASEOPTS) $(FCSUFFIX)" TRADFLAG="-traditional" AR="$(AR)" ARFLAGS="$(ARFLAGS)" \
++          FC="$(DM_FC) $(PROMOTION) -I. $(LIBMVEC) $(FCDEBUG) $(FCBASEOPTS) $(FCSUFFIX)" TRADFLAG="-traditional" AR="$(AR)" ARFLAGS="$(ARFLAGS)" \
+           FIXED="$(FORMAT_FIXED)" )
+ 
+ $(WRF_SRC_ROOT_DIR)/external/RSL_LITE/librsl_lite.a :
+ 	( cd $(WRF_SRC_ROOT_DIR)/external/RSL_LITE ; make $(J) CC="$(CC) $(CFLAGS)" \
+-          FC="$(FC) $(FCFLAGS) $(OMP) $(PROMOTION) $(BYTESWAPIO)" \
++          FC="$(FC) $(FCFLAGS) $(LIBMVEC) $(OMP) $(PROMOTION) $(BYTESWAPIO)" \
+           CPP="$(CPP) -I. $(ARCHFLAGS) $(OMPCPP) $(TRADFLAG)" AR="$(AR)" ARFLAGS="$(ARFLAGS)" ;\
+           $(RANLIB) $(WRF_SRC_ROOT_DIR)/external/RSL_LITE/librsl_lite.a )
+ 
+@@ -332,7 +332,7 @@ wrf_ioapi_includes :
+ 
+ wrfio_esmf :
+ 	( cd $(WRF_SRC_ROOT_DIR)/external/io_esmf ; \
+-          make FC="$(FC) $(PROMOTION) $(FCDEBUG) $(FCBASEOPTS) $(ESMF_MOD_INC)" \
++          make FC="$(FC) $(PROMOTION) $(LIBMVEC) $(FCDEBUG) $(FCBASEOPTS) $(ESMF_MOD_INC)" \
+           RANLIB="$(RANLIB)" CPP="$(CPP) $(POUND_DEF) " AR="$(AR)" ARFLAGS="$(ARFLAGS)" )
+ 
+ #	There is probably no reason to modify these rules
+@@ -357,7 +357,7 @@ wrfio_esmf :
+           $(WRF_SRC_ROOT_DIR)/var/build/da_name_space.pl $*.f90 > $*.f90.tmp ; \
+           mv $*.f90.tmp $*.f90 ; \
+         fi
+-	$(FC) -o $@ -c $(FCFLAGS) $(OMP) $(MODULE_DIRS) $(PROMOTION) $(FCSUFFIX) $*.f90
++	$(FC) -o $@ -c $(FCFLAGS) $(LIBMVEC) $(OMP) $(MODULE_DIRS) $(PROMOTION) $(FCSUFFIX) $*.f90
+         
+ 
+ .F.f90:
+@@ -370,7 +370,7 @@ wrfio_esmf :
+ 
+ .f90.o:
+ 	$(RM) $@
+-	$(FC) -o $@ -c $(FCFLAGS) $(PROMOTION) $(FCSUFFIX) $*.f90
++	$(FC) -o $@ -c $(FCFLAGS) $(LIBMVEC) $(PROMOTION) $(FCSUFFIX) $*.f90
+ 
+ setfeenv.o : setfeenv.c
+ 	$(RM) $@
+@@ -474,10 +474,10 @@ wrf_tsin.o :
+         fi
+ 	if $(FGREP) '!$$OMP' $*.f90 ; then \
+           if [ -n "$(OMP)" ] ; then echo COMPILING $*.F WITH OMP ; fi ; \
+-	  $(FC) -c $(PROMOTION) $(FCNOOPT) $(FCBASEOPTS) $(MODULE_DIRS) $(FCSUFFIX) $(OMP) $*.f90 ; \
++	  $(FC) -c $(PROMOTION) $(FCNOOPT) $(LIBMVEC) $(FCBASEOPTS) $(MODULE_DIRS) $(FCSUFFIX) $(OMP) $*.f90 ; \
+         else \
+           if [ -n "$(OMP)" ] ; then echo COMPILING $*.F WITHOUT OMP ; fi ; \
+-	  $(FC) -c $(PROMOTION) $(FCNOOPT) $(FCBASEOPTS) $(MODULE_DIRS) $(FCSUFFIX) $*.f90 ; \
++	  $(FC) -c $(PROMOTION) $(FCNOOPT) $(LIBMVEC) $(FCBASEOPTS) $(MODULE_DIRS) $(FCSUFFIX) $*.f90 ; \
+         fi
+ 
+ #solve_em.o :
+@@ -497,10 +497,10 @@ module_sf_ruclsm.o :
+ 	if $(FGREP) '!$$OMP' $*.f90 ; then \
+           echo COMPILING $*.F WITH OMP ; \
+           if [ -n "$(OMP)" ] ; then echo COMPILING $*.F WITH OMP ; fi ; \
+-	  $(FC) -c $(PROMOTION) $(FCREDUCEDOPT) $(FCBASEOPTS) $(MODULE_DIRS) $(FCSUFFIX) $(OMP) $*.f90 ; \
++	  $(FC) -c $(PROMOTION) $(FCREDUCEDOPT) $(LIBMVEC) $(FCBASEOPTS) $(MODULE_DIRS) $(FCSUFFIX) $(OMP) $*.f90 ; \
+         else \
+           if [ -n "$(OMP)" ] ; then echo COMPILING $*.F WITHOUT OMP ; fi ; \
+-	  $(FC) -c $(PROMOTION) $(FCREDUCEDOPT) $(FCBASEOPTS) $(MODULE_DIRS) $(FCSUFFIX) $*.f90 ; \
++	  $(FC) -c $(PROMOTION) $(FCREDUCEDOPT) $(LIBMVEC) $(FCBASEOPTS) $(MODULE_DIRS) $(FCSUFFIX) $*.f90 ; \
+         fi
+ 
+ # compile without OMP
+@@ -547,4 +547,4 @@ module_configure.o :
+           mv $*.f90.tmp $*.f90 ; \
+         fi
+ 	$(RM) $*.b $*.bb
+-	$(FC) -c $(PROMOTION) $(FCSUFFIX) $(FCNOOPT) $(FCBASEOPTS) $(MODULE_DIRS) $*.f90
++	$(FC) -c $(PROMOTION) $(FCSUFFIX) $(LIBMVEC) $(FCNOOPT) $(FCBASEOPTS) $(MODULE_DIRS) $*.f90
diff --git a/var/spack/repos/builtin/packages/wrf/patches/3.9/configure_aocc.patch b/var/spack/repos/builtin/packages/wrf/patches/3.9/configure_aocc.patch
new file mode 100644
index 0000000000..13b41a46b2
--- /dev/null
+++ b/var/spack/repos/builtin/packages/wrf/patches/3.9/configure_aocc.patch
@@ -0,0 +1,55 @@
+--- WRF-3.9.1.1/arch/configure_new.defaults	2017-08-29 01:59:47.000000000 +0530
++++ WRF-3.9.1.1/arch/configure_391_aocc22.defaults	2020-12-23 10:06:29.764955610 +0530
+@@ -1917,6 +1917,52 @@
+ CC_TOOLS        =      $(SCC) 
+ 
+ #insert new stanza here
++############################################################
++#ARCH    AMD EPYC Linux x86_64 AOCC Compilers #dm+sm
++#
++DESCRIPTION     =       AMD AOCC ($SFC/$SCC): EPYC
++DMPARALLEL      =       1
++OMPCPP          =       -D_OPENMP
++OMP             =       -fopenmp
++OMPCC           =       -fopenmp -Mpreprocess
++SFC             =       flang
++SCC             =       clang
++CCOMP           =       clang
++DM_FC           =       mpif90 -DMPI2_SUPPORT
++DM_CC           =       mpicc -DMPI2_SUPPORT
++FC              =       $(DM_FC)
++CC              =       $(DM_CC) -DFSEEKO64_OK
++LD              =       $(FC)
++RWORDSIZE       =       $(NATIVE_RWORDSIZE)
++PROMOTION       =
++ARCH_LOCAL      =       -DNONSTANDARD_SYSTEM_SUBR -DWRF_USE_CLM
++LIBMVEC         =       -mllvm --vector-library=LIBMVEC
++AMDARCHOPT      =       -march=native
++AOCCOPT         =       -O3 -m64 -Ofast -ffast-math $(AMDARCHOPT)
++CFLAGS_LOCAL    =       -w $(AOCCOPT)
++LDFLAGS_LOCAL   =       -lm -ltirpc -lamdlibm  -ljemalloc -lmvec $(AOCCOPT)
++CPLUSPLUSLIB    =
++ESMF_LDFLAG     =       $(CPLUSPLUSLIB)
++FCOPTIM         =       $(AOCCOPT) -fopenmp
++FCREDUCEDOPT    =       -O2 -Ofast -ffast-math
++FCNOOPT         =       -O0 -DFCNOOPT -fopenmp
++FCDEBUG         =       #-g
++FORMAT_FIXED    =       -Mfixed
++FORMAT_FREE     =       -Mfreeform
++FCSUFFIX        =
++BYTESWAPIO      =       -Mbyteswapio
++FCBASEOPTS_NO_G =       $(FORMAT_FREE) $(BYTESWAPIO) -fopenmp
++FCBASEOPTS      =       $(FCBASEOPTS_NO_G) $(FCDEBUG) -DBASEOPTS -fopenmp
++MODULE_SRCH_FLAG =
++TRADFLAG        =       -traditional
++CPP             =       /lib/cpp -P
++AR              =       llvm-ar
++ARFLAGS         =       ru
++M4              =       m4
++RANLIB          =       llvm-ranlib
++RLFLAGS         =
++CC_TOOLS        =       $(SCC)
++
+ 
+ ###########################################################
+ #ARCH    Fujitsu FX10/FX100 Linux x86_64 SPARC64IXfx/SPARC64Xlfx, mpifrtpx and mpifccpx compilers #serial smpar dmpar dm+sm