[AMD][rocm-openmp-extras] Update for ROCm 4.3. (#25482)

- Added new checksums for 4.3.
- Now using llvm-amdgpu ~openmp in order to use the rocm-device-libs
  build as external project in llvm-amdgpu package. We still need
  to pull device-libs in using resource for the build as some headers
  are not installed.
- Updated symlink creation to now remove an existing link if  present
  to avoid issues on partial reinstalls when debugging.
- Adjusted the flang_warning to be a part of Cmake options instead of
  a filter_file for better compatability.
- The dependency on hsa-rocr-dev created some problems as type was changed
  to the default build/link. This issue was because ROCr uses libelf and
  the build of openmp expects elfutils. When link is specified libelf
  was being found in the path first, causing errors. This was
  introduced with the llvm-amdgpu external project build of device-libs.
- On a bare bone installation of sles15 it was noted that libquadmath0 was
  needed as a dependency. On 18.04 gcc-multilib was also needed.

* Workaround for libelf headers being used instead of elfutils.

1 files changed, 61 insertions, 33 deletions

diff --git a/var/spack/repos/builtin/packages/rocm-openmp-extras/package.py b/var/spack/repos/builtin/packages/rocm-openmp-extras/package.py
index 32af84d329..baf58a7a3c 100644
--- a/var/spack/repos/builtin/packages/rocm-openmp-extras/package.py
+++ b/var/spack/repos/builtin/packages/rocm-openmp-extras/package.py
@@ -19,7 +19,8 @@ aomp = [
     "808fca9bdefb109d5bcbbc9f5b59c564a6d422488869e986516f2a7233eda235",
     "aa75455cf1d333419e5310117678e5789c5222f7cb05b05e3dfacef855c55d84",
     "9e6ed2c7bdc3b4af069751b5d3e92913fd5ac318ae844f68bd78c5def990a8f7",
-    "c368d39ba9c1bc8b0edbe66edaa3f2a4ff5649c2bd16f499ac19dfd1591dec5a"
+    "c368d39ba9c1bc8b0edbe66edaa3f2a4ff5649c2bd16f499ac19dfd1591dec5a",
+    "c2b1a61a15fdf8d50c7c7a1ad75512f059c53a7bd5afe85f69e984f1174aa74a"
 ]
 
 devlib = [
@@ -27,7 +28,8 @@ devlib = [
     "bca9291385d6bdc91a8b39a46f0fd816157d38abb1725ff5222e6a0daa0834cc",
     "d0aa495f9b63f6d8cf8ac668f4dc61831d996e9ae3f15280052a37b9d7670d2a",
     "f5f5aa6bfbd83ff80a968fa332f80220256447c4ccb71c36f1fbd2b4a8e9fc1b",
-    "34a2ac39b9bb7cfa8175cbab05d30e7f3c06aaffce99eed5f79c616d0f910f5f"
+    "34a2ac39b9bb7cfa8175cbab05d30e7f3c06aaffce99eed5f79c616d0f910f5f",
+    "055a67e63da6491c84cd45865500043553fb33c44d538313dd87040a6f3826f2"
 ]
 
 llvm = [
@@ -35,7 +37,8 @@ llvm = [
     "8262aff88c1ff6c4deb4da5a4f8cda1bf90668950e2b911f93f73edaee53b370",
     "aa1f80f429fded465e86bcfaef72255da1af1c5c52d58a4c979bc2f6c2da5a69",
     "244e38d824fa7dfa8d0edf3c036b3c84e9c17a16791828e4b745a8d31eb374ae",
-    "751eca1d18595b565cfafa01c3cb43efb9107874865a60c80d6760ba83edb661"
+    "751eca1d18595b565cfafa01c3cb43efb9107874865a60c80d6760ba83edb661",
+    "1567d349cd3bcd2c217b3ecec2f70abccd5e9248bd2c3c9f21d4cdb44897fc87"
 ]
 
 flang = [
@@ -43,7 +46,8 @@ flang = [
     "3990d39ff1c908b150f464f0653a123d94be30802f9cad6af18fbb560c4b412e",
     "f3e19699ce4ac404f41ffe08ef4546e31e2e741d8deb403b5477659e054275d5",
     "f41f661425534b5cfb20e2c0efd9d0800609dc3876ee9c3f76f026d36abbfa35",
-    "d6c3f3aaa289251a433d99d1cffe432812093089ae876a6863295a15066c1eaf"
+    "d6c3f3aaa289251a433d99d1cffe432812093089ae876a6863295a15066c1eaf",
+    "13d3525078fd1c569f7c8ea7fce439b04f6b03814bbe88600c08f95c788e7802"
 ]
 
 extras = [
@@ -51,10 +55,11 @@ extras = [
     "5d98d34aff97416d8b5b9e16e7cf474580f8de8a73bd0e549c4440a3c5df4ef5",
     "51cc8a7c5943e1d9bc657fc9b9797f45e3ce6a4e544d3d3a967c7cd0185a0510",
     "91fdfadb94aa6afc1942124d0953ddc80c297fa75de1897fb42ac8e7dea51ab9",
-    "31bbe70b51c259a54370d021ae63528a1740b5477a22412685afd14150fff6f4"
+    "31bbe70b51c259a54370d021ae63528a1740b5477a22412685afd14150fff6f4",
+    "ec6cc4a9c24f098496de3206714dafe9a714f06afacfe21d53a4e6344f9cb4c9"
 ]
 
-versions = ['3.9.0', '3.10.0', '4.0.0', '4.1.0', '4.2.0']
+versions = ['3.9.0', '3.10.0', '4.0.0', '4.1.0', '4.2.0', '4.3.0']
 versions_dict = dict()
 components = ['aomp', 'devlib', 'llvm', 'flang', 'extras']
 component_hashes = [aomp, devlib, llvm, flang, extras]
@@ -70,9 +75,10 @@ class RocmOpenmpExtras(Package):
     """OpenMP support for ROCm LLVM."""
 
     homepage = tools_url + "/aomp"
-    url = tools_url + "/aomp/archive/rocm-4.2.0.tar.gz"
+    url = tools_url + "/aomp/archive/rocm-4.3.0.tar.gz"
 
     maintainers = ['srekolam', 'arjun-raj-kuppala', 'estewart08']
+    version('4.3.0', sha256=versions_dict['4.3.0']['aomp'])
     version('4.2.0', sha256=versions_dict['4.2.0']['aomp'])
     version('4.1.0', sha256=versions_dict['4.1.0']['aomp'])
     version('4.0.0', sha256=versions_dict['4.0.0']['aomp'])
@@ -88,13 +94,11 @@ class RocmOpenmpExtras(Package):
     depends_on('elfutils', type=('build', 'link'))
     depends_on('libffi', type=('build', 'link'))
 
-    for ver in ['3.9.0', '3.10.0', '4.0.0', '4.1.0', '4.2.0']:
+    for ver in ['3.9.0', '3.10.0', '4.0.0', '4.1.0', '4.2.0', '4.3.0']:
         depends_on('hsakmt-roct@' + ver, when='@' + ver)
         depends_on('comgr@' + ver, when='@' + ver)
         depends_on('hsa-rocr-dev@' + ver, when='@' + ver)
-        # standalone rocm-device-libs
-        depends_on('rocm-device-libs@' + ver, when='@' + ver)
-        depends_on('llvm-amdgpu@{0} ~rocm-device-libs ~openmp'.format(ver),
+        depends_on('llvm-amdgpu@{0} ~openmp'.format(ver),
                    when='@' + ver)
 
         # tag changed to 'rocm-' in 4.0.0
@@ -141,7 +145,7 @@ class RocmOpenmpExtras(Package):
             when='@' + ver)
 
     def setup_run_environment(self, env):
-        devlibs_prefix = self.spec['rocm-device-libs'].prefix
+        devlibs_prefix = self.spec['llvm-amdgpu'].prefix
         openmp_extras_prefix = self.spec['rocm-openmp-extras'].prefix
         llvm_prefix = self.spec['llvm-amdgpu'].prefix
         env.set('AOMP', '{0}'.format(llvm_prefix))
@@ -169,22 +173,22 @@ class RocmOpenmpExtras(Package):
 
     def patch(self):
         src = self.stage.source_path
-        flang_warning = '-Wno-incompatible-pointer-types-discards-qualifiers)'
         aomp_extras = '{0}/rocm-openmp-extras/aomp-extras/aomp-device-libs'
         libomptarget = \
             '{0}/rocm-openmp-extras/llvm-project/openmp/libomptarget'
         flang = '{0}/rocm-openmp-extras/flang/'
 
+        # If not in a git repo the STRIP command will have an empty
+        # argument. This is fixed in later versions.
+        if self.spec.version == Version('4.3.0'):
+            filter_file('STRIP ${FLANG_SHA}', 'STRIP 0',
+                        flang.format(src) + 'CMakeLists.txt', string=True)
+
         if self.spec.version < Version('4.1.0'):
             plugin = '/plugins/hsa/CMakeLists.txt'
         else:
-            # Spack thinks some warnings from the flang build are errors.
-            # Disable those warnings.
-            filter_file('PRIVATE -fPIC)',
-                        'PRIVATE -fPIC PRIVATE ' + flang_warning,
-                        flang.format(src) + 'runtime/flang/CMakeLists.txt',
-                        string=True)
             plugin = '/plugins/amdgpu/CMakeLists.txt'
+
         filter_file(
             '{ROCM_DIR}/amdgcn/bitcode', '{DEVICE_LIBS_DIR}',
             aomp_extras.format(src) + '/aompextras/CMakeLists.txt',
@@ -257,7 +261,7 @@ class RocmOpenmpExtras(Package):
         src = self.stage.source_path
         gfx_list = "gfx700;gfx701;gfx801;gfx803;gfx900;gfx902;gfx906;gfx908"
         openmp_extras_prefix = self.spec['rocm-openmp-extras'].prefix
-        devlibs_prefix = self.spec['rocm-device-libs'].prefix
+        devlibs_prefix = self.spec['llvm-amdgpu'].prefix
         devlibs_src = '{0}/rocm-openmp-extras/rocm-device-libs'.format(src)
         hsa_prefix = self.spec['hsa-rocr-dev'].prefix
         hsakmt_prefix = self.spec['hsakmt-roct'].prefix
@@ -268,22 +272,30 @@ class RocmOpenmpExtras(Package):
         omp_lib_dir = '{0}/lib'.format(openmp_extras_prefix)
         bin_dir = '{0}/bin'.format(llvm_prefix)
         lib_dir = '{0}/lib'.format(llvm_prefix)
+        flang_warning = '-Wno-incompatible-pointer-types-discards-qualifiers'
+        libpgmath = '/rocm-openmp-extras/flang/runtime/libpgmath/lib/common'
+        elfutils_inc = spec['elfutils'].prefix.include
 
         # flang1 and flang2 symlink needed for build of flang-runtime
         # libdevice symlink to rocm-openmp-extras for runtime
         # libdebug symlink to rocm-openmp-extras for runtime
-        if not (os.path.islink((os.path.join(bin_dir, 'flang1')))):
-            os.symlink(os.path.join(omp_bin_dir, 'flang1'),
-                       os.path.join(bin_dir, 'flang1'))
-        if not (os.path.islink((os.path.join(bin_dir, 'flang2')))):
-            os.symlink(os.path.join(omp_bin_dir, 'flang2'),
-                       os.path.join(bin_dir, 'flang2'))
-        if not (os.path.islink((os.path.join(lib_dir, 'libdevice')))):
-            os.symlink(os.path.join(omp_lib_dir, 'libdevice'),
-                       os.path.join(lib_dir, 'libdevice'))
-        if not (os.path.islink((os.path.join(llvm_prefix, 'lib-debug')))):
-            os.symlink(os.path.join(openmp_extras_prefix, 'lib-debug'),
-                       os.path.join(llvm_prefix, 'lib-debug'))
+        if (os.path.islink((os.path.join(bin_dir, 'flang1')))):
+            os.unlink(os.path.join(bin_dir, 'flang1'))
+        if (os.path.islink((os.path.join(bin_dir, 'flang2')))):
+            os.unlink(os.path.join(bin_dir, 'flang2'))
+        if (os.path.islink((os.path.join(lib_dir, 'libdevice')))):
+            os.unlink(os.path.join(lib_dir, 'libdevice'))
+        if (os.path.islink((os.path.join(llvm_prefix, 'lib-debug')))):
+            os.unlink(os.path.join(llvm_prefix, 'lib-debug'))
+
+        os.symlink(os.path.join(omp_bin_dir, 'flang1'),
+                   os.path.join(bin_dir, 'flang1'))
+        os.symlink(os.path.join(omp_bin_dir, 'flang2'),
+                   os.path.join(bin_dir, 'flang2'))
+        os.symlink(os.path.join(omp_lib_dir, 'libdevice'),
+                   os.path.join(lib_dir, 'libdevice'))
+        os.symlink(os.path.join(openmp_extras_prefix, 'lib-debug'),
+                   os.path.join(llvm_prefix, 'lib-debug'))
 
         # Set cmake args
         components = dict()
@@ -299,6 +311,11 @@ class RocmOpenmpExtras(Package):
         ]
 
         # Shared cmake configuration for openmp, openmp-debug
+        # Due to hsa-rocr-dev using libelf instead of elfutils
+        # the build of openmp fails because the include path
+        # for libelf is placed before elfutils in SPACK_INCLUDE_DIRS.
+        # Passing the elfutils include path via cmake options is a
+        # workaround until hsa-rocr-dev switches to elfutils.
         openmp_common_args = [
             '-DROCM_DIR={0}'.format(hsa_prefix),
             '-DDEVICE_LIBS_DIR={0}/amdgcn/bitcode'.format(devlibs_prefix),
@@ -316,7 +333,9 @@ class RocmOpenmpExtras(Package):
             '-DOPENMP_ENABLE_LIBOMPTARGET=1',
             '-DOPENMP_ENABLE_LIBOMPTARGET_HSA=1',
             '-DLLVM_MAIN_INCLUDE_DIR={0}{1}'.format(src, llvm_inc),
-            '-DLLVM_INSTALL_PREFIX={0}'.format(llvm_prefix)
+            '-DLLVM_INSTALL_PREFIX={0}'.format(llvm_prefix),
+            '-DCMAKE_C_FLAGS=-isystem{0}'.format(elfutils_inc),
+            '-DCMAKE_CXX_FLAGS=-isystem{0}'.format(elfutils_inc)
         ]
 
         if self.spec.version < Version('4.1.0'):
@@ -349,6 +368,15 @@ class RocmOpenmpExtras(Package):
             '-DCMAKE_Fortran_COMPILER={0}/flang'.format(bin_dir),
             '-DLLVM_TARGETS_TO_BUILD=AMDGPU;x86'
         ]
+        if self.spec.version >= Version('4.2.0'):
+            # Spack thinks some warnings from the flang build are errors.
+            # Disable those warnings in C and CXX flags.
+            flang_common_args += [
+                '-DCMAKE_CXX_FLAGS={0}'.format(flang_warning) +
+                ' -I{0}{1}'.format(src, libpgmath),
+                '-DCMAKE_C_FLAGS={0}'.format(flang_warning) +
+                ' -I{0}{1}'.format(src, libpgmath)
+            ]
 
         components['pgmath'] = [
             '../rocm-openmp-extras/flang/runtime/libpgmath'