path: root/var/spack/repos/builtin/packages/aomp/package.py

# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)

import re

from spack.package import *

tools_url = "https://github.com/ROCm-Developer-Tools"
compute_url = "https://github.com/RadeonOpenCompute"


aomp = [
    "e4526489833896bbc47ba865e0d115fab278ce269789a8c99a97f444595f5f6a",
    "970374c3acb9dda8b9a17d7a579dbaab48fac731db8fdce566a65abee37e5ed3",
    "86f90d6505eccdb2840069cadf57f7111d4685653c4974cf65fb22b172e55478",
    "14fc6867af0b17e3bff8cb42cb36f509c95a29b7a933a106bf6778de21f6c123",
    "ce29cead5391a4a13f2c567e2e059de9291888d24985460725e43a91b740be7a",
]

devlib = [
    "dce3a4ba672c4a2da4c2260ee4dc96ff6dd51877f5e7e1993cb107372a35a378",
    "b3a114180bf184b3b829c356067bc6a98021d52c1c6f9db6bc57272ebafc5f1d",
    "e82cc9a8eb7d92de02cabb856583e28f17a05c8cf9c97aec5275608ef1a38574",
    "c99f45dacf5967aef9a31e3731011b9c142446d4a12bac69774998976f2576d7",
    "bca9291385d6bdc91a8b39a46f0fd816157d38abb1725ff5222e6a0daa0834cc",
]

llvm = [
    "b4fd7305dc57887eec17cce77bbf42215db46a4a3d14d8e517ab92f4e200b29d",
    "89b967de5e79f6df7c62fdc12529671fa30989ae7b634d5a7c7996629ec1140e",
    "98deabedb6cb3067ee960a643099631902507f236e4d9dc65b3e0f8d659eb55c",
    "f0a0b9fec0626878340a15742e73a56f155090011716461edcb069dcf05e6b30",
    "3ff18a8bd31d5b55232327e574dfa3556cf26787e105d0ba99411c5687325a8d",
]

flang = [
    "cc27f8bfb49257b7a4f0b03f4ba5e06a28dcb6c337065c4201b6075dd2d5bc48",
    "1fe07a0da20eb66a2a2aa8d354bf95c6f216ec38cc4a051e98041e0d13c34b36",
    "54cc6a9706dba6d7808258632ed40fa6493838edb309709d3b25e0f9b02507f8",
    "43d57bcc87fab092ac242e36da62588a87b6fa91f9e81fdb330159497afdecb3",
    "81674bf3c9d8fd9b16fb3e5c66a870537c25ff8302fc1b162ab9e95944167163",
]

extras = [
    "5dbf27f58b8114318208b97ba99a90483b78eebbcad4117cac6881441977e855",
    "adaf7670b2497ff3ac09636e0dd30f666a5a5b742ecdcb8551d722102dcfbd85",
    "4460a4f4b03022947f536221483e85dcd9b07064a54516ec103a1939c3f587b5",
    "014fca1fba54997c6db0e84822df274fb6807698b6856da4f737f38f10ab0e5d",
    "ee146cff4b9ee7aae90d7bb1d6b4957839232be0e7dab1865e0ae39832f8f795",
]

# Used only for 3.5.0
hip = ["86eb7749ff6f6c5f6851cd6c528504d42f9286967324a50dd0dd54a6a74cacc7"]

vdi = ["b21866c7c23dc536356db139b88b6beb3c97f58658836974a7fc167feb31ad7f"]

opencl = ["8963fcd5a167583b3db8b94363778d4df4593bfce8141e1d3c32a59fb64a0cf6"]

versions = ["3.5.0", "3.7.0", "3.8.0", "3.9.0", "3.10.0"]
versions_dict = dict()  # type: Dict[str, Dict[str, str]]
hashes = [aomp, devlib, llvm, flang, extras]
hashes_35 = [aomp, devlib, llvm, flang, extras, hip, vdi, opencl]
components = ["aomp", "devlib", "llvm", "flang", "extras"]
components_35 = ["aomp", "devlib", "llvm", "flang", "extras", "hip", "vdi", "opencl"]

for outer_index, item in enumerate(versions):
    if item == "3.5.0":
        use_components = components_35
        use_hashes = hashes_35
    else:
        use_components = components
        use_hashes = hashes
    for inner_index, component in enumerate(use_hashes):
        versions_dict.setdefault(item, {})[use_components[inner_index]] = use_hashes[inner_index][
            outer_index
        ]


class Aomp(Package):
    """llvm openmp compiler from AMD."""

    homepage = tools_url + "/aomp"
    url = tools_url + "/aomp/archive/rocm-3.10.0.tar.gz"

    maintainers("srekolam", "arjun-raj-kuppala", "estewart08")

    tags = ["e4s"]

    version("3.10.0", sha256=versions_dict["3.10.0"]["aomp"])
    version("3.9.0", sha256=versions_dict["3.9.0"]["aomp"])

    # Cmake above 3.18 would fail the build on 3.5.0
    depends_on("cmake@3:", type="build")
    depends_on("cmake@3:3.17", when="@3.5.0", type="build")

    depends_on("python@3:", type="build", when="@3.9.0:")
    depends_on("py-setuptools", when="@3.9.0:", type="build")

    depends_on("gl@4.5:", type=("build", "link"))
    depends_on("py-pip", when="@3.8.0:", type="build")
    depends_on("py-wheel", when="@3.8.0:", type=("build", "run"))
    depends_on("perl-data-dumper", type="build")
    depends_on("awk", type="build")
    depends_on("elfutils", type=("build", "link"))
    depends_on("libffi", type=("build", "link"))

    for ver in ["3.5.0", "3.7.0", "3.8.0", "3.9.0", "3.10.0"]:
        depends_on("hsakmt-roct@" + ver, when="@" + ver)
        depends_on("comgr@" + ver, type="build", when="@" + ver)
        depends_on("hsa-rocr-dev@" + ver, when="@" + ver)
        depends_on("rocm-device-libs@" + ver, when="@" + ver)

        if ver != "3.5.0":
            depends_on("hip@" + ver, when="@" + ver)
            depends_on("hip-rocclr@" + ver, when="@" + ver)

        if ver == "3.9.0" or ver == "3.10.0":
            depends_on("rocm-gdb@" + ver, when="@" + ver)

        resource(
            name="rocm-device-libs",
            url=compute_url + "/ROCm-Device-Libs/archive/rocm-" + ver + ".tar.gz",
            sha256=versions_dict[ver]["devlib"],
            expand=True,
            destination="aomp-dir",
            placement="rocm-device-libs",
            when="@" + ver,
        )

        resource(
            name="amd-llvm-project",
            url=tools_url + "/amd-llvm-project/archive/rocm-" + ver + ".tar.gz",
            sha256=versions_dict[ver]["llvm"],
            expand=True,
            destination="aomp-dir",
            placement="amd-llvm-project",
            when="@" + ver,
        )

        resource(
            name="flang",
            url=tools_url + "/flang/archive/rocm-" + ver + ".tar.gz",
            sha256=versions_dict[ver]["flang"],
            expand=True,
            destination="aomp-dir",
            placement="flang",
            when="@" + ver,
        )

        resource(
            name="aomp-extras",
            url=tools_url + "/aomp-extras/archive/rocm-" + ver + ".tar.gz",
            sha256=versions_dict[ver]["extras"],
            expand=True,
            destination="aomp-dir",
            placement="aomp-extras",
            when="@" + ver,
        )

        if ver == "3.5.0":
            resource(
                name="hip-on-vdi",
                url=tools_url + "/hip/archive/aomp-3.5.0.tar.gz",
                sha256=versions_dict["3.5.0"]["hip"],
                expand=True,
                destination="aomp-dir",
                placement="hip-on-vdi",
                when="@3.5.0",
            )

            resource(
                name="vdi",
                url=tools_url + "/rocclr/archive/aomp-3.5.0.tar.gz",
                sha256=versions_dict["3.5.0"]["vdi"],
                expand=True,
                destination="aomp-dir",
                placement="vdi",
                when="@3.5.0",
            )

            resource(
                name="opencl-on-vdi",
                sha256=versions_dict["3.5.0"]["opencl"],
                url=compute_url + "/ROCm-OpenCL-Runtime/archive/aomp-3.5.0.tar.gz",
                expand=True,
                destination="aomp-dir",
                placement="opencl-on-vdi",
                when="@3.5.0",
            )

    # Copy source files over for debug build in 3.9.0
    patch(
        "0001-Add-cmake-option-for-copying-source-for-debugging.patch",
        working_dir="aomp-dir/amd-llvm-project",
        when="@3.9.0:",
    )

    # Revert back to .amdgcn.bc naming scheme for 3.8.0
    patch(
        "0001-Add-amdgcn-to-devicelibs-bitcode-names-3.8.patch",
        working_dir="aomp-dir/amd-llvm-project",
        when="@3.8.0",
    )

    # Revert back to .amdgcn.bc naming scheme for 3.7.0
    patch(
        "0001-Add-amdgcn-to-devicelibs-bitcode-names.patch",
        working_dir="aomp-dir/amd-llvm-project",
        when="@3.7.0",
    )

    def patch(self):
        # Make sure python2.7 is used for the generation of hip header
        if self.spec.version == Version("3.5.0"):
            kwargs = {"ignore_absent": False, "backup": False, "string": False}
            with working_dir("aomp-dir/hip-on-vdi"):
                match = "^#!/usr/bin/python"
                python = self.spec["python"].command.path
                substitute = "#!{python}".format(python=python)
                files = ["hip_prof_gen.py", "vdi/hip_prof_gen.py"]
                filter_file(match, substitute, *files, **kwargs)
        src = self.stage.source_path
        libomptarget = "{0}/aomp-dir/amd-llvm-project/openmp/libomptarget"
        aomp_extras = "{0}/aomp-dir/aomp-extras/aomp-device-libs"
        flang = "{0}/aomp-dir/flang/"

        if self.spec.version >= Version("3.9.0"):
            filter_file(
                "ADDITIONAL_VERSIONS 2.7",
                "ADDITIONAL_VERSIONS 3",
                flang.format(src) + "CMakeLists.txt",
            )

        if self.spec.version >= Version("3.8.0"):
            filter_file(
                "{CMAKE_INSTALL_PREFIX}",
                "{HSA_INCLUDE}",
                libomptarget.format(src) + "/hostrpc/services/CMakeLists.txt",
            )

            filter_file(
                "CONFIG",
                "CONFIG PATHS ${CMAKE_INSTALL_PREFIX} NO_DEFAULT_PATH",
                libomptarget.format(src) + "/../libompd/test/CMakeLists.txt",
            )

        if self.spec.version != Version("3.5.0"):
            filter_file(
                "{ROCM_DIR}/aomp/amdgcn/bitcode",
                "{DEVICE_LIBS_DIR}",
                libomptarget.format(src) + "/hostrpc/CMakeLists.txt",
                libomptarget.format(src) + "/deviceRTLs/amdgcn/CMakeLists.txt",
            )

        if self.spec.version == Version("3.5.0"):
            filter_file(
                "{ROCM_DIR}/lib/bitcode",
                "{DEVICE_LIBS_DIR}",
                libomptarget.format(src) + "/deviceRTLs/hostcall/CMakeLists.txt",
            )

        filter_file(
            "{ROCM_DIR}/lib/bitcode",
            "{DEVICE_LIBS_DIR}",
            aomp_extras.format(src) + "/aompextras/CMakeLists.txt",
            aomp_extras.format(src) + "/libm/CMakeLists.txt",
            libomptarget.format(src) + "/deviceRTLs/amdgcn/CMakeLists.txt",
            string=True,
        )

        filter_file(
            r"${ROCM_DIR}/hsa/include ${ROCM_DIR}/hsa/include/hsa",
            "${HSA_INCLUDE}/hsa/include ${HSA_INCLUDE}/hsa/include/hsa",
            libomptarget.format(src) + "/plugins/hsa/CMakeLists.txt",
            string=True,
        )

        filter_file(
            "{ROCM_DIR}/hsa/lib",
            "{HSA_LIB}",
            libomptarget.format(src) + "/plugins/hsa/CMakeLists.txt",
        )

        filter_file(
            r"{ROCM_DIR}/lib\)",
            "{HSAKMT_LIB})\nset(HSAKMT_LIB64 ${HSAKMT_LIB64})",
            libomptarget.format(src) + "/plugins/hsa/CMakeLists.txt",
        )

        filter_file(
            r"-L${LIBOMPTARGET_DEP_LIBHSAKMT_LIBRARIES_DIRS}",
            "-L${LIBOMPTARGET_DEP_LIBHSAKMT_LIBRARIES_DIRS} -L${HSAKMT_LIB64}",
            libomptarget.format(src) + "/plugins/hsa/CMakeLists.txt",
            string=True,
        )

        filter_file(
            r"-rpath,${LIBOMPTARGET_DEP_LIBHSAKMT_LIBRARIES_DIRS}",
            "-rpath,${LIBOMPTARGET_DEP_LIBHSAKMT_LIBRARIES_DIRS}" + ",-rpath,${HSAKMT_LIB64}",
            libomptarget.format(src) + "/plugins/hsa/CMakeLists.txt",
            string=True,
        )

        filter_file(
            "{ROCM_DIR}/include",
            "{COMGR_INCLUDE}",
            libomptarget.format(src) + "/plugins/hsa/CMakeLists.txt",
        )

        filter_file(
            "{ROCM_DIR}/include",
            "{COMGR_INCLUDE}",
            libomptarget.format(src) + "/plugins/hsa/CMakeLists.txt",
        )

        filter_file(
            r"-L${LLVM_LIBDIR}${OPENMP_LIBDIR_SUFFIX}",
            "-L${LLVM_LIBDIR}${OPENMP_LIBDIR_SUFFIX} -L${COMGR_LIB}",
            libomptarget.format(src) + "/plugins/hsa/CMakeLists.txt",
            string=True,
        )

        filter_file(
            r"rpath,${LLVM_LIBDIR}${OPENMP_LIBDIR_SUFFIX}",
            "rpath,${LLVM_LIBDIR}${OPENMP_LIBDIR_SUFFIX}" + "-Wl,-rpath,${COMGR_LIB}",
            libomptarget.format(src) + "/plugins/hsa/CMakeLists.txt",
            string=True,
        )

    def setup_run_environment(self, env):
        devlibs_prefix = self.spec["rocm-device-libs"].prefix
        aomp_prefix = self.spec["aomp"].prefix
        env.set("HIP_DEVICE_LIB_PATH", "{0}/amdgcn/bitcode".format(format(devlibs_prefix)))
        env.set("AOMP", "{0}".format(format(aomp_prefix)))

    def setup_build_environment(self, env):
        aomp_prefix = self.spec["aomp"].prefix
        env.set("AOMP", "{0}".format(format(aomp_prefix)))
        env.set("FC", "{0}/bin/flang".format(format(aomp_prefix)))
        env.set("GFXLIST", "gfx700 gfx701 gfx801 gfx803 gfx900 gfx902 gfx906 gfx908")

    def install(self, spec, prefix):
        src = self.stage.source_path
        gfx_list = "gfx700;gfx701;gfx801;gfx803;gfx900;gfx902;gfx906;gfx908"
        aomp_prefix = self.spec["aomp"].prefix
        devlibs_prefix = self.spec["rocm-device-libs"].prefix
        hsa_prefix = self.spec["hsa-rocr-dev"].prefix
        hsakmt_prefix = self.spec["hsakmt-roct"].prefix
        comgr_prefix = self.spec["comgr"].prefix
        opencl_src = "/aomp-dir/opencl-on-vdi/api/opencl"
        omp_src = "/aomp-dir/amd-llvm-project/openmp"
        debug_map_format = "-fdebug-prefix-map={0}{1}={2}".format(src, omp_src, aomp_prefix)

        if self.spec.version >= Version("3.9.0"):
            bitcode_dir = "/amdgcn/bitcode"
        else:
            bitcode_dir = "/lib"

        components = dict()
        components["amd-llvm-project"] = [
            "../aomp-dir/amd-llvm-project/llvm",
            "-DLLVM_ENABLE_PROJECTS=clang;lld;compiler-rt",
            "-DCMAKE_BUILD_TYPE=release",
            "-DLLVM_ENABLE_ASSERTIONS=ON",
            "-DLLVM_TARGETS_TO_BUILD=AMDGPU;X86",
            "-DCMAKE_C_COMPILER={0}".format(self.compiler.cc),
            "-DCMAKE_CXX_COMPILER={0}".format(self.compiler.cxx),
            "-DCMAKE_ASM_COMPILER={0}".format(self.compiler.cc),
            "-DBUG_REPORT_URL=https://github.com/ROCm-Developer-Tools/aomp",
            "-DLLVM_ENABLE_BINDINGS=OFF",
            "-DLLVM_INCLUDE_BENCHMARKS=OFF",
            "-DLLVM_BUILD_TESTS=OFF",
            "-DLLVM_INCLUDE_TESTS=OFF",
            "-DCLANG_INCLUDE_TESTS=OFF",
            "-DCMAKE_VERBOSE_MAKEFILE=1",
            "-DCMAKE_INSTALL_RPATH_USE_LINK_PATH=FALSE",
        ]

        if self.spec.version == Version("3.5.0"):
            components["vdi"] = [
                "../aomp-dir/vdi",
                "-DUSE_COMGR_LIBRARY=yes",
                "-DOPENCL_DIR={0}{1}".format(src, opencl_src),
            ]

            components["hip-on-vdi"] = [
                "../aomp-dir/hip-on-vdi",
                "-DVDI_ROOT={0}/aomp-dir/vdi".format(src),
                "-DHIP_COMPILER=clang",
                "-DHIP_PLATFORM=vdi",
                "-DVDI_DIR={0}/aomp-dir/vdi".format(src),
                "-DHSA_PATH={0}".format(hsa_prefix),
                "-DLIBVDI_STATIC_DIR={0}/spack-build-vdi".format(src),
                "-DCMAKE_CXX_FLAGS=-Wno-ignored-attributes",
            ]

        components["aomp-extras"] = [
            "../aomp-dir/aomp-extras",
            "-DROCM_PATH=$ROCM_DIR ",
            "-DDEVICE_LIBS_DIR={0}{1}".format(devlibs_prefix, bitcode_dir),
            "-DAOMP_STANDALONE_BUILD=0",
            "-DDEVICELIBS_ROOT={0}/aomp-dir/rocm-device-libs".format(src),
            "-DCMAKE_VERBOSE_MAKEFILE=1",
        ]

        openmp_common_args = [
            "-DROCM_DIR={0}".format(hsa_prefix),
            "-DDEVICE_LIBS_DIR={0}{1}".format(devlibs_prefix, bitcode_dir),
            "-DAOMP_STANDALONE_BUILD=0",
            "-DDEVICELIBS_ROOT={0}/aomp-dir/rocm-device-libs".format(src),
            "-DOPENMP_TEST_C_COMPILER={0}/bin/clang".format(aomp_prefix),
            "-DOPENMP_TEST_CXX_COMPILER={0}/bin/clang++".format(aomp_prefix),
            "-DLIBOMPTARGET_AMDGCN_GFXLIST={0}".format(gfx_list),
            "-DLIBOMP_COPY_EXPORTS=OFF",
            "-DHSA_INCLUDE={0}".format(hsa_prefix),
            "-DHSA_LIB={0}/lib".format(hsa_prefix),
            "-DHSAKMT_LIB={0}/lib".format(hsakmt_prefix),
            "-DHSAKMT_LIB64={0}/lib64".format(hsakmt_prefix),
            "-DCOMGR_INCLUDE={0}/include".format(comgr_prefix),
            "-DCOMGR_LIB={0}/lib".format(comgr_prefix),
            "-DOPENMP_ENABLE_LIBOMPTARGET=1",
            "-DOPENMP_ENABLE_LIBOMPTARGET_HSA=1",
        ]

        components["openmp"] = ["../aomp-dir/amd-llvm-project/openmp"]
        components["openmp"] += openmp_common_args

        components["openmp-debug"] = [
            "../aomp-dir/amd-llvm-project/openmp",
            "-DLIBOMPTARGET_NVPTX_DEBUG=ON",
            "-DOPENMP_ENABLE_LIBOMPTARGET=1",
            "-DOPENMP_ENABLE_LIBOMPTARGET_HSA=1" "-DCMAKE_CXX_FLAGS=-g",
            "-DCMAKE_C_FLAGS=-g",
        ]

        if self.spec.version >= Version("3.9.0"):
            components["openmp-debug"] += [
                "-DENABLE_SOURCE_COPY=ON",
                "-DOPENMP_SOURCE_DEBUG_MAP={0}".format(debug_map_format),
            ]

        if self.spec.version >= Version("3.8.0"):
            components["openmp-debug"] += [
                "-DLIBOMP_ARCH=x86_64",
                "-DLIBOMP_OMP_VERSION=50",
                "-DLIBOMP_OMPT_SUPPORT=ON",
                "-DLIBOMP_USE_DEBUGGER=ON",
                "-DLIBOMP_CFLAGS=-O0",
                "-DLIBOMP_CPPFLAGS=-O0",
                "-DLIBOMP_OMPD_ENABLED=ON",
                "-DLIBOMP_OMPD_SUPPORT=ON",
                "-DLIBOMP_OMPT_DEBUG=ON",
            ]

        components["openmp-debug"] += openmp_common_args

        flang_common_args = [
            "-DLLVM_ENABLE_ASSERTIONS=ON",
            "-DLLVM_CONFIG={0}/bin/llvm-config".format(aomp_prefix),
            "-DCMAKE_CXX_COMPILER={0}/bin/clang++".format(aomp_prefix),
            "-DCMAKE_C_COMPILER={0}/bin/clang".format(aomp_prefix),
            "-DCMAKE_Fortran_COMPILER={0}/bin/flang".format(aomp_prefix),
            "-DLLVM_TARGETS_TO_BUILD=AMDGPU;x86",
        ]

        components["pgmath"] = ["../aomp-dir/flang/runtime/libpgmath"]
        components["pgmath"] += flang_common_args

        components["flang"] = [
            "../aomp-dir/flang",
            "-DFLANG_OPENMP_GPU_AMD=ON",
            "-DFLANG_OPENMP_GPU_NVIDIA=ON",
        ]

        components["flang"] += flang_common_args

        components["flang-runtime"] = [
            "../aomp-dir/flang",
            "-DLLVM_INSTALL_RUNTIME=ON",
            "-DFLANG_BUILD_RUNTIME=ON",
            "-DOPENMP_BUILD_DIR={0}/spack-build-openmp/runtime/src".format(src),
        ]

        components["flang-runtime"] += flang_common_args

        if self.spec.version != Version("3.5.0"):
            build_order = [
                "amd-llvm-project",
                "aomp-extras",
                "openmp",
                "openmp-debug",
                "pgmath",
                "flang",
                "flang-runtime",
            ]
        elif self.spec.version == Version("3.5.0"):
            build_order = [
                "amd-llvm-project",
                "vdi",
                "hip-on-vdi",
                "aomp-extras",
                "openmp",
                "openmp-debug",
                "pgmath",
                "flang",
                "flang-runtime",
            ]

        # Override standard CMAKE_BUILD_TYPE
        for arg in std_cmake_args:
            found = re.search("CMAKE_BUILD_TYPE", arg)
            if found:
                std_cmake_args.remove(arg)

        for component in build_order:
            with working_dir("spack-build-{0}".format(component), create=True):
                cmake_args = components[component]
                cmake_args.extend(std_cmake_args)
                # OpenMP build needs to be run twice(Release, Debug)
                if component == "openmp-debug":
                    cmake_args.append("-DCMAKE_BUILD_TYPE=Debug")
                else:
                    cmake_args.append("-DCMAKE_BUILD_TYPE=Release")
                cmake(*cmake_args)
                make()
                make("install")