From f57626a7c4a05cc11972a43eba498a943b17a08d Mon Sep 17 00:00:00 2001 From: Robert Cohn Date: Mon, 22 Mar 2021 20:35:45 -0400 Subject: Oneapi packages: update URLs, environment management, and dependencies (#22202) * Replace URL computation in base IntelOneApiPackage class with defining URLs in component packages (this is expected to be simpler for now) * Add component_dir property that all oneAPI component packages must define. This property names a directory that should exist after installation completes (useful for making sure the install was successful) and also defines the search location for the component's environment update script. * Add needed dependencies for components (e.g. intel-oneapi-dnn requires intel-oneapi-tbb). The compilers provided by intel-oneapi-compilers need some components under certain circumstances (e.g. when enabling SYCL support) but these were omitted since the libraries should only be linked when a dependent package requests that feature * Remove individual setup_run_environment implementations and use IntelOneApiPackage superclass method which sources vars.sh (located in a subdirectory of component_dir) * Add documentation for IntelOneApiPackge build system Co-authored-by: Vasily Danilin --- lib/spack/docs/build_systems.rst | 1 + .../docs/build_systems/inteloneapipackage.rst | 137 +++++++++++++++++++++ lib/spack/docs/build_systems/intelpackage.rst | 5 + lib/spack/spack/build_systems/oneapi.py | 78 ++++++------ 4 files changed, 185 insertions(+), 36 deletions(-) create mode 100644 lib/spack/docs/build_systems/inteloneapipackage.rst (limited to 'lib') diff --git a/lib/spack/docs/build_systems.rst b/lib/spack/docs/build_systems.rst index 784730ce40..59b9bb643a 100644 --- a/lib/spack/docs/build_systems.rst +++ b/lib/spack/docs/build_systems.rst @@ -59,6 +59,7 @@ on these ideas for each distinct build system that Spack supports: build_systems/bundlepackage build_systems/cudapackage + build_systems/inteloneapipackage build_systems/intelpackage build_systems/rocmpackage build_systems/custompackage diff --git a/lib/spack/docs/build_systems/inteloneapipackage.rst b/lib/spack/docs/build_systems/inteloneapipackage.rst new file mode 100644 index 0000000000..036fc738ad --- /dev/null +++ b/lib/spack/docs/build_systems/inteloneapipackage.rst @@ -0,0 +1,137 @@ +.. Copyright 2013-2021 Lawrence Livermore National Security, LLC and other + Spack Project Developers. See the top-level COPYRIGHT file for details. + + SPDX-License-Identifier: (Apache-2.0 OR MIT) + +.. _inteloneapipackage: + + +==================== + IntelOneapiPackage +==================== + + +.. contents:: + + +oneAPI packages in Spack +======================== + +Spack can install and use the Intel oneAPI products. You may either +use spack to install the oneAPI tools or use the `Intel +installers`_. After installation, you may use the tools directly, or +use Spack to build packages with the tools. + +The Spack Python class ``IntelOneapiPackage`` is a base class that is +used by ``IntelOneapiCompilers``, ``IntelOneapiMkl``, +``IntelOneapiTbb`` and other classes to implement the oneAPI +packages. See the :ref: for the full list of available +oneAPI packages or use:: + + spack list -d oneAPI + +For more information on a specific package, do:: + + spack info + +Intel no longer releases new versions of Parallel Studio, which can be +used in Spack via the :ref:. All of its components can +now be found in oneAPI. + +Example +======= + +We start with a simple example that will be sufficient for most +users. Install the oneAPI compilers:: + + spack install intel-oneapi-compilers + +Add the oneAPI compilers to the set of compilers that Spack can use:: + + spack compiler add `spack location -i intel-oneapi-compilers`/compiler/latest/linux/bin/intel64 + spack compiler add `spack location -i intel-oneapi-compilers`/compiler/latest/linux/bin + +This adds the compilers to your ``compilers.yaml``. Verify that the +compilers are available:: + + spack compiler list + +The ``intel-oneapi-compilers`` package includes 2 families of +compilers: + +* ``intel``: ``icc``, ``icpc``, ``ifort``. Intel's *classic* + compilers. +* ``oneapi``: ``icx``, ``icpx``, ``ifx``. Intel's new generation of + compilers based on LLVM. + +To build the ``patchelf`` Spack package with ``icc``, do:: + + spack install patchelf%intel + +To build with with ``icx``, do :: + + spack install patchelf%oneapi + +In addition to compilers, oneAPI contains many libraries. The ``hdf5`` +package works with any compatible MPI implementation. To build +``hdf5`` with Intel oneAPI MPI do:: + + spack install hdf5 +mpi ^intel-oneapi-mpi + +Using an Externally Installed oneAPI +==================================== + +Spack can also use oneAPI tools that are manually installed with +`Intel Installers`_. The procedures for configuring Spack to use +external compilers and libraries are different. + +Compilers +--------- + +To use the compilers, add some information about the installation to +``compilers.yaml``. For most users, it is sufficient to do:: + + spack compiler add /opt/intel/oneapi/compiler/latest/linux/bin/intel64 + spack compiler add /opt/intel/oneapi/compiler/latest/linux/bin + +Adapt the paths above if you did not install the tools in the default +location. After adding the compilers, using them in Spack will be +exactly the same as if you had installed the +``intel-oneapi-compilers`` package. Another option is to manually add +the configuration to ``compilers.yaml`` as described in :ref:`Compiler +configuration `. + + +Using oneAPI Tools Installed by Spack +===================================== + +Spack can be a convenient way to install and configure compilers and +libaries, even if you do not intend to build a Spack package. If you +want to build a Makefile project using Spack-installed oneAPI compilers, +then use spack to configure your environment:: + + spack load intel-oneapi-compilers + +And then you can build with:: + + CXX=icpx make + +You can also use Spack-installed libraries. For example:: + + spack load intel-oneapi-mkl + +Will update your environment CPATH, LIBRARY_PATH, and other +environment variables for building an application with MKL. + +More information +================ + +This section describes basic use of oneAPI, especially if it has +changed compared to Parallel Studio. See :ref: for more +information on :ref:, +:ref:, +:ref:, and +:ref:. + + +.. _`Intel installers`: https://software.intel.com/content/www/us/en/develop/documentation/installation-guide-for-intel-oneapi-toolkits-linux/top.html diff --git a/lib/spack/docs/build_systems/intelpackage.rst b/lib/spack/docs/build_systems/intelpackage.rst index 2c1723bbbc..2d7187bf81 100644 --- a/lib/spack/docs/build_systems/intelpackage.rst +++ b/lib/spack/docs/build_systems/intelpackage.rst @@ -137,6 +137,7 @@ If you need to save disk space or installation time, you could install the ``intel`` compilers-only subset (0.6 GB) and just the library packages you need, for example ``intel-mpi`` (0.5 GB) and ``intel-mkl`` (2.5 GB). +.. _intel-unrelated-packages: """""""""""""""""""" Unrelated packages @@ -358,6 +359,8 @@ affected by an advanced third method: Next, visit section `Selecting Intel Compilers`_ to learn how to tell Spack to use the newly configured compilers. +.. _intel-integrating-external-libraries: + """""""""""""""""""""""""""""""""" Integrating external libraries """""""""""""""""""""""""""""""""" @@ -834,6 +837,7 @@ for example: compiler: [ intel@18, intel@17, gcc@4.4.7, gcc@4.9.3, gcc@7.3.0, ] +.. _intel-virtual-packages: """""""""""""""""""""""""""""""""""""""""""""""" Selecting libraries to satisfy virtual packages @@ -907,6 +911,7 @@ With the proper installation as detailed above, no special steps should be required when a client package specifically (and thus deliberately) requests an Intel package as dependency, this being one of the target use cases for Spack. +.. _using-mkl-tips: """"""""""""""""""""""""""""""""""""""""""""""" Tips for configuring client packages to use MKL diff --git a/lib/spack/spack/build_systems/oneapi.py b/lib/spack/spack/build_systems/oneapi.py index 5babdc5a75..e1a0c66a7e 100644 --- a/lib/spack/spack/build_systems/oneapi.py +++ b/lib/spack/spack/build_systems/oneapi.py @@ -7,9 +7,11 @@ """ -from os.path import dirname, isdir +from sys import platform +from os.path import basename, dirname, isdir, join from spack.package import Package +from spack.util.environment import EnvironmentModifications from spack.util.executable import Executable from llnl.util.filesystem import find_headers, find_libraries @@ -22,46 +24,50 @@ class IntelOneApiPackage(Package): phases = ['install'] - def component_info(self, - dir_name, - components, - releases, - url_name): - self._dir_name = dir_name - self._components = components - self._releases = releases - self._url_name = url_name + # oneAPI license does not allow mirroring outside of the + # organization (e.g. University/Company). + redistribute_source = False - def url_for_version(self, version): - release = self._release(version) - return 'https://registrationcenter-download.intel.com/akdlm/irc_nas/%s/%s' % ( - release['irc_id'], self._oneapi_file(version, release)) + @property + def component_dir(self): + """Subdirectory for this component in the install prefix.""" + raise NotImplementedError + + def install(self, spec, prefix, installer_path=None): + """Shared install method for all oneapi packages.""" + + # intel-oneapi-compilers overrides the installer_path when + # installing fortran, which comes from a spack resource + if installer_path is None: + installer_path = basename(self.url_for_version(spec.version)) + + if platform == 'linux': + bash = Executable('bash') + + # Installer writes files in ~/intel set HOME so it goes to prefix + bash.add_default_env('HOME', prefix) + + bash(installer_path, + '-s', '-a', '-s', '--action', 'install', + '--eula', 'accept', + '--install-dir', prefix) - def install(self, spec, prefix): - bash = Executable('bash') + # Some installers have a bug and do not return an error code when failing + if not isdir(join(prefix, self.component_dir)): + raise RuntimeError('install failed') - # Installer writes files in ~/intel set HOME so it goes to prefix - bash.add_default_env('HOME', prefix) + def setup_run_environment(self, env): - version = spec.versions.lowest() - release = self._release(version) - bash('./%s' % self._oneapi_file(version, release), - '-s', '-a', '-s', '--action', 'install', - '--eula', 'accept', - '--components', - self._components, - '--install-dir', prefix) + """Adds environment variables to the generated module file. - # - # Helper functions - # + These environment variables come from running: - def _release(self, version): - return self._releases[str(version)] + .. code-block:: console - def _oneapi_file(self, version, release): - return 'l_%s_p_%s.%s_offline.sh' % ( - self._url_name, version, release['build']) + $ source {prefix}/setvars.sh --force + """ + env.extend(EnvironmentModifications.from_sourcing_file( + join(self.prefix, self.component_dir, 'latest/env/vars.sh'))) class IntelOneApiLibraryPackage(IntelOneApiPackage): @@ -70,11 +76,11 @@ class IntelOneApiLibraryPackage(IntelOneApiPackage): @property def headers(self): include_path = '%s/%s/latest/include' % ( - self.prefix, self._dir_name) + self.prefix, self.component_dir) return find_headers('*', include_path, recursive=True) @property def libs(self): - lib_path = '%s/%s/latest/lib/intel64' % (self.prefix, self._dir_name) + lib_path = '%s/%s/latest/lib/intel64' % (self.prefix, self.component_dir) lib_path = lib_path if isdir(lib_path) else dirname(lib_path) return find_libraries('*', root=lib_path, shared=True, recursive=True) -- cgit v1.2.3-70-g09d2