From db657d938d38775e4364a6917cc78cb9cdb0b133 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Wed, 16 Aug 2017 12:21:07 -0500 Subject: Refactor IntelInstaller into IntelPackage base class (#4300) * Refactor IntelInstaller into IntelPackage base class * Move license attributes from __init__ to class-level * Flake8 fixes: remove unused imports * Fix logic that writes the silent.cfg file * More specific version numbers for Intel MPI * Rework logic that selects components to install * Final changes necessary to get intel package working * Various updates to intel-parallel-studio * Add latest version of every Intel package * Add environment variables for Intel packages * Update env vars for intel package * Finalize components for intel-parallel-studio package Adds a +tbb variant to intel-parallel-studio. The tbb package was renamed to intel-tbb. Now both intel-tbb and intel-parallel-studio+tbb provide tbb. * Overhaul environment variables set by intel-parallel-studio * Point dependent packages to the correct MPI wrappers * Never default to intel-parallel-studio * Gather env vars by sourcing setup scripts * Use mpiicc instead of mpicc when using Intel compiler * Undo change to ARCH * Add changes from intel-mpi to intel-parallel-studio * Add comment explaining mpicc vs mpiicc * Prepend env vars containing 'PATH' or separators * Flake8 fix * Fix bugs in from_sourcing_file * Indentation fix * Prepend, not set if contains separator * Fix license symlinking broken by changes to intel-parallel-studio * Use comments instead of docstrings to document attributes * Flake8 fixes * Use a set instead of a list to prevent duplicate components * Fix MKL and MPI library linking directories * Remove +all variant from intel-parallel-studio * It is not possible to build with MKL, GCC, and OpenMP at this time * Found a workaround for locating GCC libraries * Typos and variable names * Fix initialization of empty LibraryList --- lib/spack/docs/packaging_guide.rst | 3 + lib/spack/spack/__init__.py | 2 + lib/spack/spack/build_environment.py | 4 +- lib/spack/spack/build_systems/intel.py | 192 +++++++++++++++++++++++++++++++++ lib/spack/spack/cmd/configure.py | 1 + lib/spack/spack/cmd/create.py | 10 ++ lib/spack/spack/environment.py | 182 ++++++++++++++++++------------- lib/spack/spack/package.py | 73 +++++++------ lib/spack/spack/test/environment.py | 11 +- 9 files changed, 366 insertions(+), 112 deletions(-) create mode 100644 lib/spack/spack/build_systems/intel.py (limited to 'lib') diff --git a/lib/spack/docs/packaging_guide.rst b/lib/spack/docs/packaging_guide.rst index 7e789bc294..e400272e59 100644 --- a/lib/spack/docs/packaging_guide.rst +++ b/lib/spack/docs/packaging_guide.rst @@ -2136,6 +2136,9 @@ The classes that are currently provided by Spack are: | :py:class:`.PerlPackage` | Specialized class for | | | :py:class:`.Perl` extensions | +-------------------------------+----------------------------------+ + | :py:class:`.IntelPackage` | Specialized class for licensed | + | | Intel software | + +-------------------------------+----------------------------------+ .. note:: diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index e8a010bb26..21280f0001 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -178,6 +178,7 @@ from spack.build_systems.waf import WafPackage from spack.build_systems.python import PythonPackage from spack.build_systems.r import RPackage from spack.build_systems.perl import PerlPackage +from spack.build_systems.intel import IntelPackage __all__ += [ 'run_before', @@ -193,6 +194,7 @@ __all__ += [ 'PythonPackage', 'RPackage', 'PerlPackage', + 'IntelPackage', ] from spack.version import Version, ver diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 34c9cd56d2..620445fe1c 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -229,7 +229,7 @@ def set_build_environment_variables(pkg, env, dirty=False): # Install root prefix env.set(SPACK_INSTALL, spack.store.root) - # Stuff in here sanitizes the build environemnt to eliminate + # Stuff in here sanitizes the build environment to eliminate # anything the user has set that may interfere. if not dirty: # Remove these vars from the environment during build because they @@ -518,7 +518,7 @@ def fork(pkg, function, dirty=False): Args: - pkg (PackageBase): package whose environemnt we should set up the + pkg (PackageBase): package whose environment we should set up the forked process for. function (callable): argless function to run in the child process. diff --git a/lib/spack/spack/build_systems/intel.py b/lib/spack/spack/build_systems/intel.py new file mode 100644 index 0000000000..a97f15d62c --- /dev/null +++ b/lib/spack/spack/build_systems/intel.py @@ -0,0 +1,192 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## + +import os +import xml.etree.ElementTree as ET + +from llnl.util.filesystem import install, join_path +from spack.package import PackageBase, run_after +from spack.util.executable import Executable + + +def _valid_components(): + """A generator that yields valid components.""" + + tree = ET.parse('pset/mediaconfig.xml') + root = tree.getroot() + + components = root.findall('.//Abbr') + for component in components: + yield component.text + + +class IntelPackage(PackageBase): + """Specialized class for licensed Intel software. + + This class provides two phases that can be overridden: + + 1. :py:meth:`~.IntelPackage.configure` + 2. :py:meth:`~.IntelPackage.install` + + They both have sensible defaults and for many packages the + only thing necessary will be to override ``setup_environment`` + to set the appropriate environment variables. + """ + #: Phases of an Intel package + phases = ['configure', 'install'] + + #: This attribute is used in UI queries that need to know the build + #: system base class + build_system_class = 'IntelPackage' + + #: By default, we assume that all Intel software requires a license. + #: This can be overridden for packages that do not require a license. + license_required = True + + #: Comment symbol used in the ``license.lic`` file + license_comment = '#' + + #: Location where Intel searches for a license file + license_files = ['Licenses/license.lic'] + + #: Environment variables that Intel searches for a license file + license_vars = ['INTEL_LICENSE_FILE'] + + #: URL providing information on how to acquire a license key + license_url = 'https://software.intel.com/en-us/articles/intel-license-manager-faq' + + #: Components of the package to install. + #: By default, install 'ALL' components. + components = ['ALL'] + + @property + def _filtered_components(self): + """Returns a list or set of valid components that match + the requested components from ``components``.""" + + # Don't filter 'ALL' + if self.components == ['ALL']: + return self.components + + # mediaconfig.xml is known to contain duplicate components. + # If more than one copy of the same component is used, you + # will get an error message about invalid components. + # Use a set to store components to prevent duplicates. + matches = set() + + for valid in _valid_components(): + for requested in self.components: + if valid.startswith(requested): + matches.add(valid) + + return matches + + @property + def global_license_file(self): + """Returns the path where a global license file should be stored. + + All Intel software shares the same license, so we store it in a + common 'intel' directory.""" + return join_path(self.global_license_dir, 'intel', + os.path.basename(self.license_files[0])) + + def configure(self, spec, prefix): + """Writes the ``silent.cfg`` file used to configure the installation. + + See https://software.intel.com/en-us/articles/configuration-file-format + """ + # Patterns used to check silent configuration file + # + # anythingpat - any string + # filepat - the file location pattern (/path/to/license.lic) + # lspat - the license server address pattern (0123@hostname) + # snpat - the serial number pattern (ABCD-01234567) + config = { + # Accept EULA, valid values are: {accept, decline} + 'ACCEPT_EULA': 'accept', + + # Optional error behavior, valid values are: {yes, no} + 'CONTINUE_WITH_OPTIONAL_ERROR': 'yes', + + # Install location, valid values are: {/opt/intel, filepat} + 'PSET_INSTALL_DIR': prefix, + + # Continue with overwrite of existing installation directory, + # valid values are: {yes, no} + 'CONTINUE_WITH_INSTALLDIR_OVERWRITE': 'yes', + + # List of components to install, + # valid values are: {ALL, DEFAULTS, anythingpat} + 'COMPONENTS': ';'.join(self._filtered_components), + + # Installation mode, valid values are: {install, repair, uninstall} + 'PSET_MODE': 'install', + + # Directory for non-RPM database, valid values are: {filepat} + 'NONRPM_DB_DIR': prefix, + + # Perform validation of digital signatures of RPM files, + # valid values are: {yes, no} + 'SIGNING_ENABLED': 'no', + + # Select target architecture of your applications, + # valid values are: {IA32, INTEL64, ALL} + 'ARCH_SELECTED': 'ALL', + } + + # Not all Intel software requires a license. Trying to specify + # one anyway will cause the installation to fail. + if self.license_required: + config.update({ + # License file or license server, + # valid values are: {lspat, filepat} + 'ACTIVATION_LICENSE_FILE': self.global_license_file, + + # Activation type, valid values are: {exist_lic, + # license_server, license_file, trial_lic, serial_number} + 'ACTIVATION_TYPE': 'license_file', + + # Intel(R) Software Improvement Program opt-in, + # valid values are: {yes, no} + 'PHONEHOME_SEND_USAGE_DATA': 'no', + }) + + with open('silent.cfg', 'w') as cfg: + for key in config: + cfg.write('{0}={1}\n'.format(key, config[key])) + + def install(self, spec, prefix): + """Runs the ``install.sh`` installation script.""" + + install_script = Executable('./install.sh') + install_script('--silent', 'silent.cfg') + + @run_after('install') + def save_silent_cfg(self): + """Copies the silent.cfg configuration file to ``/.spack``.""" + install('silent.cfg', join_path(self.prefix, '.spack')) + + # Check that self.prefix is there after installation + run_after('install')(PackageBase.sanity_check_prefix) diff --git a/lib/spack/spack/cmd/configure.py b/lib/spack/spack/cmd/configure.py index 562582fe09..c8588334a5 100644 --- a/lib/spack/spack/cmd/configure.py +++ b/lib/spack/spack/cmd/configure.py @@ -41,6 +41,7 @@ build_system_to_phase = { QMakePackage: 'qmake', WafPackage: 'configure', PerlPackage: 'configure', + IntelPackage: 'configure', } diff --git a/lib/spack/spack/cmd/create.py b/lib/spack/spack/cmd/create.py index ca49eb03fa..fd1e5f9fd2 100644 --- a/lib/spack/spack/cmd/create.py +++ b/lib/spack/spack/cmd/create.py @@ -370,6 +370,15 @@ class MakefilePackageTemplate(PackageTemplate): # makefile.filter('CC = .*', 'CC = cc')""" +class IntelPackageTemplate(PackageTemplate): + """Provides appropriate overrides for licensed Intel software""" + + base_class_name = 'IntelPackage' + + body = """\ + # FIXME: Override `setup_environment` if necessary.""" + + templates = { 'autotools': AutotoolsPackageTemplate, 'autoreconf': AutoreconfPackageTemplate, @@ -384,6 +393,7 @@ templates = { 'perlbuild': PerlbuildPackageTemplate, 'octave': OctavePackageTemplate, 'makefile': MakefilePackageTemplate, + 'intel': IntelPackageTemplate, 'generic': PackageTemplate, } diff --git a/lib/spack/spack/environment.py b/lib/spack/spack/environment.py index ff278dd6b6..567e54e356 100644 --- a/lib/spack/spack/environment.py +++ b/lib/spack/spack/environment.py @@ -284,132 +284,157 @@ class EnvironmentModifications(object): x.execute() @staticmethod - def from_sourcing_files(*args, **kwargs): - """Returns modifications that would be made by sourcing files. - - Args: - *args (list of str): list of files to be sourced + def from_sourcing_file(filename, *args, **kwargs): + """Returns modifications that would be made by sourcing a file. + + Parameters: + filename (str): The file to source + *args (list of str): Arguments to pass on the command line + + Keyword Arguments: + shell (str): The shell to use (default: ``bash``) + shell_options (str): Options passed to the shell (default: ``-c``) + source_command (str): The command to run (default: ``source``) + suppress_output (str): Redirect used to suppress output of command + (default: ``&> /dev/null``) + concatenate_on_success (str): Operator used to execute a command + only when the previous command succeeds (default: ``&&``) Returns: EnvironmentModifications: an object that, if executed, has - the same effect on the environment as sourcing the files - passed as parameters + the same effect on the environment as sourcing the file """ - env = EnvironmentModifications() + # Check if the file actually exists + if not os.path.isfile(filename): + msg = 'Trying to source non-existing file: {0}'.format(filename) + raise RuntimeError(msg) + + # Kwargs parsing and default values + shell = kwargs.get('shell', '/bin/bash') + shell_options = kwargs.get('shell_options', '-c') + source_command = kwargs.get('source_command', 'source') + suppress_output = kwargs.get('suppress_output', '&> /dev/null') + concatenate_on_success = kwargs.get('concatenate_on_success', '&&') - # Check if the files are actually there - files = [line.split(' ')[0] for line in args] - non_existing = [file for file in files if not os.path.isfile(file)] - if non_existing: - message = 'trying to source non-existing files\n' - message += '\n'.join(non_existing) - raise RuntimeError(message) - - # Relevant kwd parameters and formats - info = dict(kwargs) - info.setdefault('shell', '/bin/bash') - info.setdefault('shell_options', '-c') - info.setdefault('source_command', 'source') - info.setdefault('suppress_output', '&> /dev/null') - info.setdefault('concatenate_on_success', '&&') - - shell = '{shell}'.format(**info) - shell_options = '{shell_options}'.format(**info) - source_file = '{source_command} {file} {concatenate_on_success}' - - dump_cmd = "import os, json; print(json.dumps(dict(os.environ)))" - dump_environment = 'python -c "%s"' % dump_cmd + source_file = [source_command, filename] + source_file.extend(args) + source_file = ' '.join(source_file) + + dump_cmd = 'import os, json; print(json.dumps(dict(os.environ)))' + dump_environment = 'python -c "{0}"'.format(dump_cmd) # Construct the command that will be executed - command = [source_file.format(file=file, **info) for file in args] - command.append(dump_environment) - command = ' '.join(command) command = [ shell, shell_options, - command + ' '.join([ + source_file, suppress_output, + concatenate_on_success, dump_environment, + ]), ] - # Try to source all the files, + # Try to source the file proc = subprocess.Popen( command, stdout=subprocess.PIPE, env=os.environ) proc.wait() + if proc.returncode != 0: - raise RuntimeError('sourcing files returned a non-zero exit code') + msg = 'Sourcing file {0} returned a non-zero exit code'.format( + filename) + raise RuntimeError(msg) + output = ''.join([line.decode('utf-8') for line in proc.stdout]) - # Construct a dictionaries of the environment before and after - # sourcing the files, so that we can diff them. - this_environment = dict(os.environ) - after_source_env = json.loads(output) + # Construct dictionaries of the environment before and after + # sourcing the file, so that we can diff them. + env_before = dict(os.environ) + env_after = json.loads(output) # If we're in python2, convert to str objects instead of unicode # like json gives us. We can't put unicode in os.environ anyway. if sys.version_info[0] < 3: - after_source_env = dict((k.encode('utf-8'), v.encode('utf-8')) - for k, v in after_source_env.items()) + env_after = dict((k.encode('utf-8'), v.encode('utf-8')) + for k, v in env_after.items()) # Filter variables that are not related to sourcing a file - to_be_filtered = 'SHLVL', '_', 'PWD', 'OLDPWD' - for d in after_source_env, this_environment: + to_be_filtered = 'SHLVL', '_', 'PWD', 'OLDPWD', 'PS2' + for d in env_after, env_before: for name in to_be_filtered: d.pop(name, None) # Fill the EnvironmentModifications instance + env = EnvironmentModifications() # New variables - new_variables = set(after_source_env) - set(this_environment) - for x in new_variables: - env.set(x, after_source_env[x]) + new_variables = set(env_after) - set(env_before) # Variables that have been unset - unset_variables = set(this_environment) - set(after_source_env) - for x in unset_variables: - env.unset(x) + unset_variables = set(env_before) - set(env_after) # Variables that have been modified common_variables = set( - this_environment).intersection(set(after_source_env)) + env_before).intersection(set(env_after)) modified_variables = [x for x in common_variables - if this_environment[x] != after_source_env[x]] + if env_before[x] != env_after[x]] - def return_separator_if_any(first_value, second_value): + def return_separator_if_any(*args): separators = ':', ';' for separator in separators: - if separator in first_value and separator in second_value: - return separator + for arg in args: + if separator in arg: + return separator return None - for x in modified_variables: - current = this_environment[x] - modified = after_source_env[x] - sep = return_separator_if_any(current, modified) - if sep is None: - # We just need to set the variable to the new value - env.set(x, after_source_env[x]) + # Add variables to env. + # Assume that variables with 'PATH' in the name or that contain + # separators like ':' or ';' are more likely to be paths + for x in new_variables: + sep = return_separator_if_any(env_after[x]) + if sep: + env.prepend_path(x, env_after[x], separator=sep) + elif 'PATH' in x: + env.prepend_path(x, env_after[x]) else: - current_list = current.split(sep) - modified_list = modified.split(sep) + # We just need to set the variable to the new value + env.set(x, env_after[x]) + + for x in unset_variables: + env.unset(x) + + for x in modified_variables: + before = env_before[x] + after = env_after[x] + sep = return_separator_if_any(before, after) + if sep: + before_list = before.split(sep) + after_list = after.split(sep) + + # Filter out empty strings + before_list = list(filter(None, before_list)) + after_list = list(filter(None, after_list)) + # Paths that have been removed remove_list = [ - ii for ii in current_list if ii not in modified_list] - # Check that nothing has been added in the middle of vurrent - # list + ii for ii in before_list if ii not in after_list] + # Check that nothing has been added in the middle of + # before_list remaining_list = [ - ii for ii in current_list if ii in modified_list] - start = modified_list.index(remaining_list[0]) - end = modified_list.index(remaining_list[-1]) - search = sep.join(modified_list[start:end + 1]) - - if search not in current: + ii for ii in before_list if ii in after_list] + try: + start = after_list.index(remaining_list[0]) + end = after_list.index(remaining_list[-1]) + search = sep.join(after_list[start:end + 1]) + except IndexError: + env.prepend_path(x, env_after[x]) + + if search not in before: # We just need to set the variable to the new value - env.set(x, after_source_env[x]) - break + env.prepend_path(x, env_after[x]) else: try: - prepend_list = modified_list[:start] + prepend_list = after_list[:start] except KeyError: prepend_list = [] try: - append_list = modified_list[end + 1:] + append_list = after_list[end + 1:] except KeyError: append_list = [] @@ -419,6 +444,9 @@ class EnvironmentModifications(object): env.append_path(x, item) for item in prepend_list: env.prepend_path(x, item) + else: + # We just need to set the variable to the new value + env.set(x, env_after[x]) return env diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 86df0ec947..8c849573a7 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -484,38 +484,65 @@ class PackageBase(with_metaclass(PackageMeta, object)): # # These are default values for instance variables. # - """By default we build in parallel. Subclasses can override this.""" + + #: By default we build in parallel. Subclasses can override this. parallel = True - """# jobs to use for parallel make. If set, overrides default of ncpus.""" + #: # jobs to use for parallel make. If set, overrides default of ncpus. make_jobs = spack.build_jobs - """By default do not run tests within package's install()""" + #: By default do not run tests within package's install() run_tests = False # FIXME: this is a bad object-oriented design, should be moved to Clang. - """By default do not setup mockup XCode on macOS with Clang""" + #: By default do not setup mockup XCode on macOS with Clang use_xcode = False - """Most packages are NOT extendable. Set to True if you want extensions.""" + #: Most packages are NOT extendable. Set to True if you want extensions. extendable = False - """When True, add RPATHs for the entire DAG. When False, add RPATHs only - for immediate dependencies.""" + #: When True, add RPATHs for the entire DAG. When False, add RPATHs only + #: for immediate dependencies. transitive_rpaths = True - """List of prefix-relative file paths (or a single path). If these do - not exist after install, or if they exist but are not files, - sanity checks fail. - """ + #: List of prefix-relative file paths (or a single path). If these do + #: not exist after install, or if they exist but are not files, + #: sanity checks fail. sanity_check_is_file = [] - """List of prefix-relative directory paths (or a single path). If - these do not exist after install, or if they exist but are not - directories, sanity checks will fail. - """ + #: List of prefix-relative directory paths (or a single path). If + #: these do not exist after install, or if they exist but are not + #: directories, sanity checks will fail. sanity_check_is_dir = [] + # + # Set default licensing information + # + + #: Boolean. If set to ``True``, this software requires a license. + #: If set to ``False``, all of the ``license_*`` attributes will + #: be ignored. Defaults to ``False``. + license_required = False + + #: String. Contains the symbol used by the license manager to denote + #: a comment. Defaults to ``#``. + license_comment = '#' + + #: List of strings. These are files that the software searches for when + #: looking for a license. All file paths must be relative to the + #: installation directory. More complex packages like Intel may require + #: multiple licenses for individual components. Defaults to the empty list. + license_files = [] + + #: List of strings. Environment variables that can be set to tell the + #: software where to look for a license if it is not in the usual location. + #: Defaults to the empty list. + license_vars = [] + + #: String. A URL pointing to license setup instructions for the software. + #: Defaults to the empty string. + license_url = '' + def __init__(self, spec): # this determines how the package should be built. self.spec = spec @@ -569,22 +596,6 @@ class PackageBase(with_metaclass(PackageMeta, object)): if not hasattr(self, 'list_depth'): self.list_depth = 0 - # Set default licensing information - if not hasattr(self, 'license_required'): - self.license_required = False - - if not hasattr(self, 'license_comment'): - self.license_comment = '#' - - if not hasattr(self, 'license_files'): - self.license_files = [] - - if not hasattr(self, 'license_vars'): - self.license_vars = [] - - if not hasattr(self, 'license_url'): - self.license_url = None - # Set up some internal variables for timing. self._fetch_time = 0.0 self._total_time = 0.0 diff --git a/lib/spack/spack/test/environment.py b/lib/spack/spack/test/environment.py index 7f3b7bf2bd..d07223221c 100644 --- a/lib/spack/spack/test/environment.py +++ b/lib/spack/spack/test/environment.py @@ -89,7 +89,7 @@ def files_to_be_sourced(): files = [ os.path.join(datadir, 'sourceme_first.sh'), os.path.join(datadir, 'sourceme_second.sh'), - os.path.join(datadir, 'sourceme_parameters.sh intel64'), + os.path.join(datadir, 'sourceme_parameters.sh'), os.path.join(datadir, 'sourceme_unicode.sh') ] @@ -224,7 +224,14 @@ def test_source_files(files_to_be_sourced): """Tests the construction of a list of environment modifications that are the result of sourcing a file. """ - env = EnvironmentModifications.from_sourcing_files(*files_to_be_sourced) + env = EnvironmentModifications() + for filename in files_to_be_sourced: + if filename.endswith('sourceme_parameters.sh'): + env.extend(EnvironmentModifications.from_sourcing_file( + filename, 'intel64')) + else: + env.extend(EnvironmentModifications.from_sourcing_file(filename)) + modifications = env.group_by_name() # This is sensitive to the user's environment; can include -- cgit v1.2.3-70-g09d2