From dd3762d0f936aca41d11deb033e56d68d2036f2f Mon Sep 17 00:00:00 2001 From: Greg Becker Date: Tue, 5 May 2020 13:58:46 -0700 Subject: cray platform: support cray Cluster and XC type machines (#12989) Cray has two machine types. "XC" machines are the larger machines more common in HPC, but "Cluster" machines are also cropping up at some HPC sites. Cluster machines run a slightly different form of the CrayPE programming environment, and often come without default modules loaded. Cluster machines also run different versions of some software, and run a linux distro on the backend nodes instead of running Compute Node Linux (CNL). Below are the changes made to support "Cluster" machines in Spack. Some of these changes are semi-related general upkeep of the cray platform. * cray platform: detect properly after module purge * cray platform: support machines running OSs other than CNL Make Cray backend OS delegate to LinuxDistro when no cle_release file favor backend over frontend OS when name clashes * cray platform: target detection uses multiple strategies This commit improves the robustness of target detection on Cray by trying multiple strategies. The first one that produces results wins. If nothing is found only the generic family of the frontend host is used as a target. * cray-libsci: add package from NERSC * build_env: unload cray-libsci module when not explicitly needed cray-libsci is a package in Spack. The cray PrgEnv modules load it implicitly when we set up the compiler. We now unload it after setting up the compiler and only reload it when requested via external package. * util/module_cmd: more robust module parsing Cray modules have documentation inside the module that is visible to the `module show` command. Spack module parsing is now robust to documentation inside modules. * cce compiler: uses clang flags for versions >= 9.0 * build_env: push CRAY_LD_LIBRARY_PATH into everything Some Cray modules add paths to CRAY_LD_LIBRARY_PATH instead of LD_LIBRARY_PATH. This has performance benefits at load time, but leads to Spack builds not finding their dependencies from external modules. Spack now prepends CRAY_LD_LIBRARY_PATH to LD_LIBRARY_PATH before beginning the build. * mvapich2: setup cray compilers when on cray previously, mpich was the only mpi implementation to support cray systems (because it is the MPI on Cray XC systems). Cray cluster systems use mvapich2, which now supports cray compiler wrappers. * build_env: clean pkgconf from environment Cray modules silently add pkgconf to the user environment This can break builds that do not user pkgconf. Now we remove it frmo the environment and add it again if it is in the spec. * cray platform: cheat modules for rome/zen2 module on naples/zen node Cray modules for naples/zen architecture currently specify rome/zen2. For now, we detect this and return zen for modules named `craype-x86-rome`. * compiler: compiler default versions When detecting compiler default versions for target/compiler compatibility checks, Spack previously ran the compiler without setting up its environment. Now we setup a temporary environment to run the compiler with its modules to detect its version. * compilers/cce: improve logic to determine C/C++ std flags * tests: fix existing tests to play nicely with new cray support * tests: test new functionality Some new functionality can only be tested on a cray system. Add tests for what can be tested on a linux system. Co-authored-by: Massimiliano Culpo --- lib/spack/spack/architecture.py | 9 +- lib/spack/spack/build_environment.py | 13 +- lib/spack/spack/compiler.py | 40 +++++ lib/spack/spack/compilers/cce.py | 23 ++- lib/spack/spack/operating_systems/cnl.py | 151 ------------------- lib/spack/spack/operating_systems/cray_backend.py | 166 +++++++++++++++++++++ lib/spack/spack/platforms/cray.py | 154 +++++++++++++------ lib/spack/spack/spec.py | 6 +- lib/spack/spack/test/architecture.py | 7 +- lib/spack/spack/test/cmd/dev_build.py | 11 +- lib/spack/spack/test/compilers.py | 55 ++++++- lib/spack/spack/test/operating_system.py | 22 +-- lib/spack/spack/util/module_cmd.py | 8 +- .../repos/builtin/packages/cray-libsci/package.py | 50 ++++++- .../repos/builtin/packages/mvapich2/package.py | 30 +++- 15 files changed, 503 insertions(+), 242 deletions(-) delete mode 100644 lib/spack/spack/operating_systems/cnl.py create mode 100644 lib/spack/spack/operating_systems/cray_backend.py diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 38ed5baa7b..963fecd375 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -209,14 +209,15 @@ class Target(object): compiler_version = compiler.version version_number, suffix = cpu.version_components(compiler.version) if not version_number or suffix not in ('', 'apple'): - # Try to deduce the correct version. Depending on where this - # function is called we might get either a CompilerSpec or a - # fully fledged compiler object + # Try to deduce the underlying version of the compiler, regardless + # of its name in compilers.yaml. Depending on where this function + # is called we might get either a CompilerSpec or a fully fledged + # compiler object. import spack.spec if isinstance(compiler, spack.spec.CompilerSpec): compiler = spack.compilers.compilers_for_spec(compiler).pop() try: - compiler_version = compiler.cc_version(compiler.cc) + compiler_version = compiler.get_real_version() except spack.util.executable.ProcessError as e: # log this and just return compiler.version instead tty.debug(str(e)) diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 5e6ea00ce6..8d84cdaf65 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -60,7 +60,7 @@ from spack.util.environment import ( from spack.util.environment import system_dirs from spack.error import NoLibrariesError, NoHeadersError from spack.util.executable import Executable -from spack.util.module_cmd import load_module, get_path_from_module +from spack.util.module_cmd import load_module, get_path_from_module, module from spack.util.log_parse import parse_log_events, make_log_context @@ -141,12 +141,18 @@ def clean_environment(): # can affect how some packages find libraries. We want to make # sure that builds never pull in unintended external dependencies. env.unset('LD_LIBRARY_PATH') + env.unset('CRAY_LD_LIBRARY_PATH') env.unset('LIBRARY_PATH') env.unset('CPATH') env.unset('LD_RUN_PATH') env.unset('DYLD_LIBRARY_PATH') env.unset('DYLD_FALLBACK_LIBRARY_PATH') + # Remove all pkgconfig stuff from craype + for varname in os.environ.keys(): + if 'PKGCONF' in varname: + env.unset(varname) + build_lang = spack.config.get('config:build_language') if build_lang: # Override language-related variables. This can be used to force @@ -717,6 +723,11 @@ def setup_package(pkg, dirty): load_module("cce") load_module(mod) + # kludge to handle cray libsci being automatically loaded by PrgEnv + # modules on cray platform. Module unload does no damage when + # unnecessary + module('unload', 'cray-libsci') + if pkg.architecture.target.module_name: load_module(pkg.architecture.target.module_name) diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index 8afbe48c0c..94dbb190d2 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -19,6 +19,7 @@ import spack.error import spack.spec import spack.architecture import spack.util.executable +import spack.util.module_cmd import spack.compilers from spack.util.environment import filter_system_paths @@ -434,6 +435,45 @@ class Compiler(object): Position Independent Code (PIC).""" return '-fPIC' + # Note: This is not a class method. The class methods are used to detect + # compilers on PATH based systems, and do not set up the run environment of + # the compiler. This method can be called on `module` based systems as well + def get_real_version(self): + """Query the compiler for its version. + + This is the "real" compiler version, regardless of what is in the + compilers.yaml file, which the user can change to name their compiler. + + Use the runtime environment of the compiler (modules and environment + modifications) to enable the compiler to run properly on any platform. + """ + # store environment to replace later + backup_env = os.environ.copy() + + # load modules and set env variables + for module in self.modules: + # On cray, mic-knl module cannot be loaded without cce module + # See: https://github.com/spack/spack/issues/3153 + if os.environ.get("CRAY_CPU_TARGET") == 'mic-knl': + spack.util.module_cmd.load_module('cce') + spack.util.module_cmd.load_module(module) + + # apply other compiler environment changes + env = spack.util.environment.EnvironmentModifications() + env.extend(spack.schema.environment.parse(self.environment)) + env.apply_modifications() + + cc = spack.util.executable.Executable(self.cc) + output = cc(self.version_argument, + output=str, error=str, + ignore_errors=tuple(self.ignore_version_errors)) + + # Restore environment + os.environ.clear() + os.environ.update(backup_env) + + return self.extract_version_from_output(output) + # # Compiler classes have methods for querying the version of # specific compiler executables. This is used when discovering compilers. diff --git a/lib/spack/spack/compilers/cce.py b/lib/spack/spack/compilers/cce.py index 7aedb55a5d..0d30a69d3e 100644 --- a/lib/spack/spack/compilers/cce.py +++ b/lib/spack/spack/compilers/cce.py @@ -32,7 +32,12 @@ class Cce(Compiler): 'f77': 'cce/ftn', 'fc': 'cce/ftn'} - version_argument = '-V' + @property + def version_argument(self): + if self.version >= ver('9.0'): + return '--version' + return '-V' + version_regex = r'[Vv]ersion.*?(\d+(\.\d+)+)' @classmethod @@ -41,17 +46,23 @@ class Cce(Compiler): @property def openmp_flag(self): + if self.version >= ver('9.0'): + return '-fopenmp' return "-h omp" @property def cxx11_flag(self): + if self.version >= ver('9.0'): + return '-std=c++11' return "-h std=c++11" @property def c99_flag(self): - if self.version >= ver('8.4'): - return '-h stc=c99,noconform,gnu' - if self.version >= ver('8.1'): + if self.version >= ver('9.0'): + return '-std=c99' + elif self.version >= ver('8.4'): + return '-h std=c99,noconform,gnu' + elif self.version >= ver('8.1'): return '-h c99,noconform,gnu' raise UnsupportedCompilerFlag(self, 'the C99 standard', @@ -60,7 +71,9 @@ class Cce(Compiler): @property def c11_flag(self): - if self.version >= ver('8.5'): + if self.version >= ver('9.0'): + return '-std=c11' + elif self.version >= ver('8.5'): return '-h std=c11,noconform,gnu' raise UnsupportedCompilerFlag(self, 'the C11 standard', diff --git a/lib/spack/spack/operating_systems/cnl.py b/lib/spack/spack/operating_systems/cnl.py deleted file mode 100644 index 3d4036cb47..0000000000 --- a/lib/spack/spack/operating_systems/cnl.py +++ /dev/null @@ -1,151 +0,0 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) - -import os -import re - -import llnl.util.tty as tty - -import spack.error -import spack.version -from spack.architecture import OperatingSystem -from spack.util.module_cmd import module - -#: Possible locations of the Cray CLE release file, -#: which we look at to get the CNL OS version. -_cle_release_file = '/etc/opt/cray/release/cle-release' -_clerelease_file = '/etc/opt/cray/release/clerelease' - - -def read_cle_release_file(): - """Read the CLE release file and return a dict with its attributes. - - This file is present on newer versions of Cray. - - The release file looks something like this:: - - RELEASE=6.0.UP07 - BUILD=6.0.7424 - ... - - The dictionary we produce looks like this:: - - { - "RELEASE": "6.0.UP07", - "BUILD": "6.0.7424", - ... - } - - Returns: - dict: dictionary of release attributes - """ - with open(_cle_release_file) as release_file: - result = {} - for line in release_file: - # use partition instead of split() to ensure we only split on - # the first '=' in the line. - key, _, value = line.partition('=') - result[key] = value.strip() - return result - - -def read_clerelease_file(): - """Read the CLE release file and return the Cray OS version. - - This file is present on older versions of Cray. - - The release file looks something like this:: - - 5.2.UP04 - - Returns: - str: the Cray OS version - """ - with open(_clerelease_file) as release_file: - for line in release_file: - return line.strip() - - -class Cnl(OperatingSystem): - """Compute Node Linux (CNL) is the operating system used for the Cray XC - series super computers. It is a very stripped down version of GNU/Linux. - Any compilers found through this operating system will be used with - modules. If updated, user must make sure that version and name are - updated to indicate that OS has been upgraded (or downgraded) - """ - - def __init__(self): - name = 'cnl' - version = self._detect_crayos_version() - super(Cnl, self).__init__(name, version) - self.modulecmd = module - - def __str__(self): - return self.name + str(self.version) - - @classmethod - def _detect_crayos_version(cls): - if os.path.isfile(_cle_release_file): - release_attrs = read_cle_release_file() - v = spack.version.Version(release_attrs['RELEASE']) - return v[0] - elif os.path.isfile(_clerelease_file): - v = read_clerelease_file() - return spack.version.Version(v)[0] - else: - raise spack.error.UnsupportedPlatformError( - 'Unable to detect Cray OS version') - - def arguments_to_detect_version_fn(self, paths): - import spack.compilers - - command_arguments = [] - for compiler_name in spack.compilers.supported_compilers(): - cmp_cls = spack.compilers.class_for_compiler_name(compiler_name) - - # If the compiler doesn't have a corresponding - # Programming Environment, skip to the next - if cmp_cls.PrgEnv is None: - continue - - if cmp_cls.PrgEnv_compiler is None: - tty.die('Must supply PrgEnv_compiler with PrgEnv') - - compiler_id = spack.compilers.CompilerID(self, compiler_name, None) - detect_version_args = spack.compilers.DetectVersionArgs( - id=compiler_id, variation=(None, None), - language='cc', path='cc' - ) - command_arguments.append(detect_version_args) - return command_arguments - - def detect_version(self, detect_version_args): - import spack.compilers - modulecmd = self.modulecmd - compiler_name = detect_version_args.id.compiler_name - compiler_cls = spack.compilers.class_for_compiler_name(compiler_name) - output = modulecmd('avail', compiler_cls.PrgEnv_compiler) - version_regex = r'(%s)/([\d\.]+[\d])' % compiler_cls.PrgEnv_compiler - matches = re.findall(version_regex, output) - version = tuple(version for _, version in matches) - compiler_id = detect_version_args.id - value = detect_version_args._replace( - id=compiler_id._replace(version=version) - ) - return value, None - - def make_compilers(self, compiler_id, paths): - import spack.spec - name = compiler_id.compiler_name - cmp_cls = spack.compilers.class_for_compiler_name(name) - compilers = [] - for v in compiler_id.version: - comp = cmp_cls( - spack.spec.CompilerSpec(name + '@' + v), - self, "any", - ['cc', 'CC', 'ftn'], [cmp_cls.PrgEnv, name + '/' + v]) - - compilers.append(comp) - return compilers diff --git a/lib/spack/spack/operating_systems/cray_backend.py b/lib/spack/spack/operating_systems/cray_backend.py new file mode 100644 index 0000000000..91c0e6ae98 --- /dev/null +++ b/lib/spack/spack/operating_systems/cray_backend.py @@ -0,0 +1,166 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import os +import re + +import llnl.util.tty as tty + +import spack.error +import spack.version +from spack.operating_systems.linux_distro import LinuxDistro +from spack.util.module_cmd import module + +#: Possible locations of the Cray CLE release file, +#: which we look at to get the CNL OS version. +_cle_release_file = '/etc/opt/cray/release/cle-release' +_clerelease_file = '/etc/opt/cray/release/clerelease' + + +def read_cle_release_file(): + """Read the CLE release file and return a dict with its attributes. + + This file is present on newer versions of Cray. + + The release file looks something like this:: + + RELEASE=6.0.UP07 + BUILD=6.0.7424 + ... + + The dictionary we produce looks like this:: + + { + "RELEASE": "6.0.UP07", + "BUILD": "6.0.7424", + ... + } + + Returns: + dict: dictionary of release attributes + """ + with open(_cle_release_file) as release_file: + result = {} + for line in release_file: + # use partition instead of split() to ensure we only split on + # the first '=' in the line. + key, _, value = line.partition('=') + result[key] = value.strip() + return result + + +def read_clerelease_file(): + """Read the CLE release file and return the Cray OS version. + + This file is present on older versions of Cray. + + The release file looks something like this:: + + 5.2.UP04 + + Returns: + str: the Cray OS version + """ + with open(_clerelease_file) as release_file: + for line in release_file: + return line.strip() + + +class CrayBackend(LinuxDistro): + """Compute Node Linux (CNL) is the operating system used for the Cray XC + series super computers. It is a very stripped down version of GNU/Linux. + Any compilers found through this operating system will be used with + modules. If updated, user must make sure that version and name are + updated to indicate that OS has been upgraded (or downgraded) + """ + + def __init__(self): + name = 'cnl' + version = self._detect_crayos_version() + if version: + # If we found a CrayOS version, we do not want the information + # from LinuxDistro. In order to skip the logic from + # external.distro.linux_distribution, while still calling __init__ + # methods further up the MRO, we skip LinuxDistro in the MRO and + # call the OperatingSystem superclass __init__ method + super(LinuxDistro, self).__init__(name, version) + else: + super(CrayBackend, self).__init__() + self.modulecmd = module + + def __str__(self): + return self.name + str(self.version) + + @classmethod + def _detect_crayos_version(cls): + if os.path.isfile(_cle_release_file): + release_attrs = read_cle_release_file() + v = spack.version.Version(release_attrs['RELEASE']) + return v[0] + elif os.path.isfile(_clerelease_file): + v = read_clerelease_file() + return spack.version.Version(v)[0] + else: + # Not all Cray systems run CNL on the backend. + # Systems running in what Cray calls "cluster" mode run other + # linux OSs under the Cray PE. + # So if we don't detect any Cray OS version on the system, + # we return None. We can't ever be sure we will get a Cray OS + # version. + # Returning None allows the calling code to test for the value + # being "True-ish" rather than requiring a try/except block. + return None + + def arguments_to_detect_version_fn(self, paths): + import spack.compilers + + command_arguments = [] + for compiler_name in spack.compilers.supported_compilers(): + cmp_cls = spack.compilers.class_for_compiler_name(compiler_name) + + # If the compiler doesn't have a corresponding + # Programming Environment, skip to the next + if cmp_cls.PrgEnv is None: + continue + + if cmp_cls.PrgEnv_compiler is None: + tty.die('Must supply PrgEnv_compiler with PrgEnv') + + compiler_id = spack.compilers.CompilerID(self, compiler_name, None) + detect_version_args = spack.compilers.DetectVersionArgs( + id=compiler_id, variation=(None, None), + language='cc', path='cc' + ) + command_arguments.append(detect_version_args) + return command_arguments + + def detect_version(self, detect_version_args): + import spack.compilers + modulecmd = self.modulecmd + compiler_name = detect_version_args.id.compiler_name + compiler_cls = spack.compilers.class_for_compiler_name(compiler_name) + output = modulecmd('avail', compiler_cls.PrgEnv_compiler) + version_regex = r'(%s)/([\d\.]+[\d])' % compiler_cls.PrgEnv_compiler + matches = re.findall(version_regex, output) + version = tuple(version for _, version in matches) + compiler_id = detect_version_args.id + value = detect_version_args._replace( + id=compiler_id._replace(version=version) + ) + return value, None + + def make_compilers(self, compiler_id, paths): + import spack.spec + name = compiler_id.compiler_name + cmp_cls = spack.compilers.class_for_compiler_name(name) + compilers = [] + for v in compiler_id.version: + comp = cmp_cls( + spack.spec.CompilerSpec(name + '@' + v), + self, "any", + ['cc', 'CC', 'ftn'], [cmp_cls.PrgEnv, name + '/' + v]) + + compilers.append(comp) + return compilers diff --git a/lib/spack/spack/platforms/cray.py b/lib/spack/spack/platforms/cray.py index 6e8c79ef0c..8c5fe525e6 100644 --- a/lib/spack/spack/platforms/cray.py +++ b/lib/spack/spack/platforms/cray.py @@ -4,30 +4,29 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os +import os.path import re +import platform +import llnl.util.cpu as cpu import llnl.util.tty as tty from spack.paths import build_env_path from spack.util.executable import Executable from spack.architecture import Platform, Target, NoPlatformError from spack.operating_systems.cray_frontend import CrayFrontend -from spack.operating_systems.cnl import Cnl +from spack.operating_systems.cray_backend import CrayBackend from spack.util.module_cmd import module -def _get_modules_in_modulecmd_output(output): - '''Return list of valid modules parsed from modulecmd output string.''' - return [i for i in output.splitlines() - if len(i.split()) == 1] +_craype_name_to_target_name = { + 'x86-cascadelake': 'cascadelake', + 'x86-naples': 'zen', + 'x86-rome': 'zen', # Cheating because we have the wrong modules on rzcrayz + 'x86-skylake': 'skylake-avx512' +} -def _fill_craype_targets_from_modules(targets, modules): - '''Extend CrayPE CPU targets list with those found in list of modules.''' - # Craype- module prefixes that are not valid CPU targets. - non_targets = ('hugepages', 'network', 'target', 'accel', 'xtpe') - pattern = r'craype-(?!{0})(\S*)'.format('|'.join(non_targets)) - for mod in modules: - if 'craype-' in mod: - targets.extend(re.findall(pattern, mod)) +def _target_name_from_craype_target_name(name): + return _craype_name_to_target_name.get(name, name) class Cray(Platform): @@ -47,40 +46,34 @@ class Cray(Platform): # Make all craype targets available. for target in self._avail_targets(): - name = target.replace('-', '_') + name = _target_name_from_craype_target_name(target) self.add_target(name, Target(name, 'craype-%s' % target)) - self.add_target("x86_64", Target("x86_64")) - self.add_target("front_end", Target("x86_64")) - self.front_end = "x86_64" - - # Get aliased targets from config or best guess from environment: - for name in ('front_end', 'back_end'): - _target = getattr(self, name, None) - if _target is None: - _target = os.environ.get('SPACK_' + name.upper()) - if _target is None and name == 'back_end': - _target = self._default_target_from_env() - if _target is not None: - safe_name = _target.replace('-', '_') - setattr(self, name, safe_name) - self.add_target(name, self.targets[safe_name]) - - if self.back_end is not None: - self.default = self.back_end - self.add_target('default', self.targets[self.back_end]) - else: + self.back_end = os.environ.get('SPACK_BACK_END', + self._default_target_from_env()) + self.default = self.back_end + if self.back_end not in self.targets: + # We didn't find a target module for the backend raise NoPlatformError() + # Setup frontend targets + for name in cpu.targets: + if name not in self.targets: + self.add_target(name, Target(name)) + self.front_end = os.environ.get('SPACK_FRONT_END', cpu.host().name) + if self.front_end not in self.targets: + self.add_target(self.front_end, Target(self.front_end)) + front_distro = CrayFrontend() - back_distro = Cnl() + back_distro = CrayBackend() self.default_os = str(back_distro) self.back_os = self.default_os self.front_os = str(front_distro) self.add_operating_system(self.back_os, back_distro) - self.add_operating_system(self.front_os, front_distro) + if self.front_os != self.back_os: + self.add_operating_system(self.front_os, front_distro) @classmethod def setup_platform_environment(cls, pkg, env): @@ -104,9 +97,28 @@ class Cray(Platform): env.append_path("PKG_CONFIG_PATH", "/usr/lib64/pkgconfig") env.append_path("PKG_CONFIG_PATH", "/usr/local/lib64/pkgconfig") + # CRAY_LD_LIBRARY_PATH is used at build time by the cray compiler + # wrappers to augment LD_LIBRARY_PATH. This is to avoid long load + # times at runtime. This behavior is not always respected on cray + # "cluster" systems, so we reproduce it here. + if os.environ.get('CRAY_LD_LIBRARY_PATH'): + env.prepend_path('LD_LIBRARY_PATH', + os.environ['CRAY_LD_LIBRARY_PATH']) + @classmethod def detect(cls): - return os.environ.get('CRAYPE_VERSION') is not None + """ + Detect whether this system is a cray machine. + + We detect the cray platform based on the availability through `module` + of the cray programming environment. If this environment is available, + we can use it to find compilers, target modules, etc. If the cray + programming environment is not available via modules, then we will + treat it as a standard linux system, as the cray compiler wrappers + and other componenets of the cray programming environment are + irrelevant without module support. + """ + return 'opt/cray' in os.environ.get('MODULEPATH', '') def _default_target_from_env(self): '''Set and return the default CrayPE target loaded in a clean login @@ -119,22 +131,66 @@ class Cray(Platform): if getattr(self, 'default', None) is None: bash = Executable('/bin/bash') output = bash( - '-lc', 'echo $CRAY_CPU_TARGET', + '--norc', '--noprofile', '-lc', 'echo $CRAY_CPU_TARGET', env={'TERM': os.environ.get('TERM', '')}, - output=str, - error=os.devnull + output=str, error=os.devnull ) - output = ''.join(output.split()) # remove all whitespace - if output: - self.default = output - tty.debug("Found default module:%s" % self.default) - return self.default + default_from_module = ''.join(output.split()) # rm all whitespace + if default_from_module: + tty.debug("Found default module:%s" % default_from_module) + return default_from_module + else: + front_end = cpu.host().name + if front_end in list( + map(lambda x: _target_name_from_craype_target_name(x), + self._avail_targets()) + ): + tty.debug("default to front-end architecture") + return cpu.host().name + else: + return platform.machine() def _avail_targets(self): '''Return a list of available CrayPE CPU targets.''' + + def modules_in_output(output): + """Returns a list of valid modules parsed from modulecmd output""" + return [i for i in re.split(r'\s\s+|\n', output)] + + def target_names_from_modules(modules): + # Craype- module prefixes that are not valid CPU targets. + targets = [] + for mod in modules: + if 'craype-' in mod: + name = mod[7:] + _n = name.replace('-', '_') # test for mic-knl/mic_knl + is_target_name = name in cpu.targets or _n in cpu.targets + is_cray_target_name = name in _craype_name_to_target_name + if is_target_name or is_cray_target_name: + targets.append(name) + + return targets + + def modules_from_listdir(): + craype_default_path = '/opt/cray/pe/craype/default/modulefiles' + if os.path.isdir(craype_default_path): + return os.listdir(craype_default_path) + return None + if getattr(self, '_craype_targets', None) is None: - output = module('avail', '-t', 'craype-') - craype_modules = _get_modules_in_modulecmd_output(output) - self._craype_targets = targets = [] - _fill_craype_targets_from_modules(targets, craype_modules) + strategies = [ + lambda: modules_in_output(module('avail', '-t', 'craype-')), + modules_from_listdir + ] + for available_craype_modules in strategies: + craype_modules = available_craype_modules() + craype_targets = target_names_from_modules(craype_modules) + if craype_targets: + self._craype_targets = craype_targets + break + else: + # If nothing is found add platform.machine() + # to avoid Spack erroring out + self._craype_targets = [platform.machine()] + return self._craype_targets diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index c6fe2da762..0b9500246a 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -2239,7 +2239,11 @@ class Spec(object): for mod in compiler.modules: md.load_module(mod) - s.external_path = md.get_path_from_module(s.external_module) + # get the path from the module + # the package can override the default + s.external_path = getattr(s.package, 'external_prefix', + md.get_path_from_module( + s.external_module)) # Mark everything in the spec as concrete, as well. self._mark_concrete() diff --git a/lib/spack/spack/test/architecture.py b/lib/spack/spack/test/architecture.py index 552bc324bf..48cec134d2 100644 --- a/lib/spack/spack/test/architecture.py +++ b/lib/spack/spack/test/architecture.py @@ -1,4 +1,3 @@ - # Copyright 2013-2020 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # @@ -41,7 +40,7 @@ def test_dict_functions_for_architecture(): def test_platform(): output_platform_class = spack.architecture.real_platform() - if os.environ.get('CRAYPE_VERSION') is not None: + if os.path.exists('/opt/cray/pe'): my_platform_class = Cray() elif os.path.exists('/bgsys'): my_platform_class = Bgq() @@ -210,8 +209,8 @@ def test_optimization_flags_with_custom_versions( target = spack.architecture.Target(target_str) if real_version: monkeypatch.setattr( - spack.compiler.Compiler, 'cc_version', lambda x, y: real_version - ) + spack.compiler.Compiler, 'get_real_version', + lambda x: real_version) opt_flags = target.optimization_flags(compiler) assert opt_flags == expected_flags diff --git a/lib/spack/spack/test/cmd/dev_build.py b/lib/spack/spack/test/cmd/dev_build.py index 5a7dfc273c..37c40e787d 100644 --- a/lib/spack/spack/test/cmd/dev_build.py +++ b/lib/spack/spack/test/cmd/dev_build.py @@ -80,8 +80,17 @@ def test_dev_build_drop_in(tmpdir, mock_packages, monkeypatch, install_mockery): def print_spack_cc(*args): # Eat arguments and print environment variable to test - print(os.environ['CC']) + print(os.environ.get('CC', '')) monkeypatch.setattr(os, 'execvp', print_spack_cc) + + # `module unload cray-libsci` in test environment causes failure + # It does not fail for actual installs + # build_environment.py imports module directly, so we monkeypatch it there + # rather than in module_cmd + def module(*args): + pass + monkeypatch.setattr(spack.build_environment, 'module', module) + output = dev_build('-b', 'edit', '--drop-in', 'sh', 'dev-build-test-install@0.0.0') assert "lib/spack/env" in output diff --git a/lib/spack/spack/test/compilers.py b/lib/spack/spack/test/compilers.py index 24115ba562..586bb215cf 100644 --- a/lib/spack/spack/test/compilers.py +++ b/lib/spack/spack/test/compilers.py @@ -6,10 +6,13 @@ import pytest import sys +import os from copy import copy from six import iteritems +import llnl.util.filesystem as fs + import spack.spec import spack.compiler import spack.compilers as compilers @@ -259,7 +262,7 @@ def test_cce_flags(): supported_flag_test("cxx11_flag", "-h std=c++11", "cce@1.0") unsupported_flag_test("c99_flag", "cce@8.0") supported_flag_test("c99_flag", "-h c99,noconform,gnu", "cce@8.1") - supported_flag_test("c99_flag", "-h stc=c99,noconform,gnu", "cce@8.4") + supported_flag_test("c99_flag", "-h std=c99,noconform,gnu", "cce@8.4") unsupported_flag_test("c11_flag", "cce@8.4") supported_flag_test("c11_flag", "-h std=c11,noconform,gnu", "cce@8.5") supported_flag_test("cc_pic_flag", "-h PIC", "cce@1.0") @@ -615,3 +618,53 @@ def test_raising_if_compiler_target_is_over_specific(config): cfg = spack.compilers.get_compiler_config() with pytest.raises(ValueError): spack.compilers.get_compilers(cfg, 'gcc@9.0.1', arch_spec) + + +def test_compiler_get_real_version(working_env, monkeypatch, tmpdir): + # Test variables + test_version = '2.2.2' + + # Create compiler + gcc = str(tmpdir.join('gcc')) + with open(gcc, 'w') as f: + f.write("""#!/bin/bash +if [[ $CMP_ON == "1" ]]; then + echo "$CMP_VER" +fi +""") + fs.set_executable(gcc) + + # Add compiler to config + compiler_info = { + 'spec': 'gcc@foo', + 'paths': { + 'cc': gcc, + 'cxx': None, + 'f77': None, + 'fc': None, + }, + 'flags': {}, + 'operating_system': 'fake', + 'target': 'fake', + 'modules': ['turn_on'], + 'environment': { + 'set': {'CMP_VER': test_version}, + }, + 'extra_rpaths': [], + } + compiler_dict = {'compiler': compiler_info} + + # Set module load to turn compiler on + def module(*args): + if args[0] == 'show': + return '' + elif args[0] == 'load': + os.environ['CMP_ON'] = "1" + monkeypatch.setattr(spack.util.module_cmd, 'module', module) + + # Run and confirm output + compilers = spack.compilers.get_compilers([compiler_dict]) + assert len(compilers) == 1 + compiler = compilers[0] + version = compiler.get_real_version() + assert version == test_version diff --git a/lib/spack/spack/test/operating_system.py b/lib/spack/spack/test/operating_system.py index 221712e5ef..97def3feda 100644 --- a/lib/spack/spack/test/operating_system.py +++ b/lib/spack/spack/test/operating_system.py @@ -3,7 +3,7 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) -import spack.operating_systems.cnl as cnl +import spack.operating_systems.cray_backend as cray_backend def test_read_cle_release_file(tmpdir, monkeypatch): @@ -20,8 +20,9 @@ PATCHSET=35-201906112304 DUMMY=foo=bar """) - monkeypatch.setattr(cnl, '_cle_release_file', str(cle_release_path)) - attrs = cnl.read_cle_release_file() + monkeypatch.setattr(cray_backend, '_cle_release_file', + str(cle_release_path)) + attrs = cray_backend.read_cle_release_file() assert attrs['RELEASE'] == '6.0.UP07' assert attrs['BUILD'] == '6.0.7424' @@ -31,7 +32,7 @@ DUMMY=foo=bar assert attrs['PATCHSET'] == '35-201906112304' assert attrs['DUMMY'] == 'foo=bar' - assert cnl.Cnl._detect_crayos_version() == 6 + assert cray_backend.CrayBackend._detect_crayos_version() == 6 def test_read_clerelease_file(tmpdir, monkeypatch): @@ -40,12 +41,12 @@ def test_read_clerelease_file(tmpdir, monkeypatch): with clerelease_path.open('w') as f: f.write('5.2.UP04\n') - monkeypatch.setattr(cnl, '_clerelease_file', str(clerelease_path)) - v = cnl.read_clerelease_file() + monkeypatch.setattr(cray_backend, '_clerelease_file', str(clerelease_path)) + v = cray_backend.read_clerelease_file() assert v == '5.2.UP04' - assert cnl.Cnl._detect_crayos_version() == 5 + assert cray_backend.CrayBackend._detect_crayos_version() == 5 def test_cle_release_precedence(tmpdir, monkeypatch): @@ -67,7 +68,8 @@ DUMMY=foo=bar with clerelease_path.open('w') as f: f.write('5.2.UP04\n') - monkeypatch.setattr(cnl, '_clerelease_file', str(clerelease_path)) - monkeypatch.setattr(cnl, '_cle_release_file', str(cle_release_path)) + monkeypatch.setattr(cray_backend, '_clerelease_file', str(clerelease_path)) + monkeypatch.setattr(cray_backend, '_cle_release_file', + str(cle_release_path)) - assert cnl.Cnl._detect_crayos_version() == 6 + assert cray_backend.CrayBackend._detect_crayos_version() == 6 diff --git a/lib/spack/spack/util/module_cmd.py b/lib/spack/spack/util/module_cmd.py index 74790156ae..143ad3d43e 100644 --- a/lib/spack/spack/util/module_cmd.py +++ b/lib/spack/spack/util/module_cmd.py @@ -87,7 +87,13 @@ def get_path_args_from_module_line(line): words_and_symbols = line.split(lua_quote) path_arg = words_and_symbols[-2] else: - path_arg = line.split()[2] + # The path arg is the 3rd "word" of the line in a TCL module + # OPERATION VAR_NAME PATH_ARG + words = line.split() + if len(words) > 2: + path_arg = line.split()[2] + else: + return [] paths = path_arg.split(':') return paths diff --git a/var/spack/repos/builtin/packages/cray-libsci/package.py b/var/spack/repos/builtin/packages/cray-libsci/package.py index c0313a1e39..d391f471c3 100755 --- a/var/spack/repos/builtin/packages/cray-libsci/package.py +++ b/var/spack/repos/builtin/packages/cray-libsci/package.py @@ -2,10 +2,9 @@ # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) - -from llnl.util.filesystem import LibraryList -from spack import * -import os +from spack.concretize import NoBuildError +from spack.util.module_cmd import module +from spack.util.module_cmd import get_path_args_from_module_line class CrayLibsci(Package): @@ -22,14 +21,53 @@ class CrayLibsci(Package): version("16.06.1") version("16.03.1") + variant("shared", default=True, description="enable shared libs") + variant("openmp", default=False, description="link with openmp") + variant("mpi", default=False, description="link with mpi libs") + provides("blas") provides("lapack") provides("scalapack") - # NOTE: Cray compiler wrappers already include linking for the following + canonical_names = { + 'gcc': 'GNU', + 'cce': 'CRAY', + 'intel': 'INTEL', + } + + @property + def modname(self): + return "cray-libsci/{0}".format(self.version) + + @property + def external_prefix(self): + libsci_module = module("show", self.modname).splitlines() + + for line in libsci_module: + if "CRAY_LIBSCI_PREFIX_DIR" in line: + return get_path_args_from_module_line(line)[0] + @property def blas_libs(self): - return LibraryList(os.path.join(self.prefix.lib, 'libsci.so')) + shared = True if "+shared" in self.spec else False + compiler = self.spec.compiler.name + + if "+openmp" in self.spec and "+mpi" in self.spec: + lib = "libsci_{0}_mpi_mp" + elif "+openmp" in self.spec: + lib = "libsci_{0}_mp" + elif "+mpi" in self.spec: + lib = "libsci_{0}_mpi" + else: + lib = "libsci_{0}" + + libname = lib.format(self.canonical_names[compiler].lower()) + + return find_libraries( + libname, + root=self.prefix.lib, + shared=shared, + recursive=False) @property def lapack_libs(self): diff --git a/var/spack/repos/builtin/packages/mvapich2/package.py b/var/spack/repos/builtin/packages/mvapich2/package.py index f6301a564a..d8398b336d 100644 --- a/var/spack/repos/builtin/packages/mvapich2/package.py +++ b/var/spack/repos/builtin/packages/mvapich2/package.py @@ -208,10 +208,17 @@ class Mvapich2(AutotoolsPackage): env.set('SLURM_MPI_TYPE', 'pmi2') def setup_dependent_build_environment(self, env, dependent_spec): - env.set('MPICC', os.path.join(self.prefix.bin, 'mpicc')) - env.set('MPICXX', os.path.join(self.prefix.bin, 'mpicxx')) - env.set('MPIF77', os.path.join(self.prefix.bin, 'mpif77')) - env.set('MPIF90', os.path.join(self.prefix.bin, 'mpif90')) + # On Cray, the regular compiler wrappers *are* the MPI wrappers. + if 'platform=cray' in self.spec: + env.set('MPICC', spack_cc) + env.set('MPICXX', spack_cxx) + env.set('MPIF77', spack_fc) + env.set('MPIF90', spack_fc) + else: + env.set('MPICC', join_path(self.prefix.bin, 'mpicc')) + env.set('MPICXX', join_path(self.prefix.bin, 'mpicxx')) + env.set('MPIF77', join_path(self.prefix.bin, 'mpif77')) + env.set('MPIF90', join_path(self.prefix.bin, 'mpif90')) env.set('MPICH_CC', spack_cc) env.set('MPICH_CXX', spack_cxx) @@ -220,10 +227,17 @@ class Mvapich2(AutotoolsPackage): env.set('MPICH_FC', spack_fc) def setup_dependent_package(self, module, dependent_spec): - self.spec.mpicc = os.path.join(self.prefix.bin, 'mpicc') - self.spec.mpicxx = os.path.join(self.prefix.bin, 'mpicxx') - self.spec.mpifc = os.path.join(self.prefix.bin, 'mpif90') - self.spec.mpif77 = os.path.join(self.prefix.bin, 'mpif77') + if 'platform=cray' in self.spec: + self.spec.mpicc = spack_cc + self.spec.mpicxx = spack_cxx + self.spec.mpifc = spack_fc + self.spec.mpif77 = spack_f77 + else: + self.spec.mpicc = join_path(self.prefix.bin, 'mpicc') + self.spec.mpicxx = join_path(self.prefix.bin, 'mpicxx') + self.spec.mpifc = join_path(self.prefix.bin, 'mpif90') + self.spec.mpif77 = join_path(self.prefix.bin, 'mpif77') + self.spec.mpicxx_shared_libs = [ os.path.join(self.prefix.lib, 'libmpicxx.{0}'.format(dso_suffix)), os.path.join(self.prefix.lib, 'libmpi.{0}'.format(dso_suffix)) -- cgit v1.2.3-60-g2f50