From 0858c281e493fc0d83b59f2977defb9895af70c6 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Sun, 8 May 2022 17:51:27 -0700 Subject: Cray manifest file: accept "nvidia" as "nvhpc" (#30428) * create function for translating compiler names on specs/compiler entries in manifest * add tests for translating compiler names on spec/compiler entries * use higher-level function in test and add comment to prefer testing via higher-level function * opensuse clingo check should not fail on account of this pr, but I cannot get it to pass by restarting via CI UI --- lib/spack/spack/cray_manifest.py | 32 +++++++++- lib/spack/spack/test/cray_manifest.py | 113 ++++++++++++++++++++++++++++++---- 2 files changed, 132 insertions(+), 13 deletions(-) diff --git a/lib/spack/spack/cray_manifest.py b/lib/spack/spack/cray_manifest.py index 6b9af13a90..b78888d258 100644 --- a/lib/spack/spack/cray_manifest.py +++ b/lib/spack/spack/cray_manifest.py @@ -18,9 +18,37 @@ from spack.schema.cray_manifest import schema as manifest_schema #: packages here. default_path = '/opt/cray/pe/cpe-descriptive-manifest/' +compiler_name_translation = { + 'nvidia': 'nvhpc', +} + + +def translated_compiler_name(manifest_compiler_name): + """ + When creating a Compiler object, Spack expects a name matching + one of the classes in `spack.compilers`. Names in the Cray manifest + may differ; for cases where we know the name refers to a compiler in + Spack, this function translates it automatically. + + This function will raise an error if there is no recorded translation + and the name doesn't match a known compiler name. + """ + if manifest_compiler_name in compiler_name_translation: + return compiler_name_translation[manifest_compiler_name] + elif manifest_compiler_name in spack.compilers.supported_compilers(): + return manifest_compiler_name + else: + # Try to fail quickly. This can occur in two cases: (1) the compiler + # definition (2) a spec can specify a compiler that doesn't exist; the + # first will be caught when creating compiler definition. The second + # will result in Specs with associated undefined compilers. + raise spack.compilers.UnknownCompilerError( + "Manifest parsing - unknown compiler: {0}" + .format(manifest_compiler_name)) + def compiler_from_entry(entry): - compiler_name = entry['name'] + compiler_name = translated_compiler_name(entry['name']) paths = entry['executables'] version = entry['version'] arch = entry['arch'] @@ -49,7 +77,7 @@ def spec_from_entry(entry): if 'compiler' in entry: compiler_format = "%{name}@{version}" compiler_str = compiler_format.format( - name=entry['compiler']['name'], + name=translated_compiler_name(entry['compiler']['name']), version=entry['compiler']['version'] ) diff --git a/lib/spack/spack/test/cray_manifest.py b/lib/spack/spack/test/cray_manifest.py index d93990c677..97bd7e1d14 100644 --- a/lib/spack/spack/test/cray_manifest.py +++ b/lib/spack/spack/test/cray_manifest.py @@ -3,6 +3,12 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) +""" +Note that where possible, this should produce specs using `entries_to_specs` +rather than `spec_from_entry`, since the former does additional work to +establish dependency relationships (and in general the manifest-parsing +logic needs to consume all related specs in a single pass). +""" import json import pytest @@ -106,14 +112,38 @@ class JsonArchEntry(object): class JsonCompilerEntry(object): - def __init__(self, name, version): + def __init__(self, name, version, arch=None, executables=None): self.name = name self.version = version + if not arch: + arch = { + "os": "centos8", + "target": "x86_64" + } + if not executables: + executables = { + "cc": "/path/to/compiler/cc", + "cxx": "/path/to/compiler/cxx", + "fc": "/path/to/compiler/fc" + } + self.arch = arch + self.executables = executables - def to_dict(self): + def compiler_json(self): + return { + 'name': self.name, + 'version': self.version, + 'arch': self.arch, + 'executables': self.executables, + } + + def spec_json(self): + """The compiler spec only lists the name/version, not + arch/executables. + """ return { 'name': self.name, - 'version': self.version + 'version': self.version, } @@ -123,11 +153,20 @@ _common_arch = JsonArchEntry( target='haswell' ).to_dict() - +# Intended to match example_compiler_entry above _common_compiler = JsonCompilerEntry( name='gcc', - version='10.2.0' -).to_dict() + version='10.2.0', + arch={ + "os": "centos8", + "target": "x86_64" + }, + executables={ + "cc": "/path/to/compiler/cc", + "cxx": "/path/to/compiler/cxx", + "fc": "/path/to/compiler/fc" + } +) def test_compatibility(): @@ -142,7 +181,7 @@ def test_compatibility(): prefix='/path/to/packagey-install/', version='1.0', arch=_common_arch, - compiler=_common_compiler, + compiler=_common_compiler.spec_json(), dependencies={}, parameters={} ) @@ -153,7 +192,7 @@ def test_compatibility(): prefix='/path/to/packagex-install/', version='1.0', arch=_common_arch, - compiler=_common_compiler, + compiler=_common_compiler.spec_json(), dependencies=dict([y.as_dependency(deptypes=['link'])]), parameters={'precision': ['double', 'float']} ) @@ -180,7 +219,7 @@ def generate_openmpi_entries(): prefix='/path/to/hwloc-install/', version='2.0.3', arch=_common_arch, - compiler=_common_compiler, + compiler=_common_compiler.spec_json(), dependencies={}, parameters={} ) @@ -194,7 +233,7 @@ def generate_openmpi_entries(): prefix='/path/to/openmpi-install/', version='4.1.0', arch=_common_arch, - compiler=_common_compiler, + compiler=_common_compiler.spec_json(), dependencies=dict([hwloc.as_dependency(deptypes=['link'])]), parameters={ 'internal-hwloc': False, @@ -206,7 +245,7 @@ def generate_openmpi_entries(): return [openmpi, hwloc] -def test_spec_conversion(): +def test_generate_specs_from_manifest(): """Given JSON entries, check that we can form a set of Specs including dependency references. """ @@ -216,6 +255,58 @@ def test_spec_conversion(): assert openmpi_spec['hwloc'] +def test_translate_compiler_name(): + nvidia_compiler = JsonCompilerEntry( + name='nvidia', + version='19.1', + executables={ + "cc": "/path/to/compiler/nvc", + "cxx": "/path/to/compiler/nvc++", + } + ) + + compiler = compiler_from_entry(nvidia_compiler.compiler_json()) + assert compiler.name == 'nvhpc' + + spec_json = JsonSpecEntry( + name='hwloc', + hash='hwlocfakehashaaa', + prefix='/path/to/hwloc-install/', + version='2.0.3', + arch=_common_arch, + compiler=nvidia_compiler.spec_json(), + dependencies={}, + parameters={} + ).to_dict() + + spec, = entries_to_specs([spec_json]).values() + assert spec.compiler.name == 'nvhpc' + + +def test_failed_translate_compiler_name(): + unknown_compiler = JsonCompilerEntry( + name='unknown', + version='1.0' + ) + + with pytest.raises(spack.compilers.UnknownCompilerError): + compiler_from_entry(unknown_compiler.compiler_json()) + + spec_json = JsonSpecEntry( + name='packagey', + hash='hash-of-y', + prefix='/path/to/packagey-install/', + version='1.0', + arch=_common_arch, + compiler=unknown_compiler.spec_json(), + dependencies={}, + parameters={} + ).to_dict() + + with pytest.raises(spack.compilers.UnknownCompilerError): + entries_to_specs([spec_json]) + + def create_manifest_content(): return { 'specs': list(x.to_dict() for x in generate_openmpi_entries()), -- cgit v1.2.3-70-g09d2