diff options
author | Massimiliano Culpo <massimiliano.culpo@gmail.com> | 2024-09-21 14:05:41 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2024-09-21 14:05:41 +0200 |
commit | b93c57cab972d676c9540e01137fd5e0af87fa5a (patch) | |
tree | 154fd078706aab23f08478ab6b09a828be8a2883 | |
parent | 35ae2743d9888098a8f1409a327aeefa2f97f4b0 (diff) | |
download | spack-b93c57cab972d676c9540e01137fd5e0af87fa5a.tar.gz spack-b93c57cab972d676c9540e01137fd5e0af87fa5a.tar.bz2 spack-b93c57cab972d676c9540e01137fd5e0af87fa5a.tar.xz spack-b93c57cab972d676c9540e01137fd5e0af87fa5a.zip |
Remove `spack.target` from code (#46503)
The `spack.target.Target` class is a weird entity, that is just needed to:
1. Sort microarchitectures in lists deterministically
2. Being able to use microarchitectures in hashed containers
This PR removes it, and uses `archspec.cpu.Microarchitecture` directly. To sort lists, we use a proper `key=` when needed. Being able to use `Microarchitecture` objects in sets is achieved by updating the external `archspec`.
Signed-off-by: Massimiliano Culpo <massimiliano.culpo@gmail.com>
28 files changed, 201 insertions, 326 deletions
diff --git a/lib/spack/docs/conf.py b/lib/spack/docs/conf.py index 95b711db93..c80a661abd 100644 --- a/lib/spack/docs/conf.py +++ b/lib/spack/docs/conf.py @@ -219,6 +219,7 @@ nitpick_ignore = [ ("py:class", "spack.install_test.Pb"), ("py:class", "spack.filesystem_view.SimpleFilesystemView"), ("py:class", "spack.traverse.EdgeAndDepth"), + ("py:class", "archspec.cpu.microarchitecture.Microarchitecture"), ] # The reST default role (used for this markup: `text`) to use for all documents. diff --git a/lib/spack/external/__init__.py b/lib/spack/external/__init__.py index b6a35925d5..603cb412c8 100644 --- a/lib/spack/external/__init__.py +++ b/lib/spack/external/__init__.py @@ -18,7 +18,7 @@ archspec * Homepage: https://pypi.python.org/pypi/archspec * Usage: Labeling, comparison and detection of microarchitectures -* Version: 0.2.5-dev (commit cbb1fd5eb397a70d466e5160b393b87b0dbcc78f) +* Version: 0.2.5-dev (commit bceb39528ac49dd0c876b2e9bf3e7482e9c2be4a) astunparse ---------------- diff --git a/lib/spack/external/archspec/cpu/microarchitecture.py b/lib/spack/external/archspec/cpu/microarchitecture.py index 1ffe51d918..3f4e30fab5 100644 --- a/lib/spack/external/archspec/cpu/microarchitecture.py +++ b/lib/spack/external/archspec/cpu/microarchitecture.py @@ -115,6 +115,9 @@ class Microarchitecture: and self.cpu_part == other.cpu_part ) + def __hash__(self): + return hash(self.name) + @coerce_target_names def __ne__(self, other): return not self == other diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 26eb0e79f6..fd0adbf39d 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -45,6 +45,8 @@ from enum import Flag, auto from itertools import chain from typing import Dict, List, Set, Tuple +import archspec.cpu + import llnl.util.tty as tty from llnl.string import plural from llnl.util.filesystem import join_path @@ -358,7 +360,7 @@ def set_compiler_environment_variables(pkg, env): _add_werror_handling(keep_werror, env) # Set the target parameters that the compiler will add - isa_arg = spec.architecture.target.optimization_flags(compiler) + isa_arg = optimization_flags(compiler, spec.target) env.set("SPACK_TARGET_ARGS", isa_arg) # Trap spack-tracked compiler flags as appropriate. @@ -403,6 +405,36 @@ def set_compiler_environment_variables(pkg, env): return env +def optimization_flags(compiler, target): + if spack.compilers.is_mixed_toolchain(compiler): + msg = ( + "microarchitecture specific optimizations are not " + "supported yet on mixed compiler toolchains [check" + f" {compiler.name}@{compiler.version} for further details]" + ) + tty.debug(msg) + return "" + + # Try to check if the current compiler comes with a version number or + # has an unexpected suffix. If so, treat it as a compiler with a + # custom spec. + compiler_version = compiler.version + version_number, suffix = archspec.cpu.version_components(compiler.version) + if not version_number or suffix: + try: + compiler_version = compiler.real_version + except spack.util.executable.ProcessError as e: + # log this and just return compiler.version instead + tty.debug(str(e)) + + try: + result = target.optimization_flags(compiler.name, compiler_version.dotted_numeric_string) + except (ValueError, archspec.cpu.UnsupportedMicroarchitecture): + result = "" + + return result + + def set_wrapper_variables(pkg, env): """Set environment variables used by the Spack compiler wrapper (which have the prefix `SPACK_`) and also add the compiler wrappers to PATH. @@ -783,7 +815,6 @@ def setup_package(pkg, dirty, context: Context = Context.BUILD): # Platform specific setup goes before package specific setup. This is for setting # defaults like MACOSX_DEPLOYMENT_TARGET on macOS. platform = spack.platforms.by_name(pkg.spec.architecture.platform) - target = platform.target(pkg.spec.architecture.target) platform.setup_platform_environment(pkg, env_mods) tty.debug("setup_package: grabbing modifications from dependencies") @@ -808,9 +839,6 @@ def setup_package(pkg, dirty, context: Context = Context.BUILD): for mod in pkg.compiler.modules: load_module(mod) - if target and target.module_name: - load_module(target.module_name) - load_external_modules(pkg) implicit_rpaths = pkg.compiler.implicit_rpaths() diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index 6b03810826..bfb781685f 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -558,7 +558,7 @@ def get_compilers(config, cspec=None, arch_spec=None): except KeyError: # TODO: Check if this exception handling makes sense, or if we # TODO: need to change / refactor tests - family = arch_spec.target + family = str(arch_spec.target) except AttributeError: assert arch_spec is None @@ -803,12 +803,11 @@ class CompilerConfigFactory: if not spec.architecture: host_platform = spack.platforms.host() operating_system = host_platform.operating_system("default_os") - target = host_platform.target("default_target").microarchitecture + target = host_platform.target("default_target") else: target = spec.architecture.target if not target: target = spack.platforms.host().target("default_target") - target = target.microarchitecture operating_system = spec.os if not operating_system: diff --git a/lib/spack/spack/patch.py b/lib/spack/spack/patch.py index 50393ecfad..a0f4152317 100644 --- a/lib/spack/spack/patch.py +++ b/lib/spack/spack/patch.py @@ -15,7 +15,7 @@ from llnl.url import allowed_archive import spack import spack.error -import spack.fetch_strategy as fs +import spack.fetch_strategy import spack.mirror import spack.repo import spack.stage @@ -314,11 +314,15 @@ class UrlPatch(Patch): # Two checksums, one for compressed file, one for its contents if self.archive_sha256 and self.sha256: - fetcher: fs.FetchStrategy = fs.FetchAndVerifyExpandedFile( - self.url, archive_sha256=self.archive_sha256, expanded_sha256=self.sha256 + fetcher: spack.fetch_strategy.FetchStrategy = ( + spack.fetch_strategy.FetchAndVerifyExpandedFile( + self.url, archive_sha256=self.archive_sha256, expanded_sha256=self.sha256 + ) ) else: - fetcher = fs.URLFetchStrategy(url=self.url, sha256=self.sha256, expand=False) + fetcher = spack.fetch_strategy.URLFetchStrategy( + url=self.url, sha256=self.sha256, expand=False + ) # The same package can have multiple patches with the same name but # with different contents, therefore apply a subset of the hash. @@ -397,7 +401,7 @@ def from_dict( sha256 = dictionary["sha256"] checker = Checker(sha256) if patch.path and not checker.check(patch.path): - raise fs.ChecksumError( + raise spack.fetch_strategy.ChecksumError( "sha256 checksum failed for %s" % patch.path, "Expected %s but got %s " % (sha256, checker.sum) + "Patch may have changed since concretization.", diff --git a/lib/spack/spack/platforms/_platform.py b/lib/spack/spack/platforms/_platform.py index c165cf9f33..8668958966 100644 --- a/lib/spack/spack/platforms/_platform.py +++ b/lib/spack/spack/platforms/_platform.py @@ -4,6 +4,8 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) from typing import Optional +import archspec.cpu + import llnl.util.lang import spack.error @@ -60,7 +62,7 @@ class Platform: self.operating_sys = {} self.name = name - def add_target(self, name, target): + def add_target(self, name: str, target: archspec.cpu.Microarchitecture) -> None: """Used by the platform specific subclass to list available targets. Raises an error if the platform specifies a name that is reserved by spack as an alias. @@ -70,6 +72,10 @@ class Platform: raise ValueError(msg.format(name)) self.targets[name] = target + def _add_archspec_targets(self): + for name, microarchitecture in archspec.cpu.TARGETS.items(): + self.add_target(name, microarchitecture) + def target(self, name): """This is a getter method for the target dictionary that handles defaulting based on the values provided by default, diff --git a/lib/spack/spack/platforms/darwin.py b/lib/spack/spack/platforms/darwin.py index 1b7a5927f4..7ea6a09c58 100644 --- a/lib/spack/spack/platforms/darwin.py +++ b/lib/spack/spack/platforms/darwin.py @@ -7,7 +7,6 @@ import platform as py_platform import archspec.cpu -import spack.target from spack.operating_systems.mac_os import MacOs from spack.version import Version @@ -21,9 +20,7 @@ class Darwin(Platform): def __init__(self): super().__init__("darwin") - - for name in archspec.cpu.TARGETS: - self.add_target(name, spack.target.Target(name)) + self._add_archspec_targets() self.default = archspec.cpu.host().name self.front_end = self.default diff --git a/lib/spack/spack/platforms/freebsd.py b/lib/spack/spack/platforms/freebsd.py index 4485550789..af9b1a0934 100644 --- a/lib/spack/spack/platforms/freebsd.py +++ b/lib/spack/spack/platforms/freebsd.py @@ -6,7 +6,6 @@ import platform import archspec.cpu -import spack.target from spack.operating_systems.freebsd import FreeBSDOs from ._platform import Platform @@ -18,8 +17,7 @@ class FreeBSD(Platform): def __init__(self): super().__init__("freebsd") - for name in archspec.cpu.TARGETS: - self.add_target(name, spack.target.Target(name)) + self._add_archspec_targets() # Get specific default self.default = archspec.cpu.host().name diff --git a/lib/spack/spack/platforms/linux.py b/lib/spack/spack/platforms/linux.py index 2be5b51cb2..aede37d868 100644 --- a/lib/spack/spack/platforms/linux.py +++ b/lib/spack/spack/platforms/linux.py @@ -6,7 +6,6 @@ import platform import archspec.cpu -import spack.target from spack.operating_systems.linux_distro import LinuxDistro from ._platform import Platform @@ -18,8 +17,7 @@ class Linux(Platform): def __init__(self): super().__init__("linux") - for name in archspec.cpu.TARGETS: - self.add_target(name, spack.target.Target(name)) + self._add_archspec_targets() # Get specific default self.default = archspec.cpu.host().name diff --git a/lib/spack/spack/platforms/test.py b/lib/spack/spack/platforms/test.py index 9ead66ab27..8633efc7cb 100644 --- a/lib/spack/spack/platforms/test.py +++ b/lib/spack/spack/platforms/test.py @@ -4,8 +4,9 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) import platform +import archspec.cpu + import spack.operating_systems -import spack.target from ._platform import Platform @@ -32,8 +33,8 @@ class Test(Platform): def __init__(self, name=None): name = name or "test" super().__init__(name) - self.add_target(self.default, spack.target.Target(self.default)) - self.add_target(self.front_end, spack.target.Target(self.front_end)) + self.add_target(self.default, archspec.cpu.TARGETS[self.default]) + self.add_target(self.front_end, archspec.cpu.TARGETS[self.front_end]) self.add_operating_system( self.default_os, spack.operating_systems.OperatingSystem("debian", 6) diff --git a/lib/spack/spack/platforms/windows.py b/lib/spack/spack/platforms/windows.py index c00382e198..8cc89b4b19 100755 --- a/lib/spack/spack/platforms/windows.py +++ b/lib/spack/spack/platforms/windows.py @@ -7,7 +7,6 @@ import platform import archspec.cpu -import spack.target from spack.operating_systems.windows_os import WindowsOs from ._platform import Platform @@ -18,9 +17,7 @@ class Windows(Platform): def __init__(self): super().__init__("windows") - - for name in archspec.cpu.TARGETS: - self.add_target(name, spack.target.Target(name)) + self._add_archspec_targets() self.default = archspec.cpu.host().name self.front_end = self.default diff --git a/lib/spack/spack/solver/asp.py b/lib/spack/spack/solver/asp.py index d8d052f311..3fd8b9cc8d 100644 --- a/lib/spack/spack/solver/asp.py +++ b/lib/spack/spack/solver/asp.py @@ -2479,7 +2479,7 @@ class SpackSolverSetup: return allowed_targets cache = {} - for target_constraint in sorted(self.target_constraints): + for target_constraint in sorted(self.target_constraints, key=lambda x: x.name): # Construct the list of allowed targets for this constraint allowed_targets = [] for single_constraint in str(target_constraint).split(","): @@ -3237,7 +3237,7 @@ class CompilerParser: candidate = KnownCompiler( spec=spec.compiler, os=str(spec.architecture.os), - target=str(spec.architecture.target.microarchitecture.family), + target=str(spec.architecture.target.family), available=False, compiler_obj=None, ) diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index fb1a05f37c..1c566349a4 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -61,6 +61,8 @@ import socket import warnings from typing import Any, Callable, Dict, List, Match, Optional, Set, Tuple, Union +import archspec.cpu + import llnl.path import llnl.string import llnl.util.filesystem as fs @@ -82,7 +84,6 @@ import spack.provider_index import spack.repo import spack.solver import spack.store -import spack.target import spack.traverse as traverse import spack.util.executable import spack.util.hash @@ -213,6 +214,12 @@ def ensure_modern_format_string(fmt: str) -> None: ) +def _make_microarchitecture(name: str) -> archspec.cpu.Microarchitecture: + if isinstance(name, archspec.cpu.Microarchitecture): + return name + return archspec.cpu.TARGETS.get(name, archspec.cpu.generic_microarchitecture(name)) + + @lang.lazy_lexicographic_ordering class ArchSpec: """Aggregate the target platform, the operating system and the target microarchitecture.""" @@ -301,7 +308,10 @@ class ArchSpec: def _cmp_iter(self): yield self.platform yield self.os - yield self.target + if self.target is None: + yield self.target + else: + yield self.target.name @property def platform(self): @@ -360,10 +370,10 @@ class ArchSpec: # will assumed to be the host machine's platform. def target_or_none(t): - if isinstance(t, spack.target.Target): + if isinstance(t, archspec.cpu.Microarchitecture): return t if t and t != "None": - return spack.target.Target(t) + return _make_microarchitecture(t) return None value = target_or_none(value) @@ -452,10 +462,11 @@ class ArchSpec: results = self._target_intersection(other) attribute_str = ",".join(results) - if self.target == attribute_str: + intersection_target = _make_microarchitecture(attribute_str) + if self.target == intersection_target: return False - self.target = attribute_str + self.target = intersection_target return True def _target_intersection(self, other): @@ -473,7 +484,7 @@ class ArchSpec: # s_target_range is a concrete target # get a microarchitecture reference for at least one side # of each comparison so we can use archspec comparators - s_comp = spack.target.Target(s_min).microarchitecture + s_comp = _make_microarchitecture(s_min) if not o_sep: if s_min == o_min: results.append(s_min) @@ -481,21 +492,21 @@ class ArchSpec: results.append(s_min) elif not o_sep: # "cast" to microarchitecture - o_comp = spack.target.Target(o_min).microarchitecture + o_comp = _make_microarchitecture(o_min) if (not s_min or o_comp >= s_min) and (not s_max or o_comp <= s_max): results.append(o_min) else: # Take intersection of two ranges # Lots of comparisons needed - _s_min = spack.target.Target(s_min).microarchitecture - _s_max = spack.target.Target(s_max).microarchitecture - _o_min = spack.target.Target(o_min).microarchitecture - _o_max = spack.target.Target(o_max).microarchitecture + _s_min = _make_microarchitecture(s_min) + _s_max = _make_microarchitecture(s_max) + _o_min = _make_microarchitecture(o_min) + _o_max = _make_microarchitecture(o_max) n_min = s_min if _s_min >= _o_min else o_min n_max = s_max if _s_max <= _o_max else o_max - _n_min = spack.target.Target(n_min).microarchitecture - _n_max = spack.target.Target(n_max).microarchitecture + _n_min = _make_microarchitecture(n_min) + _n_max = _make_microarchitecture(n_max) if _n_min == _n_max: results.append(n_min) elif not n_min or not n_max or _n_min < _n_max: @@ -548,12 +559,18 @@ class ArchSpec: ) def to_dict(self): + # Generic targets represent either an architecture family (like x86_64) + # or a custom micro-architecture + if self.target.vendor == "generic": + target_data = str(self.target) + else: + # Get rid of compiler flag information before turning the uarch into a dict + uarch_dict = self.target.to_dict() + uarch_dict.pop("compilers", None) + target_data = syaml.syaml_dict(uarch_dict.items()) + d = syaml.syaml_dict( - [ - ("platform", self.platform), - ("platform_os", self.os), - ("target", self.target.to_dict_or_value()), - ] + [("platform", self.platform), ("platform_os", self.os), ("target", target_data)] ) return syaml.syaml_dict([("arch", d)]) @@ -561,7 +578,10 @@ class ArchSpec: def from_dict(d): """Import an ArchSpec from raw YAML/JSON data""" arch = d["arch"] - target = spack.target.Target.from_dict_or_value(arch["target"]) + target_name = arch["target"] + if not isinstance(target_name, str): + target_name = target_name["name"] + target = _make_microarchitecture(target_name) return ArchSpec((arch["platform"], arch["platform_os"], target)) def __str__(self): @@ -4120,9 +4140,7 @@ class Spec: @property def target(self): - # This property returns the underlying microarchitecture object - # to give to the attribute the appropriate comparison semantic - return self.architecture.target.microarchitecture + return self.architecture.target @property def build_spec(self): diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index ee3b4e95e5..8b4efcf387 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -33,7 +33,6 @@ from llnl.util.tty.color import colorize import spack.caches import spack.config import spack.error -import spack.fetch_strategy as fs import spack.mirror import spack.resource import spack.spec @@ -43,6 +42,7 @@ import spack.util.parallel import spack.util.path as sup import spack.util.pattern as pattern import spack.util.url as url_util +from spack import fetch_strategy as fs # breaks a cycle from spack.util.crypto import bit_length, prefix_bits from spack.util.editor import editor, executable from spack.version import StandardVersion, VersionList @@ -352,8 +352,8 @@ class Stage(LockableStagingDir): url_or_fetch_strategy, *, name=None, - mirror_paths: Optional[spack.mirror.MirrorLayout] = None, - mirrors: Optional[Iterable[spack.mirror.Mirror]] = None, + mirror_paths: Optional["spack.mirror.MirrorLayout"] = None, + mirrors: Optional[Iterable["spack.mirror.Mirror"]] = None, keep=False, path=None, lock=True, @@ -464,7 +464,7 @@ class Stage(LockableStagingDir): """Returns the well-known source directory path.""" return os.path.join(self.path, _source_path_subdir) - def _generate_fetchers(self, mirror_only=False) -> Generator[fs.FetchStrategy, None, None]: + def _generate_fetchers(self, mirror_only=False) -> Generator["fs.FetchStrategy", None, None]: fetchers: List[fs.FetchStrategy] = [] if not mirror_only: fetchers.append(self.default_fetcher) @@ -600,7 +600,7 @@ class Stage(LockableStagingDir): spack.caches.FETCH_CACHE.store(self.fetcher, self.mirror_layout.path) def cache_mirror( - self, mirror: spack.caches.MirrorCache, stats: spack.mirror.MirrorStats + self, mirror: "spack.caches.MirrorCache", stats: "spack.mirror.MirrorStats" ) -> None: """Perform a fetch if the resource is not already cached @@ -668,7 +668,7 @@ class Stage(LockableStagingDir): class ResourceStage(Stage): def __init__( self, - fetch_strategy: fs.FetchStrategy, + fetch_strategy: "fs.FetchStrategy", root: Stage, resource: spack.resource.Resource, **kwargs, diff --git a/lib/spack/spack/target.py b/lib/spack/spack/target.py deleted file mode 100644 index 4a78196865..0000000000 --- a/lib/spack/spack/target.py +++ /dev/null @@ -1,161 +0,0 @@ -# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) -import functools - -import archspec.cpu - -import llnl.util.tty as tty - -import spack.compiler -import spack.compilers -import spack.spec -import spack.util.executable -import spack.util.spack_yaml as syaml - - -def _ensure_other_is_target(method): - """In a single argument method, ensure that the argument is an - instance of ``Target``. - """ - - @functools.wraps(method) - def _impl(self, other): - if isinstance(other, str): - other = Target(other) - - if not isinstance(other, Target): - return NotImplemented - - return method(self, other) - - return _impl - - -class Target: - def __init__(self, name, module_name=None): - """Target models microarchitectures and their compatibility. - - Args: - name (str or Microarchitecture): microarchitecture of the target - module_name (str): optional module name to get access to the - current target. This is typically used on machines - like Cray (e.g. craype-compiler) - """ - if not isinstance(name, archspec.cpu.Microarchitecture): - name = archspec.cpu.TARGETS.get(name, archspec.cpu.generic_microarchitecture(name)) - self.microarchitecture = name - self.module_name = module_name - - @property - def name(self): - return self.microarchitecture.name - - @_ensure_other_is_target - def __eq__(self, other): - return ( - self.microarchitecture == other.microarchitecture - and self.module_name == other.module_name - ) - - def __ne__(self, other): - # This method is necessary as long as we support Python 2. In Python 3 - # __ne__ defaults to the implementation below - return not self == other - - @_ensure_other_is_target - def __lt__(self, other): - # TODO: In the future it would be convenient to say - # TODO: `spec.architecture.target < other.architecture.target` - # TODO: and change the semantic of the comparison operators - - # This is needed to sort deterministically specs in a list. - # It doesn't implement a total ordering semantic. - return self.microarchitecture.name < other.microarchitecture.name - - def __hash__(self): - return hash((self.name, self.module_name)) - - @staticmethod - def from_dict_or_value(dict_or_value): - # A string here represents a generic target (like x86_64 or ppc64) or - # a custom micro-architecture - if isinstance(dict_or_value, str): - return Target(dict_or_value) - - # TODO: From a dict we actually retrieve much more information than - # TODO: just the name. We can use that information to reconstruct an - # TODO: "old" micro-architecture or check the current definition. - target_info = dict_or_value - return Target(target_info["name"]) - - def to_dict_or_value(self): - """Returns a dict or a value representing the current target. - - String values are used to keep backward compatibility with generic - targets, like e.g. x86_64 or ppc64. More specific micro-architectures - will return a dictionary which contains information on the name, - features, vendor, generation and parents of the current target. - """ - # Generic targets represent either an architecture - # family (like x86_64) or a custom micro-architecture - if self.microarchitecture.vendor == "generic": - return str(self) - - # Get rid of compiler flag information before turning the uarch into a dict - uarch_dict = self.microarchitecture.to_dict() - uarch_dict.pop("compilers", None) - return syaml.syaml_dict(uarch_dict.items()) - - def __repr__(self): - cls_name = self.__class__.__name__ - fmt = cls_name + "({0}, {1})" - return fmt.format(repr(self.microarchitecture), repr(self.module_name)) - - def __str__(self): - return str(self.microarchitecture) - - def __contains__(self, cpu_flag): - return cpu_flag in self.microarchitecture - - def optimization_flags(self, compiler): - """Returns the flags needed to optimize for this target using - the compiler passed as argument. - - Args: - compiler (spack.spec.CompilerSpec or spack.compiler.Compiler): object that - contains both the name and the version of the compiler we want to use - """ - # Mixed toolchains are not supported yet - if isinstance(compiler, spack.compiler.Compiler): - if spack.compilers.is_mixed_toolchain(compiler): - msg = ( - "microarchitecture specific optimizations are not " - "supported yet on mixed compiler toolchains [check" - " {0.name}@{0.version} for further details]" - ) - tty.debug(msg.format(compiler)) - return "" - - # Try to check if the current compiler comes with a version number or - # has an unexpected suffix. If so, treat it as a compiler with a - # custom spec. - compiler_version = compiler.version - version_number, suffix = archspec.cpu.version_components(compiler.version) - if not version_number or suffix: - # Try to deduce the underlying version of the compiler, regardless - # of its name in compilers.yaml. Depending on where this function - # is called we might get either a CompilerSpec or a fully fledged - # compiler object. - if isinstance(compiler, spack.spec.CompilerSpec): - compiler = spack.compilers.compilers_for_spec(compiler).pop() - try: - compiler_version = compiler.real_version - except spack.util.executable.ProcessError as e: - # log this and just return compiler.version instead - tty.debug(str(e)) - - return self.microarchitecture.optimization_flags( - compiler.name, compiler_version.dotted_numeric_string - ) diff --git a/lib/spack/spack/test/architecture.py b/lib/spack/spack/test/architecture.py index cbd8a0b9d7..3d489a22cc 100644 --- a/lib/spack/spack/test/architecture.py +++ b/lib/spack/spack/test/architecture.py @@ -12,7 +12,6 @@ import spack.compilers import spack.concretize import spack.operating_systems import spack.platforms -import spack.target from spack.spec import ArchSpec, Spec @@ -84,25 +83,6 @@ def test_operating_system_conversion_to_dict(): @pytest.mark.parametrize( - "cpu_flag,target_name", - [ - # Test that specific flags can be used in queries - ("ssse3", "haswell"), - ("popcnt", "nehalem"), - ("avx512f", "skylake_avx512"), - ("avx512ifma", "icelake"), - # Test that proxy flags can be used in queries too - ("sse3", "nehalem"), - ("avx512", "skylake_avx512"), - ("avx512", "icelake"), - ], -) -def test_target_container_semantic(cpu_flag, target_name): - target = spack.target.Target(target_name) - assert cpu_flag in target - - -@pytest.mark.parametrize( "item,architecture_str", [ # We can search the architecture string representation @@ -118,68 +98,6 @@ def test_arch_spec_container_semantic(item, architecture_str): assert item in architecture -@pytest.mark.parametrize( - "compiler_spec,target_name,expected_flags", - [ - # Homogeneous compilers - ("gcc@4.7.2", "ivybridge", "-march=core-avx-i -mtune=core-avx-i"), - ("clang@3.5", "x86_64", "-march=x86-64 -mtune=generic"), - ("apple-clang@9.1.0", "x86_64", "-march=x86-64"), - # Mixed toolchain - ("clang@8.0.0", "broadwell", ""), - ], -) -@pytest.mark.filterwarnings("ignore:microarchitecture specific") -@pytest.mark.not_on_windows("Windows doesn't support the compiler wrapper") -def test_optimization_flags(compiler_spec, target_name, expected_flags, compiler_factory): - target = spack.target.Target(target_name) - compiler_dict = compiler_factory(spec=compiler_spec, operating_system="")["compiler"] - if compiler_spec == "clang@8.0.0": - compiler_dict["paths"] = { - "cc": "/path/to/clang-8", - "cxx": "/path/to/clang++-8", - "f77": "/path/to/gfortran-9", - "fc": "/path/to/gfortran-9", - } - compiler = spack.compilers.compiler_from_dict(compiler_dict) - - opt_flags = target.optimization_flags(compiler) - assert opt_flags == expected_flags - - -@pytest.mark.parametrize( - "compiler_str,real_version,target_str,expected_flags", - [ - ("gcc@=9.2.0", None, "haswell", "-march=haswell -mtune=haswell"), - # Check that custom string versions are accepted - ("gcc@=10foo", "9.2.0", "icelake", "-march=icelake-client -mtune=icelake-client"), - # Check that we run version detection (4.4.0 doesn't support icelake) - ("gcc@=4.4.0-special", "9.2.0", "icelake", "-march=icelake-client -mtune=icelake-client"), - # Check that the special case for Apple's clang is treated correctly - # i.e. it won't try to detect the version again - ("apple-clang@=9.1.0", None, "x86_64", "-march=x86-64"), - ], -) -def test_optimization_flags_with_custom_versions( - compiler_str, - real_version, - target_str, - expected_flags, - monkeypatch, - mutable_config, - compiler_factory, -): - target = spack.target.Target(target_str) - compiler_dict = compiler_factory(spec=compiler_str, operating_system="redhat6") - mutable_config.set("compilers", [compiler_dict]) - if real_version: - monkeypatch.setattr(spack.compiler.Compiler, "get_real_version", lambda x: real_version) - compiler = spack.compilers.compiler_from_dict(compiler_dict["compiler"]) - - opt_flags = target.optimization_flags(compiler) - assert opt_flags == expected_flags - - @pytest.mark.regression("15306") @pytest.mark.parametrize( "architecture_tuple,constraint_tuple", diff --git a/lib/spack/spack/test/build_environment.py b/lib/spack/spack/test/build_environment.py index c3ccaee029..f435a893a1 100644 --- a/lib/spack/spack/test/build_environment.py +++ b/lib/spack/spack/test/build_environment.py @@ -9,6 +9,8 @@ import sys import pytest +import archspec.cpu + from llnl.path import Path, convert_to_platform_path from llnl.util.filesystem import HeaderList, LibraryList @@ -737,3 +739,64 @@ def test_rpath_with_duplicate_link_deps(): assert child in rpath_deps assert runtime_2 in rpath_deps assert runtime_1 not in rpath_deps + + +@pytest.mark.parametrize( + "compiler_spec,target_name,expected_flags", + [ + # Homogeneous compilers + ("gcc@4.7.2", "ivybridge", "-march=core-avx-i -mtune=core-avx-i"), + ("clang@3.5", "x86_64", "-march=x86-64 -mtune=generic"), + ("apple-clang@9.1.0", "x86_64", "-march=x86-64"), + # Mixed toolchain + ("clang@8.0.0", "broadwell", ""), + ], +) +@pytest.mark.filterwarnings("ignore:microarchitecture specific") +@pytest.mark.not_on_windows("Windows doesn't support the compiler wrapper") +def test_optimization_flags(compiler_spec, target_name, expected_flags, compiler_factory): + target = archspec.cpu.TARGETS[target_name] + compiler_dict = compiler_factory(spec=compiler_spec, operating_system="")["compiler"] + if compiler_spec == "clang@8.0.0": + compiler_dict["paths"] = { + "cc": "/path/to/clang-8", + "cxx": "/path/to/clang++-8", + "f77": "/path/to/gfortran-9", + "fc": "/path/to/gfortran-9", + } + compiler = spack.compilers.compiler_from_dict(compiler_dict) + opt_flags = spack.build_environment.optimization_flags(compiler, target) + assert opt_flags == expected_flags + + +@pytest.mark.parametrize( + "compiler_str,real_version,target_str,expected_flags", + [ + ("gcc@=9.2.0", None, "haswell", "-march=haswell -mtune=haswell"), + # Check that custom string versions are accepted + ("gcc@=10foo", "9.2.0", "icelake", "-march=icelake-client -mtune=icelake-client"), + # Check that we run version detection (4.4.0 doesn't support icelake) + ("gcc@=4.4.0-special", "9.2.0", "icelake", "-march=icelake-client -mtune=icelake-client"), + # Check that the special case for Apple's clang is treated correctly + # i.e. it won't try to detect the version again + ("apple-clang@=9.1.0", None, "x86_64", "-march=x86-64"), + ], +) +def test_optimization_flags_with_custom_versions( + compiler_str, + real_version, + target_str, + expected_flags, + monkeypatch, + mutable_config, + compiler_factory, +): + target = archspec.cpu.TARGETS[target_str] + compiler_dict = compiler_factory(spec=compiler_str, operating_system="redhat6") + mutable_config.set("compilers", [compiler_dict]) + if real_version: + monkeypatch.setattr(spack.compiler.Compiler, "get_real_version", lambda x: real_version) + compiler = spack.compilers.compiler_from_dict(compiler_dict["compiler"]) + + opt_flags = spack.build_environment.optimization_flags(compiler, target) + assert opt_flags == expected_flags diff --git a/lib/spack/spack/test/config.py b/lib/spack/spack/test/config.py index ddf21ffb05..be727f1c56 100644 --- a/lib/spack/spack/test/config.py +++ b/lib/spack/spack/test/config.py @@ -407,7 +407,7 @@ def test_substitute_config_variables(mock_low_high_config, monkeypatch): ) == os.path.abspath(os.path.join("foo", "test", "bar")) host_target = spack.platforms.host().target("default_target") - host_target_family = str(host_target.microarchitecture.family) + host_target_family = str(host_target.family) assert spack_path.canonicalize_path( os.path.join("foo", "$target_family", "bar") ) == os.path.abspath(os.path.join("foo", host_target_family, "bar")) diff --git a/lib/spack/spack/util/path.py b/lib/spack/spack/util/path.py index e0fa29dbcc..a91972c79b 100644 --- a/lib/spack/spack/util/path.py +++ b/lib/spack/spack/util/path.py @@ -71,7 +71,7 @@ def replacements(): "operating_system": lambda: arch.os, "os": lambda: arch.os, "target": lambda: arch.target, - "target_family": lambda: arch.target.microarchitecture.family, + "target_family": lambda: arch.target.family, "date": lambda: date.today().strftime("%Y-%m-%d"), "env": lambda: ev.active_environment().path if ev.active_environment() else NOMATCH, } diff --git a/var/spack/repos/builtin/packages/amdfftw/package.py b/var/spack/repos/builtin/packages/amdfftw/package.py index 6ac400a013..f508c4162b 100644 --- a/var/spack/repos/builtin/packages/amdfftw/package.py +++ b/var/spack/repos/builtin/packages/amdfftw/package.py @@ -7,6 +7,7 @@ import os from llnl.util import tty +from spack.build_environment import optimization_flags from spack.package import * from spack.pkg.builtin.fftw import FftwBase @@ -213,10 +214,7 @@ class Amdfftw(FftwBase): # variable to set AMD_ARCH configure option. # Spack user can not directly use AMD_ARCH for this purpose but should # use target variable to set appropriate -march option in AMD_ARCH. - arch = spec.architecture - options.append( - "AMD_ARCH={0}".format(arch.target.optimization_flags(spec.compiler).split("=")[-1]) - ) + options.append(f"AMD_ARCH={optimization_flags(self.compiler, spec.target)}") # Specific SIMD support. # float and double precisions are supported diff --git a/var/spack/repos/builtin/packages/arbor/package.py b/var/spack/repos/builtin/packages/arbor/package.py index 832078c7bd..415914b7ba 100644 --- a/var/spack/repos/builtin/packages/arbor/package.py +++ b/var/spack/repos/builtin/packages/arbor/package.py @@ -2,7 +2,7 @@ # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) - +from spack.build_environment import optimization_flags from spack.package import * @@ -110,21 +110,23 @@ class Arbor(CMakePackage, CudaPackage): return ["all", "html"] if "+doc" in self.spec else ["all"] def cmake_args(self): + spec = self.spec args = [ self.define_from_variant("ARB_WITH_ASSERTIONS", "assertions"), self.define_from_variant("ARB_WITH_MPI", "mpi"), self.define_from_variant("ARB_WITH_PYTHON", "python"), self.define_from_variant("ARB_VECTORIZE", "vectorize"), + self.define("ARB_ARCH", "none"), + self.define("ARB_CXX_FLAGS_TARGET", optimization_flags(self.compiler, spec.target)), ] if self.spec.satisfies("+cuda"): - args.append("-DARB_GPU=cuda") - args.append(self.define_from_variant("ARB_USE_GPU_RNG", "gpu_rng")) - - # query spack for the architecture-specific compiler flags set by its wrapper - args.append("-DARB_ARCH=none") - opt_flags = self.spec.architecture.target.optimization_flags(self.spec.compiler) - args.append("-DARB_CXX_FLAGS_TARGET=" + opt_flags) + args.extend( + [ + self.define("ARB_GPU", "cuda"), + self.define_from_variant("ARB_USE_GPU_RNG", "gpu_rng"), + ] + ) return args diff --git a/var/spack/repos/builtin/packages/hpcc/package.py b/var/spack/repos/builtin/packages/hpcc/package.py index 4483426024..e8a9acb6de 100644 --- a/var/spack/repos/builtin/packages/hpcc/package.py +++ b/var/spack/repos/builtin/packages/hpcc/package.py @@ -7,6 +7,7 @@ import os import platform import re +from spack.build_environment import optimization_flags from spack.package import * @@ -161,7 +162,7 @@ class Hpcc(MakefilePackage): if spec.satisfies("%intel"): # with intel-parallel-studio+mpi the '-march' arguments # are not passed to icc - arch_opt = spec.architecture.target.optimization_flags(spec.compiler) + arch_opt = optimization_flags(self.compiler, spec.target) self.config["@CCFLAGS@"] = f"-O3 -restrict -ansi-alias -ip {arch_opt}" self.config["@CCNOOPT@"] = "-restrict" self._write_make_arch(spec, prefix) diff --git a/var/spack/repos/builtin/packages/lammps/package.py b/var/spack/repos/builtin/packages/lammps/package.py index 5c1f9c0588..be1697543e 100644 --- a/var/spack/repos/builtin/packages/lammps/package.py +++ b/var/spack/repos/builtin/packages/lammps/package.py @@ -5,6 +5,7 @@ import datetime as dt import os +from spack.build_environment import optimization_flags from spack.package import * @@ -898,7 +899,7 @@ class Lammps(CMakePackage, CudaPackage, ROCmPackage, PythonExtension): args.append(self.define("CMAKE_CXX_FLAGS_RELWITHDEBINFO", cxx_flags)) # Overwrite generic cpu tune option - cmake_tune_flags = spec.architecture.target.optimization_flags(spec.compiler) + cmake_tune_flags = optimization_flags(self.compiler, spec.target) args.append(self.define("CMAKE_TUNE_FLAGS", cmake_tune_flags)) args.append(self.define_from_variant("LAMMPS_SIZES", "lammps_sizes")) diff --git a/var/spack/repos/builtin/packages/namd/package.py b/var/spack/repos/builtin/packages/namd/package.py index 2e9fc20f79..47fcb3b03c 100644 --- a/var/spack/repos/builtin/packages/namd/package.py +++ b/var/spack/repos/builtin/packages/namd/package.py @@ -9,6 +9,7 @@ import sys import llnl.util.tty as tty +from spack.build_environment import optimization_flags from spack.package import * @@ -175,7 +176,7 @@ class Namd(MakefilePackage, CudaPackage, ROCmPackage): # this options are take from the default provided # configuration files # https://github.com/UIUC-PPL/charm/pull/2778 - archopt = spec.architecture.target.optimization_flags(spec.compiler) + archopt = optimization_flags(self.compiler, spec.target) if self.spec.satisfies("^charmpp@:6.10.1"): optims_opts = { diff --git a/var/spack/repos/builtin/packages/neuron/package.py b/var/spack/repos/builtin/packages/neuron/package.py index 2887ea9342..2cb16fdd03 100644 --- a/var/spack/repos/builtin/packages/neuron/package.py +++ b/var/spack/repos/builtin/packages/neuron/package.py @@ -3,6 +3,7 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) +from spack.build_environment import optimization_flags from spack.package import * @@ -149,7 +150,7 @@ class Neuron(CMakePackage): # add cpu arch specific optimisation flags to CMake so that they are passed # to embedded Makefile that neuron has for compiling MOD files - compilation_flags = self.spec.architecture.target.optimization_flags(self.spec.compiler) + compilation_flags = optimization_flags(self.compiler, self.spec.target) args.append(self.define("CMAKE_CXX_FLAGS", compilation_flags)) return args diff --git a/var/spack/repos/builtin/packages/py-tensorflow/package.py b/var/spack/repos/builtin/packages/py-tensorflow/package.py index 9935be8b29..dfe4f72e6b 100644 --- a/var/spack/repos/builtin/packages/py-tensorflow/package.py +++ b/var/spack/repos/builtin/packages/py-tensorflow/package.py @@ -8,6 +8,7 @@ import os import sys import tempfile +from spack.build_environment import optimization_flags from spack.package import * rocm_dependencies = [ @@ -656,7 +657,7 @@ class PyTensorflow(Package, CudaPackage, ROCmPackage, PythonExtension): # Please specify optimization flags to use during compilation when # bazel option '--config=opt' is specified - env.set("CC_OPT_FLAGS", spec.architecture.target.optimization_flags(spec.compiler)) + env.set("CC_OPT_FLAGS", optimization_flags(self.compiler, spec.target)) # Would you like to interactively configure ./WORKSPACE for # Android builds? diff --git a/var/spack/repos/builtin/packages/wgl/package.py b/var/spack/repos/builtin/packages/wgl/package.py index b420cdcf64..9d02f3c139 100644 --- a/var/spack/repos/builtin/packages/wgl/package.py +++ b/var/spack/repos/builtin/packages/wgl/package.py @@ -83,7 +83,7 @@ class Wgl(Package): return variants def _spec_arch_to_sdk_arch(self): - spec_arch = str(self.spec.architecture.target.microarchitecture.family).lower() + spec_arch = str(self.spec.architecture.target.family).lower() _64bit = "64" in spec_arch arm = "arm" in spec_arch if arm: |