diff options
50 files changed, 3040 insertions, 540 deletions
diff --git a/lib/spack/docs/basic_usage.rst b/lib/spack/docs/basic_usage.rst index d2aad69d3b..8d81db4cb4 100644 --- a/lib/spack/docs/basic_usage.rst +++ b/lib/spack/docs/basic_usage.rst @@ -848,18 +848,111 @@ that executables will run without the need to set ``LD_LIBRARY_PATH``. Architecture specifiers ^^^^^^^^^^^^^^^^^^^^^^^ -The architecture can be specified by using the reserved -words ``target`` and/or ``os`` (``target=x86-64 os=debian7``). You can also -use the triplet form of platform, operating system and processor. +Each node in the dependency graph of a spec has an architecture attribute. +This attribute is a triplet of platform, operating system and processor. +You can specify the elements either separately, by using +the reserved keywords ``platform``, ``os`` and ``target``: + +.. code-block:: console + + $ spack install libelf platform=linux + $ spack install libelf os=ubuntu18.04 + $ spack install libelf target=broadwell + +or together by using the reserved keyword ``arch``: .. code-block:: console $ spack install libelf arch=cray-CNL10-haswell -Users on non-Cray systems won't have to worry about specifying the architecture. -Spack will autodetect what kind of operating system is on your machine as well -as the processor. For more information on how the architecture can be -used on Cray machines, see :ref:`cray-support` +Normally users don't have to bother specifying the architecture +if they are installing software for their current host as in that case the +values will be detected automatically. + +.. admonition:: Cray machines + + The situation is a little bit different for Cray machines and a detailed + explanation on how the architecture can be set on them can be found at :ref:`cray-support` + +.. _support-for-microarchitectures: + +""""""""""""""""""""""""""""""""""""""" +Support for specific microarchitectures +""""""""""""""""""""""""""""""""""""""" + +Spack knows how to detect and optimize for many specific microarchitectures +(including recent Intel, AMD and IBM chips) and encodes this information in +the ``target`` portion of the architecture specification. A complete list of +the microarchitectures known to Spack can be obtained in the following way: + +.. command-output:: spack arch --known-targets + +When a spec is installed Spack matches the compiler being used with the +microarchitecture being targeted to inject appropriate optimization flags +at compile time. Giving a command such as the following: + +.. code-block:: console + + $ spack install zlib%gcc@9.0.1 target=icelake + +will produce compilation lines similar to: + +.. code-block:: console + + $ /usr/bin/gcc-9 -march=icelake-client -mtune=icelake-client -c ztest10532.c + $ /usr/bin/gcc-9 -march=icelake-client -mtune=icelake-client -c -fPIC -O2 ztest10532. + ... + +where the flags ``-march=icelake-client -mtune=icelake-client`` are injected +by Spack based on the requested target and compiler. + +If Spack knows that the requested compiler can't optimize for the current target +or can't build binaries for that target at all, it will exit with a meaningful error message: + +.. code-block:: console + + $ spack install zlib%gcc@5.5.0 target=icelake + ==> Error: cannot produce optimized binary for micro-architecture "icelake" with gcc@5.5.0 [supported compiler versions are 8:] + +When instead an old compiler is selected on a recent enough microarchitecture but there is +no explicit ``target`` specification, Spack will optimize for the best match it can find instead +of failing: + +.. code-block:: console + + $ spack arch + linux-ubuntu18.04-broadwell + + $ spack spec zlib%gcc@4.8 + Input spec + -------------------------------- + zlib%gcc@4.8 + + Concretized + -------------------------------- + zlib@1.2.11%gcc@4.8+optimize+pic+shared arch=linux-ubuntu18.04-haswell + + $ spack spec zlib%gcc@9.0.1 + Input spec + -------------------------------- + zlib%gcc@9.0.1 + + Concretized + -------------------------------- + zlib@1.2.11%gcc@9.0.1+optimize+pic+shared arch=linux-ubuntu18.04-broadwell + +In the snippet above, for instance, the microarchitecture was demoted to ``haswell`` when +compiling with ``gcc@4.8`` since support to optimize for ``broadwell`` starts from ``gcc@4.9:``. + +Finally if Spack has no information to match compiler and target, it will +proceed with the installation but avoid injecting any microarchitecture +specific flags. + +.. warning:: + + Currently Spack doesn't print any warning to the user if it has no information + on which optimization flags should be used for a given compiler. This behavior + might change in the future. .. _sec-virtual-dependencies: diff --git a/lib/spack/docs/packaging_guide.rst b/lib/spack/docs/packaging_guide.rst index 4b062c8a1a..1505b9a097 100644 --- a/lib/spack/docs/packaging_guide.rst +++ b/lib/spack/docs/packaging_guide.rst @@ -3213,6 +3213,127 @@ the two functions is that ``satisfies()`` tests whether spec constraints overlap at all, while ``in`` tests whether a spec or any of its dependencies satisfy the provided spec. +^^^^^^^^^^^^^^^^^^^^^^^ +Architecture specifiers +^^^^^^^^^^^^^^^^^^^^^^^ + +As mentioned in :ref:`support-for-microarchitectures` each node in a concretized spec +object has an architecture attribute which is a triplet of ``platform``, ``os`` and ``target``. +Each of these three items can be queried to take decisions when configuring, building or +installing a package. + +"""""""""""""""""""""""""""""""""""""""""""""" +Querying the platform and the operating system +"""""""""""""""""""""""""""""""""""""""""""""" + +Sometimes the actions to be taken to install a package might differ depending on the +platform we are installing for. If that is the case we can use conditionals: + +.. code-block:: python + + if spec.platform == 'darwin': + # Actions that are specific to Darwin + args.append('--darwin-specific-flag') + +and branch based on the current spec platform. If we need to make a package directive +conditional on the platform we can instead employ the usual spec syntax and pass the +corresponding constraint to the appropriate argument of that directive: + +.. code-block:: python + + class Libnl(AutotoolsPackage): + + conflicts('platform=darwin', msg='libnl requires FreeBSD or Linux') + +Similar considerations are also valid for the ``os`` part of a spec's architecture. +For instance: + +.. code-block:: python + + class Glib(AutotoolsPackage) + + patch('old-kernels.patch', when='os=centos6') + +will apply the patch only when the operating system is Centos 6. + +.. note:: + + Even though experienced Python programmers might recognize that there are other ways + to retrieve information on the platform: + + .. code-block:: python + + if sys.platform == 'darwin': + # Actions that are specific to Darwin + args.append('--darwin-specific-flag') + + querying the spec architecture's platform should be considered the preferred. The key difference + is that a query on ``sys.platform``, or anything similar, is always bound to the host on which the + interpreter running Spack is located and as such it won't work correctly in environments where + cross-compilation is required. + +""""""""""""""""""""""""""""""""""""" +Querying the target microarchitecture +""""""""""""""""""""""""""""""""""""" + +The third item of the architecture tuple is the ``target`` which abstracts the information on the +CPU microarchitecture. A list of all the targets known to Spack can be obtained via the +command line: + +.. command-output:: spack arch --known-targets + +Within directives each of the names above can be used to match a particular target: + +.. code-block:: python + + class Julia(Package): + # This patch is only applied on icelake microarchitectures + patch("icelake.patch", when="target=icelake") + +in a similar way to what we have seen before for ``platform`` and ``os``. +Where ``target`` objects really shine though is when they are used in methods +called at configure, build or install time. In that case we can test targets +for supported features, for instance: + +.. code-block:: python + + if 'avx512' in spec.target: + args.append('--with-avx512') + +The snippet above will append the ``--with-avx512`` item to a list of arguments only if the corresponding +feature is supported by the current target. Sometimes we need to take different actions based +on the architecture family and not on the specific microarchitecture. In those cases +we can check the ``family`` attribute: + +.. code-block:: python + + if spec.target.family == 'ppc64le': + args.append('--enable-power') + +Possible values for the ``family`` attribute are displayed by ``spack arch --known-targets`` +under the "Generic architectures (families)" header. +Finally it's possible to perform actions based on whether the current microarchitecture +is compatible with a known one: + +.. code-block:: python + + if spec.target > 'haswell': + args.append('--needs-at-least-haswell') + +The snippet above will add an item to a list of configure options only if the current +architecture is a superset of ``haswell`` or, said otherwise, only if the current +architecture is a later microarchitecture still compatible with ``haswell``. + +.. admonition:: Using Spack on unknown microarchitectures + + If Spack is used on an unknown microarchitecture it will try to perform a best match + of the features it detects and will select the closest microarchitecture it has + information for. In case nothing matches, it will create on the fly a new generic + architecture. This is done to allow users to still be able to use Spack + for their work. The software built won't be probably as optimized as it could but just + as you need a newer compiler to build for newer architectures, you may need newer + versions of Spack for new architectures to be correctly labeled. + ^^^^^^^^^^^^^^^^^^^^^^ Accessing Dependencies ^^^^^^^^^^^^^^^^^^^^^^ diff --git a/lib/spack/env/cc b/lib/spack/env/cc index c224652a34..8246539a00 100755 --- a/lib/spack/env/cc +++ b/lib/spack/env/cc @@ -374,7 +374,7 @@ case "$mode" in CXX) flags=("${flags[@]}" "${SPACK_CXXFLAGS[@]}") ;; esac - args=(${SPACK_TARGET_ARGS[@]} "${args[@]}") + flags=(${SPACK_TARGET_ARGS[@]} "${flags[@]}") ;; esac diff --git a/lib/spack/llnl/util/cpu/__init__.py b/lib/spack/llnl/util/cpu/__init__.py new file mode 100644 index 0000000000..eda6fa9c36 --- /dev/null +++ b/lib/spack/llnl/util/cpu/__init__.py @@ -0,0 +1,16 @@ +# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +from .microarchitecture import Microarchitecture, UnsupportedMicroarchitecture +from .microarchitecture import targets, generic_microarchitecture +from .detect import host + +__all__ = [ + 'Microarchitecture', + 'UnsupportedMicroarchitecture', + 'targets', + 'generic_microarchitecture', + 'host' +] diff --git a/lib/spack/llnl/util/cpu/alias.py b/lib/spack/llnl/util/cpu/alias.py new file mode 100644 index 0000000000..f52ecf7354 --- /dev/null +++ b/lib/spack/llnl/util/cpu/alias.py @@ -0,0 +1,102 @@ +# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +#: Known predicates that can be used to construct feature aliases + +from .schema import targets_json, LazyDictionary, properties + +_feature_alias_predicate = {} + + +class FeatureAliasTest(object): + """A test that must be passed for a feature alias to succeed. + + Args: + rules (dict): dictionary of rules to be met. Each key must be a + valid alias predicate + """ + def __init__(self, rules): + self.rules = rules + self.predicates = [] + for name, args in rules.items(): + self.predicates.append(_feature_alias_predicate[name](args)) + + def __call__(self, microarchitecture): + return all( + feature_test(microarchitecture) for feature_test in self.predicates + ) + + +def _feature_aliases(): + """Returns the dictionary of all defined feature aliases.""" + json_data = targets_json['feature_aliases'] + aliases = {} + for alias, rules in json_data.items(): + aliases[alias] = FeatureAliasTest(rules) + return aliases + + +feature_aliases = LazyDictionary(_feature_aliases) + + +def alias_predicate(predicate_schema): + """Decorator to register a predicate that can be used to define + feature aliases. + + Args: + predicate_schema (dict): schema to be enforced in + microarchitectures.json for the predicate + """ + def decorator(func): + name = func.__name__ + + # Check we didn't register anything else with the same name + if name in _feature_alias_predicate: + msg = 'the alias predicate "{0}" already exists'.format(name) + raise KeyError(msg) + + # Update the overall schema + alias_schema = properties['feature_aliases']['patternProperties'] + alias_schema[r'([\w]*)']['properties'].update( + {name: predicate_schema} + ) + # Register the predicate + _feature_alias_predicate[name] = func + + return func + return decorator + + +@alias_predicate(predicate_schema={'type': 'string'}) +def reason(motivation_for_the_alias): + """This predicate returns always True and it's there to allow writing + a documentation string in the JSON file to explain why an alias is needed. + """ + return lambda x: True + + +@alias_predicate(predicate_schema={ + 'type': 'array', + 'items': {'type': 'string'} +}) +def any_of(list_of_features): + """Returns a predicate that is True if any of the feature in the + list is in the microarchitecture being tested, False otherwise. + """ + def _impl(microarchitecture): + return any(x in microarchitecture for x in list_of_features) + return _impl + + +@alias_predicate(predicate_schema={ + 'type': 'array', + 'items': {'type': 'string'} +}) +def families(list_of_families): + """Returns a predicate that is True if the architecture family of + the microarchitecture being tested is in the list, False otherwise. + """ + def _impl(microarchitecture): + return str(microarchitecture.family) in list_of_families + return _impl diff --git a/lib/spack/llnl/util/cpu/detect.py b/lib/spack/llnl/util/cpu/detect.py new file mode 100644 index 0000000000..40a427a5da --- /dev/null +++ b/lib/spack/llnl/util/cpu/detect.py @@ -0,0 +1,216 @@ +# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +import collections +import functools +import platform +import re +import subprocess +import sys + +import six + +from .microarchitecture import generic_microarchitecture, targets + +#: Mapping from operating systems to chain of commands +#: to obtain a dictionary of raw info on the current cpu +info_factory = collections.defaultdict(list) + +#: Mapping from micro-architecture families (x86_64, ppc64le, etc.) to +#: functions checking the compatibility of the host with a given target +compatibility_checks = {} + + +def info_dict(operating_system): + """Decorator to mark functions that are meant to return raw info on + the current cpu. + + Args: + operating_system (str or tuple): operating system for which the marked + function is a viable factory of raw info dictionaries. + """ + def decorator(factory): + info_factory[operating_system].append(factory) + + @functools.wraps(factory) + def _impl(): + info = factory() + + # Check that info contains a few mandatory fields + msg = 'field "{0}" is missing from raw info dictionary' + assert 'vendor_id' in info, msg.format('vendor_id') + assert 'flags' in info, msg.format('flags') + assert 'model' in info, msg.format('model') + assert 'model_name' in info, msg.format('model_name') + + return info + + return _impl + + return decorator + + +@info_dict(operating_system='Linux') +def proc_cpuinfo(): + """Returns a raw info dictionary by parsing the first entry of + ``/proc/cpuinfo`` + """ + info = {} + with open('/proc/cpuinfo') as file: + for line in file: + key, separator, value = line.partition(':') + + # If there's no separator and info was already populated + # according to what's written here: + # + # http://www.linfo.org/proc_cpuinfo.html + # + # we are on a blank line separating two cpus. Exit early as + # we want to read just the first entry in /proc/cpuinfo + if separator != ':' and info: + break + + info[key.strip()] = value.strip() + return info + + +def check_output(args): + if sys.version_info[:2] == (2, 6): + return subprocess.run( + args, check=True, stdout=subprocess.PIPE).stdout # nopyqver + else: + return subprocess.check_output(args) # nopyqver + + +@info_dict(operating_system='Darwin') +def sysctl(): + """Returns a raw info dictionary parsing the output of sysctl.""" + + info = {} + info['vendor_id'] = check_output( + ['sysctl', '-n', 'machdep.cpu.vendor'] + ).strip() + info['flags'] = check_output( + ['sysctl', '-n', 'machdep.cpu.features'] + ).strip().lower() + info['flags'] += ' ' + check_output( + ['sysctl', '-n', 'machdep.cpu.leaf7_features'] + ).strip().lower() + info['model'] = check_output( + ['sysctl', '-n', 'machdep.cpu.model'] + ).strip() + info['model name'] = check_output( + ['sysctl', '-n', 'machdep.cpu.brand_string'] + ).strip() + + # Super hacky way to deal with slight representation differences + # Would be better to somehow consider these "identical" + if 'sse4.1' in info['flags']: + info['flags'] += ' sse4_1' + if 'sse4.2' in info['flags']: + info['flags'] += ' sse4_2' + if 'avx1.0' in info['flags']: + info['flags'] += ' avx' + + return info + + +def raw_info_dictionary(): + """Returns a dictionary with information on the cpu of the current host. + + This function calls all the viable factories one after the other until + there's one that is able to produce the requested information. + """ + info = {} + for factory in info_factory[platform.system()]: + try: + info = factory() + except Exception: + pass + + if info: + break + + return info + + +def compatible_microarchitectures(info): + """Returns an unordered list of known micro-architectures that are + compatible with the info dictionary passed as argument. + + Args: + info (dict): dictionary containing information on the host cpu + """ + architecture_family = platform.machine() + # If a tester is not registered, be conservative and assume no known + # target is compatible with the host + tester = compatibility_checks.get(architecture_family, lambda x, y: False) + return [x for x in targets.values() if tester(info, x)] or \ + [generic_microarchitecture(architecture_family)] + + +def host(): + """Detects the host micro-architecture and returns it.""" + # Retrieve a dictionary with raw information on the host's cpu + info = raw_info_dictionary() + + # Get a list of possible candidates for this micro-architecture + candidates = compatible_microarchitectures(info) + + # Reverse sort of the depth for the inheritance tree among only targets we + # can use. This gets the newest target we satisfy. + return sorted(candidates, key=lambda t: len(t.ancestors), reverse=True)[0] + + +def compatibility_check(architecture_family): + """Decorator to register a function as a proper compatibility check. + + A compatibility check function takes the raw info dictionary as a first + argument and an arbitrary target as the second argument. It returns True + if the target is compatible with the info dictionary, False otherwise. + + Args: + architecture_family (str or tuple): architecture family for which + this test can be used, e.g. x86_64 or ppc64le etc. + """ + # Turn the argument into something iterable + if isinstance(architecture_family, six.string_types): + architecture_family = (architecture_family,) + + def decorator(func): + # TODO: on removal of Python 2.6 support this can be re-written as + # TODO: an update + a dict comprehension + for arch_family in architecture_family: + compatibility_checks[arch_family] = func + + return func + + return decorator + + +@compatibility_check(architecture_family=('ppc64le', 'ppc64')) +def compatibility_check_for_power(info, target): + basename = platform.machine() + generation_match = re.search(r'POWER(\d+)', info.get('cpu', '')) + generation = int(generation_match.group(1)) + + # We can use a target if it descends from our machine type and our + # generation (9 for POWER9, etc) is at least its generation. + arch_root = targets[basename] + return (target == arch_root or arch_root in target.ancestors) \ + and target.generation <= generation + + +@compatibility_check(architecture_family='x86_64') +def compatibility_check_for_x86_64(info, target): + basename = 'x86_64' + vendor = info.get('vendor_id', 'generic') + features = set(info.get('flags', '').split()) + + # We can use a target if it descends from our machine type, is from our + # vendor, and we have all of its features + arch_root = targets[basename] + return (target == arch_root or arch_root in target.ancestors) \ + and (target.vendor == vendor or target.vendor == 'generic') \ + and target.features.issubset(features) diff --git a/lib/spack/llnl/util/cpu/microarchitecture.py b/lib/spack/llnl/util/cpu/microarchitecture.py new file mode 100644 index 0000000000..319eec28c7 --- /dev/null +++ b/lib/spack/llnl/util/cpu/microarchitecture.py @@ -0,0 +1,343 @@ +# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +import functools +import platform +import warnings + +try: + from collections.abc import Sequence +except ImportError: + from collections import Sequence + +import six + +import llnl.util +import llnl.util.cpu.alias +import llnl.util.cpu.schema + +from .schema import LazyDictionary +from .alias import feature_aliases + + +def coerce_target_names(func): + """Decorator that automatically converts a known target name to a proper + Microarchitecture object. + """ + @functools.wraps(func) + def _impl(self, other): + if isinstance(other, six.string_types): + if other not in targets: + msg = '"{0}" is not a valid target name' + raise ValueError(msg.format(other)) + other = targets[other] + + return func(self, other) + return _impl + + +class Microarchitecture(object): + #: Aliases for micro-architecture's features + feature_aliases = feature_aliases + + def __init__( + self, name, parents, vendor, features, compilers, generation=0 + ): + """Represents a specific CPU micro-architecture. + + Args: + name (str): name of the micro-architecture (e.g. skylake). + parents (list): list of parents micro-architectures, if any. + Parenthood is considered by cpu features and not + chronologically. As such each micro-architecture is + compatible with its ancestors. For example "skylake", + which has "broadwell" as a parent, supports running binaries + optimized for "broadwell". + vendor (str): vendor of the micro-architecture + features (list of str): supported CPU flags. Note that the semantic + of the flags in this field might vary among architectures, if + at all present. For instance x86_64 processors will list all + the flags supported by a given CPU while Arm processors will + list instead only the flags that have been added on top of the + base model for the current micro-architecture. + compilers (dict): compiler support to generate tuned code for this + micro-architecture. This dictionary has as keys names of + supported compilers, while values are list of dictionaries + with fields: + + * name: name of the micro-architecture according to the + compiler. This is the name passed to the ``-march`` option + or similar. Not needed if the name is the same as that + passed in as argument above. + * versions: versions that support this micro-architecture. + + generation (int): generation of the micro-architecture, if + relevant. + """ + self.name = name + self.parents = parents + self.vendor = vendor + self.features = features + self.compilers = compilers + self.generation = generation + + @property + def ancestors(self): + value = self.parents[:] + for parent in self.parents: + value.extend(a for a in parent.ancestors if a not in value) + return value + + def _to_set(self): + """Returns a set of the nodes in this microarchitecture DAG.""" + # This function is used to implement subset semantics with + # comparison operators + return set([str(self)] + [str(x) for x in self.ancestors]) + + @coerce_target_names + def __eq__(self, other): + if not isinstance(other, Microarchitecture): + return NotImplemented + + return (self.name == other.name and + self.vendor == other.vendor and + self.features == other.features and + self.ancestors == other.ancestors and + self.compilers == other.compilers and + self.generation == other.generation) + + @coerce_target_names + def __ne__(self, other): + return not self == other + + @coerce_target_names + def __lt__(self, other): + if not isinstance(other, Microarchitecture): + return NotImplemented + + return self._to_set() < other._to_set() + + @coerce_target_names + def __le__(self, other): + return (self == other) or (self < other) + + @coerce_target_names + def __gt__(self, other): + if not isinstance(other, Microarchitecture): + return NotImplemented + + return self._to_set() > other._to_set() + + @coerce_target_names + def __ge__(self, other): + return (self == other) or (self > other) + + def __repr__(self): + cls_name = self.__class__.__name__ + fmt = cls_name + '({0.name!r}, {0.parents!r}, {0.vendor!r}, ' \ + '{0.features!r}, {0.compilers!r}, {0.generation!r})' + return fmt.format(self) + + def __str__(self): + return self.name + + def __contains__(self, feature): + # Feature must be of a string type, so be defensive about that + if not isinstance(feature, six.string_types): + msg = 'only objects of string types are accepted [got {0}]' + raise TypeError(msg.format(str(type(feature)))) + + # Here we look first in the raw features, and fall-back to + # feature aliases if not match was found + if feature in self.features: + return True + + # Check if the alias is defined, if not it will return False + match_alias = Microarchitecture.feature_aliases.get( + feature, lambda x: False + ) + return match_alias(self) + + @property + def family(self): + """Returns the architecture family a given target belongs to""" + roots = [x for x in [self] + self.ancestors if not x.ancestors] + msg = "a target is expected to belong to just one architecture family" + msg += "[found {0}]".format(', '.join(str(x) for x in roots)) + assert len(roots) == 1, msg + + return roots.pop() + + def to_dict(self, return_list_of_items=False): + """Returns a dictionary representation of this object. + + Args: + return_list_of_items (bool): if True returns an ordered list of + items instead of the dictionary + """ + list_of_items = [ + ('name', str(self.name)), + ('vendor', str(self.vendor)), + ('features', sorted( + str(x) for x in self.features + )), + ('generation', self.generation), + ('parents', [str(x) for x in self.parents]) + ] + if return_list_of_items: + return list_of_items + + return dict(list_of_items) + + def optimization_flags(self, compiler, version): + """Returns a string containing the optimization flags that needs + to be used to produce code optimized for this micro-architecture. + + If there is no information on the compiler passed as argument the + function returns an empty string. If it is known that the compiler + version we want to use does not support this architecture the function + raises an exception. + + Args: + compiler (str): name of the compiler to be used + version (str): version of the compiler to be used + """ + # If we don't have information on compiler return an empty string + if compiler not in self.compilers: + return '' + + # If we have information on this compiler we need to check the + # version being used + compiler_info = self.compilers[compiler] + + # Normalize the entries to have a uniform treatment in the code below + if not isinstance(compiler_info, Sequence): + compiler_info = [compiler_info] + + def satisfies_constraint(entry, version): + min_version, max_version = entry['versions'].split(':') + + # Check version suffixes + min_version, _, min_suffix = min_version.partition('-') + max_version, _, max_suffix = max_version.partition('-') + version, _, suffix = version.partition('-') + + # If the suffixes are not all equal there's no match + if suffix != min_suffix or suffix != max_suffix: + return False + + # Assume compiler versions fit into semver + tuplify = lambda x: tuple(int(y) for y in x.split('.')) + + version = tuplify(version) + if min_version: + min_version = tuplify(min_version) + if min_version > version: + return False + + if max_version: + max_version = tuplify(max_version) + if max_version < version: + return False + + return True + + for compiler_entry in compiler_info: + if satisfies_constraint(compiler_entry, version): + flags_fmt = compiler_entry['flags'] + # If there's no field name, use the name of the + # micro-architecture + compiler_entry.setdefault('name', self.name) + + # Check if we need to emit a warning + warning_message = compiler_entry.get('warnings', None) + if warning_message: + warnings.warn(warning_message) + + flags = flags_fmt.format(**compiler_entry) + return flags + + msg = ("cannot produce optimized binary for micro-architecture '{0}'" + " with {1}@{2} [supported compiler versions are {3}]") + msg = msg.format(self.name, compiler, version, + ', '.join([x['versions'] for x in compiler_info])) + raise UnsupportedMicroarchitecture(msg) + + +def generic_microarchitecture(name): + """Returns a generic micro-architecture with no vendor and no features. + + Args: + name (str): name of the micro-architecture + """ + return Microarchitecture( + name, parents=[], vendor='generic', features=[], compilers={} + ) + + +def _known_microarchitectures(): + """Returns a dictionary of the known micro-architectures. If the + current host platform is unknown adds it too as a generic target. + """ + + # TODO: Simplify this logic using object_pairs_hook to OrderedDict + # TODO: when we stop supporting python2.6 + + def fill_target_from_dict(name, data, targets): + """Recursively fills targets by adding the micro-architecture + passed as argument and all its ancestors. + + Args: + name (str): micro-architecture to be added to targets. + data (dict): raw data loaded from JSON. + targets (dict): dictionary that maps micro-architecture names + to ``Microarchitecture`` objects + """ + values = data[name] + + # Get direct parents of target + parent_names = values['from'] + if isinstance(parent_names, six.string_types): + parent_names = [parent_names] + if parent_names is None: + parent_names = [] + for p in parent_names: + # Recursively fill parents so they exist before we add them + if p in targets: + continue + fill_target_from_dict(p, data, targets) + parents = [targets.get(p) for p in parent_names] + + vendor = values['vendor'] + features = set(values['features']) + compilers = values.get('compilers', {}) + generation = values.get('generation', 0) + + targets[name] = Microarchitecture( + name, parents, vendor, features, compilers, generation + ) + + targets = {} + data = llnl.util.cpu.schema.targets_json['microarchitectures'] + for name in data: + if name in targets: + # name was already brought in as ancestor to a target + continue + fill_target_from_dict(name, data, targets) + + # Add the host platform if not present + host_platform = platform.machine() + targets.setdefault(host_platform, generic_microarchitecture(host_platform)) + + return targets + + +#: Dictionary of known micro-architectures +targets = LazyDictionary(_known_microarchitectures) + + +class UnsupportedMicroarchitecture(ValueError): + """Raised if a compiler version does not support optimization for a given + micro-architecture. + """ diff --git a/lib/spack/llnl/util/cpu/microarchitectures.json b/lib/spack/llnl/util/cpu/microarchitectures.json new file mode 100644 index 0000000000..2745f51411 --- /dev/null +++ b/lib/spack/llnl/util/cpu/microarchitectures.json @@ -0,0 +1,832 @@ +{ + "microarchitectures": { + "x86": { + "from": null, + "vendor": "generic", + "features": [] + }, + "i686": { + "from": "x86", + "vendor": "GenuineIntel", + "features": [] + }, + "pentium2": { + "from": "i686", + "vendor": "GenuineIntel", + "features": [ + "mmx" + ] + }, + "pentium3": { + "from": "pentium2", + "vendor": "GenuineIntel", + "features": [ + "mmx", + "sse" + ] + }, + "pentium4": { + "from": "pentium3", + "vendor": "GenuineIntel", + "features": [ + "mmx", + "sse", + "sse2" + ] + }, + "prescott": { + "from": "pentium4", + "vendor": "GenuineIntel", + "features": [ + "mmx", + "sse", + "sse2", + "sse3" + ] + }, + "x86_64": { + "from": null, + "vendor": "generic", + "features": [], + "compilers": { + "gcc": { + "versions": "4:", + "name": "x86-64", + "flags": "-march={name} -mtune={name}" + } + } + }, + "nocona": { + "from": "x86_64", + "vendor": "GenuineIntel", + "features": [ + "mmx", + "sse", + "sse2", + "sse3" + ], + "compilers": { + "gcc": { + "versions": "4:", + "flags": "-march={name} -mtune={name}" + } + } + }, + "core2": { + "from": "nocona", + "vendor": "GenuineIntel", + "features": [ + "mmx", + "sse", + "sse2", + "ssse3" + ], + "compilers": { + "gcc": { + "versions": "4:", + "flags": "-march={name} -mtune={name}" + } + } + }, + "nehalem": { + "from": "core2", + "vendor": "GenuineIntel", + "features": [ + "mmx", + "sse", + "sse2", + "ssse3", + "sse4_1", + "sse4_2", + "popcnt" + ], + "compilers": { + "gcc": [ + { + "versions": "4.9:", + "flags": "-march={name} -mtune={name}" + }, + { + "versions": "4.6:4.8.5", + "name": "corei7", + "flags": "-march={name} -mtune={name}" + } + ] + } + }, + "westmere": { + "from": "nehalem", + "vendor": "GenuineIntel", + "features": [ + "mmx", + "sse", + "sse2", + "ssse3", + "sse4_1", + "sse4_2", + "popcnt", + "aes", + "pclmulqdq" + ], + "compilers": { + "gcc": { + "versions": "4.9:", + "flags": "-march={name} -mtune={name}" + } + } + }, + "sandybridge": { + "from": "westmere", + "vendor": "GenuineIntel", + "features": [ + "mmx", + "sse", + "sse2", + "ssse3", + "sse4_1", + "sse4_2", + "popcnt", + "aes", + "pclmulqdq", + "avx" + ], + "compilers": { + "gcc": [ + { + "versions": "4.9:", + "flags": "-march={name} -mtune={name}" + }, + { + "versions": "4.6:4.8.5", + "name": "corei7-avx", + "flags": "-march={name} -mtune={name}" + } + ] + } + }, + "ivybridge": { + "from": "sandybridge", + "vendor": "GenuineIntel", + "features": [ + "mmx", + "sse", + "sse2", + "ssse3", + "sse4_1", + "sse4_2", + "popcnt", + "aes", + "pclmulqdq", + "avx", + "rdrand", + "f16c" + ], + "compilers": { + "gcc": [ + { + "versions": "4.9:", + "flags": "-march={name} -mtune={name}" + }, + { + "versions": ":4.8.5", + "name": "core-avx-i", + "flags": "-march={name} -mtune={name}" + } + ] + } + }, + "haswell": { + "from": "ivybridge", + "vendor": "GenuineIntel", + "features": [ + "mmx", + "sse", + "sse2", + "ssse3", + "sse4_1", + "sse4_2", + "popcnt", + "aes", + "pclmulqdq", + "avx", + "rdrand", + "f16c", + "movbe", + "fma", + "avx2", + "bmi1", + "bmi2" + ], + "compilers": { + "gcc": [ + { + "versions": "4.9:", + "flags": "-march={name} -mtune={name}" + }, + { + "versions": ":4.8.5", + "name": "core-avx2", + "flags": "-march={name} -mtune={name}" + } + ] + } + }, + "broadwell": { + "from": "haswell", + "vendor": "GenuineIntel", + "features": [ + "mmx", + "sse", + "sse2", + "ssse3", + "sse4_1", + "sse4_2", + "popcnt", + "aes", + "pclmulqdq", + "avx", + "rdrand", + "f16c", + "movbe", + "fma", + "avx2", + "bmi1", + "bmi2", + "rdseed", + "adx" + ], + "compilers": { + "gcc": { + "versions": "4.9:", + "flags": "-march={name} -mtune={name}" + } + } + }, + "skylake": { + "from": "broadwell", + "vendor": "GenuineIntel", + "features": [ + "mmx", + "sse", + "sse2", + "ssse3", + "sse4_1", + "sse4_2", + "popcnt", + "aes", + "pclmulqdq", + "avx", + "rdrand", + "f16c", + "movbe", + "fma", + "avx2", + "bmi1", + "bmi2", + "rdseed", + "adx", + "clflushopt", + "xsavec", + "xsaveopt" + ], + "compilers": { + "gcc": { + "versions": "5.3:", + "flags": "-march={name} -mtune={name}" + } + } + }, + "skylake_avx512": { + "from": "skylake", + "vendor": "GenuineIntel", + "features": [ + "mmx", + "sse", + "sse2", + "ssse3", + "sse4_1", + "sse4_2", + "popcnt", + "aes", + "pclmulqdq", + "avx", + "rdrand", + "f16c", + "movbe", + "fma", + "avx2", + "bmi1", + "bmi2", + "rdseed", + "adx", + "clflushopt", + "xsavec", + "xsaveopt", + "avx512f", + "clwb", + "avx512vl", + "avx512bw", + "avx512dq", + "avx512cd" + ], + "compilers": { + "gcc": { + "name": "skylake-avx512", + "versions": "5.3:", + "flags": "-march={name} -mtune={name}" + } + } + }, + "cannonlake": { + "from": "skylake", + "vendor": "GenuineIntel", + "features": [ + "mmx", + "sse", + "sse2", + "ssse3", + "sse4_1", + "sse4_2", + "popcnt", + "aes", + "pclmulqdq", + "avx", + "rdrand", + "f16c", + "movbe", + "fma", + "avx2", + "bmi1", + "bmi2", + "rdseed", + "adx", + "clflushopt", + "xsavec", + "xsaveopt", + "avx512f", + "avx512vl", + "avx512bw", + "avx512dq", + "avx512cd", + "avx512vbmi", + "avx512ifma", + "sha", + "umip" + ], + "compilers": { + "gcc": { + "versions": "8:", + "flags": "-march={name} -mtune={name}" + } + } + }, + "cascadelake": { + "from": "skylake_avx512", + "vendor": "GenuineIntel", + "features": [ + "mmx", + "sse", + "sse2", + "ssse3", + "sse4_1", + "sse4_2", + "popcnt", + "aes", + "pclmulqdq", + "avx", + "rdrand", + "f16c", + "movbe", + "fma", + "avx2", + "bmi1", + "bmi2", + "rdseed", + "adx", + "clflushopt", + "xsavec", + "xsaveopt", + "avx512f", + "clwb", + "avx512vl", + "avx512bw", + "avx512dq", + "avx512cd", + "avx512vnni" + ], + "compilers": { + "gcc": { + "versions": "9:", + "flags": "-march={name} -mtune={name}" + } + } + }, + "icelake": { + "from": [ + "cascadelake", + "cannonlake" + ], + "vendor": "GenuineIntel", + "features": [ + "mmx", + "sse", + "sse2", + "ssse3", + "sse4_1", + "sse4_2", + "popcnt", + "aes", + "pclmulqdq", + "avx", + "rdrand", + "f16c", + "movbe", + "fma", + "avx2", + "bmi1", + "bmi2", + "rdseed", + "adx", + "clflushopt", + "xsavec", + "xsaveopt", + "avx512f", + "avx512vl", + "avx512bw", + "avx512dq", + "avx512cd", + "avx512vbmi", + "avx512ifma", + "sha", + "umip", + "clwb", + "rdpid", + "gfni", + "avx512vbmi2", + "avx512vpopcntdq", + "avx512bitalg", + "avx512vnni", + "vpclmulqdq", + "vaes" + ], + "compilers": { + "gcc": { + "name": "icelake-client", + "versions": "8:", + "flags": "-march={name} -mtune={name}" + } + } + }, + "barcelona": { + "from": "x86_64", + "vendor": "AuthenticAMD", + "features": [ + "mmx", + "sse", + "sse2", + "sse4a", + "abm" + ] + }, + "bulldozer": { + "from": "barcelona", + "vendor": "AuthenticAMD", + "features": [ + "mmx", + "sse", + "sse2", + "sse4a", + "abm", + "avx", + "xop", + "lwp", + "aes", + "pclmulqdq", + "cx16", + "ssse3", + "sse4_1", + "sse4_2" + ], + "compilers": { + "gcc": { + "name": "bdver1", + "versions": "4.6:", + "flags": "-march={name} -mtune={name}" + } + } + }, + "piledriver": { + "from": "bulldozer", + "vendor": "AuthenticAMD", + "features": [ + "mmx", + "sse", + "sse2", + "sse4a", + "abm", + "avx", + "aes", + "pclmulqdq", + "cx16", + "ssse3", + "sse4_1", + "sse4_2", + "bmi1", + "f16c", + "fma" + ], + "compilers": { + "gcc": { + "name": "bdver2", + "versions": "4.7:", + "flags": "-march={name} -mtune={name}" + } + } + }, + "steamroller": { + "from": "piledriver", + "vendor": "AuthenticAMD", + "features": [ + "mmx", + "sse", + "sse2", + "sse4a", + "abm", + "avx", + "aes", + "pclmulqdq", + "cx16", + "ssse3", + "sse4_1", + "sse4_2", + "bmi1", + "f16c", + "fma", + "fsgsbase" + ], + "compilers": { + "gcc": { + "name": "bdver3", + "versions": "4.8:", + "flags": "-march={name} -mtune={name}" + } + } + }, + "excavator": { + "from": "steamroller", + "vendor": "AuthenticAMD", + "features": [ + "mmx", + "sse", + "sse2", + "sse4a", + "abm", + "avx", + "aes", + "pclmulqdq", + "cx16", + "ssse3", + "sse4_1", + "sse4_2", + "bmi1", + "f16c", + "fma", + "fsgsbase", + "bmi2", + "avx2", + "movbe" + ], + "compilers": { + "gcc": { + "name": "bdver4", + "versions": "4.9:", + "flags": "-march={name} -mtune={name}" + } + } + }, + "zen": { + "from": "excavator", + "vendor": "AuthenticAMD", + "features": [ + "bmi1", + "bmi2", + "f16c", + "fma", + "fsgsbase", + "avx", + "avx2", + "rdseed", + "clzero", + "aes", + "pclmulqdq", + "cx16", + "movbe", + "mmx", + "sse", + "sse2", + "sse4a", + "ssse3", + "sse4_1", + "sse4_2", + "abm", + "xsavec", + "xsaveopt", + "clflushopt", + "popcnt" + ], + "compilers": { + "gcc": { + "name": "znver1", + "versions": "6:", + "flags": "-march={name} -mtune={name}" + } + } + }, + "zen2": { + "from": "zen", + "vendor": "AuthenticAMD", + "features": [ + "bmi1", + "bmi2", + "f16c", + "fma", + "fsgsbase", + "avx", + "avx2", + "rdseed", + "clzero", + "aes", + "pclmulqdq", + "cx16", + "movbe", + "mmx", + "sse", + "sse2", + "sse4a", + "ssse3", + "sse4_1", + "sse4_2", + "abm", + "xsavec", + "xsaveopt", + "clflushopt", + "popcnt", + "clwb" + ], + "compilers": { + "gcc": { + "name": "znver2", + "versions": "9:", + "flags": "-march={name} -mtune={name}" + } + } + }, + "ppc64": { + "from": null, + "vendor": "generic", + "features": [], + "compilers": { + "gcc": { + "versions": "4:", + "flags": "-mcpu={name} -mtune={name}" + } + } + }, + "power7": { + "from": "ppc64", + "vendor": "IBM", + "generation": 7, + "features": [], + "compilers": { + "gcc": { + "versions": "4.5:", + "flags": "-mcpu={name} -mtune={name}" + } + } + }, + "power8": { + "from": "power7", + "vendor": "IBM", + "generation": 8, + "features": [], + "compilers": { + "gcc": [ + { + "versions": "4.9:", + "flags": "-mcpu={name} -mtune={name}" + }, + { + "versions": "4.8:4.8.5", + "warnings": "Using GCC 4.8 to optimize for Power 8 might not work if you are not on Red Hat Enterprise Linux 7, where a custom backport of the feature has been done. Upstream support from GCC starts in version 4.9", + "flags": "-mcpu={name} -mtune={name}" + } + ] + } + }, + "power9": { + "from": "power8", + "vendor": "IBM", + "generation": 9, + "features": [], + "compilers": { + "gcc": { + "versions": "6:", + "flags": "-mcpu={name} -mtune={name}" + } + } + }, + "ppc64le": { + "from": null, + "vendor": "generic", + "features": [], + "compilers": { + "gcc": { + "versions": "4:", + "flags": "-mcpu={name} -mtune={name}" + } + } + }, + "power8le": { + "from": "ppc64le", + "vendor": "IBM", + "generation": 8, + "features": [], + "compilers": { + "gcc": [ + { + "versions": "4.9:", + "name": "power8", + "flags": "-mcpu={name} -mtune={name}" + }, + { + "versions": "4.8:4.8.5", + "warnings": "Using GCC 4.8 to optimize for Power 8 might not work if you are not on Red Hat Enterprise Linux 7, where a custom backport of the feature has been done. Upstream support from GCC starts in version 4.9", + "name": "power8", + "flags": "-mcpu={name} -mtune={name}" + } + ] + } + }, + "power9le": { + "from": "power8le", + "vendor": "IBM", + "generation": 9, + "features": [], + "compilers": { + "gcc": { + "name": "power9", + "versions": "6:", + "flags": "-mcpu={name} -mtune={name}" + } + } + }, + "aarch64": { + "from": null, + "vendor": "generic", + "features": [], + "compilers": { + "gcc": { + "versions": "4:", + "flags": "-march=armv8-a -mtune=generic" + } + } + } + }, + "feature_aliases": { + "sse3": { + "reason": "ssse3 is a superset of sse3 and might be the only one listed", + "any_of": [ + "ssse3" + ] + }, + "avx512": { + "reason": "avx512 indicates generic support for any of the avx512 instruction sets", + "any_of": [ + "avx512f", + "avx512vl", + "avx512bw", + "avx512dq", + "avx512cd" + ] + }, + "altivec": { + "reason": "altivec is supported by Power PC architectures, but might not be listed in features", + "families": [ + "ppc64le", + "ppc64" + ] + }, + "sse4.1": { + "reason": "permits to refer to sse4_1 also as sse4.1", + "any_of": [ + "sse4_1" + ] + }, + "sse4.2": { + "reason": "permits to refer to sse4_2 also as sse4.2", + "any_of": [ + "sse4_2" + ] + } + } +} diff --git a/lib/spack/llnl/util/cpu/schema.py b/lib/spack/llnl/util/cpu/schema.py new file mode 100644 index 0000000000..d13d014c01 --- /dev/null +++ b/lib/spack/llnl/util/cpu/schema.py @@ -0,0 +1,133 @@ +# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +import json +import os.path + +try: + from collections.abc import MutableMapping +except ImportError: + from collections import MutableMapping + +compilers_schema = { + 'type': 'object', + 'properties': { + 'versions': {'type': 'string'}, + 'name': {'type': 'string'}, + 'flags': {'type': 'string'} + }, + 'required': ['versions', 'flags'] +} + +properties = { + 'microarchitectures': { + 'type': 'object', + 'patternProperties': { + r'([\w]*)': { + 'type': 'object', + 'properties': { + 'from': { + 'anyOf': [ + # More than one parent + {'type': 'array', 'items': {'type': 'string'}}, + # Exactly one parent + {'type': 'string'}, + # No parent + {'type': 'null'} + ] + }, + 'vendor': { + 'type': 'string' + }, + 'features': { + 'type': 'array', + 'items': {'type': 'string'} + }, + 'compilers': { + 'type': 'object', + 'patternProperties': { + r'([\w]*)': { + 'anyOf': [ + compilers_schema, + { + 'type': 'array', + 'items': compilers_schema + } + ] + } + } + } + }, + 'required': ['from', 'vendor', 'features'] + } + } + }, + 'feature_aliases': { + 'type': 'object', + 'patternProperties': { + r'([\w]*)': { + 'type': 'object', + 'properties': {}, + 'additionalProperties': False + } + }, + + } +} + +schema = { + '$schema': 'http://json-schema.org/schema#', + 'title': 'Schema for microarchitecture definitions and feature aliases', + 'type': 'object', + 'additionalProperties': False, + 'properties': properties, +} + + +class LazyDictionary(MutableMapping): + """Lazy dictionary that gets constructed on first access to any object key + + Args: + factory (callable): factory function to construct the dictionary + """ + + def __init__(self, factory, *args, **kwargs): + self.factory = factory + self.args = args + self.kwargs = kwargs + self._data = None + + @property + def data(self): + if self._data is None: + self._data = self.factory(*self.args, **self.kwargs) + return self._data + + def __getitem__(self, key): + return self.data[key] + + def __setitem__(self, key, value): + self.data[key] = value + + def __delitem__(self, key): + del self.data[key] + + def __iter__(self): + return iter(self.data) + + def __len__(self): + return len(self.data) + + +def _load_targets_json(): + """Loads ``microarchitectures.json`` in memory.""" + directory_name = os.path.dirname(os.path.abspath(__file__)) + filename = os.path.join(directory_name, 'microarchitectures.json') + with open(filename, 'r') as f: + return json.load(f) + + +#: In memory representation of the data in microarchitectures.json, +#: loaded on first access +targets_json = LazyDictionary(_load_targets_json) diff --git a/lib/spack/llnl/util/cpu_name.py b/lib/spack/llnl/util/cpu_name.py deleted file mode 100644 index 9687252ab9..0000000000 --- a/lib/spack/llnl/util/cpu_name.py +++ /dev/null @@ -1,226 +0,0 @@ -# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) - -import platform -import re -import subprocess -import sys - - -# Tuple of name, flags added, flags removed (default []) -_intel_32 = [ - ('i686', []), - ('pentium2', ['mmx']), - ('pentium3', ['sse']), - ('pentium4', ['sse2']), - ('prescott', ['sse3']), - ] - -_intel_64 = [ # commenting out the ones that aren't shown through sysctl - ('nocona', ['mmx', 'sse', 'sse2', 'sse3']),#lm - ('core2', ['ssse3'], ['sse3']), - ('nehalem', ['sse4_1', 'sse4_2', 'popcnt']), - ('westmere', ['aes', 'pclmulqdq']), - ('sandybridge', ['avx']), - ('ivybridge', ['rdrand', 'f16c']),#fsgsbase (is it RDWRFSGS on darwin?) - ('haswell', ['movbe', 'fma', 'avx2', 'bmi1', 'bmi2']), - ('broadwell', ['rdseed', 'adx']), - ('skylake', ['xsavec', 'xsaves']) - ] - -# We will need to build on these and combine with names when intel releases -# further avx512 processors. -# _intel_avx12 = ['avx512f', 'avx512cd'] - - -_amd_10_names = [ - ('barcelona', ['mmx', 'sse', 'sse2', 'sse3', 'sse4a', 'abm']) - ] - -_amd_14_names = [ - ('btver1', ['mmx', 'sse', 'sse2', 'sse3', 'ssse3', 'sse4a', 'cx16', - 'abm']),#lm - ] - -_amd_15_names = [ - ('bdver1', ['avx', 'aes', 'pclmulqdq', 'cx16', 'sse', 'sse2', 'sse3', - 'ssse3', 'sse4a', 'sse4_1', 'sse4_2', 'abm']),#xop, lwp - ('bdver2', ['bmi1', 'f16c', 'fma',]),#tba? - ('bdver3', ['fsgsbase']), - ('bdver4', ['bmi2', 'movbe', 'avx2']) - ] - -_amd_16_names = [ - ('btver2', ['mmx', 'sse', 'sse2', 'sse3', 'ssse3', 'sse4a', 'cx16', - 'abm', 'movbe', 'f16c', 'bmi1', 'avx', 'pclmulqdq', - 'aes', 'sse4_1', 'sse4_2']),#lm - ] - -_amd_17_names = [ - ('znver1', ['bmi1', 'bmi2', 'f16c', 'fma', 'fsgsbase', 'avx', 'avx2', - 'rdseed', 'mwaitx', 'clzero', 'aes', 'pclmulqdq', 'cx16', - 'movbe', 'mmx', 'sse', 'sse2', 'sse3', 'ssse3', 'sse4a', - 'sse4_1', 'sse4_2', 'abm', 'xsavec', 'xsaves', - 'clflushopt', 'popcnt', 'adcx']) - ] - -_amd_numbers = { - 0x10: _amd_10_names, - 0x14: _amd_14_names, - 0x15: _amd_15_names, - 0x16: _amd_16_names, - 0x17: _amd_17_names - } - -def supported_target_names(): - intel_names = set(t[0] for t in _intel_64) - intel_names |= set(t[0] for t in _intel_32) - amd_names = set() - for family in _amd_numbers: - amd_names |= set(t[0] for t in _amd_numbers[family]) - power_names = set('power' + str(d) for d in range(7, 10)) - return intel_names | amd_names | power_names - -def create_dict_from_cpuinfo(): - # Initialize cpuinfo from file - cpuinfo = {} - try: - with open('/proc/cpuinfo') as file: - text = file.readlines() - for line in text: - if line.strip(): - key, _, value = line.partition(':') - cpuinfo[key.strip()] = value.strip() - except IOError: - return None - return cpuinfo - -def check_output(args): - if sys.version_info >= (3, 0): - return subprocess.run(args, check=True, stdout=PIPE).stdout # nopyqver - else: - return subprocess.check_output(args) # nopyqver - -def create_dict_from_sysctl(): - cpuinfo = {} - try: - cpuinfo['vendor_id'] = check_output(['sysctl', '-n', - 'machdep.cpu.vendor']).strip() - cpuinfo['flags'] = check_output(['sysctl', '-n', - 'machdep.cpu.features']).strip().lower() - cpuinfo['flags'] += ' ' + check_output(['sysctl', '-n', - 'machdep.cpu.leaf7_features']).strip().lower() - cpuinfo['model'] = check_output(['sysctl', '-n', - 'machdep.cpu.model']).strip() - cpuinfo['model name'] = check_output(['sysctl', '-n', - 'machdep.cpu.brand_string']).strip() - - # Super hacky way to deal with slight representation differences - # Would be better to somehow consider these "identical" - if 'sse4.1' in cpuinfo['flags']: - cpuinfo['flags'] += ' sse4_1' - if 'sse4.2' in cpuinfo['flags']: - cpuinfo['flags'] += ' sse4_2' - if 'avx1.0' in cpuinfo['flags']: - cpuinfo['flags'] += ' avx' - except: - pass - return cpuinfo - -def get_cpu_name(): - name = get_cpu_name_helper(platform.system()) - return name if name else platform.machine() - -def get_cpu_name_helper(system): - # TODO: Elsewhere create dict of codenames (targets) and flag sets. - # Return cpu name or an empty string if one cannot be determined. - cpuinfo = {} - if system == 'Linux': - cpuinfo = create_dict_from_cpuinfo() - elif system == 'Darwin': - cpuinfo = create_dict_from_sysctl() - if not cpuinfo: - return '' - - if 'vendor_id' in cpuinfo and cpuinfo['vendor_id'] == 'GenuineIntel': - if 'model name' not in cpuinfo or 'flags' not in cpuinfo: - # We don't have the information we need to determine the - # microarchitecture name - return '' - return get_intel_cpu_name(cpuinfo) - elif 'vendor_id' in cpuinfo and cpuinfo['vendor_id'] == 'AuthenticAMD': - if 'cpu family' not in cpuinfo or 'flags' not in cpuinfo: - # We don't have the information we need to determine the - # microarchitecture name - return '' - return get_amd_cpu_name(cpuinfo) - elif 'cpu' in cpuinfo and 'POWER' in cpuinfo['cpu']: - return get_ibm_cpu_name(cpuinfo['cpu']) - else: - return '' - -def get_ibm_cpu_name(cpu): - power_pattern = re.compile('POWER(\d+)') - power_match = power_pattern.search(cpu) - if power_match: - if 'le' in platform.machine(): - return 'power' + power_match.group(1) + 'le' - return 'power' + power_match.group(1) - else: - return '' - -def get_intel_cpu_name(cpuinfo): - model_name = cpuinfo['model name'] - if 'Atom' in model_name: - return 'atom' - elif 'Quark' in model_name: - return 'quark' - elif 'Xeon' in model_name and 'Phi' in model_name: - # This is hacky and needs to be extended for newer avx512 chips - return 'knl' - else: - ret = '' - flag_list = cpuinfo['flags'].split() - proc_flags = [] - for _intel_processors in [_intel_32, _intel_64]: - for entry in _intel_processors: - try: - proc, flags_added, flags_removed = entry - except ValueError: - proc, flags_added = entry - flags_removed = [] - proc_flags = list(filter(lambda x: x not in flags_removed, proc_flags)) - proc_flags.extend(flags_added) - if all(f in flag_list for f in proc_flags): - ret = proc - return ret - -def get_amd_cpu_name(cpuinfo): - #TODO: Learn what the "canonical" granularity of naming - # is for AMD processors, implement dict as for intel. - ret = '' - flag_list = cpuinfo['flags'].split() - model_number = int(cpuinfo['cpu family']) - flags_dict = _amd_numbers[model_number] - proc_flags = [] - for proc, proc_flags_added in flags_dict: - proc_flags.extend(proc_flags_added) - if all(f in flag_list for f in proc_flags): - ret = proc - else: - break - return ret - -"""IDEA: In build_environment.setup_compiler_environment, include a -call to compiler.tuning_flags(spec.architecture.target). For gcc this -would return "-march=%s" % str(spec.architecture.target). We only call -this if the target is a valid tuning target (I.e. not -platform.machine(), but a more specific target we successfully -discovered. - -Then set -SPACK_TUNING_FLAGS=compiler.tuning_flags(spec.architecture.target) -This way the compiler wrapper can just add $SPACK_TUNING_FLAGS to the -eventual command.""" diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 4234ea35a9..c67b63cba0 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -56,50 +56,137 @@ set. The user can set the front-end and back-end operating setting by the class attributes front_os and back_os. The operating system as described earlier, will be responsible for compiler detection. """ +import functools import inspect +import six + +import llnl.util.cpu as cpu import llnl.util.tty as tty from llnl.util.lang import memoized, list_modules, key_ordering -from llnl.util.cpu_name import get_cpu_name import spack.compiler import spack.paths import spack.error as serr +import spack.version from spack.util.naming import mod_to_class from spack.util.spack_yaml import syaml_dict class NoPlatformError(serr.SpackError): - def __init__(self): super(NoPlatformError, self).__init__( "Could not determine a platform for this machine.") -@key_ordering -class Target(object): - """ Target is the processor of the host machine. - The host machine may have different front-end and back-end targets, - especially if it is a Cray machine. The target will have a name and - also the module_name (e.g craype-compiler). Targets will also - recognize which platform they came from using the set_platform method. - Targets will have compiler finding strategies +def _ensure_other_is_target(method): + """Decorator to be used in dunder methods taking a single argument to + ensure that the argument is an instance of ``Target`` too. """ + @functools.wraps(method) + def _impl(self, other): + if isinstance(other, six.string_types): + other = Target(other) + + if not isinstance(other, Target): + return NotImplemented + + return method(self, other) + return _impl + + +class Target(object): def __init__(self, name, module_name=None): - self.name = name # case of cray "ivybridge" but if it's x86_64 - self.module_name = module_name # craype-ivybridge + """Target models microarchitectures and their compatibility. + + Args: + name (str or Microarchitecture):micro-architecture of the + target + module_name (str): optional module name to get access to the + current target. This is typically used on machines + like Cray (e.g. craype-compiler) + """ + if not isinstance(name, cpu.Microarchitecture): + name = cpu.targets.get( + name, cpu.generic_microarchitecture(name) + ) + self.microarchitecture = name + self.module_name = module_name - # Sets only the platform name to avoid recursiveness + @property + def name(self): + return self.microarchitecture.name - def _cmp_key(self): - return (self.name, self.module_name) + @_ensure_other_is_target + def __eq__(self, other): + return self.microarchitecture == other.microarchitecture and \ + self.module_name == other.module_name + + def __ne__(self, other): + # This method is necessary as long as we support Python 2. In Python 3 + # __ne__ defaults to the implementation below + return not self == other + + @_ensure_other_is_target + def __lt__(self, other): + # TODO: In the future it would be convenient to say + # TODO: `spec.architecture.target < other.architecture.target` + # TODO: and change the semantic of the comparison operators + + # This is needed to sort deterministically specs in a list. + # It doesn't implement a total ordering semantic. + return self.microarchitecture.name < other.microarchitecture.name + + def __hash__(self): + return hash((self.name, self.module_name)) + + @staticmethod + def from_dict_or_value(dict_or_value): + # A string here represents a generic target (like x86_64 or ppc64) or + # a custom micro-architecture + if isinstance(dict_or_value, six.string_types): + return Target(dict_or_value) + + # TODO: From a dict we actually retrieve much more information than + # TODO: just the name. We can use that information to reconstruct an + # TODO: "old" micro-architecture or check the current definition. + target_info = dict_or_value + return Target(target_info['name']) + + def to_dict_or_value(self): + """Returns a dict or a value representing the current target. + + String values are used to keep backward compatibility with generic + targets, like e.g. x86_64 or ppc64. More specific micro-architectures + will return a dictionary which contains information on the name, + features, vendor, generation and parents of the current target. + """ + # Generic targets represent either an architecture + # family (like x86_64) or a custom micro-architecture + if self.microarchitecture.vendor == 'generic': + return str(self) + + return syaml_dict( + self.microarchitecture.to_dict(return_list_of_items=True) + ) def __repr__(self): - return self.__str__() + cls_name = self.__class__.__name__ + fmt = cls_name + '({0}, {1})' + return fmt.format(repr(self.microarchitecture), + repr(self.module_name)) def __str__(self): - return self.name + return str(self.microarchitecture) + + def __contains__(self, cpu_flag): + return cpu_flag in self.microarchitecture + + def optimization_flags(self, compiler): + return self.microarchitecture.optimization_flags( + compiler.name, str(compiler.version) + ) @key_ordering @@ -142,6 +229,8 @@ class Platform(object): front-end, and back-end. This can be overwritten by a subclass for which we want to provide further aliasing options. """ + # TODO: Check if we can avoid using strings here + name = str(name) if name == 'default_target': name = self.default elif name == 'frontend' or name == 'fe': @@ -296,7 +385,7 @@ class Arch(object): else: os = self.os if isinstance(self.target, Target): - target = self.target.name + target = self.target.microarchitecture else: target = self.target return (platform, os, target) @@ -306,7 +395,7 @@ class Arch(object): d = syaml_dict([ ('platform', str_or_none(self.platform)), ('platform_os', str_or_none(self.os)), - ('target', str_or_none(self.target))]) + ('target', self.target.to_dict_or_value())]) return syaml_dict([('arch', d)]) @staticmethod @@ -336,9 +425,9 @@ def verify_platform(platform_name): def arch_for_spec(arch_spec): - """Transforms the given architecture spec into an architecture objct.""" + """Transforms the given architecture spec into an architecture object.""" arch_spec = spack.spec.ArchSpec(arch_spec) - assert(arch_spec.concrete) + assert arch_spec.concrete arch_plat = get_platform(arch_spec.platform) if not (arch_plat.operating_system(arch_spec.os) and diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 1bd27f26a3..3702a989c3 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -200,23 +200,9 @@ def set_compiler_environment_variables(pkg, env): env.set('SPACK_F77_RPATH_ARG', compiler.f77_rpath_arg) env.set('SPACK_FC_RPATH_ARG', compiler.fc_rpath_arg) - # Set the tuning parameters that the compiler will add - isa_target = compiler.isa_name_for_target(spec.architecture.target) - if spec.variants['tuning'].value == 'generic': - tuning_target = 'generic' - else: - tuning_target = compiler.tuning_name_for_target( - spec.architecture.target - ) - if compiler.isa_flag and isa_target: - isa_arg = '{0}={1}'.format(compiler.isa_flag, isa_target) - else: - isa_arg = '' - if compiler.tuning_flag and tuning_target: - tuning_arg = '{0}={1}'.format(compiler.tuning_flag, tuning_target) - else: - tuning_arg = '' - env.set('SPACK_TARGET_ARGS', '{0} {1}'.format(isa_arg, tuning_arg)) + # Set the target parameters that the compiler will add + isa_arg = spec.architecture.target.optimization_flags(compiler) + env.set('SPACK_TARGET_ARGS', isa_arg) # Trap spack-tracked compiler flags as appropriate. # env_flags are easy to accidentally override. diff --git a/lib/spack/spack/cmd/__init__.py b/lib/spack/spack/cmd/__init__.py index f4d3b18c9f..cefec644a2 100644 --- a/lib/spack/spack/cmd/__init__.py +++ b/lib/spack/spack/cmd/__init__.py @@ -10,6 +10,8 @@ import re import sys import argparse +import six + import llnl.util.tty as tty from llnl.util.lang import attr_setdefault, index_by from llnl.util.tty.colify import colify @@ -132,7 +134,7 @@ def parse_specs(args, **kwargs): tests = kwargs.get('tests', False) try: - sargs = args if isinstance(args, basestring) else ' '.join(args) + sargs = args if isinstance(args, six.string_types) else ' '.join(args) specs = spack.spec.parse(sargs) for spec in specs: if concretize: diff --git a/lib/spack/spack/cmd/arch.py b/lib/spack/spack/cmd/arch.py index 707a91cc2a..081722c05b 100644 --- a/lib/spack/spack/cmd/arch.py +++ b/lib/spack/spack/cmd/arch.py @@ -5,6 +5,11 @@ from __future__ import print_function +import collections + +import llnl.util.cpu +import llnl.util.tty.colify as colify +import llnl.util.tty.color as color import spack.architecture as architecture description = "print architecture information about this machine" @@ -13,6 +18,10 @@ level = "short" def setup_parser(subparser): + subparser.add_argument( + '--known-targets', action='store_true', + help='show a list of all known targets and exit' + ) parts = subparser.add_mutually_exclusive_group() parts2 = subparser.add_mutually_exclusive_group() parts.add_argument( @@ -32,7 +41,41 @@ def setup_parser(subparser): help='print backend') +def display_targets(targets): + """Prints a human readable list of the targets passed as argument.""" + by_vendor = collections.defaultdict(list) + for _, target in targets.items(): + by_vendor[target.vendor].append(target) + + def display_target_group(header, target_group): + print(header) + colify.colify(target_group, indent=4) + print('') + + generic_architectures = by_vendor.pop('generic', None) + if generic_architectures: + header = color.colorize(r'@*B{Generic architectures (families)}') + group = sorted(generic_architectures, key=lambda x: str(x)) + display_target_group(header, group) + + for vendor, vendor_targets in by_vendor.items(): + by_family = collections.defaultdict(list) + for t in vendor_targets: + by_family[str(t.family)].append(t) + + for family, group in by_family.items(): + vendor = color.colorize(r'@*B{' + vendor + r'}') + family = color.colorize(r'@*B{' + family + r'}') + header = ' - '.join([vendor, family]) + group = sorted(group, key=lambda x: len(x.ancestors)) + display_target_group(header, group) + + def arch(parser, args): + if args.known_targets: + display_targets(llnl.util.cpu.targets) + return + if args.frontend: arch = architecture.Arch(architecture.platform(), 'frontend', 'frontend') diff --git a/lib/spack/spack/cmd/compiler.py b/lib/spack/spack/cmd/compiler.py index 5f7eaa7d45..c52a3c7081 100644 --- a/lib/spack/spack/cmd/compiler.py +++ b/lib/spack/spack/cmd/compiler.py @@ -83,7 +83,7 @@ def compiler_find(args): compilers = [c for c in spack.compilers.find_compilers(paths)] new_compilers = [] for c in compilers: - arch_spec = ArchSpec(None, c.operating_system, c.target) + arch_spec = ArchSpec((None, c.operating_system, c.target)) same_specs = spack.compilers.compilers_for_spec( c.spec, arch_spec, init_config=False) diff --git a/lib/spack/spack/cmd/mirror.py b/lib/spack/spack/cmd/mirror.py index b7cea46d81..723e310ad6 100644 --- a/lib/spack/spack/cmd/mirror.py +++ b/lib/spack/spack/cmd/mirror.py @@ -153,7 +153,7 @@ def mirror_create(args): """Create a directory to be used as a spack mirror, and fill it with package archives.""" # try to parse specs from the command line first. - with spack.concretize.concretizer.disable_compiler_existence_check(): + with spack.concretize.disable_compiler_existence_check(): specs = spack.cmd.parse_specs(args.specs, concretize=True) # If there is a file, parse each line as a spec and add it to the list. diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index 18ec3bc71c..2152152255 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -222,14 +222,6 @@ class Compiler(object): def fc_rpath_arg(self): return '-Wl,-rpath,' - @property - def isa_flag(self): - return '-march' - - @property - def tuning_flag(self): - return '-mtune' - # Cray PrgEnv name that can be used to load this compiler PrgEnv = None # Name of module used to switch versions of this compiler @@ -428,12 +420,6 @@ class Compiler(object): def fc_version(cls, fc): return cls.default_version(fc) - def isa_name_for_target(self, target): - return str(target) - - def tuning_name_for_target(self, target): - return str(target) - @classmethod def search_regexps(cls, language): # Compile all the regular expressions used for files beforehand. diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index bc2d06e954..206aa17e1b 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -10,13 +10,12 @@ import collections import itertools import multiprocessing.pool import os -import platform as py_platform import six import llnl.util.lang import llnl.util.filesystem as fs import llnl.util.tty as tty -from llnl.util.cpu_name import get_cpu_name +import llnl.util.cpu as cpu import spack.paths import spack.error @@ -402,8 +401,18 @@ def get_compilers(config, cspec=None, arch_spec=None): # any given arch spec. If the compiler has no assigned # target this is an old compiler config file, skip this logic. target = items.get('target', None) - if arch_spec and target and (target != arch_spec.target and - target != 'any'): + + try: + current_target = llnl.util.cpu.targets[str(arch_spec.target)] + family = str(current_target.family) + except KeyError: + # TODO: Check if this exception handling makes sense, or if we + # TODO: need to change / refactor tests + family = arch_spec.target + except AttributeError: + assert arch_spec is None + + if arch_spec and target and (target != family and target != 'any'): continue compilers.append(_compiler_from_config_entry(items)) @@ -646,8 +655,9 @@ def make_compiler_list(detected_versions): compiler_cls = spack.compilers.class_for_compiler_name(compiler_name) spec = spack.spec.CompilerSpec(compiler_cls.name, version) paths = [paths.get(l, None) for l in ('cc', 'cxx', 'f77', 'fc')] + target = cpu.host() compiler = compiler_cls( - spec, operating_system, get_cpu_name(), paths + spec, operating_system, str(target.family), paths ) return [compiler] diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 136a9d67f0..ed8f6ac2e7 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -16,6 +16,7 @@ TODO: make this customizable and allow users to configure """ from __future__ import print_function +import platform import os.path import tempfile import llnl.util.filesystem as fs @@ -24,9 +25,9 @@ import llnl.util.tty as tty from itertools import chain from functools_backport import reverse_order from contextlib import contextmanager -from six import iteritems import llnl.util.lang +import llnl.util.cpu as cpu import spack.repo import spack.abi @@ -40,10 +41,6 @@ from spack.version import ver, Version, VersionList, VersionRange from spack.package_prefs import PackagePrefs, spec_externals, is_spec_buildable -#: Concretizer singleton -concretizer = llnl.util.lang.Singleton(lambda: Concretizer()) - - #: impements rudimentary logic for ABI compatibility _abi = llnl.util.lang.Singleton(lambda: spack.abi.ABI()) @@ -52,25 +49,17 @@ class Concretizer(object): """You can subclass this class to override some of the default concretization strategies, or you can override all of them. """ - def __init__(self): - # controls whether we check that compiler versions actually exist - # during concretization. Used for testing and for mirror creation - self.check_for_compiler_existence = not config.get( - 'config:install_missing_compilers', False) - - @contextmanager - def disable_compiler_existence_check(self): - saved = self.check_for_compiler_existence - self.check_for_compiler_existence = False - yield - self.check_for_compiler_existence = saved - - @contextmanager - def enable_compiler_existence_check(self): - saved = self.check_for_compiler_existence - self.check_for_compiler_existence = True - yield - self.check_for_compiler_existence = saved + #: Controls whether we check that compiler versions actually exist + #: during concretization. Used for testing and for mirror creation + check_for_compiler_existence = None + + def __init__(self, abstract_spec=None): + if Concretizer.check_for_compiler_existence is None: + Concretizer.check_for_compiler_existence = not config.get( + 'config:install_missing_compilers', False + ) + self.abstract_spec = abstract_spec + self._adjust_target_answer_generator = None def _valid_virtuals_and_externals(self, spec): """Returns a list of candidate virtual dep providers and external @@ -234,30 +223,91 @@ class Concretizer(object): DAG has an architecture, then use the root otherwise use the defaults on the platform. """ - try: - # Get the nearest architecture with any fields set - nearest = next(p for p in spec.traverse(direction='parents') - if (p.architecture and p is not spec)) - nearest_arch = nearest.architecture - except StopIteration: - # Default to the system architecture if nothing set - nearest_arch = spack.spec.ArchSpec(spack.architecture.sys_type()) - - spec_changed = False - # ensure type safety for the architecture if spec.architecture is None: spec.architecture = spack.spec.ArchSpec() - spec_changed = True - # replace each of the fields (platform, os, target) separately - nearest_dict = nearest_arch.to_cmp_dict() - replacement_fields = [k for k, v in iteritems(nearest_dict) - if v and not getattr(spec.architecture, k)] - for field in replacement_fields: - setattr(spec.architecture, field, getattr(nearest_arch, field)) - spec_changed = True + if spec.architecture.platform and \ + (spec.architecture.os and spec.architecture.target): + return False + + # Get platform of nearest spec with a platform, including spec + # If spec has a platform, easy + if spec.architecture.platform: + new_plat = spack.architecture.get_platform( + spec.architecture.platform) + else: + # Else if anyone else has a platform, take the closest one + # Search up, then down, along build/link deps first + # Then any nearest. Algorithm from compilerspec search + platform_spec = find_spec( + spec, lambda x: x.architecture and x.architecture.platform + ) + if platform_spec: + new_plat = spack.architecture.get_platform( + platform_spec.architecture.platform) + else: + # If no platform anywhere in this spec, grab the default + new_plat = spack.architecture.platform() + + # Get nearest spec with relevant platform and an os + # Generally, same algorithm as finding platform, except we only + # consider specs that have a platform + if spec.architecture.os: + new_os = spec.architecture.os + else: + new_os_spec = find_spec( + spec, lambda x: (x.architecture and + x.architecture.platform == str(new_plat) and + x.architecture.os) + ) + if new_os_spec: + new_os = new_os_spec.architecture.os + else: + new_os = new_plat.operating_system('default_os') + + # Get the nearest spec with relevant platform and a target + # Generally, same algorithm as finding os + if spec.architecture.target: + new_target = spec.architecture.target + else: + new_target_spec = find_spec( + spec, lambda x: (x.architecture and + x.architecture.platform == str(new_plat) and + x.architecture.target) + ) + if new_target_spec: + new_target = new_target_spec.architecture.target + else: + # To get default platform, consider package prefs + if PackagePrefs.has_preferred_targets(spec.name): + target_prefs = PackagePrefs(spec.name, 'target') + target_specs = [spack.spec.Spec('target=%s' % tname) + for tname in cpu.targets] + + def tspec_filter(s): + # Filter target specs by whether the architecture + # family is the current machine type. This ensures + # we only consider x86_64 targets when on an + # x86_64 machine, etc. This may need to change to + # enable setting cross compiling as a default + target = cpu.targets[s.architecture.target] + arch_family_name = target.family.name + return arch_family_name == platform.machine() + + # Sort filtered targets by package prefs + target_specs = list(filter(tspec_filter, target_specs)) + target_specs.sort(key=target_prefs) + + new_target = target_specs[0].architecture.target + else: + new_target = new_plat.target('default_target') + # Construct new architecture, compute whether spec changed + arch_spec = (str(new_plat), str(new_os), str(new_target)) + new_arch = spack.spec.ArchSpec(arch_spec) + spec_changed = new_arch != spec.architecture + spec.architecture = new_arch return spec_changed def concretize_variants(self, spec): @@ -316,7 +366,7 @@ class Concretizer(object): other_spec = spec if spec.compiler else find_spec( spec, lambda x: x.compiler, spec.root) other_compiler = other_spec.compiler - assert(other_spec) + assert other_spec # Check if the compiler is already fully specified if other_compiler and other_compiler.concrete: @@ -368,7 +418,7 @@ class Concretizer(object): _compiler_concretization_failure( other_compiler, spec.architecture) - assert(spec.compiler.concrete) + assert spec.compiler.concrete return True # things changed. def concretize_compiler_flags(self, spec): @@ -429,6 +479,94 @@ class Concretizer(object): return ret + def adjust_target(self, spec): + """Adjusts the target microarchitecture if the compiler is too old + to support the default one. + + Args: + spec: spec to be concretized + + Returns: + True if spec was modified, False otherwise + """ + # To minimize the impact on performance this function will attempt + # to adjust the target only at the very first call. It will just + # return False on subsequent calls. The way this is achieved is by + # initializing a generator and making this function return the next + # answer. + def _make_only_one_call(spec): + yield self._adjust_target(spec) + while True: + yield False + + if self._adjust_target_answer_generator is None: + self._adjust_target_answer_generator = _make_only_one_call(spec) + + return next(self._adjust_target_answer_generator) + + def _adjust_target(self, spec): + """Assumes that the architecture and the compiler have been + set already and checks if the current target microarchitecture + is the default and can be optimized by the compiler. + + If not, downgrades the microarchitecture until a suitable one + is found. If none can be found raise an error. + + Args: + spec: spec to be concretized + + Returns: + True if any modification happened, False otherwise + """ + import llnl.util.cpu + + # Try to adjust the target only if it is the default + # target for this platform + current_target = spec.architecture.target + current_platform = spack.architecture.get_platform( + spec.architecture.platform + ) + if current_target != current_platform.target('default_target') or \ + (self.abstract_spec.architecture is not None and + self.abstract_spec.architecture.target is not None): + return False + + try: + current_target.optimization_flags(spec.compiler) + except llnl.util.cpu.UnsupportedMicroarchitecture: + microarchitecture = current_target.microarchitecture + for ancestor in microarchitecture.ancestors: + candidate = None + try: + candidate = spack.architecture.Target(ancestor) + candidate.optimization_flags(spec.compiler) + except llnl.util.cpu.UnsupportedMicroarchitecture: + continue + + if candidate is not None: + spec.architecture.target = candidate + return True + else: + raise + + return False + + +@contextmanager +def disable_compiler_existence_check(): + saved = Concretizer.check_for_compiler_existence + Concretizer.check_for_compiler_existence = False + yield + Concretizer.check_for_compiler_existence = saved + + +@contextmanager +def enable_compiler_existence_check(): + saved = Concretizer.check_for_compiler_existence + Concretizer.check_for_compiler_existence = True + yield + Concretizer.check_for_compiler_existence = saved + def find_spec(spec, condition, default=None): """Searches the dag from spec in an intelligent order and looks diff --git a/lib/spack/spack/main.py b/lib/spack/spack/main.py index 69f430b5d5..a2220a9ba8 100644 --- a/lib/spack/spack/main.py +++ b/lib/spack/spack/main.py @@ -17,6 +17,7 @@ import inspect import pstats import argparse import traceback +import warnings from six import StringIO import llnl.util.tty as tty @@ -389,8 +390,16 @@ def make_argument_parser(**kwargs): return parser +def send_warning_to_tty(message, *args): + """Redirects messages to tty.warn.""" + tty.warn(message) + + def setup_main_options(args): """Configure spack globals based on the basic options.""" + # Assign a custom function to show warnings + warnings.showwarning = send_warning_to_tty + # Set up environment based on args. tty.set_verbose(args.verbose) tty.set_debug(args.debug) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index dd700bd646..23763edf10 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -61,7 +61,6 @@ from spack.util.environment import dump_environment from spack.util.package_hash import package_hash from spack.version import Version from spack.package_prefs import get_package_dir_permissions, get_package_group -from spack.directives import variant """Allowed URL schemes for spack packages.""" _ALLOWED_URL_SCHEMES = ["http", "https", "ftp", "file", "git"] @@ -508,10 +507,6 @@ class PackageBase(with_metaclass(PackageMeta, PackageViewMixin, object)): metadata_attrs = ['homepage', 'url', 'list_url', 'extendable', 'parallel', 'make_jobs'] - # Add the universal variant "tuning" with values generic | specific - variant('tuning', values=('generic', 'specific'), default='generic', - description='Set compiler tuning generic or to target') - def __init__(self, spec): # this determines how the package should be built. self.spec = spec diff --git a/lib/spack/spack/package_prefs.py b/lib/spack/spack/package_prefs.py index f9945a7ea0..e73ce7c947 100644 --- a/lib/spack/spack/package_prefs.py +++ b/lib/spack/spack/package_prefs.py @@ -137,7 +137,10 @@ class PackagePrefs(object): order = order.get(vpkg) if order: - return [str(s).strip() for s in order] + ret = [str(s).strip() for s in order] + if component == 'target': + ret = ['target=%s' % tname for tname in ret] + return ret return [] @@ -169,6 +172,11 @@ class PackagePrefs(object): return bool(cls.order_for_package(pkgname, 'providers', vpkg, False)) @classmethod + def has_preferred_targets(cls, pkg_name): + """Whether specific package has a preferred vpkg providers.""" + return bool(cls.order_for_package(pkg_name, 'target')) + + @classmethod def preferred_variants(cls, pkg_name): """Return a VariantMap of preferred variants/values for a spec.""" for pkg in (pkg_name, 'all'): diff --git a/lib/spack/spack/platforms/darwin.py b/lib/spack/spack/platforms/darwin.py index 2d580d8850..7c36812d9d 100644 --- a/lib/spack/spack/platforms/darwin.py +++ b/lib/spack/spack/platforms/darwin.py @@ -4,7 +4,7 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) import platform -from llnl.util.cpu_name import get_cpu_name +import llnl.util.cpu as cpu from spack.architecture import Platform, Target from spack.operating_systems.mac_os import MacOs @@ -15,33 +15,13 @@ class Darwin(Platform): def __init__(self): super(Darwin, self).__init__('darwin') - # TODO: These are probably overkill - # Add Intel architectures - self.add_target('haswell', Target('haswell')) - self.add_target('broadwell', Target('broadwell')) - self.add_target('ivybridge', Target('ivybridge')) - self.add_target('sandybridge', Target('sandybridge')) - self.add_target('core2', Target('core2')) - - # Add "basic" architectures - self.add_target('x86_64', Target('x86_64')) - self.add_target('ppc64le', Target('ppc64le')) - self.add_target('ppc64', Target('ppc64')) - - # Add IBM architectures - self.add_target('power7', Target('power7')) - self.add_target('power8', Target('power8')) - self.add_target('power8le', Target('power8le')) - self.add_target('power9', Target('power9')) - self.add_target('power9le', Target('power9le')) - - self.default = get_cpu_name() + for name in cpu.targets: + self.add_target(name, Target(name)) + + self.default = cpu.host().name self.front_end = self.default self.back_end = self.default - if self.default not in self.targets: - self.add_target(self.default, Target(self.default)) - mac_os = MacOs() self.default_os = str(mac_os) diff --git a/lib/spack/spack/platforms/linux.py b/lib/spack/spack/platforms/linux.py index 23f10358fb..d2165c7d79 100644 --- a/lib/spack/spack/platforms/linux.py +++ b/lib/spack/spack/platforms/linux.py @@ -6,7 +6,8 @@ import platform from spack.architecture import Platform, Target from spack.operating_systems.linux_distro import LinuxDistro -from llnl.util.cpu_name import get_cpu_name +import llnl.util.cpu as cpu + class Linux(Platform): priority = 90 @@ -14,41 +15,14 @@ class Linux(Platform): def __init__(self): super(Linux, self).__init__('linux') - # Add "basic" architectures - self.add_target('x86_64', Target('x86_64')) - self.add_target('ppc64le', Target('ppc64le')) - self.add_target('ppc64', Target('ppc64')) - - # Add Intel architectures - self.add_target('haswell', Target('haswell')) - self.add_target('broadwell', Target('broadwell')) - self.add_target('ivybridge', Target('ivybridge')) - self.add_target('sandybridge', Target('sandybridge')) - self.add_target('knl', Target('knl')) - - # Add IBM architectures - self.add_target('power7', Target('power7')) - self.add_target('power8', Target('power8')) - self.add_target('power8le', Target('power8le')) - self.add_target('power9', Target('power9')) - self.add_target('power9le', Target('power9le')) - # Eternal TODO: Add more architectures as needed. + for name in cpu.targets: + self.add_target(name, Target(name)) # Get specific default - self.default = get_cpu_name() + self.default = cpu.host().name self.front_end = self.default self.back_end = self.default - if not self.default: - # Fall back on more general name. - # This will likely fall in "basic" architectures list - self.default = platform.machine() - self.front_end = self.default - self.back_end = self.default - - if self.default not in self.targets: - self.add_target(self.default, Target(self.default)) - linux_dist = LinuxDistro() self.default_os = str(linux_dist) self.front_os = self.default_os diff --git a/lib/spack/spack/platforms/test.py b/lib/spack/spack/platforms/test.py index 0af258933d..c9bbd02dfa 100644 --- a/lib/spack/spack/platforms/test.py +++ b/lib/spack/spack/platforms/test.py @@ -9,7 +9,7 @@ from spack.architecture import OperatingSystem class Test(Platform): priority = 1000000 - front_end = 'x86_32' + front_end = 'x86' back_end = 'x86_64' default = 'x86_64' diff --git a/lib/spack/spack/schema/packages.py b/lib/spack/spack/schema/packages.py index c276c4a174..f97d82d83a 100644 --- a/lib/spack/spack/schema/packages.py +++ b/lib/spack/spack/schema/packages.py @@ -28,6 +28,12 @@ properties = { # version strings 'items': {'anyOf': [{'type': 'string'}, {'type': 'number'}]}}, + 'target': { + 'type': 'array', + 'default': [], + # target names + 'items': {'type': 'string'}, + }, 'compiler': { 'type': 'array', 'default': [], diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index dca06101b8..99246ebb02 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -85,10 +85,9 @@ import itertools import os import re +import six + from operator import attrgetter -from six import StringIO -from six import string_types -from six import iteritems from llnl.util.filesystem import find_headers, find_libraries, is_exe from llnl.util.lang import key_ordering, HashableMap, ObjectWrapper, dedupe @@ -216,37 +215,47 @@ def colorize_spec(spec): @key_ordering class ArchSpec(object): - """ The ArchSpec class represents an abstract architecture specification - that a package should be built with. At its core, each ArchSpec is - comprised of three elements: a platform (e.g. Linux), an OS (e.g. - RHEL6), and a target (e.g. x86_64). - """ + def __init__(self, spec_or_platform_tuple=(None, None, None)): + """ Architecture specification a package should be built with. - # TODO: Formalize the specifications for architectures and then use - # the appropriate parser here to read these specifications. - def __init__(self, *args): - to_attr_string = lambda s: str(s) if s and s != "None" else None + Each ArchSpec is comprised of three elements: a platform (e.g. Linux), + an OS (e.g. RHEL6), and a target (e.g. x86_64). + + Args: + spec_or_platform_tuple (ArchSpec or str or tuple): if an ArchSpec + is passed it will be duplicated into the new instance. + Otherwise information on platform, OS and target should be + passed in either as a spec string or as a tuple. + """ + # If another instance of ArchSpec was passed, duplicate it + if isinstance(spec_or_platform_tuple, ArchSpec): + self._dup(spec_or_platform_tuple) + return - self.platform, self.os, self.target = (None, None, None) + # If the argument to __init__ is a spec string, parse it + # and construct an ArchSpec + def _string_or_none(s): + if s and s != 'None': + return str(s) + return None - if len(args) == 1: - spec_like = args[0] - if isinstance(spec_like, ArchSpec): - self._dup(spec_like) - elif isinstance(spec_like, string_types): - spec_fields = spec_like.split("-") + if isinstance(spec_or_platform_tuple, six.string_types): + spec_fields = spec_or_platform_tuple.split("-") + msg = "invalid arch spec [{0}]" + assert len(spec_fields) == 3, msg.format(spec_or_platform_tuple) - if len(spec_fields) == 3: - self.platform, self.os, self.target = tuple( - to_attr_string(f) for f in spec_fields) - else: - raise ValueError("%s is an invalid arch spec" % spec_like) - elif len(args) == 3: - self.platform = to_attr_string(args[0]) - self.os = to_attr_string(args[1]) - self.target = to_attr_string(args[2]) - elif len(args) != 0: - raise TypeError("Can't make arch spec from %s" % args) + platform, operating_system, target = spec_fields + platform_tuple = _string_or_none(platform),\ + _string_or_none(operating_system), target + + if isinstance(spec_or_platform_tuple, tuple): + platform, operating_system, target = spec_or_platform_tuple + platform_tuple = _string_or_none(platform), \ + _string_or_none(operating_system), target + msg = "invalid arch spec tuple [{0}]" + assert len(platform_tuple) == 3, msg.format(platform_tuple) + + self.platform, self.os, self.target = platform_tuple def _autospec(self, spec_like): if isinstance(spec_like, ArchSpec): @@ -254,7 +263,7 @@ class ArchSpec(object): return ArchSpec(spec_like) def _cmp_key(self): - return (self.platform, self.os, self.target) + return self.platform, self.os, self.target def _dup(self, other): self.platform = other.platform @@ -263,29 +272,29 @@ class ArchSpec(object): @property def platform(self): + """The platform of the architecture.""" return self._platform @platform.setter def platform(self, value): - """ The platform of the architecture spec will be verified as a - supported Spack platform before it's set to ensure all specs - refer to valid platforms. - """ + # The platform of the architecture spec will be verified as a + # supported Spack platform before it's set to ensure all specs + # refer to valid platforms. value = str(value) if value is not None else None self._platform = value @property def os(self): + """The OS of this ArchSpec.""" return self._os @os.setter def os(self, value): - """ The OS of the architecture spec will update the platform field - if the OS is set to one of the reserved OS types so that the - default OS type can be resolved. Since the reserved OS - information is only available for the host machine, the platform - will assumed to be the host machine's platform. - """ + # The OS of the architecture spec will update the platform field + # if the OS is set to one of the reserved OS types so that the + # default OS type can be resolved. Since the reserved OS + # information is only available for the host machine, the platform + # will assumed to be the host machine's platform. value = str(value) if value is not None else None if value in spack.architecture.Platform.reserved_oss: @@ -305,19 +314,27 @@ class ArchSpec(object): @property def target(self): + """The target of the architecture.""" return self._target @target.setter def target(self, value): - """ The target of the architecture spec will update the platform field - if the target is set to one of the reserved target types so that - the default target type can be resolved. Since the reserved target - information is only available for the host machine, the platform - will assumed to be the host machine's platform. - """ - value = str(value) if value is not None else None + # The target of the architecture spec will update the platform field + # if the target is set to one of the reserved target types so that + # the default target type can be resolved. Since the reserved target + # information is only available for the host machine, the platform + # will assumed to be the host machine's platform. + + def target_or_none(t): + if isinstance(t, spack.architecture.Target): + return t + if t and t != 'None': + return spack.architecture.Target(t) + return None - if value in spack.architecture.Platform.reserved_targets: + value = target_or_none(value) + + if str(value) in spack.architecture.Platform.reserved_targets: curr_platform = str(spack.architecture.platform()) self.platform = self.platform or curr_platform @@ -328,25 +345,76 @@ class ArchSpec(object): (value, self.platform, curr_platform)) spec_platform = spack.architecture.get_platform(self.platform) - value = str(spec_platform.target(value)) + value = spec_platform.target(value) self._target = value def satisfies(self, other, strict=False): + """Predicate to check if this spec satisfies a constraint. + + Args: + other (ArchSpec or str): constraint on the current instance + strict (bool): if ``False`` the function checks if the current + instance *might* eventually satisfy the constraint. If + ``True`` it check if the constraint is satisfied right now. + + Returns: + True if the constraint is satisfied, False otherwise. + """ other = self._autospec(other) - sdict, odict = self.to_cmp_dict(), other.to_cmp_dict() - if strict or self.concrete: - return all(getattr(self, attr) == getattr(other, attr) - for attr in odict if odict[attr]) - else: - return all(getattr(self, attr) == getattr(other, attr) - for attr in odict if sdict[attr] and odict[attr]) + # Check platform and os + for attribute in ('platform', 'os'): + other_attribute = getattr(other, attribute) + self_attribute = getattr(self, attribute) + if strict or self.concrete: + if other_attribute and self_attribute != other_attribute: + return False + else: + if other_attribute and self_attribute and \ + self_attribute != other_attribute: + return False + + # Check target + return self._satisfies_target(other.target, strict=strict) + + def _satisfies_target(self, other_target, strict): + self_target = self.target + + need_to_check = bool(other_target) if strict or self.concrete \ + else bool(other_target and self_target) + + # If there's no need to check we are fine + if not need_to_check: + return True + + for target_range in str(other_target).split(','): + t_min, sep, t_max = target_range.partition(':') + + # Checking against a single specific target + if not sep and self_target == t_min: + return True + + min_ok = self_target.microarchitecture >= t_min if t_min else True + max_ok = self_target.microarchitecture <= t_max if t_max else True + + if min_ok and max_ok: + return True + + return False def constrain(self, other): - """ Projects all architecture fields that are specified in the given - spec onto the instance spec if they're missing from the instance - spec. This will only work if the two specs are compatible. + """Projects all architecture fields that are specified in the given + spec onto the instance spec if they're missing from the instance + spec. + + This will only work if the two specs are compatible. + + Args: + other (ArchSpec or str): constraints to be added + + Returns: + True if the current instance was constrained, False otherwise. """ other = self._autospec(other) @@ -354,8 +422,8 @@ class ArchSpec(object): raise UnsatisfiableArchitectureSpecError(self, other) constrained = False - for attr, svalue in iteritems(self.to_cmp_dict()): - ovalue = getattr(other, attr) + for attr in ('platform', 'os', 'target'): + svalue, ovalue = getattr(self, attr), getattr(other, attr) if svalue is None and ovalue is not None: setattr(self, attr, ovalue) constrained = True @@ -363,26 +431,22 @@ class ArchSpec(object): return constrained def copy(self): + """Copy the current instance and returns the clone.""" clone = ArchSpec.__new__(ArchSpec) clone._dup(self) return clone @property def concrete(self): - return all(v for k, v in iteritems(self.to_cmp_dict())) - - def to_cmp_dict(self): - """Returns a dictionary that can be used for field comparison.""" - return dict([ - ('platform', self.platform), - ('os', self.os), - ('target', self.target)]) + """True if the spec is concrete, False otherwise""" + # return all(v for k, v in six.iteritems(self.to_cmp_dict())) + return self.platform and self.os and self.target def to_dict(self): d = syaml_dict([ ('platform', self.platform), ('platform_os', self.os), - ('target', self.target)]) + ('target', self.target.to_dict_or_value())]) return syaml_dict([('arch', d)]) @staticmethod @@ -400,22 +464,26 @@ class ArchSpec(object): """ if not isinstance(d['arch'], dict): - return ArchSpec('spack09', 'unknown', d['arch']) + return ArchSpec(('spack09', 'unknown', d['arch'])) d = d['arch'] - if 'platform_os' in d: - return ArchSpec(d['platform'], d['platform_os'], d['target']) - else: - return ArchSpec(d['platform'], d['os'], d['target']) + + operating_system = d.get('platform_os', None) or d['os'] + target = spack.architecture.Target.from_dict_or_value(d['target']) + + return ArchSpec((d['platform'], operating_system, target)) def __str__(self): return "%s-%s-%s" % (self.platform, self.os, self.target) def __repr__(self): + # TODO: this needs to be changed (repr is meant to return valid + # TODO: Python code to return an instance equivalent to the current + # TODO: one). return str(self) def __contains__(self, string): - return string in str(self) + return string in str(self) or string in self.target @key_ordering @@ -430,7 +498,7 @@ class CompilerSpec(object): arg = args[0] # If there is one argument, it's either another CompilerSpec # to copy or a string to parse - if isinstance(arg, string_types): + if isinstance(arg, six.string_types): c = SpecParser().parse_compiler(arg) self.name = c.name self.versions = c.versions @@ -618,7 +686,7 @@ class FlagMap(HashableMap): return clone def _cmp_key(self): - return tuple((k, tuple(v)) for k, v in sorted(iteritems(self))) + return tuple((k, tuple(v)) for k, v in sorted(six.iteritems(self))) def __str__(self): sorted_keys = [k for k in sorted(self.keys()) if self[k] != []] @@ -945,7 +1013,7 @@ class Spec(object): self.external_module = external_module self._full_hash = full_hash - if isinstance(spec_like, string_types): + if isinstance(spec_like, six.string_types): spec_list = SpecParser(self).parse(spec_like) if len(spec_list) > 1: raise ValueError("More than one spec in string: " + spec_like) @@ -1032,9 +1100,9 @@ class Spec(object): if not self.architecture: new_vals = tuple(kwargs.get(arg, None) for arg in arch_attrs) - self.architecture = ArchSpec(*new_vals) + self.architecture = ArchSpec(new_vals) else: - new_attrvals = [(a, v) for a, v in iteritems(kwargs) + new_attrvals = [(a, v) for a, v in six.iteritems(kwargs) if a in arch_attrs] for new_attr, new_value in new_attrvals: if getattr(self.architecture, new_attr): @@ -1186,7 +1254,7 @@ class Spec(object): # get initial values for kwargs depth = kwargs.get('depth', False) key_fun = kwargs.get('key', id) - if isinstance(key_fun, string_types): + if isinstance(key_fun, six.string_types): key_fun = attrgetter(key_fun) yield_root = kwargs.get('root', True) cover = kwargs.get('cover', 'nodes') @@ -1659,7 +1727,7 @@ class Spec(object): formats so that reindex will work on old specs/databases. """ for dep_name, elt in dependency_dict.items(): - if isinstance(elt, string_types): + if isinstance(elt, six.string_types): # original format, elt is just the dependency hash. dag_hash, deptypes = elt, ['build', 'link'] elif isinstance(elt, tuple): @@ -1813,7 +1881,7 @@ class Spec(object): # Recurse on dependencies for s, s_dependencies in dep_like.items(): - if isinstance(s, string_types): + if isinstance(s, six.string_types): dag_node, dependency_types = name_and_dependency_types(s) else: dag_node, dependency_types = spec_and_dependency_types(s) @@ -1884,7 +1952,7 @@ class Spec(object): tty.debug(e) raise sjson.SpackJSONError("error parsing JSON spec:", str(e)) - def _concretize_helper(self, presets=None, visited=None): + def _concretize_helper(self, concretizer, presets=None, visited=None): """Recursive helper function for concretize(). This concretizes everything bottom-up. As things are concretized, they're added to the presets, and ancestors @@ -1906,8 +1974,9 @@ class Spec(object): # Concretize deps first -- this is a bottom-up process. for name in sorted(self._dependencies.keys()): - changed |= self._dependencies[ - name].spec._concretize_helper(presets, visited) + changed |= self._dependencies[name].spec._concretize_helper( + concretizer, presets, visited + ) if self.name in presets: changed |= self.constrain(presets[self.name]) @@ -1916,11 +1985,10 @@ class Spec(object): # to presets below, their constraints will all be merged, but we'll # still need to select a concrete package later. if not self.virtual: - import spack.concretize - concretizer = spack.concretize.concretizer changed |= any( (concretizer.concretize_architecture(self), concretizer.concretize_compiler(self), + concretizer.adjust_target(self), # flags must be concretized after compiler concretizer.concretize_compiler_flags(self), concretizer.concretize_version(self), @@ -1945,7 +2013,7 @@ class Spec(object): if concrete.name not in dependent._dependencies: dependent._add_dependency(concrete, deptypes) - def _expand_virtual_packages(self): + def _expand_virtual_packages(self, concretizer): """Find virtual packages in this spec, replace them with providers, and normalize again to include the provider's (potentially virtual) dependencies. Repeat until there are no virtual deps. @@ -1985,8 +2053,6 @@ class Spec(object): if not replacement: # Get a list of possible replacements in order of # preference. - import spack.concretize - concretizer = spack.concretize.concretizer candidates = concretizer.choose_virtual_or_external(spec) # Try the replacements in order, skipping any that cause @@ -2075,12 +2141,13 @@ class Spec(object): force = False user_spec_deps = self.flat_dependencies(copy=False) - + import spack.concretize + concretizer = spack.concretize.Concretizer(self.copy()) while changed: changes = (self.normalize(force, tests=tests, user_spec_deps=user_spec_deps), - self._expand_virtual_packages(), - self._concretize_helper()) + self._expand_virtual_packages(concretizer), + self._concretize_helper(concretizer)) changed = any(changes) force = True @@ -2190,6 +2257,10 @@ class Spec(object): if matches: raise ConflictsInSpecError(self, matches) + # Check if we can produce an optimized binary (will throw if + # there are declared inconsistencies) + self.architecture.target.optimization_flags(self.compiler) + def _mark_concrete(self, value=True): """Mark this spec and its dependencies as concrete. @@ -3283,7 +3354,7 @@ class Spec(object): color = kwargs.get('color', False) transform = kwargs.get('transform', {}) - out = StringIO() + out = six.StringIO() def write(s, c=None): f = cescape(s) @@ -3515,7 +3586,7 @@ class Spec(object): (k.upper(), v) for k, v in kwargs.get('transform', {}).items()) length = len(format_string) - out = StringIO() + out = six.StringIO() named = escape = compiler = False named_str = fmt = '' @@ -3790,6 +3861,20 @@ class Spec(object): def __repr__(self): return str(self) + @property + def platform(self): + return self.architecture.platform + + @property + def os(self): + return self.architecture.os + + @property + def target(self): + # This property returns the underlying microarchitecture object + # to give to the attribute the appropriate comparison semantic + return self.architecture.target.microarchitecture + class LazySpecCache(collections.defaultdict): """Cache for Specs that uses a spec_like as key, and computes lazily diff --git a/lib/spack/spack/test/architecture.py b/lib/spack/spack/test/architecture.py index 76598ce620..b36219aa76 100644 --- a/lib/spack/spack/test/architecture.py +++ b/lib/spack/spack/test/architecture.py @@ -7,10 +7,11 @@ """ Test checks if the architecture class is created correctly and also that the functions are looking for the correct architecture name """ -import itertools import os import platform as py_platform +import pytest + import spack.architecture from spack.spec import Spec from spack.platforms.cray import Cray @@ -82,7 +83,7 @@ def test_user_front_end_input(config): """ platform = spack.architecture.platform() frontend_os = str(platform.operating_system('frontend')) - frontend_target = str(platform.target('frontend')) + frontend_target = platform.target('frontend') frontend_spec = Spec('libelf os=frontend target=frontend') frontend_spec.concretize() @@ -97,7 +98,7 @@ def test_user_back_end_input(config): """ platform = spack.architecture.platform() backend_os = str(platform.operating_system("backend")) - backend_target = str(platform.target("backend")) + backend_target = platform.target("backend") backend_spec = Spec("libelf os=backend target=backend") backend_spec.concretize() @@ -109,7 +110,7 @@ def test_user_back_end_input(config): def test_user_defaults(config): platform = spack.architecture.platform() default_os = str(platform.operating_system("default_os")) - default_target = str(platform.target("default_target")) + default_target = platform.target("default_target") default_spec = Spec("libelf") # default is no args default_spec.concretize() @@ -118,29 +119,20 @@ def test_user_defaults(config): assert default_target == default_spec.architecture.target -def test_user_input_combination(config): +@pytest.mark.parametrize('operating_system', [ + x for x in spack.architecture.platform().operating_sys +] + ["fe", "be", "frontend", "backend"]) +@pytest.mark.parametrize('target', [ + x for x in spack.architecture.platform().targets +] + ["fe", "be", "frontend", "backend"]) +def test_user_input_combination(config, operating_system, target): platform = spack.architecture.platform() - os_list = list(platform.operating_sys.keys()) - target_list = list(platform.targets.keys()) - additional = ["fe", "be", "frontend", "backend"] - - os_list.extend(additional) - target_list.extend(additional) - - combinations = itertools.product(os_list, target_list) - results = [] - for arch in combinations: - o, t = arch - spec = Spec("libelf os=%s target=%s" % (o, t)) - spec.concretize() - results.append( - spec.architecture.os == str(platform.operating_system(o)) - ) - results.append( - spec.architecture.target == str(platform.target(t)) - ) - res = all(results) - assert res + spec = Spec("libelf os=%s target=%s" % (operating_system, target)) + spec.concretize() + assert spec.architecture.os == str( + platform.operating_system(operating_system) + ) + assert spec.architecture.target == platform.target(target) def test_operating_system_conversion_to_dict(): @@ -148,3 +140,32 @@ def test_operating_system_conversion_to_dict(): assert operating_system.to_dict() == { 'name': 'os', 'version': '1.0' } + + +@pytest.mark.parametrize('cpu_flag,target_name', [ + # Test that specific flags can be used in queries + ('ssse3', 'haswell'), + ('popcnt', 'nehalem'), + ('avx512f', 'skylake_avx512'), + ('avx512ifma', 'icelake'), + # Test that proxy flags can be used in queries too + ('sse3', 'nehalem'), + ('avx512', 'skylake_avx512'), + ('avx512', 'icelake'), +]) +def test_target_container_semantic(cpu_flag, target_name): + target = spack.architecture.Target(target_name) + assert cpu_flag in target + + +@pytest.mark.parametrize('item,architecture_str', [ + # We can search the architecture string representation + ('linux', 'linux-ubuntu18.04-haswell'), + ('ubuntu', 'linux-ubuntu18.04-haswell'), + ('haswell', 'linux-ubuntu18.04-haswell'), + # We can also search flags of the target, + ('avx512', 'linux-ubuntu18.04-icelake'), +]) +def test_arch_spec_container_semantic(item, architecture_str): + architecture = spack.spec.ArchSpec(architecture_str) + assert item in architecture diff --git a/lib/spack/spack/test/cmd/arch.py b/lib/spack/spack/test/cmd/arch.py index 43c1433a51..66c8f8e5dc 100644 --- a/lib/spack/spack/test/cmd/arch.py +++ b/lib/spack/spack/test/cmd/arch.py @@ -44,3 +44,7 @@ def test_arch_target(): arch('--target') arch('-f', '-t') arch('-b', '-t') + + +def test_display_targets(): + arch('--known-targets') diff --git a/lib/spack/spack/test/cmd/find.py b/lib/spack/spack/test/cmd/find.py index c457b8ffac..55706c264c 100644 --- a/lib/spack/spack/test/cmd/find.py +++ b/lib/spack/spack/test/cmd/find.py @@ -294,3 +294,9 @@ def test_find_no_sections(database, config): output = find("--no-groups") assert "-----------" not in output assert "==>" not in output + + +@pytest.mark.db +def test_find_command_basic_usage(database): + output = find() + assert 'mpileaks' in output diff --git a/lib/spack/spack/test/compilers.py b/lib/spack/spack/test/compilers.py index 0728d57699..171d390de6 100644 --- a/lib/spack/spack/test/compilers.py +++ b/lib/spack/spack/test/compilers.py @@ -70,7 +70,7 @@ def test_multiple_conflicting_compiler_definitions(mutable_config): compiler_config[0]['compiler']['paths']['f77'] = 'f77' mutable_config.update_config('compilers', compiler_config) - arch_spec = spack.spec.ArchSpec('test', 'test', 'test') + arch_spec = spack.spec.ArchSpec(('test', 'test', 'test')) cspec = compiler_config[0]['compiler']['spec'] cmp = compilers.compiler_for_spec(cspec, arch_spec) assert cmp.f77 == 'f77' diff --git a/lib/spack/spack/test/concretize.py b/lib/spack/spack/test/concretize.py index b82993dd8e..5243ba7681 100644 --- a/lib/spack/spack/test/concretize.py +++ b/lib/spack/spack/test/concretize.py @@ -16,6 +16,7 @@ from spack.spec import ConflictsInSpecError, SpecError from spack.version import ver from spack.test.conftest import MockPackage, MockPackageMultiRepo import spack.compilers +import spack.platforms.test def check_spec(abstract, concrete): @@ -81,6 +82,16 @@ def spec(request): return request.param +@pytest.fixture(params=[ + 'haswell', 'broadwell', 'skylake', 'icelake' +]) +def current_host(request, monkeypatch): + target = llnl.util.cpu.targets[request.param] + monkeypatch.setattr(llnl.util.cpu, 'host', lambda: target) + monkeypatch.setattr(spack.platforms.test.Test, 'default', request.param) + return target + + @pytest.mark.usefixtures('config', 'mock_packages') class TestConcretize(object): def test_concretize(self, spec): @@ -133,12 +144,12 @@ class TestConcretize(object): assert concrete['mpich2'].satisfies('mpich2@1.3.1:1.4') def test_concretize_enable_disable_compiler_existence_check(self): - with spack.concretize.concretizer.enable_compiler_existence_check(): + with spack.concretize.enable_compiler_existence_check(): with pytest.raises( spack.concretize.UnavailableCompilerVersionError): check_concretize('dttop %gcc@100.100') - with spack.concretize.concretizer.disable_compiler_existence_check(): + with spack.concretize.disable_compiler_existence_check(): spec = check_concretize('dttop %gcc@100.100') assert spec.satisfies('%gcc@100.100') assert spec['dtlink3'].satisfies('%gcc@100.100') @@ -273,7 +284,7 @@ class TestConcretize(object): def test_no_matching_compiler_specs(self, mock_config): # only relevant when not building compilers as needed - with spack.concretize.concretizer.enable_compiler_existence_check(): + with spack.concretize.enable_compiler_existence_check(): s = Spec('a %gcc@0.0.0') with pytest.raises( spack.concretize.UnavailableCompilerVersionError): @@ -571,3 +582,17 @@ class TestConcretize(object): """Test concretization failures for no-version packages.""" with pytest.raises(NoValidVersionError, match="no valid versions"): Spec(spec).concretized() + + @pytest.mark.parametrize('spec, best_achievable', [ + ('mpileaks%gcc@4.8', 'haswell'), + ('mpileaks%gcc@5.3.0', 'skylake_avx512') + ]) + def test_adjusting_default_target_based_on_compiler( + self, spec, best_achievable, current_host + ): + best_achievable = llnl.util.cpu.targets[best_achievable] + expected = best_achievable if best_achievable < current_host \ + else current_host + with spack.concretize.disable_compiler_existence_check(): + s = Spec(spec).concretized() + assert str(s.architecture.target) == str(expected) diff --git a/lib/spack/spack/test/data/modules/lmod/alter_environment.yaml b/lib/spack/spack/test/data/modules/lmod/alter_environment.yaml index caa01560f1..4936f4aa52 100644 --- a/lib/spack/spack/test/data/modules/lmod/alter_environment.yaml +++ b/lib/spack/spack/test/data/modules/lmod/alter_environment.yaml @@ -22,6 +22,6 @@ lmod: unset: - BAR - 'platform=test target=x86_32': + 'platform=test target=x86': load: - 'foo/bar' diff --git a/lib/spack/spack/test/data/modules/tcl/alter_environment.yaml b/lib/spack/spack/test/data/modules/tcl/alter_environment.yaml index a55ad94b0e..d3c7eec79e 100644 --- a/lib/spack/spack/test/data/modules/tcl/alter_environment.yaml +++ b/lib/spack/spack/test/data/modules/tcl/alter_environment.yaml @@ -17,6 +17,6 @@ tcl: unset: - BAR - 'platform=test target=x86_32': + 'platform=test target=x86': load: - 'foo/bar' diff --git a/lib/spack/spack/test/data/modules/tcl/invalid_token_in_env_var_name.yaml b/lib/spack/spack/test/data/modules/tcl/invalid_token_in_env_var_name.yaml index d536c994fa..bed866fe90 100644 --- a/lib/spack/spack/test/data/modules/tcl/invalid_token_in_env_var_name.yaml +++ b/lib/spack/spack/test/data/modules/tcl/invalid_token_in_env_var_name.yaml @@ -16,6 +16,6 @@ tcl: unset: - BAR - 'platform=test target=x86_32': + 'platform=test target=x86': load: - 'foo/bar' diff --git a/lib/spack/spack/test/data/targets/bgq-rhel6-power7 b/lib/spack/spack/test/data/targets/bgq-rhel6-power7 new file mode 100644 index 0000000000..39da8a49fe --- /dev/null +++ b/lib/spack/spack/test/data/targets/bgq-rhel6-power7 @@ -0,0 +1,4 @@ +processor : 0 +cpu : POWER7 (architected), altivec supported +clock : 3720.000000MHz +revision : 2.1 (pvr 003f 0201) diff --git a/lib/spack/spack/test/data/targets/darwin-mojave-broadwell b/lib/spack/spack/test/data/targets/darwin-mojave-broadwell new file mode 100644 index 0000000000..67dc28a966 --- /dev/null +++ b/lib/spack/spack/test/data/targets/darwin-mojave-broadwell @@ -0,0 +1,5 @@ +machdep.cpu.vendor: GenuineIntel +machdep.cpu.features: FPU VME DE PSE TSC MSR PAE MCE CX8 APIC SEP MTRR PGE MCA CMOV PAT PSE36 CLFSH DS ACPI MMX FXSR SSE SSE2 SS HTT TM PBE SSE3 PCLMULQDQ DTES64 MON DSCPL VMX EST TM2 SSSE3 FMA CX16 TPR PDCM SSE4.1 SSE4.2 x2APIC MOVBE POPCNT AES PCID XSAVE OSXSAVE SEGLIM64 TSCTMR AVX1.0 RDRAND F16C +machdep.cpu.leaf7_features: RDWRFSGS TSC_THREAD_OFFSET SGX BMI1 HLE AVX2 SMEP BMI2 ERMS INVPCID RTM FPU_CSDS MPX RDSEED ADX SMAP CLFSOPT IPT MDCLEAR TSXFA IBRS STIBP L1DF SSBD +machdep.cpu.model: 94 +machdep.cpu.brand_string: Intel(R) Core(TM) i7-6700K CPU @ 4.00GHz diff --git a/lib/spack/spack/test/data/targets/darwin-mojave-ivybridge b/lib/spack/spack/test/data/targets/darwin-mojave-ivybridge new file mode 100644 index 0000000000..6a52d6c19d --- /dev/null +++ b/lib/spack/spack/test/data/targets/darwin-mojave-ivybridge @@ -0,0 +1,59 @@ +machdep.cpu.max_basic: 13 +machdep.cpu.max_ext: 2147483656 +machdep.cpu.vendor: GenuineIntel +machdep.cpu.brand_string: Intel(R) Core(TM) i5-3230M CPU @ 2.60GHz +machdep.cpu.family: 6 +machdep.cpu.model: 58 +machdep.cpu.extmodel: 3 +machdep.cpu.extfamily: 0 +machdep.cpu.stepping: 9 +machdep.cpu.feature_bits: 9203919201183202303 +machdep.cpu.leaf7_feature_bits: 641 0 +machdep.cpu.leaf7_feature_bits_edx: 2617246720 +machdep.cpu.extfeature_bits: 4967106816 +machdep.cpu.signature: 198313 +machdep.cpu.brand: 0 +machdep.cpu.features: FPU VME DE PSE TSC MSR PAE MCE CX8 APIC SEP MTRR PGE MCA CMOV PAT PSE36 CLFSH DS ACPI MMX FXSR SSE SSE2 SS HTT TM PBE SSE3 PCLMULQDQ DTES64 MON DSCPL VMX EST TM2 SSSE3 CX16 TPR PDCM SSE4.1 SSE4.2 x2APIC POPCNT AES PCID XSAVE OSXSAVE TSCTMR AVX1.0 RDRAND F16C +machdep.cpu.leaf7_features: RDWRFSGS SMEP ERMS MDCLEAR IBRS STIBP L1DF SSBD +machdep.cpu.extfeatures: SYSCALL XD EM64T LAHF RDTSCP TSCI +machdep.cpu.logical_per_package: 16 +machdep.cpu.cores_per_package: 8 +machdep.cpu.microcode_version: 33 +machdep.cpu.processor_flag: 4 +machdep.cpu.mwait.linesize_min: 64 +machdep.cpu.mwait.linesize_max: 64 +machdep.cpu.mwait.extensions: 3 +machdep.cpu.mwait.sub_Cstates: 135456 +machdep.cpu.thermal.sensor: 1 +machdep.cpu.thermal.dynamic_acceleration: 1 +machdep.cpu.thermal.invariant_APIC_timer: 1 +machdep.cpu.thermal.thresholds: 2 +machdep.cpu.thermal.ACNT_MCNT: 1 +machdep.cpu.thermal.core_power_limits: 1 +machdep.cpu.thermal.fine_grain_clock_mod: 1 +machdep.cpu.thermal.package_thermal_intr: 1 +machdep.cpu.thermal.hardware_feedback: 0 +machdep.cpu.thermal.energy_policy: 0 +machdep.cpu.xsave.extended_state: 7 832 832 0 +machdep.cpu.xsave.extended_state1: 1 0 0 0 +machdep.cpu.arch_perf.version: 3 +machdep.cpu.arch_perf.number: 4 +machdep.cpu.arch_perf.width: 48 +machdep.cpu.arch_perf.events_number: 7 +machdep.cpu.arch_perf.events: 0 +machdep.cpu.arch_perf.fixed_number: 3 +machdep.cpu.arch_perf.fixed_width: 48 +machdep.cpu.cache.linesize: 64 +machdep.cpu.cache.L2_associativity: 8 +machdep.cpu.cache.size: 256 +machdep.cpu.tlb.inst.small: 64 +machdep.cpu.tlb.inst.large: 8 +machdep.cpu.tlb.data.small: 64 +machdep.cpu.tlb.data.large: 32 +machdep.cpu.tlb.shared: 512 +machdep.cpu.address_bits.physical: 36 +machdep.cpu.address_bits.virtual: 48 +machdep.cpu.core_count: 2 +machdep.cpu.thread_count: 4 +machdep.cpu.tsc_ccc.numerator: 0 +machdep.cpu.tsc_ccc.denominator: 0 diff --git a/lib/spack/spack/test/data/targets/linux-centos7-power8le b/lib/spack/spack/test/data/targets/linux-centos7-power8le new file mode 100644 index 0000000000..f21ca85323 --- /dev/null +++ b/lib/spack/spack/test/data/targets/linux-centos7-power8le @@ -0,0 +1,4 @@ +processor : 0 +cpu : POWER8NVL (raw), altivec supported +clock : 2061.000000MHz +revision : 1.0 (pvr 004c 0100) diff --git a/lib/spack/spack/test/data/targets/linux-rhel7-broadwell b/lib/spack/spack/test/data/targets/linux-rhel7-broadwell new file mode 100644 index 0000000000..427cd85bf2 --- /dev/null +++ b/lib/spack/spack/test/data/targets/linux-rhel7-broadwell @@ -0,0 +1,25 @@ +processor : 0 +vendor_id : GenuineIntel +cpu family : 6 +model : 79 +model name : Intel(R) Xeon(R) CPU E5-2690 v4 @ 2.60GHz +stepping : 1 +microcode : 0xb00002a +cpu MHz : 2600.000 +cache size : 35840 KB +physical id : 0 +siblings : 14 +core id : 0 +cpu cores : 14 +apicid : 0 +initial apicid : 0 +fpu : yes +fpu_exception : yes +cpuid level : 20 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc aperfmperf eagerfpu pni pclmulqdq dtes64 ds_cpl vmx smx est tm2 ssse3 fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch epb cat_l3 cdp_l3 invpcid_single intel_pt tpr_shadow vnmi flexpriority ept vpid fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm cqm rdt_a rdseed adx smap xsaveopt cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local ibpb ibrs stibp dtherm arat pln pts spec_ctrl intel_stibp +bogomips : 5188.06 +clflush size : 64 +cache_alignment : 64 +address sizes : 46 bits physical, 48 bits virtual +power management: diff --git a/lib/spack/spack/test/data/targets/linux-rhel7-haswell b/lib/spack/spack/test/data/targets/linux-rhel7-haswell new file mode 100644 index 0000000000..7bb5aa958f --- /dev/null +++ b/lib/spack/spack/test/data/targets/linux-rhel7-haswell @@ -0,0 +1,25 @@ +processor : 0 +vendor_id : GenuineIntel +cpu family : 6 +model : 63 +model name : Intel(R) Xeon(R) CPU E5-2680 v3 @ 2.50GHz +stepping : 2 +microcode : 0x3c +cpu MHz : 1757.910 +cache size : 30720 KB +physical id : 0 +siblings : 12 +core id : 0 +cpu cores : 12 +apicid : 0 +initial apicid : 0 +fpu : yes +fpu_exception : yes +cpuid level : 15 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc aperfmperf eagerfpu pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm epb invpcid_single tpr_shadow vnmi flexpriority ept vpid fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm xsaveopt cqm_llc cqm_occup_llc ibpb ibrs stibp dtherm arat pln pts spec_ctrl intel_stibp +bogomips : 4987.97 +clflush size : 64 +cache_alignment : 64 +address sizes : 46 bits physical, 48 bits virtual +power management: diff --git a/lib/spack/spack/test/data/targets/linux-rhel7-ivybridge b/lib/spack/spack/test/data/targets/linux-rhel7-ivybridge new file mode 100644 index 0000000000..7d6cfe4daa --- /dev/null +++ b/lib/spack/spack/test/data/targets/linux-rhel7-ivybridge @@ -0,0 +1,25 @@ +processor : 0 +vendor_id : GenuineIntel +cpu family : 6 +model : 62 +model name : Intel(R) Xeon(R) CPU E5-2650 v2 @ 2.60GHz +stepping : 4 +microcode : 0x42c +cpu MHz : 1862.554 +cache size : 20480 KB +physical id : 0 +siblings : 8 +core id : 0 +cpu cores : 8 +apicid : 0 +initial apicid : 0 +fpu : yes +fpu_exception : yes +cpuid level : 13 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc aperfmperf eagerfpu pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm epb tpr_shadow vnmi flexpriority ept vpid fsgsbase smep erms xsaveopt ibpb ibrs stibp dtherm ida arat pln pts spec_ctrl intel_stibp +bogomips : 5200.15 +clflush size : 64 +cache_alignment : 64 +address sizes : 46 bits physical, 48 bits virtual +power management: diff --git a/lib/spack/spack/test/data/targets/linux-rhel7-skylake_avx512 b/lib/spack/spack/test/data/targets/linux-rhel7-skylake_avx512 new file mode 100644 index 0000000000..e9ab1987dd --- /dev/null +++ b/lib/spack/spack/test/data/targets/linux-rhel7-skylake_avx512 @@ -0,0 +1,25 @@ +processor : 0 +vendor_id : GenuineIntel +cpu family : 6 +model : 85 +model name : Intel(R) Xeon(R) Gold 6132 CPU @ 2.60GHz +stepping : 4 +microcode : 0x200004d +cpu MHz : 2600.000 +cache size : 19712 KB +physical id : 0 +siblings : 14 +core id : 0 +cpu cores : 14 +apicid : 0 +initial apicid : 0 +fpu : yes +fpu_exception : yes +cpuid level : 22 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc aperfmperf eagerfpu pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch epb cat_l3 cdp_l3 invpcid_single intel_pt tpr_shadow vnmi flexpriority ept vpid fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm cqm mpx rdt_a avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local ibpb ibrs stibp dtherm ida arat pln pts spec_ctrl intel_stibp ssbd +bogomips : 5200.00 +clflush size : 64 +cache_alignment : 64 +address sizes : 46 bits physical, 48 bits virtual +power management: diff --git a/lib/spack/spack/test/data/targets/linux-rhel7-zen b/lib/spack/spack/test/data/targets/linux-rhel7-zen new file mode 100644 index 0000000000..014dec3359 --- /dev/null +++ b/lib/spack/spack/test/data/targets/linux-rhel7-zen @@ -0,0 +1,26 @@ +processor : 0 +vendor_id : AuthenticAMD +cpu family : 23 +model : 1 +model name : AMD EPYC 7301 16-Core Processor +stepping : 2 +microcode : 0x8001227 +cpu MHz : 2200.000 +cache size : 512 KB +physical id : 0 +siblings : 32 +core id : 0 +cpu cores : 16 +apicid : 0 +initial apicid : 0 +fpu : yes +fpu_exception : yes +cpuid level : 13 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc art rep_good nopl nonstop_tsc extd_apicid amd_dcm aperfmperf eagerfpu pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_l2 cpb hw_pstate retpoline_amd ssbd ibpb vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt sha_ni xsaveopt xsavec xgetbv1 clzero irperf xsaveerptr arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif overflow_recov succor smca +bogomips : 4399.40 +TLB size : 2560 4K pages +clflush size : 64 +cache_alignment : 64 +address sizes : 48 bits physical, 48 bits virtual +power management: ts ttp tm hwpstate cpb eff_freq_ro [13] [14] diff --git a/lib/spack/spack/test/data/targets/linux-ubuntu18.04-broadwell b/lib/spack/spack/test/data/targets/linux-ubuntu18.04-broadwell new file mode 100644 index 0000000000..bbe287ebf1 --- /dev/null +++ b/lib/spack/spack/test/data/targets/linux-ubuntu18.04-broadwell @@ -0,0 +1,26 @@ +processor : 0 +vendor_id : GenuineIntel +cpu family : 6 +model : 61 +model name : Intel(R) Core(TM) i5-5200U CPU @ 2.20GHz +stepping : 4 +microcode : 0x2d +cpu MHz : 1944.124 +cache size : 3072 KB +physical id : 0 +siblings : 4 +core id : 0 +cpu cores : 2 +apicid : 0 +initial apicid : 0 +fpu : yes +fpu_exception : yes +cpuid level : 20 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb invpcid_single pti ssbd ibrs ibpb stibp tpr_shadow vnmi flexpriority ept vpid fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid rdseed adx smap intel_pt xsaveopt dtherm ida arat pln pts md_clear flush_l1d +bugs : cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass l1tf mds +bogomips : 4389.80 +clflush size : 64 +cache_alignment : 64 +address sizes : 39 bits physical, 48 bits virtual +power management: diff --git a/lib/spack/spack/test/llnl/util/cpu.py b/lib/spack/spack/test/llnl/util/cpu.py new file mode 100644 index 0000000000..6f6b9fec4d --- /dev/null +++ b/lib/spack/spack/test/llnl/util/cpu.py @@ -0,0 +1,233 @@ +# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import pytest + +import contextlib +import os.path + +import jsonschema + +import llnl.util.cpu +import llnl.util.cpu.detect + +import spack.paths + +# This is needed to check that with repr we could create equivalent objects +from llnl.util.cpu import Microarchitecture # noqa + + +@pytest.fixture(params=[ + 'linux-ubuntu18.04-broadwell', + 'linux-rhel7-broadwell', + 'linux-rhel7-skylake_avx512', + 'linux-rhel7-ivybridge', + 'linux-rhel7-haswell', + 'linux-rhel7-zen', + 'linux-centos7-power8le', + 'darwin-mojave-ivybridge', + 'darwin-mojave-broadwell', + 'bgq-rhel6-power7' +]) +def expected_target(request, monkeypatch): + cpu = llnl.util.cpu + platform, operating_system, target = request.param.split('-') + + architecture_family = llnl.util.cpu.targets[target].family + monkeypatch.setattr( + cpu.detect.platform, 'machine', lambda: str(architecture_family) + ) + + # Monkeypatch for linux + if platform in ('linux', 'bgq'): + monkeypatch.setattr(cpu.detect.platform, 'system', lambda: 'Linux') + + @contextlib.contextmanager + def _open(not_used_arg): + filename = os.path.join( + spack.paths.test_path, 'data', 'targets', request.param + ) + with open(filename) as f: + yield f + + monkeypatch.setattr(cpu.detect, 'open', _open, raising=False) + + elif platform == 'darwin': + monkeypatch.setattr(cpu.detect.platform, 'system', lambda: 'Darwin') + + filename = os.path.join( + spack.paths.test_path, 'data', 'targets', request.param + ) + info = {} + with open(filename) as f: + for line in f: + key, value = line.split(':') + info[key.strip()] = value.strip() + + def _check_output(args): + current_key = args[-1] + return info[current_key] + + monkeypatch.setattr(cpu.detect, 'check_output', _check_output) + + return llnl.util.cpu.targets[target] + + +@pytest.fixture(params=[x for x in llnl.util.cpu.targets]) +def supported_target(request): + return request.param + + +def test_target_detection(expected_target): + detected_target = llnl.util.cpu.host() + assert detected_target == expected_target + + +def test_no_dashes_in_target_names(supported_target): + assert '-' not in supported_target + + +def test_str_conversion(supported_target): + assert supported_target == str(llnl.util.cpu.targets[supported_target]) + + +def test_repr_conversion(supported_target): + target = llnl.util.cpu.targets[supported_target] + assert eval(repr(target)) == target + + +def test_equality(supported_target): + target = llnl.util.cpu.targets[supported_target] + + for name, other_target in llnl.util.cpu.targets.items(): + if name == supported_target: + assert other_target == target + else: + assert other_target != target + + +@pytest.mark.parametrize('operation,expected_result', [ + # Test microarchitectures that are ordered with respect to each other + ('x86_64 < skylake', True), + ('icelake > skylake', True), + ('piledriver <= zen', True), + ('zen2 >= zen', True), + ('zen >= zen', True), + # Test unrelated microarchitectures + ('power8 < skylake', False), + ('power8 <= skylake', False), + ('skylake < power8', False), + ('skylake <= power8', False), + # Test microarchitectures of the same family that are not a "subset" + # of each other + ('cascadelake > cannonlake', False), + ('cascadelake < cannonlake', False), + ('cascadelake <= cannonlake', False), + ('cascadelake >= cannonlake', False), + ('cascadelake == cannonlake', False), + ('cascadelake != cannonlake', True) +]) +def test_partial_ordering(operation, expected_result): + target, operator, other_target = operation.split() + target = llnl.util.cpu.targets[target] + other_target = llnl.util.cpu.targets[other_target] + code = 'target ' + operator + 'other_target' + assert eval(code) is expected_result + + +@pytest.mark.parametrize('target_name,expected_family', [ + ('skylake', 'x86_64'), + ('zen', 'x86_64'), + ('pentium2', 'x86'), +]) +def test_architecture_family(target_name, expected_family): + target = llnl.util.cpu.targets[target_name] + assert str(target.family) == expected_family + + +@pytest.mark.parametrize('target_name,feature', [ + ('skylake', 'avx2'), + ('icelake', 'avx512f'), + # Test feature aliases + ('icelake', 'avx512'), + ('skylake', 'sse3'), + ('power8', 'altivec'), + ('broadwell', 'sse4.1'), +]) +def test_features_query(target_name, feature): + target = llnl.util.cpu.targets[target_name] + assert feature in target + + +@pytest.mark.parametrize('target_name,wrong_feature', [ + ('skylake', 1), + ('bulldozer', llnl.util.cpu.targets['x86_64']) +]) +def test_wrong_types_for_features_query(target_name, wrong_feature): + target = llnl.util.cpu.targets[target_name] + with pytest.raises(TypeError, match='only objects of string types'): + assert wrong_feature in target + + +def test_generic_microarchitecture(): + generic_march = llnl.util.cpu.generic_microarchitecture('foo') + + assert generic_march.name == 'foo' + assert not generic_march.features + assert not generic_march.ancestors + assert generic_march.vendor == 'generic' + + +def test_target_json_schema(): + # The file microarchitectures.json contains static data i.e. data that is + # not meant to be modified by users directly. It is thus sufficient to + # validate it only once during unit tests. + json_data = llnl.util.cpu.schema.targets_json.data + jsonschema.validate(json_data, llnl.util.cpu.schema.schema) + + +@pytest.mark.parametrize('target_name,compiler,version,expected_flags', [ + ('x86_64', 'gcc', '4.9.3', '-march=x86-64 -mtune=x86-64'), + ('nocona', 'gcc', '4.9.3', '-march=nocona -mtune=nocona'), + ('nehalem', 'gcc', '4.9.3', '-march=nehalem -mtune=nehalem'), + ('nehalem', 'gcc', '4.8.5', '-march=corei7 -mtune=corei7'), + ('sandybridge', 'gcc', '4.8.5', '-march=corei7-avx -mtune=corei7-avx'), + # Test that an unknown compiler returns an empty string + ('sandybridge', 'unknown', '4.8.5', ''), +]) +def test_optimization_flags(target_name, compiler, version, expected_flags): + target = llnl.util.cpu.targets[target_name] + flags = target.optimization_flags(compiler, version) + assert flags == expected_flags + + +@pytest.mark.parametrize('target_name,compiler,version', [ + ('excavator', 'gcc', '4.8.5') +]) +def test_unsupported_optimization_flags(target_name, compiler, version): + target = llnl.util.cpu.targets[target_name] + with pytest.raises( + llnl.util.cpu.UnsupportedMicroarchitecture, + matches='cannot produce optimized binary' + ): + target.optimization_flags(compiler, version) + + +@pytest.mark.parametrize('operation,expected_result', [ + # In the tests below we won't convert the right hand side to + # Microarchitecture, so that automatic conversion from a known + # target name will be tested + ('cascadelake > cannonlake', False), + ('cascadelake < cannonlake', False), + ('cascadelake <= cannonlake', False), + ('cascadelake >= cannonlake', False), + ('cascadelake == cannonlake', False), + ('cascadelake != cannonlake', True) +]) +def test_automatic_conversion_on_comparisons(operation, expected_result): + target, operator, other_target = operation.split() + target = llnl.util.cpu.targets[target] + code = 'target ' + operator + 'other_target' + assert eval(code) is expected_result diff --git a/lib/spack/spack/test/modules/lmod.py b/lib/spack/spack/test/modules/lmod.py index c9570c9a56..7ee173d660 100644 --- a/lib/spack/spack/test/modules/lmod.py +++ b/lib/spack/spack/test/modules/lmod.py @@ -124,7 +124,7 @@ class TestLmod(object): assert len([x for x in content if 'unsetenv("BAR")' in x]) == 1 content = modulefile_content( - 'libdwarf %clang platform=test target=x86_32' + 'libdwarf %clang platform=test target=x86' ) assert len( diff --git a/lib/spack/spack/test/modules/tcl.py b/lib/spack/spack/test/modules/tcl.py index 663fc86570..7bb058cd13 100644 --- a/lib/spack/spack/test/modules/tcl.py +++ b/lib/spack/spack/test/modules/tcl.py @@ -112,7 +112,7 @@ class TestTcl(object): assert len([x for x in content if 'setenv MPILEAKS_ROOT' in x]) == 1 content = modulefile_content( - 'libdwarf %clang platform=test target=x86_32' + 'libdwarf %clang platform=test target=x86' ) assert len([x for x in content diff --git a/lib/spack/spack/test/spec_semantics.py b/lib/spack/spack/test/spec_semantics.py index c3e18af44d..0c52bd1c10 100644 --- a/lib/spack/spack/test/spec_semantics.py +++ b/lib/spack/spack/test/spec_semantics.py @@ -183,13 +183,13 @@ class TestSpecSematics(object): check_unsatisfiable( 'foo platform=linux', - 'platform=test os=redhat6 target=x86_32') + 'platform=test os=redhat6 target=x86') check_unsatisfiable( 'foo os=redhat6', 'platform=test os=debian6 target=x86_64') check_unsatisfiable( 'foo target=x86_64', - 'platform=test os=redhat6 target=x86_32') + 'platform=test os=redhat6 target=x86') check_satisfies( 'foo arch=test-None-None', @@ -217,8 +217,8 @@ class TestSpecSematics(object): 'foo platform=test target=default_target os=default_os', 'platform=test os=default_os') check_unsatisfiable( - 'foo platform=test target=x86_32 os=redhat6', - 'platform=linux target=x86_32 os=redhat6') + 'foo platform=test target=x86 os=redhat6', + 'platform=linux target=x86 os=redhat6') def test_satisfies_dependencies(self): check_satisfies('mpileaks^mpich', '^mpich') @@ -944,3 +944,35 @@ class TestSpecSematics(object): with pytest.raises(SpecError): spec.prefix + + def test_forwarding_of_architecture_attributes(self): + spec = Spec('libelf').concretized() + + # Check that we can still access each member through + # the architecture attribute + assert 'test' in spec.architecture + assert 'debian' in spec.architecture + assert 'x86_64' in spec.architecture + + # Check that we forward the platform and os attribute correctly + assert spec.platform == 'test' + assert spec.os == 'debian6' + + # Check that the target is also forwarded correctly and supports + # all the operators we expect + assert spec.target == 'x86_64' + assert spec.target.family == 'x86_64' + assert 'avx512' not in spec.target + assert spec.target < 'broadwell' + + @pytest.mark.parametrize('spec,constraint,expected_result', [ + ('libelf target=haswell', 'target=broadwell', False), + ('libelf target=haswell', 'target=haswell', True), + ('libelf target=haswell', 'target=x86_64:', True), + ('libelf target=haswell', 'target=:haswell', True), + ('libelf target=haswell', 'target=icelake,:nocona', False), + ('libelf target=haswell', 'target=haswell,:nocona', True), + ]) + def test_target_constraints(self, spec, constraint, expected_result): + s = Spec(spec) + assert s.satisfies(constraint) is expected_result diff --git a/lib/spack/spack/test/spec_syntax.py b/lib/spack/spack/test/spec_syntax.py index 238b21f535..a87fa5188d 100644 --- a/lib/spack/spack/test/spec_syntax.py +++ b/lib/spack/spack/test/spec_syntax.py @@ -105,6 +105,7 @@ class TestSpecSyntax(object): """Check that the provided spec parses to the provided token list.""" spec = shlex.split(str(spec)) lex_output = sp.SpecLexer().lex(spec) + assert len(tokens) == len(lex_output), "unexpected number of tokens" for tok, spec_tok in zip(tokens, lex_output): if tok.type == sp.ID or tok.type == sp.VAL: assert tok == spec_tok @@ -184,7 +185,7 @@ class TestSpecSyntax(object): self.check_parse( "mvapich_foo" " ^_openmpi@1.2:1.4,1.6%intel@12.1 debug=2 ~qt_4" - " ^stackwalker@8.1_1e arch=test-redhat6-x86_32") + " ^stackwalker@8.1_1e arch=test-redhat6-x86") def test_canonicalize(self): self.check_parse( @@ -749,3 +750,16 @@ class TestSpecSyntax(object): "mvapich_foo debug= 4 " "^ _openmpi @1.2 : 1.4 , 1.6 % intel @ 12.1 : 12.6 + debug - qt_4 " "^ stackwalker @ 8.1_1e") + + @pytest.mark.parametrize('expected_tokens,spec_string', [ + ([Token(sp.ID, 'target'), + Token(sp.EQ, '='), + Token(sp.VAL, 'broadwell')], + 'target=broadwell'), + ([Token(sp.ID, 'target'), + Token(sp.EQ, '='), + Token(sp.VAL, ':broadwell,icelake')], + 'target=:broadwell,icelake') + ]) + def test_target_tokenization(self, expected_tokens, spec_string): + self.check_lex(expected_tokens, spec_string) |