summaryrefslogtreecommitdiff
path: root/lib
diff options
context:
space:
mode:
Diffstat (limited to 'lib')
-rwxr-xr-xlib/spack/env/cc4
-rw-r--r--lib/spack/llnl/util/cpu_name.py226
-rw-r--r--lib/spack/spack/architecture.py3
-rw-r--r--lib/spack/spack/build_environment.py23
-rw-r--r--lib/spack/spack/cmd/__init__.py3
-rw-r--r--lib/spack/spack/compiler.py15
-rw-r--r--lib/spack/spack/compilers/__init__.py3
-rw-r--r--lib/spack/spack/package.py8
-rw-r--r--lib/spack/spack/platforms/darwin.py33
-rw-r--r--lib/spack/spack/platforms/linux.py36
-rw-r--r--lib/spack/spack/test/build_environment.py10
-rw-r--r--lib/spack/spack/test/cc.py3
12 files changed, 350 insertions, 17 deletions
diff --git a/lib/spack/env/cc b/lib/spack/env/cc
index c7ea8c793b..c224652a34 100755
--- a/lib/spack/env/cc
+++ b/lib/spack/env/cc
@@ -32,6 +32,7 @@ parameters=(
SPACK_CXX_RPATH_ARG
SPACK_F77_RPATH_ARG
SPACK_FC_RPATH_ARG
+ SPACK_TARGET_ARGS
SPACK_SHORT_SPEC
SPACK_SYSTEM_DIRS
)
@@ -78,7 +79,7 @@ function system_dir {
}
for param in "${parameters[@]}"; do
- if [[ -z ${!param} ]]; then
+ if [[ -z ${!param+x} ]]; then
die "Spack compiler must be run from Spack! Input '$param' is missing."
fi
done
@@ -373,6 +374,7 @@ case "$mode" in
CXX)
flags=("${flags[@]}" "${SPACK_CXXFLAGS[@]}") ;;
esac
+ args=(${SPACK_TARGET_ARGS[@]} "${args[@]}")
;;
esac
diff --git a/lib/spack/llnl/util/cpu_name.py b/lib/spack/llnl/util/cpu_name.py
new file mode 100644
index 0000000000..9687252ab9
--- /dev/null
+++ b/lib/spack/llnl/util/cpu_name.py
@@ -0,0 +1,226 @@
+# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+import platform
+import re
+import subprocess
+import sys
+
+
+# Tuple of name, flags added, flags removed (default [])
+_intel_32 = [
+ ('i686', []),
+ ('pentium2', ['mmx']),
+ ('pentium3', ['sse']),
+ ('pentium4', ['sse2']),
+ ('prescott', ['sse3']),
+ ]
+
+_intel_64 = [ # commenting out the ones that aren't shown through sysctl
+ ('nocona', ['mmx', 'sse', 'sse2', 'sse3']),#lm
+ ('core2', ['ssse3'], ['sse3']),
+ ('nehalem', ['sse4_1', 'sse4_2', 'popcnt']),
+ ('westmere', ['aes', 'pclmulqdq']),
+ ('sandybridge', ['avx']),
+ ('ivybridge', ['rdrand', 'f16c']),#fsgsbase (is it RDWRFSGS on darwin?)
+ ('haswell', ['movbe', 'fma', 'avx2', 'bmi1', 'bmi2']),
+ ('broadwell', ['rdseed', 'adx']),
+ ('skylake', ['xsavec', 'xsaves'])
+ ]
+
+# We will need to build on these and combine with names when intel releases
+# further avx512 processors.
+# _intel_avx12 = ['avx512f', 'avx512cd']
+
+
+_amd_10_names = [
+ ('barcelona', ['mmx', 'sse', 'sse2', 'sse3', 'sse4a', 'abm'])
+ ]
+
+_amd_14_names = [
+ ('btver1', ['mmx', 'sse', 'sse2', 'sse3', 'ssse3', 'sse4a', 'cx16',
+ 'abm']),#lm
+ ]
+
+_amd_15_names = [
+ ('bdver1', ['avx', 'aes', 'pclmulqdq', 'cx16', 'sse', 'sse2', 'sse3',
+ 'ssse3', 'sse4a', 'sse4_1', 'sse4_2', 'abm']),#xop, lwp
+ ('bdver2', ['bmi1', 'f16c', 'fma',]),#tba?
+ ('bdver3', ['fsgsbase']),
+ ('bdver4', ['bmi2', 'movbe', 'avx2'])
+ ]
+
+_amd_16_names = [
+ ('btver2', ['mmx', 'sse', 'sse2', 'sse3', 'ssse3', 'sse4a', 'cx16',
+ 'abm', 'movbe', 'f16c', 'bmi1', 'avx', 'pclmulqdq',
+ 'aes', 'sse4_1', 'sse4_2']),#lm
+ ]
+
+_amd_17_names = [
+ ('znver1', ['bmi1', 'bmi2', 'f16c', 'fma', 'fsgsbase', 'avx', 'avx2',
+ 'rdseed', 'mwaitx', 'clzero', 'aes', 'pclmulqdq', 'cx16',
+ 'movbe', 'mmx', 'sse', 'sse2', 'sse3', 'ssse3', 'sse4a',
+ 'sse4_1', 'sse4_2', 'abm', 'xsavec', 'xsaves',
+ 'clflushopt', 'popcnt', 'adcx'])
+ ]
+
+_amd_numbers = {
+ 0x10: _amd_10_names,
+ 0x14: _amd_14_names,
+ 0x15: _amd_15_names,
+ 0x16: _amd_16_names,
+ 0x17: _amd_17_names
+ }
+
+def supported_target_names():
+ intel_names = set(t[0] for t in _intel_64)
+ intel_names |= set(t[0] for t in _intel_32)
+ amd_names = set()
+ for family in _amd_numbers:
+ amd_names |= set(t[0] for t in _amd_numbers[family])
+ power_names = set('power' + str(d) for d in range(7, 10))
+ return intel_names | amd_names | power_names
+
+def create_dict_from_cpuinfo():
+ # Initialize cpuinfo from file
+ cpuinfo = {}
+ try:
+ with open('/proc/cpuinfo') as file:
+ text = file.readlines()
+ for line in text:
+ if line.strip():
+ key, _, value = line.partition(':')
+ cpuinfo[key.strip()] = value.strip()
+ except IOError:
+ return None
+ return cpuinfo
+
+def check_output(args):
+ if sys.version_info >= (3, 0):
+ return subprocess.run(args, check=True, stdout=PIPE).stdout # nopyqver
+ else:
+ return subprocess.check_output(args) # nopyqver
+
+def create_dict_from_sysctl():
+ cpuinfo = {}
+ try:
+ cpuinfo['vendor_id'] = check_output(['sysctl', '-n',
+ 'machdep.cpu.vendor']).strip()
+ cpuinfo['flags'] = check_output(['sysctl', '-n',
+ 'machdep.cpu.features']).strip().lower()
+ cpuinfo['flags'] += ' ' + check_output(['sysctl', '-n',
+ 'machdep.cpu.leaf7_features']).strip().lower()
+ cpuinfo['model'] = check_output(['sysctl', '-n',
+ 'machdep.cpu.model']).strip()
+ cpuinfo['model name'] = check_output(['sysctl', '-n',
+ 'machdep.cpu.brand_string']).strip()
+
+ # Super hacky way to deal with slight representation differences
+ # Would be better to somehow consider these "identical"
+ if 'sse4.1' in cpuinfo['flags']:
+ cpuinfo['flags'] += ' sse4_1'
+ if 'sse4.2' in cpuinfo['flags']:
+ cpuinfo['flags'] += ' sse4_2'
+ if 'avx1.0' in cpuinfo['flags']:
+ cpuinfo['flags'] += ' avx'
+ except:
+ pass
+ return cpuinfo
+
+def get_cpu_name():
+ name = get_cpu_name_helper(platform.system())
+ return name if name else platform.machine()
+
+def get_cpu_name_helper(system):
+ # TODO: Elsewhere create dict of codenames (targets) and flag sets.
+ # Return cpu name or an empty string if one cannot be determined.
+ cpuinfo = {}
+ if system == 'Linux':
+ cpuinfo = create_dict_from_cpuinfo()
+ elif system == 'Darwin':
+ cpuinfo = create_dict_from_sysctl()
+ if not cpuinfo:
+ return ''
+
+ if 'vendor_id' in cpuinfo and cpuinfo['vendor_id'] == 'GenuineIntel':
+ if 'model name' not in cpuinfo or 'flags' not in cpuinfo:
+ # We don't have the information we need to determine the
+ # microarchitecture name
+ return ''
+ return get_intel_cpu_name(cpuinfo)
+ elif 'vendor_id' in cpuinfo and cpuinfo['vendor_id'] == 'AuthenticAMD':
+ if 'cpu family' not in cpuinfo or 'flags' not in cpuinfo:
+ # We don't have the information we need to determine the
+ # microarchitecture name
+ return ''
+ return get_amd_cpu_name(cpuinfo)
+ elif 'cpu' in cpuinfo and 'POWER' in cpuinfo['cpu']:
+ return get_ibm_cpu_name(cpuinfo['cpu'])
+ else:
+ return ''
+
+def get_ibm_cpu_name(cpu):
+ power_pattern = re.compile('POWER(\d+)')
+ power_match = power_pattern.search(cpu)
+ if power_match:
+ if 'le' in platform.machine():
+ return 'power' + power_match.group(1) + 'le'
+ return 'power' + power_match.group(1)
+ else:
+ return ''
+
+def get_intel_cpu_name(cpuinfo):
+ model_name = cpuinfo['model name']
+ if 'Atom' in model_name:
+ return 'atom'
+ elif 'Quark' in model_name:
+ return 'quark'
+ elif 'Xeon' in model_name and 'Phi' in model_name:
+ # This is hacky and needs to be extended for newer avx512 chips
+ return 'knl'
+ else:
+ ret = ''
+ flag_list = cpuinfo['flags'].split()
+ proc_flags = []
+ for _intel_processors in [_intel_32, _intel_64]:
+ for entry in _intel_processors:
+ try:
+ proc, flags_added, flags_removed = entry
+ except ValueError:
+ proc, flags_added = entry
+ flags_removed = []
+ proc_flags = list(filter(lambda x: x not in flags_removed, proc_flags))
+ proc_flags.extend(flags_added)
+ if all(f in flag_list for f in proc_flags):
+ ret = proc
+ return ret
+
+def get_amd_cpu_name(cpuinfo):
+ #TODO: Learn what the "canonical" granularity of naming
+ # is for AMD processors, implement dict as for intel.
+ ret = ''
+ flag_list = cpuinfo['flags'].split()
+ model_number = int(cpuinfo['cpu family'])
+ flags_dict = _amd_numbers[model_number]
+ proc_flags = []
+ for proc, proc_flags_added in flags_dict:
+ proc_flags.extend(proc_flags_added)
+ if all(f in flag_list for f in proc_flags):
+ ret = proc
+ else:
+ break
+ return ret
+
+"""IDEA: In build_environment.setup_compiler_environment, include a
+call to compiler.tuning_flags(spec.architecture.target). For gcc this
+would return "-march=%s" % str(spec.architecture.target). We only call
+this if the target is a valid tuning target (I.e. not
+platform.machine(), but a more specific target we successfully
+discovered.
+
+Then set
+SPACK_TUNING_FLAGS=compiler.tuning_flags(spec.architecture.target)
+This way the compiler wrapper can just add $SPACK_TUNING_FLAGS to the
+eventual command."""
diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py
index aded5290d8..4234ea35a9 100644
--- a/lib/spack/spack/architecture.py
+++ b/lib/spack/spack/architecture.py
@@ -60,6 +60,7 @@ import inspect
import llnl.util.tty as tty
from llnl.util.lang import memoized, list_modules, key_ordering
+from llnl.util.cpu_name import get_cpu_name
import spack.compiler
import spack.paths
@@ -226,7 +227,7 @@ class OperatingSystem(object):
return self.__str__()
def _cmp_key(self):
- return self.name, self.version
+ return (self.name, self.version)
def to_dict(self):
return {
diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py
index 877ae09ab5..1bd27f26a3 100644
--- a/lib/spack/spack/build_environment.py
+++ b/lib/spack/spack/build_environment.py
@@ -168,6 +168,7 @@ def clean_environment():
def set_compiler_environment_variables(pkg, env):
assert pkg.spec.concrete
compiler = pkg.compiler
+ spec = pkg.spec
# Set compiler variables used by CMake and autotools
assert all(key in compiler.link_paths for key in (
@@ -199,6 +200,24 @@ def set_compiler_environment_variables(pkg, env):
env.set('SPACK_F77_RPATH_ARG', compiler.f77_rpath_arg)
env.set('SPACK_FC_RPATH_ARG', compiler.fc_rpath_arg)
+ # Set the tuning parameters that the compiler will add
+ isa_target = compiler.isa_name_for_target(spec.architecture.target)
+ if spec.variants['tuning'].value == 'generic':
+ tuning_target = 'generic'
+ else:
+ tuning_target = compiler.tuning_name_for_target(
+ spec.architecture.target
+ )
+ if compiler.isa_flag and isa_target:
+ isa_arg = '{0}={1}'.format(compiler.isa_flag, isa_target)
+ else:
+ isa_arg = ''
+ if compiler.tuning_flag and tuning_target:
+ tuning_arg = '{0}={1}'.format(compiler.tuning_flag, tuning_target)
+ else:
+ tuning_arg = ''
+ env.set('SPACK_TARGET_ARGS', '{0} {1}'.format(isa_arg, tuning_arg))
+
# Trap spack-tracked compiler flags as appropriate.
# env_flags are easy to accidentally override.
inject_flags = {}
@@ -217,7 +236,7 @@ def set_compiler_environment_variables(pkg, env):
handler = pkg.flag_handler.__func__
else:
handler = pkg.flag_handler.im_func
- injf, envf, bsf = handler(pkg, flag, pkg.spec.compiler_flags[flag])
+ injf, envf, bsf = handler(pkg, flag, spec.compiler_flags[flag])
inject_flags[flag] = injf or []
env_flags[flag] = envf or []
build_system_flags[flag] = bsf or []
@@ -234,7 +253,7 @@ def set_compiler_environment_variables(pkg, env):
env.set(flag.upper(), ' '.join(f for f in env_flags[flag]))
pkg.flags_to_build_system_args(build_system_flags)
- env.set('SPACK_COMPILER_SPEC', str(pkg.spec.compiler))
+ env.set('SPACK_COMPILER_SPEC', str(spec.compiler))
env.set('SPACK_SYSTEM_DIRS', ':'.join(system_dirs))
diff --git a/lib/spack/spack/cmd/__init__.py b/lib/spack/spack/cmd/__init__.py
index cf6e3b7c3e..f4d3b18c9f 100644
--- a/lib/spack/spack/cmd/__init__.py
+++ b/lib/spack/spack/cmd/__init__.py
@@ -132,7 +132,8 @@ def parse_specs(args, **kwargs):
tests = kwargs.get('tests', False)
try:
- specs = spack.spec.parse(args)
+ sargs = args if isinstance(args, basestring) else ' '.join(args)
+ specs = spack.spec.parse(sargs)
for spec in specs:
if concretize:
spec.concretize(tests=tests) # implies normalize
diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py
index b9f68f80f1..18ec3bc71c 100644
--- a/lib/spack/spack/compiler.py
+++ b/lib/spack/spack/compiler.py
@@ -221,6 +221,15 @@ class Compiler(object):
@property
def fc_rpath_arg(self):
return '-Wl,-rpath,'
+
+ @property
+ def isa_flag(self):
+ return '-march'
+
+ @property
+ def tuning_flag(self):
+ return '-mtune'
+
# Cray PrgEnv name that can be used to load this compiler
PrgEnv = None
# Name of module used to switch versions of this compiler
@@ -419,6 +428,12 @@ class Compiler(object):
def fc_version(cls, fc):
return cls.default_version(fc)
+ def isa_name_for_target(self, target):
+ return str(target)
+
+ def tuning_name_for_target(self, target):
+ return str(target)
+
@classmethod
def search_regexps(cls, language):
# Compile all the regular expressions used for files beforehand.
diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py
index 929cd07ae9..bc2d06e954 100644
--- a/lib/spack/spack/compilers/__init__.py
+++ b/lib/spack/spack/compilers/__init__.py
@@ -16,6 +16,7 @@ import six
import llnl.util.lang
import llnl.util.filesystem as fs
import llnl.util.tty as tty
+from llnl.util.cpu_name import get_cpu_name
import spack.paths
import spack.error
@@ -646,7 +647,7 @@ def make_compiler_list(detected_versions):
spec = spack.spec.CompilerSpec(compiler_cls.name, version)
paths = [paths.get(l, None) for l in ('cc', 'cxx', 'f77', 'fc')]
compiler = compiler_cls(
- spec, operating_system, py_platform.machine(), paths
+ spec, operating_system, get_cpu_name(), paths
)
return [compiler]
diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py
index 50e483ffd6..dd700bd646 100644
--- a/lib/spack/spack/package.py
+++ b/lib/spack/spack/package.py
@@ -61,6 +61,10 @@ from spack.util.environment import dump_environment
from spack.util.package_hash import package_hash
from spack.version import Version
from spack.package_prefs import get_package_dir_permissions, get_package_group
+from spack.directives import variant
+
+"""Allowed URL schemes for spack packages."""
+_ALLOWED_URL_SCHEMES = ["http", "https", "ftp", "file", "git"]
# Filename for the Spack build/install log.
@@ -504,6 +508,10 @@ class PackageBase(with_metaclass(PackageMeta, PackageViewMixin, object)):
metadata_attrs = ['homepage', 'url', 'list_url', 'extendable', 'parallel',
'make_jobs']
+ # Add the universal variant "tuning" with values generic | specific
+ variant('tuning', values=('generic', 'specific'), default='generic',
+ description='Set compiler tuning generic or to target')
+
def __init__(self, spec):
# this determines how the package should be built.
self.spec = spec
diff --git a/lib/spack/spack/platforms/darwin.py b/lib/spack/spack/platforms/darwin.py
index 4ef18754af..2d580d8850 100644
--- a/lib/spack/spack/platforms/darwin.py
+++ b/lib/spack/spack/platforms/darwin.py
@@ -4,19 +4,44 @@
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
import platform
+from llnl.util.cpu_name import get_cpu_name
from spack.architecture import Platform, Target
from spack.operating_systems.mac_os import MacOs
class Darwin(Platform):
priority = 89
- front_end = 'x86_64'
- back_end = 'x86_64'
- default = 'x86_64'
def __init__(self):
super(Darwin, self).__init__('darwin')
- self.add_target(self.default, Target(self.default))
+
+ # TODO: These are probably overkill
+ # Add Intel architectures
+ self.add_target('haswell', Target('haswell'))
+ self.add_target('broadwell', Target('broadwell'))
+ self.add_target('ivybridge', Target('ivybridge'))
+ self.add_target('sandybridge', Target('sandybridge'))
+ self.add_target('core2', Target('core2'))
+
+ # Add "basic" architectures
+ self.add_target('x86_64', Target('x86_64'))
+ self.add_target('ppc64le', Target('ppc64le'))
+ self.add_target('ppc64', Target('ppc64'))
+
+ # Add IBM architectures
+ self.add_target('power7', Target('power7'))
+ self.add_target('power8', Target('power8'))
+ self.add_target('power8le', Target('power8le'))
+ self.add_target('power9', Target('power9'))
+ self.add_target('power9le', Target('power9le'))
+
+ self.default = get_cpu_name()
+ self.front_end = self.default
+ self.back_end = self.default
+
+ if self.default not in self.targets:
+ self.add_target(self.default, Target(self.default))
+
mac_os = MacOs()
self.default_os = str(mac_os)
diff --git a/lib/spack/spack/platforms/linux.py b/lib/spack/spack/platforms/linux.py
index b08b8e279f..23f10358fb 100644
--- a/lib/spack/spack/platforms/linux.py
+++ b/lib/spack/spack/platforms/linux.py
@@ -6,19 +6,45 @@
import platform
from spack.architecture import Platform, Target
from spack.operating_systems.linux_distro import LinuxDistro
-
+from llnl.util.cpu_name import get_cpu_name
class Linux(Platform):
priority = 90
def __init__(self):
super(Linux, self).__init__('linux')
+
+ # Add "basic" architectures
self.add_target('x86_64', Target('x86_64'))
self.add_target('ppc64le', Target('ppc64le'))
-
- self.default = platform.machine()
- self.front_end = platform.machine()
- self.back_end = platform.machine()
+ self.add_target('ppc64', Target('ppc64'))
+
+ # Add Intel architectures
+ self.add_target('haswell', Target('haswell'))
+ self.add_target('broadwell', Target('broadwell'))
+ self.add_target('ivybridge', Target('ivybridge'))
+ self.add_target('sandybridge', Target('sandybridge'))
+ self.add_target('knl', Target('knl'))
+
+ # Add IBM architectures
+ self.add_target('power7', Target('power7'))
+ self.add_target('power8', Target('power8'))
+ self.add_target('power8le', Target('power8le'))
+ self.add_target('power9', Target('power9'))
+ self.add_target('power9le', Target('power9le'))
+ # Eternal TODO: Add more architectures as needed.
+
+ # Get specific default
+ self.default = get_cpu_name()
+ self.front_end = self.default
+ self.back_end = self.default
+
+ if not self.default:
+ # Fall back on more general name.
+ # This will likely fall in "basic" architectures list
+ self.default = platform.machine()
+ self.front_end = self.default
+ self.back_end = self.default
if self.default not in self.targets:
self.add_target(self.default, Target(self.default))
diff --git a/lib/spack/spack/test/build_environment.py b/lib/spack/spack/test/build_environment.py
index 1146073db5..e30e4a7c16 100644
--- a/lib/spack/spack/test/build_environment.py
+++ b/lib/spack/spack/test/build_environment.py
@@ -42,14 +42,22 @@ def build_environment(working_env):
os.environ['SPACK_CXX_RPATH_ARG'] = "-Wl,-rpath,"
os.environ['SPACK_F77_RPATH_ARG'] = "-Wl,-rpath,"
os.environ['SPACK_FC_RPATH_ARG'] = "-Wl,-rpath,"
-
os.environ['SPACK_SYSTEM_DIRS'] = '/usr/include /usr/lib'
+ os.environ['SPACK_TARGET_ARGS'] = ''
if 'SPACK_DEPENDENCIES' in os.environ:
del os.environ['SPACK_DEPENDENCIES']
yield {'cc': cc, 'cxx': cxx, 'fc': fc}
+ for name in ('SPACK_CC', 'SPACK_CXX', 'SPACK_FC', 'SPACK_PREFIX',
+ 'SPACK_ENV_PATH', 'SPACK_DEBUG_LOG_DIR',
+ 'SPACK_COMPILER_SPEC', 'SPACK_SHORT_SPEC',
+ 'SPACK_CC_RPATH_ARG', 'SPACK_CXX_RPATH_ARG',
+ 'SPACK_F77_RPATH_ARG', 'SPACK_FC_RPATH_ARG',
+ 'SPACK_TARGET_ARGS'):
+ del os.environ[name]
+
def test_static_to_shared_library(build_environment):
os.environ['SPACK_TEST_COMMAND'] = 'dump-args'
diff --git a/lib/spack/spack/test/cc.py b/lib/spack/spack/test/cc.py
index 3e8d5d8c70..a24f2b7d4e 100644
--- a/lib/spack/spack/test/cc.py
+++ b/lib/spack/spack/test/cc.py
@@ -102,7 +102,8 @@ def wrapper_environment():
SPACK_FC_RPATH_ARG='-Wl,-rpath,',
SPACK_LINK_DIRS=None,
SPACK_INCLUDE_DIRS=None,
- SPACK_RPATH_DIRS=None):
+ SPACK_RPATH_DIRS=None,
+ SPACK_TARGET_ARGS=''):
yield