From 14599f09be6cd6eac4dc4d9d2d01a0ad001ae3f5 Mon Sep 17 00:00:00 2001 From: Massimiliano Culpo Date: Thu, 25 Jun 2020 18:18:48 +0200 Subject: Separate Apple Clang from LLVM Clang (#17110) * Separate Apple Clang from LLVM Clang Apple Clang is a compiler of its own. All places referring to "-apple" suffix have been updated. * Hack to use a dash in 'apple-clang' To be able to use autodoc from Sphinx we need a valid Python name for the module that contains Apple's Clang code. * Updated packages to account for the existence of apple-clang Co-authored-by: Adam J. Stewart * Added unit test for XCode related functions Co-authored-by: Gregory Becker Co-authored-by: Adam J. Stewart --- lib/spack/docs/basic_usage.rst | 4 +- lib/spack/docs/getting_started.rst | 2 +- lib/spack/docs/packaging_guide.rst | 8 +- lib/spack/docs/repositories.rst | 20 +-- lib/spack/llnl/util/cpu/microarchitecture.py | 29 ++-- lib/spack/llnl/util/cpu/microarchitectures.json | 8 +- lib/spack/spack/ci.py | 12 +- lib/spack/spack/compilers/__init__.py | 16 +- lib/spack/spack/compilers/apple_clang.py | 165 +++++++++++++++++++ lib/spack/spack/compilers/clang.py | 209 +++--------------------- lib/spack/spack/spec.py | 8 +- lib/spack/spack/test/architecture.py | 6 +- lib/spack/spack/test/compilers/basics.py | 142 +++++++++++++--- lib/spack/spack/test/compilers/detection.py | 17 +- lib/spack/spack/test/concretize.py | 2 +- lib/spack/spack/test/conftest.py | 2 +- lib/spack/spack/test/data/config/compilers.yaml | 2 +- 17 files changed, 396 insertions(+), 256 deletions(-) create mode 100644 lib/spack/spack/compilers/apple_clang.py (limited to 'lib') diff --git a/lib/spack/docs/basic_usage.rst b/lib/spack/docs/basic_usage.rst index 8acebeb0e6..93c5858d93 100644 --- a/lib/spack/docs/basic_usage.rst +++ b/lib/spack/docs/basic_usage.rst @@ -627,8 +627,8 @@ output metadata on specs and all dependencies as json: "target": "x86_64" }, "compiler": { - "name": "clang", - "version": "10.0.0-apple" + "name": "apple-clang", + "version": "10.0.0" }, "namespace": "builtin", "parameters": { diff --git a/lib/spack/docs/getting_started.rst b/lib/spack/docs/getting_started.rst index 1fbddbd4a9..226b1f0883 100644 --- a/lib/spack/docs/getting_started.rst +++ b/lib/spack/docs/getting_started.rst @@ -478,7 +478,7 @@ Fortran. cxx: /usr/bin/clang++ f77: /path/to/bin/gfortran fc: /path/to/bin/gfortran - spec: clang@11.0.0-apple + spec: apple-clang@11.0.0 If you used Spack to install GCC, you can get the installation prefix by diff --git a/lib/spack/docs/packaging_guide.rst b/lib/spack/docs/packaging_guide.rst index 533036852b..d3a888b1fc 100644 --- a/lib/spack/docs/packaging_guide.rst +++ b/lib/spack/docs/packaging_guide.rst @@ -1675,15 +1675,15 @@ can see the patches that would be applied to ``m4``:: Concretized -------------------------------- - m4@1.4.18%clang@9.0.0-apple patches=3877ab548f88597ab2327a2230ee048d2d07ace1062efe81fc92e91b7f39cd00,c0a408fbffb7255fcc75e26bd8edab116fc81d216bfd18b473668b7739a4158e,fc9b61654a3ba1a8d6cd78ce087e7c96366c290bc8d2c299f09828d793b853c8 +sigsegv arch=darwin-highsierra-x86_64 - ^libsigsegv@2.11%clang@9.0.0-apple arch=darwin-highsierra-x86_64 + m4@1.4.18%apple-clang@9.0.0 patches=3877ab548f88597ab2327a2230ee048d2d07ace1062efe81fc92e91b7f39cd00,c0a408fbffb7255fcc75e26bd8edab116fc81d216bfd18b473668b7739a4158e,fc9b61654a3ba1a8d6cd78ce087e7c96366c290bc8d2c299f09828d793b853c8 +sigsegv arch=darwin-highsierra-x86_64 + ^libsigsegv@2.11%apple-clang@9.0.0 arch=darwin-highsierra-x86_64 You can also see patches that have been applied to installed packages with ``spack find -v``:: $ spack find -v m4 ==> 1 installed package - -- darwin-highsierra-x86_64 / clang@9.0.0-apple ----------------- + -- darwin-highsierra-x86_64 / apple-clang@9.0.0 ----------------- m4@1.4.18 patches=3877ab548f88597ab2327a2230ee048d2d07ace1062efe81fc92e91b7f39cd00,c0a408fbffb7255fcc75e26bd8edab116fc81d216bfd18b473668b7739a4158e,fc9b61654a3ba1a8d6cd78ce087e7c96366c290bc8d2c299f09828d793b853c8 +sigsegv .. _cmd-spack-resource: @@ -1713,7 +1713,7 @@ wonder where the extra boost patches are coming from:: $ spack spec dealii ^boost@1.68.0 ^hdf5+fortran | grep '\^boost' ^boost@1.68.0 - ^boost@1.68.0%clang@9.0.0-apple+atomic+chrono~clanglibcpp cxxstd=default +date_time~debug+exception+filesystem+graph~icu+iostreams+locale+log+math~mpi+multithreaded~numpy patches=2ab6c72d03dec6a4ae20220a9dfd5c8c572c5294252155b85c6874d97c323199,b37164268f34f7133cbc9a4066ae98fda08adf51e1172223f6a969909216870f ~pic+program_options~python+random+regex+serialization+shared+signals~singlethreaded+system~taggedlayout+test+thread+timer~versionedlayout+wave arch=darwin-highsierra-x86_64 + ^boost@1.68.0%apple-clang@9.0.0+atomic+chrono~clanglibcpp cxxstd=default +date_time~debug+exception+filesystem+graph~icu+iostreams+locale+log+math~mpi+multithreaded~numpy patches=2ab6c72d03dec6a4ae20220a9dfd5c8c572c5294252155b85c6874d97c323199,b37164268f34f7133cbc9a4066ae98fda08adf51e1172223f6a969909216870f ~pic+program_options~python+random+regex+serialization+shared+signals~singlethreaded+system~taggedlayout+test+thread+timer~versionedlayout+wave arch=darwin-highsierra-x86_64 $ spack resource show b37164268 b37164268f34f7133cbc9a4066ae98fda08adf51e1172223f6a969909216870f path: /home/spackuser/src/spack/var/spack/repos/builtin/packages/dealii/boost_1.68.0.patch diff --git a/lib/spack/docs/repositories.rst b/lib/spack/docs/repositories.rst index 6c2050a4be..4a2c163886 100644 --- a/lib/spack/docs/repositories.rst +++ b/lib/spack/docs/repositories.rst @@ -280,16 +280,16 @@ you install it, you can use ``spack spec -N``: Concretized -------------------------------- - builtin.hdf5@1.10.0-patch1%clang@7.0.2-apple+cxx~debug+fortran+mpi+shared~szip~threadsafe arch=darwin-elcapitan-x86_64 - ^builtin.openmpi@2.0.1%clang@7.0.2-apple~mxm~pmi~psm~psm2~slurm~sqlite3~thread_multiple~tm~verbs+vt arch=darwin-elcapitan-x86_64 - ^builtin.hwloc@1.11.4%clang@7.0.2-apple arch=darwin-elcapitan-x86_64 - ^builtin.libpciaccess@0.13.4%clang@7.0.2-apple arch=darwin-elcapitan-x86_64 - ^builtin.libtool@2.4.6%clang@7.0.2-apple arch=darwin-elcapitan-x86_64 - ^builtin.m4@1.4.17%clang@7.0.2-apple+sigsegv arch=darwin-elcapitan-x86_64 - ^builtin.libsigsegv@2.10%clang@7.0.2-apple arch=darwin-elcapitan-x86_64 - ^builtin.pkg-config@0.29.1%clang@7.0.2-apple+internal_glib arch=darwin-elcapitan-x86_64 - ^builtin.util-macros@1.19.0%clang@7.0.2-apple arch=darwin-elcapitan-x86_64 - ^builtin.zlib@1.2.8%clang@7.0.2-apple+pic arch=darwin-elcapitan-x86_64 + builtin.hdf5@1.10.0-patch1%apple-clang@7.0.2+cxx~debug+fortran+mpi+shared~szip~threadsafe arch=darwin-elcapitan-x86_64 + ^builtin.openmpi@2.0.1%apple-clang@7.0.2~mxm~pmi~psm~psm2~slurm~sqlite3~thread_multiple~tm~verbs+vt arch=darwin-elcapitan-x86_64 + ^builtin.hwloc@1.11.4%apple-clang@7.0.2 arch=darwin-elcapitan-x86_64 + ^builtin.libpciaccess@0.13.4%apple-clang@7.0.2 arch=darwin-elcapitan-x86_64 + ^builtin.libtool@2.4.6%apple-clang@7.0.2 arch=darwin-elcapitan-x86_64 + ^builtin.m4@1.4.17%apple-clang@7.0.2+sigsegv arch=darwin-elcapitan-x86_64 + ^builtin.libsigsegv@2.10%apple-clang@7.0.2 arch=darwin-elcapitan-x86_64 + ^builtin.pkg-config@0.29.1%apple-clang@7.0.2+internal_glib arch=darwin-elcapitan-x86_64 + ^builtin.util-macros@1.19.0%apple-clang@7.0.2 arch=darwin-elcapitan-x86_64 + ^builtin.zlib@1.2.8%apple-clang@7.0.2+pic arch=darwin-elcapitan-x86_64 .. warning:: diff --git a/lib/spack/llnl/util/cpu/microarchitecture.py b/lib/spack/llnl/util/cpu/microarchitecture.py index f507837f89..284bfebb7e 100644 --- a/lib/spack/llnl/util/cpu/microarchitecture.py +++ b/lib/spack/llnl/util/cpu/microarchitecture.py @@ -204,10 +204,22 @@ class Microarchitecture(object): compiler (str): name of the compiler to be used version (str): version of the compiler to be used """ - # If we don't have information on compiler return an empty string - if compiler not in self.compilers: + # If we don't have information on compiler at all + # return an empty string + if compiler not in self.family.compilers: return '' + # If we have information but it stops before this + # microarchitecture, fall back to the best known target + if compiler not in self.compilers: + best_target = [ + x for x in self.ancestors if compiler in x.compilers + ][0] + msg = ("'{0}' compiler is known to optimize up to the '{1}'" + " microarchitecture in the '{2}' architecture family") + msg = msg.format(compiler, best_target, best_target.family) + raise UnsupportedMicroarchitecture(msg) + # If we have information on this compiler we need to check the # version being used compiler_info = self.compilers[compiler] @@ -219,15 +231,10 @@ class Microarchitecture(object): def satisfies_constraint(entry, version): min_version, max_version = entry['versions'].split(':') - # Check version suffixes - min_version, min_suffix = version_components(min_version) - max_version, max_suffix = version_components(max_version) - version, suffix = version_components(version) - - # If the suffixes are not all equal there's no match - if ((suffix != min_suffix and min_version) or - (suffix != max_suffix and max_version)): - return False + # Extract numeric part of the version + min_version, _ = version_components(min_version) + max_version, _ = version_components(max_version) + version, _ = version_components(version) # Assume compiler versions fit into semver tuplify = lambda x: tuple(int(y) for y in x.split('.')) diff --git a/lib/spack/llnl/util/cpu/microarchitectures.json b/lib/spack/llnl/util/cpu/microarchitectures.json index 51411d4a24..e94146bf85 100644 --- a/lib/spack/llnl/util/cpu/microarchitectures.json +++ b/lib/spack/llnl/util/cpu/microarchitectures.json @@ -61,12 +61,14 @@ "flags": "-march={name} -mtune={name}" } ], - "clang": [ + "apple-clang": [ { - "versions": "0.0.0-apple:", + "versions": ":", "name": "x86-64", "flags": "-march={name}" - }, + } + ], + "clang": [ { "versions": ":", "name": "x86-64", diff --git a/lib/spack/spack/ci.py b/lib/spack/spack/ci.py index ce74abf29c..230f25ccee 100644 --- a/lib/spack/spack/ci.py +++ b/lib/spack/spack/ci.py @@ -345,18 +345,18 @@ def compute_spec_deps(spec_list): ], "specs": [ { - "root_spec": "readline@7.0%clang@9.1.0-apple arch=darwin-...", - "spec": "readline@7.0%clang@9.1.0-apple arch=darwin-highs...", + "root_spec": "readline@7.0%apple-clang@9.1.0 arch=darwin-...", + "spec": "readline@7.0%apple-clang@9.1.0 arch=darwin-highs...", "label": "readline/ip6aiun" }, { - "root_spec": "readline@7.0%clang@9.1.0-apple arch=darwin-...", - "spec": "ncurses@6.1%clang@9.1.0-apple arch=darwin-highsi...", + "root_spec": "readline@7.0%apple-clang@9.1.0 arch=darwin-...", + "spec": "ncurses@6.1%apple-clang@9.1.0 arch=darwin-highsi...", "label": "ncurses/y43rifz" }, { - "root_spec": "readline@7.0%clang@9.1.0-apple arch=darwin-...", - "spec": "pkgconf@1.5.4%clang@9.1.0-apple arch=darwin-high...", + "root_spec": "readline@7.0%apple-clang@9.1.0 arch=darwin-...", + "spec": "pkgconf@1.5.4%apple-clang@9.1.0 arch=darwin-high...", "label": "pkgconf/eg355zb" } ] diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index 771459f7f3..dfa750cc4d 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -245,7 +245,9 @@ def supported_compilers(): See available_compilers() to get a list of all the available versions of supported compilers. """ - return sorted(name for name in + # Hack to be able to call the compiler `apple-clang` while still + # using a valid python name for the module + return sorted(name if name != 'apple_clang' else 'apple-clang' for name in llnl.util.lang.list_modules(spack.paths.compilers_path)) @@ -469,7 +471,13 @@ def class_for_compiler_name(compiler_name): """Given a compiler module name, get the corresponding Compiler class.""" assert(supported(compiler_name)) - file_path = os.path.join(spack.paths.compilers_path, compiler_name + ".py") + # Hack to be able to call the compiler `apple-clang` while still + # using a valid python name for the module + module_name = compiler_name + if compiler_name == 'apple-clang': + module_name = compiler_name.replace('-', '_') + + file_path = os.path.join(spack.paths.compilers_path, module_name + ".py") compiler_mod = simp.load_source(_imported_compilers_module, file_path) cls = getattr(compiler_mod, mod_to_class(compiler_name)) @@ -662,7 +670,7 @@ def make_compiler_list(detected_versions): operating_system, compiler_name, version = cmp_id compiler_cls = spack.compilers.class_for_compiler_name(compiler_name) spec = spack.spec.CompilerSpec(compiler_cls.name, version) - paths = [paths.get(l, None) for l in ('cc', 'cxx', 'f77', 'fc')] + paths = [paths.get(x, None) for x in ('cc', 'cxx', 'f77', 'fc')] target = cpu.host() compiler = compiler_cls( spec, operating_system, str(target.family), paths @@ -716,6 +724,8 @@ def is_mixed_toolchain(compiler): toolchains.add(compiler_cls.__name__) if len(toolchains) > 1: + if toolchains == set(['Clang', 'AppleClang']): + return False tty.debug("[TOOLCHAINS] {0}".format(toolchains)) return True diff --git a/lib/spack/spack/compilers/apple_clang.py b/lib/spack/spack/compilers/apple_clang.py new file mode 100644 index 0000000000..8ef58550ea --- /dev/null +++ b/lib/spack/spack/compilers/apple_clang.py @@ -0,0 +1,165 @@ +# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +import os.path +import re +import shutil + +import llnl.util.tty as tty +import llnl.util.lang +import spack.compiler +import spack.compilers.clang +import spack.util.executable +import spack.version + + +class AppleClang(spack.compilers.clang.Clang): + openmp_flag = "-Xpreprocessor -fopenmp" + + @classmethod + @llnl.util.lang.memoized + def extract_version_from_output(cls, output): + ver = 'unknown' + match = re.search( + # Apple's LLVM compiler has its own versions, so suffix them. + r'^Apple (?:LLVM|clang) version ([^ )]+)', output + ) + if match: + ver = match.group(match.lastindex) + return ver + + @property + def cxx11_flag(self): + # Adapted from CMake's AppleClang-CXX rules + # Spack's AppleClang detection only valid from Xcode >= 4.6 + if self.version < spack.version.ver('4.0.0'): + raise spack.compiler.UnsupportedCompilerFlag( + self, "the C++11 standard", "cxx11_flag", "Xcode < 4.0.0" + ) + return "-std=c++11" + + @property + def cxx14_flag(self): + # Adapted from CMake's rules for AppleClang + if self.version < spack.version.ver('5.1.0'): + raise spack.compiler.UnsupportedCompilerFlag( + self, "the C++14 standard", "cxx14_flag", "Xcode < 5.1.0" + ) + elif self.version < spack.version.ver('6.1.0'): + return "-std=c++1y" + + return "-std=c++14" + + @property + def cxx17_flag(self): + # Adapted from CMake's rules for AppleClang + if self.version < spack.version.ver('6.1.0'): + raise spack.compiler.UnsupportedCompilerFlag( + self, "the C++17 standard", "cxx17_flag", "Xcode < 6.1.0" + ) + return "-std=c++1z" + + def setup_custom_environment(self, pkg, env): + """Set the DEVELOPER_DIR environment for the Xcode toolchain. + + On macOS, not all buildsystems support querying CC and CXX for the + compilers to use and instead query the Xcode toolchain for what + compiler to run. This side-steps the spack wrappers. In order to inject + spack into this setup, we need to copy (a subset of) Xcode.app and + replace the compiler executables with symlinks to the spack wrapper. + Currently, the stage is used to store the Xcode.app copies. We then set + the 'DEVELOPER_DIR' environment variables to cause the xcrun and + related tools to use this Xcode.app. + """ + super(AppleClang, self).setup_custom_environment(pkg, env) + + if not pkg.use_xcode: + # if we do it for all packages, we get into big troubles with MPI: + # filter_compilers(self) will use mockup XCode compilers on macOS + # with Clang. Those point to Spack's compiler wrappers and + # consequently render MPI non-functional outside of Spack. + return + + # Use special XCode versions of compiler wrappers when using XCode + # Overwrites build_environment's setting of SPACK_CC and SPACK_CXX + xcrun = spack.util.executable.Executable('xcrun') + xcode_clang = xcrun('-f', 'clang', output=str).strip() + xcode_clangpp = xcrun('-f', 'clang++', output=str).strip() + env.set('SPACK_CC', xcode_clang, force=True) + env.set('SPACK_CXX', xcode_clangpp, force=True) + + xcode_select = spack.util.executable.Executable('xcode-select') + + # Get the path of the active developer directory + real_root = xcode_select('--print-path', output=str).strip() + + # The path name can be used to determine whether the full Xcode suite + # or just the command-line tools are installed + if real_root.endswith('Developer'): + # The full Xcode suite is installed + pass + else: + if real_root.endswith('CommandLineTools'): + # Only the command-line tools are installed + msg = 'It appears that you have the Xcode command-line tools ' + msg += 'but not the full Xcode suite installed.\n' + + else: + # Xcode is not installed + msg = 'It appears that you do not have Xcode installed.\n' + + msg += 'In order to use Spack to build the requested application, ' + msg += 'you need the full Xcode suite. It can be installed ' + msg += 'through the App Store. Make sure you launch the ' + msg += 'application and accept the license agreement.\n' + + raise OSError(msg) + + real_root = os.path.dirname(os.path.dirname(real_root)) + developer_root = os.path.join(spack.stage.get_stage_root(), + 'xcode-select', + self.name, + str(self.version)) + xcode_link = os.path.join(developer_root, 'Xcode.app') + + if not os.path.exists(developer_root): + tty.warn('Copying Xcode from %s to %s in order to add spack ' + 'wrappers to it. Please do not interrupt.' + % (real_root, developer_root)) + + # We need to make a new Xcode.app instance, but with symlinks to + # the spack wrappers for the compilers it ships. This is necessary + # because some projects insist on just asking xcrun and related + # tools where the compiler runs. These tools are very hard to trick + # as they do realpath and end up ignoring the symlinks in a + # "softer" tree of nothing but symlinks in the right places. + shutil.copytree( + real_root, developer_root, symlinks=True, + ignore=shutil.ignore_patterns( + 'AppleTV*.platform', 'Watch*.platform', 'iPhone*.platform', + 'Documentation', 'swift*' + )) + + real_dirs = [ + 'Toolchains/XcodeDefault.xctoolchain/usr/bin', + 'usr/bin', + ] + + bins = ['c++', 'c89', 'c99', 'cc', 'clang', 'clang++', 'cpp'] + + for real_dir in real_dirs: + dev_dir = os.path.join(developer_root, + 'Contents', + 'Developer', + real_dir) + for fname in os.listdir(dev_dir): + if fname in bins: + os.unlink(os.path.join(dev_dir, fname)) + os.symlink( + os.path.join(spack.paths.build_env_path, 'cc'), + os.path.join(dev_dir, fname)) + + os.symlink(developer_root, xcode_link) + + env.set('DEVELOPER_DIR', xcode_link) diff --git a/lib/spack/spack/compilers/clang.py b/lib/spack/spack/compilers/clang.py index 846c3609dc..f158ff3276 100644 --- a/lib/spack/spack/compilers/clang.py +++ b/lib/spack/spack/compilers/clang.py @@ -4,17 +4,11 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) import re -import os import sys -from shutil import copytree, ignore_patterns import llnl.util.lang -import llnl.util.tty as tty -import spack.paths -import spack.stage from spack.compiler import Compiler, UnsupportedCompilerFlag -from spack.util.executable import Executable from spack.version import ver @@ -88,88 +82,41 @@ class Clang(Compiler): return link_paths - @property - def is_apple(self): - ver_string = str(self.version) - return ver_string.endswith('-apple') - @property def verbose_flag(self): return "-v" - @property - def openmp_flag(self): - if self.is_apple: - return "-Xpreprocessor -fopenmp" - else: - return "-fopenmp" + openmp_flag = "-fopenmp" @property def cxx11_flag(self): - if self.is_apple: - # Adapted from CMake's AppleClang-CXX rules - # Spack's AppleClang detection only valid from Xcode >= 4.6 - if self.version < ver('4.0.0'): - raise UnsupportedCompilerFlag(self, - "the C++11 standard", - "cxx11_flag", - "Xcode < 4.0.0") - else: - return "-std=c++11" - else: - if self.version < ver('3.3'): - raise UnsupportedCompilerFlag(self, - "the C++11 standard", - "cxx11_flag", - "< 3.3") - else: - return "-std=c++11" + if self.version < ver('3.3'): + raise UnsupportedCompilerFlag( + self, "the C++11 standard", "cxx11_flag", "< 3.3" + ) + return "-std=c++11" @property def cxx14_flag(self): - if self.is_apple: - # Adapted from CMake's rules for AppleClang - if self.version < ver('5.1.0'): - raise UnsupportedCompilerFlag(self, - "the C++14 standard", - "cxx14_flag", - "Xcode < 5.1.0") - elif self.version < ver('6.1.0'): - return "-std=c++1y" - else: - return "-std=c++14" - else: - if self.version < ver('3.4'): - raise UnsupportedCompilerFlag(self, - "the C++14 standard", - "cxx14_flag", - "< 3.5") - elif self.version < ver('3.5'): - return "-std=c++1y" - else: - return "-std=c++14" + if self.version < ver('3.4'): + raise UnsupportedCompilerFlag( + self, "the C++14 standard", "cxx14_flag", "< 3.5" + ) + elif self.version < ver('3.5'): + return "-std=c++1y" + + return "-std=c++14" @property def cxx17_flag(self): - if self.is_apple: - # Adapted from CMake's rules for AppleClang - if self.version < ver('6.1.0'): - raise UnsupportedCompilerFlag(self, - "the C++17 standard", - "cxx17_flag", - "Xcode < 6.1.0") - else: - return "-std=c++1z" - else: - if self.version < ver('3.5'): - raise UnsupportedCompilerFlag(self, - "the C++17 standard", - "cxx17_flag", - "< 3.5") - elif self.version < ver('5.0'): - return "-std=c++1z" - else: - return "-std=c++17" + if self.version < ver('3.5'): + raise UnsupportedCompilerFlag( + self, "the C++17 standard", "cxx17_flag", "< 3.5" + ) + elif self.version < ver('5.0'): + return "-std=c++1z" + + return "-std=c++17" @property def c99_flag(self): @@ -207,9 +154,10 @@ class Clang(Compiler): @llnl.util.lang.memoized def extract_version_from_output(cls, output): ver = 'unknown' + if 'Apple' in output: + return ver + match = re.search( - # Apple's LLVM compiler has its own versions, so suffix them. - r'^Apple (?:LLVM|clang) version ([^ )]+)|' # Normal clang compiler versions are left as-is r'clang version ([^ )]+)-svn[~.\w\d-]*|' # Don't include hyphenated patch numbers in the version @@ -219,8 +167,7 @@ class Clang(Compiler): output ) if match: - suffix = '-apple' if match.lastindex == 1 else '' - ver = match.group(match.lastindex) + suffix + ver = match.group(match.lastindex) return ver @classmethod @@ -235,107 +182,3 @@ class Clang(Compiler): @classmethod def f77_version(cls, f77): return cls.fc_version(f77) - - def setup_custom_environment(self, pkg, env): - """Set the DEVELOPER_DIR environment for the Xcode toolchain. - - On macOS, not all buildsystems support querying CC and CXX for the - compilers to use and instead query the Xcode toolchain for what - compiler to run. This side-steps the spack wrappers. In order to inject - spack into this setup, we need to copy (a subset of) Xcode.app and - replace the compiler executables with symlinks to the spack wrapper. - Currently, the stage is used to store the Xcode.app copies. We then set - the 'DEVELOPER_DIR' environment variables to cause the xcrun and - related tools to use this Xcode.app. - """ - super(Clang, self).setup_custom_environment(pkg, env) - - if not self.is_apple or not pkg.use_xcode: - # if we do it for all packages, we get into big troubles with MPI: - # filter_compilers(self) will use mockup XCode compilers on macOS - # with Clang. Those point to Spack's compiler wrappers and - # consequently render MPI non-functional outside of Spack. - return - - # Use special XCode versions of compiler wrappers when using XCode - # Overwrites build_environment's setting of SPACK_CC and SPACK_CXX - xcrun = Executable('xcrun') - xcode_clang = xcrun('-f', 'clang', output=str).strip() - xcode_clangpp = xcrun('-f', 'clang++', output=str).strip() - env.set('SPACK_CC', xcode_clang, force=True) - env.set('SPACK_CXX', xcode_clangpp, force=True) - - xcode_select = Executable('xcode-select') - - # Get the path of the active developer directory - real_root = xcode_select('--print-path', output=str).strip() - - # The path name can be used to determine whether the full Xcode suite - # or just the command-line tools are installed - if real_root.endswith('Developer'): - # The full Xcode suite is installed - pass - else: - if real_root.endswith('CommandLineTools'): - # Only the command-line tools are installed - msg = 'It appears that you have the Xcode command-line tools ' - msg += 'but not the full Xcode suite installed.\n' - - else: - # Xcode is not installed - msg = 'It appears that you do not have Xcode installed.\n' - - msg += 'In order to use Spack to build the requested application, ' - msg += 'you need the full Xcode suite. It can be installed ' - msg += 'through the App Store. Make sure you launch the ' - msg += 'application and accept the license agreement.\n' - - raise OSError(msg) - - real_root = os.path.dirname(os.path.dirname(real_root)) - developer_root = os.path.join(spack.stage.get_stage_root(), - 'xcode-select', - self.name, - str(self.version)) - xcode_link = os.path.join(developer_root, 'Xcode.app') - - if not os.path.exists(developer_root): - tty.warn('Copying Xcode from %s to %s in order to add spack ' - 'wrappers to it. Please do not interrupt.' - % (real_root, developer_root)) - - # We need to make a new Xcode.app instance, but with symlinks to - # the spack wrappers for the compilers it ships. This is necessary - # because some projects insist on just asking xcrun and related - # tools where the compiler runs. These tools are very hard to trick - # as they do realpath and end up ignoring the symlinks in a - # "softer" tree of nothing but symlinks in the right places. - copytree(real_root, developer_root, symlinks=True, - ignore=ignore_patterns('AppleTV*.platform', - 'Watch*.platform', - 'iPhone*.platform', - 'Documentation', - 'swift*')) - - real_dirs = [ - 'Toolchains/XcodeDefault.xctoolchain/usr/bin', - 'usr/bin', - ] - - bins = ['c++', 'c89', 'c99', 'cc', 'clang', 'clang++', 'cpp'] - - for real_dir in real_dirs: - dev_dir = os.path.join(developer_root, - 'Contents', - 'Developer', - real_dir) - for fname in os.listdir(dev_dir): - if fname in bins: - os.unlink(os.path.join(dev_dir, fname)) - os.symlink( - os.path.join(spack.paths.build_env_path, 'cc'), - os.path.join(dev_dir, fname)) - - os.symlink(developer_root, xcode_link) - - env.set('DEVELOPER_DIR', xcode_link) diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index b09d72003e..b23142e101 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -1448,8 +1448,8 @@ class Spec(object): 'target': 'x86_64', }, 'compiler': { - 'name': 'clang', - 'version': '10.0.0-apple', + 'name': 'apple-clang', + 'version': '10.0.0', }, 'namespace': 'builtin', 'parameters': { @@ -1554,8 +1554,8 @@ class Spec(object): 'target': 'x86_64', }, 'compiler': { - 'name': 'clang', - 'version': '10.0.0-apple', + 'name': 'apple-clang', + 'version': '10.0.0', }, 'namespace': 'builtin', 'parameters': { diff --git a/lib/spack/spack/test/architecture.py b/lib/spack/spack/test/architecture.py index 48cec134d2..80d3fc72c0 100644 --- a/lib/spack/spack/test/architecture.py +++ b/lib/spack/spack/test/architecture.py @@ -176,8 +176,8 @@ def test_arch_spec_container_semantic(item, architecture_str): # Check mixed toolchains ('clang@8.0.0', 'broadwell', ''), ('clang@3.5', 'x86_64', '-march=x86-64 -mtune=generic'), - # Check clang compilers with 'apple' suffix - ('clang@9.1.0-apple', 'x86_64', '-march=x86-64') + # Check Apple's Clang compilers + ('apple-clang@9.1.0', 'x86_64', '-march=x86-64') ]) @pytest.mark.filterwarnings("ignore:microarchitecture specific") def test_optimization_flags( @@ -200,7 +200,7 @@ def test_optimization_flags( '-march=icelake-client -mtune=icelake-client'), # Check that the special case for Apple's clang is treated correctly # i.e. it won't try to detect the version again - (spack.spec.CompilerSpec('clang@9.1.0-apple'), None, 'x86_64', + (spack.spec.CompilerSpec('apple-clang@9.1.0'), None, 'x86_64', '-march=x86-64'), ]) def test_optimization_flags_with_custom_versions( diff --git a/lib/spack/spack/test/compilers/basics.py b/lib/spack/spack/test/compilers/basics.py index 32f298ae2b..825975b21a 100644 --- a/lib/spack/spack/test/compilers/basics.py +++ b/lib/spack/spack/test/compilers/basics.py @@ -7,6 +7,7 @@ import pytest import sys import os +import shutil from copy import copy from six import iteritems @@ -16,6 +17,8 @@ import llnl.util.filesystem as fs import spack.spec import spack.compiler import spack.compilers as compilers +import spack.spec +import spack.util.environment from spack.compiler import Compiler from spack.util.executable import ProcessError @@ -130,7 +133,7 @@ def test_compiler_flags_from_config_are_grouped(): # Utility function to test most flags. default_compiler_entry = { - 'spec': 'clang@2.0.0-apple', + 'spec': 'apple-clang@2.0.0', 'operating_system': 'foo-os', 'paths': { 'cc': 'cc-path', @@ -370,26 +373,27 @@ def test_cce_flags(): 'cce@1.0') -def test_clang_flags(): - # Apple Clang. +def test_apple_clang_flags(): supported_flag_test( - "openmp_flag", "-Xpreprocessor -fopenmp", "clang@2.0.0-apple") - unsupported_flag_test("cxx11_flag", "clang@2.0.0-apple") - supported_flag_test("cxx11_flag", "-std=c++11", "clang@4.0.0-apple") - unsupported_flag_test("cxx14_flag", "clang@5.0.0-apple") - supported_flag_test("cxx14_flag", "-std=c++1y", "clang@5.1.0-apple") - supported_flag_test("cxx14_flag", "-std=c++14", "clang@6.1.0-apple") - unsupported_flag_test("cxx17_flag", "clang@6.0.0-apple") - supported_flag_test("cxx17_flag", "-std=c++1z", "clang@6.1.0-apple") - supported_flag_test("c99_flag", "-std=c99", "clang@6.1.0-apple") - unsupported_flag_test("c11_flag", "clang@6.0.0-apple") - supported_flag_test("c11_flag", "-std=c11", "clang@6.1.0-apple") - supported_flag_test("cc_pic_flag", "-fPIC", "clang@2.0.0-apple") - supported_flag_test("cxx_pic_flag", "-fPIC", "clang@2.0.0-apple") - supported_flag_test("f77_pic_flag", "-fPIC", "clang@2.0.0-apple") - supported_flag_test("fc_pic_flag", "-fPIC", "clang@2.0.0-apple") - - # non-Apple Clang. + "openmp_flag", "-Xpreprocessor -fopenmp", "apple-clang@2.0.0" + ) + unsupported_flag_test("cxx11_flag", "apple-clang@2.0.0") + supported_flag_test("cxx11_flag", "-std=c++11", "apple-clang@4.0.0") + unsupported_flag_test("cxx14_flag", "apple-clang@5.0.0") + supported_flag_test("cxx14_flag", "-std=c++1y", "apple-clang@5.1.0") + supported_flag_test("cxx14_flag", "-std=c++14", "apple-clang@6.1.0") + unsupported_flag_test("cxx17_flag", "apple-clang@6.0.0") + supported_flag_test("cxx17_flag", "-std=c++1z", "apple-clang@6.1.0") + supported_flag_test("c99_flag", "-std=c99", "apple-clang@6.1.0") + unsupported_flag_test("c11_flag", "apple-clang@6.0.0") + supported_flag_test("c11_flag", "-std=c11", "apple-clang@6.1.0") + supported_flag_test("cc_pic_flag", "-fPIC", "apple-clang@2.0.0") + supported_flag_test("cxx_pic_flag", "-fPIC", "apple-clang@2.0.0") + supported_flag_test("f77_pic_flag", "-fPIC", "apple-clang@2.0.0") + supported_flag_test("fc_pic_flag", "-fPIC", "apple-clang@2.0.0") + + +def test_clang_flags(): supported_flag_test("version_argument", "--version", "clang@foo.bar") supported_flag_test("openmp_flag", "-fopenmp", "clang@3.3") unsupported_flag_test("cxx11_flag", "clang@3.2") @@ -713,3 +717,101 @@ fi except ProcessError: # Confirm environment does not change after failed call assert 'SPACK_TEST_CMP_ON' not in os.environ + + +def test_apple_clang_setup_environment(mock_executable, monkeypatch): + """Test a code path that is taken only if the package uses + Xcode on MacOS. + """ + class MockPackage(object): + use_xcode = False + + apple_clang_cls = spack.compilers.class_for_compiler_name('apple-clang') + compiler = apple_clang_cls( + spack.spec.CompilerSpec('apple-clang@11.0.0'), 'catalina', 'x86_64', [ + '/usr/bin/clang', '/usr/bin/clang++', None, None + ] + ) + env = spack.util.environment.EnvironmentModifications() + # Check a package that doesn't use xcode and ensure we don't add changes + # to the environment + pkg = MockPackage() + compiler.setup_custom_environment(pkg, env) + assert not env + + # Prepare mock executables to fake the Xcode environment + xcrun = mock_executable('xcrun', """ +if [[ "$2" == "clang" ]] ; then + echo "/Library/Developer/CommandLineTools/usr/bin/clang" +fi +if [[ "$2" == "clang++" ]] ; then + echo "/Library/Developer/CommandLineTools/usr/bin/clang++" +fi +""") + mock_executable('xcode-select', """ +echo "/Library/Developer" +""") + bin_dir = os.path.dirname(xcrun) + monkeypatch.setenv('PATH', bin_dir, prepend=os.pathsep) + + def noop(*args, **kwargs): + pass + + real_listdir = os.listdir + + def _listdir(path): + if not os.path.exists(path): + return [] + return real_listdir(path) + + # Set a few operations to noop + monkeypatch.setattr(shutil, 'copytree', noop) + monkeypatch.setattr(os, 'unlink', noop) + monkeypatch.setattr(os, 'symlink', noop) + monkeypatch.setattr(os, 'listdir', _listdir) + + # Qt is so far the only package that uses this code path, change + # introduced in https://github.com/spack/spack/pull/1832 + pkg.use_xcode = True + compiler.setup_custom_environment(pkg, env) + assert len(env) == 3 + assert env.env_modifications[0].name == 'SPACK_CC' + assert env.env_modifications[1].name == 'SPACK_CXX' + assert env.env_modifications[2].name == 'DEVELOPER_DIR' + + +@pytest.mark.parametrize('xcode_select_output', [ + '', '/Library/Developer/CommandLineTools' +]) +def test_xcode_not_available( + xcode_select_output, mock_executable, monkeypatch +): + # Prepare mock executables to fake the Xcode environment + xcrun = mock_executable('xcrun', """ + if [[ "$2" == "clang" ]] ; then + echo "/Library/Developer/CommandLineTools/usr/bin/clang" + fi + if [[ "$2" == "clang++" ]] ; then + echo "/Library/Developer/CommandLineTools/usr/bin/clang++" + fi + """) + mock_executable('xcode-select', """ + echo "{0}" + """.format(xcode_select_output)) + bin_dir = os.path.dirname(xcrun) + monkeypatch.setenv('PATH', bin_dir, prepend=os.pathsep) + # Prepare compiler + apple_clang_cls = spack.compilers.class_for_compiler_name('apple-clang') + compiler = apple_clang_cls( + spack.spec.CompilerSpec('apple-clang@11.0.0'), 'catalina', 'x86_64', [ + '/usr/bin/clang', '/usr/bin/clang++', None, None + ] + ) + env = spack.util.environment.EnvironmentModifications() + + class MockPackage(object): + use_xcode = True + + pkg = MockPackage() + with pytest.raises(OSError): + compiler.setup_custom_environment(pkg, env) diff --git a/lib/spack/spack/test/compilers/detection.py b/lib/spack/spack/test/compilers/detection.py index 90311ad2d3..b38cf89b5d 100644 --- a/lib/spack/spack/test/compilers/detection.py +++ b/lib/spack/spack/test/compilers/detection.py @@ -53,11 +53,22 @@ def test_cce_version_detection(version_str, expected_version): 'Target: x86_64-apple-darwin18.7.0\n' 'Thread model: posix\n' 'InstalledDir: /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin\n', # noqa - '11.0.0-apple'), + '11.0.0'), ('Apple LLVM version 7.0.2 (clang-700.1.81)\n' 'Target: x86_64-apple-darwin15.2.0\n' - 'Thread model: posix\n', '7.0.2-apple'), - # Other platforms + 'Thread model: posix\n', '7.0.2'), +]) +def test_apple_clang_version_detection( + version_str, expected_version +): + cls = spack.compilers.class_for_compiler_name('apple-clang') + version = cls.extract_version_from_output(version_str) + assert version == expected_version + + +@pytest.mark.regression('10191') +@pytest.mark.parametrize('version_str,expected_version', [ + # LLVM Clang ('clang version 6.0.1-svn334776-1~exp1~20181018152737.116 (branches/release_60)\n' # noqa 'Target: x86_64-pc-linux-gnu\n' 'Thread model: posix\n' diff --git a/lib/spack/spack/test/concretize.py b/lib/spack/spack/test/concretize.py index fd0e185168..cfe0748c4b 100644 --- a/lib/spack/spack/test/concretize.py +++ b/lib/spack/spack/test/concretize.py @@ -608,7 +608,7 @@ class TestConcretize(object): ('mpileaks%gcc@4.8', 'haswell'), ('mpileaks%gcc@5.3.0', 'broadwell'), # Apple's clang always falls back to x86-64 for now - ('mpileaks%clang@9.1.0-apple', 'x86_64') + ('mpileaks%apple-clang@9.1.0', 'x86_64') ]) @pytest.mark.regression('13361') def test_adjusting_default_target_based_on_compiler( diff --git a/lib/spack/spack/test/conftest.py b/lib/spack/spack/test/conftest.py index 1efdd6ba95..9a292821f5 100644 --- a/lib/spack/spack/test/conftest.py +++ b/lib/spack/spack/test/conftest.py @@ -1145,7 +1145,7 @@ def mock_executable(tmpdir): import jinja2 def _factory(name, output, subdir=('bin',)): - f = tmpdir.mkdir(*subdir).join(name) + f = tmpdir.ensure(*subdir, dir=True).join(name) t = jinja2.Template('#!/bin/bash\n{{ output }}\n') f.write(t.render(output=output)) f.chmod(0o755) diff --git a/lib/spack/spack/test/data/config/compilers.yaml b/lib/spack/spack/test/data/config/compilers.yaml index 7aec138473..3a2db05e72 100644 --- a/lib/spack/spack/test/data/config/compilers.yaml +++ b/lib/spack/spack/test/data/config/compilers.yaml @@ -127,7 +127,7 @@ compilers: cxxflags: -O3 modules: 'None' - compiler: - spec: clang@9.1.0-apple + spec: apple-clang@9.1.0 operating_system: elcapitan paths: cc: /path/to/clang -- cgit v1.2.3-70-g09d2