From 3a994032f8143a2bdbfe5c61baeb12b1bd2fed60 Mon Sep 17 00:00:00 2001 From: John Parent Date: Fri, 22 Oct 2021 14:00:02 -0400 Subject: Spack on Windows package ports CMake - Windows Bootstrap (#25825) Remove hardcoded cmake compiler (#26410) Revert breaking cmake changes Ensure no autotools on Windows Perl on Windows (#26612) Python source build windows (#26313) Reconfigure sysconf for Windows Python2.6 compatibility Fxixup new sbang tests for windows Ruby support (#28287) Add NASM support (#28319) Add mock Ninja package for testing --- lib/spack/docs/getting_started.rst | 8 ---- lib/spack/llnl/util/tty/__init__.py | 2 +- lib/spack/spack/build_environment.py | 4 +- lib/spack/spack/build_systems/autotools.py | 3 +- lib/spack/spack/build_systems/cmake.py | 30 +++++-------- lib/spack/spack/cmd/unit_test.py | 2 +- lib/spack/spack/compilers/msvc.py | 70 +++++++++++++++++++++--------- lib/spack/spack/detection/path.py | 1 - lib/spack/spack/fetch_strategy.py | 5 ++- lib/spack/spack/platforms/_platform.py | 5 +++ lib/spack/spack/test/bootstrap.py | 2 +- lib/spack/spack/test/cmd/unit_test.py | 4 +- lib/spack/spack/test/cmd/url.py | 1 + lib/spack/spack/test/relocate.py | 1 + lib/spack/spack/test/sbang.py | 13 ++++++ 15 files changed, 93 insertions(+), 58 deletions(-) (limited to 'lib') diff --git a/lib/spack/docs/getting_started.rst b/lib/spack/docs/getting_started.rst index a904f0416a..8ba0a799a9 100644 --- a/lib/spack/docs/getting_started.rst +++ b/lib/spack/docs/getting_started.rst @@ -1587,14 +1587,6 @@ When given the option of adjusting your ``PATH``, choose the ``Git from the command line and also from 3rd-party software`` option. This will automatically update your ``PATH`` variable to include the ``git`` command. -"""" -Perl -"""" - -Perl is a flexible and feature-rich programming language that comes built-in -on Unix boxes but needs to be installed externally for Windows users. Fortunately, -you can find the most recent release at https://www.perl.org/get.html. - """" NASM """" diff --git a/lib/spack/llnl/util/tty/__init__.py b/lib/spack/llnl/util/tty/__init__.py index b244b8a58d..5b8729d7ec 100644 --- a/lib/spack/llnl/util/tty/__init__.py +++ b/lib/spack/llnl/util/tty/__init__.py @@ -146,7 +146,7 @@ def process_stacktrace(countback): file_list = [] for frame in st: # Check that the file is a spack file - if frame[0].find("/spack") >= 0: + if frame[0].find(os.path.sep + "spack") >= 0: file_list.append(frame[0]) # We use commonprefix to find what the spack 'root' directory is. root_dir = os.path.commonprefix(file_list) diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 7888dee139..3281a24013 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -528,7 +528,9 @@ def _set_variables_for_single_module(pkg, module): m.cmake = Executable('cmake') m.ctest = MakeExecutable('ctest', jobs) - # Standard build system arguments + if os.name == 'nt': + m.nmake = Executable('nmake') + # Standard CMake arguments m.std_cmake_args = spack.build_systems.cmake.CMakePackage._std_args(pkg) m.std_meson_args = spack.build_systems.meson.MesonPackage._std_args(pkg) m.std_pip_args = spack.build_systems.python.PythonPackage._std_args(pkg) diff --git a/lib/spack/spack/build_systems/autotools.py b/lib/spack/spack/build_systems/autotools.py index 46a3ccddf7..a644c92622 100644 --- a/lib/spack/spack/build_systems/autotools.py +++ b/lib/spack/spack/build_systems/autotools.py @@ -14,7 +14,7 @@ import llnl.util.tty as tty from llnl.util.filesystem import force_remove, working_dir from spack.build_environment import InstallError -from spack.directives import depends_on +from spack.directives import conflicts, depends_on from spack.operating_systems.mac_os import macos_version from spack.package import PackageBase, run_after, run_before from spack.util.executable import Executable @@ -104,6 +104,7 @@ class AutotoolsPackage(PackageBase): depends_on('gnuconfig', type='build', when='target=ppc64le:') depends_on('gnuconfig', type='build', when='target=aarch64:') depends_on('gnuconfig', type='build', when='target=riscv64:') + conflicts('platform=windows') @property def _removed_la_files_log(self): diff --git a/lib/spack/spack/build_systems/cmake.py b/lib/spack/spack/build_systems/cmake.py index 979a6bb024..591d73a5c4 100644 --- a/lib/spack/spack/build_systems/cmake.py +++ b/lib/spack/spack/build_systems/cmake.py @@ -93,15 +93,11 @@ class CMakePackage(PackageBase): #: See https://cmake.org/cmake/help/latest/manual/cmake-generators.7.html #: for more information. - variant('generator', - default='Make' if sys.platform != 'win32' else 'Ninja', - description='Build system to generate', - values=('Make', 'Ninja')) + generator = "Unix Makefiles" - depends_on('ninja', when='generator=Ninja') - - generatorMap = {'Make': 'Unix Makefiles', - 'Ninja': 'Ninja'} + if sys.platform == 'win32': + generator = "Ninja" + depends_on('ninja') # https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html variant('build_type', default='RelWithDebInfo', @@ -150,10 +146,10 @@ class CMakePackage(PackageBase): """Computes the standard cmake arguments for a generic package""" try: - pkg.generator = pkg.spec.variants['generator'].value - except KeyError: - pkg.generator = 'Make' if sys.platform != 'win32' else 'Ninja' - primary_generator = CMakePackage.generatorMap[pkg.generator] + if not pkg.generator: + raise AttributeError + except AttributeError: + pkg.generator = CMakePackage.generator try: build_type = pkg.spec.variants['build_type'].value @@ -167,22 +163,16 @@ class CMakePackage(PackageBase): define = CMakePackage.define args = [ - '-G', primary_generator, + '-G', pkg.generator, define('CMAKE_INSTALL_PREFIX', pkg.prefix.replace('\\', '/')), define('CMAKE_BUILD_TYPE', build_type), - define('CMAKE_C_COMPILER:FILEPATH', pkg.compiler.cc.replace('\\', '/')), - define('CMAKE_CXX_COMPILER:FILEPATH', pkg.compiler.cxx.replace('\\', '/')) ] - if pkg.compiler.fc is not None: - args.append(define('CMAKE_Fortran_COMPILER:FILEPATH', - pkg.compiler.fc.replace('\\', '/'))) - # CMAKE_INTERPROCEDURAL_OPTIMIZATION only exists for CMake >= 3.9 if pkg.spec.satisfies('^cmake@3.9:'): args.append(define('CMAKE_INTERPROCEDURAL_OPTIMIZATION', ipo)) - if primary_generator == 'Unix Makefiles': + if pkg.generator == 'Unix Makefiles': args.append(define('CMAKE_VERBOSE_MAKEFILE', True)) if platform.mac_ver()[0]: diff --git a/lib/spack/spack/cmd/unit_test.py b/lib/spack/spack/cmd/unit_test.py index 0faa05b834..05c308b84c 100644 --- a/lib/spack/spack/cmd/unit_test.py +++ b/lib/spack/spack/cmd/unit_test.py @@ -130,7 +130,7 @@ def do_list(args, extra_args): # in the future - so this manipulation might be fragile if nodetype.lower() == 'function': name_parts.append(item) - key_end = os.path.join(*[x[1] for x in key_parts]) + key_end = os.path.join(*key_parts[-1][1].split('/')) key = next(f for f in files if f.endswith(key_end)) tests[key].add(tuple(x[1] for x in name_parts)) elif nodetype.lower() == 'class': diff --git a/lib/spack/spack/compilers/msvc.py b/lib/spack/spack/compilers/msvc.py index d0ae6921ea..f51583caf4 100644 --- a/lib/spack/spack/compilers/msvc.py +++ b/lib/spack/spack/compilers/msvc.py @@ -4,13 +4,37 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) import os +import re import subprocess import sys -from typing import List # novm +from distutils.version import StrictVersion +from typing import Dict, List, Set # novm import spack.operating_systems.windows_os import spack.util.executable from spack.compiler import Compiler +from spack.error import SpackError + +avail_fc_version = set() # type: Set[str] +fc_path = dict() # type: Dict[str, str] + +fortran_mapping = { + '2021.3.0': '19.29.30133', + '2021.2.1': '19.28.29913', + '2021.2.0': '19.28.29334', + '2021.1.0': '19.28.29333', +} + + +def get_valid_fortran_pth(comp_ver): + cl_ver = str(comp_ver).split('@')[1] + sort_fn = lambda fc_ver: StrictVersion(fc_ver) + sort_fc_ver = sorted(list(avail_fc_version), key=sort_fn) + for ver in sort_fc_ver: + if ver in fortran_mapping: + if StrictVersion(cl_ver) <= StrictVersion(fortran_mapping[ver]): + return fc_path[ver] + return None class Msvc(Compiler): @@ -46,6 +70,8 @@ class Msvc(Compiler): # file based on compiler executable path. def __init__(self, *args, **kwargs): + new_pth = [pth if pth else get_valid_fortran_pth(args[0]) for pth in args[3]] + args[3][:] = new_pth super(Msvc, self).__init__(*args, **kwargs) if os.getenv("ONEAPI_ROOT"): # If this found, it sets all the vars @@ -65,6 +91,12 @@ class Msvc(Compiler): def pic_flag(self): return "" + @property + def msvc_version(self): + ver = re.search(Msvc.version_regex, self.cc).group(1) + ver = "".join(ver.split('.')[:2])[:-1] + return "MSVC" + ver + def setup_custom_environment(self, pkg, env): """Set environment variables for MSVC using the Microsoft-provided script.""" @@ -81,44 +113,40 @@ class Msvc(Compiler): if sys.version_info[0] >= 3: out = out.decode('utf-16le', errors='replace') # novermin - int_env = { # novermin - key.lower(): value - for key, _, value in - (line.partition('=') for line in out.splitlines()) - if key and value - } + int_env = dict((key.lower(), value) for key, _, value in + (line.partition('=') for line in out.splitlines()) + if key and value) if 'path' in int_env: env.set_path('PATH', int_env['path'].split(';')) env.set_path('INCLUDE', int_env.get('include', '').split(';')) env.set_path('LIB', int_env.get('lib', '').split(';')) + + env.set('CC', self.cc) + env.set('CXX', self.cxx) + env.set('FC', self.fc) + env.set('F77', self.f77) else: # Should not this be an exception? print("Cannot pull msvc compiler information in Python 2.6 or below") - # fc_version only loads the ifx compiler into the first MSVC stanza; - # if there are other versions of Microsoft VS installed and detected, they - # will only have cl.exe as the C/C++ compiler - @classmethod def fc_version(cls, fc): # We're using intel for the Fortran compilers, which exist if # ONEAPI_ROOT is a meaningful variable + fc_ver = cls.default_version(fc) + avail_fc_version.add(fc_ver) + fc_path[fc_ver] = fc if os.getenv("ONEAPI_ROOT"): try: sps = spack.operating_systems.windows_os.WindowsOs.compiler_search_paths - except Exception: - print("sps not found.") - raise - try: - clp = spack.util.executable.which_string("cl", path=sps) - except Exception: - print("cl not found.") - raise + except AttributeError: + raise SpackError("Windows compiler search paths not established") + clp = spack.util.executable.which_string("cl", path=sps) ver = cls.default_version(clp) - return ver else: - return cls.default_version(fc) + ver = fc_ver + return ver @classmethod def f77_version(cls, f77): diff --git a/lib/spack/spack/detection/path.py b/lib/spack/spack/detection/path.py index f280af4520..ee98e23841 100644 --- a/lib/spack/spack/detection/path.py +++ b/lib/spack/spack/detection/path.py @@ -12,7 +12,6 @@ import re import sys import warnings - import llnl.util.filesystem import llnl.util.tty diff --git a/lib/spack/spack/fetch_strategy.py b/lib/spack/spack/fetch_strategy.py index f9823fb648..2435da4b2f 100644 --- a/lib/spack/spack/fetch_strategy.py +++ b/lib/spack/spack/fetch_strategy.py @@ -353,8 +353,9 @@ class URLFetchStrategy(FetchStrategy): # Telling urllib to check if url is accessible try: url, headers, response = spack.util.web.read_from_url(url) - except spack.util.web.SpackWebError: - msg = "Urllib fetch failed to verify url {0}".format(url) + except spack.util.web.SpackWebError as werr: + msg = "Urllib fetch failed to verify url\ + {0}\n with error {1}".format(url, werr) raise FailedDownloadError(url, msg) return (response.getcode() is None or response.getcode() == 200) diff --git a/lib/spack/spack/platforms/_platform.py b/lib/spack/spack/platforms/_platform.py index b68b7f1299..a19381b8d4 100644 --- a/lib/spack/spack/platforms/_platform.py +++ b/lib/spack/spack/platforms/_platform.py @@ -2,6 +2,8 @@ # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) +import platform + import llnl.util.lang import spack.error @@ -143,3 +145,6 @@ class Platform(object): for o in sorted(self.operating_sys.values()): yield o._cmp_iter yield oses + + def is_64bit(self): + return platform.machine().endswith('64') diff --git a/lib/spack/spack/test/bootstrap.py b/lib/spack/spack/test/bootstrap.py index 8b72a98ad3..d753e2cdfa 100644 --- a/lib/spack/spack/test/bootstrap.py +++ b/lib/spack/spack/test/bootstrap.py @@ -131,7 +131,7 @@ spack: # Don't trigger evaluation here with spack.bootstrap.ensure_bootstrap_configuration(): pass - assert str(spack.store.root) == '/tmp/store' + assert str(spack.store.root) == sep + os.path.join('tmp', 'store') def test_nested_use_of_context_manager(mutable_config): diff --git a/lib/spack/spack/test/cmd/unit_test.py b/lib/spack/spack/test/cmd/unit_test.py index 276996501b..aa31282fdb 100644 --- a/lib/spack/spack/test/cmd/unit_test.py +++ b/lib/spack/spack/test/cmd/unit_test.py @@ -3,10 +3,12 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) +import os + from spack.main import SpackCommand spack_test = SpackCommand('unit-test') -cmd_test_py = 'lib/spack/spack/test/cmd/unit_test.py' +cmd_test_py = os.path.join('lib', 'spack', 'spack', 'test', 'cmd', 'unit_test.py') def test_list(): diff --git a/lib/spack/spack/test/cmd/url.py b/lib/spack/spack/test/cmd/url.py index 204f3a81b1..1e4c0d391a 100644 --- a/lib/spack/spack/test/cmd/url.py +++ b/lib/spack/spack/test/cmd/url.py @@ -3,6 +3,7 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) import re +import sys import pytest diff --git a/lib/spack/spack/test/relocate.py b/lib/spack/spack/test/relocate.py index a64052c7c8..4ef9ae4bb3 100644 --- a/lib/spack/spack/test/relocate.py +++ b/lib/spack/spack/test/relocate.py @@ -2,6 +2,7 @@ # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) +import os import os.path import re import shutil diff --git a/lib/spack/spack/test/sbang.py b/lib/spack/spack/test/sbang.py index aa9d9b10dd..f0cc1835d9 100644 --- a/lib/spack/spack/test/sbang.py +++ b/lib/spack/spack/test/sbang.py @@ -11,6 +11,7 @@ import grp import os import shutil import stat +import sys import tempfile import pytest @@ -193,6 +194,8 @@ def test_shebang_interpreter_regex(shebang, interpreter): sbang.get_interpreter(shebang) == interpreter +@pytest.mark.skipif(sys.platform == 'win32', + reason="Not supported on Windows (yet)") def test_shebang_handling(script_dir, sbang_line): sbang.filter_shebangs_in_directory(script_dir.tempdir) @@ -339,6 +342,8 @@ def test_install_user_sbang(install_mockery, configure_user_perms): run_test_install_sbang(False) +@pytest.mark.skipif(sys.platform == 'win32', + reason="Not supported on Windows (yet)") def test_install_sbang_too_long(tmpdir): root = str(tmpdir) num_extend = sbang.system_shebang_limit - len(root) - len('/bin/sbang') @@ -357,6 +362,8 @@ def test_install_sbang_too_long(tmpdir): assert 'cannot patch' in err +@pytest.mark.skipif(sys.platform == 'win32', + reason="Not supported on Windows (yet)") def test_sbang_hook_skips_nonexecutable_blobs(tmpdir): # Write a binary blob to non-executable.sh, with a long interpreter "path" # consisting of invalid UTF-8. The latter is technically not really necessary for @@ -374,6 +381,8 @@ def test_sbang_hook_skips_nonexecutable_blobs(tmpdir): assert b'sbang' not in f.readline() +@pytest.mark.skipif(sys.platform == 'win32', + reason="Not supported on Windows (yet)") def test_sbang_handles_non_utf8_files(tmpdir): # We have an executable with a copyright sign as filename contents = (b'#!' + b'\xa9' * sbang.system_shebang_limit + @@ -408,6 +417,8 @@ def shebang_limits_system_8_spack_16(): sbang.spack_shebang_limit = spack_limit +@pytest.mark.skipif(sys.platform == 'win32', + reason="Not supported on Windows (yet)") def test_shebang_exceeds_spack_shebang_limit(shebang_limits_system_8_spack_16, tmpdir): """Tests whether shebangs longer than Spack's limit are skipped""" file = str(tmpdir.join('longer_than_spack_limit.sh')) @@ -421,6 +432,8 @@ def test_shebang_exceeds_spack_shebang_limit(shebang_limits_system_8_spack_16, t assert b'sbang' not in f.read() +@pytest.mark.skipif(sys.platform == 'win32', + reason="Not supported on Windows (yet)") def test_sbang_hook_handles_non_writable_files_preserving_permissions(tmpdir): path = str(tmpdir.join('file.sh')) with open(path, 'w') as f: -- cgit v1.2.3-70-g09d2