From 31daf0f2b6bd769135e088294eff93a94342c3d2 Mon Sep 17 00:00:00 2001 From: John Parent Date: Fri, 22 Oct 2021 12:25:46 -0400 Subject: Add Windows to platform and target changes Add compiler hint to the root spec for Windows Reporters on Windows (#26038) Reporters use Jinja2 as the templating engine, and Jinja2 indexes templates by Unix separators, even on Windows, so search using Unix paths on all systems. Support patching on win via git (#25871) Handle GRP on windows --- lib/spack/spack/bootstrap.py | 4 +++- lib/spack/spack/cmd/modules/lmod.py | 1 - lib/spack/spack/hooks/sbang.py | 8 +++++++- lib/spack/spack/operating_systems/__init__.py | 6 ++++-- lib/spack/spack/operating_systems/windows_os.py | 7 ++++--- lib/spack/spack/patch.py | 11 +++++++++-- lib/spack/spack/platforms/__init__.py | 2 ++ lib/spack/spack/platforms/_functions.py | 3 ++- lib/spack/spack/platforms/windows.py | 6 ++++-- lib/spack/spack/reporters/cdash.py | 13 +++++++------ lib/spack/spack/reporters/junit.py | 5 ++--- lib/spack/spack/test/sbang.py | 4 +++- 12 files changed, 47 insertions(+), 23 deletions(-) diff --git a/lib/spack/spack/bootstrap.py b/lib/spack/spack/bootstrap.py index c44f217ccd..335999a120 100644 --- a/lib/spack/spack/bootstrap.py +++ b/lib/spack/spack/bootstrap.py @@ -727,9 +727,11 @@ def _root_spec(spec_str): spec_str (str): spec to be bootstrapped. Must be without compiler and target. """ # Add a proper compiler hint to the root spec. We use GCC for - # everything but MacOS. + # everything but MacOS and Windows. if str(spack.platforms.host()) == 'darwin': spec_str += ' %apple-clang' + elif str(spack.platforms.host()) == 'windows': + spec_str += ' %msvc' else: spec_str += ' %gcc' diff --git a/lib/spack/spack/cmd/modules/lmod.py b/lib/spack/spack/cmd/modules/lmod.py index abd89b3a18..fec663e53e 100644 --- a/lib/spack/spack/cmd/modules/lmod.py +++ b/lib/spack/spack/cmd/modules/lmod.py @@ -61,7 +61,6 @@ def setdefault(module_type, specs, args): writer = spack.modules.module_types['lmod'](spec, args.module_set_name) writer.update_module_defaults() - module_folder = os.path.dirname(writer.layout.filename) module_basename = os.path.basename(writer.layout.filename) with llnl.util.filesystem.working_dir(module_folder): diff --git a/lib/spack/spack/hooks/sbang.py b/lib/spack/spack/hooks/sbang.py index 75eff66281..2793906ebd 100644 --- a/lib/spack/spack/hooks/sbang.py +++ b/lib/spack/spack/hooks/sbang.py @@ -4,7 +4,6 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) import filecmp -import grp import os import re import shutil @@ -15,6 +14,7 @@ import tempfile import llnl.util.filesystem as fs import llnl.util.tty as tty +import spack.error import spack.package_prefs import spack.paths import spack.spec @@ -28,6 +28,12 @@ if sys.platform == 'darwin': else: system_shebang_limit = 127 +#: Groupdb does not exist on Windows, prevent imports +#: on supported systems +is_windows = str(spack.platforms.host()) == 'windows' +if not is_windows: + import grp + #: Spack itself also limits the shebang line to at most 4KB, which should be plenty. spack_shebang_limit = 4096 diff --git a/lib/spack/spack/operating_systems/__init__.py b/lib/spack/spack/operating_systems/__init__.py index bd2377d80c..5a029ae5a8 100644 --- a/lib/spack/spack/operating_systems/__init__.py +++ b/lib/spack/spack/operating_systems/__init__.py @@ -7,14 +7,16 @@ from .cray_backend import CrayBackend from .cray_frontend import CrayFrontend from .linux_distro import LinuxDistro from .mac_os import MacOs +from .windows_os import WindowsOs __all__ = [ 'OperatingSystem', 'LinuxDistro', 'MacOs', 'CrayFrontend', - 'CrayBackend' + 'CrayBackend', + 'WindowsOs' ] #: List of all the Operating Systems known to Spack -operating_systems = [LinuxDistro, MacOs, CrayFrontend, CrayBackend] +operating_systems = [LinuxDistro, MacOs, CrayFrontend, CrayBackend, WindowsOs] diff --git a/lib/spack/spack/operating_systems/windows_os.py b/lib/spack/spack/operating_systems/windows_os.py index 7d5937c4a9..06113a4434 100755 --- a/lib/spack/spack/operating_systems/windows_os.py +++ b/lib/spack/spack/operating_systems/windows_os.py @@ -8,9 +8,10 @@ import os import subprocess import sys -from spack.architecture import OperatingSystem from spack.version import Version +from ._operating_system import OperatingSystem + # FIXME: To get the actual Windows version, we need a python that runs # natively on Windows, not Cygwin. @@ -37,14 +38,14 @@ class WindowsOs(OperatingSystem): extra_args = {} if sys.version_info[:3] >= (3, 6, 0): extra_args = {'encoding': 'mbcs', 'errors': 'strict'} - paths = subprocess.check_output([ # novermin + paths = subprocess.check_output([ # type: ignore[call-overload] # novermin os.path.join(root, "Microsoft Visual Studio", "Installer", "vswhere.exe"), "-prerelease", "-requires", "Microsoft.VisualStudio.Component.VC.Tools.x86.x64", "-property", "installationPath", "-products", "*", - ], **extra_args).strip() # type: ignore[call-overload] + ], **extra_args).strip() if (3, 0) <= sys.version_info[:2] <= (3, 5): paths = paths.decode() vs_install_paths = paths.split('\n') diff --git a/lib/spack/spack/patch.py b/lib/spack/spack/patch.py index d248966ede..64f3919029 100644 --- a/lib/spack/spack/patch.py +++ b/lib/spack/spack/patch.py @@ -19,7 +19,7 @@ import spack.stage import spack.util.spack_json as sjson from spack.util.compression import allowed_archive from spack.util.crypto import Checker, checksum -from spack.util.executable import which +from spack.util.executable import which, which_string def apply_patch(stage, patch_path, level=1, working_dir='.'): @@ -32,7 +32,14 @@ def apply_patch(stage, patch_path, level=1, working_dir='.'): working_dir (str): relative path *within* the stage to change to (default '.') """ - patch = which("patch", required=True) + git_utils_path = os.environ.get('PATH', '') + if os.name == 'nt': + git = which_string('git', required=True) + git_root = os.path.dirname(git).split('/')[:-1] + git_root.extend(['usr', 'bin']) + git_utils_path = os.sep.join(git_root) + + patch = which("patch", required=True, path=git_utils_path) with llnl.util.filesystem.working_dir(stage.source_path): patch('-s', '-p', str(level), diff --git a/lib/spack/spack/platforms/__init__.py b/lib/spack/spack/platforms/__init__.py index 83de03de12..50acc3ec42 100644 --- a/lib/spack/spack/platforms/__init__.py +++ b/lib/spack/spack/platforms/__init__.py @@ -10,6 +10,7 @@ from .cray import Cray from .darwin import Darwin from .linux import Linux from .test import Test +from .windows import Windows __all__ = [ 'Platform', @@ -17,6 +18,7 @@ __all__ = [ 'Darwin', 'Linux', 'Test', + 'Windows', 'platforms', 'host', 'by_name', diff --git a/lib/spack/spack/platforms/_functions.py b/lib/spack/spack/platforms/_functions.py index 7d3a440a3a..503be5afcb 100644 --- a/lib/spack/spack/platforms/_functions.py +++ b/lib/spack/spack/platforms/_functions.py @@ -12,9 +12,10 @@ from .cray import Cray from .darwin import Darwin from .linux import Linux from .test import Test +from .windows import Windows #: List of all the platform classes known to Spack -platforms = [Cray, Darwin, Linux, Test] +platforms = [Cray, Darwin, Linux, Windows, Test] @llnl.util.lang.memoized diff --git a/lib/spack/spack/platforms/windows.py b/lib/spack/spack/platforms/windows.py index 7c9ed1443d..552e520193 100755 --- a/lib/spack/spack/platforms/windows.py +++ b/lib/spack/spack/platforms/windows.py @@ -7,9 +7,11 @@ import platform import archspec.cpu -from spack.architecture import Platform, Target +import spack.target from spack.operating_systems.windows_os import WindowsOs +from ._platform import Platform + class Windows(Platform): priority = 101 @@ -20,7 +22,7 @@ class Windows(Platform): super(Windows, self).__init__('windows') for name in archspec.cpu.TARGETS: - self.add_target(name, Target(name)) + self.add_target(name, spack.target.Target(name)) self.default = archspec.cpu.host().name self.front_end = self.default diff --git a/lib/spack/spack/reporters/cdash.py b/lib/spack/spack/reporters/cdash.py index 212283f39e..06f4479b14 100644 --- a/lib/spack/spack/reporters/cdash.py +++ b/lib/spack/spack/reporters/cdash.py @@ -7,6 +7,7 @@ import collections import hashlib import os.path import platform +import posixpath import re import socket import time @@ -61,7 +62,7 @@ class CDash(Reporter): def __init__(self, args): Reporter.__init__(self, args) self.success = True - self.template_dir = os.path.join('reports', 'cdash') + self.template_dir = posixpath.join('reports', 'cdash') self.cdash_upload_url = args.cdash_upload_url if self.cdash_upload_url: @@ -219,11 +220,11 @@ class CDash(Reporter): if phase != 'update': # Update.xml stores site information differently # than the rest of the CTest XML files. - site_template = os.path.join(self.template_dir, 'Site.xml') + site_template = posixpath.join(self.template_dir, 'Site.xml') t = env.get_template(site_template) f.write(t.render(report_data)) - phase_template = os.path.join(self.template_dir, report_name) + phase_template = posixpath.join(self.template_dir, report_name) t = env.get_template(phase_template) f.write(t.render(report_data)) self.upload(phase_report) @@ -346,11 +347,11 @@ class CDash(Reporter): if phase != 'update': # Update.xml stores site information differently # than the rest of the CTest XML files. - site_template = os.path.join(self.template_dir, 'Site.xml') + site_template = posixpath.join(self.template_dir, 'Site.xml') t = env.get_template(site_template) f.write(t.render(report_data)) - phase_template = os.path.join(self.template_dir, report_name) + phase_template = posixpath.join(self.template_dir, report_name) t = env.get_template(phase_template) f.write(t.render(report_data)) self.upload(phase_report) @@ -376,7 +377,7 @@ class CDash(Reporter): report_data['update']['log'] = msg env = spack.tengine.make_environment() - update_template = os.path.join(self.template_dir, 'Update.xml') + update_template = posixpath.join(self.template_dir, 'Update.xml') t = env.get_template(update_template) output_filename = os.path.join(directory_name, 'Update.xml') with open(output_filename, 'w') as f: diff --git a/lib/spack/spack/reporters/junit.py b/lib/spack/spack/reporters/junit.py index 323010c46f..bcd777e532 100644 --- a/lib/spack/spack/reporters/junit.py +++ b/lib/spack/spack/reporters/junit.py @@ -3,8 +3,7 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) -import os -import os.path +import posixpath import spack.build_environment import spack.fetch_strategy @@ -19,7 +18,7 @@ class JUnit(Reporter): def __init__(self, args): Reporter.__init__(self, args) - self.template_file = os.path.join('reports', 'junit.xml') + self.template_file = posixpath.join('reports', 'junit.xml') def build_report(self, filename, report_data): # Write the report diff --git a/lib/spack/spack/test/sbang.py b/lib/spack/spack/test/sbang.py index f0cc1835d9..4488d8845a 100644 --- a/lib/spack/spack/test/sbang.py +++ b/lib/spack/spack/test/sbang.py @@ -7,7 +7,6 @@ Test that Spack's shebang filtering works correctly. """ import filecmp -import grp import os import shutil import stat @@ -24,6 +23,9 @@ import spack.store import spack.util.spack_yaml as syaml from spack.util.executable import which +if sys.platform != 'win32': + import grp + too_long = sbang.system_shebang_limit + 1 -- cgit v1.2.3-70-g09d2