Windows: add registry query and SDK/WDK packages (#33021)

* Add a WindowsRegistryView class, which can query for existing
  package installations on Windows. This is particularly important
  because some Windows packages (including those added here)
  do not allow two simultaneous installs, and this can be
  queried in order to provide a clear error message.
* Consolidate external path detection logic for Windows into
  WindowsKitExternalPaths and WindowsCompilerExternalPaths objects.
* Add external-only packages win-sdk and wgl
* Add win-wdk (including external detection) which depends on
  win-sdk
* Replace prior msmpi implementation with a source-based install
  (depends on win-wdk). This install can control the install
  destination (unlike the binary installation).
* Update MSVC compiler to choose vcvars based on win-sdk dependency
* Provide "msbuild" module-level variable to packages during build
* When creating symlinks on Windows, need to explicitly specify when
  a symlink target is a directory
* executables_in_path no-longer defaults to using PATH (this is
  now expected to be taken care of by the caller)

6 files changed, 409 insertions, 21 deletions

diff --git a/var/spack/repos/builtin/packages/msmpi/ifort_compat.patch b/var/spack/repos/builtin/packages/msmpi/ifort_compat.patch
new file mode 100644
index 0000000000..54902d8f5f
--- /dev/null
+++ b/var/spack/repos/builtin/packages/msmpi/ifort_compat.patch
@@ -0,0 +1,34 @@
+diff --git a/src/mpi/msmpi/dll/msmpi.vcxproj b/src/mpi/msmpi/dll/msmpi.vcxproj
+index 255b9f5..cc4f096 100644
+--- a/src/mpi/msmpi/dll/msmpi.vcxproj
++++ b/src/mpi/msmpi/dll/msmpi.vcxproj
+@@ -57,6 +57,9 @@
+         $(OutDir)\..\mpi_debugger\mpi_debugger.lib;
+         $(CRT_Libs);
+       </AdditionalDependencies>
++      <AdditionalLibraryDirectories>
++        $(SPACK_IFORT)compiler\lib\intel64
++      </AdditionalLibraryDirectories>
+       <ModuleDefinitionFile>.\msmpi.def</ModuleDefinitionFile>
+     </Link>
+   </ItemDefinitionGroup>
+diff --git a/src/mpi/msmpi/fortran/lib/mpifort.vcxproj b/src/mpi/msmpi/fortran/lib/mpifort.vcxproj
+index 24bd29d..57d0292 100644
+--- a/src/mpi/msmpi/fortran/lib/mpifort.vcxproj
++++ b/src/mpi/msmpi/fortran/lib/mpifort.vcxproj
+@@ -8,12 +8,12 @@
+   </PropertyGroup>
+   <Target Name="CompileFortran" AfterTargets="ClCompile" Inputs="@(ForCompile)" Outputs="@(ForCompile->'$(O)%(FileName).obj')">
+     <PropertyGroup Condition="'$(BuildArchitecture)'=='i386'">
+-      <Fort_Flags>-fno-underscoring -D_X86_=1 -Di386=1 -march=x86-64 -m32</Fort_Flags>
++      <Fort_Flags>/D _X86_=1 /D i386=1 -march=x86-64 -m32</Fort_Flags>
+     </PropertyGroup>
+     <PropertyGroup Condition="'$(BuildArchitecture)'=='amd64'">
+-      <Fort_Flags>-fno-underscoring -D_WIN64 -D_AMD64_ -DAMD64</Fort_Flags>
++      <Fort_Flags>/D _WIN64=1 /D _AMD64_=1 /D AMD64=1</Fort_Flags>
+     </PropertyGroup>
+-    <Exec Command="$(GFORTRAN_BIN)\gfortran.exe -I$(MPI_INC_ROOT) -c %(ForCompile.Identity) $(Fort_Flags)  -o $(O)\%(ForCompile.FileName).obj" />
++    <Exec Command="$(IFORT_BIN)\ifort.exe /I$(MPI_INC_ROOT) /c %(ForCompile.Identity) $(Fort_Flags) /names:lowercase /assume:nounderscore /o $(O)\%(ForCompile.FileName).obj" />
+     <ItemGroup>
+       <Lib Condition="'$(ConfigurationType)'=='StaticLibrary'" Include="@(ForCompile->'$(O)\%(Filename).obj')" />
+     </ItemGroup>
diff --git a/var/spack/repos/builtin/packages/msmpi/package.py b/var/spack/repos/builtin/packages/msmpi/package.py
index 0e5584cf29..a2206d797f 100644
--- a/var/spack/repos/builtin/packages/msmpi/package.py
+++ b/var/spack/repos/builtin/packages/msmpi/package.py
@@ -3,40 +3,61 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
+import os
+import platform
+import re
+
+from spack.build_systems.generic import GenericBuilder
 from spack.package import *
 
 
 class Msmpi(Package):
-    """A Windows-specced build of MPICH provided directly by
-    Microsoft Support Team
-    """
+    """MSMPI is a Windows port of MPICH provided by the Windows team"""
 
-    homepage = "https://www.microsoft.com/en-us/download/default.aspx"
-    maintainers = ["jpopelar"]
+    homepage = "https://docs.microsoft.com/en-us/message-passing-interface/microsoft-mpi"
+    url = "https://github.com/microsoft/Microsoft-MPI/archive/refs/tags/v10.1.1.tar.gz"
+    git = "https://github.com/microsoft/Microsoft-MPI.git"
 
     executable = ["mpiexec.exe"]
 
-    version(
-        "10.0",
-        sha256="7dae13797627726f67fab9c1d251aec2df9ecd25939984645ec05748bdffd396",
-        extension="exe",
-        expand=False,
-    )
+    version("10.1.1", sha256="63c7da941fc4ffb05a0f97bd54a67968c71f63389a0d162d3182eabba1beab3d")
+    version("10.0.0", sha256="cfb53cf53c3cf0d4935ab58be13f013a0f7ccb1189109a5b8eea0fcfdcaef8c1")
 
     provides("mpi")
 
-    conflicts("platform=linux")
-    conflicts("platform=darwin")
-    conflicts("platform=cray")
+    depends_on("win-wdk")
 
-    def url_for_version(self, version):
-        return "https://download.microsoft.com/download/A/E/0/AE002626-9D9D-448D-8197-1EA510E297CE/msmpisetup.exe"
+    patch("ifort_compat.patch")
 
-    def determine_version(self, exe):
-        output = Executable("mpiexec.exe")
+    @classmethod
+    def determine_version(cls, exe):
+        output = Executable(exe)()
         ver_str = re.search("[Version ([0-9.]+)]", output)
         return Version(ver_str.group(0)) if ver_str else None
 
+
+class GenericBuilder(GenericBuilder):
+    def setup_build_environment(self, env):
+        ifort_root = os.path.join(*self.compiler.fc.split(os.path.sep)[:-2])
+        env.set("SPACK_IFORT", ifort_root)
+
+    def is_64bit(self):
+        return platform.machine().endswith("64")
+
+    def build_command_line(self):
+        args = ["-noLogo"]
+        ifort_bin = self.compiler.fc
+        if not ifort_bin:
+            raise InstallError(
+                "Cannot install MSMPI without fortran"
+                "please select a compiler with fortran support."
+            )
+        args.append("/p:IFORT_BIN=%s" % os.path.dirname(ifort_bin))
+        args.append("/p:VCToolsVersion=%s" % self.compiler.msvc_version)
+        args.append("/p:WindowsTargetPlatformVersion=%s" % str(self.spec["wdk"].version))
+        args.append("/p:PlatformToolset=%s" % self.compiler.cc_version)
+        return args
+
     def install(self, spec, prefix):
-        installer = Executable("msmpisetup.exe")
-        installer("-unattend")
+        with working_dir(self.stage.build_directory, create=True):
+            msbuild(*self.build_command_line())
diff --git a/var/spack/repos/builtin/packages/perl/package.py b/var/spack/repos/builtin/packages/perl/package.py
index 76ee382ef4..cca48bf2cf 100644
--- a/var/spack/repos/builtin/packages/perl/package.py
+++ b/var/spack/repos/builtin/packages/perl/package.py
@@ -229,7 +229,7 @@ class Perl(Package):  # Perl doesn't use Autotools, it should subclass Package
     def nmake_arguments(self):
         args = []
         if self.spec.satisfies("%msvc"):
-            args.append("CCTYPE=%s" % self.compiler.msvc_version)
+            args.append("CCTYPE=%s" % self.compiler.short_msvc_version)
         else:
             raise RuntimeError("Perl unsupported for non MSVC compilers on Windows")
         args.append("INST_TOP=%s" % self.prefix.replace("/", "\\"))
diff --git a/var/spack/repos/builtin/packages/wgl/package.py b/var/spack/repos/builtin/packages/wgl/package.py
new file mode 100644
index 0000000000..0c119b769e
--- /dev/null
+++ b/var/spack/repos/builtin/packages/wgl/package.py
@@ -0,0 +1,96 @@
+# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+import os
+import re
+
+from spack.package import *
+
+
+class Wgl(Package):
+    """External WGl and Windows OpenGL emulation representation in Spack"""
+
+    homepage = "https://learn.microsoft.com/en-us/windows/win32/opengl/wgl-and-windows-reference"
+    has_code = False
+
+    # hard code the extension as shared lib
+    libraries = ["OpenGL32.Lib"]
+
+    # versions here are in no way related to actual WGL versions
+    # (there is only one on a system at a time)
+    # but instead reflects the Windows Kit version that a particular WGL library file is found in
+    # Windows Kits are intended to be more or less contained environments so this allows us to
+    # marshall our SDK and WDK to their respective WGLs. The purpose here is to better reflect
+    # the concept of an MS build toolchain version W.R.T. to MSVC
+    version("10.0.22621")
+    version("10.0.19041")
+    version("10.0.18362")
+    version("10.0.17763")
+    version("10.0.17134")
+    version("10.0.16299")
+    version("10.0.15063")
+    version("10.0.14393")
+    version("10.0.10586")
+    version("10.0.26639")
+
+    # As per https://github.com/spack/spack/pull/31748 this provisory version represents
+    # an arbitrary openGL version designed for maximum compatibility with calling packages
+    # this current version simply reflects the latest OpenGL vesion available at the time of
+    # package creation and is set in a way that all specs currently depending on GL are
+    # satisfied appropriately
+    provides("gl@4.6")
+
+    variant("plat", values=("x64", "x86", "arm", "arm64"), default="x64")
+
+    # WGL exists on all Windows systems post win 98, however the headers
+    # needed to use OpenGL are found in the SDK (GL/gl.h)
+    # Dep is needed to consolidate sdk version to locate header files for
+    # version of SDK being used
+    depends_on("win-sdk@10.0.19041", when="@10.0.19041")
+    depends_on("win-sdk@10.0.18362", when="@10.0.18362")
+    depends_on("win-sdk@10.0.17763", when="@10.0.17763")
+    depends_on("win-sdk@10.0.17134", when="@10.0.17134")
+    depends_on("win-sdk@10.0.16299", when="@10.0.16299")
+    depends_on("win-sdk@10.0.15063", when="@10.0.15063")
+    depends_on("win-sdk@10.0.14393", when="@10.0.14393")
+
+    # WGL has no meaning on other platforms, should not be able to spec
+    for plat in ["linux", "darwin", "cray"]:
+        conflicts("platform=%s" % plat)
+
+    @classmethod
+    def determine_version(cls, lib):
+        """Allow for WGL to be externally detectable"""
+        version_match_pat = re.compile(r"[0-9][0-9].[0-9]+.[0-9][0-9][0-9][0-9][0-9]")
+        ver_str = re.search(version_match_pat, lib)
+        return ver_str if not ver_str else Version(ver_str.group())
+
+    @classmethod
+    def determine_variants(cls, libs, ver_str):
+        """Allow for determination of toolchain arch for detected WGL"""
+        variants = []
+        for lib in libs:
+            base, lib_name = os.path.split(lib)
+            _, arch = os.path.split(base)
+            variants.append("plat=%s" % arch)
+        return variants
+
+    # As noted above, the headers neccesary to include
+    @property
+    def headers(self):
+        return find_headers("GL/gl.h", root=self.spec["win-sdk"].prefix.includes, recursive=True)
+
+    @property
+    def libs(self):
+        return find_libraries("opengl32", shared=False, root=self.prefix, recursive=True)
+
+    def install(self, spec, prefix):
+        raise RuntimeError(
+            "This package is not installable from Spack\
+            and should be installed on the system prior to Spack use.\
+                If not installed this package should be installed via\
+                    the Visual Studio installer in order to use the \
+                        MSVC compiler on Windows."
+        )
diff --git a/var/spack/repos/builtin/packages/win-sdk/package.py b/var/spack/repos/builtin/packages/win-sdk/package.py
new file mode 100644
index 0000000000..8ac2992205
--- /dev/null
+++ b/var/spack/repos/builtin/packages/win-sdk/package.py
@@ -0,0 +1,90 @@
+# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+
+import os
+import re
+
+from spack.package import *
+
+
+class WinSdk(Package):
+    """
+    Windows Desktop C++ development SDK
+    Spack packaged used to define search heuristics
+    to locate the SDK on a filesystem
+    """
+
+    homepage = "https://developer.microsoft.com/en-us/windows/downloads/windows-sdk/"
+    has_code = False
+
+    # The sdk has many libraries and executables. Record one for detection purposes
+    libraries = ["rcdll.dll"]
+
+    version("10.0.22621")
+    version("10.0.19041")
+    version("10.0.18362")
+    version("10.0.17763")
+    version("10.0.17134")
+    version("10.0.16299")
+    version("10.0.15063")
+    version("10.0.14393")
+    version("10.0.10586")
+    version("10.0.26639")
+
+    variant("plat", values=("x64", "x86", "arm", "arm64"), default="x64")
+
+    # WinSDK versions depend on compatible compilers
+    # WDK versions do as well, but due to their one to one dep on the SDK
+    # we can ensure that requirment here
+    # WinSDK is very backwards compatible, however older
+    # MSVC editions may have problems with newer SDKs
+    conflicts("%msvc@:19.16.00000", when="@10.0.19041")
+    conflicts("%msvc@:19.16.00000", when="@10.0.18362")
+    conflicts("%msvc@:19.15.00000", when="@10.0.17763")
+    conflicts("%msvc@:19.14.00000", when="@10.0.17134")
+    conflicts("%msvc@:19.11.00000", when="@10.0.16299")
+    conflicts("%msvc@:19.10.00000", when="@10.0.15063")
+    conflicts("%msvc@:19.10.00000", when="@10.0.14393")
+    conflicts("%msvc@:19.00.00000", when="@10.0.10586")
+
+    # For now we don't support Windows development env
+    # on other platforms
+    for plat in ["linux", "darwin", "cray"]:
+        conflicts("platform=%s" % plat)
+
+    @classmethod
+    def determine_version(cls, lib):
+        """
+        WinSDK that we would like to
+        be discoverable externally by Spack.
+        """
+        # This version is found in the package's path
+        # not by calling an exe or a libraries name
+        version_match_pat = re.compile(r"[0-9][0-9].[0-9]+.[0-9][0-9][0-9][0-9][0-9]")
+        ver_str = re.search(version_match_pat, lib)
+        return ver_str if not ver_str else Version(ver_str.group())
+
+    @classmethod
+    def determine_variants(cls, libs, ver_str):
+        """Allow for determination of toolchain arch for detected WGL"""
+        variants = []
+        for lib in libs:
+            base, lib_name = os.path.split(lib)
+            _, arch = os.path.split(base)
+            variants.append("plat=%s" % arch)
+        return variants
+
+    def install(self, spec, prefix):
+        raise RuntimeError(
+            "This package is not installable from Spack\
+            and should be installed on the system prior to Spack use.\
+                If not installed this package should be installed via\
+                    the Visual Studio installer in order to use the \
+                        MSVC compiler on Windows."
+            "If absolutely neccesary this SDK can be installed directly from Microsoft\
+                but this approach is not recommended unless you know what you're doing \
+                    or if you're on Windows 11 you have no choice for the moment."
+        )
diff --git a/var/spack/repos/builtin/packages/win-wdk/package.py b/var/spack/repos/builtin/packages/win-wdk/package.py
new file mode 100644
index 0000000000..652932ccf8
--- /dev/null
+++ b/var/spack/repos/builtin/packages/win-wdk/package.py
@@ -0,0 +1,147 @@
+# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+
+import glob
+import os
+import re
+
+import spack.util.windows_registry as winreg
+from spack.package import *
+
+
+class WinWdk(Package):
+    """
+    Windows Driver Kit development package
+    """
+
+    homepage = "https://learn.microsoft.com/en-us/windows-hardware/drivers/"
+
+    # The wdk has many libraries and executables. Record one for detection purposes
+    libraries = ["mmos.lib"]
+
+    version(
+        "10.0.19041",
+        sha256="5f4ea0c55af099f97cb569a927c3a290c211f17edcfc65009f5b9253b9827925",
+        url="https://go.microsoft.com/fwlink/?linkid=2128854",
+        expand=False,
+    )
+    version(
+        "10.0.18362",
+        sha256="c35057cb294096c63bbea093e5024a5fb4120103b20c13fa755c92f227b644e5",
+        url="https://go.microsoft.com/fwlink/?linkid=2085767",
+        expand=False,
+    )
+    version(
+        "10.0.17763",
+        sha256="e6e5a57bf0a58242363cd6ca4762f44739f19351efc06cad382cca944b097235",
+        url="https://go.microsoft.com/fwlink/?linkid=2026156",
+        expand=False,
+    )
+    version(
+        "10.0.17134",
+        sha256="48e636117bb7bfe66b1ade793cc8e885c42c880fadaee471782d31b5c4d13e9b",
+        url="https://go.microsoft.com/fwlink/?linkid=873060",
+        expand=False,
+    )
+    version(
+        "10.0.16299",
+        sha256="14efbcc849e5977417e962f1cd68357d21abf27393110b9d95983ad03fc22ef4",
+        url="https://go.microsoft.com/fwlink/p/?linkid=859232",
+        expand=False,
+    )
+    version(
+        "10.0.15063",
+        sha256="489b497111bc791d9021b3573bfd93086a28b598c7325ab255e81c6f5d80a820",
+        url="https://go.microsoft.com/fwlink/p/?LinkID=845980",
+        expand=False,
+    )
+    version(
+        "10.0.14393",
+        sha256="0bfb2ac9db446e0d98c29ef7341a8c8e8e7aa24bc72b00c5704a88b13f48b3cb",
+        url="https://go.microsoft.com/fwlink/p/?LinkId=526733",
+        expand=False,
+    )
+
+    variant("plat", values=("x64", "x86", "arm", "arm64"), default="x64")
+
+    # need one to one dep on SDK per https://github.com/MicrosoftDocs/windows-driver-docs/issues/1550
+    # additionally, the WDK needs to be paired with a version of the Windows SDK
+    # as per https://learn.microsoft.com/en-us/windows-hardware/drivers/download-the-wdk#download-icon-step-2-install-windows-11-version-22h2-sdk
+    depends_on("win-sdk@10.0.19041", when="@10.0.19041")
+    depends_on("win-sdk@10.0.18362", when="@10.0.18362")
+    depends_on("win-sdk@10.0.17763", when="@10.0.17763")
+    depends_on("win-sdk@10.0.17134", when="@10.0.17134")
+    depends_on("win-sdk@10.0.16299", when="@10.0.16299")
+    depends_on("win-sdk@10.0.15063", when="@10.0.15063")
+    depends_on("win-sdk@10.0.14393", when="@10.0.14393")
+
+    for plat in ["linux", "darwin", "cray"]:
+        conflicts("platform=%s" % plat)
+
+    @classmethod
+    def determine_version(cls, lib):
+        """
+        WDK is a set of drivers that we would like to
+        be discoverable externally by Spack.
+        The lib does not provide the WDK
+        version so we derive from the lib path
+        """
+        version_match_pat = re.compile(r"[0-9][0-9].[0-9]+.[0-9][0-9][0-9][0-9][0-9]")
+        ver_str = re.search(version_match_pat, lib)
+        return ver_str if not ver_str else Version(ver_str.group())
+
+    @classmethod
+    def determine_variants(cls, libs, ver_str):
+        """Allow for determination of toolchain arch for detected WGL"""
+        variants = []
+        for lib in libs:
+            base, lib_name = os.path.split(lib)
+            _, arch = os.path.split(base)
+            variants.append("plat=%s" % arch)
+        return variants
+
+    def setup_dependent_environment(self):
+        # This points to all core build extensions needed to build
+        # drivers on Windows
+        os.environ["WDKContentRoot"] = self.prefix
+
+    @run_before("install")
+    def rename_downloaded_executable(self):
+        """WGL download is named by fetch based on name derived from Link redirection
+        This name is not properly formated so that Windows understands it as an executable
+        We rename so as to allow Windows to run the WGL installer"""
+        installer = glob.glob(os.path.join(self.stage.source_path, "linkid=**"))
+        if len(installer) > 1:
+            raise RuntimeError(
+                "Fetch has failed, unable to determine installer path from:\n%s"
+                % "\n".join(installer)
+            )
+        installer = installer[0]
+        os.rename(installer, os.path.join(self.stage.source_path, "wdksetup.exe"))
+
+    def install(self, spec, prefix):
+        install_args = ["/features", "+", "/quiet", "/installpath", self.prefix]
+        with working_dir(self.stage.source_path):
+            try:
+                Executable("wdksetup.exe")(*install_args)
+            except ProcessError as pe:
+                reg = winreg.WindowsRegistryView(
+                    "SOFTWARE\\Microsoft\\Windows Kits\\Installed Roots",
+                    root_key=spack.util.windows_registry.HKEY_LOCAL_MACHINE,
+                )
+                if not reg:
+                    # No Kits are available, failure was genuine
+                    raise pe
+                else:
+                    versions = [str(subkey) for subkey in reg.get_subkeys()]
+                    versions = ",".join(versions) if len(versions) > 1 else versions[0]
+                    plural = "s" if len(versions) > 1 else ""
+                    raise InstallError(
+                        "Cannot install WDK version %s. "
+                        "Version%s %s already present on system."
+                        "Please run `spack external find win-wdk` to use the WDK"
+                        % (self.version, plural, versions)
+                    )