openssl: prefer 3.x (#36729)

* openssl: prefer 3.x

This PR is not intended to be merged immediately, but it would be good
to see what packages fail to build in CI so that we can get proper
version constraints on openssl (before all packages update and support
both openssl 1 and 3)

* Disable assembly for 3.x %oneapi

* cmake: depend on spack curl, to deal with curl - openssl compat

* also make zlib external

* remove overly strict & unsafe requirement on py-cryptographty patch version number

* update openssl compat bounds in py-cryptography

* smaller diff

* Make libssh2 an autotools/cmake package

* fix weird upperbound in libssh2 as there is not openssl v2

* libssh2: pc file lists plain -lssl -lcrypto w/o leading -L flag, confusing libgit2 parsing of pkg-config output

* Actually fix the issue in libssh2: its pc file looks broken

7 files changed, 99 insertions, 51 deletions

diff --git a/share/spack/gitlab/cloud_pipelines/stacks/e4s-oneapi/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/e4s-oneapi/spack.yaml
index 62004d5986..461560d592 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/e4s-oneapi/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/e4s-oneapi/spack.yaml
@@ -42,7 +42,7 @@ spack:
     mpich:
       require: '@4.1.1 ~wrapperrpath ~hwloc'
     py-cryptography:
-      require: '@38.0.1'
+      require: '@38.0'
     unzip:
       require: '%gcc'
     binutils:
diff --git a/var/spack/repos/builtin/packages/cmake/package.py b/var/spack/repos/builtin/packages/cmake/package.py
index a70f04152d..2cc358f41b 100644
--- a/var/spack/repos/builtin/packages/cmake/package.py
+++ b/var/spack/repos/builtin/packages/cmake/package.py
@@ -210,10 +210,20 @@ class Cmake(Package):
     # transparent to patch Spack's versions of CMake's dependencies.
     conflicts("+ownlibs %nvhpc")
 
+    # Use Spack's curl even if +ownlibs, since that allows us to make use of
+    # the conflicts on the curl package for TLS libs like OpenSSL.
+    # In the past we let CMake build a vendored copy of curl, but had to
+    # provide Spack's TLS libs anyways, which is not flexible, and actually
+    # leads to issues where we have to keep track of the vendored curl version
+    # and its conflicts with OpenSSL.
+    depends_on("curl")
+
+    # When using curl, cmake defaults to using system zlib too, probably because
+    # curl already depends on zlib. Therefore, also unconditionaly depend on zlib.
+    depends_on("zlib")
+
     with when("~ownlibs"):
-        depends_on("curl")
         depends_on("expat")
-        depends_on("zlib")
         # expat/zlib are used in CMake/CTest, so why not require them in libarchive.
         depends_on("libarchive@3.1.0: xar=expat compression=zlib")
         depends_on("libarchive@3.3.3:", when="@3.15.0:")
@@ -222,11 +232,6 @@ class Cmake(Package):
         depends_on("libuv@1.10.0:", when="@3.12.0:")
         depends_on("rhash", when="@3.8.0:")
 
-    for plat in ["darwin", "linux", "cray"]:
-        with when("+ownlibs platform=%s" % plat):
-            depends_on("openssl")
-            depends_on("openssl@:1.0", when="@:3.6.9")
-
     depends_on("qt", when="+qt")
     depends_on("ncurses", when="+ncurses")
 
@@ -311,11 +316,6 @@ class Cmake(Package):
                 flags.append(self.compiler.cxx11_flag)
         return (flags, None, None)
 
-    def setup_build_environment(self, env):
-        spec = self.spec
-        if "+ownlibs" in spec and "platform=windows" not in spec:
-            env.set("OPENSSL_ROOT_DIR", spec["openssl"].prefix)
-
     def bootstrap_args(self):
         spec = self.spec
         args = []
@@ -355,6 +355,9 @@ class Cmake(Package):
                     # use CMake-provided library to avoid circular dependency
                     args.append("--no-system-jsoncpp")
 
+            # Whatever +/~ownlibs, use system curl.
+            args.append("--system-curl")
+
             if "+qt" in spec:
                 args.append("--qt-gui")
             else:
@@ -369,21 +372,15 @@ class Cmake(Package):
         else:
             args.append("-DCMAKE_INSTALL_PREFIX=%s" % self.prefix)
 
-        args.append("-DCMAKE_BUILD_TYPE={0}".format(self.spec.variants["build_type"].value))
-
-        # Install CMake correctly, even if `spack install` runs
-        # inside a ctest environment
-        args.append("-DCMake_TEST_INSTALL=OFF")
-
-        # When building our own private copy of curl we still require an
-        # external openssl.
-        if "+ownlibs" in spec:
-            if "platform=windows" in spec:
-                args.append("-DCMAKE_USE_OPENSSL=OFF")
-            else:
-                args.append("-DCMAKE_USE_OPENSSL=ON")
-
-        args.append("-DBUILD_CursesDialog=%s" % str("+ncurses" in spec))
+        args.extend(
+            [
+                f"-DCMAKE_BUILD_TYPE={self.spec.variants['build_type'].value}",
+                # Install CMake correctly, even if `spack install` runs
+                # inside a ctest environment
+                "-DCMake_TEST_INSTALL=OFF",
+                f"-DBUILD_CursesDialog={'ON' if '+ncurses' in spec else 'OFF'}",
+            ]
+        )
 
         # Make CMake find its own dependencies.
         rpaths = spack.build_environment.get_rpaths(self)
diff --git a/var/spack/repos/builtin/packages/curl/package.py b/var/spack/repos/builtin/packages/curl/package.py
index b4f24e1a8c..f8a50829ac 100644
--- a/var/spack/repos/builtin/packages/curl/package.py
+++ b/var/spack/repos/builtin/packages/curl/package.py
@@ -278,7 +278,13 @@ class Curl(NMakePackage, AutotoolsPackage):
     depends_on("mbedtls@2: +pic", when="@7.79: tls=mbedtls")
     depends_on("mbedtls@:2 +pic", when="@:7.78 tls=mbedtls")
     depends_on("nss", when="tls=nss")
-    depends_on("openssl", when="tls=openssl")
+
+    with when("tls=openssl"):
+        depends_on("openssl")
+        # Since https://github.com/curl/curl/commit/ee36e86ce8f77a017c49b8312814c33f4b969565
+        # there is OpenSSL 3 detection.
+        depends_on("openssl@:1", when="@:7.76")
+
     depends_on("libidn2", when="+libidn2")
     depends_on("zlib")
     depends_on("nghttp2", when="+nghttp2")
diff --git a/var/spack/repos/builtin/packages/libssh2/package.py b/var/spack/repos/builtin/packages/libssh2/package.py
index f50bbfdf61..49619345e6 100644
--- a/var/spack/repos/builtin/packages/libssh2/package.py
+++ b/var/spack/repos/builtin/packages/libssh2/package.py
@@ -6,12 +6,13 @@
 from spack.package import *
 
 
-class Libssh2(CMakePackage):
+class Libssh2(AutotoolsPackage, CMakePackage):
     """libssh2 is a client-side C library implementing the SSH2 protocol"""
 
     homepage = "https://www.libssh2.org/"
     url = "https://www.libssh2.org/download/libssh2-1.7.0.tar.gz"
 
+    version("1.11.0", sha256="3736161e41e2693324deb38c26cfdc3efe6209d634ba4258db1cecff6a5ad461")
     version("1.10.0", sha256="2d64e90f3ded394b91d3a2e774ca203a4179f69aebee03003e5a6fa621e41d51")
     version("1.9.0", sha256="d5fb8bd563305fd1074dda90bd053fb2d29fc4bce048d182f96eaa466dfadafd")
     version("1.8.0", sha256="39f34e2f6835f4b992cafe8625073a88e5a28ba78f83e8099610a7b3af4676d4")
@@ -20,21 +21,37 @@ class Libssh2(CMakePackage):
         "1.4.3", sha256="eac6f85f9df9db2e6386906a6227eb2cd7b3245739561cad7d6dc1d5d021b96d"
     )  # CentOS7
 
-    variant("crypto", default="openssl", values=("openssl", "mbedtls"), multi=False)
+    build_system("autotools", "cmake", default="autotools")
+
+    variant("crypto", default="openssl", values=("openssl", conditional("mbedtls", when="@1.8:")))
     variant("shared", default=True, description="Build shared libraries")
 
-    conflicts("crypto=mbedtls", when="@:1.7", msg="mbedtls only available from 1.8.0")
+    with when("build_system=cmake"):
+        depends_on("cmake@2.8.11:", type="build")
+        # on macOS ensure CMP0042 is on (default in cmake 3.0+)
+        depends_on("cmake@3:", type="build", when="platform=darwin")
+
+    with when("crypto=openssl"):
+        depends_on("openssl")
+        depends_on("openssl@:1", when="@:1.9")
 
-    depends_on("cmake@2.8.11:", type="build")
-    depends_on("openssl", when="crypto=openssl")
-    depends_on("openssl@:2", when="@:1.9 crypto=openssl")
     depends_on("mbedtls@:2 +pic", when="crypto=mbedtls")
     depends_on("zlib")
     depends_on("xz")
 
+    # libssh2 adds its own deps in the pc file even when doing shared linking,
+    # and fails to prepend the -L flags, which is causing issues in libgit2, as
+    # it tries to locate e.g. libssl in the dirs of the pc file's -L flags, and
+    # cannot find the lib.
+    patch("pr-1114.patch", when="@1.7:")
+
+
+class CMakeBuilder(spack.build_systems.cmake.CMakeBuilder):
     def cmake_args(self):
         args = [
-            self.define("BUILD_TESTING", "OFF"),
+            self.define("BUILD_TESTING", False),
+            self.define("RUN_DOCKER_TESTS", False),
+            self.define("BUILD_EXAMPLES", False),
             self.define_from_variant("BUILD_SHARED_LIBS", "shared"),
         ]
 
@@ -47,13 +64,17 @@ class Libssh2(CMakePackage):
 
         return args
 
-    @run_after("install")
-    def darwin_fix(self):
-        # The shared library is not installed correctly on Darwin; fix this
-        if self.spec.satisfies("platform=darwin"):
-            fix_darwin_install_name(self.prefix.lib)
 
-    def check(self):
-        # Docker is required to run tests
-        if which("docker"):
-            make("test")
+class AutotoolsBuilder(spack.build_systems.autotools.AutotoolsBuilder):
+    def configure_args(self):
+        args = ["--disable-tests", "--disable-docker-tests", "--disable-examples-build"]
+        args += self.enable_or_disable("shared")
+
+        crypto = self.spec.variants["crypto"].value
+
+        if crypto == "openssl":
+            args.append(f"--with-libssl-prefix={self.spec['openssl'].prefix}")
+        elif crypto == "mbedtls":
+            args.append(f"--with-libmbedcrypto-prefix={self.spec['mbedtls'].prefix}")
+
+        return args
diff --git a/var/spack/repos/builtin/packages/libssh2/pr-1114.patch b/var/spack/repos/builtin/packages/libssh2/pr-1114.patch
new file mode 100644
index 0000000000..160c52c06e
--- /dev/null
+++ b/var/spack/repos/builtin/packages/libssh2/pr-1114.patch
@@ -0,0 +1,21 @@
+From 052b624e113912bb97bad877b81ddba22d37ba96 Mon Sep 17 00:00:00 2001
+From: Harmen Stoppels <harmenstoppels@gmail.com>
+Date: Sun, 9 Jul 2023 21:36:07 +0200
+Subject: [PATCH] Don't put `@LIBS@` in pc file
+
+---
+ libssh2.pc.in | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/libssh2.pc.in b/libssh2.pc.in
+index 4fdfaa924..9cbebb7f0 100644
+--- a/libssh2.pc.in
++++ b/libssh2.pc.in
+@@ -15,6 +15,6 @@ URL: https://www.libssh2.org/
+ Description: Library for SSH-based communication
+ Version: @LIBSSH2VER@
+ Requires.private: @LIBSREQUIRED@
+-Libs: -L${libdir} -lssh2 @LIBS@
++Libs: -L${libdir} -lssh2
+ Libs.private: @LIBS@
+ Cflags: -I${includedir}
diff --git a/var/spack/repos/builtin/packages/openssl/package.py b/var/spack/repos/builtin/packages/openssl/package.py
index e1fd117a20..a0d6c7428d 100644
--- a/var/spack/repos/builtin/packages/openssl/package.py
+++ b/var/spack/repos/builtin/packages/openssl/package.py
@@ -73,11 +73,7 @@ class Openssl(Package):  # Uses Fake Autotools, should subclass Package
 
     # The latest stable version is the 1.1.1 series. This is also our Long Term
     # Support (LTS) version, supported until 11th September 2023.
-    version(
-        "1.1.1u",
-        sha256="e2f8d84b523eecd06c7be7626830370300fbcc15386bf5142d72758f6963ebc6",
-        preferred=True,
-    )
+    version("1.1.1u", sha256="e2f8d84b523eecd06c7be7626830370300fbcc15386bf5142d72758f6963ebc6")
     version(
         "1.1.1t",
         sha256="8dee9b24bdb1dcbf0c3d1e9b02fb8f6bf22165e807f45adeb7c9677536859d3b",
@@ -403,6 +399,10 @@ class Openssl(Package):  # Uses Fake Autotools, should subclass Package
             # nvhpc segfaults NVC++-F-0000-Internal compiler error.
             # gen_llvm_expr(): unknown opcode       0  (crypto/rsa/rsa_oaep.c: 248)
             options.append("no-asm")
+        elif spec.satisfies("@3: %oneapi"):
+            # Last tested on oneapi@2023.1.0 for x86_64:
+            # crypto/md5/md5-x86_64.s:684:31: error: expected string
+            options.append("no-asm")
 
         # The default glibc provided by CentOS 7 does not provide proper
         # atomic support when using the NVIDIA compilers
diff --git a/var/spack/repos/builtin/packages/py-cryptography/package.py b/var/spack/repos/builtin/packages/py-cryptography/package.py
index 110b0642e6..cdbbe863f7 100644
--- a/var/spack/repos/builtin/packages/py-cryptography/package.py
+++ b/var/spack/repos/builtin/packages/py-cryptography/package.py
@@ -47,8 +47,11 @@ class PyCryptography(PythonPackage):
     depends_on("py-six@1.4.1:", type=("build", "run"), when="@:3.3")
     depends_on("py-idna@2.1:", type=("build", "run"), when="@:2.4")  # deprecated
     depends_on("py-idna@2.1:", type=("build", "run"), when="@2.5: +idna")  # deprecated
-    depends_on("openssl@:1.0", when="@:1.8.1")
+
     depends_on("openssl")
+    depends_on("openssl@:1.0", when="@:1.8.1")
+    depends_on("openssl@:1.1", when="@:3.4")
+    depends_on("openssl@1.1.1:", when="@39:")
 
     # To fix https://github.com/spack/spack/issues/29669
     # https://community.home-assistant.io/t/error-failed-building-wheel-for-cryptography/352020/14