py-pyspark: new versions (#35129)

* py-pyspark: new versions

* py-py4j: new versions

* py-py4j: more style

* py-py4j: even more style

* py-spark, py4j: review remarks

2 files changed, 29 insertions, 8 deletions

diff --git a/var/spack/repos/builtin/packages/py-py4j/package.py b/var/spack/repos/builtin/packages/py-py4j/package.py
index bcc17e4456..414affc65d 100644
--- a/var/spack/repos/builtin/packages/py-py4j/package.py
+++ b/var/spack/repos/builtin/packages/py-py4j/package.py
@@ -13,14 +13,23 @@ class PyPy4j(PythonPackage):
     homepage = "https://www.py4j.org/"
     pypi = "py4j/py4j-0.10.4.zip"
 
-    version(
-        "0.10.9",
-        url="https://files.pythonhosted.org/packages/10/ed/9dad8808c34bc5b0b1a44b13c0bddeea0c8df59956eb20ac2fecc373f322/py4j-0.10.9.tar.gz",
-        sha256="36ec57f43ff8ced260a18aa9a4e46c3500a730cac8860e259cbaa546c2b9db2f",
-    )
+    version("0.10.9.5", sha256="276a4a3c5a2154df1860ef3303a927460e02e97b047dc0a47c1c3fb8cce34db6")
+    version("0.10.9.3", sha256="0d92844da4cb747155b9563c44fc322c9a1562b3ef0979ae692dbde732d784dd")
+    version("0.10.9", sha256="36ec57f43ff8ced260a18aa9a4e46c3500a730cac8860e259cbaa546c2b9db2f")
     version("0.10.7", sha256="721189616b3a7d28212dfb2e7c6a1dd5147b03105f1fc37ff2432acd0e863fa5")
     version("0.10.6", sha256="d3e7ac7c2171c290eba87e70aa5095b7eb6d6ad34789c007c88d550d9f575083")
     version("0.10.4", sha256="406fbfdbcbbb398739f61fafd25724670a405a668eb08c1721d832eadce06aae")
     version("0.10.3", sha256="f4570108ad014dd52a65c2288418e31cb8227b5ecc39ad7fc7fe98314f7a26f2")
 
     depends_on("py-setuptools", type="build")
+
+    def url_for_version(self, version):
+        url = "https://pypi.io/packages/source/p/py4j/"
+
+        if version < Version("0.10.9"):
+            url += "py4j-{0}.zip"
+        else:
+            url += "py4j-{0}.tar.gz"
+
+        url = url.format(version)
+        return url
diff --git a/var/spack/repos/builtin/packages/py-pyspark/package.py b/var/spack/repos/builtin/packages/py-pyspark/package.py
index b5abc03d0b..541852cbbd 100644
--- a/var/spack/repos/builtin/packages/py-pyspark/package.py
+++ b/var/spack/repos/builtin/packages/py-pyspark/package.py
@@ -12,15 +12,27 @@ class PyPyspark(PythonPackage):
     homepage = "https://spark.apache.org"
     pypi = "pyspark/pyspark-3.0.1.tar.gz"
 
+    version("3.3.1", sha256="e99fa7de92be406884bfd831c32b9306a3a99de44cfc39a2eefb6ed07445d5fa")
+    version("3.3.0", sha256="7ebe8e9505647b4d124d5a82fca60dfd3891021cf8ad6c5ec88777eeece92cf7")
+    version("3.2.1", sha256="0b81359262ec6e9ac78c353344e7de026027d140c6def949ff0d80ab70f89a54")
+    version("3.1.3", sha256="39ac641ef5559a3d1286154779fc990316e9934520853615ae4785c1af52d14b")
+    version("3.1.2", sha256="5e25ebb18756e9715f4d26848cc7e558035025da74b4fc325a0ebc05ff538e65")
     version("3.0.1", sha256="38b485d3634a86c9a2923c39c8f08f003fdd0e0a3d7f07114b2fb4392ce60479")
     version("2.4.4", sha256="13655eb113b8cf5f3f85b24fd92f86c4668a723723bd68949d028fa0df2cf694")
     version("2.3.2", sha256="7fb3b4fe47edb0fb78cecec37e0f2a728590f17ef6a49eae55141a7a374c07c8")
     version("2.3.0", sha256="0b3536910e154c36a94239f0ba0a201f476aadc72006409e5787198ffd01986e")
 
+    depends_on("python@3.7:", when="@3.3.0:", type=("build", "run"))
+    depends_on("python@3.6:", when="@3:3.2.3", type=("build", "run"))
     depends_on("python@2.7:3.7", when="@:2", type=("build", "run"))
     depends_on("python@2.7:", type=("build", "run"))
     depends_on("py-setuptools", type="build")
-    depends_on("py-py4j@0.10.9", when="@3.0.1", type=("build", "run"))
-    depends_on("py-py4j@0.10.7", when="@2.4.4", type=("build", "run"))
-    depends_on("py-py4j@0.10.7", when="@2.3.2", type=("build", "run"))
+    depends_on("py-py4j@0.10.9.5", when="@3.3.0:", type=("build", "run"))
+    depends_on("py-py4j@0.10.9.3", when="@3.2.1", type=("build", "run"))
+    depends_on("py-py4j@0.10.9", when="@3.0.1:3.1.3", type=("build", "run"))
+    depends_on("py-py4j@0.10.7", when="@2.3.2:2.4.4", type=("build", "run"))
     depends_on("py-py4j@0.10.6", when="@2.3.0", type=("build", "run"))
+
+    def setup_run_environment(self, env):
+        env.set("PYSPARK_PYTHON", self.spec["python"].command.path)
+        env.set("PYSPARK_DRIVER_PYTHON", self.spec["python"].command.path)