PyTorch: update ecosystem (#45431)

5 files changed, 40 insertions, 17 deletions

diff --git a/var/spack/repos/builtin/packages/py-torch/package.py b/var/spack/repos/builtin/packages/py-torch/package.py
index 663f7d4901..327034ed7b 100644
--- a/var/spack/repos/builtin/packages/py-torch/package.py
+++ b/var/spack/repos/builtin/packages/py-torch/package.py
@@ -26,6 +26,7 @@ class PyTorch(PythonPackage, CudaPackage, ROCmPackage):
     license("BSD-3-Clause")
 
     version("main", branch="main")
+    version("2.4.0", tag="v2.4.0", commit="d990dada86a8ad94882b5c23e859b88c0c255bda")
     version("2.3.1", tag="v2.3.1", commit="63d5e9221bedd1546b7d364b5ce4171547db12a9")
     version("2.3.0", tag="v2.3.0", commit="97ff6cfd9c86c5c09d7ce775ab64ec5c99230f5d")
     version("2.2.2", tag="v2.2.2", commit="39901f229520a5256505ec24782f716ee7ddc843")
@@ -56,9 +57,8 @@ class PyTorch(PythonPackage, CudaPackage, ROCmPackage):
     version("1.5.0", tag="v1.5.0", commit="4ff3872a2099993bf7e8c588f7182f3df777205b")
     version("1.4.1", tag="v1.4.1", commit="74044638f755cd8667bedc73da4dbda4aa64c948")
 
-    depends_on("c", type="build")  # generated
-    depends_on("cxx", type="build")  # generated
-    depends_on("fortran", type="build")  # generated
+    depends_on("c", type="build")
+    depends_on("cxx", type="build")
 
     is_darwin = sys.platform == "darwin"
 
@@ -159,7 +159,6 @@ class PyTorch(PythonPackage, CudaPackage, ROCmPackage):
         depends_on("py-networkx", when="@2:")
         depends_on("py-jinja2", when="@2:")
         depends_on("py-fsspec", when="@2.1:")
-        depends_on("mkl@2021.1.1:2021.4.0", when="@2.3: platform=windows")
 
         # pyproject.toml
         depends_on("py-setuptools")
@@ -236,14 +235,16 @@ class PyTorch(PythonPackage, CudaPackage, ROCmPackage):
         depends_on("py-pybind11@2.10.0", when="@1.13:1")
         depends_on("py-pybind11@2.6.2", when="@1.8:1.12")
         depends_on("py-pybind11@2.3.0", when="@:1.7")
-    depends_on("sleef@3.5.1_2020-12-22", when="@1.8:")
+    depends_on("sleef@3.6.0_2024-03-20", when="@2.4:")
+    depends_on("sleef@3.5.1_2020-12-22", when="@1.8:2.3")
     depends_on("sleef@3.4.0_2019-07-30", when="@1.6:1.7")
 
     # Optional dependencies
     with default_args(type=("build", "link", "run")):
         # cmake/public/cuda.cmake
+        depends_on("cuda@11:", when="@2.4:+cuda")
         # https://github.com/pytorch/pytorch/issues/122169
-        depends_on("cuda@11:12.3", when="@2:+cuda")
+        depends_on("cuda@11:12.3", when="@2.0:2.3+cuda")
         depends_on("cuda@10.2:12.3", when="@1.11:1+cuda")
         # https://discuss.pytorch.org/t/compiling-1-10-1-from-source-with-gcc-11-and-cuda-11-5/140971
         depends_on("cuda@10.2:11.4", when="@1.10+cuda")
@@ -287,6 +288,7 @@ class PyTorch(PythonPackage, CudaPackage, ROCmPackage):
 
     # Historical dependencies
     with default_args(type=("build", "run")):
+        depends_on("mkl@2021.1.1:2021.4.0", when="@2.3 platform=windows")
         depends_on("py-cffi", when="@:1")
         depends_on("py-future", when="@1.5:1")
         depends_on("py-six", when="@1.13:1")
@@ -559,6 +561,11 @@ class PyTorch(PythonPackage, CudaPackage, ROCmPackage):
             env.set("CUDNN_INCLUDE_DIR", self.spec["cudnn"].prefix.include)
             env.set("CUDNN_LIBRARY", self.spec["cudnn"].libs[0])
 
+        # Flash attention has very high memory requirements that may cause the build to fail
+        # https://github.com/pytorch/pytorch/issues/111526
+        # https://github.com/pytorch/pytorch/issues/124018
+        env.set("USE_FLASH_ATTENTION", "OFF")
+
         enable_or_disable("fbgemm")
         enable_or_disable("kineto")
         enable_or_disable("magma")
diff --git a/var/spack/repos/builtin/packages/py-torchaudio/package.py b/var/spack/repos/builtin/packages/py-torchaudio/package.py
index f7ac883ac7..cf4894cbfd 100644
--- a/var/spack/repos/builtin/packages/py-torchaudio/package.py
+++ b/var/spack/repos/builtin/packages/py-torchaudio/package.py
@@ -18,6 +18,7 @@ class PyTorchaudio(PythonPackage):
     maintainers("adamjstewart")
 
     version("main", branch="main")
+    version("2.4.0", tag="v2.4.0", commit="69d40773dc4ed86643820c21a8a880e4d074a46e")
     version("2.3.1", tag="v2.3.1", commit="3edcf69e78a3c9a3077a11159861422440ec7d4a")
     version("2.3.0", tag="v2.3.0", commit="952ea7457bcc3ed0669e7741ff23015c426d6322")
     version("2.2.2", tag="v2.2.2", commit="cefdb369247668e1dba74de503d4d996124b6b11")
@@ -48,8 +49,8 @@ class PyTorchaudio(PythonPackage):
     version("0.5.0", tag="v0.5.0", commit="09494ea545738538f9db2dceeffe10d421060ee5")
     version("0.4.0", tag="v0.4.0", commit="8afed303af3de41f3586007079c0534543c8f663")
 
-    depends_on("c", type="build")  # generated
-    depends_on("cxx", type="build")  # generated
+    depends_on("c", type="build")
+    depends_on("cxx", type="build")
 
     with default_args(type=("build", "link", "run")):
         # Based on PyPI wheel availability
@@ -60,6 +61,7 @@ class PyTorchaudio(PythonPackage):
         depends_on("python@:3.8", when="@:0.7.0")
 
         depends_on("py-torch@main", when="@main")
+        depends_on("py-torch@2.4.0", when="@2.4.0")
         depends_on("py-torch@2.3.1", when="@2.3.1")
         depends_on("py-torch@2.3.0", when="@2.3.0")
         depends_on("py-torch@2.2.2", when="@2.2.2")
diff --git a/var/spack/repos/builtin/packages/py-torchdata/package.py b/var/spack/repos/builtin/packages/py-torchdata/package.py
index 8ace2d51ff..db1b37305e 100644
--- a/var/spack/repos/builtin/packages/py-torchdata/package.py
+++ b/var/spack/repos/builtin/packages/py-torchdata/package.py
@@ -18,6 +18,7 @@ class PyTorchdata(PythonPackage):
     license("BSD-3-Clause")
 
     version("main", branch="main")
+    version("0.8.0", sha256="d5d27b264e79d7d00ad4998f14d097b770332d979672dceb6d038caf204f1208")
     version("0.7.1", sha256="ef9bbdcee759b53c3c9d99e76eb0a66da33d36bfb7f859a25a9b5e737a51fa23")
     version("0.7.0", sha256="0b444719c3abc67201ed0fea92ea9c4100e7f36551ba0d19a09446cc11154eb3")
     version("0.6.1", sha256="c596db251c5e6550db3f00e4308ee7112585cca4d6a1c82a433478fd86693257")
@@ -28,10 +29,11 @@ class PyTorchdata(PythonPackage):
     version("0.4.0", sha256="b4ec446a701680faa620fcb828b98ba36a63fa79da62a1e568d4a683889172da")
     version("0.3.0", sha256="ac36188bf133cf5f1041a28ccb3ee82ba52d4b5d99617be37d64d740acd6cfd4")
 
-    depends_on("cxx", type="build")  # generated
+    depends_on("cxx", type="build")
 
     # https://github.com/pytorch/data#version-compatibility
-    depends_on("python@3.8:3.11", when="@0.6:", type=("build", "run"))
+    depends_on("python@3.8:3.12", when="@0.8:", type=("build", "run"))
+    depends_on("python@3.8:3.11", when="@0.6:0.7", type=("build", "run"))
     depends_on("python@3.7:3.10", when="@:0.5", type=("build", "run"))
 
     # pyproject.toml
@@ -43,6 +45,7 @@ class PyTorchdata(PythonPackage):
 
     # https://github.com/pytorch/data#version-compatibility
     depends_on("py-torch@main", when="@main", type=("build", "run"))
+    depends_on("py-torch@2.4.0", when="@0.8.0", type=("build", "run"))
     depends_on("py-torch@2.1.1", when="@0.7.1", type=("build", "run"))
     depends_on("py-torch@2.1.0", when="@0.7.0", type=("build", "run"))
     depends_on("py-torch@2.0.1", when="@0.6.1", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-torchvision/package.py b/var/spack/repos/builtin/packages/py-torchvision/package.py
index 543c744ee3..2bfe45fe87 100644
--- a/var/spack/repos/builtin/packages/py-torchvision/package.py
+++ b/var/spack/repos/builtin/packages/py-torchvision/package.py
@@ -19,6 +19,7 @@ class PyTorchvision(PythonPackage):
     license("BSD-3-Clause")
 
     version("main", branch="main")
+    version("0.19.0", sha256="4c499d0a412b5a21d55ac3c0a37e80ecd7e1f002f2a7b6b3b38a2de2544acbb6")
     version("0.18.1", sha256="347d472a9ceecc44e0bee1eda140d63cfaffc74a54ec07d4b98da7698ce75516")
     version("0.18.0", sha256="3e61cbac33986a862a59cd733fd65da8b2c2a6160a66556cfa0e850f62fd43c7")
     version("0.17.2", sha256="0f9304acd77aafb7cfaf3fd5e318b2986ecc73547394b971d710eacd59f3e78e")
@@ -51,14 +52,17 @@ class PyTorchvision(PythonPackage):
     version("0.6.0", sha256="02de11b3abe6882de4032ce86dab9c7794cbc84369b44d04e667486580f0f1f7")
     version("0.5.0", sha256="eb9afc93df3d174d975ee0914057a9522f5272310b4d56c150b955c287a4d74d")
 
-    depends_on("cxx", type="build")  # generated
+    depends_on("cxx", type="build")
 
     desc = "Enable support for native encoding/decoding of {} formats in torchvision.io"
     variant("png", default=True, description=desc.format("PNG"))
     variant("jpeg", default=True, description=desc.format("JPEG"))
-    variant("nvjpeg", default=False, description=desc.format("JPEG"))
+    variant("nvjpeg", default=False, description=desc.format("NVJPEG"))
     variant("ffmpeg", default=False, description=desc.format("FFMPEG"))
     variant("video_codec", default=False, description=desc.format("video_codec"))
+    # torchvision does not yet support disabling giflib:
+    # https://github.com/pytorch/vision/pull/8406#discussion_r1590926939
+    # variant("gif", default=False, description=desc.format("GIF"), when="@0.19:")
 
     with default_args(type=("build", "link", "run")):
         # Based on PyPI wheel availability
@@ -70,6 +74,7 @@ class PyTorchvision(PythonPackage):
 
         # https://github.com/pytorch/vision#installation
         depends_on("py-torch@main", when="@main")
+        depends_on("py-torch@2.4.0", when="@0.19.0")
         depends_on("py-torch@2.3.1", when="@0.18.1")
         depends_on("py-torch@2.3.0", when="@0.18.0")
         depends_on("py-torch@2.2.2", when="@0.17.2")
@@ -118,6 +123,9 @@ class PyTorchvision(PythonPackage):
     depends_on("cuda", when="+nvjpeg")
     depends_on("ffmpeg@3.1:", when="+ffmpeg")
     depends_on("cuda", when="+video_codec")
+    # torchvision does not yet support externally-installed giflib:
+    # https://github.com/pytorch/vision/pull/8406#discussion_r1590926939
+    # depends_on("giflib", when="+gif")
 
     # Historical dependencies
     depends_on("py-requests", when="@0.12:0.17.0", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/sleef/package.py b/var/spack/repos/builtin/packages/sleef/package.py
index 02cf199e62..5518519896 100644
--- a/var/spack/repos/builtin/packages/sleef/package.py
+++ b/var/spack/repos/builtin/packages/sleef/package.py
@@ -19,8 +19,11 @@ class Sleef(CMakePackage):
 
     version("master", branch="master")
     version("3.6.1", commit="6ee14bcae5fe92c2ff8b000d5a01102dab08d774", preferred=True)
-    version("3.6", commit="a99491afee2bae0b11e9ffbf3211349f43a5fd10")
-    version("3.5.1_2020-12-22", commit="e0a003ee838b75d11763aa9c3ef17bf71a725bff")  # py-torch@1.8:
+    version("3.6.0_2024-03-20", commit="60e76d2bce17d278b439d9da17177c8f957a9e9b")  # py-torch@2.4:
+    version("3.6.0", commit="a99491afee2bae0b11e9ffbf3211349f43a5fd10")
+    version(
+        "3.5.1_2020-12-22", commit="e0a003ee838b75d11763aa9c3ef17bf71a725bff"
+    )  # py-torch@1.8:2.3
     version("3.5.1", sha256="415ee9b1bcc5816989d3d4d92afd0cd3f9ee89cbd5a33eb008e69751e40438ab")
     version(
         "3.4.0_2019-07-30", commit="7f523de651585fe25cade462efccca647dcc8d02"
@@ -38,15 +41,15 @@ class Sleef(CMakePackage):
         deprecated=True,
     )
     version(
-        "3.2_2018-05-09", commit="6ff7a135a1e31979d1e1844a2e7171dfbd34f54f", deprecated=True
+        "3.2.0_2018-05-09", commit="6ff7a135a1e31979d1e1844a2e7171dfbd34f54f", deprecated=True
     )  # py-torch@0.4.1:1.0
     version(
-        "3.2",
+        "3.2.0",
         sha256="3130c5966e204e6d6a3ace81e543d12b5b21f60897f1c185bfa587c1bd77bee2",
         deprecated=True,
     )
 
-    depends_on("c", type="build")  # generated
+    depends_on("c", type="build")
 
     generator("ninja")
     depends_on("cmake@3.18:", type="build")