py-transformers: add v4.24.0 (#33716)

* py-transformers: add v4.24.0

* Internet access still required

6 files changed, 32 insertions, 10 deletions

diff --git a/share/spack/gitlab/cloud_pipelines/stacks/ml-cpu/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/ml-cpu/spack.yaml
index a03ecbae69..6b6280d66c 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/ml-cpu/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/ml-cpu/spack.yaml
@@ -23,6 +23,9 @@ spack:
     # Horovod
     - py-horovod
 
+    # Hugging Face
+    - py-transformers
+
     # JAX
     # https://github.com/google/jax/issues/12614
     # - py-jax
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/ml-cuda/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/ml-cuda/spack.yaml
index ada465fd1a..9d050b5b01 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/ml-cuda/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/ml-cuda/spack.yaml
@@ -26,6 +26,9 @@ spack:
     # Horovod
     - py-horovod
 
+    # Hugging Face
+    - py-transformers
+
     # JAX
     # https://github.com/google/jax/issues/12614
     # - py-jax
diff --git a/share/spack/gitlab/cloud_pipelines/stacks/ml-rocm/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/ml-rocm/spack.yaml
index bbc6159f59..f1c5fa3956 100644
--- a/share/spack/gitlab/cloud_pipelines/stacks/ml-rocm/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/ml-rocm/spack.yaml
@@ -28,6 +28,9 @@ spack:
     # Horovod
     - py-horovod
 
+    # Hugging Face
+    - py-transformers
+
     # JAX
     # https://github.com/google/jax/issues/12614
     # - py-jax
diff --git a/var/spack/repos/builtin/packages/py-huggingface-hub/package.py b/var/spack/repos/builtin/packages/py-huggingface-hub/package.py
index f9d5fec83a..be59e5b60c 100644
--- a/var/spack/repos/builtin/packages/py-huggingface-hub/package.py
+++ b/var/spack/repos/builtin/packages/py-huggingface-hub/package.py
@@ -14,13 +14,18 @@ class PyHuggingfaceHub(PythonPackage):
     homepage = "https://github.com/huggingface/huggingface_hub"
     pypi = "huggingface_hub/huggingface_hub-0.0.10.tar.gz"
 
+    version("0.10.1", sha256="5c188d5b16bec4b78449f8681f9975ff9d321c16046cc29bcf0d7e464ff29276")
     version("0.0.10", sha256="556765e4c7edd2d2c4c733809bae1069dca20e10ff043870ec40d53e498efae2")
     version("0.0.8", sha256="be5b9a7ed36437bb10a780d500154d426798ec16803ff3406f7a61107e4ebfc2")
 
+    depends_on("python@3.7:", when="@0.10:", type=("build", "run"))
     depends_on("python@3.6:", type=("build", "run"))
     depends_on("py-setuptools", type="build")
     depends_on("py-filelock", type=("build", "run"))
     depends_on("py-requests", type=("build", "run"))
     depends_on("py-tqdm", type=("build", "run"))
+    depends_on("py-pyyaml@5.1:", when="@0.10:", type=("build", "run"))
+    depends_on("py-typing-extensions@3.7.4.3:", when="@0.10:", type=("build", "run"))
     depends_on("py-typing-extensions", when="@0.0.10:", type=("build", "run"))
     depends_on("py-importlib-metadata", when="^python@:3.7", type=("build", "run"))
+    depends_on("py-packaging@20.9:", when="@0.10:", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-tokenizers/package.py b/var/spack/repos/builtin/packages/py-tokenizers/package.py
index 138fe08466..24befd2986 100644
--- a/var/spack/repos/builtin/packages/py-tokenizers/package.py
+++ b/var/spack/repos/builtin/packages/py-tokenizers/package.py
@@ -13,15 +13,15 @@ class PyTokenizers(PythonPackage):
     homepage = "https://github.com/huggingface/tokenizers"
     pypi = "tokenizers/tokenizers-0.6.0.tar.gz"
 
+    version("0.13.1", sha256="3333d1cee5c8f47c96362ea0abc1f81c77c9b92c6c3d11cbf1d01985f0d5cf1d")
     version("0.10.3", sha256="1a5d3b596c6d3a237e1ad7f46c472d467b0246be7fd1a364f12576eb8db8f7e6")
     version("0.6.0", sha256="1da11fbfb4f73be695bed0d655576097d09a137a16dceab2f66399716afaffac")
     version("0.5.2", sha256="b5a235f9c71d04d4925df6c4fa13b13f1d03f9b7ac302b89f8120790c4f742bc")
 
     depends_on("py-setuptools", type="build")
     depends_on("py-setuptools-rust", type="build")
-    depends_on("rust@nightly", type="build")
 
     # TODO: This package currently requires internet access to install.
-    # Also, a nightly or dev version of rust is required to build.
+    # Also, a nightly or dev version of rust is required to build older versions.
     # https://github.com/huggingface/tokenizers/issues/176
     # https://github.com/PyO3/pyo3/issues/5
diff --git a/var/spack/repos/builtin/packages/py-transformers/package.py b/var/spack/repos/builtin/packages/py-transformers/package.py
index b0e3fbd1fd..d2643b732c 100644
--- a/var/spack/repos/builtin/packages/py-transformers/package.py
+++ b/var/spack/repos/builtin/packages/py-transformers/package.py
@@ -16,24 +16,32 @@ class PyTransformers(PythonPackage):
 
     maintainers = ["adamjstewart"]
 
+    version("4.24.0", sha256="486f353a8e594002e48be0e2aba723d96eda839e63bfe274702a4b5eda85559b")
     version("4.6.1", sha256="83dbff763b7e7dc57cbef1a6b849655d4fcab6bffdd955c5e8bea12a4f76dc10")
     version("2.8.0", sha256="b9f29cdfd39c28f29e0806c321270dea337d6174a7aa60daf9625bf83dbb12ee")
 
+    depends_on("python@3.7:", when="@4.24:", type=("build", "run"))
     depends_on("python@3.6:", type=("build", "run"))
     depends_on("py-setuptools", type="build")
-
-    depends_on("py-dataclasses", when="^python@:3.6", type=("build", "run"))
-    depends_on("py-importlib-metadata", when="@4.6.1: ^python@:3.7", type=("build", "run"))
+    depends_on("py-importlib-metadata", when="@4.6: ^python@:3.7", type=("build", "run"))
     depends_on("py-filelock", type=("build", "run"))
-    depends_on("py-huggingface-hub@0.0.8", when="@4.6.1:", type=("build", "run"))
+    depends_on("py-huggingface-hub@0.10:0", when="@4.24:", type=("build", "run"))
+    depends_on("py-huggingface-hub@0.0.8", when="@4.6.1", type=("build", "run"))
+    depends_on("py-numpy@1.17:", when="@4.6:", type=("build", "run"))
     depends_on("py-numpy", type=("build", "run"))
-    depends_on("py-numpy@1.17:", when="@4.6.1:", type=("build", "run"))
-    depends_on("py-packaging", when="@4.6.1:", type=("build", "run"))
+    depends_on("py-packaging@20:", when="@4.24:", type=("build", "run"))
+    depends_on("py-packaging", when="@4.6.1", type=("build", "run"))
+    depends_on("py-pyyaml@5.1:", when="@4.24:", type=("build", "run"))
     depends_on("py-regex@:2019.12.16,2019.12.18:", type=("build", "run"))
     depends_on("py-requests", type=("build", "run"))
-    depends_on("py-sacremoses", type=("build", "run"))
+    depends_on("py-tokenizers@0.11.1:0.11.2,0.11.4:0.13", when="@4.24:", type=("build", "run"))
+    depends_on("py-tokenizers@0.10.1:0.10", when="@4.6.1", type=("build", "run"))
     depends_on("py-tokenizers@0.5.2", when="@2.8.0", type=("build", "run"))
-    depends_on("py-tokenizers@0.10.1:0.10", when="@4.6.1:", type=("build", "run"))
     depends_on("py-tqdm@4.27:", type=("build", "run"))
+
+    # Historical requirements
+    depends_on("py-dataclasses", when="@4.6.1 ^python@:3.6", type=("build", "run"))
+    depends_on("py-sacremoses", when="@:4.6", type=("build", "run"))
     depends_on("py-boto3", when="@2.8.0", type=("build", "run"))
+    depends_on("py-dataclasses", when="@2.8.0 ^python@:3.6", type=("build", "run"))
     depends_on("py-sentencepiece", when="@2.8.0", type=("build", "run"))