Add py-mlflow and its dependencies (#34463)

* Add py-mlflow and its dependencies
* mlflow: fix syntax error in package.py
* py-mlflow: cleanup
  Process review remarks, add missing dependencies, add skinny variant
* Apply suggestions from code review
* Fix flake8 issues
* More formatting fixes
* Fix py-waitress dependency version
* py-mlflow: platform-specific dependency
* Update var/spack/repos/builtin/packages/py-mlflow/package.py
* Update var/spack/repos/builtin/packages/py-mlflow/package.py
* Process review remarks
* Fix typo in dependency version
* py-shap: fix dependencies
* py-arrow: fix dependencies
* py-slicer: remove py-setuptools explicit version
* py-pyarrow: dataset variant and pass options through environment
  It appears there are some issues when using `pip install` instead of
  `python setup.py` - this setup_build_environment should fix that.
* py-pyarrow: review remark
* Decouple setup_build_environment from install_options
* py-pyarrow: style
* Bump licenses to 2023

---------

Co-authored-by: Tamara Dahlgren <35777542+tldahlgren@users.noreply.github.com>
Co-authored-by: Adam J. Stewart <ajstewart426@gmail.com>
Co-authored-by: Matthias Wolf <matthias.wolf@epfl.ch>

10 files changed, 199 insertions, 1 deletions

diff --git a/var/spack/repos/builtin/packages/arrow/package.py b/var/spack/repos/builtin/packages/arrow/package.py
index b40b86a120..3281affb1a 100644
--- a/var/spack/repos/builtin/packages/arrow/package.py
+++ b/var/spack/repos/builtin/packages/arrow/package.py
@@ -15,6 +15,7 @@ class Arrow(CMakePackage, CudaPackage):
     homepage = "https://arrow.apache.org"
     url = "https://github.com/apache/arrow/archive/apache-arrow-0.9.0.tar.gz"
 
+    version("10.0.1", sha256="28c3e0402bc1c3c1e047b6e26cedb8d1d89b2b9497d576af24b0b700eef11701")
     version("9.0.0", sha256="bb187b4b0af8dcc027fffed3700a7b891c9f76c9b63ad8925b4afb8257a2bb1b")
     version("8.0.0", sha256="19ece12de48e51ce4287d2dee00dc358fbc5ff02f41629d16076f77b8579e272")
     version("7.0.0", sha256="57e13c62f27b710e1de54fd30faed612aefa22aa41fa2c0c3bacd204dd18a8f3")
diff --git a/var/spack/repos/builtin/packages/py-databricks-cli/package.py b/var/spack/repos/builtin/packages/py-databricks-cli/package.py
new file mode 100644
index 0000000000..69b604dc97
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-databricks-cli/package.py
@@ -0,0 +1,26 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class PyDatabricksCli(PythonPackage):
+    """A command line interface for Databricks."""
+
+    homepage = "https://pypi.org/project/databricks-cli/"
+    pypi = "databricks-cli/databricks-cli-0.17.4.tar.gz"
+
+    version("0.17.4", sha256="bc0c4dd082f033cb6d7978cacaca5261698efe3a4c70f52f98762c38db925ce0")
+    version("0.14.3", sha256="bdf89a3917a3f8f8b99163e38d40e66dc478c7408954747f145cd09816b05e2c")
+
+    depends_on("python@3.6:", type=("build", "run"))
+    depends_on("py-setuptools", type=("build", "run"))
+
+    depends_on("py-click@7.0:", type=("build", "run"))
+    depends_on("py-pyjwt@1.7.0:", type=("build", "run"))
+    depends_on("py-oauthlib@3.1.0:", type=("build", "run"))
+    depends_on("py-requests@2.17.3:", type=("build", "run"))
+    depends_on("py-tabulate@0.7.7:", type=("build", "run"))
+    depends_on("py-six@1.10.0:", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-importlib-metadata/package.py b/var/spack/repos/builtin/packages/py-importlib-metadata/package.py
index 930e067f3a..d3021be604 100644
--- a/var/spack/repos/builtin/packages/py-importlib-metadata/package.py
+++ b/var/spack/repos/builtin/packages/py-importlib-metadata/package.py
@@ -13,6 +13,7 @@ class PyImportlibMetadata(PythonPackage):
     pypi = "importlib_metadata/importlib_metadata-1.2.0.tar.gz"
     git = "https://github.com/python/importlib_metadata"
 
+    version("5.1.0", sha256="d5059f9f1e8e41f80e9c56c2ee58811450c31984dfa625329ffd7c0dad88a73b")
     version("4.12.0", sha256="637245b8bab2b6502fcbc752cc4b7a6f6243bb02b31c5c26156ad103d3d45670")
     version("4.11.1", sha256="175f4ee440a0317f6e8d81b7f8d4869f93316170a65ad2b007d2929186c8052c")
     version("4.8.2", sha256="75bdec14c397f528724c1bfd9709d660b33a4d2e77387a3358f20b848bb5e5fb")
diff --git a/var/spack/repos/builtin/packages/py-mlflow/package.py b/var/spack/repos/builtin/packages/py-mlflow/package.py
new file mode 100644
index 0000000000..a162194d92
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-mlflow/package.py
@@ -0,0 +1,54 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class PyMlflow(PythonPackage):
+    """MLflow: A Platform for ML Development and Productionization."""
+
+    homepage = "https://pypi.org/project/mlflow/"
+    pypi = "mlflow/mlflow-2.0.1.tar.gz"
+
+    version("2.0.1", sha256="7ce6caf3c6acb022d6f5ce8a0995a92be1db524ae16aade1f83da661cdf993de")
+    version("1.17.0", sha256="4898c58899e3101e09e2b37cf5bee7db04c5d73389a56942d3ef5a5e4396799e")
+
+    depends_on("python@3.6:", type=("build", "run"), when="@1.17.0:")
+    depends_on("python@3.7:", type=("build", "run"), when="@1.24.0:")
+    depends_on("python@3.8:", type=("build", "run"), when="@2.0.1:")
+    depends_on("py-setuptools", type="build")
+
+    depends_on("py-click@7.0:8", type=("build", "run"))
+    depends_on("py-cloudpickle@:2", type=("build", "run"))
+    depends_on("py-databricks-cli@0.8.7:0", type=("build", "run"))
+    depends_on("py-entrypoints@:0", type=("build", "run"))
+    depends_on("py-gitpython@2.1.0:3", type=("build", "run"))
+    depends_on("py-pyyaml@5.1:6", type=("build", "run"))
+    depends_on("py-protobuf@3.12.0:4", type=("build", "run"))
+    depends_on("py-pytz@:2022", type=("build", "run"))
+    depends_on("py-requests@2.17.3:2", type=("build", "run"))
+    depends_on("py-packaging@:21", type=("build", "run"))
+    depends_on("py-importlib-metadata@3.7:4.6,4.7.1:5", type=("build", "run"))
+    depends_on("py-sqlparse@0.4.0:0", type=("build", "run"))
+
+    depends_on("py-alembic@:1", type=("build", "run"))
+    depends_on("py-docker@4.0.0:6", type=("build", "run"))
+    depends_on("py-flask@:2", type=("build", "run"))
+    depends_on("py-numpy@:1", type=("build", "run"))
+    depends_on("py-scipy@:1", type=("build", "run"))
+    depends_on("py-pandas@:1", type=("build", "run"))
+    depends_on("py-querystring-parser@:1", type=("build", "run"))
+    depends_on("py-sqlalchemy@1.4.0:1", type=("build", "run"))
+    for platform in ["linux", "darwin", "cray"]:
+        depends_on("py-gunicorn@:20", type=("build", "run"), when=f"platform={platform}")
+    depends_on("py-waitress@:2", type=("build", "run"), when="platform=windows")
+    depends_on("py-scikit-learn@:1", type=("build", "run"))
+    depends_on("py-pyarrow@4.0.0:10", type=("build", "run"))
+    depends_on("py-shap@0.40:0", type=("build", "run"))
+    depends_on("py-markdown@3.3:3", type=("build", "run"))
+    for platform in ["linux", "darwin", "cray"]:
+        depends_on("py-jinja2@2.11:3", type=("build", "run"), when=f"platform={platform}")
+    depends_on("py-jinja2@3.0:3", type=("build", "run"), when="platform=windows")
+    depends_on("py-matplotlib@:3", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-prometheus-flask-exporter/package.py b/var/spack/repos/builtin/packages/py-prometheus-flask-exporter/package.py
new file mode 100644
index 0000000000..b026dd3859
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-prometheus-flask-exporter/package.py
@@ -0,0 +1,20 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class PyPrometheusFlaskExporter(PythonPackage):
+    """Prometheus metrics exporter for Flask."""
+
+    homepage = "https://pypi.org/project/prometheus-flask-exporter/"
+    pypi = "prometheus-flask-exporter/prometheus_flask_exporter-0.21.0.tar.gz"
+
+    version("0.21.0", sha256="ebbc016c1e3d16e7cd39fe651a6c52ac68779858b2d5d1be6ddbc9e66f7fc29f")
+    version("0.18.2", sha256="fc487e385d95cb5efd045d6a315c4ecf68c42661e7bfde0526af75ed3c4f8c1b")
+
+    depends_on("py-setuptools", type="build")
+    depends_on("py-prometheus-client", type=("build", "run"))
+    depends_on("py-flask", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-pyarrow/package.py b/var/spack/repos/builtin/packages/py-pyarrow/package.py
index 7b0b2054d8..b5524ba9a0 100644
--- a/var/spack/repos/builtin/packages/py-pyarrow/package.py
+++ b/var/spack/repos/builtin/packages/py-pyarrow/package.py
@@ -15,6 +15,7 @@ class PyPyarrow(PythonPackage, CudaPackage):
     homepage = "https://arrow.apache.org"
     pypi = "pyarrow/pyarrow-0.17.1.tar.gz"
 
+    version("10.0.1", sha256="1a14f57a5f472ce8234f2964cd5184cccaa8df7e04568c64edc33b23eb285dd5")
     version("8.0.0", sha256="4a18a211ed888f1ac0b0ebcb99e2d9a3e913a481120ee9b1fe33d3fedb945d4e")
     version("7.0.0", sha256="da656cad3c23a2ebb6a307ab01d35fce22f7850059cffafcb90d12590f8f4f38")
     version("4.0.1", sha256="11517f0b4f4acbab0c37c674b4d1aad3c3dfea0f6b1bb322e921555258101ab3")
@@ -27,6 +28,7 @@ class PyPyarrow(PythonPackage, CudaPackage):
 
     variant("parquet", default=False, description="Build with Parquet support")
     variant("orc", default=False, description="Build with orc support")
+    variant("dataset", default=True, description="Build with Dataset support")
 
     depends_on("cmake@3.0.0:", type="build")
     depends_on("pkgconfig", type="build")
@@ -35,13 +37,13 @@ class PyPyarrow(PythonPackage, CudaPackage):
     depends_on("python@3.7:", type=("build", "run"), when="@7.0.0:")
     depends_on("py-setuptools", type="build")
     depends_on("py-setuptools@38.6.0:", type="build", when="@7.0.0:")
+    depends_on("py-setuptools@40.1.0:", type="build", when="@10.0.1:")
     depends_on("py-setuptools-scm", type="build", when="@0.15.0:")
     depends_on("py-cython", type="build")
     depends_on("py-cython@0.29:", type="build", when="@0.15.0:")
     depends_on("py-cython@0.29.22:", type="build", when="@8.0.0:")
     depends_on("py-numpy@1.14:", type=("build", "run"), when="@0.15.0:")
     depends_on("py-numpy@1.16.6:", type=("build", "run"), when="@3.0.0:")
-    depends_on("py-six@1.0.0:", type=("build", "run"), when="@0.15.0")
 
     arrow_versions = (
         "@0.9.0",
@@ -53,6 +55,7 @@ class PyPyarrow(PythonPackage, CudaPackage):
         "@4.0.1",
         "@7.0.0",
         "@8.0.0",
+        "@10.0.1",
     )
     for v in arrow_versions:
         depends_on("arrow+python" + v, when=v)
@@ -62,6 +65,16 @@ class PyPyarrow(PythonPackage, CudaPackage):
 
     patch("for_aarch64.patch", when="@0 target=aarch64:")
 
+    def setup_build_environment(self, env):
+        if spec.satisfies("+parquet"):
+            env.set("PYARROW_PARQUET", "1")
+        if spec.satisfies("+cuda"):
+            env.set("PYARROW_WITH_CUDA", "1")
+        if spec.satisfies("+orc"):
+            env.set("PYARROW_WITH_ORC", "1")
+        if spec.satisfies("+dataset"):
+            env.set("PYARROW_WITH_DATASET", "1")
+
     def install_options(self, spec, prefix):
         args = []
         if spec.satisfies("+parquet"):
@@ -70,4 +83,6 @@ class PyPyarrow(PythonPackage, CudaPackage):
             args.append("--with-cuda")
         if spec.satisfies("+orc"):
             args.append("--with-orc")
+        if spec.satisfies("+dataset"):
+            args.append("--with-dataset")
         return args
diff --git a/var/spack/repos/builtin/packages/py-querystring-parser/package.py b/var/spack/repos/builtin/packages/py-querystring-parser/package.py
new file mode 100644
index 0000000000..d441e6c9c7
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-querystring-parser/package.py
@@ -0,0 +1,18 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class PyQuerystringParser(PythonPackage):
+    """QueryString parser that correctly handles nested dictionaries."""
+
+    homepage = "https://pypi.org/project/querystring-parser/"
+    pypi = "querystring-parser/querystring_parser-1.2.4.tar.gz"
+
+    version("1.2.4", sha256="644fce1cffe0530453b43a83a38094dbe422ccba8c9b2f2a1c00280e14ca8a62")
+
+    depends_on("py-six", type=("build", "run"))
+    depends_on("py-setuptools", type=("build"))
diff --git a/var/spack/repos/builtin/packages/py-shap/package.py b/var/spack/repos/builtin/packages/py-shap/package.py
new file mode 100644
index 0000000000..3ed5a9e4b4
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-shap/package.py
@@ -0,0 +1,28 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class PyShap(PythonPackage):
+    """SHAP (SHapley Additive exPlanations): a unified approach
+    to explain the output of any machine learning model."""
+
+    homepage = "https://github.com/slundberg/shap/"
+    pypi = "shap/shap-0.41.0.tar.gz"
+
+    version("0.41.0", sha256="a49ea4d65aadbc845a695fa3d7ea0bdfc8c928b8e213b0feedf5868ade4b3ca5")
+
+    depends_on("py-setuptools", type="build")
+
+    depends_on("py-numpy", type=("build", "run"))
+    depends_on("py-scipy", type=("build", "run"))
+    depends_on("py-scikit-learn", type=("build", "run"))
+    depends_on("py-pandas", type=("build", "run"))
+    depends_on("py-tqdm@4.25.1:", type=("build", "run"))
+    depends_on("py-packaging@20.10:", type=("build", "run"))
+    depends_on("py-slicer@0.0.7", type=("build", "run"))
+    depends_on("py-numba", type=("build", "run"))
+    depends_on("py-cloudpickle", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-slicer/package.py b/var/spack/repos/builtin/packages/py-slicer/package.py
new file mode 100644
index 0000000000..3479da03a7
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-slicer/package.py
@@ -0,0 +1,18 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class PySlicer(PythonPackage):
+    """slicer wraps tensor-like objects and provides a uniform slicing interface via __getitem__"""
+
+    homepage = "https://github.com/interpretml/slicer"
+    pypi = "slicer/slicer-0.0.7.tar.gz"
+
+    version("0.0.7", sha256="f5d5f7b45f98d155b9c0ba6554fa9770c6b26d5793a3e77a1030fb56910ebeec")
+
+    depends_on("py-setuptools", type="build")
+    depends_on("python@3.6:", type=("build", "run"))
diff --git a/var/spack/repos/builtin/packages/py-waitress/package.py b/var/spack/repos/builtin/packages/py-waitress/package.py
new file mode 100644
index 0000000000..d09d6b2d69
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-waitress/package.py
@@ -0,0 +1,17 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class PyWaitress(PythonPackage):
+    """Waitress: a production-quality pure-Python WSGI server with very acceptable performance."""
+
+    homepage = "https://github.com/Pylons/waitress/"
+    pypi = "waitress/waitress-2.1.2.tar.gz"
+
+    version("2.1.2", sha256="780a4082c5fbc0fde6a2fcfe5e26e6efc1e8f425730863c04085769781f51eba")
+
+    depends_on("py-setuptools@41:", type="build")