Support newer version of pyspark for autologging (mlflow#9163)

Signed-off-by: Serena Ruan <[email protected]>
BenWilson2 · Jul 31, 2023 · 2dc8664 · 2dc8664
1 parent f9578c8
commit 2dc8664
Show file tree

Hide file tree

Showing 3 changed files with 19 additions and 21 deletions.
diff --git a/mlflow/ml-package-versions.yml b/mlflow/ml-package-versions.yml
@@ -59,7 +59,7 @@ pytorch-lightning:
 
   autologging:
     minimum: "1.0.5"
-    maximum: "2.0.5"
+    maximum: "2.0.6"
     requirements:
       ">= 0.0.0": ["scikit-learn", "torchvision", "protobuf<4.0.0", "tensorboard"]
       "< 1.2.0": ["torch<1.11.0"]
@@ -333,7 +333,7 @@ spark:
 
   models:
     minimum: "3.0.0"
-    maximum: "3.4.1"
+    maximum: "3.5.0"
     # NB: Allow unreleased maximum versions for the pyspark package to support models and
     # autologging use cases in environments where newer versions of pyspark are available
     # prior to their release on PyPI (e.g. Databricks)
@@ -354,7 +354,7 @@ spark:
 
   autologging:
     minimum: "3.0.0"
-    maximum: "3.4.1"
+    maximum: "3.5.0"
     # NB: Allow unreleased maximum versions for the pyspark package to support models and
     # autologging use cases in environments where newer versions of pyspark are available
     # prior to their release on PyPI (e.g. Databricks)
@@ -466,7 +466,7 @@ h2o:
     pip_release: "h2o"
   models:
     minimum: "3.40.0.1"
-    maximum: "3.42.0.1"
+    maximum: "3.42.0.2"
     run: |
       pytest tests/h2o
 
@@ -475,7 +475,7 @@ shap:
     pip_release: "shap"
   models:
     minimum: "0.41.0"
-    maximum: "0.42.0"
+    maximum: "0.42.1"
     requirements:
       "<= 0.41.0": ["numpy<1.24.0"]
     run: |
@@ -502,7 +502,7 @@ transformers:
       pip install git+https://github.com/huggingface/transformers
   models:
     minimum: "4.25.1"
-    maximum: "4.30.2"
+    maximum: "4.31.0"
     requirements:
       ">= 0.0.0": [
           "datasets",
@@ -522,7 +522,7 @@ transformers:
       pytest tests/transformers/test_transformers_model_export.py
   autologging:
     minimum: "4.25.1"
-    maximum: "4.30.2"
+    maximum: "4.31.0"
     requirements:
       ">= 0.0.0":
         [
@@ -562,7 +562,7 @@ langchain:
       pip install git+https://github.com/hwchase17/langchain#subdirectory=libs/langchain
   models:
     minimum: "0.0.169"
-    maximum: "0.0.232"
+    maximum: "0.0.244"
     requirements:
       ">= 0.0.0":
         [

diff --git a/mlflow/ml_package_versions.py b/mlflow/ml_package_versions.py
@@ -34,7 +34,7 @@
         },
         "autologging": {
             "minimum": "1.0.5",
-            "maximum": "2.0.5"
+            "maximum": "2.0.6"
         }
     },
     "tensorflow": {
@@ -148,11 +148,11 @@
         },
         "models": {
             "minimum": "3.0.0",
-            "maximum": "3.4.1"
+            "maximum": "3.5.0"
         },
         "autologging": {
             "minimum": "3.0.0",
-            "maximum": "3.4.1"
+            "maximum": "3.5.0"
         }
     },
     "mleap": {
@@ -197,7 +197,7 @@
         },
         "models": {
             "minimum": "3.40.0.1",
-            "maximum": "3.42.0.1"
+            "maximum": "3.42.0.2"
         }
     },
     "shap": {
@@ -206,7 +206,7 @@
         },
         "models": {
             "minimum": "0.41.0",
-            "maximum": "0.42.0"
+            "maximum": "0.42.1"
         }
     },
     "paddle": {
@@ -224,11 +224,11 @@
         },
         "models": {
             "minimum": "4.25.1",
-            "maximum": "4.30.2"
+            "maximum": "4.31.0"
         },
         "autologging": {
             "minimum": "4.25.1",
-            "maximum": "4.30.2"
+            "maximum": "4.31.0"
         }
     },
     "openai": {
@@ -246,7 +246,7 @@
         },
         "models": {
             "minimum": "0.0.169",
-            "maximum": "0.0.232"
+            "maximum": "0.0.244"
         }
     },
     "sentence_transformers": {

diff --git a/tests/autologging/test_autologging_utils.py b/tests/autologging/test_autologging_utils.py
@@ -798,17 +798,15 @@ def test_is_autologging_integration_supported(flavor, module_version, expected_r
     ("flavor", "module_version", "expected_result"),
     [
         ("pyspark.ml", "3.10.1.dev0", False),
+        ("pyspark.ml", "3.5.0.dev0", True),
         ("pyspark.ml", "3.3.0.dev0", True),
         ("pyspark.ml", "3.2.1.dev0", True),
         ("pyspark.ml", "3.1.2.dev0", True),
         ("pyspark.ml", "3.0.1.dev0", True),
-        ("pyspark.ml", "3.0.0.dev0", False),
+        ("pyspark.ml", "3.0.0.dev0", True),
+        ("pyspark.ml", "2.4.8.dev0", False),
     ],
 )
-@mock.patch(
-    "mlflow.utils.autologging_utils.versioning._ML_PACKAGE_VERSIONS",
-    _module_version_info_dict_patch,
-)
 def test_dev_version_pyspark_is_supported_in_databricks(flavor, module_version, expected_result):
     module_name, _ = FLAVOR_TO_MODULE_NAME_AND_VERSION_INFO_KEY[flavor]
     with mock.patch(module_name + ".__version__", module_version):