Merge pull request #2170 from adriangonz/azure-dep-optional

Make Azure dependency optional

doc/source/python/python_module.md

@@ -44,6 +44,20 @@ libraries that `seldon-core` requires for a single multi-cloud one.
 This discussion is currently open on [issue #1028](https://github.com/SeldonIO/seldon-core/issues/1028).
 Feedback and suggestions are welcome!
 
+### Azure Blob Storage support
+
+As part of the options to store your trained model, Seldon Core adds optional
+support to fetch them from Azure Blob Storage.
+We are aware that users will usually only require one of the storage backends.
+Therefore, to avoid bloating the `seldon-core` package, the Azure Blob Storage
+dependencies are not installed by default.
+
+To include the optional Azure support, you can install `seldon-core` as:
+
+```bash
+$ pip install seldon-core[azure]
+```
+
 ### Install all extra dependencies
 
 If you want to install `seldon-core` with all its extra dependencies, you can

python/seldon_core/imports_helper.py

@@ -7,6 +7,7 @@
 # not
 _TF_PRESENT = False
 _GCS_PRESENT = False
+_AZURE_PRESENT = False
 
 try:
     #  Fix for https://github.com/SeldonIO/seldon-core/issues/1076
@@ -35,7 +36,7 @@
         or
 
             $ pip install seldon_core[all]
-    """
+        """
     )
     logger.info(notice)
 
@@ -57,6 +58,27 @@
         or
 
             $ pip install seldon_core[all]
-    """
+        """
+    )
+    logger.info(notice)
+
+try:
+    from azure.storage.blob import BlockBlobService  # noqa: F401
+
+    _AZURE_PRESENT = True
+except ImportError:
+    _AZURE_PRESENT = False
+    notice = textwrap.dedent(
+        """
+        Support for Azure Blob Storage is not installed.
+        If you want to download resources from Azure Blob Storage
+        install `azure-storage-blob` or install `seldon_core` as
+
+            $ pip install seldon_core[azure]
+
+        or
+
+            $ pip install seldon_core[all]
+        """
     )
     logger.info(notice)

python/seldon_core/storage.py

@@ -23,15 +23,18 @@
 import os
 import re
 from urllib.parse import urlparse
-from azure.storage.blob import BlockBlobService
 from minio import Minio
-from seldon_core.imports_helper import _GCS_PRESENT
+from seldon_core.imports_helper import _AZURE_PRESENT, _GCS_PRESENT
 from seldon_core.utils import getenv
 
 if _GCS_PRESENT:
     from google.auth import exceptions
     from google.cloud import storage
 
+if _AZURE_PRESENT:
+    from azure.storage.blob import BlockBlobService
+
+
 _GCS_PREFIX = "gs://"
 _S3_PREFIX = "s3://"
 _BLOB_RE = "https://(.+?).blob.core.windows.net/(.+)"

python/setup.cfg

@@ -12,6 +12,7 @@ envlist =
   py3
   tensorflow
   gcs
+  azure
   all
 
 [testenv]
@@ -28,6 +29,9 @@ extras = tensorflow
 [testenv:gcs]
 extras = gcs
 
+[testenv:azure]
+extras = azure
+
 [testenv:all]
 extras = all
 

python/setup.py

@@ -8,7 +8,11 @@
     exec(fp.read(), version)
 
 # Extra dependencies, with special 'all' key
-extras = {"tensorflow": ["tensorflow"], "gcs": ["google-cloud-storage >= 1.16.0"]}
+extras = {
+    "tensorflow": ["tensorflow"],
+    "gcs": ["google-cloud-storage >= 1.16.0"],
+    "azure": ["azure-storage-blob >= 2.0.1, < 3.0.0"],
+}
 all_extra_deps = chain.from_iterable(extras.values())
 extras["all"] = list(set(all_extra_deps))
 
@@ -41,7 +45,6 @@
         "PyYAML<5.4",
         "gunicorn >= 19.9.0, < 20.1.0",
         "minio >= 4.0.9, < 6.0.0",
-        "azure-storage-blob >= 2.0.1, < 3.0.0",
         "setuptools >= 41.0.0",
         "prometheus_client >= 0.7.1, < 0.9.0",
     ],

python/tests/test_azure_storage.py

@@ -19,9 +19,15 @@
 import unittest.mock as mock
 import itertools
 import pytest
-from azure.common import AzureMissingResourceHttpError
 import seldon_core
 
+from seldon_core.imports_helper import _AZURE_PRESENT
+
+from .utils import skipif_azure_missing
+
+if _AZURE_PRESENT:
+    from azure.common import AzureMissingResourceHttpError
+
 
 def create_mock_item(path):
     mock_obj = mock.MagicMock()
@@ -47,6 +53,7 @@ def get_call_args(call_args_list):
 # pylint: disable=protected-access
 
 
+@skipif_azure_missing
 @mock.patch("seldon_core.storage.os.makedirs")
 @mock.patch("seldon_core.storage.BlockBlobService")
 def test_blob(mock_storage, mock_makedirs):  # pylint: disable=unused-argument
@@ -69,6 +76,7 @@ def test_blob(mock_storage, mock_makedirs):  # pylint: disable=unused-argument
     mock_storage.assert_called_with(account_name="seldon_core")
 
 
+@skipif_azure_missing
 @mock.patch("seldon_core.storage.os.makedirs")
 @mock.patch("seldon_core.storage.Storage._get_azure_storage_token")
 @mock.patch("seldon_core.storage.BlockBlobService")
@@ -99,6 +107,7 @@ def test_secure_blob(
     ]
 
 
+@skipif_azure_missing
 @mock.patch("seldon_core.storage.os.makedirs")
 @mock.patch("seldon_core.storage.BlockBlobService")
 def test_deep_blob(mock_storage, mock_makedirs):  # pylint: disable=unused-argument
@@ -123,6 +132,7 @@ def test_deep_blob(mock_storage, mock_makedirs):  # pylint: disable=unused-argum
     assert actual_calls == expected_calls
 
 
+@skipif_azure_missing
 @mock.patch("seldon_core.storage.os.makedirs")
 @mock.patch("seldon_core.storage.BlockBlobService")
 def test_blob_file(mock_storage, mock_makedirs):  # pylint: disable=unused-argument
@@ -145,6 +155,7 @@ def test_blob_file(mock_storage, mock_makedirs):  # pylint: disable=unused-argum
     assert actual_calls == expected_calls
 
 
+@skipif_azure_missing
 @mock.patch("seldon_core.storage.os.makedirs")
 @mock.patch("seldon_core.storage.BlockBlobService")
 def test_blob_fq_file(mock_storage, mock_makedirs):  # pylint: disable=unused-argument
@@ -167,6 +178,7 @@ def test_blob_fq_file(mock_storage, mock_makedirs):  # pylint: disable=unused-ar
     assert actual_calls == expected_calls
 
 
+@skipif_azure_missing
 @mock.patch("seldon_core.storage.os.makedirs")
 @mock.patch("seldon_core.storage.BlockBlobService")
 def test_blob_no_prefix(mock_storage, mock_makedirs):  # pylint: disable=unused-argument

python/tests/test_storage.py

@@ -20,7 +20,7 @@
 import seldon_core
 from minio import Minio, error
 import unittest.mock as mock
-from .utils import skipif_gcs_missing
+from .utils import skipif_gcs_missing, skipif_azure_missing
 from seldon_core.imports_helper import _GCS_PRESENT
 
 if _GCS_PRESENT:
@@ -61,6 +61,7 @@ def test_mock_gcs(mock_storage):
     assert seldon_core.Storage.download(gcs_path)
 
 
+@skipif_azure_missing
 def test_storage_blob_exception():
     blob_path = "https://accountname.blob.core.windows.net/container/some/blob/"
     with pytest.raises(Exception):

python/tests/utils.py

@@ -1,9 +1,14 @@
 import pytest
-from seldon_core.imports_helper import _TF_PRESENT, _GCS_PRESENT
+from seldon_core.imports_helper import _TF_PRESENT, _GCS_PRESENT, _AZURE_PRESENT
 
 skipif_tf_missing = pytest.mark.skipif(
     not _TF_PRESENT, reason="tensorflow is not present"
 )
+
 skipif_gcs_missing = pytest.mark.skipif(
     not _GCS_PRESENT, reason="google-cloud-storage is not present"
 )
+
+skipif_azure_missing = pytest.mark.skipif(
+    not _AZURE_PRESENT, reason="azure-storage-blob is not present"
+)