Update Storage dependencies (#833)

* remove unused storage imports

* correct test lints

* update azure-storage dependency

* fix storage blob test lint issues

* clean up storage test

* correct blob issues

* correct file upload with new APIs

* update storageblob for ansible-test

plugins/module_utils/azure_rm_common.py

@@ -247,8 +247,7 @@ def default_api_version(self):
     from azure.mgmt.containerservice import ContainerServiceClient
     from azure.mgmt.marketplaceordering import MarketplaceOrderingAgreements
     from azure.mgmt.trafficmanager import TrafficManagerManagementClient
-    from azure.storage.cloudstorageaccount import CloudStorageAccount
-    from azure.storage.blob import PageBlobService, BlockBlobService
+    from azure.storage.blob import BlobServiceClient
     from adal.authentication_context import AuthenticationContext
     from azure.mgmt.authorization import AuthorizationManagementClient
     from azure.mgmt.sql import SqlManagementClient
@@ -681,30 +680,22 @@ def check_provisioning_state(self, azure_object, requested_state='present'):
                 self.fail("Error {0} has a provisioning state of {1}. Expecting state to be {2}.".format(
                     azure_object.name, azure_object.provisioning_state, AZURE_SUCCESS_STATE))
 
-    def get_blob_client(self, resource_group_name, storage_account_name, storage_blob_type='block'):
-        keys = dict()
+    def get_blob_service_client(self, resource_group_name, storage_account_name):
         try:
-            # Get keys from the storage account
-            self.log('Getting keys')
-            account_keys = self.storage_client.storage_accounts.list_keys(resource_group_name, storage_account_name)
+            self.log("Getting storage account detail")
+            account = self.storage_client.storage_accounts.get_properties(resource_group_name=resource_group_name, account_name=storage_account_name)
+            account_keys = self.storage_client.storage_accounts.list_keys(resource_group_name=resource_group_name, account_name=storage_account_name)
         except Exception as exc:
-            self.fail("Error getting keys for account {0} - {1}".format(storage_account_name, str(exc)))
+            self.fail("Error getting storage account detail for {0}: {1}".format(storage_account_name, str(exc)))
 
         try:
-            self.log('Create blob service')
-            if storage_blob_type == 'page':
-                return PageBlobService(endpoint_suffix=self._cloud_environment.suffixes.storage_endpoint,
-                                       account_name=storage_account_name,
-                                       account_key=account_keys.keys[0].value)
-            elif storage_blob_type == 'block':
-                return BlockBlobService(endpoint_suffix=self._cloud_environment.suffixes.storage_endpoint,
-                                        account_name=storage_account_name,
-                                        account_key=account_keys.keys[0].value)
-            else:
-                raise Exception("Invalid storage blob type defined.")
+            self.log("Create blob service client")
+            return BlobServiceClient(
+                account_url=account.primary_endpoints.blob,
+                credential=account_keys.keys[0].value,
+            )
         except Exception as exc:
-            self.fail("Error creating blob service client for storage account {0} - {1}".format(storage_account_name,
-                                                                                                str(exc)))
+            self.fail("Error creating blob service client for storage account {0} - {1}".format(storage_account_name, str(exc)))
 
     def create_default_pip(self, resource_group, location, public_ip_name, allocation_method='Dynamic', sku=None):
         '''

plugins/modules/azure_rm_storageaccount.py

@@ -405,7 +405,6 @@
 
 try:
     from azure.core.exceptions import ResourceNotFoundError
-    from azure.storage.cloudstorageaccount import CloudStorageAccount
     from azure.common import AzureMissingResourceHttpError
 except ImportError:
     # This is handled in azure_rm_common
@@ -868,14 +867,14 @@ def account_has_blob_containers(self):
         not be deleted.
         '''
         self.log('Checking for existing blob containers')
-        blob_service = self.get_blob_client(self.resource_group, self.name)
+        blob_service = self.get_blob_service_client(self.resource_group, self.name)
         try:
             response = blob_service.list_containers()
         except Exception:
             # No blob storage available?
             return False
 
-        if len(response.items) > 0:
+        if len(list(response)) > 0:
             return True
         return False
 

plugins/modules/azure_rm_storageblob.py

@@ -190,8 +190,8 @@
 import mimetypes
 
 try:
-    from azure.storage.blob.models import ContentSettings
-    from azure.common import AzureMissingResourceHttpError, AzureHttpError
+    from azure.storage.blob._models import BlobType, ContentSettings
+    from azure.core.exceptions import ResourceNotFoundError
 except ImportError:
     # This is handled in azure_rm_common
     pass
@@ -226,7 +226,7 @@ def __init__(self):
 
         mutually_exclusive = [('src', 'dest'), ('src', 'batch_upload_src'), ('dest', 'batch_upload_src')]
 
-        self.blob_client = None
+        self.blob_service_client = None
         self.blob_details = None
         self.storage_account_name = None
         self.blob = None
@@ -264,8 +264,10 @@ def exec_module(self, **kwargs):
 
         # add file path validation
 
-        self.blob_client = self.get_blob_client(self.resource_group, self.storage_account_name, self.blob_type)
+        self.blob_service_client = self.get_blob_service_client(self.resource_group, self.storage_account_name)
         self.container_obj = self.get_container()
+        if self.blob:
+            self.blob_obj = self.get_blob()
 
         if self.state == 'present':
             if not self.container_obj:
@@ -283,11 +285,9 @@ def exec_module(self, **kwargs):
 
             if self.blob:
                 # create, update or download blob
-                self.blob_obj = self.get_blob()
                 if self.src and self.src_is_valid():
                     if self.blob_obj and not self.force:
-                        self.log("Cannot upload to {0}. Blob with that name already exists. "
-                                 "Use the force option".format(self.blob))
+                        self.log("Cannot upload to {0}. Blob with that name already exists. Use the force option".format(self.blob))
                     else:
                         self.upload_blob()
                 elif self.dest and self.dest_is_valid():
@@ -373,28 +373,40 @@ def _guess_content_type(file_path, original):
                 blob_path = _normalize_blob_file_path(self.batch_upload_dst, blob_path)
             if not self.check_mode:
                 try:
-                    self.blob_client.create_blob_from_path(self.container, blob_path, src,
-                                                           metadata=self.tags, content_settings=_guess_content_type(src, content_settings))
-                except AzureHttpError as exc:
+                    client = self.blob_service_client.get_blob_client(container=self.container, blob=blob_path)
+                    with open(src, "rb") as data:
+                        client.upload_blob(data=data,
+                                           blob_type=self.get_blob_type(self.blob_type),
+                                           metadata=self.tags,
+                                           content_settings=_guess_content_type(src, content_settings))
+                except Exception as exc:
                     self.fail("Error creating blob {0} - {1}".format(src, str(exc)))
             self.results['actions'].append('created blob from {0}'.format(src))
 
         self.results['changed'] = True
         self.results['container'] = self.container_obj
 
+    def get_blob_type(self, blob_type):
+        if blob_type == "block":
+            return BlobType.BlockBlob
+        elif blob_type == "page":
+            return BlobType.PageBlob
+        else:
+            return BlobType.AppendBlob
+
     def get_container(self):
         result = {}
         container = None
         if self.container:
             try:
-                container = self.blob_client.get_container_properties(self.container)
-            except AzureMissingResourceHttpError:
+                container = self.blob_service_client.get_container_client(container=self.container).get_container_properties()
+            except ResourceNotFoundError:
                 pass
         if container:
             result = dict(
-                name=container.name,
-                tags=container.metadata,
-                last_modified=container.properties.last_modified.strftime('%d-%b-%Y %H:%M:%S %z'),
+                name=container["name"],
+                tags=container["metadata"],
+                last_modified=container["last_modified"].strftime('%d-%b-%Y %H:%M:%S %z'),
             )
         return result
 
@@ -403,23 +415,23 @@ def get_blob(self):
         blob = None
         if self.blob:
             try:
-                blob = self.blob_client.get_blob_properties(self.container, self.blob)
-            except AzureMissingResourceHttpError:
+                blob = self.blob_service_client.get_blob_client(container=self.container, blob=self.blob).get_blob_properties()
+            except ResourceNotFoundError:
                 pass
         if blob:
             result = dict(
-                name=blob.name,
-                tags=blob.metadata,
-                last_modified=blob.properties.last_modified.strftime('%d-%b-%Y %H:%M:%S %z'),
-                type=blob.properties.blob_type,
-                content_length=blob.properties.content_length,
+                name=blob["name"],
+                tags=blob["metadata"],
+                last_modified=blob["last_modified"].strftime('%d-%b-%Y %H:%M:%S %z'),
+                type=blob["blob_type"],
+                content_length=blob["size"],
                 content_settings=dict(
-                    content_type=blob.properties.content_settings.content_type,
-                    content_encoding=blob.properties.content_settings.content_encoding,
-                    content_language=blob.properties.content_settings.content_language,
-                    content_disposition=blob.properties.content_settings.content_disposition,
-                    cache_control=blob.properties.content_settings.cache_control,
-                    content_md5=blob.properties.content_settings.content_md5
+                    content_type=blob["content_settings"]["content_type"],
+                    content_encoding=blob["content_settings"]["content_encoding"],
+                    content_language=blob["content_settings"]["content_language"],
+                    content_disposition=blob["content_settings"]["content_disposition"],
+                    cache_control=blob["content_settings"]["cache_control"],
+                    content_md5=blob["content_settings"]["content_md5"],
                 )
             )
         return result
@@ -434,8 +446,9 @@ def create_container(self):
 
         if not self.check_mode:
             try:
-                self.blob_client.create_container(self.container, metadata=tags, public_access=self.public_access)
-            except AzureHttpError as exc:
+                client = self.blob_service_client.get_container_client(container=self.container)
+                client.create_container(metadata=tags, public_access=self.public_access)
+            except Exception as exc:
                 self.fail("Error creating container {0} - {1}".format(self.container, str(exc)))
         self.container_obj = self.get_container()
         self.results['changed'] = True
@@ -456,9 +469,14 @@ def upload_blob(self):
             )
         if not self.check_mode:
             try:
-                self.blob_client.create_blob_from_path(self.container, self.blob, self.src,
-                                                       metadata=self.tags, content_settings=content_settings)
-            except AzureHttpError as exc:
+                client = self.blob_service_client.get_blob_client(container=self.container, blob=self.blob)
+                with open(self.src, "rb") as data:
+                    client.upload_blob(data=data,
+                                       blob_type=self.get_blob_type(self.blob_type),
+                                       metadata=self.tags,
+                                       content_settings=content_settings,
+                                       overwrite=self.force)
+            except Exception as exc:
                 self.fail("Error creating blob {0} - {1}".format(self.blob, str(exc)))
 
         self.blob_obj = self.get_blob()
@@ -470,7 +488,10 @@ def upload_blob(self):
     def download_blob(self):
         if not self.check_mode:
             try:
-                self.blob_client.get_blob_to_path(self.container, self.blob, self.dest)
+                client = self.blob_service_client.get_blob_client(container=self.container, blob=self.blob)
+                with open(self.dest, "wb") as blob_stream:
+                    blob_data = client.download_blob()
+                    blob_data.readinto(blob_stream)
             except Exception as exc:
                 self.fail("Failed to download blob {0}:{1} to {2} - {3}".format(self.container,
                                                                                 self.blob,
@@ -527,27 +548,27 @@ def dest_is_valid(self):
     def delete_container(self):
         if not self.check_mode:
             try:
-                self.blob_client.delete_container(self.container)
-            except AzureHttpError as exc:
+                self.blob_service_client.get_container_client(container=self.container).delete_container()
+            except Exception as exc:
                 self.fail("Error deleting container {0} - {1}".format(self.container, str(exc)))
 
         self.results['changed'] = True
         self.results['actions'].append('deleted container {0}'.format(self.container))
 
     def container_has_blobs(self):
         try:
-            list_generator = self.blob_client.list_blobs(self.container)
-        except AzureHttpError as exc:
+            blobs = self.blob_service_client.get_container_client(container=self.container).list_blobs()
+        except Exception as exc:
             self.fail("Error list blobs in {0} - {1}".format(self.container, str(exc)))
-        if len(list_generator.items) > 0:
+        if len(list(blobs)) > 0:
             return True
         return False
 
     def delete_blob(self):
         if not self.check_mode:
             try:
-                self.blob_client.delete_blob(self.container, self.blob)
-            except AzureHttpError as exc:
+                self.blob_service_client.get_container_client(container=self.container).delete_blob(blob=self.blob)
+            except Exception as exc:
                 self.fail("Error deleting blob {0}:{1} - {2}".format(self.container, self.blob, str(exc)))
 
         self.results['changed'] = True
@@ -557,8 +578,8 @@ def delete_blob(self):
     def update_container_tags(self, tags):
         if not self.check_mode:
             try:
-                self.blob_client.set_container_metadata(self.container, metadata=tags)
-            except AzureHttpError as exc:
+                self.blob_service_client.get_container_client(container=self.container).set_container_metadata(metadata=tags)
+            except Exception as exc:
                 self.fail("Error updating container tags {0} - {1}".format(self.container, str(exc)))
         self.container_obj = self.get_container()
         self.results['changed'] = True
@@ -568,8 +589,8 @@ def update_container_tags(self, tags):
     def update_blob_tags(self, tags):
         if not self.check_mode:
             try:
-                self.blob_client.set_blob_metadata(self.container, self.blob, metadata=tags)
-            except AzureHttpError as exc:
+                self.blob_service_client.get_blob_client(container=self.container, blob=self.blob).set_blob_metadata(metadata=tags)
+            except Exception as exc:
                 self.fail("Update blob tags {0}:{1} - {2}".format(self.container, self.blob, str(exc)))
         self.blob_obj = self.get_blob()
         self.results['changed'] = True
@@ -604,8 +625,8 @@ def update_blob_content_settings(self):
         )
         if not self.check_mode:
             try:
-                self.blob_client.set_blob_properties(self.container, self.blob, content_settings=content_settings)
-            except AzureHttpError as exc:
+                self.blob_service_client.get_blob_client(container=self.container, blob=self.blob).set_http_headers(content_settings=content_settings)
+            except Exception as exc:
                 self.fail("Update blob content settings {0}:{1} - {2}".format(self.container, self.blob, str(exc)))
 
         self.blob_obj = self.get_blob()

plugins/modules/azure_rm_virtualmachine.py

@@ -2030,12 +2030,12 @@ def delete_vm_storage(self, vhd_uris):
             container_name = blob_parts['containername']
             blob_name = blob_parts['blobname']
 
-            blob_client = self.get_blob_client(self.resource_group, storage_account_name)
+            blob_service_client = self.get_blob_service_client(self.resource_group, storage_account_name)
 
             self.log("Delete blob {0}:{1}".format(container_name, blob_name))
             self.results['actions'].append("Deleted blob {0}:{1}".format(container_name, blob_name))
             try:
-                blob_client.delete_blob(container_name, blob_name)
+                blob_service_client.get_blob_client(container=container_name, blob=blob_name).delete_blob()
             except Exception as exc:
                 self.fail("Error deleting blob {0}:{1} - {2}".format(container_name, blob_name, str(exc)))
         return True

requirements-azure.txt

@@ -34,7 +34,7 @@ azure-mgmt-storage==19.0.0
 azure-mgmt-trafficmanager==0.50.0
 azure-mgmt-web==0.41.0
 azure-nspkg==2.0.0
-azure-storage==0.35.1
+azure-storage-blob==12.11.0
 msrest==0.6.21
 msrestazure==0.6.4
 azure-keyvault==1.0.0a1