diff --git a/plugins/module_utils/azure_rm_common.py b/plugins/module_utils/azure_rm_common.py index ce2e2640d..79130c881 100644 --- a/plugins/module_utils/azure_rm_common.py +++ b/plugins/module_utils/azure_rm_common.py @@ -247,8 +247,7 @@ def default_api_version(self): from azure.mgmt.containerservice import ContainerServiceClient from azure.mgmt.marketplaceordering import MarketplaceOrderingAgreements from azure.mgmt.trafficmanager import TrafficManagerManagementClient - from azure.storage.cloudstorageaccount import CloudStorageAccount - from azure.storage.blob import PageBlobService, BlockBlobService + from azure.storage.blob import BlobServiceClient from adal.authentication_context import AuthenticationContext from azure.mgmt.authorization import AuthorizationManagementClient from azure.mgmt.sql import SqlManagementClient @@ -681,30 +680,22 @@ def check_provisioning_state(self, azure_object, requested_state='present'): self.fail("Error {0} has a provisioning state of {1}. Expecting state to be {2}.".format( azure_object.name, azure_object.provisioning_state, AZURE_SUCCESS_STATE)) - def get_blob_client(self, resource_group_name, storage_account_name, storage_blob_type='block'): - keys = dict() + def get_blob_service_client(self, resource_group_name, storage_account_name): try: - # Get keys from the storage account - self.log('Getting keys') - account_keys = self.storage_client.storage_accounts.list_keys(resource_group_name, storage_account_name) + self.log("Getting storage account detail") + account = self.storage_client.storage_accounts.get_properties(resource_group_name=resource_group_name, account_name=storage_account_name) + account_keys = self.storage_client.storage_accounts.list_keys(resource_group_name=resource_group_name, account_name=storage_account_name) except Exception as exc: - self.fail("Error getting keys for account {0} - {1}".format(storage_account_name, str(exc))) + self.fail("Error getting storage account detail for {0}: {1}".format(storage_account_name, str(exc))) try: - self.log('Create blob service') - if storage_blob_type == 'page': - return PageBlobService(endpoint_suffix=self._cloud_environment.suffixes.storage_endpoint, - account_name=storage_account_name, - account_key=account_keys.keys[0].value) - elif storage_blob_type == 'block': - return BlockBlobService(endpoint_suffix=self._cloud_environment.suffixes.storage_endpoint, - account_name=storage_account_name, - account_key=account_keys.keys[0].value) - else: - raise Exception("Invalid storage blob type defined.") + self.log("Create blob service client") + return BlobServiceClient( + account_url=account.primary_endpoints.blob, + credential=account_keys.keys[0].value, + ) except Exception as exc: - self.fail("Error creating blob service client for storage account {0} - {1}".format(storage_account_name, - str(exc))) + self.fail("Error creating blob service client for storage account {0} - {1}".format(storage_account_name, str(exc))) def create_default_pip(self, resource_group, location, public_ip_name, allocation_method='Dynamic', sku=None): ''' diff --git a/plugins/modules/azure_rm_storageaccount.py b/plugins/modules/azure_rm_storageaccount.py index bdbd27570..bbfa05cba 100644 --- a/plugins/modules/azure_rm_storageaccount.py +++ b/plugins/modules/azure_rm_storageaccount.py @@ -405,7 +405,6 @@ try: from azure.core.exceptions import ResourceNotFoundError - from azure.storage.cloudstorageaccount import CloudStorageAccount from azure.common import AzureMissingResourceHttpError except ImportError: # This is handled in azure_rm_common @@ -868,14 +867,14 @@ def account_has_blob_containers(self): not be deleted. ''' self.log('Checking for existing blob containers') - blob_service = self.get_blob_client(self.resource_group, self.name) + blob_service = self.get_blob_service_client(self.resource_group, self.name) try: response = blob_service.list_containers() except Exception: # No blob storage available? return False - if len(response.items) > 0: + if len(list(response)) > 0: return True return False diff --git a/plugins/modules/azure_rm_storageblob.py b/plugins/modules/azure_rm_storageblob.py index 05f0d21d7..f936f8618 100644 --- a/plugins/modules/azure_rm_storageblob.py +++ b/plugins/modules/azure_rm_storageblob.py @@ -190,8 +190,8 @@ import mimetypes try: - from azure.storage.blob.models import ContentSettings - from azure.common import AzureMissingResourceHttpError, AzureHttpError + from azure.storage.blob._models import BlobType, ContentSettings + from azure.core.exceptions import ResourceNotFoundError except ImportError: # This is handled in azure_rm_common pass @@ -226,7 +226,7 @@ def __init__(self): mutually_exclusive = [('src', 'dest'), ('src', 'batch_upload_src'), ('dest', 'batch_upload_src')] - self.blob_client = None + self.blob_service_client = None self.blob_details = None self.storage_account_name = None self.blob = None @@ -264,8 +264,10 @@ def exec_module(self, **kwargs): # add file path validation - self.blob_client = self.get_blob_client(self.resource_group, self.storage_account_name, self.blob_type) + self.blob_service_client = self.get_blob_service_client(self.resource_group, self.storage_account_name) self.container_obj = self.get_container() + if self.blob: + self.blob_obj = self.get_blob() if self.state == 'present': if not self.container_obj: @@ -283,11 +285,9 @@ def exec_module(self, **kwargs): if self.blob: # create, update or download blob - self.blob_obj = self.get_blob() if self.src and self.src_is_valid(): if self.blob_obj and not self.force: - self.log("Cannot upload to {0}. Blob with that name already exists. " - "Use the force option".format(self.blob)) + self.log("Cannot upload to {0}. Blob with that name already exists. Use the force option".format(self.blob)) else: self.upload_blob() elif self.dest and self.dest_is_valid(): @@ -373,28 +373,40 @@ def _guess_content_type(file_path, original): blob_path = _normalize_blob_file_path(self.batch_upload_dst, blob_path) if not self.check_mode: try: - self.blob_client.create_blob_from_path(self.container, blob_path, src, - metadata=self.tags, content_settings=_guess_content_type(src, content_settings)) - except AzureHttpError as exc: + client = self.blob_service_client.get_blob_client(container=self.container, blob=blob_path) + with open(src, "rb") as data: + client.upload_blob(data=data, + blob_type=self.get_blob_type(self.blob_type), + metadata=self.tags, + content_settings=_guess_content_type(src, content_settings)) + except Exception as exc: self.fail("Error creating blob {0} - {1}".format(src, str(exc))) self.results['actions'].append('created blob from {0}'.format(src)) self.results['changed'] = True self.results['container'] = self.container_obj + def get_blob_type(self, blob_type): + if blob_type == "block": + return BlobType.BlockBlob + elif blob_type == "page": + return BlobType.PageBlob + else: + return BlobType.AppendBlob + def get_container(self): result = {} container = None if self.container: try: - container = self.blob_client.get_container_properties(self.container) - except AzureMissingResourceHttpError: + container = self.blob_service_client.get_container_client(container=self.container).get_container_properties() + except ResourceNotFoundError: pass if container: result = dict( - name=container.name, - tags=container.metadata, - last_modified=container.properties.last_modified.strftime('%d-%b-%Y %H:%M:%S %z'), + name=container["name"], + tags=container["metadata"], + last_modified=container["last_modified"].strftime('%d-%b-%Y %H:%M:%S %z'), ) return result @@ -403,23 +415,23 @@ def get_blob(self): blob = None if self.blob: try: - blob = self.blob_client.get_blob_properties(self.container, self.blob) - except AzureMissingResourceHttpError: + blob = self.blob_service_client.get_blob_client(container=self.container, blob=self.blob).get_blob_properties() + except ResourceNotFoundError: pass if blob: result = dict( - name=blob.name, - tags=blob.metadata, - last_modified=blob.properties.last_modified.strftime('%d-%b-%Y %H:%M:%S %z'), - type=blob.properties.blob_type, - content_length=blob.properties.content_length, + name=blob["name"], + tags=blob["metadata"], + last_modified=blob["last_modified"].strftime('%d-%b-%Y %H:%M:%S %z'), + type=blob["blob_type"], + content_length=blob["size"], content_settings=dict( - content_type=blob.properties.content_settings.content_type, - content_encoding=blob.properties.content_settings.content_encoding, - content_language=blob.properties.content_settings.content_language, - content_disposition=blob.properties.content_settings.content_disposition, - cache_control=blob.properties.content_settings.cache_control, - content_md5=blob.properties.content_settings.content_md5 + content_type=blob["content_settings"]["content_type"], + content_encoding=blob["content_settings"]["content_encoding"], + content_language=blob["content_settings"]["content_language"], + content_disposition=blob["content_settings"]["content_disposition"], + cache_control=blob["content_settings"]["cache_control"], + content_md5=blob["content_settings"]["content_md5"], ) ) return result @@ -434,8 +446,9 @@ def create_container(self): if not self.check_mode: try: - self.blob_client.create_container(self.container, metadata=tags, public_access=self.public_access) - except AzureHttpError as exc: + client = self.blob_service_client.get_container_client(container=self.container) + client.create_container(metadata=tags, public_access=self.public_access) + except Exception as exc: self.fail("Error creating container {0} - {1}".format(self.container, str(exc))) self.container_obj = self.get_container() self.results['changed'] = True @@ -456,9 +469,14 @@ def upload_blob(self): ) if not self.check_mode: try: - self.blob_client.create_blob_from_path(self.container, self.blob, self.src, - metadata=self.tags, content_settings=content_settings) - except AzureHttpError as exc: + client = self.blob_service_client.get_blob_client(container=self.container, blob=self.blob) + with open(self.src, "rb") as data: + client.upload_blob(data=data, + blob_type=self.get_blob_type(self.blob_type), + metadata=self.tags, + content_settings=content_settings, + overwrite=self.force) + except Exception as exc: self.fail("Error creating blob {0} - {1}".format(self.blob, str(exc))) self.blob_obj = self.get_blob() @@ -470,7 +488,10 @@ def upload_blob(self): def download_blob(self): if not self.check_mode: try: - self.blob_client.get_blob_to_path(self.container, self.blob, self.dest) + client = self.blob_service_client.get_blob_client(container=self.container, blob=self.blob) + with open(self.dest, "wb") as blob_stream: + blob_data = client.download_blob() + blob_data.readinto(blob_stream) except Exception as exc: self.fail("Failed to download blob {0}:{1} to {2} - {3}".format(self.container, self.blob, @@ -527,8 +548,8 @@ def dest_is_valid(self): def delete_container(self): if not self.check_mode: try: - self.blob_client.delete_container(self.container) - except AzureHttpError as exc: + self.blob_service_client.get_container_client(container=self.container).delete_container() + except Exception as exc: self.fail("Error deleting container {0} - {1}".format(self.container, str(exc))) self.results['changed'] = True @@ -536,18 +557,18 @@ def delete_container(self): def container_has_blobs(self): try: - list_generator = self.blob_client.list_blobs(self.container) - except AzureHttpError as exc: + blobs = self.blob_service_client.get_container_client(container=self.container).list_blobs() + except Exception as exc: self.fail("Error list blobs in {0} - {1}".format(self.container, str(exc))) - if len(list_generator.items) > 0: + if len(list(blobs)) > 0: return True return False def delete_blob(self): if not self.check_mode: try: - self.blob_client.delete_blob(self.container, self.blob) - except AzureHttpError as exc: + self.blob_service_client.get_container_client(container=self.container).delete_blob(blob=self.blob) + except Exception as exc: self.fail("Error deleting blob {0}:{1} - {2}".format(self.container, self.blob, str(exc))) self.results['changed'] = True @@ -557,8 +578,8 @@ def delete_blob(self): def update_container_tags(self, tags): if not self.check_mode: try: - self.blob_client.set_container_metadata(self.container, metadata=tags) - except AzureHttpError as exc: + self.blob_service_client.get_container_client(container=self.container).set_container_metadata(metadata=tags) + except Exception as exc: self.fail("Error updating container tags {0} - {1}".format(self.container, str(exc))) self.container_obj = self.get_container() self.results['changed'] = True @@ -568,8 +589,8 @@ def update_container_tags(self, tags): def update_blob_tags(self, tags): if not self.check_mode: try: - self.blob_client.set_blob_metadata(self.container, self.blob, metadata=tags) - except AzureHttpError as exc: + self.blob_service_client.get_blob_client(container=self.container, blob=self.blob).set_blob_metadata(metadata=tags) + except Exception as exc: self.fail("Update blob tags {0}:{1} - {2}".format(self.container, self.blob, str(exc))) self.blob_obj = self.get_blob() self.results['changed'] = True @@ -604,8 +625,8 @@ def update_blob_content_settings(self): ) if not self.check_mode: try: - self.blob_client.set_blob_properties(self.container, self.blob, content_settings=content_settings) - except AzureHttpError as exc: + self.blob_service_client.get_blob_client(container=self.container, blob=self.blob).set_http_headers(content_settings=content_settings) + except Exception as exc: self.fail("Update blob content settings {0}:{1} - {2}".format(self.container, self.blob, str(exc))) self.blob_obj = self.get_blob() diff --git a/plugins/modules/azure_rm_virtualmachine.py b/plugins/modules/azure_rm_virtualmachine.py index 79d3838fc..c404744d1 100644 --- a/plugins/modules/azure_rm_virtualmachine.py +++ b/plugins/modules/azure_rm_virtualmachine.py @@ -2030,12 +2030,12 @@ def delete_vm_storage(self, vhd_uris): container_name = blob_parts['containername'] blob_name = blob_parts['blobname'] - blob_client = self.get_blob_client(self.resource_group, storage_account_name) + blob_service_client = self.get_blob_service_client(self.resource_group, storage_account_name) self.log("Delete blob {0}:{1}".format(container_name, blob_name)) self.results['actions'].append("Deleted blob {0}:{1}".format(container_name, blob_name)) try: - blob_client.delete_blob(container_name, blob_name) + blob_service_client.get_blob_client(container=container_name, blob=blob_name).delete_blob() except Exception as exc: self.fail("Error deleting blob {0}:{1} - {2}".format(container_name, blob_name, str(exc))) return True diff --git a/requirements-azure.txt b/requirements-azure.txt index 5147ca1c2..12492fa2d 100644 --- a/requirements-azure.txt +++ b/requirements-azure.txt @@ -34,7 +34,7 @@ azure-mgmt-storage==19.0.0 azure-mgmt-trafficmanager==0.50.0 azure-mgmt-web==0.41.0 azure-nspkg==2.0.0 -azure-storage==0.35.1 +azure-storage-blob==12.11.0 msrest==0.6.21 msrestazure==0.6.4 azure-keyvault==1.0.0a1 diff --git a/tests/integration/targets/azure_rm_storageaccount/tasks/main.yml b/tests/integration/targets/azure_rm_storageaccount/tasks/main.yml index fa3c22ed0..8a33a7c6f 100644 --- a/tests/integration/targets/azure_rm_storageaccount/tasks/main.yml +++ b/tests/integration/targets/azure_rm_storageaccount/tasks/main.yml @@ -9,8 +9,7 @@ name: "invalid_char$" account_type: Standard_LRS register: output - ignore_errors: yes - + ignore_errors: true - name: Check intentional name failure. assert: that: @@ -22,7 +21,7 @@ resource_group: "{{ resource_group }}" name: "{{ item }}" state: absent - force_delete_nonempty: True + force_delete_nonempty: true loop: - "{{ storage_account_name_default }}" - "{{ storage_account_name_explicit }}" @@ -34,7 +33,6 @@ name: "{{ storage_account_name_default }}" account_type: Standard_LRS register: defaults_output - - name: Assert status succeeded and results match expectations assert: that: @@ -53,7 +51,6 @@ account_type: Premium_ZRS kind: FileStorage register: filestorage_output - - name: Assert status succeeded and results match I(kind=FileStorage) assert: that: @@ -65,7 +62,7 @@ access_tier: Hot account_type: Premium_LRS allow_blob_public_access: False - append_tags: no + append_tags: false blob_cors: - allowed_origins: - http://www.example.com/ @@ -95,7 +92,6 @@ test: test galaxy: galaxy register: explicit_output - - name: Assert status succeeded and correct parameter results assert: that: @@ -114,7 +110,7 @@ access_tier: Hot account_type: Premium_LRS allow_blob_public_access: False - append_tags: no + append_tags: false blob_cors: - allowed_origins: - http://www.example.com/ @@ -144,7 +140,6 @@ test: test galaxy: galaxy register: output - - name: Assert that properties have not changed assert: that: @@ -172,7 +167,6 @@ resource_group: "{{ resource_group }}" name: "{{ storage_account_name_explicit }}" register: output - - name: Assert that properties have not changed assert: that: @@ -200,7 +194,7 @@ resource_group: "{{ resource_group }}" name: "{{ storage_account_name_default }}" allow_blob_public_access: False - append_tags: no + append_tags: false blob_cors: - allowed_origins: - http://www.example.com/ @@ -227,7 +221,6 @@ test: test galaxy: galaxy register: output - - name: Assert account change success assert: that: @@ -249,8 +242,7 @@ name: "{{ storage_account_name_default }}" account_type: Premium_LRS register: output - ignore_errors: yes - + ignore_errors: true - name: Assert account type change failed assert: that: @@ -263,10 +255,9 @@ name: "{{ storage_account_name_default }}" custom_domain: name: ansible.com - use_sub_domain: no - ignore_errors: yes + use_sub_domain: false + ignore_errors: true register: output - - name: Assert CNAME failure assert: that: @@ -280,20 +271,18 @@ - test - galaxy register: output - - assert: - that: output.storageaccounts | length >= 1 + that: output.storageaccounts | length >= 1 - name: Update account tags azure_rm_storageaccount: resource_group: "{{ resource_group }}" name: "{{ storage_account_name_explicit }}" - append_tags: no + append_tags: false tags: testing: testing delete: never register: output - - assert: that: - "output.state.tags | length == 2" @@ -321,12 +310,10 @@ - output.storageaccounts[0].network_acls.bypass == "AzureServices" - output.storageaccounts[0].network_acls.default_action == "Deny" - output.storageaccounts[0].network_acls.ip_rules | length == 1 - - name: List storage accounts by resource group. azure_rm_storageaccount_info: resource_group: "{{ resource_group }}" register: output - - assert: that: - "output.storageaccounts | length >= 2" diff --git a/tests/integration/targets/azure_rm_storageblob/aliases b/tests/integration/targets/azure_rm_storageblob/aliases index e82395432..aa77c071a 100644 --- a/tests/integration/targets/azure_rm_storageblob/aliases +++ b/tests/integration/targets/azure_rm_storageblob/aliases @@ -1,4 +1,3 @@ cloud/azure shippable/azure/group2 destructive -unstable diff --git a/tests/integration/targets/azure_rm_storageblob/tasks/main.yml b/tests/integration/targets/azure_rm_storageblob/tasks/main.yml index 8137738d1..8ad07331f 100644 --- a/tests/integration/targets/azure_rm_storageblob/tasks/main.yml +++ b/tests/integration/targets/azure_rm_storageblob/tasks/main.yml @@ -1,6 +1,7 @@ - name: Create storage account name set_fact: - storage_account: "sb{{ resource_group | hash('md5') | truncate(22, True, '') }}" + storage_account: "sb{{ resource_group | hash('md5') | truncate(22, True, '') }}" + test1_file: "./targets/azure_rm_storageblob/files/Ratings.png" - name: Create storage account azure_rm_storageaccount: @@ -20,16 +21,12 @@ account_name: "{{ storage_account }}" container_name: my-blobs blob: 'Ratings.png' - src: './targets/azure_rm_storageblob/files/Ratings.png' + src: '{{ test1_file }}' content_type: image/png tags: - val1: foo - val2: bar - force: yes - -- name: storage blob seems to have some timing issues - wait_for: - delay: 10 + val1: foo + val2: bar + force: true - name: Upload blob idempotence azure_rm_storageblob: @@ -37,15 +34,14 @@ account_name: "{{ storage_account }}" container_name: my-blobs blob: 'Ratings.png' - src: './targets/azure_rm_storageblob/files/Ratings.png' + src: '{{ test1_file }}' content_type: image/png tags: - val1: foo - val2: bar + val1: foo + val2: bar register: upload_facts - - assert: - that: "not upload_facts.changed" + that: "not upload_facts.changed" - name: Download file idempotence azure_rm_storageblob: @@ -53,11 +49,10 @@ account_name: "{{ storage_account }}" container_name: my-blobs blob: 'Ratings.png' - dest: './targets/azure_rm_storageblob/files/Ratings.png' + dest: '{{ test1_file }}' register: download_results - - assert: - that: not download_results.changed + that: not download_results.changed - file: path="/tmp/Ratings.png" state=absent @@ -69,14 +64,13 @@ blob: 'Ratings.png' dest: '/tmp/Ratings.png' register: download_results - - assert: - that: "download_results.changed" + that: "download_results.changed" - find: paths='/tmp' patterns="Ratings.png" register: find_results - -- assert: { that: "find_results['matched'] == 1" } +- assert: + that: "find_results['matched'] == 1" - name: Do not delete container that has blobs azure_rm_storageblob: @@ -85,9 +79,8 @@ container_name: my-blobs state: absent register: output - - assert: - that: "not output.changed" + that: "not output.changed" - name: Delete blob object azure_rm_storageblob: @@ -97,9 +90,8 @@ blob: "Ratings.png" state: absent register: output - - assert: - that: "output.changed" + that: "output.changed" - name: Delete container azure_rm_storageblob: @@ -108,9 +100,8 @@ container_name: my-blobs state: absent register: output - - assert: - that: "output.changed" + that: "output.changed" - name: Delete storage account azure_rm_storageaccount: