Fixes to storage dependencies in CI

scripts/dev_setup.py

@@ -88,6 +88,11 @@ def pip_command(command, additional_dir=".", error_ok=False):
     content_packages.remove("azure-common")
 content_packages.insert(2, "azure-common")
 
+if 'azure-core' in content_packages:
+    content_packages.remove('azure-core')
+content_packages.insert(1, 'azure-core')
+
+
 print("Running dev setup...")
 print("Root directory '{}'\n".format(root_dir))
 

sdk/core/azure-servicemanagement-legacy/dev_requirements.txt

@@ -1 +1,2 @@
 -e ../../../tools/azure-sdk-tools
+-e ../../storage/azure-storage-blob

sdk/core/azure-servicemanagement-legacy/tests/legacy_mgmt_testcase.py

@@ -82,13 +82,18 @@ def create_service_management(self, service_class):
         return service
 
     def _create_storage_service(self, service_class, settings, account_name=None, account_key=None):
-        account_name = account_name or settings.STORAGE_ACCOUNT_NAME
-        account_key = account_key or settings.STORAGE_ACCOUNT_KEY
+        try:
+            account_name_arg = account_name or settings.STORAGE_ACCOUNT_NAME
+        except NameError:
+            account_name_arg = settings.STORAGE_ACCOUNT_NAME
+        try:
+            account_key_arg = account_key or settings.STORAGE_ACCOUNT_KEY
+        except NameError:
+            account_key_arg = settings.STORAGE_ACCOUNT_KEY
         session = Session()
         service = service_class(
-            account_name,
-            account_key,
-            request_session=session,
+            "https://{}.blob.core.windows.net".format(account_name_arg),
+            credentials=ccount_key_arg
         )
         self._set_service_options(service, settings)
         return service

sdk/core/azure-servicemanagement-legacy/tests/test_legacy_mgmt_misc.py

@@ -40,8 +40,12 @@
     parse_response_for_async_op,
     get_certificate_from_publish_settings,
 )
-from azure.storage.blob import PageBlobService, BlockBlobService
-from azure.storage.blob.models import PublicAccess
+from azure.core.exceptions import HttpResponseError, ResourceExistsError
+from azure.storage.blob import BlobServiceClient
+from azure.storage.blob.common import (
+    BlobType,
+    PublicAccess
+)
 from testutils.common_recordingtestcase import (
     TestMode,
     record,
@@ -85,9 +89,7 @@ def setUp(self):
         super(LegacyMgmtMiscTest, self).setUp()
 
         self.sms = self.create_service_management(ServiceManagementService)
-
-        self.bc = self._create_storage_service(PageBlobService, self.settings)
-        self.bbc = self._create_storage_service(BlockBlobService, self.settings)
+        self.bsc = self._create_storage_service(BlobServiceClient, self.settings)
 
         self.hosted_service_name = self.get_resource_name('utsvc')
         self.container_name = self.get_resource_name('utctnr')
@@ -166,7 +168,7 @@ def tearDown(self):
                     pass
 
             try:
-                self.bc.delete_container(self.container_name)
+                self.bsc.delete_container(self.container_name)
             except:
                 pass
 
@@ -284,14 +286,19 @@ def _make_blob_url(self, storage_account_name, container_name, blob_name):
 
     def _create_container_and_block_blob(self, container_name, blob_name,
                                          blob_data):
-        self.bc.create_container(container_name, None, 'container', False)
-        self.bbc.create_blob_from_bytes(
-            container_name, blob_name, blob_data)
+        try:
+            container = self.bsc.create_container(container_name, public_access='container')
+        except ResourceExistsError:
+            container = self.bsc.get_container_client(container_name)
+        container.upload_blob(blob_name, blob_data)
 
     def _create_container_and_page_blob(self, container_name, blob_name,
                                         content_length):
-        self.bc.create_container(container_name, None, 'container', False)
-        self.bc.create_blob_from_bytes(container_name, blob_name, b'')
+        try:
+            container = self.bsc.create_container(container_name, public_access='container')
+        except ResourceExistsError:
+            container = self.bsc.get_container_client(container_name)
+        container.upload_blob(blob_name, b'', blob_type=BlobType.PageBlob)
 
     def _upload_file_to_block_blob(self, file_path, blob_name):
         data = open(file_path, 'rb').read()
@@ -303,14 +310,13 @@ def _upload_file_to_block_blob(self, file_path, blob_name):
 
     def _upload_chunks(self, file_path, blob_name, chunk_size):
         index = 0
+        blob = self.bsc.get_blob_client(self.container_name, blob_name)
         with open(file_path, 'rb') as f:
             while True:
                 data = f.read(chunk_size)
                 if data:
                     length = len(data)
-                    self.bc.update_page(
-                        self.container_name, blob_name, data,
-                        index, index + length - 1)
+                    blob.upload_page(data, index, index + length - 1)
                     index += length
                 else:
                     break
@@ -379,7 +385,8 @@ def _os_image_exists(self, image_name):
 
     def _blob_exists(self, container_name, blob_name):
         try:
-            props = self.bc.get_blob_properties(container_name, blob_name)
+            blob = self.bsc.get_blob_client(container_name, blob_name)
+            props = blob.get_blob_properties()
             return props is not None
         except:
             return False
@@ -564,11 +571,13 @@ def _create_vm_image(self, image_name):
 
     def _copy_linux_os_vhd_to_container(self):
         blob_name = 'imagecopy.vhd'
-        self.bc.create_container(self.container_name,
-                                 public_access=PublicAccess.Blob)
-        resp = self.bc.copy_blob(self.container_name, blob_name,
-                                 self.settings.LINUX_OS_VHD)
-        return self.bc.make_blob_url(self.container_name, blob_name)
+        try:
+            container = self.bsc.create_container(self.container_name, public_access=PublicAccess.Blob)
+        except ResourceExistsError:
+            container = self.bsc.get_container_client(self.container_name)
+        blob = container.get_blob_client(blob_name)
+        blob.copy_blob_from_url(self.settings.LINUX_OS_VHD)
+        return blob.url
 
     #--Test cases for http passthroughs --------------------------------------
     @record

sdk/keyvault/azure-keyvault/dev_requirements.txt

@@ -1,4 +1,5 @@
 -e ../../authorization/azure-mgmt-authorization
 -e ../../core/azure-core
+-e ../../storage/azure-storage-blob
 -e ../azure-mgmt-keyvault
 -e ../../../tools/azure-sdk-tools

sdk/keyvault/azure-keyvault/tests/test_storage.py

@@ -1,4 +1,7 @@
 import uuid
+
+import pytest
+
 from devtools_testutils import AzureMgmtTestCase, ResourceGroupPreparer, StorageAccountPreparer
 from keyvault_preparer import KeyVaultPreparer
 from keyvault_testcase import KeyvaultTestCase
@@ -9,6 +12,7 @@
 
 class KeyVaultSecretTest(KeyvaultTestCase):
 
+    @pytest.mark.skip(reason="Cannot use a ServicePrincipal to execute set_storage_account. See issue azure-sdk-for-python#5858.")
     @ResourceGroupPreparer()
     @StorageAccountPreparer(name_prefix='kvsa1')
     @KeyVaultPreparer()
@@ -66,16 +70,22 @@ def create_account_sas_definition(self, storage_account_name, vault_url):
         """
         Creates an account sas definition, to manage storage account and its entities.
         """
-        from azure.storage.common import SharedAccessSignature, CloudStorageAccount
+        from azure.storage.common import SharedAccessSignature
+        from azure.storage.blob import BlobServiceClient
         from azure.keyvault.models import SasTokenType, SasDefinitionAttributes
         from azure.keyvault import SecretId
 
         # To create an account sas definition in the vault we must first create the template. The
         # template_uri for an account sas definition is the intended account sas token signed with an arbitrary key.
         # Use the SharedAccessSignature class from azure.storage.common to create a account sas token
+        storage_account_key = '00000000'
+        container_name='blobcontainer'
+        blob_name='blob1'
+        blob_data = u'test blob1 data'
+
         sas = SharedAccessSignature(account_name=storage_account_name,
                                     # don't sign the template with the storage account key use key 00000000
-                                    account_key='00000000')
+                                    account_key=storage_account_key)
         account_sas_template = sas.generate_account(services='bfqt',  # all services blob, file, queue and table
                                                     resource_types='sco',  # all resources service, template, object
                                                     permission='acdlpruw',
@@ -104,15 +114,16 @@ def create_account_sas_definition(self, storage_account_name, vault_url):
                                                          secret_version=sas_secret_id.version).value
 
         # create the cloud storage account object
-        cloud_storage_account = CloudStorageAccount(account_name=storage_account_name,
-                                                    sas_token=acct_sas_token)
+        blob_conn_string = 'EndpointSuffix=blob.core.windows.net;AccountKey={key};AccountName={hostname};DefaultEndpointsProtocol=https;SharedAccessSignature={token}'.format(
+            token=acct_sas_token[1:],
+            hostname=storage_account_name,
+            key=storage_account_key)
+
 
         # create a blob with the account sas token
-        blob_service = cloud_storage_account.create_block_blob_service()
-        blob_service.create_container('blobcontainer')
-        blob_service.create_blob_from_text(container_name='blobcontainer',
-                                           blob_name='blob1',
-                                           text=u'test blob1 data')
+        blob_service = BlobServiceClient.from_connection_string(conn_str=blob_conn_string)
+        container = blob_service.create_container(container_name)
+        container.upload_blob(blob_name, blob_data)
 
     def create_blob_sas_defintion(self, storage_account_name, vault_url):
         """

sdk/storage/azure-storage-blob/MANIFEST.in

@@ -1,5 +1,4 @@
-recursive-include tests *.py *.yaml
-include *.rst
+include *.md
 include azure/__init__.py
 include azure/storage/__init__.py
-
+include LICENSE.txt

sdk/storage/azure-storage-blob/azure/__init__.py

@@ -1 +1 @@
-__path__ = __import__('pkgutil').extend_path(__path__, __name__)
+__path__ = __import__('pkgutil').extend_path(__path__, __name__)  # type: str

sdk/storage/azure-storage-blob/azure/storage/__init__.py

@@ -1 +1 @@
-__path__ = __import__('pkgutil').extend_path(__path__, __name__)
+__path__ = __import__('pkgutil').extend_path(__path__, __name__)  # type: str