From 9e1334f59356f4f75b9ef8d08ac71f252a66d2bf Mon Sep 17 00:00:00 2001 From: Shishir Patil Date: Thu, 27 Jan 2022 12:57:25 -0800 Subject: [PATCH 1/2] multiple regions --- skylark/obj_store/azure_interface.py | 11 ++++------- skylark/obj_store/azure_keys.py | 16 ++++++++++++++++ skylark/test/test_azure_interface.py | 6 +++--- 3 files changed, 23 insertions(+), 10 deletions(-) create mode 100644 skylark/obj_store/azure_keys.py diff --git a/skylark/obj_store/azure_interface.py b/skylark/obj_store/azure_interface.py index 61872037d..69f485b1a 100644 --- a/skylark/obj_store/azure_interface.py +++ b/skylark/obj_store/azure_interface.py @@ -6,6 +6,7 @@ import os, uuid, time from azure.storage.blob import BlobServiceClient, BlobClient, ContainerClient, __version__, BlobBlock +from skylark.obj_store.azure_keys import azure_storage_credentials from skylark.obj_store.object_store_interface import NoSuchObjectException, ObjectStoreInterface, ObjectStoreObject from azure.core.exceptions import HttpResponseError, ResourceExistsError, ResourceNotFoundError @@ -20,19 +21,15 @@ class AzureInterface(ObjectStoreInterface): def __init__(self, azure_region, container_name): # TODO: the azure region should get corresponding os.getenv() self.azure_region = azure_region + assert self.azure_region in azure_storage_credentials self.container_name = container_name self.bucket_name = self.container_name # For compatibility self.pending_downloads, self.completed_downloads = 0, 0 self.pending_uploads, self.completed_uploads = 0, 0 - # Retrieve the connection string for use with the application. The storage - # connection string is stored in an environment variable on the machine - # running the application called AZURE_STORAGE_CONNECTION_STRING. If the environment variable is - # created after the application is launched in a console or with Visual Studio, - # the shell or application needs to be closed and reloaded to take the - # environment variable into account. - self._connect_str = os.getenv("AZURE_STORAGE_CONNECTION_STRING") + # Connection strings are stored in azure_keys.py + self._connect_str = azure_storage_credentials[self.azure_region]['connection_string'] # Create the BlobServiceClient object which will be used to create a container client self.blob_service_client = BlobServiceClient.from_connection_string(self._connect_str) diff --git a/skylark/obj_store/azure_keys.py b/skylark/obj_store/azure_keys.py new file mode 100644 index 000000000..e563bd451 --- /dev/null +++ b/skylark/obj_store/azure_keys.py @@ -0,0 +1,16 @@ +''' +Azure account config +=== +Performance: standard +subscription: skylark-azure-paras +redundancy: locally-redundant storage (LRS) +Routing Preferance: Microsoft Network Routing +''' +azure_storage_credentials = { "eastus":{'name':'skyeastus','connection_string':'DefaultEndpointsProtocol=https;AccountName=skyuseast;AccountKey=BgxPfOR5GB+0SR7B+qIgSly1Ih+M2xfOhtqxornE18N+2MyULBqH1QG7lmro/+o3UncwUuc8m4AduqFY3sC7UA==;EndpointSuffix=core.windows.net'}, +"westus":{'name':'skyuswest','connection_string':'DefaultEndpointsProtocol=https;AccountName=skyuswest;AccountKey=vvBaJdH8ndD65f5VGuUTUSiHzKse82+ogr+1qfaEAXFjtcgm7WAol78eHPNE4liSE/79QrfoMxz3MtgA0iIXNw==;EndpointSuffix=core.windows.net'}, +"centralus":{'name':'skycentralus','connection_string':'DefaultEndpointsProtocol=https;AccountName=skycentralus;AccountKey=xSIiDWw10JZHuS6reLJdzmebxBwctRwpd/hNOJ4C/ciKvy2ez57oRN7ZF5A3ETY495A2wcO+Lutf0feyEdWU2A==;EndpointSuffix=core.windows.net'} +} + + + + diff --git a/skylark/test/test_azure_interface.py b/skylark/test/test_azure_interface.py index 67f7188f3..92f3faa2f 100644 --- a/skylark/test/test_azure_interface.py +++ b/skylark/test/test_azure_interface.py @@ -8,9 +8,9 @@ def test_azure_interface(): - azure_interface = AzureInterface(f"us-east1", f"sky-us-east-2") - assert azure_interface.bucket_name == "sky-us-east-2" - assert azure_interface.azure_region == "us-east1" + azure_interface = AzureInterface(f"eastus", f"sky-us-east-1") + assert azure_interface.bucket_name == "sky-us-east-1" + assert azure_interface.azure_region == "eastus" azure_interface.create_bucket() # generate file and upload From fa1e01ab6073c0beaa2186663fa5db7c5617497f Mon Sep 17 00:00:00 2001 From: Shishir Patil Date: Thu, 27 Jan 2022 21:02:36 +0000 Subject: [PATCH 2/2] Black --- skylark/obj_store/azure_interface.py | 2 +- skylark/obj_store/azure_keys.py | 24 +++++++++++++++--------- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/skylark/obj_store/azure_interface.py b/skylark/obj_store/azure_interface.py index 69f485b1a..67463f1c6 100644 --- a/skylark/obj_store/azure_interface.py +++ b/skylark/obj_store/azure_interface.py @@ -29,7 +29,7 @@ def __init__(self, azure_region, container_name): self.pending_uploads, self.completed_uploads = 0, 0 # Connection strings are stored in azure_keys.py - self._connect_str = azure_storage_credentials[self.azure_region]['connection_string'] + self._connect_str = azure_storage_credentials[self.azure_region]["connection_string"] # Create the BlobServiceClient object which will be used to create a container client self.blob_service_client = BlobServiceClient.from_connection_string(self._connect_str) diff --git a/skylark/obj_store/azure_keys.py b/skylark/obj_store/azure_keys.py index e563bd451..e7ce6e476 100644 --- a/skylark/obj_store/azure_keys.py +++ b/skylark/obj_store/azure_keys.py @@ -1,16 +1,22 @@ -''' +""" Azure account config === Performance: standard subscription: skylark-azure-paras redundancy: locally-redundant storage (LRS) Routing Preferance: Microsoft Network Routing -''' -azure_storage_credentials = { "eastus":{'name':'skyeastus','connection_string':'DefaultEndpointsProtocol=https;AccountName=skyuseast;AccountKey=BgxPfOR5GB+0SR7B+qIgSly1Ih+M2xfOhtqxornE18N+2MyULBqH1QG7lmro/+o3UncwUuc8m4AduqFY3sC7UA==;EndpointSuffix=core.windows.net'}, -"westus":{'name':'skyuswest','connection_string':'DefaultEndpointsProtocol=https;AccountName=skyuswest;AccountKey=vvBaJdH8ndD65f5VGuUTUSiHzKse82+ogr+1qfaEAXFjtcgm7WAol78eHPNE4liSE/79QrfoMxz3MtgA0iIXNw==;EndpointSuffix=core.windows.net'}, -"centralus":{'name':'skycentralus','connection_string':'DefaultEndpointsProtocol=https;AccountName=skycentralus;AccountKey=xSIiDWw10JZHuS6reLJdzmebxBwctRwpd/hNOJ4C/ciKvy2ez57oRN7ZF5A3ETY495A2wcO+Lutf0feyEdWU2A==;EndpointSuffix=core.windows.net'} +""" +azure_storage_credentials = { + "eastus": { + "name": "skyeastus", + "connection_string": "DefaultEndpointsProtocol=https;AccountName=skyuseast;AccountKey=BgxPfOR5GB+0SR7B+qIgSly1Ih+M2xfOhtqxornE18N+2MyULBqH1QG7lmro/+o3UncwUuc8m4AduqFY3sC7UA==;EndpointSuffix=core.windows.net", + }, + "westus": { + "name": "skyuswest", + "connection_string": "DefaultEndpointsProtocol=https;AccountName=skyuswest;AccountKey=vvBaJdH8ndD65f5VGuUTUSiHzKse82+ogr+1qfaEAXFjtcgm7WAol78eHPNE4liSE/79QrfoMxz3MtgA0iIXNw==;EndpointSuffix=core.windows.net", + }, + "centralus": { + "name": "skycentralus", + "connection_string": "DefaultEndpointsProtocol=https;AccountName=skycentralus;AccountKey=xSIiDWw10JZHuS6reLJdzmebxBwctRwpd/hNOJ4C/ciKvy2ez57oRN7ZF5A3ETY495A2wcO+Lutf0feyEdWU2A==;EndpointSuffix=core.windows.net", + }, } - - - -