Add SAS Token Authentication Support to Azure Repo Plugin (elastic#42982

)

* Added setting for SAS token
* Added support for the token in tests
* Relates elastic#42117

docs/plugins/repository-azure.asciidoc

@@ -19,7 +19,11 @@ bin/elasticsearch-keystore add azure.client.default.account
 bin/elasticsearch-keystore add azure.client.default.key
 ----------------------------------------------------------------
 
-Where `account` is the azure account name and `key` the azure secret key.
+Where `account` is the azure account name and `key` the azure secret key. Instead of an azure secret key under `key`, you can alternatively
+define a shared access signatures (SAS) token under `sas_token` to use for authentication instead. When using an SAS token instead of an
+account key, the SAS token must have read (r), write (w), list (l), and delete (d) permissions for the repository base path and
+all its contents. These permissions need to be granted for the blob service (b) and apply to resource types service (s), container (c), and
+object (o).
 These settings are used by the repository's internal azure client.
 
 Note that you can also define more than one account:
@@ -29,14 +33,14 @@ Note that you can also define more than one account:
 bin/elasticsearch-keystore add azure.client.default.account
 bin/elasticsearch-keystore add azure.client.default.key
 bin/elasticsearch-keystore add azure.client.secondary.account
-bin/elasticsearch-keystore add azure.client.secondary.key
+bin/elasticsearch-keystore add azure.client.secondary.sas_token
 ----------------------------------------------------------------
 
 `default` is the default account name which will be used by a repository,
 unless you set an explicit one in the
 <<repository-azure-repository-settings, repository settings>>.
 
-Both `account` and `key` storage settings are
+The `account`, `key`, and `sas_token` storage settings are
 {ref}/secure-settings.html#reloadable-secure-settings[reloadable]. After you
 reload the settings, the internal azure clients, which are used to transfer the
 snapshot, will utilize the latest settings from the keystore.

plugins/repository-azure/build.gradle

@@ -76,6 +76,7 @@ String azureAccount = System.getenv("azure_storage_account")
 String azureKey = System.getenv("azure_storage_key")
 String azureContainer = System.getenv("azure_storage_container")
 String azureBasePath = System.getenv("azure_storage_base_path")
+String azureSasToken = System.getenv("azure_storage_sas_token")
 
 test {
   exclude '**/AzureStorageCleanupThirdPartyTests.class'
@@ -85,10 +86,11 @@ task thirdPartyTest(type: Test) {
   include '**/AzureStorageCleanupThirdPartyTests.class'
   systemProperty 'test.azure.account', azureAccount ? azureAccount : ""
   systemProperty 'test.azure.key', azureKey ? azureKey : ""
+  systemProperty 'test.azure.sas_token', azureSasToken ? azureSasToken : ""
   systemProperty 'test.azure.container', azureContainer ? azureContainer : ""
   systemProperty 'test.azure.base', azureBasePath ? azureBasePath : ""
 }
 
-if (azureAccount || azureKey || azureContainer || azureBasePath) {
+if (azureAccount || azureKey || azureContainer || azureBasePath || azureSasToken) {
   check.dependsOn(thirdPartyTest)
 }

plugins/repository-azure/qa/microsoft-azure-storage/build.gradle

@@ -29,12 +29,14 @@ String azureAccount = System.getenv("azure_storage_account")
 String azureKey = System.getenv("azure_storage_key")
 String azureContainer = System.getenv("azure_storage_container")
 String azureBasePath = System.getenv("azure_storage_base_path")
+String azureSasToken = System.getenv("azure_storage_sas_token")
 
-if (!azureAccount && !azureKey && !azureContainer && !azureBasePath) {
+if (!azureAccount && !azureKey && !azureContainer && !azureBasePath && !azureSasToken) {
     azureAccount = 'azure_integration_test_account'
     azureKey = 'YXp1cmVfaW50ZWdyYXRpb25fdGVzdF9rZXk=' // The key is "azure_integration_test_key" encoded using base64
     azureContainer = 'container_test'
     azureBasePath = 'integration_test'
+    azureSasToken = ''
     useFixture = true
 }
 
@@ -63,7 +65,14 @@ integTest {
 testClusters.integTest {
     plugin file(project(':plugins:repository-azure').bundlePlugin.archiveFile)
     keystore 'azure.client.integration_test.account', azureAccount
-    keystore 'azure.client.integration_test.key', azureKey
+    if (azureKey != null && azureKey.isEmpty() == false) {
+        println "Using access key in external service tests."
+        keystore 'azure.client.integration_test.key', azureKey
+    }
+    if (azureSasToken != null && azureSasToken.isEmpty() == false) {
+        println "Using SAS token in external service tests."
+        keystore 'azure.client.integration_test.sas_token', azureSasToken
+    }
 
     if (useFixture) {
         tasks.integTest.dependsOn azureStorageFixture

plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepositoryPlugin.java

@@ -60,6 +60,7 @@ public List<Setting<?>> getSettings() {
         return Arrays.asList(
             AzureStorageSettings.ACCOUNT_SETTING,
             AzureStorageSettings.KEY_SETTING,
+            AzureStorageSettings.SAS_TOKEN_SETTING,
             AzureStorageSettings.ENDPOINT_SUFFIX_SETTING,
             AzureStorageSettings.TIMEOUT_SETTING,
             AzureStorageSettings.MAX_RETRIES_SETTING,

plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageService.java

@@ -117,7 +117,7 @@ private static CloudBlobClient buildClient(AzureStorageSettings azureStorageSett
     }
 
     private static CloudBlobClient createClient(AzureStorageSettings azureStorageSettings) throws InvalidKeyException, URISyntaxException {
-        final String connectionString = azureStorageSettings.buildConnectionString();
+        final String connectionString = azureStorageSettings.getConnectString();
         return CloudStorageAccount.parse(connectionString).createCloudBlobClient();
     }
 

plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageSettings.java

@@ -21,6 +21,7 @@
 
 import com.microsoft.azure.storage.LocationMode;
 import com.microsoft.azure.storage.RetryPolicy;
+import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.collect.MapBuilder;
 import org.elasticsearch.common.settings.SecureSetting;
@@ -53,6 +54,10 @@ final class AzureStorageSettings {
     public static final AffixSetting<SecureString> KEY_SETTING = Setting.affixKeySetting(AZURE_CLIENT_PREFIX_KEY, "key",
         key -> SecureSetting.secureString(key, null));
 
+    /** Azure SAS token */
+    public static final AffixSetting<SecureString> SAS_TOKEN_SETTING = Setting.affixKeySetting(AZURE_CLIENT_PREFIX_KEY, "sas_token",
+        key -> SecureSetting.secureString(key, null));
+
     /** max_retries: Number of retries in case of Azure errors. Defaults to 3 (RetryPolicy.DEFAULT_CLIENT_RETRY_COUNT). */
     public static final Setting<Integer> MAX_RETRIES_SETTING =
         Setting.affixKeySetting(AZURE_CLIENT_PREFIX_KEY, "max_retries",
@@ -82,29 +87,29 @@ final class AzureStorageSettings {
         PROXY_HOST_SETTING);
 
     private final String account;
-    private final String key;
+    private final String connectString;
     private final String endpointSuffix;
     private final TimeValue timeout;
     private final int maxRetries;
     private final Proxy proxy;
     private final LocationMode locationMode;
 
     // copy-constructor
-    private AzureStorageSettings(String account, String key, String endpointSuffix, TimeValue timeout, int maxRetries, Proxy proxy,
-            LocationMode locationMode) {
+    private AzureStorageSettings(String account, String connectString, String endpointSuffix, TimeValue timeout, int maxRetries,
+                                 Proxy proxy, LocationMode locationMode) {
         this.account = account;
-        this.key = key;
+        this.connectString = connectString;
         this.endpointSuffix = endpointSuffix;
         this.timeout = timeout;
         this.maxRetries = maxRetries;
         this.proxy = proxy;
         this.locationMode = locationMode;
     }
 
-    AzureStorageSettings(String account, String key, String endpointSuffix, TimeValue timeout, int maxRetries,
-                                Proxy.Type proxyType, String proxyHost, Integer proxyPort) {
+    private AzureStorageSettings(String account, String key, String sasToken, String endpointSuffix, TimeValue timeout, int maxRetries,
+                                 Proxy.Type proxyType, String proxyHost, Integer proxyPort) {
         this.account = account;
-        this.key = key;
+        this.connectString = buildConnectString(account, key, sasToken, endpointSuffix);
         this.endpointSuffix = endpointSuffix;
         this.timeout = timeout;
         this.maxRetries = maxRetries;
@@ -145,13 +150,26 @@ public Proxy getProxy() {
         return proxy;
     }
 
-    public String buildConnectionString() {
+    public String getConnectString() {
+        return connectString;
+    }
+
+    private static String buildConnectString(String account, @Nullable String key, @Nullable String sasToken, String endpointSuffix) {
+        final boolean hasSasToken = Strings.hasText(sasToken);
+        final boolean hasKey = Strings.hasText(key);
+        if (hasSasToken == false && hasKey == false) {
+            throw new SettingsException("Neither a secret key nor a shared access token was set.");
+        }
+        if (hasSasToken && hasKey) {
+            throw new SettingsException("Both a secret as well as a shared access token were set.");
+        }
         final StringBuilder connectionStringBuilder = new StringBuilder();
-        connectionStringBuilder.append("DefaultEndpointsProtocol=https")
-                .append(";AccountName=")
-                .append(account)
-                .append(";AccountKey=")
-                .append(key);
+        connectionStringBuilder.append("DefaultEndpointsProtocol=https").append(";AccountName=").append(account);
+        if (hasKey) {
+            connectionStringBuilder.append(";AccountKey=").append(key);
+        } else {
+            connectionStringBuilder.append(";SharedAccessSignature=").append(sasToken);
+        }
         if (Strings.hasText(endpointSuffix)) {
             connectionStringBuilder.append(";EndpointSuffix=").append(endpointSuffix);
         }
@@ -166,7 +184,6 @@ public LocationMode getLocationMode() {
     public String toString() {
         final StringBuilder sb = new StringBuilder("AzureStorageSettings{");
         sb.append("account='").append(account).append('\'');
-        sb.append(", key='").append(key).append('\'');
         sb.append(", timeout=").append(timeout);
         sb.append(", endpointSuffix='").append(endpointSuffix).append('\'');
         sb.append(", maxRetries=").append(maxRetries);
@@ -201,8 +218,9 @@ public static Map<String, AzureStorageSettings> load(Settings settings) {
     /** Parse settings for a single client. */
     private static AzureStorageSettings getClientSettings(Settings settings, String clientName) {
         try (SecureString account = getConfigValue(settings, clientName, ACCOUNT_SETTING);
-             SecureString key = getConfigValue(settings, clientName, KEY_SETTING)) {
-            return new AzureStorageSettings(account.toString(), key.toString(),
+             SecureString key = getConfigValue(settings, clientName, KEY_SETTING);
+             SecureString sasToken = getConfigValue(settings, clientName, SAS_TOKEN_SETTING)) {
+            return new AzureStorageSettings(account.toString(), key.toString(), sasToken.toString(),
                 getValue(settings, clientName, ENDPOINT_SUFFIX_SETTING),
                 getValue(settings, clientName, TIMEOUT_SETTING),
                 getValue(settings, clientName, MAX_RETRIES_SETTING),
@@ -228,10 +246,9 @@ static Map<String, AzureStorageSettings> overrideLocationMode(Map<String, AzureS
                                                                   LocationMode locationMode) {
         final MapBuilder<String, AzureStorageSettings> mapBuilder = new MapBuilder<>();
         for (final Map.Entry<String, AzureStorageSettings> entry : clientsSettings.entrySet()) {
-            final AzureStorageSettings azureSettings = new AzureStorageSettings(entry.getValue().account, entry.getValue().key,
-                    entry.getValue().endpointSuffix, entry.getValue().timeout, entry.getValue().maxRetries, entry.getValue().proxy,
-                    locationMode);
-            mapBuilder.put(entry.getKey(), azureSettings);
+            mapBuilder.put(entry.getKey(),
+                new AzureStorageSettings(entry.getValue().account, entry.getValue().connectString, entry.getValue().endpointSuffix,
+                    entry.getValue().timeout, entry.getValue().maxRetries, entry.getValue().proxy, locationMode));
         }
         return mapBuilder.immutableMap();
     }

plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureStorageCleanupThirdPartyTests.java

@@ -20,6 +20,7 @@
 package org.elasticsearch.repositories.azure;
 
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.MockSecureSettings;
 import org.elasticsearch.common.settings.SecureSettings;
 import org.elasticsearch.common.settings.Settings;
@@ -42,13 +43,22 @@ protected Collection<Class<? extends Plugin>> getPlugins() {
     @Override
     protected SecureSettings credentials() {
         assertThat(System.getProperty("test.azure.account"), not(blankOrNullString()));
-        assertThat(System.getProperty("test.azure.key"), not(blankOrNullString()));
+        final boolean hasSasToken = Strings.hasText(System.getProperty("test.azure.sas_token"));
+        if (hasSasToken == false) {
+            assertThat(System.getProperty("test.azure.key"), not(blankOrNullString()));
+        } else {
+            assertThat(System.getProperty("test.azure.key"), blankOrNullString());
+        }
         assertThat(System.getProperty("test.azure.container"), not(blankOrNullString()));
         assertThat(System.getProperty("test.azure.base"), not(blankOrNullString()));
 
         MockSecureSettings secureSettings = new MockSecureSettings();
         secureSettings.setString("azure.client.default.account", System.getProperty("test.azure.account"));
-        secureSettings.setString("azure.client.default.key", System.getProperty("test.azure.key"));
+        if (hasSasToken) {
+            secureSettings.setString("azure.client.default.sas_token", System.getProperty("test.azure.sas_token"));
+        } else {
+            secureSettings.setString("azure.client.default.key", System.getProperty("test.azure.key"));
+        }
         return secureSettings;
     }
 

plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureStorageServiceTests.java

@@ -158,15 +158,21 @@ public void testReinitClientWrongSettings() throws IOException {
         secureSettings2.setString("azure.client.azure1.account", "myaccount1");
         // missing key
         final Settings settings2 = Settings.builder().setSecureSettings(secureSettings2).build();
+        final MockSecureSettings secureSettings3 = new MockSecureSettings();
+        secureSettings3.setString("azure.client.azure1.account", "myaccount3");
+        secureSettings3.setString("azure.client.azure1.key", encodeKey("mykey33"));
+        secureSettings3.setString("azure.client.azure1.sas_token", encodeKey("mysasToken33"));
+        final Settings settings3 = Settings.builder().setSecureSettings(secureSettings3).build();
         try (AzureRepositoryPlugin plugin = pluginWithSettingsValidation(settings1)) {
             final AzureStorageService azureStorageService = plugin.azureStoreService;
             final CloudBlobClient client11 = azureStorageService.client("azure1").v1();
             assertThat(client11.getEndpoint().toString(), equalTo("https://myaccount1.blob.core.windows.net"));
-            plugin.reload(settings2);
+            final SettingsException e1 = expectThrows(SettingsException.class, () -> plugin.reload(settings2));
+            assertThat(e1.getMessage(), is("Neither a secret key nor a shared access token was set."));
+            final SettingsException e2 = expectThrows(SettingsException.class, () -> plugin.reload(settings3));
+            assertThat(e2.getMessage(), is("Both a secret as well as a shared access token were set."));
             // existing client untouched
             assertThat(client11.getEndpoint().toString(), equalTo("https://myaccount1.blob.core.windows.net"));
-            final SettingsException e = expectThrows(SettingsException.class, () -> azureStorageService.client("azure1"));
-            assertThat(e.getMessage(), is("Invalid azure client settings with name [azure1]"));
         }
     }