Add shared access signature authentication support

plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureCredentials.java

@@ -0,0 +1,25 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.repositories.azure;
+
+public interface AzureCredentials {
+    String buildConnectionString(String endpointSuffix);
+}
+

plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureKeyCredentials.java

@@ -0,0 +1,69 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.repositories.azure;
+
+import org.elasticsearch.common.Strings;
+
+import java.util.Objects;
+
+public class AzureKeyCredentials implements AzureCredentials {
+    private final String account;
+    private final String key;
+
+    public AzureKeyCredentials(String account, String key) {
+        this.account = account;
+        this.key = key;
+    }
+
+    public String getAccount() {
+        return account;
+    }
+
+    public String getKey() {
+        return key;
+    }
+
+    public String buildConnectionString(String endpointSuffix) {
+        final StringBuilder connectionStringBuilder = new StringBuilder();
+        connectionStringBuilder.append("DefaultEndpointsProtocol=https")
+            .append(";AccountName=")
+            .append(this.getAccount())
+            .append(";AccountKey=")
+            .append(this.getKey());
+        if (Strings.hasText(endpointSuffix)) {
+            connectionStringBuilder.append(";EndpointSuffix=").append(endpointSuffix);
+        }
+        return connectionStringBuilder.toString();
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        AzureKeyCredentials that = (AzureKeyCredentials) o;
+        return Objects.equals(account, that.account) &&
+            Objects.equals(key, that.key);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(account, key);
+    }
+}

plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java

@@ -139,14 +139,17 @@ protected ByteSizeValue chunkSize() {
 
     @Override
     public void initializeSnapshot(SnapshotId snapshotId, List<IndexId> indices, MetaData clusterMetadata) {
-        try {
-            final AzureBlobStore blobStore = (AzureBlobStore) blobStore();
-            if (blobStore.containerExist() == false) {
-                throw new IllegalArgumentException("The bucket [" + blobStore + "] does not exist. Please create it before "
-                        + " creating an azure snapshot repository backed by it.");
+        // Skip bucket existence check if sas_token is set since it might not hold the required permissions to list buckets
+        if(!storageService.storageSettings.containsKey("sas_token")) {
+            try {
+                final AzureBlobStore blobStore = (AzureBlobStore) blobStore();
+                if (blobStore.containerExist() == false) {
+                    throw new IllegalArgumentException("The bucket [" + blobStore + "] does not exist. Please create it before "
+                            + " creating an azure snapshot repository backed by it.");
+                }
+            } catch (URISyntaxException | StorageException e) {
+                throw new SnapshotCreationException(metadata.name(), snapshotId, e);
             }
-        } catch (URISyntaxException | StorageException e) {
-            throw new SnapshotCreationException(metadata.name(), snapshotId, e);
         }
         super.initializeSnapshot(snapshotId, indices, clusterMetadata);
     }

plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepositoryPlugin.java

@@ -60,6 +60,7 @@ public List<Setting<?>> getSettings() {
         return Arrays.asList(
             AzureStorageSettings.ACCOUNT_SETTING,
             AzureStorageSettings.KEY_SETTING,
+            AzureStorageSettings.SAS_TOKEN_SETTING,
             AzureStorageSettings.ENDPOINT_SUFFIX_SETTING,
             AzureStorageSettings.TIMEOUT_SETTING,
             AzureStorageSettings.MAX_RETRIES_SETTING,

plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureSasCredentials.java

@@ -0,0 +1,73 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.repositories.azure;
+
+import org.elasticsearch.common.Strings;
+
+import java.util.Objects;
+
+public class AzureSasCredentials implements AzureCredentials {
+    private final String account;
+    private final String sasToken;
+
+    public AzureSasCredentials(String account, String sasToken) {
+        this.account = account;
+        this.sasToken = sasToken;
+    }
+
+    public String getSasToken() {
+        return sasToken;
+    }
+
+    public String getAccount() {
+        return account;
+    }
+
+    public String buildConnectionString(String endpointSuffix) {
+        final StringBuilder connectionStringBuilder = new StringBuilder();
+        connectionStringBuilder.append("DefaultEndpointsProtocol=https")
+            .append(";AccountName=")
+            .append(this.getAccount())
+            .append(";SharedAccessSignature=")
+            .append(this.getSasToken());
+        if (Strings.hasText(endpointSuffix)) {
+            connectionStringBuilder.append(";EndpointSuffix=").append(endpointSuffix);
+        }
+        return connectionStringBuilder.toString();
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (o == null || getClass() != o.getClass()) {
+            return false;
+        }
+        if (!super.equals(o)) return false;
+        AzureSasCredentials that = (AzureSasCredentials) o;
+        return Objects.equals(sasToken, that.sasToken);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), sasToken);
+    }
+}

plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageService.java

@@ -117,6 +117,7 @@ private static CloudBlobClient buildClient(AzureStorageSettings azureStorageSett
 
     private static CloudBlobClient createClient(AzureStorageSettings azureStorageSettings) throws InvalidKeyException, URISyntaxException {
         final String connectionString = azureStorageSettings.buildConnectionString();
+
         return CloudStorageAccount.parse(connectionString).createCloudBlobClient();
     }
 

plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageSettings.java

@@ -53,58 +53,58 @@ final class AzureStorageSettings {
     public static final AffixSetting<SecureString> KEY_SETTING = Setting.affixKeySetting(AZURE_CLIENT_PREFIX_KEY, "key",
         key -> SecureSetting.secureString(key, null));
 
+    /** Azure SAS token */
+    public static final AffixSetting<SecureString> SAS_TOKEN_SETTING = Setting.affixKeySetting(AZURE_CLIENT_PREFIX_KEY, "sas_token",
+        key -> SecureSetting.secureString(key, null));
+
     /** max_retries: Number of retries in case of Azure errors. Defaults to 3 (RetryPolicy.DEFAULT_CLIENT_RETRY_COUNT). */
     public static final Setting<Integer> MAX_RETRIES_SETTING =
         Setting.affixKeySetting(AZURE_CLIENT_PREFIX_KEY, "max_retries",
             (key) -> Setting.intSetting(key, RetryPolicy.DEFAULT_CLIENT_RETRY_COUNT, Setting.Property.NodeScope),
-            ACCOUNT_SETTING, KEY_SETTING);
+            ACCOUNT_SETTING);
     /**
      * Azure endpoint suffix. Default to core.windows.net (CloudStorageAccount.DEFAULT_DNS).
      */
     public static final Setting<String> ENDPOINT_SUFFIX_SETTING = Setting.affixKeySetting(AZURE_CLIENT_PREFIX_KEY, "endpoint_suffix",
-        key -> Setting.simpleString(key, Property.NodeScope), ACCOUNT_SETTING, KEY_SETTING);
+        key -> Setting.simpleString(key, Property.NodeScope), ACCOUNT_SETTING);
 
     public static final AffixSetting<TimeValue> TIMEOUT_SETTING = Setting.affixKeySetting(AZURE_CLIENT_PREFIX_KEY, "timeout",
-        (key) -> Setting.timeSetting(key, TimeValue.timeValueMinutes(-1), Property.NodeScope), ACCOUNT_SETTING, KEY_SETTING);
+        (key) -> Setting.timeSetting(key, TimeValue.timeValueMinutes(-1), Property.NodeScope), ACCOUNT_SETTING);
 
     /** The type of the proxy to connect to azure through. Can be direct (no proxy, default), http or socks */
     public static final AffixSetting<Proxy.Type> PROXY_TYPE_SETTING = Setting.affixKeySetting(AZURE_CLIENT_PREFIX_KEY, "proxy.type",
-        (key) -> new Setting<>(key, "direct", s -> Proxy.Type.valueOf(s.toUpperCase(Locale.ROOT)), Property.NodeScope)
-        , ACCOUNT_SETTING, KEY_SETTING);
+        (key) -> new Setting<>(key, "direct", s -> Proxy.Type.valueOf(s.toUpperCase(Locale.ROOT)), Property.NodeScope), ACCOUNT_SETTING);
 
     /** The host name of a proxy to connect to azure through. */
     public static final AffixSetting<String> PROXY_HOST_SETTING = Setting.affixKeySetting(AZURE_CLIENT_PREFIX_KEY, "proxy.host",
-        (key) -> Setting.simpleString(key, Property.NodeScope), KEY_SETTING, ACCOUNT_SETTING, PROXY_TYPE_SETTING);
+        (key) -> Setting.simpleString(key, Property.NodeScope), ACCOUNT_SETTING, PROXY_TYPE_SETTING);
 
     /** The port of a proxy to connect to azure through. */
     public static final Setting<Integer> PROXY_PORT_SETTING = Setting.affixKeySetting(AZURE_CLIENT_PREFIX_KEY, "proxy.port",
-        (key) -> Setting.intSetting(key, 0, 0, 65535, Setting.Property.NodeScope), ACCOUNT_SETTING, KEY_SETTING, PROXY_TYPE_SETTING,
+        (key) -> Setting.intSetting(key, 0, 0, 65535, Setting.Property.NodeScope), ACCOUNT_SETTING, PROXY_TYPE_SETTING,
         PROXY_HOST_SETTING);
 
-    private final String account;
-    private final String key;
+    private final AzureCredentials credentials;
     private final String endpointSuffix;
     private final TimeValue timeout;
     private final int maxRetries;
     private final Proxy proxy;
     private final LocationMode locationMode;
 
     // copy-constructor
-    private AzureStorageSettings(String account, String key, String endpointSuffix, TimeValue timeout, int maxRetries, Proxy proxy,
+    private AzureStorageSettings(AzureCredentials credentials, String endpointSuffix, TimeValue timeout, int maxRetries, Proxy proxy,
             LocationMode locationMode) {
-        this.account = account;
-        this.key = key;
+        this.credentials = credentials;
         this.endpointSuffix = endpointSuffix;
         this.timeout = timeout;
         this.maxRetries = maxRetries;
         this.proxy = proxy;
         this.locationMode = locationMode;
     }
 
-    AzureStorageSettings(String account, String key, String endpointSuffix, TimeValue timeout, int maxRetries,
+    AzureStorageSettings(AzureCredentials credentials, String endpointSuffix, TimeValue timeout, int maxRetries,
                                 Proxy.Type proxyType, String proxyHost, Integer proxyPort) {
-        this.account = account;
-        this.key = key;
+        this.credentials = credentials;
         this.endpointSuffix = endpointSuffix;
         this.timeout = timeout;
         this.maxRetries = maxRetries;
@@ -146,16 +146,7 @@ public Proxy getProxy() {
     }
 
     public String buildConnectionString() {
-        final StringBuilder connectionStringBuilder = new StringBuilder();
-        connectionStringBuilder.append("DefaultEndpointsProtocol=https")
-                .append(";AccountName=")
-                .append(account)
-                .append(";AccountKey=")
-                .append(key);
-        if (Strings.hasText(endpointSuffix)) {
-            connectionStringBuilder.append(";EndpointSuffix=").append(endpointSuffix);
-        }
-        return connectionStringBuilder.toString();
+        return credentials.buildConnectionString(endpointSuffix);
     }
 
     public LocationMode getLocationMode() {
@@ -165,8 +156,7 @@ public LocationMode getLocationMode() {
     @Override
     public String toString() {
         final StringBuilder sb = new StringBuilder("AzureStorageSettings{");
-        sb.append("account='").append(account).append('\'');
-        sb.append(", key='").append(key).append('\'');
+        sb.append("credentials='").append(credentials.toString()).append('\'');
         sb.append(", timeout=").append(timeout);
         sb.append(", endpointSuffix='").append(endpointSuffix).append('\'');
         sb.append(", maxRetries=").append(maxRetries);
@@ -200,15 +190,34 @@ public static Map<String, AzureStorageSettings> load(Settings settings) {
     // pkg private for tests
     /** Parse settings for a single client. */
     private static AzureStorageSettings getClientSettings(Settings settings, String clientName) {
-        try (SecureString account = getConfigValue(settings, clientName, ACCOUNT_SETTING);
-             SecureString key = getConfigValue(settings, clientName, KEY_SETTING)) {
-            return new AzureStorageSettings(account.toString(), key.toString(),
+            return new AzureStorageSettings(loadCredentials(settings, clientName),
                 getValue(settings, clientName, ENDPOINT_SUFFIX_SETTING),
                 getValue(settings, clientName, TIMEOUT_SETTING),
                 getValue(settings, clientName, MAX_RETRIES_SETTING),
                 getValue(settings, clientName, PROXY_TYPE_SETTING),
                 getValue(settings, clientName, PROXY_HOST_SETTING),
                 getValue(settings, clientName, PROXY_PORT_SETTING));
+    }
+
+    private static AzureCredentials loadCredentials(Settings settings, String clientName) {
+        try (SecureString account = getConfigValue(settings, clientName, ACCOUNT_SETTING);
+             SecureString key = getConfigValue(settings, clientName, KEY_SETTING);
+             SecureString sasToken = getConfigValue(settings, clientName, SAS_TOKEN_SETTING)
+        ) {
+            if(key.length() != 0 && sasToken.length() != 0) {
+                throw new SettingsException("Both key and sas_token are set for [" + clientName + "] - only one must be specified");
+            }
+            if (account.length() != 0) {
+                if(key.length() != 0) {
+                    return new AzureKeyCredentials(account.toString(), key.toString());
+                } else if(sasToken.length() != 0) {
+                    return  new AzureSasCredentials(account.toString(), sasToken.toString());
+                } else {
+                    throw new SettingsException("Either key or sas_token need to be defined for azure client [" + clientName + "]");
+                }
+            } else {
+                throw new SettingsException("Missing account name for azure client [" + clientName + "]");
+            }
         }
     }
 
@@ -228,7 +237,7 @@ static Map<String, AzureStorageSettings> overrideLocationMode(Map<String, AzureS
                                                                   LocationMode locationMode) {
         final var map = new HashMap<String, AzureStorageSettings>();
         for (final Map.Entry<String, AzureStorageSettings> entry : clientsSettings.entrySet()) {
-            final AzureStorageSettings azureSettings = new AzureStorageSettings(entry.getValue().account, entry.getValue().key,
+            final AzureStorageSettings azureSettings = new AzureStorageSettings(entry.getValue().credentials,
                     entry.getValue().endpointSuffix, entry.getValue().timeout, entry.getValue().maxRetries, entry.getValue().proxy,
                     locationMode);
             map.put(entry.getKey(), azureSettings);