Support AwsSigV4 signing for Amazon OpenSearch Serverless

CHANGELOG.md

@@ -3,6 +3,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 
 ## [Unreleased]
 ### Added
+- Added support for Amazon OpenSearch Serverless request signing ([#133](https://github.com/opensearch-project/opensearch-net/pull/133))
 - Github workflow for changelog verification ([#111](https://github.com/opensearch-project/opensearch-net/pull/111))
 - Bump version to `1.2.1`([#115](https://github.com/opensearch-project/opensearch-net/pull/115))
 - Added `dependabot_changelog` workflow ([#118](https://github.com/opensearch-project/opensearch-net/pull/118))
@@ -26,6 +27,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Bumps `Octokit` from 0.32.0 to 4.0.3
 - Bumps `BenchMarkDotNet` from 0.13.1 to 0.13.3
 - Bumps `System.Reactive` from 3.1.1 to 5.0.0
-- Bumps `SharpZipLib` from 1.0.4 to `1.4.1` ([#136](https://github.com/opensearch-project/opensearch-net/pull/136))
+- Bumps `SharpZipLib` from 1.0.4 to 1.4.1 ([#136](https://github.com/opensearch-project/opensearch-net/pull/136))
+- Bumps `AWSSDK.Core` from 3.7.12.11 to 3.7.103.17
 
 [Unreleased]: https://github.com/opensearch-project/opensearch-net/compare/1.2.0...HEAD

OpenSearch.sln

@@ -76,6 +76,7 @@ EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "OpenSearch.Net.Auth.AwsSigV4", "src\OpenSearch.Net.Auth.AwsSigV4\OpenSearch.Net.Auth.AwsSigV4.csproj", "{2BDA50C8-767A-4560-9547-15AFC972A6E3}"
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Tests.Auth.AwsSigV4", "tests\Tests.Auth.AwsSigV4\Tests.Auth.AwsSigV4.csproj", "{2BCE8150-C16C-4226-B216-C7A0463ADC56}"
+EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "OpenSearch.Stack.ArtifactsApiTests", "abstractions\tests\OpenSearch.Stack.ArtifactsApiTests\OpenSearch.Stack.ArtifactsApiTests.csproj", "{1F5A7B1A-2566-481F-91B5-A63D7F939973}"
 EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "abstractions", "abstractions", "{87ABA679-F3F4-48CE-82B3-1AAE5D0A5935}"

USER_GUIDE.md

@@ -10,6 +10,7 @@
     - [Getting Started](#getting-started-1)
     - [Connecting](#connecting-1)
     - [Configuring Region & Credentials](#configuring-region--credentials)
+    - [Amazon OpenSearch Serverless](#amazon-opensearch-serverless)
   - [OpenSearch.Net](#opensearchnet)
     - [Getting Started](#getting-started-2)
     - [Connecting](#connecting-2)
@@ -26,7 +27,7 @@ Include OpenSearch.Client in your .csproj file.
 <Project>
   ...
   <ItemGroup>
-    <PackageReference Include="OpenSearch.Client" Version="1.0.0" />
+    <PackageReference Include="OpenSearch.Client" Version="1.*" />
   </ItemGroup>
 </Project>
 ```
@@ -141,7 +142,7 @@ var response = lowLevelClient.Search<SearchResponse<Tweet>>("mytweetindex", Post
 
 ## [OpenSearch.Net.Auth.AwsSigV4](src/OpenSearch.Net.Auth.AwsSigV4)
 
-An implementation of AWS SigV4 request signing for performing IAM authentication against the managed Amazon OpenSearch Service.
+An implementation of AWS SigV4 request signing for performing IAM authentication against the managed [Amazon OpenSearch Service](https://aws.amazon.com/opensearch-service/).
 It can be used with both the low-level OpenSearch.Net client as well as the higher-level OpenSearch.Client client.
 
 ### Getting Started
@@ -150,8 +151,8 @@ Include OpenSearch.Net.Auth.AwsSigV4 along with your preferred client in your .c
 <Project>
   ...
   <ItemGroup>
-    <PackageReference Include="OpenSearch.Client" Version="1.0.0" />
-    <PackageReference Include="OpenSearch.Net.Auth.AwsSigV4" Version="1.0.0" />
+    <PackageReference Include="OpenSearch.Client" Version="1.*" />
+    <PackageReference Include="OpenSearch.Net.Auth.AwsSigV4" Version="1.*" />
   </ItemGroup>
 </Project>
 ```
@@ -219,6 +220,20 @@ var config = new ConnectionSettings(endpoint, connection);
 var client = new OpenSearchClient(config);
 ```
 
+### Amazon OpenSearch Serverless
+Use the `"aoss"` service code to make requests to [Amazon OpenSearch Serverless](https://aws.amazon.com/opensearch-service/features/serverless/), otherwise all configuration options are identical to above.
+```shell
+export AWS_ACCESS_KEY_ID="..."
+export AWS_SECRET_ACCESS_KEY="..."
+export AWS_SESSION_TOKEN="..."
+```
+```csharp
+var endpoint = new Uri("https://aaabbbcccddd111222333.ap-southeast-2.aoss.amazonaws.com");
+var connection = new AwsSigV4HttpConnection(RegionEndpoint.APSoutheast2, service: AwsSigV4HttpConnection.OpenSearchServerlessService);
+var config = new ConnectionSettings(endpoint, connection);
+var client = new OpenSearchClient(config);
+```
+
 ## [OpenSearch.Net](src/OpenSearch.Net)
 
 A low-level, dependency free client that is a simple .NET wrapper for the REST API. It allows you to build and represent your own requests and responses according to you needs.

src/OpenSearch.Net.Auth.AwsSigV4/AwsSigV4HttpClientHandler.cs

@@ -20,19 +20,21 @@ internal class AwsSigV4HttpClientHandler : DelegatingHandler
 	{
 		private readonly AWSCredentials _credentials;
 		private readonly RegionEndpoint _region;
+		private readonly string _service;
 
-		public AwsSigV4HttpClientHandler(AWSCredentials credentials, RegionEndpoint region, HttpMessageHandler innerHandler)
+		public AwsSigV4HttpClientHandler(AWSCredentials credentials, RegionEndpoint region, string service, HttpMessageHandler innerHandler)
 			: base(innerHandler)
 		{
 			_credentials = credentials ?? throw new ArgumentNullException(nameof(credentials));
 			_region = region ?? throw new ArgumentNullException(nameof(region));
+			_service = service ?? throw new ArgumentNullException(nameof(service));
 		}
 
 		protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
 		{
 			var credentials = await _credentials.GetCredentialsAsync().ConfigureAwait(false);
 
-			await AwsSigV4Util.SignRequest(request, credentials, _region, DateTime.UtcNow).ConfigureAwait(false);
+			await AwsSigV4Util.SignRequest(request, credentials, _region, DateTime.UtcNow, _service).ConfigureAwait(false);
 
 			return await base.SendAsync(request, cancellationToken).ConfigureAwait(false);
 		}

src/OpenSearch.Net.Auth.AwsSigV4/AwsSigV4HttpConnection.cs

@@ -16,58 +16,67 @@ namespace OpenSearch.Net.Auth.AwsSigV4
 	/// </summary>
 	public class AwsSigV4HttpConnection : HttpConnection
 	{
+		public const string OpenSearchService = "es";
+		public const string OpenSearchServerlessService = "aoss";
+
 		private readonly AWSCredentials _credentials;
 		private readonly RegionEndpoint _region;
+		private readonly string _service;
 
 		/// <summary>
 		/// Construct a new connection discovering both the credentials and region from the environment.
 		/// </summary>
-		/// <seealso cref="AwsSigV4HttpConnection(AWSCredentials, RegionEndpoint)"/>
-		public AwsSigV4HttpConnection() : this(null, null) { }
+		/// <param name="service">The service code to use when signing, defaults to the service code for the Amazon OpenSearch Service (<c>"es"</c>).</param>
+		/// <seealso cref="AwsSigV4HttpConnection(AWSCredentials, RegionEndpoint, string)"/>
+		public AwsSigV4HttpConnection(string service = OpenSearchService) : this(null, null, service) { }
 
 		/// <summary>
 		/// Construct a new connection configured with the specified credentials and using the region discovered from the environment.
 		/// </summary>
 		/// <param name="credentials">The credentials to use when signing.</param>
-		/// <seealso cref="AwsSigV4HttpConnection(AWSCredentials, RegionEndpoint)"/>
-		public AwsSigV4HttpConnection(AWSCredentials credentials) : this(credentials, null) { }
+		/// <param name="service">The service code to use when signing, defaults to the service code for the Amazon OpenSearch Service (<c>"es"</c>).</param>
+		/// <seealso cref="AwsSigV4HttpConnection(AWSCredentials, RegionEndpoint, string)"/>
+		public AwsSigV4HttpConnection(AWSCredentials credentials, string service = OpenSearchService) : this(credentials, null, service) { }
 
 		/// <summary>
 		/// Construct a new connection configured with a specified region and using credentials discovered from the environment.
 		/// </summary>
 		/// <param name="region">The region to use when signing.</param>
-		/// <seealso cref="AwsSigV4HttpConnection(AWSCredentials, RegionEndpoint)"/>
-		public AwsSigV4HttpConnection(RegionEndpoint region) : this(null, region) { }
+		/// <param name="service">The service code to use when signing, defaults to the service code for the Amazon OpenSearch Service (<c>"es"</c>).</param>
+		/// <seealso cref="AwsSigV4HttpConnection(AWSCredentials, RegionEndpoint, string)"/>
+		public AwsSigV4HttpConnection(RegionEndpoint region, string service = OpenSearchService) : this(null, region, service) { }
 
 		/// <summary>
 		/// Construct a new connection configured with the given credentials and region.
 		/// </summary>
 		/// <param name="credentials">The credentials to use when signing, or null to have them discovered automatically by the AWS SDK.</param>
 		/// <param name="region">The region to use when signing, or null to have it discovered automatically by the AWS SDK.</param>
+		///	<param name="service">The service code to use when signing, defaults to the service code for the Amazon OpenSearch Service (<c>"es"</c>).</param>
 		/// <exception cref="ArgumentNullException">Thrown if region is null and is unable to be automatically discovered by the AWS SDK.</exception>
 		/// <remarks>
 		/// The same logic as the <a href="https://docs.aws.amazon.com/sdk-for-net/v3/developer-guide/creds-assign.html">AWS SDK for .NET</a>
 		/// is used to automatically discover the credentials and region to use if not provided explicitly.
 		/// </remarks>
-		public AwsSigV4HttpConnection(AWSCredentials credentials, RegionEndpoint region)
+		public AwsSigV4HttpConnection(AWSCredentials credentials, RegionEndpoint region, string service = OpenSearchService)
 		{
 			_credentials = credentials ?? FallbackCredentialsFactory.GetCredentials(); // FallbackCredentialsFactory throws in case of not finding credentials.
 			_region = region
 				?? FallbackRegionFactory.GetRegionEndpoint() // FallbackRegionFactory can return null.
 				?? throw new ArgumentNullException(nameof(region), "A RegionEndpoint was not provided and was unable to be determined from the environment.");
+			_service = service ?? OpenSearchService;
 		}
 
 #if DOTNETCORE
 
 		protected override System.Net.Http.HttpMessageHandler CreateHttpClientHandler(RequestData requestData) =>
-			new AwsSigV4HttpClientHandler(_credentials, _region, base.CreateHttpClientHandler(requestData));
+			new AwsSigV4HttpClientHandler(_credentials, _region, _service, base.CreateHttpClientHandler(requestData));
 
 #else
 
 		protected override System.Net.HttpWebRequest CreateHttpWebRequest(RequestData requestData)
 		{
 			var request = base.CreateHttpWebRequest(requestData);
-			AwsSigV4Util.SignRequest(request, requestData, _credentials.GetCredentials(), _region, DateTime.UtcNow);
+			AwsSigV4Util.SignRequest(request, requestData, _credentials.GetCredentials(), _region, DateTime.UtcNow, _service);
 			return request;
 		}
 

src/OpenSearch.Net.Auth.AwsSigV4/AwsSigV4Util.cs

@@ -25,34 +25,38 @@ public static async Task SignRequest(
 			HttpRequestMessage request,
 			ImmutableCredentials credentials,
 			RegionEndpoint region,
-			DateTime signingTime)
+			DateTime signingTime,
+			string service)
 		{
 			var canonicalRequest = await CanonicalRequest.From(request, credentials, signingTime).ConfigureAwait(false);
 
-			var signature = AWS4Signer.ComputeSignature(credentials, region.SystemName, signingTime, "es", canonicalRequest.SignedHeaders,
+			var signature = AWS4Signer.ComputeSignature(credentials, region.SystemName, signingTime, service, canonicalRequest.SignedHeaders,
 				canonicalRequest.ToString());
 
-			request.Headers.TryAddWithoutValidation("x-amz-date", canonicalRequest.XAmzDate);
-			request.Headers.TryAddWithoutValidation("authorization", signature.ForAuthorizationHeader);
-			if (!string.IsNullOrEmpty(canonicalRequest.XAmzSecurityToken)) request.Headers.TryAddWithoutValidation("x-amz-security-token", canonicalRequest.XAmzSecurityToken);
+			request.Headers.TryAddWithoutValidation(HeaderNames.XAmzDate, canonicalRequest.XAmzDate);
+			request.Headers.TryAddWithoutValidation(HeaderNames.XAmzContentSha256, canonicalRequest.XAmzContentSha256);
+			request.Headers.TryAddWithoutValidation(HeaderNames.Authorization, signature.ForAuthorizationHeader);
+			if (!string.IsNullOrEmpty(canonicalRequest.XAmzSecurityToken)) request.Headers.TryAddWithoutValidation(HeaderNames.XAmzSecurityToken, canonicalRequest.XAmzSecurityToken);
 		}
 #else
 		public static void SignRequest(
 			HttpWebRequest request,
 			RequestData requestData,
 			ImmutableCredentials credentials,
 			RegionEndpoint region,
-			DateTime signingTime)
+			DateTime signingTime,
+			string service)
 		{
 			var canonicalRequest = CanonicalRequest.From(request, requestData, credentials, signingTime);
 
-			var signature = AWS4Signer.ComputeSignature(credentials, region.SystemName, signingTime, "es", canonicalRequest.SignedHeaders,
+			var signature = AWS4Signer.ComputeSignature(credentials, region.SystemName, signingTime, service, canonicalRequest.SignedHeaders,
 				canonicalRequest.ToString());
 
-			request.Headers["x-amz-date"] = canonicalRequest.XAmzDate;
-			request.Headers["authorization"] = signature.ForAuthorizationHeader;
+			request.Headers[HeaderNames.XAmzDate] = canonicalRequest.XAmzDate;
+			request.Headers[HeaderNames.XAmzContentSha256] = canonicalRequest.XAmzContentSha256;
+			request.Headers[HeaderNames.Authorization] = signature.ForAuthorizationHeader;
 			if (!string.IsNullOrEmpty(canonicalRequest.XAmzSecurityToken))
-				request.Headers["x-amz-security-token"] = canonicalRequest.XAmzSecurityToken;
+				request.Headers[HeaderNames.XAmzSecurityToken] = canonicalRequest.XAmzSecurityToken;
 		}
 #endif
 	}

src/OpenSearch.Net.Auth.AwsSigV4/CanonicalRequest.cs

@@ -30,12 +30,14 @@ namespace OpenSearch.Net.Auth.AwsSigV4
 	public class CanonicalRequest
 	{
 		private static readonly IComparer<KeyValuePair<string, string>> StringPairComparer = new KeyValuePairComparer<string, string>();
+		private static readonly ISet<string> ExcludedHeaders = new HashSet<string> { HeaderNames.UserAgent, HeaderNames.ContentLength };
 
 		private readonly string _method;
 		private readonly string _path;
 		private readonly string _params;
 		private readonly SortedDictionary<string, List<string>> _headers;
-		private readonly string _contentSha256;
+
+		public string XAmzContentSha256 { get; }
 
 		public string XAmzDate { get; }
 
@@ -44,14 +46,14 @@ public class CanonicalRequest
 		public string SignedHeaders { get; }
 
 		private CanonicalRequest(string method, string path, string queryParams, SortedDictionary<string, List<string>> headers,
-			string contentSha256, string xAmzDate, string xAmzSecurityToken
+			string xAmzContentSha256, string xAmzDate, string xAmzSecurityToken
 		)
 		{
 			_method = method;
 			_path = path;
 			_params = queryParams;
 			_headers = headers;
-			_contentSha256 = contentSha256;
+			XAmzContentSha256 = xAmzContentSha256;
 			SignedHeaders = string.Join(";", _headers.Keys);
 			XAmzDate = xAmzDate;
 			XAmzSecurityToken = xAmzSecurityToken;
@@ -63,15 +65,15 @@ public static async Task<CanonicalRequest> From(HttpRequestMessage request, Immu
 		public static CanonicalRequest From(HttpWebRequest request, RequestData requestData, ImmutableCredentials credentials, DateTime signingTime)
 #endif
 		{
-			var path = AWSSDKUtils.CanonicalizeResourcePath(request.RequestUri, null, false);
+			var path = AWSSDKUtils.CanonicalizeResourcePathV2(request.RequestUri, null, false, null);
 
 #if DOTNETCORE
 			var bodyBytes = await GetBodyBytes(request).ConfigureAwait(false);
 #else
 			var bodyBytes = GetBodyBytes(requestData);
 #endif
 
-			var contentSha256 = AWSSDKUtils.ToHex(AWS4Signer.ComputeHash(bodyBytes), true);
+			var xAmzContentSha256 = AWSSDKUtils.ToHex(AWS4Signer.ComputeHash(bodyBytes), true);
 
 			var xAmzDate = AWS4Signer.FormatDateTime(signingTime, "yyyyMMddTHHmmssZ");
 
@@ -82,14 +84,15 @@ public static CanonicalRequest From(HttpWebRequest request, RequestData requestD
 			CanonicalizeHeaders(canonicalHeaders, request.Content?.Headers);
 #endif
 
-			canonicalHeaders["host"] = new List<string> { request.RequestUri.Authority };
-			canonicalHeaders["x-amz-date"] = new List<string> { xAmzDate };
+			canonicalHeaders[HeaderNames.Host] = new List<string> { request.RequestUri.Authority };
+			canonicalHeaders[HeaderNames.XAmzDate] = new List<string> { xAmzDate };
+			canonicalHeaders[HeaderNames.XAmzContentSha256] = new List<string> { xAmzContentSha256 };
 
 			string xAmzSecurityToken = null;
 			if (credentials.UseToken)
 			{
 				xAmzSecurityToken = credentials.Token;
-				canonicalHeaders["x-amz-security-token"] = new List<string> { xAmzSecurityToken };
+				canonicalHeaders[HeaderNames.XAmzSecurityToken] = new List<string> { xAmzSecurityToken };
 			}
 
 			var queryParams = HttpUtility.ParseQueryString(request.RequestUri.Query);
@@ -112,7 +115,7 @@ public static CanonicalRequest From(HttpWebRequest request, RequestData requestD
 			var method = request.Method;
 #endif
 
-			return new CanonicalRequest(method, path, paramString, canonicalHeaders, contentSha256, xAmzDate, xAmzSecurityToken);
+			return new CanonicalRequest(method, path, paramString, canonicalHeaders, xAmzContentSha256, xAmzDate, xAmzSecurityToken);
 		}
 
 #if DOTNETCORE
@@ -127,7 +130,7 @@ private static async Task<byte[]> GetBodyBytes(HttpRequestMessage request)
 			var content = new ByteArrayContent(body);
 			foreach (var pair in request.Content.Headers)
 			{
-				if (string.Equals(pair.Key, "Content-Length", StringComparison.OrdinalIgnoreCase)) continue;
+				if (string.Equals(pair.Key, HeaderNames.ContentLength, StringComparison.OrdinalIgnoreCase)) continue;
 
 				content.Headers.TryAddWithoutValidation(pair.Key, pair.Value);
 			}
@@ -173,7 +176,7 @@ NameValueCollection headers
 
 				var key = pair.Key.ToLowerInvariant();
 
-				if (key == "user-agent") continue;
+				if (ExcludedHeaders.Contains(key)) continue;
 
 				if (!canonicalHeaders.TryGetValue(key, out var dictValues))
 					dictValues = canonicalHeaders[key] = new List<string>();
@@ -192,7 +195,7 @@ public override string ToString()
 			foreach (var header in _headers) sb.Append($"{header.Key}:{string.Join(",", header.Value)}\n");
 			sb.Append('\n');
 			sb.Append($"{SignedHeaders}\n");
-			sb.Append(_contentSha256);
+			sb.Append(XAmzContentSha256);
 
 			return sb.ToString();
 		}

src/OpenSearch.Net.Auth.AwsSigV4/HeaderNames.cs

@@ -0,0 +1,20 @@
+/* SPDX-License-Identifier: Apache-2.0
+*
+* The OpenSearch Contributors require contributions made to
+* this file be licensed under the Apache-2.0 license or a
+* compatible open source license.
+*/
+
+namespace OpenSearch.Net.Auth.AwsSigV4
+{
+	internal static class HeaderNames
+	{
+		public const string Authorization = "authorization";
+		public const string ContentLength = "content-length";
+		public const string Host = "host";
+		public const string UserAgent = "user-agent";
+		public const string XAmzContentSha256 = "x-amz-content-sha256";
+		public const string XAmzDate = "x-amz-date";
+		public const string XAmzSecurityToken = "x-amz-security-token";
+	}
+}

src/OpenSearch.Net.Auth.AwsSigV4/OpenSearch.Net.Auth.AwsSigV4.csproj

@@ -20,7 +20,7 @@
     <ProjectReference Include="$(SolutionRoot)\src\OpenSearch.Net\OpenSearch.Net.csproj" />
   </ItemGroup>
   <ItemGroup>
-    <PackageReference Include="AWSSDK.Core" Version="3.7.12.11" />
+    <PackageReference Include="AWSSDK.Core" Version="3.7.103.17" />
   </ItemGroup>
   <ItemGroup Condition="'$(TargetFramework)'=='net461'">
     <Reference Include="System.Web" />