[Storage] Integrate decryption for CSE v2.1 (#46064)

* Initial commit

* Commit refactor

* removed obselete

* Added Changelog

* Fixed some LiveOnly tests

* Changelog added for Blobs and Queues

* Reverting changelog for Common and Queues

sdk/storage/Azure.Storage.Blobs/CHANGELOG.md

@@ -7,6 +7,7 @@
 ### Breaking Changes
 
 ### Bugs Fixed
+- Integrated decryption for CSE v2.1
 
 ### Other Changes
 

sdk/storage/Azure.Storage.Blobs/src/BlobClientSideDecryptor.cs

@@ -59,10 +59,10 @@ public async Task<Stream> DecryptInternal(
             int alreadyTrimmedOffset = encryptionData.EncryptionAgent.EncryptionVersion switch
             {
 #pragma warning disable CS0618 // obsolete
-                ClientSideEncryptionVersion.V1_0 => ivInStream ? Constants.ClientSideEncryption.EncryptionBlockSize : 0,
+                ClientSideEncryptionVersionInternal.V1_0 => ivInStream ? Constants.ClientSideEncryption.EncryptionBlockSize : 0,
 #pragma warning restore CS0618 // obsolete
                 // first block is special case where we don't want to communicate a trim. Otherwise communicate nonce length * 1-indexed start region + tag length * 0-indexed region
-                ClientSideEncryptionVersion.V2_0 => contentRange?.Start > 0
+                ClientSideEncryptionVersionInternal.V2_0 or ClientSideEncryptionVersionInternal.V2_1 => contentRange?.Start > 0
                     ? (-encryptionData.EncryptedRegionInfo.NonceLength * (v2StartRegion0Indexed)) - (Constants.ClientSideEncryption.V2.TagSize * v2StartRegion0Indexed)
                     : 0,
                 _ => throw Errors.ClientSideEncryption.ClientSideEncryptionVersionNotSupported()
@@ -141,12 +141,13 @@ internal static EncryptionData GetAndValidateEncryptionDataOrDefault(Metadata me
             switch (encryptionData.EncryptionAgent.EncryptionVersion)
             {
 #pragma warning disable CS0618 // obsolete
-                case ClientSideEncryptionVersion.V1_0:
+                case ClientSideEncryptionVersionInternal.V1_0:
                     _ = encryptionData.ContentEncryptionIV ?? throw Errors.ClientSideEncryption.MissingEncryptionMetadata(
                         nameof(EncryptionData.ContentEncryptionIV));
                     break;
 #pragma warning restore CS0618 // obsolete
-                case ClientSideEncryptionVersion.V2_0:
+                case ClientSideEncryptionVersionInternal.V2_0:
+                case ClientSideEncryptionVersionInternal.V2_1:
                     _ = encryptionData.EncryptedRegionInfo ?? throw Errors.ClientSideEncryption.MissingEncryptionMetadata(
                         nameof(EncryptionData.EncryptedRegionInfo));
                     break;
@@ -211,10 +212,11 @@ internal static HttpRange GetEncryptedBlobRange(HttpRange originalRange, Encrypt
             switch (encryptionData.EncryptionAgent.EncryptionVersion)
             {
 #pragma warning disable CS0618 // obsolete
-                case ClientSideEncryptionVersion.V1_0:
+                case ClientSideEncryptionVersionInternal.V1_0:
                     return GetEncryptedBlobRangeV1_0(originalRange);
 #pragma warning restore CS0618 // obsolete
-                case ClientSideEncryptionVersion.V2_0:
+                case ClientSideEncryptionVersionInternal.V2_0:
+                case ClientSideEncryptionVersionInternal.V2_1:
                     return GetEncryptedBlobRangeV2_0(originalRange, encryptionData);
                 default:
                     throw Errors.InvalidArgument(nameof(encryptionData));

sdk/storage/Azure.Storage.Blobs/tests/ClientSideEncryptionTests.cs

@@ -165,10 +165,10 @@ private async Task<byte[]> ReplicateEncryption(byte[] plaintext, BlobProperties
             switch (encryptionMetadata.EncryptionAgent.EncryptionVersion)
             {
 #pragma warning disable CS0618 // obsolete
-                case ClientSideEncryptionVersion.V1_0:
+                case ClientSideEncryptionVersionInternal.V1_0:
                     return await ReplicateEncryptionV1_0(plaintext, encryptionMetadata, keyEncryptionKey);
 #pragma warning restore CS0618 // obsolete
-                case ClientSideEncryptionVersion.V2_0:
+                case ClientSideEncryptionVersionInternal.V2_0:
                     return await ReplicateEncryptionV2_0(plaintext, encryptionMetadata, keyEncryptionKey);
                 default:
                     throw new ArgumentException("Bad version in EncryptionData");
@@ -181,7 +181,7 @@ private async Task<byte[]> ReplicateEncryption(byte[] plaintext, BlobProperties
         private async Task<byte[]> ReplicateEncryptionV1_0(byte[] plaintext, EncryptionData encryptionMetadata, IKeyEncryptionKey keyEncryptionKey)
         {
             Assert.NotNull(encryptionMetadata, "Never encrypted data.");
-            Assert.AreEqual(ClientSideEncryptionVersion.V1_0, encryptionMetadata.EncryptionAgent.EncryptionVersion);
+            Assert.AreEqual(ClientSideEncryptionVersionInternal.V1_0, encryptionMetadata.EncryptionAgent.EncryptionVersion);
 
             var explicitlyUnwrappedKey = IsAsync // can't instrument this
                 ? await keyEncryptionKey.UnwrapKeyAsync(s_algorithmName, encryptionMetadata.WrappedContentKey.EncryptedKey, s_cancellationToken).ConfigureAwait(false)
@@ -197,7 +197,7 @@ private async Task<byte[]> ReplicateEncryptionV1_0(byte[] plaintext, EncryptionD
         private async Task<byte[]> ReplicateEncryptionV2_0(byte[] plaintext, EncryptionData encryptionMetadata, IKeyEncryptionKey keyEncryptionKey)
         {
             Assert.NotNull(encryptionMetadata, "Never encrypted data.");
-            Assert.AreEqual(ClientSideEncryptionVersion.V2_0, encryptionMetadata.EncryptionAgent.EncryptionVersion);
+            Assert.AreEqual(ClientSideEncryptionVersionInternal.V2_0, encryptionMetadata.EncryptionAgent.EncryptionVersion);
 
             var explicitlyUnwrappedContent = IsAsync // can't instrument this
                 ? await keyEncryptionKey.UnwrapKeyAsync(s_algorithmName, encryptionMetadata.WrappedContentKey.EncryptedKey, s_cancellationToken).ConfigureAwait(false)
@@ -247,7 +247,7 @@ private async Task<EncryptionData> GetMockEncryptionDataAsync(byte[] cek, IKeyEn
                 EncryptionAgent = new EncryptionAgent()
                 {
                     EncryptionAlgorithm = "foo",
-                    EncryptionVersion = ClientSideEncryptionVersion.V2_0
+                    EncryptionVersion = ClientSideEncryptionVersionInternal.V2_0
                 },
                 EncryptionMode = "bar",
                 KeyWrappingMetadata = new Dictionary<string, string> { { "fizz", "buzz" } }
@@ -685,7 +685,22 @@ public async Task RoundtripWithMetadata([ValueSource("GetEncryptionVersions")] C
                     Assert.AreEqual(metadata[kvp.Key], downloadedMetadata[kvp.Key]);
                 }
                 Assert.IsTrue(downloadedMetadata.ContainsKey(EncryptionDataKey));
-                Assert.AreEqual(version, EncryptionDataSerializer.Deserialize(downloadedMetadata[EncryptionDataKey]).EncryptionAgent.EncryptionVersion);
+
+                ClientSideEncryptionVersionInternal versionInternal = ClientSideEncryptionVersionInternal.V2_0;
+                switch (version)
+                {
+#pragma warning disable CS0618 // obsolete
+                    case ClientSideEncryptionVersion.V1_0:
+                        versionInternal = ClientSideEncryptionVersionInternal.V1_0;
+                        break;
+#pragma warning restore CS0618 // obsolete
+                    case ClientSideEncryptionVersion.V2_0:
+                        versionInternal = ClientSideEncryptionVersionInternal.V2_0;
+                        break;
+                    default:
+                        throw new ArgumentException("Bad version in EncryptionData");
+                }
+                Assert.AreEqual(versionInternal, EncryptionDataSerializer.Deserialize(downloadedMetadata[EncryptionDataKey]).EncryptionAgent.EncryptionVersion);
             }
         }
 

sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/ClientSideDecryptor.cs

@@ -10,6 +10,7 @@
 using Azure.Core.Cryptography;
 using Azure.Core.Pipeline;
 using Azure.Storage.Cryptography.Models;
+using static Azure.Storage.Cryptography.Models.ClientSideEncryptionVersionExtensions;
 
 namespace Azure.Storage.Cryptography
 {
@@ -75,7 +76,7 @@ public async Task<Stream> DecryptReadInternal(
             switch (encryptionData.EncryptionAgent.EncryptionVersion)
             {
 #pragma warning disable CS0618 // obsolete
-                case ClientSideEncryptionVersion.V1_0:
+                case ClientSideEncryptionVersionInternal.V1_0:
                     return await DecryptReadInternalV1_0(
                         ciphertext,
                         encryptionData,
@@ -84,7 +85,8 @@ public async Task<Stream> DecryptReadInternal(
                         async,
                         cancellationToken).ConfigureAwait(false);
 #pragma warning restore CS0618 // obsolete
-                case ClientSideEncryptionVersion.V2_0:
+                case ClientSideEncryptionVersionInternal.V2_0:
+                case ClientSideEncryptionVersionInternal.V2_1:
                     return await DecryptInternalV2_0(
                         ciphertext,
                         encryptionData,
@@ -125,14 +127,15 @@ public async Task<Stream> DecryptWholeContentWriteInternal(
             switch (encryptionData.EncryptionAgent.EncryptionVersion)
             {
 #pragma warning disable CS0618 // obsolete
-                case ClientSideEncryptionVersion.V1_0:
+                case ClientSideEncryptionVersionInternal.V1_0:
                     return await DecryptWholeContentWriteInternalV1_0(
                         plaintextDestination,
                         encryptionData,
                         async,
                         cancellationToken).ConfigureAwait(false);
 #pragma warning restore CS0618 // obsolete
-                case ClientSideEncryptionVersion.V2_0:
+                case ClientSideEncryptionVersionInternal.V2_0:
+                case ClientSideEncryptionVersionInternal.V2_1:
                     return await DecryptInternalV2_0(
                         plaintextDestination,
                         encryptionData,
@@ -403,13 +406,14 @@ internal async Task<Memory<byte>> GetContentEncryptionKeyAsync(
             switch (encryptionData.EncryptionAgent.EncryptionVersion)
             {
 #pragma warning disable CS0618 // obsolete
-                case ClientSideEncryptionVersion.V1_0:
+                case ClientSideEncryptionVersionInternal.V1_0:
                     unwrappedKey = unwrappedContent;
                     break;
 #pragma warning restore CS0618 // obsolete
                 // v2.0 binds content encryption key with content encryption algorithm under a single keywrap.
                 // Separate key from algorithm ID and validate ID match
-                case ClientSideEncryptionVersion.V2_0:
+                case ClientSideEncryptionVersionInternal.V2_0:
+                case ClientSideEncryptionVersionInternal.V2_1:
                     string unwrappedProtocolString = Encoding.UTF8.GetString(
                         unwrappedContent,
                         index: 0,

sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/ClientSideEncryptionVersionExtensions.cs

@@ -1,6 +1,8 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT License.
 
+using System;
+
 namespace Azure.Storage.Cryptography.Models
 {
     internal static class ClientSideEncryptionVersionExtensions
@@ -9,38 +11,61 @@ public static class ClientSideEncryptionVersionString
         {
             public const string V1_0 = "1.0";
             public const string V2_0 = "2.0";
+            public const string V2_1 = "2.1";
         }
 
-        public static string Serialize(this ClientSideEncryptionVersion version)
+        public static string Serialize(this ClientSideEncryptionVersionInternal version)
         {
             switch (version)
             {
 #pragma warning disable CS0618 // obsolete
-                case ClientSideEncryptionVersion.V1_0:
+                case ClientSideEncryptionVersionInternal.V1_0:
                     return ClientSideEncryptionVersionString.V1_0;
 #pragma warning restore CS0618 // obsolete
-                case ClientSideEncryptionVersion.V2_0:
+                case ClientSideEncryptionVersionInternal.V2_0:
                     return ClientSideEncryptionVersionString.V2_0;
+                case ClientSideEncryptionVersionInternal.V2_1:
+                    return ClientSideEncryptionVersionString.V2_1;
                 default:
                     // sanity check; serialize is in this file to make it easy to add the serialization cases
                     throw Errors.ClientSideEncryption.ClientSideEncryptionVersionNotSupported();
             }
         }
 
-        public static ClientSideEncryptionVersion ToClientSideEncryptionVersion(this string versionString)
+        public static ClientSideEncryptionVersionInternal ToClientSideEncryptionVersion(this string versionString)
         {
             switch (versionString)
             {
 #pragma warning disable CS0618 // obsolete
                 case ClientSideEncryptionVersionString.V1_0:
-                    return ClientSideEncryptionVersion.V1_0;
+                    return ClientSideEncryptionVersionInternal.V1_0;
 #pragma warning restore CS0618 // obsolete
                 case ClientSideEncryptionVersionString.V2_0:
-                    return ClientSideEncryptionVersion.V2_0;
+                    return ClientSideEncryptionVersionInternal.V2_0;
+                case ClientSideEncryptionVersionString.V2_1:
+                    return ClientSideEncryptionVersionInternal.V2_1;
                 default:
                     // This library doesn't support the stated encryption version
                     throw Errors.ClientSideEncryption.ClientSideEncryptionVersionNotSupported(versionString);
             }
         }
     }
+
+    internal enum ClientSideEncryptionVersionInternal
+    {
+        /// <summary>
+        /// 1.0.
+        /// </summary>
+        V1_0 = 1,
+
+        /// <summary>
+        /// 2.0.
+        /// </summary>
+        V2_0 = 2,
+
+        /// <summary>
+        /// 2.1.
+        /// </summary>
+        V2_1 = 21
+    }
 }

sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Models/EncryptionAgent.cs

@@ -11,7 +11,7 @@ internal class EncryptionAgent
         /// <summary>
         /// The protocol version used for encryption.
         /// </summary>
-        public ClientSideEncryptionVersion EncryptionVersion { get; set; }
+        public ClientSideEncryptionVersionInternal EncryptionVersion { get; set; }
 
         /// <summary>
         /// The algorithm used for encryption.

sdk/storage/Azure.Storage.Common/src/Shared/ClientsideEncryption/Models/EncryptionData.cs

@@ -65,7 +65,7 @@ internal static async ValueTask<EncryptionData> CreateInternalV1_0(
                 {
                     EncryptionAlgorithm = ClientSideEncryptionAlgorithm.AesCbc256,
 #pragma warning disable CS0618 // obsolete
-                    EncryptionVersion = ClientSideEncryptionVersion.V1_0
+                    EncryptionVersion = ClientSideEncryptionVersionInternal.V1_0
 #pragma warning restore CS0618 // obsolete
                 },
                 KeyWrappingMetadata = new Dictionary<string, string>()
@@ -92,7 +92,7 @@ internal static async Task<EncryptionData> CreateInternalV2_0(
             // v2.0 binds content encryption key with protocol version under a single keywrap
             int keyOffset = Constants.ClientSideEncryption.V2.WrappedDataVersionLength;
             var dataToWrap = new byte[keyOffset + contentEncryptionKey.Length];
-            Encoding.UTF8.GetBytes(ClientSideEncryptionVersion.V2_0.Serialize()).CopyTo(dataToWrap, 0);
+            Encoding.UTF8.GetBytes(ClientSideEncryptionVersionInternal.V2_0.Serialize()).CopyTo(dataToWrap, 0);
             contentEncryptionKey.CopyTo(dataToWrap, keyOffset);
 
             return new EncryptionData()
@@ -101,7 +101,7 @@ internal static async Task<EncryptionData> CreateInternalV2_0(
                 EncryptionAgent = new EncryptionAgent()
                 {
                     EncryptionAlgorithm = ClientSideEncryptionAlgorithm.AesGcm256,
-                    EncryptionVersion = ClientSideEncryptionVersion.V2_0
+                    EncryptionVersion = ClientSideEncryptionVersionInternal.V2_0
                 },
                 EncryptedRegionInfo = new EncryptedRegionInfo()
                 {

sdk/storage/Azure.Storage.Queues/tests/ClientSideEncryptionTests.cs

@@ -241,18 +241,19 @@ public async Task UploadAsync(ClientSideEncryptionVersion version, int messageSi
                 switch (encryptionMetadata.EncryptionAgent.EncryptionVersion)
                 {
 #pragma warning disable CS0618 // obsolete
-                    case ClientSideEncryptionVersion.V1_0:
+                    case ClientSideEncryptionVersionInternal.V1_0:
                         explicitlyUnwrappedKey = explicitlyUnwrappedContent;
                         break;
 #pragma warning restore CS0618 // obsolete
-                    case ClientSideEncryptionVersion.V2_0:
+                    case ClientSideEncryptionVersionInternal.V2_0:
                         explicitlyUnwrappedKey = new Span<byte>(explicitlyUnwrappedContent).Slice(8).ToArray();
                         break;
                     default:
                         throw new Exception();
                 }
 
                 string expectedEncryptedMessage;
+                ClientSideEncryptionVersionInternal versionInternal = ClientSideEncryptionVersionInternal.V2_0;
                 switch (version)
                 {
 #pragma warning disable CS0618 // obsolete
@@ -261,18 +262,20 @@ public async Task UploadAsync(ClientSideEncryptionVersion version, int messageSi
                             message,
                             explicitlyUnwrappedKey,
                             encryptionMetadata.ContentEncryptionIV);
+                        versionInternal = ClientSideEncryptionVersionInternal.V1_0;
                         break;
 #pragma warning restore CS0618 // obsolete
                     case ClientSideEncryptionVersion.V2_0:
                         expectedEncryptedMessage = EncryptDataV2_0(
                             message,
                             explicitlyUnwrappedKey);
+                        versionInternal = ClientSideEncryptionVersionInternal.V2_0;
                         break;
                     default: throw new ArgumentException("Test does not support clientside encryption version");
                 }
 
                 // compare data
-                Assert.AreEqual(version, parsedEncryptedMessage.EncryptionData.EncryptionAgent.EncryptionVersion);
+                Assert.AreEqual(versionInternal, parsedEncryptedMessage.EncryptionData.EncryptionAgent.EncryptionVersion);
                 Assert.AreEqual(expectedEncryptedMessage, parsedEncryptedMessage.EncryptedMessageText);
             }
         }