Fix Bugs in new v12 Azure Storage (#868)

Also merges in the latest changes from the main branch.

Test/DurableTask.AzureStorage.Tests/MessageManagerTests.cs

@@ -0,0 +1,151 @@
+//  ----------------------------------------------------------------------------------
+//  Copyright Microsoft Corporation
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//  http://www.apache.org/licenses/LICENSE-2.0
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+//  ----------------------------------------------------------------------------------
+#nullable enable
+namespace DurableTask.AzureStorage.Tests
+{
+    using System;
+    using System.Collections.Generic;
+    using DurableTask.AzureStorage.Storage;
+    using DurableTask.Core.History;
+    using Microsoft.VisualStudio.TestTools.UnitTesting;
+    using Newtonsoft.Json;
+
+    [TestClass]
+    public class MessageManagerTests
+    {
+        [DataTestMethod]
+        [DataRow("System.Collections.Generic.Dictionary`2[[System.String, System.Private.CoreLib],[System.String, System.Private.CoreLib]]")]
+        [DataRow("System.Collections.Generic.Dictionary`2[[System.String, mscorlib],[System.String, mscorlib]]")]
+        public void DeserializesStandardTypes(string dictionaryType)
+        {
+            // Given
+            var message = GetMessage(dictionaryType);
+            var messageManager = SetupMessageManager(new PrimitiveTypeBinder());
+
+            // When
+            var deserializedMessage = messageManager.DeserializeMessageData(message);
+
+            // Then
+            Assert.IsInstanceOfType(deserializedMessage.TaskMessage.Event, typeof(ExecutionStartedEvent));
+            ExecutionStartedEvent startedEvent = (ExecutionStartedEvent)deserializedMessage.TaskMessage.Event;
+            Assert.AreEqual("tagValue", startedEvent.Tags["tag1"]);
+        }
+
+        [TestMethod]
+        public void FailsDeserializingUnknownTypes()
+        {
+            // Given
+            var message = GetMessage("RandomType");
+            var messageManager = SetupMessageManager(new KnownTypeBinder());
+
+            // When/Then
+            Assert.ThrowsException<JsonSerializationException>(() => messageManager.DeserializeMessageData(message));
+        }
+
+
+        [TestMethod]
+        public void DeserializesCustomTypes()
+        {
+            // Given
+            var message = GetMessage("KnownType");
+            var messageManager = SetupMessageManager(new KnownTypeBinder());
+
+            // When
+            var deserializedMessage = messageManager.DeserializeMessageData(message);
+
+            // Then
+            Assert.IsInstanceOfType(deserializedMessage.TaskMessage.Event, typeof(ExecutionStartedEvent));
+            ExecutionStartedEvent startedEvent = (ExecutionStartedEvent)deserializedMessage.TaskMessage.Event;
+            Assert.AreEqual("tagValue", startedEvent.Tags["tag1"]);
+        }
+
+        [DataTestMethod]
+        [DataRow("blob.bin")]
+        [DataRow("@#$%!")]
+        [DataRow("foo/bar/[email protected]")]
+        public void GetBlobUrlUnescaped(string blob)
+        {
+            var settings = new AzureStorageOrchestrationServiceSettings
+            {
+                StorageAccountClientProvider = new StorageAccountClientProvider("UseDevelopmentStorage=true"),
+            };
+
+            const string container = "@entity12345";
+            var manager = new MessageManager(settings, new AzureStorageClient(settings), container);
+            var expected = $"http://127.0.0.1:10000/devstoreaccount1/{container}/{blob}";
+            Assert.AreEqual(expected, manager.GetBlobUrl(blob));
+        }
+
+        private string GetMessage(string dictionaryType)
+            => "{\"$type\":\"DurableTask.AzureStorage.MessageData\",\"ActivityId\":\"5406d369-4369-4673-afae-6671a2fa1e57\",\"TaskMessage\":{\"$type\":\"DurableTask.Core.TaskMessage\",\"Event\":{\"$type\":\"DurableTask.Core.History.ExecutionStartedEvent\",\"OrchestrationInstance\":{\"$type\":\"DurableTask.Core.OrchestrationInstance\",\"InstanceId\":\"2.2-34a2c9d4-306e-4467-8470-a8018b2e4f11\",\"ExecutionId\":\"aae324dcc8f943e490b37ec5e5bbf9da\"},\"EventType\":0,\"ParentInstance\":null,\"Name\":\"OrchestrationName\",\"Version\":\"2.0\",\"Input\":\"input\",\"Tags\":{\"$type\":\""
+            + dictionaryType
+            + "\",\"tag1\":\"tagValue\"},\"Correlation\":null,\"ScheduledStartTime\":null,\"Generation\":0,\"EventId\":-1,\"IsPlayed\":false,\"Timestamp\":\"2023-03-24T20:53:05.9093518Z\"},\"SequenceNumber\":0,\"OrchestrationInstance\":{\"$type\":\"DurableTask.Core.OrchestrationInstance\",\"InstanceId\":\"2.2-34a2c9d4-306e-4467-8470-a8018b2e4f11\",\"ExecutionId\":\"aae324dcc8f943e490b37ec5e5bbf9da\"}},\"CompressedBlobName\":null,\"SequenceNumber\":40,\"Sender\":{\"InstanceId\":\"\",\"ExecutionId\":\"\"},\"SerializableTraceContext\":null}\r\n\r\n";
+
+        private MessageManager SetupMessageManager(ICustomTypeBinder binder)
+        {
+            var azureStorageClient = new AzureStorageClient(
+                new AzureStorageOrchestrationServiceSettings
+                {
+                    StorageAccountClientProvider = new StorageAccountClientProvider("UseDevelopmentStorage=true"),
+                });
+
+            return new MessageManager(
+                new AzureStorageOrchestrationServiceSettings { CustomMessageTypeBinder = binder },
+                azureStorageClient,
+                "$root");
+        }
+    }
+
+    internal class KnownTypeBinder : ICustomTypeBinder
+    {
+        public void BindToName(Type serializedType, out string assemblyName, out string typeName)
+        {
+            throw new NotImplementedException();
+        }
+
+        public Type? BindToType(string assemblyName, string typeName)
+        {
+            if (typeName == "KnownType")
+            {
+                return typeof(Dictionary<string, string>);
+            }
+
+            return null;
+        }
+    }
+
+    internal class PrimitiveTypeBinder : ICustomTypeBinder
+    {
+        readonly bool hasStandardLib;
+
+        public PrimitiveTypeBinder() 
+        {
+            hasStandardLib = typeof(string).AssemblyQualifiedName!.Contains("mscorlib");
+        }
+
+        public void BindToName(Type serializedType, out string assemblyName, out string typeName)
+        {
+            throw new NotImplementedException();
+        }
+
+        public Type BindToType(string assemblyName, string typeName)
+        {
+            if (hasStandardLib)
+            {
+                return Type.GetType(typeName.Replace("System.Private.CoreLib", "mscorlib"))!;
+            }
+
+            return Type.GetType(typeName.Replace("mscorlib", "System.Private.CoreLib"))!;
+        }
+    }
+}

Test/DurableTask.AzureStorage.Tests/Net/UriPathTests.cs

@@ -0,0 +1,38 @@
+//  ----------------------------------------------------------------------------------
+//  Copyright Microsoft Corporation
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//  http://www.apache.org/licenses/LICENSE-2.0
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+//  ----------------------------------------------------------------------------------
+#nullable enable
+namespace DurableTask.AzureStorage.Net
+{
+    using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+    [TestClass]
+    public class UriPathTests
+    {
+        [DataTestMethod]
+        [DataRow("", "", "")]
+        [DataRow("", "bar/baz", "bar/baz")]
+        [DataRow("foo", "", "foo")]
+        [DataRow("foo", "/", "foo/")]
+        [DataRow("foo", "bar", "foo/bar")]
+        [DataRow("foo", "/bar", "foo/bar")]
+        [DataRow("foo/", "", "foo/")]
+        [DataRow("foo/", "/", "foo/")]
+        [DataRow("foo/", "bar", "foo/bar")]
+        [DataRow("foo/", "/bar", "foo/bar")]
+        [DataRow("/foo//", "//bar/baz", "/foo///bar/baz")]
+        public void Combine(string path1, string path2, string expected)
+        {
+            Assert.AreEqual(expected, UriPath.Combine(path1, path2));
+        }
+    }
+}

Test/DurableTask.AzureStorage.Tests/Storage/TableClientExtensionsTests.cs

@@ -0,0 +1,52 @@
+//  ----------------------------------------------------------------------------------
+//  Copyright Microsoft Corporation
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//  http://www.apache.org/licenses/LICENSE-2.0
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+//  ----------------------------------------------------------------------------------
+
+namespace DurableTask.AzureStorage.Tests.Storage
+{
+    using System;
+    using Azure.Data.Tables;
+    using DurableTask.AzureStorage.Storage;
+    using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+    [TestClass]
+    public class TableClientExtensionsTests
+    {
+        const string EmulatorAccountKey = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==";
+
+        [TestMethod]
+        public void GetUri_ConnectionString()
+        {
+            TableClient client;
+
+            client = new TableClient("UseDevelopmentStorage=true", "bar");
+            Assert.AreEqual(new Uri("http://127.0.0.1:10002/devstoreaccount1/bar"), client.GetUri());
+
+            client = new TableClient($"DefaultEndpointsProtocol=https;AccountName=foo;AccountKey={EmulatorAccountKey};TableEndpoint=https://foo.table.core.windows.net/;", "bar");
+            Assert.AreEqual(new Uri("https://foo.table.core.windows.net/bar"), client.GetUri());
+        }
+
+        [TestMethod]
+        public void GetUri_ServiceEndpoint()
+        {
+            var client = new TableClient(new Uri("https://foo.table.core.windows.net/"), "bar", new TableSharedKeyCredential("foo", EmulatorAccountKey));
+            Assert.AreEqual(new Uri("https://foo.table.core.windows.net/bar"), client.GetUri());
+        }
+
+        [TestMethod]
+        public void GetUri_TableEndpoint()
+        {
+            var client = new TableClient(new Uri("https://foo.table.core.windows.net/bar"));
+            Assert.AreEqual(new Uri("https://foo.table.core.windows.net/bar"), client.GetUri());
+        }
+    }
+}

Test/DurableTask.Core.Tests/ExceptionHandlingIntegrationTests.cs

@@ -61,16 +61,16 @@ await this.worker
             if (mode == ErrorPropagationMode.SerializeExceptions)
             {
                 // The exception should be deserializable
-                InvalidOperationException e = JsonConvert.DeserializeObject<InvalidOperationException>(state.Output);
+                InvalidOperationException? e = JsonConvert.DeserializeObject<InvalidOperationException>(state.Output);
                 Assert.IsNotNull(e);
-                Assert.AreEqual("This is a test exception", e.Message);
+                Assert.AreEqual("This is a test exception", e!.Message);
             }
             else if (mode == ErrorPropagationMode.UseFailureDetails)
             {
                 // The failure details should contain the relevant exception metadata
-                FailureDetails details = JsonConvert.DeserializeObject<FailureDetails>(state.Output);
+                FailureDetails? details = JsonConvert.DeserializeObject<FailureDetails>(state.Output);
                 Assert.IsNotNull(details);
-                Assert.AreEqual(typeof(InvalidOperationException).FullName, details.ErrorType);
+                Assert.AreEqual(typeof(InvalidOperationException).FullName, details!.ErrorType);
                 Assert.IsTrue(details.IsCausedBy<InvalidOperationException>());
                 Assert.IsTrue(details.IsCausedBy<Exception>()); // check that base types work too
                 Assert.AreEqual("This is a test exception", details.ErrorMessage);

azure-pipelines-release.yml

@@ -138,16 +138,16 @@ steps:
     packDirectory: $(build.artifactStagingDirectory)
     packagesToPack: 'src/DurableTask.ServiceBus/DurableTask.ServiceBus.csproj'
 
-# - task: DotNetCoreCLI@2
-#   displayName: Generate nuget packages
-#   inputs:
-#     command: pack
-#     verbosityPack: Minimal
-#     configuration: Release
-#     nobuild: true
-#     packDirectory: $(build.artifactStagingDirectory)
-#     packagesToPack: 'src/DurableTask.AzureServiceFabric/DurableTask.AzureServiceFabric.csproj'
-#     buildProperties: 'Platform=x64'
+- task: DotNetCoreCLI@2
+  displayName: Generate nuget packages
+  inputs:
+    command: pack
+    verbosityPack: Minimal
+    configuration: Release
+    nobuild: true
+    packDirectory: $(build.artifactStagingDirectory)
+    packagesToPack: 'src/DurableTask.AzureServiceFabric/DurableTask.AzureServiceFabric.csproj'
+    buildProperties: 'Platform=x64'
 
 # Digitally sign all the nuget packages with the Microsoft certificate.
 # This appears to be an in-place signing job, which is convenient.

src/DurableTask.AzureServiceFabric/DurableTask.AzureServiceFabric.csproj

@@ -6,7 +6,7 @@
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
     <PackageId>Microsoft.Azure.DurableTask.AzureServiceFabric</PackageId>
     <GeneratePackageOnBuild>true</GeneratePackageOnBuild>
-    <Version>2.3.6</Version>
+    <Version>2.3.7</Version>
     <AssemblyVersion>$(Version)</AssemblyVersion>
     <FileVersion>$(Version)</FileVersion>
     <Title>Azure Service Fabric provider extension for the Durable Task Framework.</Title>

src/DurableTask.AzureStorage/AzureStorageOrchestrationServiceSettings.cs

@@ -150,7 +150,7 @@ public class AzureStorageOrchestrationServiceSettings
         public bool UseAppLease { get; set; } = true;
 
         /// <summary>
-        /// If UseAppLease is true, gets or sets the AppLeaaseOptions used for acquiring the lease to start the application.
+        /// If UseAppLease is true, gets or sets the AppLeaseOptions used for acquiring the lease to start the application.
         /// </summary>
         public AppLeaseOptions AppLeaseOptions { get; set; } = AppLeaseOptions.DefaultOptions;
 
@@ -160,13 +160,12 @@ public class AzureStorageOrchestrationServiceSettings
         public StorageAccountClientProvider? StorageAccountClientProvider { get; set; }
 
         /// <summary>
-        /// Gets or sets the storage provider for Azure Table Storage, which is used for the Tracking Store
-        /// that records the progress of orchestrations and entities.
+        /// Gets or sets the optional client provider for the Tracking Store that records the progress of orchestrations and entities.
         /// </summary>
         /// <remarks>
         /// If the value is <see langword="null"/>, then the <see cref="StorageAccountClientProvider"/> is used instead.
         /// </remarks>
-        public IStorageServiceClientProvider<TableServiceClient, TableClientOptions>? TrackingServiceClientProvider { get; set; }
+        public TrackingServiceClientProvider? TrackingServiceClientProvider { get; set; }
 
         /// <summary>
         ///  Should we carry over unexecuted raised events to the next iteration of an orchestration on ContinueAsNew
@@ -199,6 +198,11 @@ public class AzureStorageOrchestrationServiceSettings
         /// </summary>
         public bool DisableExecutionStartedDeduplication { get; set; }
 
+        /// <summary>
+        /// Gets or sets an optional custom type binder used when trying to deserialize queued messages.
+        /// </summary>
+        public ICustomTypeBinder? CustomMessageTypeBinder { get; set; }
+
         /// <summary>
         /// Returns bool indicating is the <see cref="TrackingServiceClientProvider"/> has been set.
         /// </summary>

src/DurableTask.AzureStorage/DurableTask.AzureStorage.csproj

@@ -8,11 +8,11 @@
     <Description>Azure Storage provider extension for the Durable Task Framework.</Description>
     <PackageTags>Azure Task Durable Orchestration Workflow Activity Reliable AzureStorage</PackageTags>
     <PackageId>Microsoft.Azure.DurableTask.AzureStorage</PackageId>
-    <PublishRepositoryUrl>true</PublishRepositoryUrl> 
-    <EmbedUntrackedSources>true</EmbedUntrackedSources> 
-    <DebugSymbols>true</DebugSymbols> 
-    <DebugType>embedded</DebugType> 
-    <IncludeSymbols>false</IncludeSymbols> 
+    <PublishRepositoryUrl>true</PublishRepositoryUrl>
+    <EmbedUntrackedSources>true</EmbedUntrackedSources>
+    <DebugSymbols>true</DebugSymbols>
+    <DebugType>embedded</DebugType>
+    <IncludeSymbols>false</IncludeSymbols>
     <!--NuGet licenseUrl and PackageIconUrl/iconUrl deprecation. -->
     <NoWarn>NU5125;NU5048</NoWarn>
   </PropertyGroup>
@@ -24,7 +24,7 @@
     <PatchVersion>0</PatchVersion>
 
     <VersionPrefix>$(MajorVersion).$(MinorVersion).$(PatchVersion)</VersionPrefix>
-    <VersionSuffix>preview.1</VersionSuffix>
+    <VersionSuffix>preview.2</VersionSuffix>
     <FileVersion>$(VersionPrefix).0</FileVersion>
     <!-- FileVersionRevision is expected to be set by the CI. This is useful for distinguishing between multiple builds of the same version. -->
     <FileVersion Condition="'$(FileVersionRevision)' != ''">$(VersionPrefix).$(FileVersionRevision)</FileVersion>
@@ -34,11 +34,14 @@
     <Version>$(VersionPrefix)-$(VersionSuffix)</Version>
   </PropertyGroup>
 
+  <!-- The below versions must align with the Functions Host -->
   <ItemGroup>
-    <PackageReference Include="Azure.Core" Version="1.27.0" />
-    <PackageReference Include="Azure.Data.Tables" Version="12.7.1" />
-    <!-- The below version of Azure.Storage.Blobs must align with the Functions Host -->
+    <PackageReference Include="Azure.Core" Version="1.25.0" />
     <PackageReference Include="Azure.Storage.Blobs" Version="12.13.0" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Azure.Data.Tables" Version="12.6.1" />
     <PackageReference Include="Azure.Storage.Queues" Version="12.12.0" />
     <PackageReference Include="Microsoft.Bcl.AsyncInterfaces" Version="6.0.0" />
     <PackageReference Include="System.Linq.Async" Version="6.0.1" />
@@ -51,7 +54,6 @@
 
   <ItemGroup Condition="'$(TargetFramework)' == 'netstandard2.0'">
     <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="2.2.0" />
-    <PackageReference Include="Newtonsoft.Json" Version="11.0.2" />
   </ItemGroup>
 
   <ItemGroup>