Merge pull request #5189 from akkadotnet/dev

v1.4.23 Release

docs/articles/clustering/cluster-sharding.md

@@ -119,6 +119,59 @@ public sealed class MessageExtractor : HashCodeMessageExtractor
 
 Using `ShardRegion.StartEntity` implies, that you're able to infer a shard id given an entity id alone. For this reason, in example above we modified a cluster sharding routing logic to make use of `HashCodeMessageExtractor` - in this variant, shard id doesn't have to be provided explicitly, as it will be computed from the hash of entity id itself. Notice a `maxNumberOfShards`, which is the maximum available number of shards allowed for this type of an actor - this value must never change during a single lifetime of a cluster. 
 
+### Remember Entities Store
+
+There are two options for the remember entities store:
+
+1. Distributed data
+2. Persistence
+
+#### Remember Entities Persistence Mode
+You can enable persistence mode (enabled by default) with:
+
+```
+akka.cluster.sharding.state-store-mode = persistence
+```
+
+This mode uses [persistence](../persistence/event-sourcing.md) to store the active shards and active entities for each shard.
+By default, cluster sharding will use the journal and snapshot store plugin defined in `akka.persistence.journal.plugin` and 
+`akka.persistence.snapshot-store.plugin` respectively; to change this behaviour, you can use these configuration:
+
+```
+akka.cluster.sharding.journal-plugin-id = <plugin>
+akka.cluster.sharding.snapshot-plugin-id = <plugin>
+```
+
+#### Remember Entities Distributed Data Mode
+
+You can enable DData mode by setting these configuration:
+
+```
+akka.cluster.sharding.state-store-mode = ddata
+```
+
+To support restarting entities after a full cluster restart (non-rolling) the remember entities store 
+is persisted to disk by distributed data. This can be disabled if not needed:
+
+```
+akka.cluster.sharding.distributed-data.durable.keys = []
+```
+
+Possible reasons for disabling remember entity storage are:
+
+- No requirement for remembering entities after a full cluster shutdown
+- Running in an environment without access to disk between restarts e.g. Kubernetes without persistent volumes
+
+For supporting remembered entities in an environment without disk storage but with access to a database, use persistence mode instead.
+
+> [!NOTE]
+> Currently, Lightning.NET library, the storage solution used to store DData in disk, is having problem
+> deploying native library files in [Linux operating system operating in x64 and ARM platforms]
+> (https://github.com/CoreyKaylor/Lightning.NET/issues/141).
+> 
+> You will need to install LightningDB in your Linux distribution manually if you wanted to use the durable DData feature.
+
+
 ### Terminating Remembered Entities
 One complication that  `akka.cluster.sharding.remember-entities = true` introduces is that your sharded entity actors can no longer be terminated through the normal Akka.NET channels, i.e. `Context.Stop(Self)`, `PoisonPill.Instance`, and the like. This is because as part of the `remember-entities` contract - the sharding system is going to insist on keeping all remembered entities alive until explictily told to stop.
 

src/NuGet.Config

@@ -7,4 +7,4 @@
     <clear />
     <add key="nuget.org" value="https://api.nuget.org/v3/index.json" />
   </packageSources>
-</configuration>
+</configuration>

src/common.props

@@ -2,7 +2,7 @@
   <PropertyGroup>
     <Copyright>Copyright © 2013-2021 Akka.NET Team</Copyright>
     <Authors>Akka.NET Team</Authors>
-    <VersionPrefix>1.4.19</VersionPrefix>
+    <VersionPrefix>1.4.23</VersionPrefix>
     <PackageIconUrl>https://getakka.net/images/akkalogo.png</PackageIconUrl>
     <PackageProjectUrl>https://github.com/akkadotnet/akka.net</PackageProjectUrl>
     <PackageLicenseUrl>https://github.com/akkadotnet/akka.net/blob/master/LICENSE</PackageLicenseUrl>

src/contrib/cluster/Akka.Cluster.Sharding.Tests.MultiNode/Akka.Cluster.Sharding.Tests.MultiNode.csproj

@@ -1,5 +1,8 @@
 <Project Sdk="Microsoft.NET.Sdk">
-  <Import Project="..\..\..\common.props" />
+    <Import Project="..\..\..\common.props" />
+
+    <!-- Needed to resolve https://github.com/akkadotnet/akka.net/pull/5180 and https://github.com/akkadotnet/akka.net/issues/5174 -->
+    <Import Project="../lib-hack/LightningDB.targets" />
 
   <PropertyGroup>
     <AssemblyTitle>Akka.Cluster.Sharding.Tests.MultiNode</AssemblyTitle>
@@ -20,12 +23,11 @@
     <PackageReference Include="xunit.runner.visualstudio" Version="$(XunitVersion)" />
     <PackageReference Include="FluentAssertions" Version="$(FluentAssertionsVersion)" />
   </ItemGroup>
-
-  <ItemGroup>
-    <Service Include="{82a7f48d-3b50-4b1e-b82e-3ada8210c358}" />
-  </ItemGroup>
-
+
   <PropertyGroup Condition=" '$(Configuration)' == 'Release' ">
     <DefineConstants>$(DefineConstants);RELEASE</DefineConstants>
   </PropertyGroup>
+
+
+
 </Project>

src/contrib/cluster/Akka.Cluster.Sharding.Tests.MultiNode/Properties/AssemblyInfo.cs

@@ -19,15 +19,4 @@
 [assembly: ComVisible(false)]
 
 // The following GUID is for the ID of the typelib if this project is exposed to COM
-[assembly: Guid("666698ab-85bc-4c91-894c-cbb4316dc1a4")]
-
-// Version information for an assembly consists of the following four values:
-//
-//      Major Version
-//      Minor Version 
-//      Build Number
-//      Revision
-//
-// You can specify all the values or you can default the Build and Revision Numbers 
-// by using the '*' as shown below:
-// [assembly: AssemblyVersion("1.0.*")]
+[assembly: Guid("666698ab-85bc-4c91-894c-cbb4316dc1a4")]

src/contrib/cluster/Akka.DistributedData.LightningDB/Akka.DistributedData.LightningDB.csproj

@@ -15,7 +15,7 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="LightningDB" Version="0.13.0" />
+    <PackageReference Include="LightningDB.Vendored.Akka" Version="0.14.0" />
   </ItemGroup>
 
   <PropertyGroup Condition=" '$(Configuration)' == 'Release' ">

src/contrib/cluster/Akka.DistributedData.Tests.MultiNode/Akka.DistributedData.Tests.MultiNode.csproj

@@ -1,5 +1,8 @@
 <Project Sdk="Microsoft.NET.Sdk">
-  <Import Project="..\..\..\common.props" />
+    <Import Project="..\..\..\common.props" />
+
+    <!-- Needed to resolve https://github.com/akkadotnet/akka.net/pull/5180 and https://github.com/akkadotnet/akka.net/issues/5174 -->
+    <Import Project="../lib-hack/LightningDB.targets" />
 
   <PropertyGroup>
     <AssemblyTitle>Akka.DistributedData.Tests.MultiNode</AssemblyTitle>
@@ -20,10 +23,6 @@
     <PackageReference Include="FluentAssertions" Version="$(FluentAssertionsVersion)" />
   </ItemGroup>
 
-  <ItemGroup>
-    <Service Include="{82a7f48d-3b50-4b1e-b82e-3ada8210c358}" />
-  </ItemGroup>
-
   <PropertyGroup Condition=" '$(Configuration)' == 'Release' ">
     <DefineConstants>$(DefineConstants);RELEASE</DefineConstants>
   </PropertyGroup>

src/contrib/cluster/lib-hack/LightningDB.targets

@@ -0,0 +1,59 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <PropertyGroup>
+    <LightningDBTargetRuntimeRelativePath Condition=" '$(LightningDBTargetRuntimeRelativePath)' == '' ">./</LightningDBTargetRuntimeRelativePath>
+  </PropertyGroup>
+  <ItemGroup Condition="'$([MSBuild]::IsOsPlatform(Windows))' And '$([System.Runtime.InteropServices.RuntimeInformation]::OSArchitecture)'=='X64'">
+    <None Include="$(MSBuildThisFileDirectory)$(LightningDBTargetRuntimeRelativePath)runtimes\win-x64\native\lmdb.dll">
+      <Visible>false</Visible>
+      <Link>lmdb.dll</Link>
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+    <None Include="$(MSBuildThisFileDirectory)$(LightningDBTargetRuntimeRelativePath)runtimes\win-x64\native\lmdbautoresize.dll">
+      <Visible>false</Visible>
+      <Link>lmdbautoresize.dll</Link>
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+  </ItemGroup>
+  <ItemGroup Condition="'$([MSBuild]::IsOsPlatform(Windows))' And '$([System.Runtime.InteropServices.RuntimeInformation]::OSArchitecture)'=='X86'">
+    <None Include="$(MSBuildThisFileDirectory)$(LightningDBTargetRuntimeRelativePath)runtimes\win-x86\native\lmdb.dll">
+      <Visible>false</Visible>
+      <Link>lmdb.dll</Link>
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+    <None Include="$(MSBuildThisFileDirectory)$(LightningDBTargetRuntimeRelativePath)runtimes\win-x86\native\lmdbautoresize.dll">
+      <Visible>false</Visible>
+      <Link>lmdbautoresize.dll</Link>
+
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+  </ItemGroup>
+  <ItemGroup Condition=" '$([MSBuild]::IsOsPlatform(OSX))' ">
+    <None Include="$(MSBuildThisFileDirectory)$(LightningDBTargetRuntimeRelativePath)runtimes\osx\native\lmdb.dylib">
+      <Visible>false</Visible>
+      <Link>lmdb.dylib</Link>
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+  </ItemGroup>
+  <ItemGroup Condition="'$([MSBuild]::IsOsPlatform(Linux))' And '$([System.Runtime.InteropServices.RuntimeInformation]::OSArchitecture)'=='Arm'">
+    <None Include="$(MSBuildThisFileDirectory)$(LightningDBTargetRuntimeRelativePath)runtimes\linux-arm\native\liblmdb.so">
+      <Visible>false</Visible>
+      <Link>liblmdb.so</Link>
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+  </ItemGroup>
+  <ItemGroup Condition="'$([MSBuild]::IsOsPlatform(Linux))' And '$([System.Runtime.InteropServices.RuntimeInformation]::OSArchitecture)'=='Arm64'">
+    <None Include="$(MSBuildThisFileDirectory)$(LightningDBTargetRuntimeRelativePath)runtimes\linux-arm64\native\liblmdb.so">
+      <Visible>false</Visible>
+      <Link>liblmdb.so</Link>
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+  </ItemGroup>
+  <ItemGroup Condition="'$([MSBuild]::IsOsPlatform(Linux))' And '$([System.Runtime.InteropServices.RuntimeInformation]::OSArchitecture)'=='X64'">
+    <None Include="$(MSBuildThisFileDirectory)$(LightningDBTargetRuntimeRelativePath)runtimes\linux-x64\native\liblmdb.so">
+      <Visible>false</Visible>
+      <Link>liblmdb.so</Link>
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+  </ItemGroup>
+</Project>

src/core/Akka.API.Tests/CoreAPISpec.ApproveRemote.approved.txt

@@ -204,7 +204,7 @@ namespace Akka.Remote
         public Akka.Actor.IInternalActorRef ActorOf(Akka.Actor.Internal.ActorSystemImpl system, Akka.Actor.Props props, Akka.Actor.IInternalActorRef supervisor, Akka.Actor.ActorPath path, bool systemService, Akka.Actor.Deploy deploy, bool lookupDeploy, bool async) { }
         protected virtual Akka.Actor.IActorRef CreateRemoteDeploymentWatcher(Akka.Actor.Internal.ActorSystemImpl system) { }
         protected virtual Akka.Actor.IInternalActorRef CreateRemoteRef(Akka.Actor.ActorPath actorPath, Akka.Actor.Address localAddress) { }
-        protected virtual Akka.Remote.RemoteActorRef CreateRemoteRef(Akka.Actor.Props props, Akka.Actor.IInternalActorRef supervisor, Akka.Actor.Address localAddress, Akka.Actor.ActorPath rpath, Akka.Actor.Deploy deployment) { }
+        protected virtual Akka.Actor.IInternalActorRef CreateRemoteRef(Akka.Actor.Props props, Akka.Actor.IInternalActorRef supervisor, Akka.Actor.Address localAddress, Akka.Actor.ActorPath rpath, Akka.Actor.Deploy deployment) { }
         protected virtual Akka.Actor.IActorRef CreateRemoteWatcher(Akka.Actor.Internal.ActorSystemImpl system) { }
         protected Akka.Remote.DefaultFailureDetectorRegistry<Akka.Actor.Address> CreateRemoteWatcherFailureDetector(Akka.Actor.ActorSystem system) { }
         public Akka.Actor.Address GetExternalAddressFor(Akka.Actor.Address address) { }

src/core/Akka.MultiNodeTestRunner/Akka.MultiNodeTestRunner.csproj

@@ -9,6 +9,7 @@
     <RuntimeIdentifiers>win10-x64</RuntimeIdentifiers>
     <PackageTags>$(AkkaPackageTags)</PackageTags>
     <NuspecFile>Akka.MultiNodeTestRunner.nuspec</NuspecFile>
+    <ValidateExecutableReferencesMatchSelfContained>false</ValidateExecutableReferencesMatchSelfContained>
   </PropertyGroup>
 
   <ItemGroup>

src/core/Akka.Remote/RemoteActorRefProvider.cs

@@ -512,15 +512,15 @@ protected virtual IInternalActorRef CreateRemoteRef(ActorPath actorPath, Address
         }
 
         /// <summary>
-        /// Used to create <see cref="RemoteActoRef"/> instances upon remote deployment to another <see cref="ActorSystem"/>.
+        /// Used to create <see cref="RemoteActorRef"/> instances upon remote deployment to another <see cref="ActorSystem"/>.
         /// </summary>
         /// <param name="props">Props of the remotely deployed actor.</param>
         /// <param name="supervisor">A reference to the local parent actor responsible for supervising the remotely deployed one.</param>
         /// <param name="localAddress">The local address of this actor.</param>
         /// <param name="rpath">The remote actor path.</param>
         /// <param name="deployment">The deployment included in this Props.</param>
         /// <returns>An <see cref="IInternalActorRef"/> instance.</returns>
-        protected virtual RemoteActorRef CreateRemoteRef(Props props, IInternalActorRef supervisor, Address localAddress, ActorPath rpath, Deploy deployment)
+        protected virtual IInternalActorRef CreateRemoteRef(Props props, IInternalActorRef supervisor, Address localAddress, ActorPath rpath, Deploy deployment)
         {
             return new RemoteActorRef(Transport, localAddress, rpath, supervisor, props, deployment);
         }
@@ -552,7 +552,7 @@ public IActorRef ResolveActorRef(string path)
         /// <returns>An <see cref="IActorRef"/> if a match was found. Otherwise nobody.</returns>
         public IActorRef InternalResolveActorRef(string path)
         {
-            if (path == String.Empty)
+            if (path == string.Empty)
                 return ActorRefs.NoSender;
 
             if (ActorPath.TryParse(path, out var actorPath))
@@ -586,10 +586,10 @@ public IActorRef ResolveActorRef(ActorPath actorPath)
         }
 
         /// <summary>
-        /// TBD
+        /// Used to translate an incoming address into an external one
         /// </summary>
-        /// <param name="address">TBD</param>
-        /// <returns>TBD</returns>
+        /// <param name="address">The incoming address</param>
+        /// <returns>The remote Address, if applicable. If not applicable <c>null</c> may be returned.</returns>
         public Address GetExternalAddressFor(Address address)
         {
             if (HasAddress(address)) { return _local.RootPath.Address; }
@@ -641,7 +641,7 @@ public void Quarantine(Address address, int? uid)
         /// All of the private internals used by <see cref="RemoteActorRefProvider"/>, namely its transport
         /// registry, remote serializers, and the <see cref="RemoteDaemon"/> instance.
         /// </summary>
-        class Internals : INoSerializationVerificationNeeded
+        private class Internals : INoSerializationVerificationNeeded
         {
             /// <summary>
             /// TBD
@@ -679,7 +679,7 @@ public Internals(RemoteTransport transport, Akka.Serialization.Serialization ser
         /// <summary>
         /// Describes the FSM states of the <see cref="RemotingTerminator"/>
         /// </summary>
-        enum TerminatorState
+        private enum TerminatorState
         {
             /// <summary>
             /// TBD
@@ -773,14 +773,7 @@ private void InitFSM()
             public sealed class TransportShutdown
             {
                 private TransportShutdown() { }
-                private static readonly TransportShutdown _instance = new TransportShutdown();
-                public static TransportShutdown Instance
-                {
-                    get
-                    {
-                        return _instance;
-                    }
-                }
+                public static TransportShutdown Instance { get; } = new TransportShutdown();
 
                 public override string ToString()
                 {
@@ -800,18 +793,16 @@ public RemoteDeadLetterActorRef(IActorRefProvider provider, ActorPath actorPath,
 
             protected override void TellInternal(object message, IActorRef sender)
             {
-                var send = message as EndpointManager.Send;
                 var deadLetter = message as DeadLetter;
-                if (send != null)
+                if (message is EndpointManager.Send send)
                 {
                     if (send.Seq == null)
                     {
                         base.TellInternal(send.Message, send.SenderOption ?? ActorRefs.NoSender);
                     }
                 }
-                else if (deadLetter?.Message is EndpointManager.Send)
+                else if (deadLetter?.Message is EndpointManager.Send deadSend)
                 {
-                    var deadSend = (EndpointManager.Send)deadLetter.Message;
                     if (deadSend.Seq == null)
                     {
                         base.TellInternal(deadSend.Message, deadSend.SenderOption ?? ActorRefs.NoSender);

src/core/Akka/Actor/ActorRef.cs

@@ -134,6 +134,10 @@ protected override void TellInternal(object message, IActorRef sender)
                     {
                         _result.TrySetResult(t);
                     }
+                    else if (message is Failure f)
+                    {
+                        _result.TrySetException(f.Exception ?? new TaskCanceledException("Task cancelled by actor via Failure message."));
+                    }
                     else
                     {
                         _result.TrySetException(new ArgumentException(
@@ -142,7 +146,7 @@ protected override void TellInternal(object message, IActorRef sender)
                 }
             }
         }
-
+        
         /// <summary>
         /// TBD
         /// </summary>

src/examples/Cluster/ClusterSharding/ClusterSharding.Node/Customers.cs

@@ -8,6 +8,7 @@
 using System;
 using System.Collections.Generic;
 using System.Text;
+using Akka.Cluster.Sharding;
 using Akka.Persistence;
 
 namespace ClusterSharding.Node
@@ -38,15 +39,22 @@ public ItemPurchased(string itemName)
 
         #endregion
 
-        public override string PersistenceId { get; } = Context.Parent.Path.Name + "/" + Context.Self.Path.Name;
+        public override string PersistenceId { get; }
 
         public ICollection<string> _purchasedItems = new List<string>();
 
-        public Customer()
+        public Customer(string persistenceId)
         {
+            PersistenceId = persistenceId;
             SetReceiveTimeout(TimeSpan.FromSeconds(60));
             Recover<ItemPurchased>(purchased => _purchasedItems.Add(purchased.ItemName));
 
+            Command<ShardRegion.StartEntity>(e =>
+            {
+                Console.WriteLine($"'{PersistenceId}' started via remember-entities");
+
+            });
+
             Command<PurchaseItem>(purchase =>
             {
                 Persist(new ItemPurchased(purchase.ItemName), purchased =>