From c166886ab47fd7afcd766ee44c14443203244c90 Mon Sep 17 00:00:00 2001 From: Gregorius Soedharmo Date: Fri, 17 Jul 2020 04:23:39 +0700 Subject: [PATCH 1/6] Add support for Azure Storage Emulator development environment --- .../Journal/AzureTableStorageJournal.cs | 5 ++++- .../AzureTableStorageJournalSettings.cs | 13 +++++++++++-- .../Snapshot/AzureBlobSnapshotStore.cs | 4 +++- .../AzureBlobSnapshotStoreSettings.cs | 19 ++++++++++++++++--- 4 files changed, 34 insertions(+), 7 deletions(-) diff --git a/src/Akka.Persistence.Azure/Journal/AzureTableStorageJournal.cs b/src/Akka.Persistence.Azure/Journal/AzureTableStorageJournal.cs index 2eec0fd..1c0d1f4 100644 --- a/src/Akka.Persistence.Azure/Journal/AzureTableStorageJournal.cs +++ b/src/Akka.Persistence.Azure/Journal/AzureTableStorageJournal.cs @@ -58,7 +58,10 @@ public AzureTableStorageJournal() { _settings = AzurePersistence.Get(Context.System).TableSettings; _serialization = new SerializationHelper(Context.System); - _storageAccount = CloudStorageAccount.Parse(_settings.ConnectionString); + _storageAccount = _settings.Development ? + CloudStorageAccount.DevelopmentStorageAccount : + CloudStorageAccount.Parse(_settings.ConnectionString); + _tableStorage = new Lazy(() => InitCloudStorage(5).Result); } diff --git a/src/Akka.Persistence.Azure/Journal/AzureTableStorageJournalSettings.cs b/src/Akka.Persistence.Azure/Journal/AzureTableStorageJournalSettings.cs index 67b47be..77ee225 100644 --- a/src/Akka.Persistence.Azure/Journal/AzureTableStorageJournalSettings.cs +++ b/src/Akka.Persistence.Azure/Journal/AzureTableStorageJournalSettings.cs @@ -24,8 +24,12 @@ public AzureTableStorageJournalSettings( string tableName, TimeSpan connectTimeout, TimeSpan requestTimeout, - bool verboseLogging) + bool verboseLogging, + bool development) { + if(string.IsNullOrWhiteSpace(tableName)) + throw new ConfigurationException("[AzureTableStorageJournal] Table name is null or empty."); + NameValidator.ValidateTableName(tableName); if (ReservedTableNames.Contains(tableName)) @@ -39,6 +43,7 @@ public AzureTableStorageJournalSettings( ConnectTimeout = connectTimeout; RequestTimeout = requestTimeout; VerboseLogging = verboseLogging; + Development = development; } /// @@ -66,6 +71,8 @@ public AzureTableStorageJournalSettings( /// public bool VerboseLogging { get; } + public bool Development { get; } + /// /// Creates an instance using the /// `akka.persistence.journal.azure-table` HOCON configuration section. @@ -79,13 +86,15 @@ public static AzureTableStorageJournalSettings Create(Config config) var connectTimeout = config.GetTimeSpan("connect-timeout", TimeSpan.FromSeconds(3)); var requestTimeout = config.GetTimeSpan("request-timeout", TimeSpan.FromSeconds(3)); var verbose = config.GetBoolean("verbose-logging", false); + var development = config.GetBoolean("development", false); return new AzureTableStorageJournalSettings( connectionString, tableName, connectTimeout, requestTimeout, - verbose); + verbose, + development); } } } \ No newline at end of file diff --git a/src/Akka.Persistence.Azure/Snapshot/AzureBlobSnapshotStore.cs b/src/Akka.Persistence.Azure/Snapshot/AzureBlobSnapshotStore.cs index c1fe15e..2edf9ac 100644 --- a/src/Akka.Persistence.Azure/Snapshot/AzureBlobSnapshotStore.cs +++ b/src/Akka.Persistence.Azure/Snapshot/AzureBlobSnapshotStore.cs @@ -36,8 +36,10 @@ public AzureBlobSnapshotStore() { _settings = AzurePersistence.Get(Context.System).BlobSettings; _serialization = new SerializationHelper(Context.System); - _storageAccount = CloudStorageAccount.Parse(_settings.ConnectionString); + _storageAccount = _settings.Development ? + CloudStorageAccount.DevelopmentStorageAccount : + CloudStorageAccount.Parse(_settings.ConnectionString); _container = new Lazy(() => InitCloudStorage().Result); } diff --git a/src/Akka.Persistence.Azure/Snapshot/AzureBlobSnapshotStoreSettings.cs b/src/Akka.Persistence.Azure/Snapshot/AzureBlobSnapshotStoreSettings.cs index 8cf2512..d40abd7 100644 --- a/src/Akka.Persistence.Azure/Snapshot/AzureBlobSnapshotStoreSettings.cs +++ b/src/Akka.Persistence.Azure/Snapshot/AzureBlobSnapshotStoreSettings.cs @@ -17,14 +17,18 @@ namespace Akka.Persistence.Azure.Snapshot public sealed class AzureBlobSnapshotStoreSettings { public AzureBlobSnapshotStoreSettings(string connectionString, string containerName, - TimeSpan connectTimeout, TimeSpan requestTimeout, bool verboseLogging) + TimeSpan connectTimeout, TimeSpan requestTimeout, bool verboseLogging, bool development) { + if (string.IsNullOrWhiteSpace(containerName)) + throw new ConfigurationException("[AzureBlobSnapshotStore] Container name is null or empty."); + NameValidator.ValidateContainerName(containerName); ConnectionString = connectionString; ContainerName = containerName; RequestTimeout = requestTimeout; ConnectTimeout = connectTimeout; VerboseLogging = verboseLogging; + Development = development; } /// @@ -52,6 +56,8 @@ public AzureBlobSnapshotStoreSettings(string connectionString, string containerN /// public bool VerboseLogging { get; } + public bool Development { get; } + /// /// Creates an instance using the /// `akka.persistence.snapshot-store.azure-blob-store` HOCON configuration section. @@ -65,8 +71,15 @@ public static AzureBlobSnapshotStoreSettings Create(Config config) var connectTimeout = config.GetTimeSpan("connect-timeout", TimeSpan.FromSeconds(3)); var requestTimeout = config.GetTimeSpan("request-timeout", TimeSpan.FromSeconds(3)); var verbose = config.GetBoolean("verbose-logging", false); - return new AzureBlobSnapshotStoreSettings(connectionString, containerName, connectTimeout, requestTimeout, - verbose); + var development = config.GetBoolean("development", false); + + return new AzureBlobSnapshotStoreSettings( + connectionString, + containerName, + connectTimeout, + requestTimeout, + verbose, + development); } } } \ No newline at end of file From 595429e8568d6ff9d0bfaf80feabed969269241d Mon Sep 17 00:00:00 2001 From: Gregorius Soedharmo Date: Fri, 17 Jul 2020 04:25:45 +0700 Subject: [PATCH 2/6] Add Config param to .ctor, so Akka.Persistence.Azure can play nicely with Akka.Cluster.Sharding --- .../Journal/AzureTableStorageJournal.cs | 8 ++++++-- .../Snapshot/AzureBlobSnapshotStore.cs | 7 +++++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/Akka.Persistence.Azure/Journal/AzureTableStorageJournal.cs b/src/Akka.Persistence.Azure/Journal/AzureTableStorageJournal.cs index 1c0d1f4..5c5aa22 100644 --- a/src/Akka.Persistence.Azure/Journal/AzureTableStorageJournal.cs +++ b/src/Akka.Persistence.Azure/Journal/AzureTableStorageJournal.cs @@ -19,6 +19,7 @@ using System.Linq; using System.Threading; using System.Threading.Tasks; +using Akka.Configuration; using Debug = System.Diagnostics.Debug; namespace Akka.Persistence.Azure.Journal @@ -54,9 +55,12 @@ public class AzureTableStorageJournal : AsyncWriteJournal private readonly Lazy _tableStorage; private readonly Dictionary> _tagSubscribers = new Dictionary>(); - public AzureTableStorageJournal() + public AzureTableStorageJournal(Config config = null) { - _settings = AzurePersistence.Get(Context.System).TableSettings; + _settings = config is null ? + AzurePersistence.Get(Context.System).TableSettings : + AzureTableStorageJournalSettings.Create(config); + _serialization = new SerializationHelper(Context.System); _storageAccount = _settings.Development ? CloudStorageAccount.DevelopmentStorageAccount : diff --git a/src/Akka.Persistence.Azure/Snapshot/AzureBlobSnapshotStore.cs b/src/Akka.Persistence.Azure/Snapshot/AzureBlobSnapshotStore.cs index 2edf9ac..6ad24fc 100644 --- a/src/Akka.Persistence.Azure/Snapshot/AzureBlobSnapshotStore.cs +++ b/src/Akka.Persistence.Azure/Snapshot/AzureBlobSnapshotStore.cs @@ -10,6 +10,7 @@ using System.Linq; using System.Threading; using System.Threading.Tasks; +using Akka.Configuration; using Akka.Event; using Akka.Persistence.Azure.Util; using Akka.Persistence.Snapshot; @@ -32,10 +33,12 @@ public class AzureBlobSnapshotStore : SnapshotStore private readonly AzureBlobSnapshotStoreSettings _settings; private readonly CloudStorageAccount _storageAccount; - public AzureBlobSnapshotStore() + public AzureBlobSnapshotStore(Config config = null) { - _settings = AzurePersistence.Get(Context.System).BlobSettings; _serialization = new SerializationHelper(Context.System); + _settings = config is null + ? AzurePersistence.Get(Context.System).BlobSettings + : AzureBlobSnapshotStoreSettings.Create(config); _storageAccount = _settings.Development ? CloudStorageAccount.DevelopmentStorageAccount : From 086703ae803b265ff1cfda8e26c0dd8440277bfa Mon Sep 17 00:00:00 2001 From: Gregorius Soedharmo Date: Fri, 17 Jul 2020 04:51:52 +0700 Subject: [PATCH 3/6] Remove code rot --- .../AzurePersistenceConfigSpec.cs | 19 --------- .../Query/AzureTableStorageQuerySettings.cs | 42 ------------------- 2 files changed, 61 deletions(-) delete mode 100644 src/Akka.Persistence.Azure/Query/AzureTableStorageQuerySettings.cs diff --git a/src/Akka.Persistence.Azure.Tests/AzurePersistenceConfigSpec.cs b/src/Akka.Persistence.Azure.Tests/AzurePersistenceConfigSpec.cs index cbbf699..4e6f553 100644 --- a/src/Akka.Persistence.Azure.Tests/AzurePersistenceConfigSpec.cs +++ b/src/Akka.Persistence.Azure.Tests/AzurePersistenceConfigSpec.cs @@ -61,25 +61,6 @@ public void ShouldParseTableConfig() tableSettings.VerboseLogging.Should().BeFalse(); } - [Fact] - public void ShouldParseQueryConfig() - { - var querySettings = - AzureTableStorageQuerySettings.Create( - ConfigurationFactory.ParseString(@"akka.persistence.query.journal.azure-table{ - class = ""classname"" - write-plugin = foo - max-buffer-size = 100 - refresh-interval = 3s - }").WithFallback(AzurePersistence.DefaultConfig) - .GetConfig("akka.persistence.query.journal.azure-table")); - - querySettings.Class.Should().Be("classname"); - querySettings.WritePlugin.Should().Be("foo"); - querySettings.MaxBufferSize.Should().Be("100"); - querySettings.RefreshInterval.Should().Be(new TimeSpan(0, 0, 3)); - } - [Theory] [InlineData("fo", "Invalid table name length")] [InlineData("1foo", "Invalid table name")] diff --git a/src/Akka.Persistence.Azure/Query/AzureTableStorageQuerySettings.cs b/src/Akka.Persistence.Azure/Query/AzureTableStorageQuerySettings.cs deleted file mode 100644 index 7543fd5..0000000 --- a/src/Akka.Persistence.Azure/Query/AzureTableStorageQuerySettings.cs +++ /dev/null @@ -1,42 +0,0 @@ -using Akka.Configuration; -using System; - -namespace Akka.Persistence.Azure.Query -{ - public sealed class AzureTableStorageQuerySettings - { - private AzureTableStorageQuerySettings( - string @class, - string writePlugin, - string maxBufferSize, - TimeSpan refreshInterval) - { - Class = @class; - MaxBufferSize = maxBufferSize; - RefreshInterval = refreshInterval; - WritePlugin = writePlugin; - } - - public string Class { get; } - - public string MaxBufferSize { get; } - - public TimeSpan RefreshInterval { get; } - - public string WritePlugin { get; } - - public static AzureTableStorageQuerySettings Create(Config config) - { - var @class = config.GetString("class"); - var writePlugin = config.GetString("write-plugin"); - var maxBufferSize = config.GetString("max-buffer-size"); - var refreshInterval = config.GetTimeSpan("refresh-interval"); - - return new AzureTableStorageQuerySettings( - @class, - writePlugin, - maxBufferSize, - refreshInterval); - } - } -} \ No newline at end of file From 4405b47dcec4f2c8ceba01dd017275051acd3d01 Mon Sep 17 00:00:00 2001 From: Gregorius Soedharmo Date: Fri, 17 Jul 2020 04:52:40 +0700 Subject: [PATCH 4/6] Add Azure Storage Emulator settings to default config --- src/Akka.Persistence.Azure/reference.conf | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/Akka.Persistence.Azure/reference.conf b/src/Akka.Persistence.Azure/reference.conf index 8ed299f..8cd57c6 100644 --- a/src/Akka.Persistence.Azure/reference.conf +++ b/src/Akka.Persistence.Azure/reference.conf @@ -27,6 +27,10 @@ akka.persistence { # dispatcher used to drive journal actor plugin-dispatcher = "akka.actor.default-dispatcher" + + # Support for Azure Storage Emulator for local development. + # Will ignore connection string settings if turned on. + development = off } } @@ -78,6 +82,10 @@ akka.persistence { # dispatcher used to drive snapshot storage actor plugin-dispatcher = "akka.actor.default-dispatcher" + + # Support for Azure Storage Emulator for local development. + # Will ignore connection string settings if turned on. + development = off } } } \ No newline at end of file From 59057738e2aeab541d53ba49f931427ff625c922 Mon Sep 17 00:00:00 2001 From: Gregorius Soedharmo Date: Fri, 17 Jul 2020 04:53:26 +0700 Subject: [PATCH 5/6] Make sure that the default query write-plugin does not point to the default akka local storage journal --- src/Akka.Persistence.Azure/reference.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Akka.Persistence.Azure/reference.conf b/src/Akka.Persistence.Azure/reference.conf index 8cd57c6..e3f09af 100644 --- a/src/Akka.Persistence.Azure/reference.conf +++ b/src/Akka.Persistence.Azure/reference.conf @@ -44,7 +44,7 @@ akka.persistence { # query journal will connect to. # If undefined (or "") it will connect to the default journal as specified by the # akka.persistence.journal.plugin property. - write-plugin = "" + write-plugin = "akka.persistence.journal.azure-table" # How many events to fetch in one query (replay) and keep buffered until they # are delivered downstreams. From 321f8c89d57a8155f61321a8b60bc9ea688bc1ce Mon Sep 17 00:00:00 2001 From: Gregorius Soedharmo Date: Fri, 17 Jul 2020 05:43:04 +0700 Subject: [PATCH 6/6] Update README.md with local development flag info --- README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/README.md b/README.md index 26825e5..d1df18a 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,16 @@ akka.persistence.snapshot-store.azure-blob-store.connection-string = "Your Azure akka.persistence.snapshot-store.azure-blob-store.container-name = "Your container name" ``` +## Using the plugin in local development environment + +You can use this plugin with Azure Storage Emulator in a local development environment by setting the development flag in the configuration file: +``` +akka.persistence.journal.azure-table.development = on +akka.persistence.snapshot-store.azure-blob-store.development = on +``` + +you do **not** need to provide a connection string for this to work, it is handled automatically by the Microsoft Azure SDK. + ## Building this solution To run the build script associated with this solution, execute the following: