feat: add observability v2.5 to worker templates (#621)

* feat: add observability v2.5 to worker templates * pr-fix: update new app insights env var in docker integration tests yaml file * pr-fix: update with correct using * pr-fix: remove managed identity in docker tests * pr-fix: use failed condition in task logs * pr-fix: use failed condition in task logs * pr-fix: use configuration in release * pr-fix: use configuration in release * pr-fix: use configuration in release * pr-sug: remove 'your' in warning message Co-authored-by: Frederik Gheysels <[email protected]>
arcus-azure · Jul 13, 2022 · c54963c · c54963c
1 parent d26e2de
commit c54963c
Show file tree

Hide file tree

Showing 7 changed files with 105 additions and 74 deletions.
diff --git a/build/templates/docker-integration-tests.yml b/build/templates/docker-integration-tests.yml
@@ -45,13 +45,14 @@ stages:
               script: |
                 Get-Content './Arcus.Demo.ServiceBus.Queue/Program.cs' | 
                   Where-Object { $_ -notmatch '#error' -and $_ -notmatch 'secretProvider: null' -and $_ -notmatch 'EmptyMessageHandler' -and $_ -notmatch 'AddServiceBusQueueMessagePump' } | 
+                  Where-Object { $_ -notmatch 'AddAzureKeyVaultWithManagedIdentity' -and $_ -notmatch '#if DEBUG' -and $_ -notmatch '#endif' } |
                   Set-Content './Arcus.Demo.ServiceBus.Queue/Program.cs'
               envVars: |
                 ARCUS_HEALTH_PORT=$(Arcus.ServiceBus.Queue.Worker.HealthPort)
                 EVENTGRID_TOPIC_URI=$(Arcus.Worker.EventGrid.TopicUri)
                 EVENTGRID_AUTH_KEY=$(Arcus.Worker.EventGrid.AuthKey)
                 ARCUS_SERVICEBUS_CONNECTIONSTRING=$(Arcus.Worker.ServiceBus.Queue.ConnectionString)
-                TELEMETRY_APPLICATIONINSIGHTS_INSTRUMENTATIONKEY=$(Arcus.ApplicationInsights.InstrumentationKey)
+                APPLICATIONINSIGHTS_CONNECTION_STRING=InstrumentationKey=$(Arcus.ApplicationInsights.InstrumentationKey)
           - template: 'run-new-project-from-template.yml'
             parameters:
               projectName: 'Arcus.Demo.ServiceBus.Topic'
@@ -60,13 +61,14 @@ stages:
               script: |
                 Get-Content './Arcus.Demo.ServiceBus.Topic/Program.cs' | 
                   Where-Object { $_ -notmatch '#error' -and $_ -notmatch 'secretProvider: null' -and $_ -notmatch 'EmptyMessageHandler' -and $_ -notmatch 'AddServiceBusTopicMessagePump' } | 
+                  Where-Object { $_ -notmatch 'AddAzureKeyVaultWithManagedIdentity' -and $_ -notmatch '#if DEBUG' -and $_ -notmatch '#endif' } |
                   Set-Content './Arcus.Demo.ServiceBus.Topic/Program.cs'
               envVars: |
                 ARCUS_HEALTH_PORT=$(Arcus.ServiceBus.Topic.Worker.HealthPort)
-                 EVENTGRID_TOPIC_URI=$(Arcus.Worker.EventGrid.TopicUri)
+                EVENTGRID_TOPIC_URI=$(Arcus.Worker.EventGrid.TopicUri)
                 EVENTGRID_AUTH_KEY=$(Arcus.Worker.EventGrid.AuthKey)
                 ARCUS_SERVICEBUS_CONNECTIONSTRING=$(Arcus.Worker.ServiceBus.Topic.ConnectionString)
-                TELEMETRY_APPLICATIONINSIGHTS_INSTRUMENTATIONKEY=$(Arcus.ApplicationInsights.InstrumentationKey)
+                APPLICATIONINSIGHTS_CONNECTION_STRING=InstrumentationKey=$(Arcus.ApplicationInsights.InstrumentationKey)
           - template: 'run-new-project-from-template.yml'
             parameters:
               projectName: 'Arcus.Demo.AzureFunctions.Databricks.JobMetrics'
@@ -131,3 +133,15 @@ stages:
               targetType: 'inline'
               script: 'docker logs Arcus.Demo.WebApi'
             condition: failed()
+          - task: PowerShell@2
+            displayName: 'Get Docker container logs for Arcus.Demo.ServiceBus.Topic'
+            inputs:
+              targetType: 'inline'
+              script: 'docker logs Arcus.Demo.ServiceBus.Topic'
+            condition: failed()
+          - task: PowerShell@2
+            displayName: 'Get Docker container logs for Arcus.Demo.ServiceBus.Queue'
+            inputs:
+              targetType: 'inline'
+              script: 'docker logs Arcus.Demo.ServiceBus.Queue'
+            condition: failed()
diff --git a/src/Arcus.Templates.ServiceBus.Queue/Arcus.Templates.ServiceBus.Queue.csproj b/src/Arcus.Templates.ServiceBus.Queue/Arcus.Templates.ServiceBus.Queue.csproj
@@ -50,10 +50,10 @@
   <ItemGroup>
     <PackageReference Include="Arcus.Messaging.Health" Version="1.2.0" />
     <PackageReference Include="Arcus.Messaging.Pumps.ServiceBus" Version="1.2.0" />
-    <PackageReference Include="Arcus.Observability.Correlation" Version="2.4.0" />
-    <PackageReference Include="Arcus.Observability.Telemetry.Core" Version="2.4.0" />
-    <PackageReference Include="Arcus.Observability.Telemetry.Serilog.Filters" Version="2.4.0" Condition="'$(Serilog)' == 'true'" />
-    <PackageReference Include="Arcus.Observability.Telemetry.Serilog.Sinks.ApplicationInsights" Version="2.4.0" Condition="'$(Serilog)' == 'true'" />
+    <PackageReference Include="Arcus.Observability.Correlation" Version="2.5.0" />
+    <PackageReference Include="Arcus.Observability.Telemetry.Core" Version="2.5.0" />
+    <PackageReference Include="Arcus.Observability.Telemetry.Serilog.Filters" Version="2.5.0" Condition="'$(Serilog)' == 'true'" />
+    <PackageReference Include="Arcus.Observability.Telemetry.Serilog.Sinks.ApplicationInsights" Version="2.5.0" Condition="'$(Serilog)' == 'true'" />
     <PackageReference Include="Arcus.Security.Core" Version="1.7.0" />
     <PackageReference Include="Arcus.Security.Providers.AzureKeyVault" Version="1.7.0" />
     <PackageReference Include="Guard.NET" Version="2.0.0" />

diff --git a/src/Arcus.Templates.ServiceBus.Queue/Program.cs b/src/Arcus.Templates.ServiceBus.Queue/Program.cs
@@ -1,4 +1,6 @@
 using System;
+using System.Threading.Tasks;
+using Arcus.Security.Core;
 using Arcus.Security.Core.Caching.Configuration;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
@@ -7,31 +9,32 @@
 using Serilog;
 using Serilog.Configuration;
 using Serilog.Events;
+using Serilog.Extensions.Hosting;
 #endif
 
 namespace Arcus.Templates.ServiceBus.Queue
 {
     public class Program
     {
 #if Serilog
-        #warning Make sure that the appsettings.json is updated with your Azure Application Insights instrumentation key.
-        private const string ApplicationInsightsInstrumentationKeyName = "APPINSIGHTS_INSTRUMENTATIONKEY";
+        #warning Make sure that the Azure Application Insights connection string key is available as a secret.
+        private const string ApplicationInsightsConnectionStringKeyName = "APPLICATIONINSIGHTS_CONNECTION_STRING";
 
 #endif
-        public static int Main(string[] args)
+        public static async Task<int> Main(string[] args)
         {
 #if Serilog
             Log.Logger = new LoggerConfiguration()
                 .MinimumLevel.Debug()
                 .WriteTo.Console()
-                .CreateLogger();
+                .CreateBootstrapLogger();
 
             try
             {
-                CreateHostBuilder(args)
-                    .Build()
-                    .Run();
-
+                IHost host = CreateHostBuilder(args).Build();
+                await ConfigureSerilogAsync(host);
+                await host.RunAsync();
+                
                 return 0;
             }
             catch (Exception exception)
@@ -70,7 +73,7 @@ public static IHostBuilder CreateHostBuilder(string[] args)
                            stores.AddAzureKeyVaultWithManagedIdentity("https://your-keyvault.vault.azure.net/", CacheConfiguration.Default);
                        })
 #if Serilog
-                       .UseSerilog(UpdateLoggerConfiguration)
+                       .UseSerilog(Log.Logger)
 #endif
                        .ConfigureServices((hostContext, services) =>
                        {
@@ -82,22 +85,28 @@ public static IHostBuilder CreateHostBuilder(string[] args)
         }
 #if Serilog
 
-        private static void UpdateLoggerConfiguration(
-            HostBuilderContext hostContext,
-            LoggerConfiguration config)
+        private static async Task ConfigureSerilogAsync(IHost host)
         {
-            config.MinimumLevel.Debug()
-                  .MinimumLevel.Override("Microsoft", LogEventLevel.Information)
-                  .Enrich.FromLogContext()
-                  .Enrich.WithVersion()
-                  .Enrich.WithComponentName("Service Bus Queue Worker")
-                  .WriteTo.Console();
+            var secretProvider = host.Services.GetRequiredService<ISecretProvider>();
+            string connectionString = await secretProvider.GetRawSecretAsync(ApplicationInsightsConnectionStringKeyName);
 
-            var instrumentationKey = hostContext.Configuration.GetValue<string>(ApplicationInsightsInstrumentationKeyName);
-            if (!string.IsNullOrWhiteSpace(instrumentationKey))
+            var reloadLogger = (ReloadableLogger) Log.Logger;
+            reloadLogger.Reload(config =>
             {
-                config.WriteTo.AzureApplicationInsights(instrumentationKey);
-            }
+                config.MinimumLevel.Debug()
+                      .MinimumLevel.Override("Microsoft", LogEventLevel.Information)
+                      .Enrich.FromLogContext()
+                      .Enrich.WithVersion()
+                      .Enrich.WithComponentName("Service Bus Queue Worker")
+                      .WriteTo.Console();
+
+                if (!string.IsNullOrWhiteSpace(connectionString))
+                {
+                    config.WriteTo.AzureApplicationInsightsWithConnectionString(connectionString);
+                }
+
+                return config;
+            });
         }
 #endif
     }

diff --git a/src/Arcus.Templates.ServiceBus.Topic/Arcus.Templates.ServiceBus.Topic.csproj b/src/Arcus.Templates.ServiceBus.Topic/Arcus.Templates.ServiceBus.Topic.csproj
@@ -50,10 +50,10 @@
   <ItemGroup>
     <PackageReference Include="Arcus.Messaging.Health" Version="1.2.0" />
     <PackageReference Include="Arcus.Messaging.Pumps.ServiceBus" Version="1.2.0" />
-    <PackageReference Include="Arcus.Observability.Correlation" Version="2.4.0" />
-    <PackageReference Include="Arcus.Observability.Telemetry.Core" Version="2.4.0" />
-    <PackageReference Include="Arcus.Observability.Telemetry.Serilog.Filters" Version="2.4.0" Condition="'$(Serilog)' == 'true'" />
-    <PackageReference Include="Arcus.Observability.Telemetry.Serilog.Sinks.ApplicationInsights" Version="2.4.0" Condition="'$(Serilog)' == 'true'" />
+    <PackageReference Include="Arcus.Observability.Correlation" Version="2.5.0" />
+    <PackageReference Include="Arcus.Observability.Telemetry.Core" Version="2.5.0" />
+    <PackageReference Include="Arcus.Observability.Telemetry.Serilog.Filters" Version="2.5.0" Condition="'$(Serilog)' == 'true'" />
+    <PackageReference Include="Arcus.Observability.Telemetry.Serilog.Sinks.ApplicationInsights" Version="2.5.0" Condition="'$(Serilog)' == 'true'" />
     <PackageReference Include="Arcus.Security.Core" Version="1.7.0" />
     <PackageReference Include="Arcus.Security.Providers.AzureKeyVault" Version="1.7.0" />
     <PackageReference Include="Guard.NET" Version="2.0.0" />

diff --git a/src/Arcus.Templates.ServiceBus.Topic/Program.cs b/src/Arcus.Templates.ServiceBus.Topic/Program.cs
@@ -1,4 +1,6 @@
 using System;
+using System.Threading.Tasks;
+using Arcus.Security.Core;
 using Arcus.Security.Core.Caching.Configuration;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
@@ -7,31 +9,32 @@
 using Serilog;
 using Serilog.Configuration;
 using Serilog.Events;
+using Serilog.Extensions.Hosting;
 #endif
 
 namespace Arcus.Templates.ServiceBus.Topic
 {
     public class Program
     {
 #if Serilog
-        #warning Make sure that the appsettings.json is updated with your Azure Application Insights instrumentation key.
-        private const string ApplicationInsightsInstrumentationKeyName = "APPINSIGHTS_INSTRUMENTATIONKEY";
-
+        #warning Make sure that the Azure Application Insights connection string key is available as a secret.
+        private const string ApplicationInsightsConnectionStringKeyName = "APPLICATIONINSIGHTS_CONNECTION_STRING";
+        
 #endif
-        public static int Main(string[] args)
+        public static async Task<int> Main(string[] args)
         {
 #if Serilog
             Log.Logger = new LoggerConfiguration()
                 .MinimumLevel.Debug()
                 .WriteTo.Console()
-                .CreateLogger();
-
+                .CreateBootstrapLogger();
+            
             try
             {
-                CreateHostBuilder(args)
-                    .Build()
-                    .Run();
-
+                IHost host = CreateHostBuilder(args).Build();
+                await ConfigureSerilogAsync(host);
+                await host.RunAsync();
+                
                 return 0;
             }
             catch (Exception exception)
@@ -44,14 +47,13 @@ public static int Main(string[] args)
                 Log.CloseAndFlush();
             }
 #else
-            CreateHostBuilder(args)
-                .Build()
-                .Run();
-
+            IHost host = CreateHostBuilder(args).Build();
+            await host.RunAsync();
+
             return 0;
 #endif
         }
-
+        
         public static IHostBuilder CreateHostBuilder(string[] args)
         {
             return Host.CreateDefaultBuilder(args)
@@ -65,12 +67,12 @@ public static IHostBuilder CreateHostBuilder(string[] args)
 //[#if DEBUG]
                            stores.AddConfiguration(config);
 //[#endif]
-
+                            
                            //#error Please provide a valid secret provider, for example Azure Key Vault: https://security.arcus-azure.net/features/secret-store/provider/key-vault
                            stores.AddAzureKeyVaultWithManagedIdentity("https://your-keyvault.vault.azure.net/", CacheConfiguration.Default);
                        })
 #if Serilog
-                       .UseSerilog(UpdateLoggerConfiguration)
+                       .UseSerilog(Log.Logger)
 #endif
                        .ConfigureServices((hostContext, services) =>
                        {
@@ -82,22 +84,28 @@ public static IHostBuilder CreateHostBuilder(string[] args)
         }
 #if Serilog
 
-        private static void UpdateLoggerConfiguration(
-            HostBuilderContext hostContext,
-            LoggerConfiguration config)
+        private static async Task ConfigureSerilogAsync(IHost host)
         {
-            config.MinimumLevel.Debug()
-                  .MinimumLevel.Override("Microsoft", LogEventLevel.Information)
-                  .Enrich.FromLogContext()
-                  .Enrich.WithVersion()
-                  .Enrich.WithComponentName("Service Bus Topic Worker")
-                  .WriteTo.Console();
+            var secretProvider = host.Services.GetRequiredService<ISecretProvider>();
+            string connectionString = await secretProvider.GetRawSecretAsync(ApplicationInsightsConnectionStringKeyName);
 
-            var instrumentationKey = hostContext.Configuration.GetValue<string>(ApplicationInsightsInstrumentationKeyName);
-            if (!string.IsNullOrWhiteSpace(instrumentationKey))
+            var reloadLogger = (ReloadableLogger) Log.Logger;
+            reloadLogger.Reload(config =>
             {
-                config.WriteTo.AzureApplicationInsights(instrumentationKey);
-            }
+                config.MinimumLevel.Debug()
+                      .MinimumLevel.Override("Microsoft", LogEventLevel.Information)
+                      .Enrich.FromLogContext()
+                      .Enrich.WithVersion()
+                      .Enrich.WithComponentName("Service Bus Topic Worker")
+                      .WriteTo.Console();
+
+                if (!string.IsNullOrWhiteSpace(connectionString))
+                {
+                    config.WriteTo.AzureApplicationInsightsWithConnectionString(connectionString);
+                }
+
+                return config;
+            });
         }
 #endif
     }

diff --git a/src/Arcus.Templates.Tests.Integration/Worker/ServiceBusWorkerProject.cs b/src/Arcus.Templates.Tests.Integration/Worker/ServiceBusWorkerProject.cs
@@ -9,6 +9,7 @@
 using Arcus.Templates.Tests.Integration.Worker.Fixture;
 using Arcus.Templates.Tests.Integration.Worker.Health;
 using Arcus.Templates.Tests.Integration.Worker.MessagePump;
+using Azure.Messaging.ServiceBus;
 using GuardNet;
 using Polly;
 using Xunit.Abstractions;
@@ -165,16 +166,13 @@ private void AddOrdersMessagePump()
             AddTypeAsFile<OrderCreatedEvent>();
             AddTypeAsFile<OrderCreatedEventData>();
             AddTypeAsFile<OrdersMessageHandler>();
-            AddTypeAsFile<SingleValueSecretProvider>();
-
-            string connectionString = _configuration.GetServiceBusConnectionString(_entity);
+
             UpdateFileInProject("Program.cs", contents => 
                 RemovesUserErrorsFromContents(contents)
                     .Replace(".MinimumLevel.Debug()", ".MinimumLevel.Verbose()")
                     .Replace("EmptyMessageHandler", nameof(OrdersMessageHandler))
                     .Replace("EmptyMessage", nameof(Order))
-                    .Replace("AddAzureKeyVaultWithManagedIdentity(\"https://your-keyvault.vault.azure.net/\", CacheConfiguration.Default)", 
-                             $"AddProvider(new {nameof(SingleValueSecretProvider)}(\"{connectionString}\"))"));
+                    .Replace("stores.AddAzureKeyVaultWithManagedIdentity(\"https://your-keyvault.vault.azure.net/\", CacheConfiguration.Default);", ""));
         }
 
         private async Task StartAsync(ServiceBusWorkerProjectOptions options)

diff --git a/src/Arcus.Templates.Tests.Integration/Worker/ServiceBusWorkerProjectOptions.cs b/src/Arcus.Templates.Tests.Integration/Worker/ServiceBusWorkerProjectOptions.cs
@@ -11,7 +11,7 @@ namespace Arcus.Templates.Tests.Integration.Worker
     /// </summary>
     public class ServiceBusWorkerProjectOptions : ProjectOptions
     {
-        private const string SerilogTelemetryInstrumentationKey = "TELEMETRY_APPLICATIONINSIGHTS_INSTRUMENTATIONKEY";
+        private const string ApplicationInsightsConnectionStringKey = "APPLICATIONINSIGHTS_CONNECTION_STRING";
 
         /// <summary>
         /// Initializes a new instance of the <see cref="ServiceBusWorkerProjectOptions"/> class.
@@ -43,14 +43,16 @@ private ServiceBusWorkerProjectOptions(
         public IEnumerable<CommandArgument> AdditionalArguments { get; }
 
         /// <summary>
-        /// 
+        /// Creates an <see cref="ServiceBusWorkerProjectOptions"/> instance that provides additional user-configurable options for the Azure Service Bus .NET Worker projects.
         /// </summary>
-        /// <param name="configuration"></param>
-        /// <returns></returns>
+        /// <param name="configuration">The integration test configuration instance to retrieve connection secrets.</param>
+        /// <exception cref="ArgumentNullException">Thrown when the <paramref name="configuration"/> is <c>null</c>.</exception>
         public static ServiceBusWorkerProjectOptions Create(TestConfig configuration)
         {
+            Guard.NotNull(configuration, nameof(configuration), "Requires a test configuration instance to retrieve additional connection secrets");
+
             string instrumentationKey = configuration.GetApplicationInsightsInstrumentationKey();
-            var commandArgument = CommandArgument.CreateSecret(SerilogTelemetryInstrumentationKey, instrumentationKey);
+            var commandArgument = CommandArgument.CreateSecret(ApplicationInsightsConnectionStringKey, $"InstrumentationKey={instrumentationKey}");
 
             return new ServiceBusWorkerProjectOptions(new[] { commandArgument });
         }
@@ -61,7 +63,7 @@ public static ServiceBusWorkerProjectOptions Create(TestConfig configuration)
         public ServiceBusWorkerProjectOptions WithExcludeSerilog()
         {
             ProjectOptions optionsWithoutSerilog = AddOption("--exclude-serilog");
-            IEnumerable<CommandArgument> argumentsWithoutSerilog = AdditionalArguments.Where(arg => arg.Name != SerilogTelemetryInstrumentationKey);
+            IEnumerable<CommandArgument> argumentsWithoutSerilog = AdditionalArguments.Where(arg => arg.Name != ApplicationInsightsConnectionStringKey);
 
             return new ServiceBusWorkerProjectOptions(argumentsWithoutSerilog, optionsWithoutSerilog);
         }