Add heartbeat support to ng.exe and other jobs in this repository (#667)

NuGet · Oct 8, 2019 · 95c5043 · 95c5043
1 parent 3f8258d
commit 95c5043
Show file tree

Hide file tree

Showing 20 changed files with 259 additions and 183 deletions.
diff --git a/src/Catalog/NuGet.Services.Metadata.Catalog.csproj b/src/Catalog/NuGet.Services.Metadata.Catalog.csproj
@@ -291,7 +291,7 @@
       <Version>4.4.5-dev-2939401</Version>
     </PackageReference>
     <PackageReference Include="NuGet.Services.Sql">
-      <Version>2.46.0</Version>
+      <Version>2.57.0</Version>
     </PackageReference>
     <PackageReference Include="NuGet.Protocol">
       <Version>5.0.0-preview1.5665</Version>
@@ -302,9 +302,6 @@
     <PackageReference Include="NuGet.StrongName.json-ld.net">
       <Version>1.0.6</Version>
     </PackageReference>
-    <PackageReference Include="Microsoft.ApplicationInsights">
-      <Version>2.2.0</Version>
-    </PackageReference>
     <PackageReference Include="Microsoft.Azure.Storage.DataMovement">
       <Version>0.6.0</Version>
     </PackageReference>
@@ -320,13 +317,13 @@
       <Version>5.0.0-preview1.5665</Version>
     </PackageReference>
     <PackageReference Include="NuGet.Services.Logging">
-      <Version>2.25.0-master-30088</Version>
+      <Version>2.57.0</Version>
     </PackageReference>
     <PackageReference Include="System.Net.Http">
       <Version>4.3.3</Version>
     </PackageReference>
     <PackageReference Include="WindowsAzure.Storage">
-      <Version>8.2.1</Version>
+      <Version>9.3.3</Version>
     </PackageReference>
   </ItemGroup>
   <ItemGroup>

diff --git a/src/Ng/Arguments.cs b/src/Ng/Arguments.cs
@@ -15,6 +15,7 @@ public static class Arguments
         public const string Gallery = "gallery";
         public const string Id = "id";
         public const string InstrumentationKey = "instrumentationkey";
+        public const string HeartbeatIntervalSeconds = "HeartbeatIntervalSeconds";
         public const string Path = "path";
         public const string Source = "source";
         public const string Verbose = "verbose";

diff --git a/src/Ng/Ng.csproj b/src/Ng/Ng.csproj
@@ -160,9 +160,6 @@
       <IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets>
       <PrivateAssets>all</PrivateAssets>
     </PackageReference>
-    <PackageReference Include="Microsoft.ApplicationInsights">
-      <Version>2.2.0</Version>
-    </PackageReference>
     <PackageReference Include="Microsoft.Extensions.DependencyInjection">
       <Version>1.1.1</Version>
     </PackageReference>
@@ -178,23 +175,20 @@
       <Version>5.0.0-preview1.5665</Version>
     </PackageReference>
     <PackageReference Include="NuGet.Services.Configuration">
-      <Version>2.2.2</Version>
+      <Version>2.57.0</Version>
     </PackageReference>
     <PackageReference Include="NuGet.Services.Logging">
-      <Version>2.25.0-master-30088</Version>
+      <Version>2.57.0</Version>
     </PackageReference>
     <PackageReference Include="NuGet.Services.Sql">
-      <Version>2.46.0</Version>
+      <Version>2.57.0</Version>
     </PackageReference>
     <PackageReference Include="NuGet.Services.Storage">
-      <Version>2.25.0-sb-queuesize-32353</Version>
+      <Version>2.57.0</Version>
     </PackageReference>
     <PackageReference Include="Serilog.Sinks.File">
       <Version>4.0.0</Version>
     </PackageReference>
-    <PackageReference Include="WindowsAzure.Storage">
-      <Version>8.2.1</Version>
-    </PackageReference>
   </ItemGroup>
   <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
   <PropertyGroup>

diff --git a/src/Ng/Program.cs b/src/Ng/Program.cs
@@ -58,7 +58,19 @@ public static async Task MainAsync(string[] args)
                 TelemetryConfiguration.Active.TelemetryInitializers.Add(new JobNameTelemetryInitializer(jobName, instanceName));
 
                 // Configure ApplicationInsights
-                ApplicationInsights.Initialize(arguments.GetOrDefault<string>(Arguments.InstrumentationKey));
+                var instrumentationKey = arguments.GetOrDefault<string>(Arguments.InstrumentationKey);
+                var heartbeatIntervalSeconds = arguments.GetOrDefault<int>(Arguments.HeartbeatIntervalSeconds);
+
+                if (heartbeatIntervalSeconds == 0)
+                {
+                    ApplicationInsights.Initialize(instrumentationKey);
+                }
+                else
+                {
+                    ApplicationInsights.Initialize(
+                        instrumentationKey,
+                        TimeSpan.FromSeconds(heartbeatIntervalSeconds));
+                }
 
                 // Create an ILoggerFactory
                 var loggerConfiguration = LoggingSetup.CreateDefaultLoggerConfiguration(withConsoleLogger: true);
@@ -83,7 +95,7 @@ public static async Task MainAsync(string[] args)
             catch (ArgumentException ae)
             {
                 _logger?.LogError("A required argument was not found or was malformed/invalid: {Exception}", ae);
-                
+
                 Console.WriteLine(job != null ? job.GetUsage() : NgJob.GetUsageBase());
             }
             catch (Exception e)

diff --git a/src/NuGet.Indexing/NuGet.Indexing.csproj b/src/NuGet.Indexing/NuGet.Indexing.csproj
@@ -188,7 +188,7 @@
       <Version>5.0.0-preview1.5665</Version>
     </PackageReference>
     <PackageReference Include="NuGet.Services.Configuration">
-      <Version>2.2.2</Version>
+      <Version>2.57.0</Version>
     </PackageReference>
     <PackageReference Include="NuGet.Services.Entities">
       <Version>4.4.5-dev-2939401</Version>
@@ -197,7 +197,7 @@
       <Version>4.3.3</Version>
     </PackageReference>
     <PackageReference Include="WindowsAzure.Storage">
-      <Version>8.2.1</Version>
+      <Version>9.3.3</Version>
     </PackageReference>
   </ItemGroup>
   <ItemGroup>

diff --git a/src/NuGet.Services.AzureSearch/NuGet.Services.AzureSearch.csproj b/src/NuGet.Services.AzureSearch/NuGet.Services.AzureSearch.csproj
@@ -243,7 +243,7 @@
       <Version>5.0.3</Version>
     </PackageReference>
     <PackageReference Include="NuGet.Services.Validation.Common.Job">
-      <Version>4.1.0-dev-2942767</Version>
+      <Version>4.2.0</Version>
     </PackageReference>
     <PackageReference Include="NuGet.Build.Tasks.Pack">
       <Version>4.8.0</Version>

diff --git a/src/NuGet.Services.BasicSearch/ApplicationInsights.config b/src/NuGet.Services.BasicSearch/ApplicationInsights.config
@@ -1,13 +1,12 @@
 <?xml version="1.0" encoding="utf-8"?>
 <ApplicationInsights xmlns="http://schemas.microsoft.com/ApplicationInsights/2013/Settings">
   <TelemetryProcessors>
-    <Add Type="Microsoft.ApplicationInsights.Extensibility.PerfCounterCollector.QuickPulse.QuickPulseTelemetryProcessor, Microsoft.AI.PerfCounterCollector"/>
     <Add Type="Microsoft.ApplicationInsights.WindowsServer.TelemetryChannel.AdaptiveSamplingTelemetryProcessor, Microsoft.AI.ServerTelemetryChannel">
       <MaxTelemetryItemsPerSecond>5</MaxTelemetryItemsPerSecond>
       <ExcludedTypes>Trace;Exception</ExcludedTypes>
     </Add>
   </TelemetryProcessors>
-  
+
   <TelemetryInitializers>
     <!--<Add Type="Microsoft.ApplicationInsights.Web.SessionTelemetryInitializer, Microsoft.AI.Web"/>-->
     <Add Type="NuGet.ApplicationInsights.Owin.OwinRequestIdTelemetryInitializer, NuGet.ApplicationInsights.Owin"/>
@@ -51,6 +50,36 @@
       -->
     </Add>
     <Add Type="Microsoft.ApplicationInsights.Extensibility.PerfCounterCollector.QuickPulse.QuickPulseTelemetryModule, Microsoft.AI.PerfCounterCollector"/>
+    <Add Type="Microsoft.ApplicationInsights.WindowsServer.AppServicesHeartbeatTelemetryModule, Microsoft.AI.WindowsServer"/>
+    <Add Type="Microsoft.ApplicationInsights.WindowsServer.AzureInstanceMetadataTelemetryModule, Microsoft.AI.WindowsServer">
+      <!--
+      Remove individual fields collected here by adding them to the ApplicationInsighs.HeartbeatProvider 
+      with the following syntax:
+      
+      <Add Type="Microsoft.ApplicationInsights.Extensibility.Implementation.Tracing.DiagnosticsTelemetryModule, Microsoft.ApplicationInsights">
+        <ExcludedHeartbeatProperties>
+          <Add>osType</Add>
+          <Add>location</Add>
+          <Add>name</Add>
+          <Add>offer</Add>
+          <Add>platformFaultDomain</Add>
+          <Add>platformUpdateDomain</Add>
+          <Add>publisher</Add>
+          <Add>sku</Add>
+          <Add>version</Add>
+          <Add>vmId</Add>
+          <Add>vmSize</Add>
+          <Add>subscriptionId</Add>
+          <Add>resourceGroupName</Add>
+          <Add>placementGroupId</Add>
+          <Add>tags</Add>
+          <Add>vmScaleSetName</Add>
+        </ExcludedHeartbeatProperties>
+      </Add>
+            
+      NOTE: exclusions will be lost upon upgrade.
+      -->
+    </Add>
     <Add Type="Microsoft.ApplicationInsights.WindowsServer.DeveloperModeWithDebuggerAttachedTelemetryModule, Microsoft.AI.WindowsServer"/>
     <Add Type="Microsoft.ApplicationInsights.WindowsServer.UnhandledExceptionTelemetryModule, Microsoft.AI.WindowsServer"/>
     <Add Type="Microsoft.ApplicationInsights.WindowsServer.UnobservedExceptionTelemetryModule, Microsoft.AI.WindowsServer"/>
@@ -61,7 +90,6 @@
         
         NOTE: handler configuration will be lost upon NuGet upgrade.
         -->
-        <Add>System.Web.Handlers.TransferRequestHandler</Add>
         <Add>Microsoft.VisualStudio.Web.PageInspector.Runtime.Tracing.RequestDataHttpHandler</Add>
         <Add>System.Web.StaticFileHandler</Add>
         <Add>System.Web.Handlers.AssemblyResourceLoader</Add>
@@ -73,11 +101,22 @@
       </Handlers>
     </Add>
     <Add Type="Microsoft.ApplicationInsights.Web.ExceptionTrackingTelemetryModule, Microsoft.AI.Web"/>
+    <Add Type="Microsoft.ApplicationInsights.Web.AspNetDiagnosticTelemetryModule, Microsoft.AI.Web"/>
   </TelemetryModules>
-  <TelemetryChannel Type="Microsoft.ApplicationInsights.WindowsServer.TelemetryChannel.ServerTelemetryChannel, Microsoft.AI.ServerTelemetryChannel"/>
-<!-- 
+  <ApplicationIdProvider Type="Microsoft.ApplicationInsights.Extensibility.Implementation.ApplicationId.ApplicationInsightsApplicationIdProvider, Microsoft.ApplicationInsights"/>
+  <TelemetrySinks>
+    <Add Name="default">
+      <TelemetryProcessors>
+        <Add Type="Microsoft.ApplicationInsights.Extensibility.PerfCounterCollector.QuickPulse.QuickPulseTelemetryProcessor, Microsoft.AI.PerfCounterCollector"/>
+        <Add Type="Microsoft.ApplicationInsights.Extensibility.AutocollectedMetricsExtractor, Microsoft.ApplicationInsights"/>
+      </TelemetryProcessors>
+      <TelemetryChannel Type="Microsoft.ApplicationInsights.WindowsServer.TelemetryChannel.ServerTelemetryChannel, Microsoft.AI.ServerTelemetryChannel"/>
+    </Add>
+  </TelemetrySinks>
+  <!-- 
     Learn more about Application Insights configuration with ApplicationInsights.config here: 
     http://go.microsoft.com/fwlink/?LinkID=513840
     
     Note: If not present, please add <InstrumentationKey>Your Key</InstrumentationKey> to the top of this file.
-  --></ApplicationInsights>
+  -->
+</ApplicationInsights>
diff --git a/src/NuGet.Services.BasicSearch/BasicSearchConfiguration.cs b/src/NuGet.Services.BasicSearch/BasicSearchConfiguration.cs
@@ -19,6 +19,9 @@ public class BasicSearchConfiguration : IndexingConfiguration
         [ConfigurationKeyPrefix(SerilogPrefix)]
         public string ApplicationInsightsInstrumentationKey { get; set; }
 
+        [ConfigurationKeyPrefix(SerilogPrefix)]
+        public int ApplicationInsightsHeartbeatIntervalSeconds { get; set; }
+
         [ConfigurationKeyPrefix(SearchPrefix)]
         [ConfigurationKey("IndexRefresh")]
         [DefaultValue(300)]

diff --git a/src/NuGet.Services.BasicSearch/NuGet.Services.BasicSearch.csproj b/src/NuGet.Services.BasicSearch/NuGet.Services.BasicSearch.csproj
@@ -130,9 +130,6 @@
       <IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets>
       <PrivateAssets>all</PrivateAssets>
     </PackageReference>
-    <PackageReference Include="Microsoft.ApplicationInsights">
-      <Version>2.2.0</Version>
-    </PackageReference>
     <PackageReference Include="Microsoft.ApplicationInsights.JavaScript">
       <Version>1.0.17-build01715</Version>
     </PackageReference>
@@ -143,7 +140,7 @@
       <Version>0.12.0-build17386</Version>
     </PackageReference>
     <PackageReference Include="Microsoft.ApplicationInsights.Web">
-      <Version>2.2.0</Version>
+      <Version>2.10.0</Version>
     </PackageReference>
     <PackageReference Include="Microsoft.Diagnostics.Instrumentation.Extensions.Intercept">
       <Version>0.12.0-build02810</Version>
@@ -164,10 +161,10 @@
       <Version>3.0.1</Version>
     </PackageReference>
     <PackageReference Include="NuGet.Services.Logging">
-      <Version>2.25.0-master-30088</Version>
+      <Version>2.57.0</Version>
     </PackageReference>
     <PackageReference Include="NuGet.Services.Owin">
-      <Version>2.2.3</Version>
+      <Version>2.57.0</Version>
     </PackageReference>
     <PackageReference Include="SerilogWeb.Classic">
       <Version>2.0.9</Version>

diff --git a/src/NuGet.Services.BasicSearch/Startup.cs b/src/NuGet.Services.BasicSearch/Startup.cs
@@ -41,16 +41,21 @@ public class Startup
         private SearchTelemetryClient _searchTelemetryClient;
         private IConfigurationFactory _configFactory;
 
-        public void Configuration(IAppBuilder app, IConfigurationFactory configFactory, Directory directory,
+        public void Configuration(
+            IAppBuilder app,
+            IConfigurationFactory configFactory,
+            Directory directory,
             ILoader loader)
         {
             _configFactory = configFactory;
             var config = GetConfiguration().Result;
 
             // Configure
-            if ( ! string.IsNullOrEmpty(config.ApplicationInsightsInstrumentationKey))
+            if (!string.IsNullOrEmpty(config.ApplicationInsightsInstrumentationKey))
             {
-                TelemetryConfiguration.Active.InstrumentationKey = config.ApplicationInsightsInstrumentationKey;
+                Logging.ApplicationInsights.Initialize(
+                    config.ApplicationInsightsInstrumentationKey,
+                    TimeSpan.FromSeconds(config.ApplicationInsightsHeartbeatIntervalSeconds));
             }
 
             // Add telemetry initializers
@@ -218,7 +223,7 @@ private async Task<BasicSearchConfiguration> GetConfiguration()
             {
                 return await _configFactory.Get<BasicSearchConfiguration>();
             }
-            catch (KeyVaultClientException e) 
+            catch (KeyVaultClientException e)
             {
                 // The status code we expect is (e.Status == HttpStatusCode.Unauthorized || e.Status == HttpStatusCode.Forbidden) but the catch is not explicit since confidence here is low.