Upgrade telemetry and add heartbeats support (#732)

NuGet · Jan 24, 2020 · 5aad4d0 · 5aad4d0
1 parent d81908c
commit 5aad4d0
Show file tree

Hide file tree

Showing 58 changed files with 522 additions and 728 deletions.
diff --git a/sign.thirdparty.props b/sign.thirdparty.props
@@ -30,7 +30,7 @@
     <ThirdPartyBinaries Include="Serilog.Enrichers.Process.dll" />
     <ThirdPartyBinaries Include="Serilog.Extensions.Logging.dll" />
     <ThirdPartyBinaries Include="Serilog.Sinks.ApplicationInsights.dll" />
-    <ThirdPartyBinaries Include="Serilog.Sinks.ColoredConsole.dll" />
+    <ThirdPartyBinaries Include="Serilog.Sinks.Console.dll" />
     <ThirdPartyBinaries Include="Serilog.Sinks.File.dll" />
     <ThirdPartyBinaries Include="SerilogTraceListener.dll" />
     <ThirdPartyBinaries Include="VDS.Common.dll" />

diff --git a/src/Catalog/NuGet.Services.Metadata.Catalog.csproj b/src/Catalog/NuGet.Services.Metadata.Catalog.csproj
@@ -292,7 +292,7 @@
       <Version>4.4.5-dev-3213233</Version>
     </PackageReference>
     <PackageReference Include="NuGet.Services.Sql">
-      <Version>2.58.0</Version>
+      <Version>2.66.0</Version>
     </PackageReference>
     <PackageReference Include="NuGet.StrongName.json-ld.net">
       <Version>1.0.6</Version>
@@ -309,7 +309,7 @@
       <PrivateAssets>all</PrivateAssets>
     </PackageReference>
     <PackageReference Include="NuGet.Services.Logging">
-      <Version>2.58.0</Version>
+      <Version>2.66.0</Version>
     </PackageReference>
     <PackageReference Include="WindowsAzure.Storage">
       <Version>9.3.3</Version>

diff --git a/src/Catalog/Telemetry/TelemetryService.cs b/src/Catalog/Telemetry/TelemetryService.cs
@@ -3,7 +3,6 @@
 
 using System;
 using System.Collections.Generic;
-using Microsoft.ApplicationInsights;
 using NuGet.Services.Logging;
 using NuGet.Services.Metadata.Catalog.Helpers;
 using NuGet.Versioning;
@@ -12,19 +11,14 @@ namespace NuGet.Services.Metadata.Catalog
 {
     public class TelemetryService : ITelemetryService
     {
-        private readonly TelemetryClientWrapper _telemetryClient;
+        private readonly ITelemetryClient _telemetryClient;
 
         public IDictionary<string, string> GlobalDimensions { get; }
 
-        public TelemetryService(TelemetryClient telemetryClient)
+        public TelemetryService(ITelemetryClient telemetryClient, IDictionary<string, string> globalDimensions)
         {
-            if (telemetryClient == null)
-            {
-                throw new ArgumentNullException(nameof(telemetryClient));
-            }
-
-            _telemetryClient = new TelemetryClientWrapper(telemetryClient);
-            GlobalDimensions = new Dictionary<string, string>();
+            _telemetryClient = telemetryClient ?? throw new ArgumentNullException(nameof(telemetryClient));
+            GlobalDimensions = globalDimensions ?? throw new ArgumentNullException(nameof(globalDimensions));
         }
 
         public void TrackCatalogIndexReadDuration(TimeSpan duration, Uri uri)

diff --git a/src/Ng/Jobs/Catalog2DnxJob.cs b/src/Ng/Jobs/Catalog2DnxJob.cs
@@ -8,6 +8,7 @@
 using Microsoft.Extensions.Logging;
 using NuGet.Protocol.Catalog;
 using NuGet.Services.Configuration;
+using NuGet.Services.Logging;
 using NuGet.Services.Metadata.Catalog;
 using NuGet.Services.Metadata.Catalog.Dnx;
 using NuGet.Services.Metadata.Catalog.Persistence;
@@ -21,8 +22,10 @@ public class Catalog2DnxJob : LoopingNgJob
         private ReadCursor _back;
         private Uri _destination;
 
-        public Catalog2DnxJob(ITelemetryService telemetryService, ILoggerFactory loggerFactory)
-            : base(telemetryService, loggerFactory)
+        public Catalog2DnxJob(ILoggerFactory loggerFactory,
+            ITelemetryClient telemetryClient,
+            IDictionary<string, string> telemetryGlobalDimensions)
+            : base(loggerFactory, telemetryClient, telemetryGlobalDimensions)
         {
         }
 

diff --git a/src/Ng/Jobs/Catalog2IconJob.cs b/src/Ng/Jobs/Catalog2IconJob.cs
@@ -10,6 +10,7 @@
 using Microsoft.Extensions.Logging;
 using NuGet.Protocol.Catalog;
 using NuGet.Services.Configuration;
+using NuGet.Services.Logging;
 using NuGet.Services.Metadata.Catalog;
 using NuGet.Services.Metadata.Catalog.Icons;
 using NuGet.Services.Metadata.Catalog.Persistence;
@@ -23,8 +24,11 @@ public class Catalog2IconJob : LoopingNgJob
         private IconsCollector _collector;
         private DurableCursor _front;
 
-        public Catalog2IconJob(ITelemetryService telemetryService, ILoggerFactory loggerFactory)
-            : base(telemetryService, loggerFactory)
+        public Catalog2IconJob(
+            ILoggerFactory loggerFactory,
+            ITelemetryClient telemetryClient,
+            IDictionary<string, string> telemetryGlobalDimensions)
+            : base(loggerFactory, telemetryClient, telemetryGlobalDimensions)
         {
         }
 

diff --git a/src/Ng/Jobs/Catalog2LuceneJob.cs b/src/Ng/Jobs/Catalog2LuceneJob.cs
@@ -11,6 +11,7 @@
 using Microsoft.Extensions.Logging;
 using NuGet.Indexing;
 using NuGet.Services.Configuration;
+using NuGet.Services.Logging;
 using NuGet.Services.Metadata.Catalog;
 
 namespace Ng.Jobs
@@ -30,8 +31,11 @@ public class Catalog2LuceneJob : LoopingNgJob
         private Func<HttpMessageHandler> _handlerFunc;
         private string _destination;
 
-        public Catalog2LuceneJob(ITelemetryService telemetryService, ILoggerFactory loggerFactory)
-            : base(telemetryService, loggerFactory)
+        public Catalog2LuceneJob(
+            ILoggerFactory loggerFactory,
+            ITelemetryClient telemetryClient,
+            IDictionary<string, string> telemetryGlobalDimensions)
+            : base(loggerFactory, telemetryClient, telemetryGlobalDimensions)
         {
         }
 

diff --git a/src/Ng/Jobs/Catalog2MonitoringJob.cs b/src/Ng/Jobs/Catalog2MonitoringJob.cs
@@ -6,6 +6,7 @@
 using System.Threading.Tasks;
 using Microsoft.Extensions.Logging;
 using NuGet.Services.Configuration;
+using NuGet.Services.Logging;
 using NuGet.Services.Metadata.Catalog;
 using NuGet.Services.Metadata.Catalog.Monitoring;
 
@@ -17,10 +18,13 @@ namespace Ng.Jobs
     /// </summary>
     public class Catalog2MonitoringJob : LoopingNgJob
     {
-        private PackageValidatorContextEnqueuer _enqueuer; 
+        private PackageValidatorContextEnqueuer _enqueuer;
 
-        public Catalog2MonitoringJob(ITelemetryService telemetryService, ILoggerFactory loggerFactory)
-            : base(telemetryService, loggerFactory)
+        public Catalog2MonitoringJob(
+            ILoggerFactory loggerFactory,
+            ITelemetryClient telemetryClient,
+            IDictionary<string, string> telemetryGlobalDimensions)
+            : base(loggerFactory, telemetryClient, telemetryGlobalDimensions)
         {
         }
 

diff --git a/src/Ng/Jobs/Catalog2PackageFixupJob.cs b/src/Ng/Jobs/Catalog2PackageFixupJob.cs
@@ -15,20 +15,20 @@
 using Microsoft.WindowsAzure.Storage.Auth;
 using NuGet.Packaging.Core;
 using NuGet.Services.Configuration;
+using NuGet.Services.Logging;
 using NuGet.Services.Metadata.Catalog;
 
 namespace Ng.Jobs
 {
     public class Catalog2PackageFixupJob : NgJob
     {
-        private const int MaximumPackageProcessingAttempts = 5;
-        private static readonly TimeSpan MaximumPackageProcessingTime = TimeSpan.FromMinutes(10);
-        private static readonly TimeSpan DonePollingInterval = TimeSpan.FromMinutes(1);
-
         private IServiceProvider _serviceProvider;
 
-        public Catalog2PackageFixupJob(ITelemetryService telemetryService, ILoggerFactory loggerFactory)
-            : base(telemetryService, loggerFactory)
+        public Catalog2PackageFixupJob(
+            ILoggerFactory loggerFactory,
+            ITelemetryClient telemetryClient,
+            IDictionary<string, string> telemetryGlobalDimensions)
+            : base(loggerFactory, telemetryClient, telemetryGlobalDimensions)
         {
             ThreadPool.SetMinThreads(MaxDegreeOfParallelism, 4);
         }
@@ -54,7 +54,7 @@ protected override void Init(IDictionary<string, string> arguments, Cancellation
 
             var services = new ServiceCollection();
 
-            services.AddSingleton(TelemetryService);
+            services.AddSingleton(new TelemetryService(TelemetryClient, GlobalTelemetryDimensions));
             services.AddSingleton(LoggerFactory);
             services.AddLogging();
 
@@ -65,7 +65,7 @@ protected override void Init(IDictionary<string, string> arguments, Cancellation
                 {
                     AllowPipelining = true
                 });
-                
+
                 httpClient.DefaultRequestHeaders.Add("User-Agent", UserAgentUtility.GetUserAgent());
                 httpClient.Timeout = TimeSpan.FromMinutes(10);
 

diff --git a/src/Ng/Jobs/Catalog2RegistrationJob.cs b/src/Ng/Jobs/Catalog2RegistrationJob.cs
@@ -7,6 +7,7 @@
 using System.Threading.Tasks;
 using Microsoft.Extensions.Logging;
 using NuGet.Services.Configuration;
+using NuGet.Services.Logging;
 using NuGet.Services.Metadata.Catalog;
 using NuGet.Services.Metadata.Catalog.Registration;
 
@@ -19,8 +20,11 @@ public class Catalog2RegistrationJob : LoopingNgJob
         private ReadCursor _back;
         private Uri _destination;
 
-        public Catalog2RegistrationJob(ITelemetryService telemetryService, ILoggerFactory loggerFactory)
-            : base(telemetryService, loggerFactory)
+        public Catalog2RegistrationJob(
+            ILoggerFactory loggerFactory,
+            ITelemetryClient telemetryClient,
+            IDictionary<string, string> telemetryGlobalDimensions)
+            : base(loggerFactory, telemetryClient, telemetryGlobalDimensions)
         {
         }
 

diff --git a/src/Ng/Jobs/CheckLuceneJob.cs b/src/Ng/Jobs/CheckLuceneJob.cs
@@ -6,15 +6,19 @@
 using System.Threading.Tasks;
 using Lucene.Net.Index;
 using Microsoft.Extensions.Logging;
-using NuGet.Services.Metadata.Catalog;
+using NuGet.Services.Logging;
 
 namespace Ng.Jobs
 {
     public class CheckLuceneJob : NgJob
     {
         private Lucene.Net.Store.Directory _directory;
 
-        public CheckLuceneJob(ITelemetryService telemetryService, ILoggerFactory loggerFactory) : base(telemetryService, loggerFactory)
+        public CheckLuceneJob(
+            ILoggerFactory loggerFactory,
+            ITelemetryClient telemetryClient,
+            IDictionary<string, string> telemetryGlobalDimensions)
+            : base(loggerFactory, telemetryClient, telemetryGlobalDimensions)
         {
         }
 

diff --git a/src/Ng/Jobs/ClearLuceneJob.cs b/src/Ng/Jobs/ClearLuceneJob.cs
@@ -4,18 +4,22 @@
 using System.Collections.Generic;
 using System.Threading;
 using System.Threading.Tasks;
-using Microsoft.Extensions.Logging;
-using Lucene.Net.Index;
 using Lucene.Net.Analysis.Standard;
-using NuGet.Services.Metadata.Catalog;
+using Lucene.Net.Index;
+using Microsoft.Extensions.Logging;
+using NuGet.Services.Logging;
 
 namespace Ng.Jobs
 {
     public class ClearLuceneJob : NgJob
     {
         private Lucene.Net.Store.Directory _directory;
 
-        public ClearLuceneJob(ITelemetryService telemetryService, ILoggerFactory loggerFactory) : base(telemetryService, loggerFactory)
+        public ClearLuceneJob(
+            ILoggerFactory loggerFactory,
+            ITelemetryClient telemetryClient,
+            IDictionary<string, string> telemetryGlobalDimensions)
+            : base(loggerFactory, telemetryClient, telemetryGlobalDimensions)
         {
         }
 
@@ -34,7 +38,7 @@ protected override void Init(IDictionary<string, string> arguments, Cancellation
         {
             _directory = CommandHelpers.GetLuceneDirectory(arguments);
         }
-        
+
         protected override Task RunInternalAsync(CancellationToken cancellationToken)
         {
             if (IndexReader.IndexExists(_directory))

diff --git a/src/Ng/Jobs/CopyLuceneJob.cs b/src/Ng/Jobs/CopyLuceneJob.cs
@@ -6,9 +6,9 @@
 using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
-using Microsoft.Extensions.Logging;
 using Lucene.Net.Store.Azure;
-using NuGet.Services.Metadata.Catalog;
+using Microsoft.Extensions.Logging;
+using NuGet.Services.Logging;
 
 namespace Ng.Jobs
 {
@@ -17,7 +17,11 @@ public class CopyLuceneJob : NgJob
         private Lucene.Net.Store.Directory _srcDirectory;
         private Lucene.Net.Store.Directory _destDirectory;
 
-        public CopyLuceneJob(ITelemetryService telemetryService, ILoggerFactory loggerFactory) : base(telemetryService, loggerFactory)
+        public CopyLuceneJob(
+            ILoggerFactory loggerFactory,
+            ITelemetryClient telemetryClient,
+            IDictionary<string, string> telemetryGlobalDimensions)
+            : base(loggerFactory, telemetryClient, telemetryGlobalDimensions)
         {
         }
 
@@ -47,7 +51,7 @@ protected override void Init(IDictionary<string, string> arguments, Cancellation
             _srcDirectory = CommandHelpers.GetCopySrcLuceneDirectory(arguments);
             _destDirectory = CommandHelpers.GetCopyDestLuceneDirectory(arguments);
         }
-        
+
         protected override Task RunInternalAsync(CancellationToken cancellationToken)
         {
             Lucene.Net.Store.Directory.Copy(_srcDirectory, _destDirectory, true);

diff --git a/src/Ng/Jobs/Db2CatalogJob.cs b/src/Ng/Jobs/Db2CatalogJob.cs
@@ -12,6 +12,7 @@
 using Ng.Helpers;
 using NuGet.Protocol;
 using NuGet.Services.Configuration;
+using NuGet.Services.Logging;
 using NuGet.Services.Metadata.Catalog;
 using NuGet.Services.Metadata.Catalog.Helpers;
 using NuGet.Services.Metadata.Catalog.Persistence;
@@ -35,8 +36,11 @@ public class Db2CatalogJob : LoopingNgJob
         protected Uri Destination;
         protected bool SkipCreatedPackagesProcessing;
 
-        public Db2CatalogJob(ITelemetryService telemetryService, ILoggerFactory loggerFactory)
-            : base(telemetryService, loggerFactory)
+        public Db2CatalogJob(
+            ILoggerFactory loggerFactory,
+            ITelemetryClient telemetryClient,
+            IDictionary<string, string> telemetryGlobalDimensions)
+            : base(loggerFactory, telemetryClient, telemetryGlobalDimensions)
         {
         }
 

diff --git a/src/Ng/Jobs/Db2LuceneJob.cs b/src/Ng/Jobs/Db2LuceneJob.cs
@@ -8,7 +8,7 @@
 using Microsoft.Extensions.Logging;
 using NuGet.Indexing;
 using NuGet.Services.Configuration;
-using NuGet.Services.Metadata.Catalog;
+using NuGet.Services.Logging;
 
 namespace Ng.Jobs
 {
@@ -19,7 +19,11 @@ public class Db2LuceneJob : NgJob
         private string _source;
         private Uri _catalogIndexUrl;
 
-        public Db2LuceneJob(ITelemetryService telemetryService, ILoggerFactory loggerFactory) : base(telemetryService, loggerFactory)
+        public Db2LuceneJob(
+            ILoggerFactory loggerFactory,
+            ITelemetryClient telemetryClient,
+            IDictionary<string, string> telemetryGlobalDimensions)
+            : base(loggerFactory, telemetryClient, telemetryGlobalDimensions)
         {
         }
 
@@ -40,7 +44,7 @@ protected override void Init(IDictionary<string, string> arguments, Cancellation
 
             _catalogIndexUrl = new Uri(_source);
         }
-        
+
         protected override Task RunInternalAsync(CancellationToken cancellationToken)
         {
             Sql2Lucene.Export(_connectionString, _catalogIndexUrl, _path, LoggerFactory);

diff --git a/src/Ng/Jobs/Db2MonitoringJob.cs b/src/Ng/Jobs/Db2MonitoringJob.cs
@@ -22,6 +22,7 @@
 using CatalogStorageFactory = NuGet.Services.Metadata.Catalog.Persistence.StorageFactory;
 using CatalogStorage = NuGet.Services.Metadata.Catalog.Persistence.Storage;
 using Constants = NuGet.Services.Metadata.Catalog.Constants;
+using NuGet.Services.Logging;
 
 namespace Ng.Jobs
 {
@@ -53,8 +54,11 @@ public class Db2MonitoringJob : LoopingNgJob
 
         private CollectorHttpClient _client;
 
-        public Db2MonitoringJob(ITelemetryService telemetryService, ILoggerFactory loggerFactory)
-            : base(telemetryService, loggerFactory)
+        public Db2MonitoringJob(
+            ILoggerFactory loggerFactory,
+            ITelemetryClient telemetryClient,
+            IDictionary<string, string> telemetryGlobalDimensions)
+            : base(loggerFactory, telemetryClient, telemetryGlobalDimensions)
         {
         }
 
@@ -143,7 +147,7 @@ protected override async Task RunInternalAsync(CancellationToken cancellationTok
         }
 
         private async Task<bool> CheckPackages(
-            IReadOnlyCollection<IPackageStatusOutdatedCheckSource> sources, 
+            IReadOnlyCollection<IPackageStatusOutdatedCheckSource> sources,
             CancellationToken cancellationToken)
         {
             Logger.LogInformation("Fetching packages to check status of.");