Merge pull request #683 from NuGet/dev

[ReleasePrep][2018.11.26]RI of dev into master
NuGet · Nov 27, 2018 · cf9c69b · cf9c69b
2 parents 2648d4a + 76a88ac
commit cf9c69b
Show file tree

Hide file tree

Showing 24 changed files with 236 additions and 34 deletions.
diff --git a/src/Gallery.CredentialExpiration/Gallery.CredentialExpiration.csproj b/src/Gallery.CredentialExpiration/Gallery.CredentialExpiration.csproj
@@ -103,7 +103,7 @@
       <Version>9.0.1</Version>
     </PackageReference>
     <PackageReference Include="NuGet.Services.Messaging.Email">
-      <Version>2.36.0</Version>
+      <Version>2.39.0</Version>
     </PackageReference>
     <PackageReference Include="NuGet.Services.Storage">
       <Version>2.1.3</Version>

diff --git a/src/NuGet.Jobs.Common/NuGet.Jobs.Common.csproj b/src/NuGet.Jobs.Common/NuGet.Jobs.Common.csproj
@@ -105,13 +105,13 @@
       <PrivateAssets>all</PrivateAssets>
     </PackageReference>
     <PackageReference Include="NuGet.Services.Configuration">
-      <Version>2.33.0</Version>
+      <Version>2.39.0</Version>
     </PackageReference>
     <PackageReference Include="NuGet.Services.Logging">
-      <Version>2.33.0</Version>
+      <Version>2.39.0</Version>
     </PackageReference>
     <PackageReference Include="NuGet.Services.Sql">
-      <Version>2.33.0</Version>
+      <Version>2.39.0</Version>
     </PackageReference>
     <PackageReference Include="System.Net.Http">
       <Version>4.3.3</Version>

diff --git a/src/NuGet.Services.Revalidate/NuGet.Services.Revalidate.csproj b/src/NuGet.Services.Revalidate/NuGet.Services.Revalidate.csproj
@@ -123,7 +123,7 @@
       <PrivateAssets>all</PrivateAssets>
     </PackageReference>
     <PackageReference Include="NuGet.Services.Status">
-      <Version>2.33.0</Version>
+      <Version>2.39.0</Version>
     </PackageReference>
   </ItemGroup>
   <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />

diff --git a/src/NuGet.Services.Validation.Orchestrator/Job.cs b/src/NuGet.Services.Validation.Orchestrator/Job.cs
@@ -47,6 +47,11 @@ namespace NuGet.Services.Validation.Orchestrator
 {
     public class Job : JobBase
     {
+        /// <summary>
+        /// The maximum number of concurrent connections that can be established to a single server.
+        /// </summary>
+        private const int MaximumConnectionsPerServer = 64;
+
         private const string ConfigurationArgument = "Configuration";
         private const string ValidateArgument = "Validate";
 
@@ -93,6 +98,8 @@ public class Job : JobBase
 
         public override void Init(IServiceContainer serviceContainer, IDictionary<string, string> jobArgsDictionary)
         {
+            ServicePointManager.DefaultConnectionLimit = MaximumConnectionsPerServer;
+
             var configurationFilename = JobConfigurationManager.GetArgument(jobArgsDictionary, ConfigurationArgument);
             _validateOnly = JobConfigurationManager.TryGetBoolArgument(jobArgsDictionary, ValidateArgument, defaultValue: false);
 

diff --git a/src/NuGet.Services.Validation.Orchestrator/NuGet.Services.Validation.Orchestrator.csproj b/src/NuGet.Services.Validation.Orchestrator/NuGet.Services.Validation.Orchestrator.csproj
@@ -167,7 +167,7 @@
       <PrivateAssets>all</PrivateAssets>
     </PackageReference>
     <PackageReference Include="NuGet.Services.Messaging.Email">
-      <Version>2.36.0</Version>
+      <Version>2.39.0</Version>
     </PackageReference>
   </ItemGroup>
   <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />

diff --git a/src/NuGet.Services.Validation.Orchestrator/Services/PackageMessageService.cs b/src/NuGet.Services.Validation.Orchestrator/Services/PackageMessageService.cs
@@ -7,6 +7,7 @@
 using Microsoft.Extensions.Options;
 using NuGet.Services.Entities;
 using NuGet.Services.Messaging.Email;
+using NuGetGallery.Infrastructure.Mail.Messages;
 
 namespace NuGet.Services.Validation.Orchestrator
 {

diff --git a/src/NuGet.Services.Validation.Orchestrator/Services/SymbolsMessageService.cs b/src/NuGet.Services.Validation.Orchestrator/Services/SymbolsMessageService.cs
@@ -7,6 +7,7 @@
 using Microsoft.Extensions.Options;
 using NuGet.Services.Entities;
 using NuGet.Services.Messaging.Email;
+using NuGetGallery.Infrastructure.Mail.Messages;
 
 namespace NuGet.Services.Validation.Orchestrator
 {

diff --git a/src/NuGet.Services.Validation.Orchestrator/Telemetry/ITelemetryService.cs b/src/NuGet.Services.Validation.Orchestrator/Telemetry/ITelemetryService.cs
@@ -17,12 +17,19 @@ public interface ITelemetryService
         /// </summary>
         void TrackDurationToValidationSetCreation(TimeSpan duration);
 
+        /// <summary>
+        /// Track how long a package's backup takes.
+        /// </summary>
+        /// <param name="validationSet">The validation set that requested the backup.</param>
+        /// <returns>Reports the duration when disposed.</returns>
+        IDisposable TrackDurationToBackupPackage(PackageValidationSet validationSet);
+
         /// <summary>
         /// A counter metric emitted when a package changes package status. This metric is not emitted if package status
         /// does not change. This metric is emitted for revalidation if the terminal state changes.
         /// </summary>
         /// <param name="fromStatus">The status that the package moved from.</param>
-        /// <param name="toStatus">The status that the package moved tp.</param>
+        /// <param name="toStatus">The status that the package moved to.</param>
         void TrackPackageStatusChange(PackageStatus fromStatus, PackageStatus toStatus);
 
         /// <summary>

diff --git a/src/NuGet.Services.Validation.Orchestrator/Telemetry/TelemetryService.cs b/src/NuGet.Services.Validation.Orchestrator/Telemetry/TelemetryService.cs
@@ -16,6 +16,7 @@ public class TelemetryService : ITelemetryService, ISubscriptionProcessorTelemet
         private const string PackageCertificatesPrefix = "PackageCertificates.";
 
         private const string DurationToValidationSetCreationSeconds = OrchestratorPrefix + "DurationToValidationSetCreationSeconds";
+        private const string DurationToBackupPackageSeconds = OrchestratorPrefix + "DurationToBackupPackageSeconds";
         private const string PackageStatusChange = OrchestratorPrefix + "PackageStatusChange";
         private const string TotalValidationDurationSeconds = OrchestratorPrefix + "TotalValidationDurationSeconds";
         private const string SentValidationTakingTooLongMessage = OrchestratorPrefix + "SentValidationTakingTooLongMessage";
@@ -31,6 +32,8 @@ public class TelemetryService : ITelemetryService, ISubscriptionProcessorTelemet
         private const string DurationToHashPackageSeconds = OrchestratorPrefix + "DurationToHashPackageSeconds";
         private const string MessageDeliveryLag = OrchestratorPrefix + "MessageDeliveryLag";
         private const string MessageEnqueueLag = OrchestratorPrefix + "MessageEnqueueLag";
+        private const string MessageHandlerDurationSeconds = OrchestratorPrefix + "MessageHandlerDurationSeconds";
+        private const string MessageLockLost = OrchestratorPrefix + "MessageLockLost";
         private const string SymbolsMessageEnqueued = OrchestratorPrefix + "SymbolsMessageEnqueued";
 
         private const string DurationToStartPackageSigningValidatorSeconds = PackageSigningPrefix + "DurationToStartSeconds";
@@ -50,6 +53,8 @@ public class TelemetryService : ITelemetryService, ISubscriptionProcessorTelemet
         private const string HashAlgorithm = "HashAlgorithm";
         private const string StreamType = "StreamType";
         private const string MessageType = "MessageType";
+        private const string CallGuid = "CallGuid";
+        private const string Handled = "Handled";
         private const string ValidationId = "ValidationId";
         private const string OperationDateTime = "OperationDateTime";
 
@@ -86,6 +91,18 @@ public void TrackDurationToValidationSetCreation(TimeSpan duration)
                 duration.TotalSeconds);
         }
 
+        public IDisposable TrackDurationToBackupPackage(PackageValidationSet validationSet)
+        {
+            return _telemetryClient.TrackDuration(
+                DurationToBackupPackageSeconds,
+                new Dictionary<string, string>
+                {
+                    { ValidationTrackingId, validationSet.ValidationTrackingId.ToString() },
+                    { PackageId, validationSet.PackageId },
+                    { NormalizedVersion, validationSet.PackageNormalizedVersion }
+                });
+        }
+
         public void TrackPackageStatusChange(PackageStatus fromStatus, PackageStatus toStatus)
         {
             _telemetryClient.TrackMetric(
@@ -238,6 +255,31 @@ public void TrackMessageDeliveryLag<TMessage>(TimeSpan deliveryLag)
                     { MessageType, typeof(TMessage).Name }
                 });
 
+        public void TrackMessageHandlerDuration<TMessage>(TimeSpan duration, Guid callGuid, bool handled)
+        {
+            _telemetryClient.TrackMetric(
+                MessageHandlerDurationSeconds,
+                duration.TotalSeconds,
+                new Dictionary<string, string>
+                {
+                    { MessageType, typeof(TMessage).Name },
+                    { CallGuid, callGuid.ToString() },
+                    { Handled, handled.ToString() }
+                });
+        }
+
+        public void TrackMessageLockLost<TMessage>(Guid callGuid)
+        {
+            _telemetryClient.TrackMetric(
+                MessageLockLost,
+                1,
+                new Dictionary<string, string>
+                {
+                    { MessageType, typeof(TMessage).Name },
+                    { CallGuid, callGuid.ToString() }
+                });
+        }
+
         public void TrackEnqueueLag<TMessage>(TimeSpan enqueueLag)
             => _telemetryClient.TrackMetric(
                 MessageEnqueueLag,

diff --git a/src/NuGet.Services.Validation.Orchestrator/ValidationPackageFileService.cs b/src/NuGet.Services.Validation.Orchestrator/ValidationPackageFileService.cs
@@ -118,19 +118,22 @@ public Task CopyValidationPackageForValidationSetAsync(PackageValidationSet vali
 
         public async Task BackupPackageFileFromValidationSetPackageAsync(PackageValidationSet validationSet)
         {
-            _logger.LogInformation(
-                "Backing up package for validation set {ValidationTrackingId} ({PackageId} {PackageVersion}).",
-                validationSet.ValidationTrackingId,
-                validationSet.PackageId,
-                validationSet.PackageNormalizedVersion);
+            using (_telemetryService.TrackDurationToBackupPackage(validationSet))
+            {
+                _logger.LogInformation(
+                    "Backing up package for validation set {ValidationTrackingId} ({PackageId} {PackageVersion}).",
+                    validationSet.ValidationTrackingId,
+                    validationSet.PackageId,
+                    validationSet.PackageNormalizedVersion);
 
-            var packageUri = await GetPackageForValidationSetReadUriAsync(
-                validationSet,
-                DateTimeOffset.UtcNow.Add(AccessDuration));
+                var packageUri = await GetPackageForValidationSetReadUriAsync(
+                    validationSet,
+                    DateTimeOffset.UtcNow.Add(AccessDuration));
 
-            using (var packageStream = await _fileDownloader.DownloadAsync(packageUri, CancellationToken.None))
-            {
-                await StorePackageFileInBackupLocationAsync(validationSet, packageStream);
+                using (var packageStream = await _fileDownloader.DownloadAsync(packageUri, CancellationToken.None))
+                {
+                    await StorePackageFileInBackupLocationAsync(validationSet, packageStream);
+                }
             }
         }
 

diff --git a/src/NuGet.SupportRequests.Notifications/NuGet.SupportRequests.Notifications.csproj b/src/NuGet.SupportRequests.Notifications/NuGet.SupportRequests.Notifications.csproj
@@ -111,7 +111,7 @@
       <Version>9.0.1</Version>
     </PackageReference>
     <PackageReference Include="NuGet.Services.Messaging.Email">
-      <Version>2.36.0</Version>
+      <Version>2.39.0</Version>
     </PackageReference>
   </ItemGroup>
   <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />

diff --git a/src/PackageHash/PackageHash.csproj b/src/PackageHash/PackageHash.csproj
@@ -81,7 +81,7 @@
       <Version>1.1.2</Version>
     </PackageReference>
     <PackageReference Include="NuGet.Services.Cursor">
-      <Version>2.22.0</Version>
+      <Version>2.39.0</Version>
     </PackageReference>
   </ItemGroup>
   <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />

diff --git a/src/PackageLagMonitor/Monitoring.PackageLag.csproj b/src/PackageLagMonitor/Monitoring.PackageLag.csproj
@@ -112,7 +112,7 @@
       <Version>0.5.0-CI-20180510-012541</Version>
     </PackageReference>
     <PackageReference Include="NuGet.Services.AzureManagement">
-      <Version>2.33.0</Version>
+      <Version>2.39.0</Version>
     </PackageReference>
     <PackageReference Include="System.Net.Http">
       <Version>4.3.3</Version>

diff --git a/src/StatusAggregator/StatusAggregator.csproj b/src/StatusAggregator/StatusAggregator.csproj
@@ -157,13 +157,13 @@
       <Version>1.1.1</Version>
     </PackageReference>
     <PackageReference Include="NuGet.Services.Incidents">
-      <Version>2.38.0</Version>
+      <Version>2.39.0</Version>
     </PackageReference>
     <PackageReference Include="NuGet.Services.Status">
-      <Version>2.38.0</Version>
+      <Version>2.39.0</Version>
     </PackageReference>
     <PackageReference Include="NuGet.Services.Status.Table">
-      <Version>2.38.0</Version>
+      <Version>2.39.0</Version>
     </PackageReference>
     <PackageReference Include="WindowsAzure.Storage">
       <Version>9.2.0</Version>

diff --git a/src/Validation.Common.Job/Storage/ProcessorPackageFileService.cs b/src/Validation.Common.Job/Storage/ProcessorPackageFileService.cs
@@ -50,7 +50,7 @@ public Task<Uri> GetReadAndDeleteUriAsync(
             var fileName = BuildFileName(packageId, packageNormalizedVersion, validationId);
 
             return _fileStorageService.GetPriviledgedFileUriAsync(
-                CoreConstants.ValidationFolderName,
+                CoreConstants.Folders.ValidationFolderName,
                 fileName,
                 FileUriPermissions.Read | FileUriPermissions.Delete,
                 DateTimeOffset.UtcNow + AccessDuration);
@@ -80,7 +80,7 @@ public Task SaveAsync(
             packageFile.Position = 0;
 
             return _fileStorageService.SaveFileAsync(
-                CoreConstants.ValidationFolderName,
+                CoreConstants.Folders.ValidationFolderName,
                 fileName,
                 packageFile,
                 overwrite: true);

diff --git a/src/Validation.Common.Job/Validation.Common.Job.csproj b/src/Validation.Common.Job/Validation.Common.Job.csproj
@@ -103,19 +103,19 @@
       <Version>5.0.0-preview1.5665</Version>
     </PackageReference>
     <PackageReference Include="NuGet.Services.ServiceBus">
-      <Version>2.33.0</Version>
+      <Version>2.39.0</Version>
     </PackageReference>
     <PackageReference Include="NuGet.Services.Storage">
-      <Version>2.33.0</Version>
+      <Version>2.39.0</Version>
     </PackageReference>
     <PackageReference Include="NuGet.Services.Validation">
-      <Version>2.33.0</Version>
+      <Version>2.39.0</Version>
     </PackageReference>
     <PackageReference Include="NuGet.Services.Validation.Issues">
-      <Version>2.33.0</Version>
+      <Version>2.39.0</Version>
     </PackageReference>
     <PackageReference Include="NuGetGallery.Core">
-      <Version>4.4.5-dev-2193892</Version>
+      <Version>4.4.5-master-2224741</Version>
     </PackageReference>
     <PackageReference Include="Serilog">
       <Version>2.5.0</Version>

diff --git a/src/Validation.PackageSigning.ProcessSignature/Telemetry/TelemetryService.cs b/src/Validation.PackageSigning.ProcessSignature/Telemetry/TelemetryService.cs
@@ -16,6 +16,8 @@ public class TelemetryService : ITelemetryService, ISubscriptionProcessorTelemet
         private const string DurationToStripRepositorySignaturesSeconds = Prefix + "DurationToStripRepositorySignaturesSeconds";
         private const string MessageDeliveryLag = Prefix + "MessageDeliveryLag";
         private const string MessageEnqueueLag = Prefix + "MessageEnqueueLag";
+        private const string MessageHandlerDurationSeconds = Prefix + "MessageHandlerDurationSeconds";
+        private const string MessageLockLost = Prefix + "MessageLockLost";
 
         private const string PackageId = "PackageId";
         private const string NormalizedVersion = "NormalizedVersion";
@@ -26,6 +28,8 @@ public class TelemetryService : ITelemetryService, ISubscriptionProcessorTelemet
         private const string OutputCounterSignatureCount = "OutputCounterSignatureCount";
         private const string Changed = "Changed";
         private const string MessageType = "MessageType";
+        private const string CallGuid = "CallGuid";
+        private const string Handled = "Handled";
 
         private readonly ITelemetryClient _telemetryClient;
 
@@ -100,5 +104,30 @@ public void TrackEnqueueLag<TMessage>(TimeSpan enqueueLag)
                 {
                     { MessageType, typeof(TMessage).Name }
                 });
+
+        public void TrackMessageHandlerDuration<TMessage>(TimeSpan duration, Guid callGuid, bool handled)
+        {
+            _telemetryClient.TrackMetric(
+                MessageHandlerDurationSeconds,
+                duration.TotalSeconds,
+                new Dictionary<string, string>
+                {
+                    { MessageType, typeof(TMessage).Name },
+                    { CallGuid, callGuid.ToString() },
+                    { Handled, handled.ToString() }
+                });
+        }
+
+        public void TrackMessageLockLost<TMessage>(Guid callGuid)
+        {
+            _telemetryClient.TrackMetric(
+                MessageLockLost,
+                1,
+                new Dictionary<string, string>
+                {
+                    { MessageType, typeof(TMessage).Name },
+                    { CallGuid, callGuid.ToString() }
+                });
+        }
     }
 }
diff --git a/src/Validation.PackageSigning.ValidateCertificate/TelemetryService.cs b/src/Validation.PackageSigning.ValidateCertificate/TelemetryService.cs
@@ -17,13 +17,17 @@ public class TelemetryService : ITelemetryService, ISubscriptionProcessorTelemet
         private const string UnableToValidateCertificate = Prefix + "UnableToValidateCertificate";
         private const string MessageDeliveryLag = Prefix + "MessageDeliveryLag";
         private const string MessageEnqueueLag = Prefix + "MessageEnqueueLag";
+        private const string MessageHandlerDurationSeconds = Prefix + "MessageHandlerDurationSeconds";
+        private const string MessageLockLost = Prefix + "MessageLockLost";
 
         private const string PackageId = "PackageId";
         private const string PackageNormalizedVersion = "PackageNormalizedVersion";
         private const string PackageSignatureId = "PackageSignatureId";
         private const string CertificateId = "CertificateId";
         private const string CertificateThumbprint = "CertificateThumbprint";
         private const string MessageType = "MessageType";
+        private const string CallGuid = "CallGuid";
+        private const string Handled = "Handled";
 
         private readonly TelemetryClient _telemetryClient;
 
@@ -87,5 +91,30 @@ public void TrackEnqueueLag<TMessage>(TimeSpan enqueueLag)
                 {
                     { MessageType, typeof(TMessage).Name }
                 });
+
+        public void TrackMessageHandlerDuration<TMessage>(TimeSpan duration, Guid callGuid, bool handled)
+        {
+            _telemetryClient.TrackMetric(
+                MessageHandlerDurationSeconds,
+                duration.TotalSeconds,
+                new Dictionary<string, string>
+                {
+                    { MessageType, typeof(TMessage).Name },
+                    { CallGuid, callGuid.ToString() },
+                    { Handled, handled.ToString() }
+                });
+        }
+
+        public void TrackMessageLockLost<TMessage>(Guid callGuid)
+        {
+            _telemetryClient.TrackMetric(
+                MessageLockLost,
+                1,
+                new Dictionary<string, string>
+                {
+                    { MessageType, typeof(TMessage).Name },
+                    { CallGuid, callGuid.ToString() }
+                });
+        }
     }
 }