Merge pull request #586 from NuGet/dev

[ReleasePrep][2018.10.15]RI of dev into master
NuGet · Oct 16, 2018 · 899fb8d · 899fb8d
2 parents 373495c + 6a65f27
commit 899fb8d
Show file tree

Hide file tree

Showing 182 changed files with 8,734 additions and 1,549 deletions.
diff --git a/build.ps1 b/build.ps1
@@ -9,7 +9,7 @@ param (
     [string]$SemanticVersion = '1.0.0-zlocal',
     [string]$Branch = 'zlocal',
     [string]$CommitSHA,
-    [string]$BuildBranch = 'c35bbc228717720bdbc610f3285259391635e90e'
+    [string]$BuildBranch = '80b8f1b4a1cfe57367881e59fca063866dcaaa42'
 )
 
 $msBuildVersion = 15;

diff --git a/src/Gallery.Maintenance/DeleteExpiredVerificationKeysTask.cs b/src/Gallery.Maintenance/DeleteExpiredVerificationKeysTask.cs
@@ -61,19 +61,21 @@ public override async Task RunAsync(Job job)
             if (expectedRowCount > 0)
             {
                 using (var connection = await job.OpenSqlConnectionAsync<GalleryDbConfiguration>())
+                using (var transaction = connection.BeginTransaction())
+                using (var command = connection.CreateCommand())
                 {
-                    using (var transaction = connection.BeginTransaction())
-                    {
-                        var numKeys = 0;
-                        var parameters = credentialKeys.Select(c => new SqlParameter("@Key" + numKeys++, SqlDbType.Int) { Value = c }).ToArray();
-
-                        rowCount = await connection.ExecuteAsync(
-                            string.Format(DeleteQuery, string.Join(",", parameters.Select(p => p.ParameterName))),
-                            parameters,
-                            transaction, _commandTimeout);
-
-                        transaction.Commit();
-                    }
+                    var numKeys = 0;
+                    var parameters = credentialKeys.Select(c => new SqlParameter("@Key" + numKeys++, SqlDbType.Int) { Value = c }).ToArray();
+                    command.Parameters.AddRange(parameters);
+
+                    command.CommandText = string.Format(DeleteQuery, string.Join(",", parameters.Select(p => p.ParameterName)));
+                    command.CommandType = CommandType.Text;
+                    command.CommandTimeout = (int)_commandTimeout.TotalSeconds;
+                    command.Transaction = transaction;
+
+                    rowCount = await command.ExecuteNonQueryAsync();
+
+                    transaction.Commit();
                 }
             }
 

diff --git a/src/NuGet.Jobs.Common/Extensions/DapperExtensions.cs b/src/NuGet.Jobs.Common/Extensions/DapperExtensions.cs
@@ -10,30 +10,6 @@ namespace System.Data.SqlClient
 {
     public static class DapperExtensions
     {
-        public static Task<int> ExecuteAsync(this SqlConnection connection, string sql, SqlParameter[] parameters = null, SqlTransaction transaction = null, TimeSpan? commandTimeout = null)
-        {
-            SqlCommand cmd = connection.CreateCommand();
-            cmd.CommandText = sql;
-            cmd.CommandType = CommandType.Text;
-
-            if (parameters != null)
-            {
-                cmd.Parameters.AddRange(parameters);
-            }
-
-            if (commandTimeout.HasValue)
-            {
-                cmd.CommandTimeout = (int)commandTimeout.Value.TotalSeconds;
-            }
-
-            if (transaction != null)
-            {
-                cmd.Transaction = transaction;
-            }
-
-            return cmd.ExecuteNonQueryAsync();
-        }
-
         public static async Task<IEnumerable<T>> QueryWithRetryAsync<T>(
             this SqlConnection connection,
             string sql,

diff --git a/src/NuGet.Jobs.Common/NuGet.Jobs.Common.csproj b/src/NuGet.Jobs.Common/NuGet.Jobs.Common.csproj
@@ -94,13 +94,13 @@
       <Version>2.27.0</Version>
     </PackageReference>
     <PackageReference Include="NuGet.Services.KeyVault">
-      <Version>2.28.0</Version>
+      <Version>2.29.0</Version>
     </PackageReference>
     <PackageReference Include="NuGet.Services.Logging">
       <Version>2.27.0</Version>
     </PackageReference>
     <PackageReference Include="NuGet.Services.Sql">
-      <Version>2.28.0</Version>
+      <Version>2.29.0</Version>
     </PackageReference>
     <PackageReference Include="System.Net.Http">
       <Version>4.3.3</Version>

diff --git a/src/NuGet.Services.Revalidate/NuGet.Services.Revalidate.csproj b/src/NuGet.Services.Revalidate/NuGet.Services.Revalidate.csproj
@@ -117,10 +117,10 @@
   </ItemGroup>
   <ItemGroup>
     <PackageReference Include="NuGet.Services.Status">
-      <Version>2.27.0</Version>
+      <Version>2.29.0</Version>
     </PackageReference>
     <PackageReference Include="NuGet.Services.Storage">
-      <Version>2.27.0</Version>
+      <Version>2.29.0</Version>
     </PackageReference>
   </ItemGroup>
   <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />

diff --git a/src/NuGet.Services.Validation.Orchestrator/Job.cs b/src/NuGet.Services.Validation.Orchestrator/Job.cs
@@ -599,6 +599,7 @@ private static void ConfigureOrchestratorSymbolTypes(IServiceCollection services
             services.AddTransient<IValidationSetProvider<SymbolPackage>, ValidationSetProvider<SymbolPackage>>();
             services.AddTransient<IMessageService<SymbolPackage>, SymbolsPackageMessageService>();
             services.AddTransient<IBrokeredMessageSerializer<SymbolsValidatorMessage>, SymbolsValidatorMessageSerializer>();
+            services.AddTransient<IBrokeredMessageSerializer<SymbolsIngesterMessage>, SymbolsIngesterMessageSerializer>();
             services.AddTransient<ISymbolsValidationEntitiesService, SymbolsValidationEntitiesService>();
         }
 
@@ -653,25 +654,24 @@ private static void ConfigureSymbolsIngester(ContainerBuilder builder)
                 .Keyed<ITopicClient>(SymbolsIngesterBindingKey);
 
             builder
-                .RegisterType<SymbolsMessageEnqueuer>()
+                .RegisterType<SymbolsIngesterMessageEnqueuer>()
                 .WithKeyedParameter(typeof(ITopicClient), SymbolsIngesterBindingKey)
                 .WithParameter(
                     (pi, ctx) => pi.ParameterType == typeof(TimeSpan?),
                     (pi, ctx) => ctx.Resolve<IOptionsSnapshot<SymbolsIngesterConfiguration>>().Value.MessageDelay)
-                .Keyed<ISymbolsMessageEnqueuer>(SymbolsIngesterBindingKey)
-                .As<ISymbolsMessageEnqueuer>();
+                .Keyed<ISymbolsIngesterMessageEnqueuer>(SymbolsIngesterBindingKey)
+                .As<ISymbolsIngesterMessageEnqueuer>();
 
             builder
                 .RegisterType<SymbolsIngester>()
                 .WithKeyedParameter(typeof(IValidatorStateService), SymbolsIngesterBindingKey)
-                .WithKeyedParameter(typeof(ISymbolsMessageEnqueuer), SymbolsIngesterBindingKey)
+                .WithKeyedParameter(typeof(ISymbolsIngesterMessageEnqueuer), SymbolsIngesterBindingKey)
                 .AsSelf();
         }
 
         private T GetRequiredService<T>()
         {
             return _serviceProvider.GetRequiredService<T>();
         }
-
     }
 }
diff --git a/src/NuGet.Services.Validation.Orchestrator/NuGet.Services.Validation.Orchestrator.csproj b/src/NuGet.Services.Validation.Orchestrator/NuGet.Services.Validation.Orchestrator.csproj
@@ -53,6 +53,7 @@
     <Compile Include="Error.cs" />
     <Compile Include="Services\MessageServiceConfiguration.cs" />
     <Compile Include="Services\SymbolsMessageService.cs" />
+    <Compile Include="Symbols\ISymbolsIngesterMessageEnqueuer.cs" />
     <Compile Include="Symbols\SymbolScanOnlyConfiguration.cs" />
     <Compile Include="Symbols\SymbolCriteriaEvaluator.cs" />
     <Compile Include="Symbols\SymbolScanValidator.cs" />
@@ -62,6 +63,7 @@
     <Compile Include="IPackageStatusProcessor.cs" />
     <Compile Include="Services\SymbolEntityService.cs" />
     <Compile Include="Symbols\ISymbolsMessageEnqueuer.cs" />
+    <Compile Include="Symbols\SymbolsIngesterMessageEnqueuer.cs" />
     <Compile Include="Symbols\SymbolsMessageEnqueuer.cs" />
     <Compile Include="Symbols\SymbolsIngesterConfiguration.cs" />
     <Compile Include="Symbols\SymbolsIngester.cs" />

diff --git a/src/NuGet.Services.Validation.Orchestrator/Symbols/ISymbolsIngesterMessageEnqueuer.cs b/src/NuGet.Services.Validation.Orchestrator/Symbols/ISymbolsIngesterMessageEnqueuer.cs
@@ -0,0 +1,18 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System.Threading.Tasks;
+using NuGet.Jobs.Validation.Symbols.Core;
+
+namespace NuGet.Services.Validation.Symbols
+{
+    public interface ISymbolsIngesterMessageEnqueuer
+    {
+        /// <summary>
+        /// Enqueues a message to one of the topics used by the Symbol Ingester.
+        /// </summary>
+        /// <param name="request">The validation request.</param>
+        /// <returns>A <see cref="Task"/> that will be completed when the execution is completed.</returns>
+        Task<SymbolsIngesterMessage> EnqueueSymbolsIngestionMessageAsync(IValidationRequest request);
+    }
+}
diff --git a/src/NuGet.Services.Validation.Orchestrator/Symbols/SymbolsIngester.cs b/src/NuGet.Services.Validation.Orchestrator/Symbols/SymbolsIngester.cs
@@ -15,13 +15,13 @@ namespace NuGet.Services.Validation.Symbols
     public class SymbolsIngester : BaseValidator, IValidator
     {
         private readonly ISymbolsValidationEntitiesService _symbolsValidationEntitiesService;
-        private readonly ISymbolsMessageEnqueuer _symbolMessageEnqueuer;
+        private readonly ISymbolsIngesterMessageEnqueuer _symbolMessageEnqueuer;
         private readonly ITelemetryService _telemetryService;
         private readonly ILogger<SymbolsIngester> _logger;
 
         public SymbolsIngester(
             ISymbolsValidationEntitiesService symbolsValidationEntitiesService,
-            ISymbolsMessageEnqueuer symbolMessageEnqueuer,
+            ISymbolsIngesterMessageEnqueuer symbolMessageEnqueuer,
             ITelemetryService telemetryService,
             ILogger<SymbolsIngester> logger)
         {
@@ -40,11 +40,12 @@ public async Task<IValidationResult> GetResultAsync(IValidationRequest request)
 
             var result = SymbolsValidationEntitiesService.ConvertToIValidationResult(await _symbolsValidationEntitiesService.GetSymbolsServerRequestAsync(request));
             _logger.LogInformation(
-                    "Symbols status {Status} for PackageId: {PackageId}, PackageNormalizedVersion {PackageNormalizedVersion}, SymbolsPackageKey {SymbolsPackageKey} ",
+                    "Symbols status {Status} for PackageId: {PackageId}, PackageNormalizedVersion {PackageNormalizedVersion}, SymbolsPackageKey {SymbolsPackageKey} ValidationId {ValidationId}",
                     result.Status,
                     request.PackageId,
                     request.PackageVersion,
-                    request.PackageKey);
+                    request.PackageKey,
+                    request.ValidationId);
 
             return result;
         }
@@ -78,9 +79,9 @@ public async Task<IValidationResult> StartAsync(IValidationRequest request)
             }
 
             _telemetryService.TrackSymbolsMessageEnqueued(ValidatorName.SymbolsIngester, request.ValidationId);
-            await _symbolMessageEnqueuer.EnqueueSymbolsValidationMessageAsync(request);
+            var message = await _symbolMessageEnqueuer.EnqueueSymbolsIngestionMessageAsync(request);
 
-            var newSymbolsRequest = SymbolsValidationEntitiesService.CreateFromValidationRequest(request, SymbolsPackageIngestRequestStatus.Ingesting);
+            var newSymbolsRequest = SymbolsValidationEntitiesService.CreateFromValidationRequest(request, SymbolsPackageIngestRequestStatus.Ingesting, message.RequestName);
             var savedSymbolRequest = await _symbolsValidationEntitiesService.AddSymbolsServerRequestAsync(newSymbolsRequest);
 
             if(savedSymbolRequest.RequestStatusKey != SymbolsPackageIngestRequestStatus.Ingesting)

diff --git a/src/NuGet.Services.Validation.Orchestrator/Symbols/SymbolsIngesterMessageEnqueuer.cs b/src/NuGet.Services.Validation.Orchestrator/Symbols/SymbolsIngesterMessageEnqueuer.cs
@@ -0,0 +1,45 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Threading.Tasks;
+using Microsoft.Extensions.Options;
+using NuGet.Jobs.Validation.Symbols.Core;
+using NuGet.Services.ServiceBus;
+
+namespace NuGet.Services.Validation.Symbols
+{
+    public class SymbolsIngesterMessageEnqueuer : ISymbolsIngesterMessageEnqueuer
+    {
+        private readonly ITopicClient _topicClient;
+        private readonly TimeSpan? _messageDelay;
+        private readonly IBrokeredMessageSerializer<SymbolsIngesterMessage> _serializer;
+
+        public SymbolsIngesterMessageEnqueuer(
+            ITopicClient topicClient,
+            IBrokeredMessageSerializer<SymbolsIngesterMessage> serializer,
+            TimeSpan? messageDelay)
+        {
+            _topicClient = topicClient ?? throw new ArgumentNullException(nameof(topicClient));
+            _serializer = serializer ?? throw new ArgumentNullException(nameof(serializer));
+            _messageDelay = messageDelay;
+        }
+
+        public async Task<SymbolsIngesterMessage> EnqueueSymbolsIngestionMessageAsync(IValidationRequest request)
+        {
+            var message = new SymbolsIngesterMessage(validationId: request.ValidationId, 
+                symbolPackageKey: request.PackageKey,
+                packageId: request.PackageId,
+                packageNormalizedVersion: request.PackageVersion,
+                snupkgUrl: request.NupkgUrl,
+                requestName : SymbolsValidationEntitiesService.CreateSymbolServerRequestNameFromValidationRequest(request));
+            var brokeredMessage = _serializer.Serialize(message);
+
+            var visibleAt = DateTimeOffset.UtcNow + (_messageDelay ?? TimeSpan.Zero);
+            brokeredMessage.ScheduledEnqueueTimeUtc = visibleAt;
+
+            await _topicClient.SendAsync(brokeredMessage);
+            return message;
+        }
+    }
+}
diff --git a/src/NuGet.SupportRequests.Notifications/Tasks/SupportRequestsNotificationScheduledTask.cs b/src/NuGet.SupportRequests.Notifications/Tasks/SupportRequestsNotificationScheduledTask.cs
@@ -15,8 +15,6 @@ internal abstract class SupportRequestsNotificationScheduledTask<TNotification>
       : IScheduledTask
         where TNotification : INotification
     {
-        private InitializationConfiguration _configuration;
-
         private readonly SupportRequestRepository _supportRequestRepository;
         private readonly MessagingService _messagingService;
 

diff --git a/src/NuGetCDNRedirect/GlobalSuppressions.cs b/src/NuGetCDNRedirect/GlobalSuppressions.cs
@@ -0,0 +1,6 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System.Diagnostics.CodeAnalysis;
+
+[module: SuppressMessage("Microsoft.Security.Web.Configuration", "CA3103:EnableFormsRequireSSL", Justification = "Forms authentication is not used, and this project is obsolete (Engineering#1737).")]
diff --git a/src/NuGetCDNRedirect/NuGetCDNRedirect.csproj b/src/NuGetCDNRedirect/NuGetCDNRedirect.csproj
@@ -183,6 +183,7 @@
     <Compile Include="Global.asax.cs">
       <DependentUpon>Global.asax</DependentUpon>
     </Compile>
+    <Compile Include="GlobalSuppressions.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
     <Compile Include="Properties\AssemblyInfo.*.cs" />
   </ItemGroup>

diff --git a/src/Search.GenerateAuxiliaryData/NestedJArrayExporter.cs b/src/Search.GenerateAuxiliaryData/NestedJArrayExporter.cs
@@ -3,7 +3,6 @@
 
 using System;
 using System.Collections.Generic;
-using System.Data;
 using System.Data.SqlClient;
 using System.Linq;
 using System.Threading.Tasks;
@@ -34,8 +33,7 @@ public NestedJArrayExporter(
 
         protected override JContainer GetResultOfQuery(SqlConnection connection)
         {
-            var command = new SqlCommand(GetEmbeddedSqlScript(SqlScript), connection);
-            command.CommandType = CommandType.Text;
+            var command = GetEmbeddedSqlCommand(connection, SqlScript);
 
             return SqlDataReaderToNestedJArrays(command.ExecuteReader(), Col0, Col1);
         }

diff --git a/src/Search.GenerateAuxiliaryData/RankingsExporter.cs b/src/Search.GenerateAuxiliaryData/RankingsExporter.cs
@@ -32,8 +32,7 @@ public RankingsExporter(
 
         protected override JContainer GetResultOfQuery(SqlConnection connection)
         {
-            var rankingsTotalCommand = new SqlCommand(GetEmbeddedSqlScript(_rankingsTotalScript), connection);
-            rankingsTotalCommand.CommandType = CommandType.Text;
+            var rankingsTotalCommand = GetEmbeddedSqlCommand(connection, _rankingsTotalScript);
             rankingsTotalCommand.Parameters.AddWithValue(_rankingCountParameterName, _rankingCount);
             rankingsTotalCommand.CommandTimeout = 60;
 

diff --git a/src/Search.GenerateAuxiliaryData/SqlExporter.cs b/src/Search.GenerateAuxiliaryData/SqlExporter.cs
@@ -5,6 +5,7 @@
 using System.Collections.Generic;
 using System.Data;
 using System.Data.SqlClient;
+using System.Diagnostics.CodeAnalysis;
 using System.IO;
 using System.Reflection;
 using System.Threading.Tasks;
@@ -34,10 +35,18 @@ public SqlExporter(
             OpenSqlConnectionAsync = openSqlConnectionAsync;
         }
 
-        protected static string GetEmbeddedSqlScript(string resourceName)
+        [SuppressMessage("Microsoft.Security", "CA2100", Justification = "Query string comes from embedded resource, not user input.")]
+        protected static SqlCommand GetEmbeddedSqlCommand(SqlConnection connection, string resourceName)
         {
-            var stream = _executingAssembly.GetManifestResourceStream(_assemblyName + "." + resourceName);
-            return new StreamReader(stream).ReadToEnd();
+            using (var reader = new StreamReader(_executingAssembly.GetManifestResourceStream(_assemblyName + "." + resourceName)))
+            {
+                var commandText = reader.ReadToEnd();
+
+                return new SqlCommand(commandText, connection)
+                {
+                    CommandType = CommandType.Text
+                };
+            }
         }
 
         public override async Task ExportAsync()

diff --git a/src/Search.GenerateAuxiliaryData/VerifiedPackagesExporter.cs b/src/Search.GenerateAuxiliaryData/VerifiedPackagesExporter.cs
@@ -29,8 +29,7 @@ public VerifiedPackagesExporter(
 
         protected override JContainer GetResultOfQuery(SqlConnection connection)
         {
-            var verifiedPackagesCommand = new SqlCommand(GetEmbeddedSqlScript(_verifiedPackagesScript), connection);
-            verifiedPackagesCommand.CommandType = CommandType.Text;
+            var verifiedPackagesCommand = GetEmbeddedSqlCommand(connection, _verifiedPackagesScript);
             verifiedPackagesCommand.CommandTimeout = 60;
 
             SqlDataReader reader = null;

diff --git a/src/Stats.AggregateCdnDownloadsInGallery/AggregateCdnDownloadsJob.cs b/src/Stats.AggregateCdnDownloadsInGallery/AggregateCdnDownloadsJob.cs
@@ -148,7 +148,14 @@ private async Task ProcessBatch(List<IPackageIdGroup> batch, SqlConnection desti
         {
             // Create a temporary table
             Logger.LogDebug("Creating temporary table...");
-            await destinationDatabase.ExecuteAsync(_createTempTable);
+
+            using (var cmd = destinationDatabase.CreateCommand())
+            {
+                cmd.CommandText = _createTempTable;
+                cmd.CommandType = CommandType.Text;
+
+                await cmd.ExecuteNonQueryAsync();
+            }
 
             // Load temporary table
             var aggregateCdnDownloadsInGalleryTable = new DataTable();
@@ -206,8 +213,14 @@ private async Task ProcessBatch(List<IPackageIdGroup> batch, SqlConnection desti
             Logger.LogInformation("Updating destination database Download Counts... ({RecordCount} package registrations to process).", batch.Count());
             stopwatch.Restart();
 
-            await destinationDatabase.ExecuteAsync(_updateFromTempTable,
-                commandTimeout: TimeSpan.FromMinutes(30));
+            using (var cmd = destinationDatabase.CreateCommand())
+            {
+                cmd.CommandText = _updateFromTempTable;
+                cmd.CommandType = CommandType.Text;
+                cmd.CommandTimeout = (int)TimeSpan.FromMinutes(30).TotalSeconds;
+
+                await cmd.ExecuteNonQueryAsync();
+            }
 
             Logger.LogInformation(
                 "Updated destination database Download Counts (took {DurationSeconds} seconds).",

diff --git a/src/Stats.RefreshClientDimension/Stats.RefreshClientDimension.csproj b/src/Stats.RefreshClientDimension/Stats.RefreshClientDimension.csproj
@@ -84,7 +84,7 @@
       <Version>9.0.1</Version>
     </PackageReference>
     <PackageReference Include="NuGet.Services.Sql">
-      <Version>2.28.0</Version>
+      <Version>2.29.0</Version>
     </PackageReference>
     <PackageReference Include="UAParser">
       <Version>1.2.0</Version>