Speed up bulk insertions

src/NuGet.Services.Revalidate/Job.cs

@@ -134,11 +134,17 @@ protected override void ConfigureJobServices(IServiceCollection services, IConfi
                 return new GalleryContext(config.ConnectionString, readOnly: false);
             });
 
+            services.AddTransient(provider =>
+            {
+                return CreateSqlConnection<ValidationDbConfiguration>();
+            });
+
             // Core
             services.AddTransient<ITelemetryService, TelemetryService>();
             services.AddTransient<ITelemetryClient, TelemetryClientWrapper>();
 
             services.AddTransient<IPackageRevalidationStateService, PackageRevalidationStateService>();
+            services.AddTransient<IPackageRevalidationInserter, PackageRevalidationInserter>();
             services.AddTransient<IRevalidationJobStateService, RevalidationJobStateService>();
             services.AddTransient<IRevalidationStateService, RevalidationStateService>();
 

src/NuGet.Services.Revalidate/NuGet.Services.Revalidate.csproj

@@ -58,12 +58,14 @@
     <Compile Include="Services\HealthService.cs" />
     <Compile Include="Services\IGalleryService.cs" />
     <Compile Include="Services\IHealthService.cs" />
+    <Compile Include="Services\IPackageRevalidationInserter.cs" />
     <Compile Include="Services\IRevalidationQueue.cs" />
     <Compile Include="Services\IRevalidationJobStateService.cs" />
     <Compile Include="Services\IPackageRevalidationStateService.cs" />
     <Compile Include="Services\IRevalidationService.cs" />
     <Compile Include="Services\IRevalidationThrottler.cs" />
     <Compile Include="Services\ISingletonService.cs" />
+    <Compile Include="Services\PackageRevalidationInserter.cs" />
     <Compile Include="Services\RevalidationOperation.cs" />
     <Compile Include="Services\RevalidationQueue.cs" />
     <Compile Include="Services\RevalidationResult.cs" />

src/NuGet.Services.Revalidate/Services/IPackageRevalidationInserter.cs

@@ -0,0 +1,14 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System.Collections.Generic;
+using System.Threading.Tasks;
+using NuGet.Services.Validation;
+
+namespace NuGet.Services.Revalidate
+{
+    public interface IPackageRevalidationInserter
+    {
+        Task AddPackageRevalidationsAsync(IReadOnlyList<PackageRevalidation> revalidations);
+    }
+}

src/NuGet.Services.Revalidate/Services/PackageRevalidationInserter.cs

@@ -0,0 +1,91 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Collections.Generic;
+using System.Data;
+using System.Data.SqlClient;
+using System.Threading.Tasks;
+using Microsoft.Extensions.Logging;
+using NuGet.Services.Validation;
+
+namespace NuGet.Services.Revalidate
+{
+    public class PackageRevalidationInserter : IPackageRevalidationInserter
+    {
+        private const string TableName = "[dbo].[PackageRevalidations]";
+
+        private const string PackageIdColumn = "PackageId";
+        private const string PackageNormalizedVersionColumn = "PackageNormalizedVersion";
+        private const string EnqueuedColumn = "Enqueued";
+        private const string ValidationTrackingIdColumn = "ValidationTrackingId";
+        private const string CompletedColumn = "Completed";
+
+        private readonly SqlConnection _connection;
+        private readonly ILogger<PackageRevalidationInserter> _logger;
+
+        public PackageRevalidationInserter(SqlConnection connection, ILogger<PackageRevalidationInserter> logger)
+        {
+            _connection = connection ?? throw new ArgumentNullException(nameof(connection));
+            _logger = logger ?? throw new ArgumentNullException(nameof(logger));
+        }
+
+        public async Task AddPackageRevalidationsAsync(IReadOnlyList<PackageRevalidation> revalidations)
+        {
+            _logger.LogDebug("Persisting package revalidations to database...");
+
+            var table = PrepareTable(revalidations);
+
+            await _connection.OpenAsync();
+
+            var bulkCopy = new SqlBulkCopy(
+                _connection,
+                SqlBulkCopyOptions.TableLock | SqlBulkCopyOptions.FireTriggers | SqlBulkCopyOptions.UseInternalTransaction,
+                externalTransaction: null);
+
+            foreach (DataColumn column in table.Columns)
+            {
+                bulkCopy.ColumnMappings.Add(column.ColumnName, column.ColumnName);
+            }
+
+            bulkCopy.DestinationTableName = TableName;
+            bulkCopy.WriteToServer(table);
+
+            _connection.Close();
+
+            _logger.LogDebug("Finished persisting package revalidations to database...");
+        }
+
+        private DataTable PrepareTable(IReadOnlyList<PackageRevalidation> revalidations)
+        {
+            // Prepare the table.
+            var table = new DataTable();
+
+            table.Columns.Add(PackageIdColumn, typeof(string));
+            table.Columns.Add(PackageNormalizedVersionColumn, typeof(string));
+            table.Columns.Add(CompletedColumn, typeof(bool));
+
+            var enqueued = table.Columns.Add(EnqueuedColumn, typeof(DateTime));
+            var trackingId = table.Columns.Add(ValidationTrackingIdColumn, typeof(Guid));
+
+            enqueued.AllowDBNull = true;
+            trackingId.AllowDBNull = true;
+
+            // Populate the table.
+            foreach (var revalidation in revalidations)
+            {
+                var row = table.NewRow();
+
+                row[PackageIdColumn] = revalidation.PackageId;
+                row[PackageNormalizedVersionColumn] = revalidation.PackageNormalizedVersion;
+                row[EnqueuedColumn] = ((object)revalidation.Enqueued) ?? DBNull.Value;
+                row[ValidationTrackingIdColumn] = ((object)revalidation.ValidationTrackingId) ?? DBNull.Value;
+                row[CompletedColumn] = revalidation.Completed;
+
+                table.Rows.Add(row);
+            }
+
+            return table;
+        }
+    }
+}

src/NuGet.Services.Revalidate/Services/PackageRevalidationStateService.cs

@@ -3,6 +3,7 @@
 
 using System;
 using System.Collections.Generic;
+using System.Data;
 using System.Data.Entity;
 using System.Data.Entity.Infrastructure;
 using System.Linq;
@@ -15,36 +16,22 @@ namespace NuGet.Services.Revalidate
     public class PackageRevalidationStateService : IPackageRevalidationStateService
     {
         private readonly IValidationEntitiesContext _context;
+        private readonly IPackageRevalidationInserter _inserter;
         private readonly ILogger<PackageRevalidationStateService> _logger;
 
-        public PackageRevalidationStateService(IValidationEntitiesContext context, ILogger<PackageRevalidationStateService> logger)
+        public PackageRevalidationStateService(
+            IValidationEntitiesContext context,
+            IPackageRevalidationInserter inserter,
+            ILogger<PackageRevalidationStateService> logger)
         {
             _context = context ?? throw new ArgumentNullException(nameof(context));
+            _inserter = inserter ?? throw new ArgumentNullException(nameof(inserter));
             _logger = logger ?? throw new ArgumentNullException(nameof(logger));
         }
 
-        public async Task AddPackageRevalidationsAsync(IReadOnlyList<PackageRevalidation> revalidations)
+        public Task AddPackageRevalidationsAsync(IReadOnlyList<PackageRevalidation> revalidations)
         {
-            var validationContext = _context as ValidationEntitiesContext;
-
-            if (validationContext != null)
-            {
-                validationContext.Configuration.AutoDetectChangesEnabled = false;
-                validationContext.Configuration.ValidateOnSaveEnabled = false;
-            }
-
-            foreach (var revalidation in revalidations)
-            {
-                _context.PackageRevalidations.Add(revalidation);
-            }
-
-            await _context.SaveChangesAsync();
-
-            if (validationContext != null)
-            {
-                validationContext.Configuration.AutoDetectChangesEnabled = true;
-                validationContext.Configuration.ValidateOnSaveEnabled = true;
-            }
+            return _inserter.AddPackageRevalidationsAsync(revalidations);
         }
 
         public async Task<int> RemovePackageRevalidationsAsync(int max)

tests/NuGet.Services.Revalidate.Tests/Services/PackageRevalidationStateServiceFacts.cs

@@ -7,6 +7,7 @@
 using System.Threading.Tasks;
 using Microsoft.Extensions.Logging;
 using Moq;
+using NuGet.Jobs.Validation;
 using NuGet.Services.Validation;
 using Tests.ContextHelpers;
 using Xunit;
@@ -15,29 +16,6 @@ namespace NuGet.Services.Revalidate.Tests.Services
 {
     public class PackageRevalidationStateServiceFacts
     {
-        public class TheAddPackageRevalidationsAsyncMethod : FactsBase
-        {
-            [Fact]
-            public async Task AddsRevalidations()
-            {
-                // Arrange
-                var revalidations = new List<PackageRevalidation>
-                {
-                    new PackageRevalidation { PackageId = "A" },
-                    new PackageRevalidation { PackageId = "B" }
-                };
-
-                _context.Mock();
-
-                // Act & Assert
-                await _target.AddPackageRevalidationsAsync(revalidations);
-
-                Assert.Equal(2, _context.Object.PackageRevalidations.Count());
-
-                _context.Verify(c => c.SaveChangesAsync(), Times.Once);
-            }
-        }
-
         public class TheRemoveRevalidationsAsyncMethod : FactsBase
         {
             [Fact]
@@ -124,6 +102,7 @@ public FactsBase()
 
                 _target = new PackageRevalidationStateService(
                     _context.Object,
+                    Mock.Of<IPackageRevalidationInserter>(),
                     Mock.Of<ILogger<PackageRevalidationStateService>>());
             }
         }

tests/Validation.PackageSigning.ScanAndSign.Tests/ScanAndSignProcessorFacts.cs

@@ -311,9 +311,12 @@ public async Task EnqueuesScanAndSignEvenIfRepositorySigningIsDisabled()
                         _request.NupkgUrl,
                         _config.V3ServiceIndexUrl,
                         It.Is<List<string>>(l =>
-                            l.Count() == 2 &&
-                            l.Contains("Billy") &&
-                            l.Contains("Bob"))),
+                            // Ensure that the owners are lexicographically ordered.
+                            l.Count() == 4 &&
+                            l[0] == "Annie" &&
+                            l[1] == "Bob" &&
+                            l[2] == "zack" &&
+                            l[3] == "Zorro")),
                     Times.Once);
 
             _validatorStateServiceMock