From 90705f393c06cfc1229dcc86890d5ce534b80be8 Mon Sep 17 00:00:00 2001 From: Loic Sharma Date: Wed, 8 Aug 2018 18:31:11 -0700 Subject: [PATCH] Speed up bulk insertions --- src/NuGet.Services.Revalidate/Job.cs | 6 ++ .../NuGet.Services.Revalidate.csproj | 2 + .../Services/IPackageRevalidationInserter.cs | 14 +++ .../Services/PackageRevalidationInserter.cs | 91 +++++++++++++++++++ .../PackageRevalidationStateService.cs | 31 ++----- .../PackageRevalidationStateServiceFacts.cs | 25 +---- .../ScanAndSignProcessorFacts.cs | 9 +- 7 files changed, 130 insertions(+), 48 deletions(-) create mode 100644 src/NuGet.Services.Revalidate/Services/IPackageRevalidationInserter.cs create mode 100644 src/NuGet.Services.Revalidate/Services/PackageRevalidationInserter.cs diff --git a/src/NuGet.Services.Revalidate/Job.cs b/src/NuGet.Services.Revalidate/Job.cs index 43f15ae56..e6e2cd79f 100644 --- a/src/NuGet.Services.Revalidate/Job.cs +++ b/src/NuGet.Services.Revalidate/Job.cs @@ -134,11 +134,17 @@ protected override void ConfigureJobServices(IServiceCollection services, IConfi return new GalleryContext(config.ConnectionString, readOnly: false); }); + services.AddTransient(provider => + { + return CreateSqlConnection(); + }); + // Core services.AddTransient(); services.AddTransient(); services.AddTransient(); + services.AddTransient(); services.AddTransient(); services.AddTransient(); diff --git a/src/NuGet.Services.Revalidate/NuGet.Services.Revalidate.csproj b/src/NuGet.Services.Revalidate/NuGet.Services.Revalidate.csproj index 15bd808c4..7c495f9b3 100644 --- a/src/NuGet.Services.Revalidate/NuGet.Services.Revalidate.csproj +++ b/src/NuGet.Services.Revalidate/NuGet.Services.Revalidate.csproj @@ -58,12 +58,14 @@ + + diff --git a/src/NuGet.Services.Revalidate/Services/IPackageRevalidationInserter.cs b/src/NuGet.Services.Revalidate/Services/IPackageRevalidationInserter.cs new file mode 100644 index 000000000..f5d52aa38 --- /dev/null +++ b/src/NuGet.Services.Revalidate/Services/IPackageRevalidationInserter.cs @@ -0,0 +1,14 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Generic; +using System.Threading.Tasks; +using NuGet.Services.Validation; + +namespace NuGet.Services.Revalidate +{ + public interface IPackageRevalidationInserter + { + Task AddPackageRevalidationsAsync(IReadOnlyList revalidations); + } +} diff --git a/src/NuGet.Services.Revalidate/Services/PackageRevalidationInserter.cs b/src/NuGet.Services.Revalidate/Services/PackageRevalidationInserter.cs new file mode 100644 index 000000000..f2a3c3b48 --- /dev/null +++ b/src/NuGet.Services.Revalidate/Services/PackageRevalidationInserter.cs @@ -0,0 +1,91 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Data; +using System.Data.SqlClient; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using NuGet.Services.Validation; + +namespace NuGet.Services.Revalidate +{ + public class PackageRevalidationInserter : IPackageRevalidationInserter + { + private const string TableName = "[dbo].[PackageRevalidations]"; + + private const string PackageIdColumn = "PackageId"; + private const string PackageNormalizedVersionColumn = "PackageNormalizedVersion"; + private const string EnqueuedColumn = "Enqueued"; + private const string ValidationTrackingIdColumn = "ValidationTrackingId"; + private const string CompletedColumn = "Completed"; + + private readonly SqlConnection _connection; + private readonly ILogger _logger; + + public PackageRevalidationInserter(SqlConnection connection, ILogger logger) + { + _connection = connection ?? throw new ArgumentNullException(nameof(connection)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task AddPackageRevalidationsAsync(IReadOnlyList revalidations) + { + _logger.LogDebug("Persisting package revalidations to database..."); + + var table = PrepareTable(revalidations); + + await _connection.OpenAsync(); + + var bulkCopy = new SqlBulkCopy( + _connection, + SqlBulkCopyOptions.TableLock | SqlBulkCopyOptions.FireTriggers | SqlBulkCopyOptions.UseInternalTransaction, + externalTransaction: null); + + foreach (DataColumn column in table.Columns) + { + bulkCopy.ColumnMappings.Add(column.ColumnName, column.ColumnName); + } + + bulkCopy.DestinationTableName = TableName; + bulkCopy.WriteToServer(table); + + _connection.Close(); + + _logger.LogDebug("Finished persisting package revalidations to database..."); + } + + private DataTable PrepareTable(IReadOnlyList revalidations) + { + // Prepare the table. + var table = new DataTable(); + + table.Columns.Add(PackageIdColumn, typeof(string)); + table.Columns.Add(PackageNormalizedVersionColumn, typeof(string)); + table.Columns.Add(CompletedColumn, typeof(bool)); + + var enqueued = table.Columns.Add(EnqueuedColumn, typeof(DateTime)); + var trackingId = table.Columns.Add(ValidationTrackingIdColumn, typeof(Guid)); + + enqueued.AllowDBNull = true; + trackingId.AllowDBNull = true; + + // Populate the table. + foreach (var revalidation in revalidations) + { + var row = table.NewRow(); + + row[PackageIdColumn] = revalidation.PackageId; + row[PackageNormalizedVersionColumn] = revalidation.PackageNormalizedVersion; + row[EnqueuedColumn] = ((object)revalidation.Enqueued) ?? DBNull.Value; + row[ValidationTrackingIdColumn] = ((object)revalidation.ValidationTrackingId) ?? DBNull.Value; + row[CompletedColumn] = revalidation.Completed; + + table.Rows.Add(row); + } + + return table; + } + } +} diff --git a/src/NuGet.Services.Revalidate/Services/PackageRevalidationStateService.cs b/src/NuGet.Services.Revalidate/Services/PackageRevalidationStateService.cs index 8f7a79c1a..fb90b8497 100644 --- a/src/NuGet.Services.Revalidate/Services/PackageRevalidationStateService.cs +++ b/src/NuGet.Services.Revalidate/Services/PackageRevalidationStateService.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; +using System.Data; using System.Data.Entity; using System.Data.Entity.Infrastructure; using System.Linq; @@ -15,36 +16,22 @@ namespace NuGet.Services.Revalidate public class PackageRevalidationStateService : IPackageRevalidationStateService { private readonly IValidationEntitiesContext _context; + private readonly IPackageRevalidationInserter _inserter; private readonly ILogger _logger; - public PackageRevalidationStateService(IValidationEntitiesContext context, ILogger logger) + public PackageRevalidationStateService( + IValidationEntitiesContext context, + IPackageRevalidationInserter inserter, + ILogger logger) { _context = context ?? throw new ArgumentNullException(nameof(context)); + _inserter = inserter ?? throw new ArgumentNullException(nameof(inserter)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); } - public async Task AddPackageRevalidationsAsync(IReadOnlyList revalidations) + public Task AddPackageRevalidationsAsync(IReadOnlyList revalidations) { - var validationContext = _context as ValidationEntitiesContext; - - if (validationContext != null) - { - validationContext.Configuration.AutoDetectChangesEnabled = false; - validationContext.Configuration.ValidateOnSaveEnabled = false; - } - - foreach (var revalidation in revalidations) - { - _context.PackageRevalidations.Add(revalidation); - } - - await _context.SaveChangesAsync(); - - if (validationContext != null) - { - validationContext.Configuration.AutoDetectChangesEnabled = true; - validationContext.Configuration.ValidateOnSaveEnabled = true; - } + return _inserter.AddPackageRevalidationsAsync(revalidations); } public async Task RemovePackageRevalidationsAsync(int max) diff --git a/tests/NuGet.Services.Revalidate.Tests/Services/PackageRevalidationStateServiceFacts.cs b/tests/NuGet.Services.Revalidate.Tests/Services/PackageRevalidationStateServiceFacts.cs index 69e976fb4..dd2ac4e9e 100644 --- a/tests/NuGet.Services.Revalidate.Tests/Services/PackageRevalidationStateServiceFacts.cs +++ b/tests/NuGet.Services.Revalidate.Tests/Services/PackageRevalidationStateServiceFacts.cs @@ -7,6 +7,7 @@ using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Moq; +using NuGet.Jobs.Validation; using NuGet.Services.Validation; using Tests.ContextHelpers; using Xunit; @@ -15,29 +16,6 @@ namespace NuGet.Services.Revalidate.Tests.Services { public class PackageRevalidationStateServiceFacts { - public class TheAddPackageRevalidationsAsyncMethod : FactsBase - { - [Fact] - public async Task AddsRevalidations() - { - // Arrange - var revalidations = new List - { - new PackageRevalidation { PackageId = "A" }, - new PackageRevalidation { PackageId = "B" } - }; - - _context.Mock(); - - // Act & Assert - await _target.AddPackageRevalidationsAsync(revalidations); - - Assert.Equal(2, _context.Object.PackageRevalidations.Count()); - - _context.Verify(c => c.SaveChangesAsync(), Times.Once); - } - } - public class TheRemoveRevalidationsAsyncMethod : FactsBase { [Fact] @@ -124,6 +102,7 @@ public FactsBase() _target = new PackageRevalidationStateService( _context.Object, + Mock.Of(), Mock.Of>()); } } diff --git a/tests/Validation.PackageSigning.ScanAndSign.Tests/ScanAndSignProcessorFacts.cs b/tests/Validation.PackageSigning.ScanAndSign.Tests/ScanAndSignProcessorFacts.cs index 0c3bdd2f0..c5682c7af 100644 --- a/tests/Validation.PackageSigning.ScanAndSign.Tests/ScanAndSignProcessorFacts.cs +++ b/tests/Validation.PackageSigning.ScanAndSign.Tests/ScanAndSignProcessorFacts.cs @@ -311,9 +311,12 @@ public async Task EnqueuesScanAndSignEvenIfRepositorySigningIsDisabled() _request.NupkgUrl, _config.V3ServiceIndexUrl, It.Is>(l => - l.Count() == 2 && - l.Contains("Billy") && - l.Contains("Bob"))), + // Ensure that the owners are lexicographically ordered. + l.Count() == 4 && + l[0] == "Annie" && + l[1] == "Bob" && + l[2] == "zack" && + l[3] == "Zorro")), Times.Once); _validatorStateServiceMock