From 182b6555e785c5b1e966c03ad34c63cb92e89210 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Sharma?= Date: Tue, 24 Jul 2018 09:08:54 -0700 Subject: [PATCH 1/3] [Revalidation] Add Package Revalidation State (#481) This change adds the state used to track whether the job has been initialized or killswitched. The state is also used to track the desired event rate, which is used to speed up the job up over time. This change does not include health checks and will not slow down revalidations if the ingestion pipeline is unhealthy. This will be addressed in follow-up changes. Addresses https://github.com/NuGet/Engineering/issues/1440 Partially addresses https://github.com/nuget/engineering/issues/1441 --- .../NuGet.Jobs.Common.csproj | 6 +- .../RevalidationConfiguration.cs | 13 + .../Initialization/InitializationManager.cs | 57 ++-- src/NuGet.Services.Revalidate/Job.cs | 4 +- .../NuGet.Services.Revalidate.csproj | 12 +- ...cs => IPackageRevalidationStateService.cs} | 20 +- .../Services/IRevalidationJobStateService.cs | 50 ++++ .../Services/IRevalidationThrottler.cs | 13 - .../Services/ITelemetryService.cs | 4 +- ....cs => PackageRevalidationStateService.cs} | 25 +- .../Services/RevalidationJobStateService.cs | 124 ++++++++ .../Services/RevalidationOperation.cs | 13 + .../Services/RevalidationService.cs | 48 ++- .../Services/RevalidationThrottler.cs | 56 ++-- .../Services/TelemetryService.cs | 13 +- .../Settings/dev.json | 12 + .../Settings/int.json | 7 + .../Settings/prod.json | 7 + src/NuGet.Services.Revalidate/readme.md | 41 +++ .../Monitoring.PackageLag.csproj | 6 +- .../Validation.Common.Job.csproj | 18 +- .../Validation.Common.Job.nuspec | 14 +- .../Validation.Common.csproj | 4 +- ...ion.PackageSigning.ProcessSignature.csproj | 2 +- .../Validation.ScanAndSign.Core.csproj | 2 +- .../Initializer/InitializationManagerFacts.cs | 87 +++++- .../NuGet.Services.Revalidate.Tests.csproj | 4 +- .../PackageRevalidationStateServiceFacts.cs} | 34 ++- .../RevalidationJobStateServiceFacts.cs | 273 ++++++++++++++++++ .../Services/RevalidationServiceFacts.cs | 139 +++++++-- .../Services/RevalidationThrottlerFacts.cs | 64 ++++ 31 files changed, 1018 insertions(+), 154 deletions(-) rename src/NuGet.Services.Revalidate/Services/{IRevalidationStateService.cs => IPackageRevalidationStateService.cs} (73%) create mode 100644 src/NuGet.Services.Revalidate/Services/IRevalidationJobStateService.cs rename src/NuGet.Services.Revalidate/Services/{RevalidationStateService.cs => PackageRevalidationStateService.cs} (78%) create mode 100644 src/NuGet.Services.Revalidate/Services/RevalidationJobStateService.cs create mode 100644 src/NuGet.Services.Revalidate/Services/RevalidationOperation.cs create mode 100644 src/NuGet.Services.Revalidate/readme.md rename tests/NuGet.Services.Revalidate.Tests/{RevalidationStateServiceFacts.cs => Services/PackageRevalidationStateServiceFacts.cs} (69%) create mode 100644 tests/NuGet.Services.Revalidate.Tests/Services/RevalidationJobStateServiceFacts.cs create mode 100644 tests/NuGet.Services.Revalidate.Tests/Services/RevalidationThrottlerFacts.cs diff --git a/src/NuGet.Jobs.Common/NuGet.Jobs.Common.csproj b/src/NuGet.Jobs.Common/NuGet.Jobs.Common.csproj index 4e1ca11ca..2dd0251bc 100644 --- a/src/NuGet.Jobs.Common/NuGet.Jobs.Common.csproj +++ b/src/NuGet.Jobs.Common/NuGet.Jobs.Common.csproj @@ -74,13 +74,13 @@ 1.50.2 - 2.25.0 + 2.27.0 - 2.26.0-master-33196 + 2.27.0 - 2.25.0 + 2.27.0 4.3.3 diff --git a/src/NuGet.Services.Revalidate/Configuration/RevalidationConfiguration.cs b/src/NuGet.Services.Revalidate/Configuration/RevalidationConfiguration.cs index e782f79de..7e47c419c 100644 --- a/src/NuGet.Services.Revalidate/Configuration/RevalidationConfiguration.cs +++ b/src/NuGet.Services.Revalidate/Configuration/RevalidationConfiguration.cs @@ -7,6 +7,19 @@ namespace NuGet.Services.Revalidate { public class RevalidationConfiguration { + /// + /// The lower limit for the desired package event rate (includes package pushes, lists, unlists, and revalidations). + /// If the ingestion pipeline remains healthy, the job will increase its rate over time. If the ingestion pipeline becomes + /// unhealthy, the job will reset its rate to this value. + /// + public int MinPackageEventRate { get; set; } + + /// + /// The revalidation job will speed up over time. This is the upper limit for the desired package event + /// rate (includes package pushes, lists, unlists, and revalidations). + /// + public int MaxPackageEventRate { get; set; } + /// /// The time before the revalidation job restarts itself. /// diff --git a/src/NuGet.Services.Revalidate/Initialization/InitializationManager.cs b/src/NuGet.Services.Revalidate/Initialization/InitializationManager.cs index b4cd2ad75..0a818cb88 100644 --- a/src/NuGet.Services.Revalidate/Initialization/InitializationManager.cs +++ b/src/NuGet.Services.Revalidate/Initialization/InitializationManager.cs @@ -15,20 +15,21 @@ public class InitializationManager { private static int BatchSize = 1000; - private readonly IRevalidationStateService _revalidationState; + private readonly IRevalidationJobStateService _jobState; + private readonly IPackageRevalidationStateService _packageState; private readonly IPackageFinder _packageFinder; private readonly InitializationConfiguration _config; private readonly ILogger _logger; public InitializationManager( - IRevalidationStateService revalidationState, + IRevalidationJobStateService jobState, + IPackageRevalidationStateService packageState, IPackageFinder packageFinder, InitializationConfiguration config, ILogger logger) { - // TODO: Accept service for settings (IsInitialized, etc...) - // See: https://github.com/NuGet/Engineering/issues/1440 - _revalidationState = revalidationState ?? throw new ArgumentNullException(nameof(revalidationState)); + _jobState = jobState ?? throw new ArgumentNullException(nameof(jobState)); + _packageState = packageState ?? throw new ArgumentNullException(nameof(packageState)); _packageFinder = packageFinder ?? throw new ArgumentNullException(nameof(packageFinder)); _config = config ?? throw new ArgumentNullException(nameof(config)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); @@ -36,8 +37,13 @@ public InitializationManager( public async Task InitializeAsync() { - // TODO: Check "IsInitialized" setting. If true, error! - // See: https://github.com/NuGet/Engineering/issues/1440 + if (await _jobState.IsInitializedAsync()) + { + _logger.LogError("Attempted to initialize the revalidation job when it is already initialized!"); + + throw new InvalidOperationException("Attempted to initialize the revalidation job when it is already initialized!"); + } + await ClearPackageRevalidationStateAsync(); // Find packages owned by Microsoft or preinstalled by Visual Studio. @@ -66,16 +72,20 @@ public async Task InitializeAsync() await InitializePackageSetAsync(PackageFinder.DependencySetName, dependencyPackages); await InitializePackageSetAsync(PackageFinder.RemainingSetName, remainingPackages); - // TODO: Set "IsInitialized" setting to true - // See: https://github.com/NuGet/Engineering/issues/1440 + await _jobState.MarkAsInitializedAsync(); } public async Task VerifyInitializationAsync() { - // TODO: Check "IsInitialized" setting. If false, error! - // See: https://github.com/NuGet/Engineering/issues/1440 + if (!await _jobState.IsInitializedAsync()) + { + _logger.LogError("Expected revalidation state to be initialized"); + + throw new Exception("Expected revalidation state to be initialized"); + } + var expectedCount = _packageFinder.AppropriatePackageCount(); - var actualCount = await _revalidationState.PackageRevalidationCountAsync(); + var actualCount = await _packageState.PackageRevalidationCountAsync(); if (actualCount != expectedCount) { @@ -93,7 +103,7 @@ private async Task ClearPackageRevalidationStateAsync() do { - removedRevalidations = await _revalidationState.RemoveRevalidationsAsync(BatchSize); + removedRevalidations = await _packageState.RemovePackageRevalidationsAsync(BatchSize); if (removedRevalidations > 0) { @@ -120,9 +130,19 @@ private async Task InitializePackageSetAsync(string setName, HashSet packag for (var chunkIndex = 0; chunkIndex < chunks.Count; chunkIndex++) { - // TODO: Check the kill switch - // See https://github.com/NuGet/Engineering/issues/1440 - _logger.LogInformation("Initializing chunk {Chunk} of {Chunks} for package set {SetName}...", + while (await _jobState.IsKillswitchActiveAsync()) + { + _logger.LogInformation( + "Delaying initialization of chunk {Chunk} of {Chunks} for package set {SetName} due to active killswitch", + chunkIndex + 1, + chunks.Count, + setName); + + await Task.Delay(_config.SleepDurationBetweenBatches); + } + + _logger.LogInformation( + "Initializing chunk {Chunk} of {Chunks} for package set {SetName}...", chunkIndex + 1, chunks.Count, setName); @@ -132,7 +152,8 @@ private async Task InitializePackageSetAsync(string setName, HashSet packag await InitializeRevalidationsAsync(chunk, versions); - _logger.LogInformation("Initialized chunk {Chunk} of {Chunks} for package set {SetName}", + _logger.LogInformation( + "Initialized chunk {Chunk} of {Chunks} for package set {SetName}", chunkIndex + 1, chunks.Count, setName); @@ -184,7 +205,7 @@ private async Task InitializeRevalidationsAsync( } } - await _revalidationState.AddPackageRevalidationsAsync(revalidations); + await _packageState.AddPackageRevalidationsAsync(revalidations); } } } diff --git a/src/NuGet.Services.Revalidate/Job.cs b/src/NuGet.Services.Revalidate/Job.cs index 2590ba358..5efb0e7bc 100644 --- a/src/NuGet.Services.Revalidate/Job.cs +++ b/src/NuGet.Services.Revalidate/Job.cs @@ -109,7 +109,9 @@ protected override void ConfigureJobServices(IServiceCollection services, IConfi services.AddTransient(); services.AddTransient(); - services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); // Initialization services.AddTransient(); diff --git a/src/NuGet.Services.Revalidate/NuGet.Services.Revalidate.csproj b/src/NuGet.Services.Revalidate/NuGet.Services.Revalidate.csproj index 20c23ccfc..56c1b21f8 100644 --- a/src/NuGet.Services.Revalidate/NuGet.Services.Revalidate.csproj +++ b/src/NuGet.Services.Revalidate/NuGet.Services.Revalidate.csproj @@ -54,14 +54,17 @@ - + + + - + + @@ -99,6 +102,11 @@ Validation.PackageSigning.Core + + + 2.27.0 + + ..\..\build diff --git a/src/NuGet.Services.Revalidate/Services/IRevalidationStateService.cs b/src/NuGet.Services.Revalidate/Services/IPackageRevalidationStateService.cs similarity index 73% rename from src/NuGet.Services.Revalidate/Services/IRevalidationStateService.cs rename to src/NuGet.Services.Revalidate/Services/IPackageRevalidationStateService.cs index c0180cff6..f72e09ecc 100644 --- a/src/NuGet.Services.Revalidate/Services/IRevalidationStateService.cs +++ b/src/NuGet.Services.Revalidate/Services/IPackageRevalidationStateService.cs @@ -7,14 +7,8 @@ namespace NuGet.Services.Revalidate { - public interface IRevalidationStateService + public interface IPackageRevalidationStateService { - /// - /// Check whether the killswitch has been activated. If it has, all revalidation operations should be halted. - /// - /// Whether the killswitch has been activated. - Task IsKillswitchActiveAsync(); - /// /// Add the new revalidations to the database. /// @@ -22,10 +16,10 @@ public interface IRevalidationStateService Task AddPackageRevalidationsAsync(IReadOnlyList revalidations); /// - /// Remove revalidations from the database. + /// Remove package revalidation entities from the database. /// /// A task that returns the number of revalidations that have been removed. - Task RemoveRevalidationsAsync(int max); + Task RemovePackageRevalidationsAsync(int max); /// /// Count the number of package revalidations in the database. @@ -33,11 +27,17 @@ public interface IRevalidationStateService /// The count of package revalidations in the database. Task PackageRevalidationCountAsync(); + /// + /// Count the number of package revalidations that were enqueued in the past hour. + /// + /// The number of enqueued revalidations. + Task CountRevalidationsEnqueuedInPastHourAsync(); + /// /// Update the package revalidation and mark is as enqueued. /// /// The revalidation to update. /// A task that completes once the revalidation has been updated. - Task MarkRevalidationAsEnqueuedAsync(PackageRevalidation revalidation); + Task MarkPackageRevalidationAsEnqueuedAsync(PackageRevalidation revalidation); } } diff --git a/src/NuGet.Services.Revalidate/Services/IRevalidationJobStateService.cs b/src/NuGet.Services.Revalidate/Services/IRevalidationJobStateService.cs new file mode 100644 index 000000000..0723cee0e --- /dev/null +++ b/src/NuGet.Services.Revalidate/Services/IRevalidationJobStateService.cs @@ -0,0 +1,50 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Threading.Tasks; + +namespace NuGet.Services.Revalidate +{ + /// + /// The state shared between the Gallery and the revalidation job. + /// + public interface IRevalidationJobStateService + { + /// + /// Check whether the revalidation state has been initialized. + /// + /// Whether the revalidation state has been initialized. + Task IsInitializedAsync(); + + /// + /// Update the settings to mark the revalidation job as initialized. + /// + /// A task that completes once the settings have been updated. + Task MarkAsInitializedAsync(); + + /// + /// Check whether the killswitch has been activated. If it has, all revalidation operations should be halted. + /// + /// Whether the killswitch has been activated. + Task IsKillswitchActiveAsync(); + + /// + /// Determine the desired package event rate per hour. Package events include package pushes, + /// edits, and revalidations. + /// + /// The desired maximum number of package events per hour. + Task GetDesiredPackageEventRateAsync(); + + /// + /// Reset the desired package event rate to the configured minimum. + /// + /// A task that completes once the rate has been reset. + Task ResetDesiredPackageEventRateAsync(); + + /// + /// Increment the desired package event rate per hour. + /// + /// A task that completes once the rate has been incremented. + Task IncreaseDesiredPackageEventRateAsync(); + } +} diff --git a/src/NuGet.Services.Revalidate/Services/IRevalidationThrottler.cs b/src/NuGet.Services.Revalidate/Services/IRevalidationThrottler.cs index 1d03969e6..e1870321f 100644 --- a/src/NuGet.Services.Revalidate/Services/IRevalidationThrottler.cs +++ b/src/NuGet.Services.Revalidate/Services/IRevalidationThrottler.cs @@ -13,19 +13,6 @@ public interface IRevalidationThrottler /// If true, no more revalidations should be performed. Task IsThrottledAsync(); - /// - /// Reset the capacity to the configured minimum value. Call this when the service's status is degraded to - /// throttle the revalidations. - /// - /// A task that completes once the capacity theshold has been reset. - Task ResetCapacityAsync(); - - /// - /// Increase the revalidation capacity by one revalidation per minute. - /// - /// A task taht completes once the capacity has been increased. - Task IncreaseCapacityAsync(); - /// /// Delay the current task to achieve the desired revalidation rate. /// diff --git a/src/NuGet.Services.Revalidate/Services/ITelemetryService.cs b/src/NuGet.Services.Revalidate/Services/ITelemetryService.cs index 01c487039..881578911 100644 --- a/src/NuGet.Services.Revalidate/Services/ITelemetryService.cs +++ b/src/NuGet.Services.Revalidate/Services/ITelemetryService.cs @@ -1,13 +1,13 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. -using System; +using NuGet.Services.Logging; namespace NuGet.Services.Revalidate { public interface ITelemetryService { - IDisposable TrackDurationToStartNextRevalidation(); + DurationMetric TrackStartNextRevalidationOperation(); void TrackPackageRevalidationMarkedAsCompleted(string packageId, string normalizedVersion); diff --git a/src/NuGet.Services.Revalidate/Services/RevalidationStateService.cs b/src/NuGet.Services.Revalidate/Services/PackageRevalidationStateService.cs similarity index 78% rename from src/NuGet.Services.Revalidate/Services/RevalidationStateService.cs rename to src/NuGet.Services.Revalidate/Services/PackageRevalidationStateService.cs index 1220486a1..8f7a79c1a 100644 --- a/src/NuGet.Services.Revalidate/Services/RevalidationStateService.cs +++ b/src/NuGet.Services.Revalidate/Services/PackageRevalidationStateService.cs @@ -12,23 +12,17 @@ namespace NuGet.Services.Revalidate { - public class RevalidationStateService : IRevalidationStateService + public class PackageRevalidationStateService : IPackageRevalidationStateService { private readonly IValidationEntitiesContext _context; - private readonly ILogger _logger; + private readonly ILogger _logger; - public RevalidationStateService(IValidationEntitiesContext context, ILogger logger) + public PackageRevalidationStateService(IValidationEntitiesContext context, ILogger logger) { _context = context ?? throw new ArgumentNullException(nameof(context)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); } - public Task IsKillswitchActiveAsync() - { - // TODO - return Task.FromResult(false); - } - public async Task AddPackageRevalidationsAsync(IReadOnlyList revalidations) { var validationContext = _context as ValidationEntitiesContext; @@ -53,7 +47,7 @@ public async Task AddPackageRevalidationsAsync(IReadOnlyList RemoveRevalidationsAsync(int max) + public async Task RemovePackageRevalidationsAsync(int max) { var revalidations = await _context.PackageRevalidations .Take(max) @@ -77,7 +71,16 @@ public async Task PackageRevalidationCountAsync() return await _context.PackageRevalidations.CountAsync(); } - public async Task MarkRevalidationAsEnqueuedAsync(PackageRevalidation revalidation) + public async Task CountRevalidationsEnqueuedInPastHourAsync() + { + var lowerBound = DateTime.UtcNow.Subtract(TimeSpan.FromHours(1)); + + return await _context.PackageRevalidations + .Where(r => r.Enqueued >= lowerBound) + .CountAsync(); + } + + public async Task MarkPackageRevalidationAsEnqueuedAsync(PackageRevalidation revalidation) { try { diff --git a/src/NuGet.Services.Revalidate/Services/RevalidationJobStateService.cs b/src/NuGet.Services.Revalidate/Services/RevalidationJobStateService.cs new file mode 100644 index 000000000..2ace054da --- /dev/null +++ b/src/NuGet.Services.Revalidate/Services/RevalidationJobStateService.cs @@ -0,0 +1,124 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using NuGetGallery; + +namespace NuGet.Services.Revalidate +{ + public class RevalidationJobStateService : IRevalidationJobStateService + { + private readonly IRevalidationStateService _state; + private readonly RevalidationConfiguration _config; + private readonly ILogger _logger; + + public RevalidationJobStateService( + IRevalidationStateService state, + RevalidationConfiguration config, + ILogger logger) + { + _state = state ?? throw new ArgumentNullException(nameof(state)); + _config = config ?? throw new ArgumentNullException(nameof(config)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task IsInitializedAsync() + { + return await GetStateValue(s => s.IsInitialized); + } + + public async Task MarkAsInitializedAsync() + { + _logger.LogInformation("Updating state as initialized"); + + await _state.UpdateStateAsync(s => s.IsInitialized = true); + } + + public async Task IsKillswitchActiveAsync() + { + return await GetStateValue(s => s.IsKillswitchActive); + } + + public async Task GetDesiredPackageEventRateAsync() + { + // Ensure the desired rate is within the configured lower and upper bounds. A desired rate that's outside + // the bounds indicates that the job was redeployed with different configuration values. + var finalState = await _state.MaybeUpdateStateAsync(state => + { + if (state.DesiredPackageEventRate < _config.MinPackageEventRate) + { + _logger.LogInformation( + "Overriding desired package event rate {ToRate} from {FromRate}", + _config.MinPackageEventRate, + state.DesiredPackageEventRate); + + state.DesiredPackageEventRate = _config.MinPackageEventRate; + + return true; + } + else if (state.DesiredPackageEventRate > _config.MaxPackageEventRate) + { + _logger.LogInformation( + "Overriding desired package event rate {ToRate} from {FromRate}", + _config.MaxPackageEventRate, + state.DesiredPackageEventRate); + + state.DesiredPackageEventRate = _config.MaxPackageEventRate; + + return true; + } + + // The rate is within the expected bounds. Don't update anything. + return false; + }); + + return finalState.DesiredPackageEventRate; + } + + public async Task ResetDesiredPackageEventRateAsync() + { + await _state.UpdateStateAsync(state => + { + _logger.LogInformation( + "Resetting desired package event rate to {ToRate} from {FromRate}", + _config.MinPackageEventRate, + state.DesiredPackageEventRate); + + state.DesiredPackageEventRate = _config.MinPackageEventRate; + }); + } + + public async Task IncreaseDesiredPackageEventRateAsync() + { + await _state.MaybeUpdateStateAsync(state => + { + // Don't update the state if we've reached the upper limit. + if (state.DesiredPackageEventRate == _config.MaxPackageEventRate) + { + _logger.LogInformation( + "Desired package event rate is at configured maximum of {MaxRate} per hour", + _config.MaxPackageEventRate); + + return false; + } + + var nextRate = Math.Min(_config.MaxPackageEventRate, state.DesiredPackageEventRate + 1); + + _logger.LogInformation( + "Increasing desired package event rate to {ToRate} from {FromRate}", + nextRate, + state.DesiredPackageEventRate); + + state.DesiredPackageEventRate = nextRate; + return true; + }); + } + + private async Task GetStateValue(Func callback) + { + return callback(await _state.GetStateAsync()); + } + } +} diff --git a/src/NuGet.Services.Revalidate/Services/RevalidationOperation.cs b/src/NuGet.Services.Revalidate/Services/RevalidationOperation.cs new file mode 100644 index 000000000..9a85e08cc --- /dev/null +++ b/src/NuGet.Services.Revalidate/Services/RevalidationOperation.cs @@ -0,0 +1,13 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +namespace NuGet.Services.Revalidate +{ + public class StartNextRevalidationOperation + { + /// + /// The result of attempting to start the next revalidation. + /// + public RevalidationResult Result { get; set; } + } +} diff --git a/src/NuGet.Services.Revalidate/Services/RevalidationService.cs b/src/NuGet.Services.Revalidate/Services/RevalidationService.cs index 46db515de..b196bf3d0 100644 --- a/src/NuGet.Services.Revalidate/Services/RevalidationService.cs +++ b/src/NuGet.Services.Revalidate/Services/RevalidationService.cs @@ -11,7 +11,8 @@ namespace NuGet.Services.Revalidate { public class RevalidationService : IRevalidationService { - private readonly IRevalidationStateService _state; + private readonly IRevalidationJobStateService _jobState; + private readonly IPackageRevalidationStateService _packageState; private readonly ISingletonService _singletonService; private readonly IRevalidationThrottler _throttler; private readonly IHealthService _healthService; @@ -22,7 +23,8 @@ public class RevalidationService : IRevalidationService private readonly ILogger _logger; public RevalidationService( - IRevalidationStateService state, + IRevalidationJobStateService jobState, + IPackageRevalidationStateService packageState, ISingletonService singletonService, IRevalidationThrottler throttler, IHealthService healthService, @@ -32,7 +34,8 @@ public RevalidationService( ITelemetryService telemetryService, ILogger logger) { - _state = state ?? throw new ArgumentNullException(nameof(state)); + _jobState = jobState ?? throw new ArgumentNullException(nameof(jobState)); + _packageState = packageState ?? throw new ArgumentNullException(nameof(packageState)); _singletonService = singletonService ?? throw new ArgumentNullException(nameof(singletonService)); _throttler = throttler ?? throw new ArgumentNullException(nameof(throttler)); _healthService = healthService ?? throw new ArgumentNullException(nameof(healthService)); @@ -45,6 +48,13 @@ public RevalidationService( public async Task RunAsync() { + if (!await _jobState.IsInitializedAsync()) + { + _logger.LogError("The revalidation service must be initialized before running revalidations"); + + throw new InvalidOperationException("The revalidation service must be initialized before running revalidations"); + } + var runTime = Stopwatch.StartNew(); do @@ -83,7 +93,19 @@ public async Task RunAsync() public async Task StartNextRevalidationAsync() { - using (_telemetryService.TrackDurationToStartNextRevalidation()) + using (var operation = _telemetryService.TrackStartNextRevalidationOperation()) + { + var result = await StartNextRevalidationInternalAsync(); + + operation.Properties.Result = result; + + return result; + } + } + + public async Task StartNextRevalidationInternalAsync() + { + try { // Don't start a revalidation if the job has been deactivated, if the ingestion pipeline is unhealthy, // or if we have reached our quota of desired revalidations. @@ -98,7 +120,7 @@ public async Task StartNextRevalidationAsync() } // Everything is in tip-top shape! Increase the throttling quota and start the next revalidation. - await _throttler.IncreaseCapacityAsync(); + await _jobState.IncreaseDesiredPackageEventRateAsync(); var revalidation = await _revalidationQueue.NextOrNullAsync(); if (revalidation == null) @@ -110,6 +132,12 @@ public async Task StartNextRevalidationAsync() return await StartRevalidationAsync(revalidation); } + catch (Exception e) + { + _logger.LogError(0, e, "Failed to start next validation due to exception, retry later..."); + + return RevalidationResult.RetryLater; + } } private async Task CanStartRevalidationAsync() @@ -121,7 +149,7 @@ public async Task StartNextRevalidationAsync() return RevalidationResult.UnrecoverableError; } - if (await _state.IsKillswitchActiveAsync()) + if (await _jobState.IsKillswitchActiveAsync()) { _logger.LogWarning("Revalidation killswitch has been activated, retry later..."); @@ -137,14 +165,14 @@ public async Task StartNextRevalidationAsync() if (!await _healthService.IsHealthyAsync()) { - _logger.LogWarning("Service appears to be unhealthy, resetting throttling capacity and retry later..."); + _logger.LogWarning("Service appears to be unhealthy, resetting the desired package event rate. Retry later..."); - await _throttler.ResetCapacityAsync(); + await _jobState.ResetDesiredPackageEventRateAsync(); return RevalidationResult.RetryLater; } - if (await _state.IsKillswitchActiveAsync()) + if (await _jobState.IsKillswitchActiveAsync()) { _logger.LogWarning("Revalidation killswitch has been activated after the throttle and health check, retry later..."); @@ -162,7 +190,7 @@ private async Task StartRevalidationAsync(PackageRevalidatio revalidation.ValidationTrackingId.Value); await _validationEnqueuer.StartValidationAsync(message); - await _state.MarkRevalidationAsEnqueuedAsync(revalidation); + await _packageState.MarkPackageRevalidationAsEnqueuedAsync(revalidation); _telemetryService.TrackPackageRevalidationStarted(revalidation.PackageId, revalidation.PackageNormalizedVersion); diff --git a/src/NuGet.Services.Revalidate/Services/RevalidationThrottler.cs b/src/NuGet.Services.Revalidate/Services/RevalidationThrottler.cs index b736d5863..4041832f7 100644 --- a/src/NuGet.Services.Revalidate/Services/RevalidationThrottler.cs +++ b/src/NuGet.Services.Revalidate/Services/RevalidationThrottler.cs @@ -9,41 +9,42 @@ namespace NuGet.Services.Revalidate { public class RevalidationThrottler : IRevalidationThrottler { + private readonly IRevalidationJobStateService _jobState; + private readonly IPackageRevalidationStateService _packageState; private readonly RevalidationConfiguration _config; private readonly ILogger _logger; - public RevalidationThrottler(RevalidationConfiguration config, ILogger logger) + public RevalidationThrottler( + IRevalidationJobStateService jobState, + IPackageRevalidationStateService packageState, + RevalidationConfiguration config, + ILogger logger) { + _jobState = jobState ?? throw new ArgumentNullException(nameof(jobState)); + _packageState = packageState ?? throw new ArgumentNullException(nameof(packageState)); _config = config ?? throw new ArgumentNullException(nameof(config)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); } - public Task IsThrottledAsync() + public async Task IsThrottledAsync() { - // TODO: - // Calculate desired event rate - // Calculate current event rate (# of revalidations + Gallery actions) - // Compare desired event rate to configured event rate. If configured rate is higher, update desired event rate. - // If current event rate is greater than or equal to desired event rate, return true; - return Task.FromResult(false); - } + var desiredRate = await _jobState.GetDesiredPackageEventRateAsync(); + var recentGalleryEvents = await CountGalleryEventsInPastHourAsync(); + var recentRevalidations = await _packageState.CountRevalidationsEnqueuedInPastHourAsync(); - public Task ResetCapacityAsync() - { - return Task.CompletedTask; - } + var revalidationQuota = desiredRate - recentRevalidations - recentGalleryEvents; - public Task IncreaseCapacityAsync() - { - return Task.CompletedTask; + return (revalidationQuota <= 0); } public async Task DelayUntilNextRevalidationAsync() { - // TODO: Calculate sleep duration to achieve desired event rate. - _logger.LogInformation("Delaying until next revalidation by sleeping for 5 minutes..."); + var desiredHourlyRate = await _jobState.GetDesiredPackageEventRateAsync(); + var sleepDuration = TimeSpan.FromHours(1.0 / desiredHourlyRate); - await Task.Delay(TimeSpan.FromMinutes(5)); + _logger.LogInformation("Delaying until next revalidation by sleeping for {SleepDuration}...", sleepDuration); + + await Task.Delay(sleepDuration); } public async Task DelayUntilRevalidationRetryAsync() @@ -54,5 +55,22 @@ public async Task DelayUntilRevalidationRetryAsync() await Task.Delay(_config.RetryLaterSleep); } + + private Task CountGalleryEventsInPastHourAsync() + { + // TODO: Count the number of package pushes, lists, and unlists. + // Run this AI query: + // + // customMetrics | where name == "PackagePush" or name == "PackageUnlisted" or name == "PackageListed" | summarize sum(value) + // + // Using this HTTP request: + // + // GET /v1/apps/46f13c7d-635f-42c3-8120-593edeaad426/query?timespan=P1D&query=customMetrics%20%7C%20where%20name%20%3D%3D%20%22PackagePush%22%20or%20name%20%3D%3D%20%22PackageUnlisted%22%20or%20name%20%3D%3D%20%22PackageListed%22%20%7C%20summarize%20sum(value)%20 HTTP/1.1 + // Host: api.applicationinsights.io + // x-api-key: my-super-secret-api-key + // + // See: https://dev.applicationinsights.io/quickstart + return Task.FromResult(0); + } } } diff --git a/src/NuGet.Services.Revalidate/Services/TelemetryService.cs b/src/NuGet.Services.Revalidate/Services/TelemetryService.cs index 9ce792992..1ea15a0e6 100644 --- a/src/NuGet.Services.Revalidate/Services/TelemetryService.cs +++ b/src/NuGet.Services.Revalidate/Services/TelemetryService.cs @@ -11,12 +11,13 @@ public class TelemetryService : ITelemetryService { private const string RevalidationPrefix = "Revalidation."; - private const string DurationToStartNextRevalidation = RevalidationPrefix + "DurationToStartNextRevalidation"; + private const string DurationToStartNextRevalidation = RevalidationPrefix + "DurationToStartNextRevalidationSeconds"; private const string PackageRevalidationMarkedAsCompleted = RevalidationPrefix + "PackageRevalidationMarkedAsCompleted"; private const string PackageRevalidationStarted = RevalidationPrefix + "PackageRevalidationStarted"; private const string PackageId = "PackageId"; private const string NormalizedVersion = "NormalizedVersion"; + private const string Result = "Result"; private readonly ITelemetryClient _client; @@ -25,9 +26,15 @@ public TelemetryService(ITelemetryClient client) _client = client ?? throw new ArgumentNullException(nameof(client)); } - public IDisposable TrackDurationToStartNextRevalidation() + public DurationMetric TrackStartNextRevalidationOperation() { - return _client.TrackDuration(DurationToStartNextRevalidation); + return _client.TrackDuration( + DurationToStartNextRevalidation, + new StartNextRevalidationOperation(), + o => new Dictionary + { + { Result, o.Result.ToString() } + }); } public void TrackPackageRevalidationMarkedAsCompleted(string packageId, string normalizedVersion) diff --git a/src/NuGet.Services.Revalidate/Settings/dev.json b/src/NuGet.Services.Revalidate/Settings/dev.json index 2964de561..d79310f37 100644 --- a/src/NuGet.Services.Revalidate/Settings/dev.json +++ b/src/NuGet.Services.Revalidate/Settings/dev.json @@ -9,18 +9,30 @@ "MaxPackageCreationDate": "2021-03-01T23:52:40.7022034+00:00", // TODO: Update this when repository signing is enabled "SleepDurationBetweenBatches": "00:00:01" }, + + "MinPackageEventRate": 120, + "MaxPackageEventRate": 500, + "Queue": { "MaximumAttempts": 5, "SleepBetweenAttempts": "00:05:00" } }, + "Storage": { + "ConnectionString": "DefaultEndpointsProtocol=https;AccountName=nugetdevlegacy;AccountKey=$$Dev-NuGetDevLegacyStorage-Key$$", + "Container": "revalidations" + }, + "GalleryDb": { "ConnectionString": "Data Source=tcp:#{Jobs.validation.GalleryDatabaseAddress};Initial Catalog=nuget-dev-0-v2gallery;Integrated Security=False;User ID=$$Dev-GalleryDBReadOnly-UserName$$;Password=$$Dev-GalleryDBReadOnly-Password$$;Connect Timeout=30;Encrypt=True" }, "ValidationDb": { "ConnectionString": "Data Source=tcp:#{Jobs.validation.DatabaseAddress};Initial Catalog=nuget-dev-validation;Integrated Security=False;User ID=$$Dev-ValidationDBWriter-UserName$$;Password=$$Dev-ValidationDBWriter-Password$$;Connect Timeout=30;Encrypt=True" }, + "ValidationStorage": { + "ConnectionString": "DefaultEndpointsProtocol=https;AccountName=nugetdevlegacy;AccountKey=$$Dev-NuGetDevLegacyStorage-Key$$" + }, "ServiceBus": { "ConnectionString": "Endpoint=sb://nugetdev.servicebus.windows.net/;SharedAccessKeyName=gallery;SharedAccessKey=$$Dev-ServiceBus-SharedAccessKey-Validation-GallerySender$$", "TopicPath": "validation" diff --git a/src/NuGet.Services.Revalidate/Settings/int.json b/src/NuGet.Services.Revalidate/Settings/int.json index 25c5abb5c..acebe3e27 100644 --- a/src/NuGet.Services.Revalidate/Settings/int.json +++ b/src/NuGet.Services.Revalidate/Settings/int.json @@ -9,6 +9,10 @@ "MaxPackageCreationDate": "2021-03-01T23:52:40.7022034+00:00", // TODO: Update this when repository signing is enabled "SleepDurationBetweenBatches": "00:00:30" }, + + "MinPackageEventRate": 120, + "MaxPackageEventRate": 500, + "Queue": { "MaximumAttempts": 5, "SleepBetweenAttempts": "00:05:00" @@ -21,6 +25,9 @@ "ValidationDb": { "ConnectionString": "Data Source=tcp:#{Jobs.validation.DatabaseAddress};Initial Catalog=nuget-int-validation;Integrated Security=False;User ID=$$Int-ValidationDBWriter-UserName$$;Password=$$Int-ValidationDBWriter-Password$$;Connect Timeout=30;Encrypt=True" }, + "ValidationStorage": { + "ConnectionString": "DefaultEndpointsProtocol=https;AccountName=nugetint0;AccountKey=$$Int-NuGetInt0Storage-Key$$" + }, "ServiceBus": { "ConnectionString": "Endpoint=sb://nugetint.servicebus.windows.net/;SharedAccessKeyName=gallery;SharedAccessKey=$$Int-ServiceBus-SharedAccessKey-Validation-GallerySender$$", "TopicPath": "validation" diff --git a/src/NuGet.Services.Revalidate/Settings/prod.json b/src/NuGet.Services.Revalidate/Settings/prod.json index 519b9d11f..f873a636d 100644 --- a/src/NuGet.Services.Revalidate/Settings/prod.json +++ b/src/NuGet.Services.Revalidate/Settings/prod.json @@ -9,6 +9,10 @@ "MaxPackageCreationDate": "2021-03-01T23:52:40.7022034+00:00", // TODO: Update this when repository signing is enabled "SleepDurationBetweenBatches": "00:00:30" }, + + "MinPackageEventRate": 120, + "MaxPackageEventRate": 500, + "Queue": { "MaximumAttempts": 5, "SleepBetweenAttempts": "00:05:00" @@ -21,6 +25,9 @@ "ValidationDb": { "ConnectionString": "Data Source=tcp:#{Jobs.validation.DatabaseAddress};Initial Catalog=nuget-prod-validation;Integrated Security=False;User ID=$$Prod-ValidationDBWriter-UserName$$;Password=$$Prod-ValidationDBWriter-Password$$;Connect Timeout=30;Encrypt=True" }, + "ValidationStorage": { + "ConnectionString": "DefaultEndpointsProtocol=https;AccountName=nugetgallery;AccountKey=$$Prod-NuGetGalleryStorage-Key$$" + }, "ServiceBus": { "ConnectionString": "Endpoint=sb://nugetprod.servicebus.windows.net/;SharedAccessKeyName=gallery;SharedAccessKey=$$Prod-ServiceBus-SharedAccessKey-Validation-GallerySender$$", "TopicPath": "validation" diff --git a/src/NuGet.Services.Revalidate/readme.md b/src/NuGet.Services.Revalidate/readme.md new file mode 100644 index 000000000..c9a28c051 --- /dev/null +++ b/src/NuGet.Services.Revalidate/readme.md @@ -0,0 +1,41 @@ +# The Revalidate Job + +This job enqueues packages revalidation as fast as possible without affecting the +health of NuGet's ingestion pipeline. It does so in two phases: + +1. Initialization phase - the job determines which packages should be revalidated. +2. Revalidation phase - packages are enqueued for revalidations + +The initialization phase MUST complete before the revalidation phase is started. + +# The Initialization Phase + +To initialize the job, run: + +``` +NuGet.Services.Revalidate.exe ^ + -Configuration "C:\Path\to\config.json" ^ + -Initialize + -VerifyInitialization + -Once +``` + +This will figure which packages should be revalidated, and the order that packages +should be revalidated. Packages are prioritized by: + +1. Packages owned by the `Microsoft` account +2. Packages installed by Visual Studio or by the .NET Core SDK +3. All remaining packages + +Pending package revalidations are stored in the `PackageRevalidations` +table, in order of priority. + +# The Revalidation Phase + +To enqueue revalidations, run: + +``` +NuGet.Services.Revalidate.exe ^ + -Configuration "C:\Path\to\config.json" ^ + -Initialize +``` \ No newline at end of file diff --git a/src/PackageLagMonitor/Monitoring.PackageLag.csproj b/src/PackageLagMonitor/Monitoring.PackageLag.csproj index a41d91d01..daa00bca3 100644 --- a/src/PackageLagMonitor/Monitoring.PackageLag.csproj +++ b/src/PackageLagMonitor/Monitoring.PackageLag.csproj @@ -111,13 +111,13 @@ 0.5.0-CI-20180510-012541 - 2.26.0 + 2.27.0 - 2.25.0 + 2.27.0 - 2.25.0 + 2.27.0 4.3.3 diff --git a/src/Validation.Common.Job/Validation.Common.Job.csproj b/src/Validation.Common.Job/Validation.Common.Job.csproj index 4b4fa7e6a..fa1a131a8 100644 --- a/src/Validation.Common.Job/Validation.Common.Job.csproj +++ b/src/Validation.Common.Job/Validation.Common.Job.csproj @@ -94,25 +94,25 @@ 4.8.0-preview4.5289 - 2.26.0-master-34394 + 2.27.0 - 2.26.0-master-34394 + 2.27.0 - 2.26.0-master-34736 + 2.27.0 - 2.26.0-master-34394 + 2.27.0 - 2.26.0-master-34394 + 2.27.0 - 2.26.0-master-34736 + 2.27.0 - 4.4.5-dev-34648 + 4.4.5-dev-35356 2.5.0 @@ -128,7 +128,9 @@ - + + Designer + diff --git a/src/Validation.Common.Job/Validation.Common.Job.nuspec b/src/Validation.Common.Job/Validation.Common.Job.nuspec index f01f8d6cd..1b70aaa35 100644 --- a/src/Validation.Common.Job/Validation.Common.Job.nuspec +++ b/src/Validation.Common.Job/Validation.Common.Job.nuspec @@ -16,13 +16,13 @@ - - - - - - - + + + + + + + diff --git a/src/Validation.Common/Validation.Common.csproj b/src/Validation.Common/Validation.Common.csproj index 716e67d13..75cc170e2 100644 --- a/src/Validation.Common/Validation.Common.csproj +++ b/src/Validation.Common/Validation.Common.csproj @@ -108,10 +108,10 @@ 4.1.0 - 2.26.0-master-33196 + 2.27.0 - 2.25.0 + 2.27.0 3.2.0 diff --git a/src/Validation.PackageSigning.ProcessSignature/Validation.PackageSigning.ProcessSignature.csproj b/src/Validation.PackageSigning.ProcessSignature/Validation.PackageSigning.ProcessSignature.csproj index a27ed8411..3d331e9f9 100644 --- a/src/Validation.PackageSigning.ProcessSignature/Validation.PackageSigning.ProcessSignature.csproj +++ b/src/Validation.PackageSigning.ProcessSignature/Validation.PackageSigning.ProcessSignature.csproj @@ -87,7 +87,7 @@ - 2.26.0-master-34394 + 2.27.0 diff --git a/src/Validation.ScanAndSign.Core/Validation.ScanAndSign.Core.csproj b/src/Validation.ScanAndSign.Core/Validation.ScanAndSign.Core.csproj index c5bc64caa..6c215c062 100644 --- a/src/Validation.ScanAndSign.Core/Validation.ScanAndSign.Core.csproj +++ b/src/Validation.ScanAndSign.Core/Validation.ScanAndSign.Core.csproj @@ -55,7 +55,7 @@ 1.1.2 - 2.26.0-master-34736 + 2.27.0 diff --git a/tests/NuGet.Services.Revalidate.Tests/Initializer/InitializationManagerFacts.cs b/tests/NuGet.Services.Revalidate.Tests/Initializer/InitializationManagerFacts.cs index 261143584..8594f9cda 100644 --- a/tests/NuGet.Services.Revalidate.Tests/Initializer/InitializationManagerFacts.cs +++ b/tests/NuGet.Services.Revalidate.Tests/Initializer/InitializationManagerFacts.cs @@ -18,6 +18,18 @@ public class InitializationManagerFacts { public class TheInitializeAsyncMethod : FactsBase { + [Fact] + public async Task ThrowsIfAlreadyInitialized() + { + // Arrange + _settings.Setup(s => s.IsInitializedAsync()).ReturnsAsync(true); + + // Act & Assert + var e = await Assert.ThrowsAsync(() => _target.InitializeAsync()); + + Assert.Equal("Attempted to initialize the revalidation job when it is already initialized!", e.Message); + } + [Fact] public async Task RemovesPreviousRevalidations() { @@ -32,7 +44,7 @@ public async Task RemovesPreviousRevalidations() var firstRemove = true; - _revalidationState.Setup(s => s.RemoveRevalidationsAsync(1000)) + _packageState.Setup(s => s.RemovePackageRevalidationsAsync(1000)) .ReturnsAsync(() => { if (firstRemove) @@ -47,7 +59,7 @@ public async Task RemovesPreviousRevalidations() // Act & assert await _target.InitializeAsync(); - _revalidationState.Verify(s => s.RemoveRevalidationsAsync(1000), Times.Exactly(2)); + _packageState.Verify(s => s.RemovePackageRevalidationsAsync(1000), Times.Exactly(2)); } [Fact] @@ -71,13 +83,12 @@ public async Task InsertsMicrosoftThenPreinstalledThenDependenciesThenAllOtherPa remainingPackages: new[] { new Package { PackageRegistrationKey = 3, DownloadCount = 20, NormalizedVersion = "3.0.0", }, - }); // Act & assert await _target.InitializeAsync(); - _revalidationState.Verify( + _packageState.Verify( s => s.AddPackageRevalidationsAsync(It.IsAny>()), Times.Exactly(4)); @@ -116,7 +127,7 @@ public async Task AddsRevalidationsByDescendingOrderOfDownloadCounts() // Act & assert await _target.InitializeAsync(); - _revalidationState.Verify( + _packageState.Verify( s => s.AddPackageRevalidationsAsync(It.IsAny>()), Times.Once); @@ -153,7 +164,7 @@ public async Task InsertsPackageVersionsByDescendingOrder() // Act & assert await _target.InitializeAsync(); - _revalidationState.Verify( + _packageState.Verify( s => s.AddPackageRevalidationsAsync(It.IsAny>()), Times.Once); @@ -209,7 +220,7 @@ public async Task PartitionsPackagesIntoBatchesOf1000OrLessVersions(int[] packag // Act & assert await _target.InitializeAsync(); - _revalidationState.Verify( + _packageState.Verify( s => s.AddPackageRevalidationsAsync(It.IsAny>()), Times.Exactly(expectedBatches)); } @@ -248,7 +259,36 @@ public static IEnumerable PartitionsPackagesIntoBatchesOf1000OrLessVer }; } - // Partitions packages by batches of 1000 versions (with intelligent weights) + [Fact] + public async Task MarksAsInitializedAfterAddingRevalidations() + { + // Arrange + Setup(remainingPackages: new[] + { + new Package { PackageRegistrationKey = 3, DownloadCount = 20, NormalizedVersion = "3.0.0", }, + }); + + int order = 0; + int addRevalidationOrder = 0; + int markAsInitializedOrder = 0; + + _packageState + .Setup(s => s.AddPackageRevalidationsAsync(It.IsAny>())) + .Callback(() => addRevalidationOrder = order++) + .Returns(Task.CompletedTask); + + _settings + .Setup(s => s.MarkAsInitializedAsync()) + .Callback(() => markAsInitializedOrder = order++) + .Returns(Task.CompletedTask); + + // Act & Assert + await _target.InitializeAsync(); + + _settings.Verify(s => s.MarkAsInitializedAsync(), Times.Once); + + Assert.True(markAsInitializedOrder > addRevalidationOrder); + } private void Setup( IEnumerable microsoftPackages = null, @@ -344,7 +384,7 @@ public TheInitializeAsyncMethod() { _revalidationBatches = new List>(); - _revalidationState + _packageState .Setup(s => s.AddPackageRevalidationsAsync(It.IsAny>())) .Callback>(r => _revalidationBatches.Add(r)) .Returns(Task.CompletedTask); @@ -353,20 +393,34 @@ public TheInitializeAsyncMethod() public class TheVerifyAsyncMethod : FactsBase { + [Fact] + public async Task ThrowsIfNotInitialized() + { + _settings.Setup(s => s.IsInitializedAsync()).ReturnsAsync(false); + + var e = await Assert.ThrowsAsync(() => _target.VerifyInitializationAsync()); + + Assert.Equal("Expected revalidation state to be initialized", e.Message); + } + [Fact] public async Task ThrowsIfAppropriatePackageCountDoesNotMatchRevalidationCount() { + _settings.Setup(s => s.IsInitializedAsync()).ReturnsAsync(true); _packageFinder.Setup(f => f.AppropriatePackageCount()).Returns(100); - _revalidationState.Setup(s => s.PackageRevalidationCountAsync()).ReturnsAsync(50); + _packageState.Setup(s => s.PackageRevalidationCountAsync()).ReturnsAsync(50); + + var e = await Assert.ThrowsAsync(() => _target.VerifyInitializationAsync()); - var exception = await Assert.ThrowsAsync(async () => await _target.VerifyInitializationAsync()); + Assert.Equal("Expected 100 revalidation, found 50", e.Message); } [Fact] public async Task DoesNotThrowIfCountsMatch() { + _settings.Setup(s => s.IsInitializedAsync()).ReturnsAsync(true); _packageFinder.Setup(f => f.AppropriatePackageCount()).Returns(100); - _revalidationState.Setup(s => s.PackageRevalidationCountAsync()).ReturnsAsync(100); + _packageState.Setup(s => s.PackageRevalidationCountAsync()).ReturnsAsync(100); await _target.VerifyInitializationAsync(); } @@ -374,7 +428,8 @@ public async Task DoesNotThrowIfCountsMatch() public class FactsBase { - public readonly Mock _revalidationState; + public readonly Mock _settings; + public readonly Mock _packageState; public readonly Mock _packageFinder; public readonly InitializationConfiguration _config; @@ -382,13 +437,15 @@ public class FactsBase public FactsBase() { - _revalidationState = new Mock(); + _settings = new Mock(); + _packageState = new Mock(); _packageFinder = new Mock(); _config = new InitializationConfiguration(); _target = new InitializationManager( - _revalidationState.Object, + _settings.Object, + _packageState.Object, _packageFinder.Object, _config, Mock.Of>()); diff --git a/tests/NuGet.Services.Revalidate.Tests/NuGet.Services.Revalidate.Tests.csproj b/tests/NuGet.Services.Revalidate.Tests/NuGet.Services.Revalidate.Tests.csproj index 0fe5be2eb..a09ec1eea 100644 --- a/tests/NuGet.Services.Revalidate.Tests/NuGet.Services.Revalidate.Tests.csproj +++ b/tests/NuGet.Services.Revalidate.Tests/NuGet.Services.Revalidate.Tests.csproj @@ -66,9 +66,11 @@ - + + + diff --git a/tests/NuGet.Services.Revalidate.Tests/RevalidationStateServiceFacts.cs b/tests/NuGet.Services.Revalidate.Tests/Services/PackageRevalidationStateServiceFacts.cs similarity index 69% rename from tests/NuGet.Services.Revalidate.Tests/RevalidationStateServiceFacts.cs rename to tests/NuGet.Services.Revalidate.Tests/Services/PackageRevalidationStateServiceFacts.cs index 3afff254a..69e976fb4 100644 --- a/tests/NuGet.Services.Revalidate.Tests/RevalidationStateServiceFacts.cs +++ b/tests/NuGet.Services.Revalidate.Tests/Services/PackageRevalidationStateServiceFacts.cs @@ -1,6 +1,7 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +using System; using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; @@ -10,9 +11,9 @@ using Tests.ContextHelpers; using Xunit; -namespace NuGet.Services.Revalidate.Tests +namespace NuGet.Services.Revalidate.Tests.Services { - public class RevalidationStateServiceFacts + public class PackageRevalidationStateServiceFacts { public class TheAddPackageRevalidationsAsyncMethod : FactsBase { @@ -50,7 +51,7 @@ public async Task RemovesRevalidations() }); // Act & Assert - var result = await _target.RemoveRevalidationsAsync(5); + var result = await _target.RemovePackageRevalidationsAsync(5); Assert.Equal(2, result); Assert.Equal(0, _context.Object.PackageRevalidations.Count()); @@ -69,7 +70,7 @@ public async Task RespectsMaxParameter() }); // Act & Assert - var result = await _target.RemoveRevalidationsAsync(1); + var result = await _target.RemovePackageRevalidationsAsync(1); Assert.Equal(1, result); Assert.Equal(1, _context.Object.PackageRevalidations.Count()); @@ -93,18 +94,37 @@ public async Task ReturnsRevalidationCount() } } + public class TheCountRevalidationsEnqueuedInPastHourAsyncMethod : FactsBase + { + [Fact] + public async Task ReturnsRevalidationCount() + { + var now = DateTime.UtcNow; + + _context.Mock(packageRevalidations: new List + { + new PackageRevalidation { PackageId = "A", Enqueued = now.Subtract(TimeSpan.FromDays(4)) }, + new PackageRevalidation { PackageId = "B", Enqueued = now.Subtract(TimeSpan.FromHours(3)) }, + new PackageRevalidation { PackageId = "C", Enqueued = now.Subtract(TimeSpan.FromMinutes(2)) }, + new PackageRevalidation { PackageId = "D", Enqueued = now.Subtract(TimeSpan.FromSeconds(1)) }, + }); + + Assert.Equal(2, await _target.CountRevalidationsEnqueuedInPastHourAsync()); + } + } + public class FactsBase { public readonly Mock _context; - public readonly RevalidationStateService _target; + public readonly PackageRevalidationStateService _target; public FactsBase() { _context = new Mock(); - _target = new RevalidationStateService( + _target = new PackageRevalidationStateService( _context.Object, - Mock.Of>()); + Mock.Of>()); } } } diff --git a/tests/NuGet.Services.Revalidate.Tests/Services/RevalidationJobStateServiceFacts.cs b/tests/NuGet.Services.Revalidate.Tests/Services/RevalidationJobStateServiceFacts.cs new file mode 100644 index 000000000..7403364bf --- /dev/null +++ b/tests/NuGet.Services.Revalidate.Tests/Services/RevalidationJobStateServiceFacts.cs @@ -0,0 +1,273 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Moq; +using NuGetGallery; +using Xunit; + +namespace NuGet.Services.Revalidate.Tests.Services +{ + public class RevalidationJobStateServiceFacts + { + public class TheIsInitializedAsyncMethod : FactsBase + { + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task ReturnsInitializedValue(bool isInitialized) + { + _state + .Setup(s => s.GetStateAsync()) + .ReturnsAsync(new RevalidationState + { + IsInitialized = isInitialized + }); + + Assert.Equal(isInitialized, await _target.IsInitializedAsync()); + + _state.Verify(s => s.GetStateAsync(), Times.Once); + } + } + + public class TheMarkAsInitializedAsyncMethod : FactsBase + { + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task UpdatesState(bool isInitialized) + { + var result = new RevalidationState + { + IsInitialized = isInitialized + }; + + _state + .Setup(s => s.UpdateStateAsync(It.IsAny>())) + .Callback((Action a) => a(result)) + .Returns(Task.CompletedTask); + + await _target.MarkAsInitializedAsync(); + + _state.Verify( + s => s.UpdateStateAsync(It.IsAny>()), + Times.Once); + + Assert.True(result.IsInitialized); + } + } + + public class TheIsKillswitchActiveAsyncMethod : FactsBase + { + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task ReturnsIsKillswitchActiveValue(bool isKillswitchActive) + { + _state + .Setup(s => s.GetStateAsync()) + .ReturnsAsync(new RevalidationState + { + IsKillswitchActive = isKillswitchActive + }); + + Assert.Equal(isKillswitchActive, await _target.IsKillswitchActiveAsync()); + + _state.Verify(s => s.GetStateAsync(), Times.Once); + } + } + + public class TheGetDesiredPackageEventRateAsyncMethod : FactsBase + { + [Fact] + public async Task ReturnsDesiredPackageEventRateValue() + { + // Arrange + bool? result = null; + var state = new RevalidationState + { + DesiredPackageEventRate = 123 + }; + + _state + .Setup(s => s.MaybeUpdateStateAsync(It.IsAny>())) + .Callback((Func a) => + { + result = a(state); + }) + .ReturnsAsync(state); + + // Act & Assert + Assert.Equal(123, await _target.GetDesiredPackageEventRateAsync()); + Assert.False(result); + + _state.Verify(s => s.MaybeUpdateStateAsync(It.IsAny>()), Times.Once); + } + + [Theory] + [InlineData(0, 100)] + [InlineData(600, 500)] + public async Task UpdatesRateIfCurrentRateOutsideofConfiguredBounds(int storedRate, int expectedRate) + { + // Arrange + bool? result = null; + var state = new RevalidationState + { + DesiredPackageEventRate = storedRate + }; + + _state + .Setup(s => s.MaybeUpdateStateAsync(It.IsAny>())) + .Callback((Func a) => + { + result = a(state); + }) + .ReturnsAsync(state); + + // Act & Assert + Assert.Equal(expectedRate, await _target.GetDesiredPackageEventRateAsync()); + Assert.True(result); + + _state.Verify(s => s.MaybeUpdateStateAsync(It.IsAny>()), Times.Once); + } + } + + public class TheResetDesiredPackageEventRateAsyncMethod : FactsBase + { + [Fact] + public async Task UpdatesState() + { + var result = new RevalidationState + { + DesiredPackageEventRate = 123 + }; + + _state + .Setup(s => s.UpdateStateAsync(It.IsAny>())) + .Callback((Action a) => a(result)) + .Returns(Task.CompletedTask); + + await _target.ResetDesiredPackageEventRateAsync(); + + _state.Verify( + s => s.UpdateStateAsync(It.IsAny>()), + Times.Once); + + Assert.Equal(100, result.DesiredPackageEventRate); + } + } + + public class TheIncreaseDesiredPackageEventRateAsyncMethod : FactsBase + { + [Fact] + public async Task IncreasesDesiredPackageEventRateValue() + { + // Arrange + bool? result = null; + var state = new RevalidationState + { + DesiredPackageEventRate = 123 + }; + + _state + .Setup(s => s.MaybeUpdateStateAsync(It.IsAny>())) + .Callback((Func a) => + { + result = a(state); + }) + .ReturnsAsync(state); + + // Act + await _target.IncreaseDesiredPackageEventRateAsync(); + + // Assert + Assert.Equal(124, state.DesiredPackageEventRate); + Assert.True(result); + + _state.Verify(s => s.MaybeUpdateStateAsync(It.IsAny>()), Times.Once); + } + + [Fact] + public async Task DoesntIncreaseIfReachedConfiguredMaxRate() + { + // Arrange + bool? result = null; + var state = new RevalidationState + { + DesiredPackageEventRate = 500 + }; + + _state + .Setup(s => s.MaybeUpdateStateAsync(It.IsAny>())) + .Callback((Func a) => + { + result = a(state); + }) + .ReturnsAsync(state); + + // Act + await _target.IncreaseDesiredPackageEventRateAsync(); + + // Assert + Assert.Equal(500, state.DesiredPackageEventRate); + Assert.False(result); + + _state.Verify(s => s.MaybeUpdateStateAsync(It.IsAny>()), Times.Once); + } + + [Fact] + public async Task ResetsToConfiguredMaxRateIfPastConfiguredMax() + { + // Arrange + bool? result = null; + var state = new RevalidationState + { + DesiredPackageEventRate = 600 + }; + + _state + .Setup(s => s.MaybeUpdateStateAsync(It.IsAny>())) + .Callback((Func a) => + { + result = a(state); + }) + .ReturnsAsync(state); + + // Act + await _target.IncreaseDesiredPackageEventRateAsync(); + + // Assert + Assert.Equal(500, state.DesiredPackageEventRate); + Assert.True(result); + + _state.Verify(s => s.MaybeUpdateStateAsync(It.IsAny>()), Times.Once); + } + } + + public class FactsBase + { + protected readonly Mock _state; + protected readonly RevalidationConfiguration _config; + + protected readonly RevalidationJobStateService _target; + + public FactsBase() + { + _state = new Mock(); + + _config = new RevalidationConfiguration + { + MinPackageEventRate = 100, + MaxPackageEventRate = 500, + }; + + _target = new RevalidationJobStateService( + _state.Object, + _config, + Mock.Of>()); + } + } + } +} diff --git a/tests/NuGet.Services.Revalidate.Tests/Services/RevalidationServiceFacts.cs b/tests/NuGet.Services.Revalidate.Tests/Services/RevalidationServiceFacts.cs index 839fea7ff..b4d72744c 100644 --- a/tests/NuGet.Services.Revalidate.Tests/Services/RevalidationServiceFacts.cs +++ b/tests/NuGet.Services.Revalidate.Tests/Services/RevalidationServiceFacts.cs @@ -2,9 +2,11 @@ // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; +using System.Collections.Generic; using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Moq; +using NuGet.Services.Logging; using NuGet.Services.Validation; using Xunit; @@ -14,6 +16,16 @@ public class RevalidationServiceFacts { public class TheRunAsyncMethod : FactsBase { + [Fact] + public async Task ThrowsIfNotInitialized() + { + Setup(isInitialized: false); + + var e = await Assert.ThrowsAsync(() => _target.RunAsync()); + + Assert.Equal("The revalidation service must be initialized before running revalidations", e.Message); + } + [Fact] public async Task ReturnsUnrecoverableError() { @@ -84,12 +96,24 @@ public async Task IfNotSingleton_ReturnsUnrecoverableError() var result = await _target.StartNextRevalidationAsync(); _singletonService.Verify(s => s.IsSingletonAsync(), Times.Once); - _throttler.Verify(t => t.IncreaseCapacityAsync(), Times.Never); + _jobState.Verify(s => s.IncreaseDesiredPackageEventRateAsync(), Times.Never); _validationEnqueuer.Verify(e => e.StartValidationAsync(It.IsAny()), Times.Never); Assert.Equal(RevalidationResult.UnrecoverableError, result); } + [Fact] + public async Task IfSingletonCheckThrows_ReturnsRetryLater() + { + // Arrange + Setup(singletonThrows: true); + + // Act & Assert + var result = await _target.StartNextRevalidationAsync(); + + Assert.Equal(RevalidationResult.RetryLater, result); + } + [Fact] public async Task IfKillswitchActive_ReturnsRetryLater() { @@ -99,13 +123,25 @@ public async Task IfKillswitchActive_ReturnsRetryLater() // Act & Assert var result = await _target.StartNextRevalidationAsync(); - _stateService.Verify(s => s.IsKillswitchActiveAsync(), Times.Once); - _throttler.Verify(t => t.IncreaseCapacityAsync(), Times.Never); + _jobState.Verify(s => s.IsKillswitchActiveAsync(), Times.Once); + _jobState.Verify(s => s.IncreaseDesiredPackageEventRateAsync(), Times.Never); _validationEnqueuer.Verify(e => e.StartValidationAsync(It.IsAny()), Times.Never); Assert.Equal(RevalidationResult.RetryLater, result); } + [Fact] + public async Task IfKillswitchThrows_ReturnsRetryLater() + { + // Arrange + Setup(killswitchThrows: true); + + // Act & Assert + var result = await _target.StartNextRevalidationAsync(); + + Assert.Equal(RevalidationResult.RetryLater, result); + } + [Fact] public async Task IfThrottled_ReturnsRetryLater() { @@ -116,12 +152,24 @@ public async Task IfThrottled_ReturnsRetryLater() var result = await _target.StartNextRevalidationAsync(); _throttler.Verify(s => s.IsThrottledAsync(), Times.Once); - _throttler.Verify(t => t.IncreaseCapacityAsync(), Times.Never); + _jobState.Verify(s => s.IncreaseDesiredPackageEventRateAsync(), Times.Never); _validationEnqueuer.Verify(e => e.StartValidationAsync(It.IsAny()), Times.Never); Assert.Equal(RevalidationResult.RetryLater, result); } + [Fact] + public async Task IfThrottledThrows_ReturnsRetryLater() + { + // Arrange + Setup(throttledThrows: true); + + // Act & Assert + var result = await _target.StartNextRevalidationAsync(); + + Assert.Equal(RevalidationResult.RetryLater, result); + } + [Fact] public async Task IfUnhealthy_ReturnsRetryLater() { @@ -131,13 +179,25 @@ public async Task IfUnhealthy_ReturnsRetryLater() // Act & Assert var result = await _target.StartNextRevalidationAsync(); - _throttler.Verify(t => t.IncreaseCapacityAsync(), Times.Never); + _jobState.Verify(s => s.IncreaseDesiredPackageEventRateAsync(), Times.Never); _healthService.Verify(h => h.IsHealthyAsync(), Times.Once); _validationEnqueuer.Verify(e => e.StartValidationAsync(It.IsAny()), Times.Never); Assert.Equal(RevalidationResult.RetryLater, result); } + [Fact] + public async Task IfUnhealthyThrows_ReturnsRetryLater() + { + // Arrange + Setup(healthyThrows: true); + + // Act & Assert + var result = await _target.StartNextRevalidationAsync(); + + Assert.Equal(RevalidationResult.RetryLater, result); + } + [Fact] public async Task IfRevalidationQueueEmpty_ReturnsRetryLater() { @@ -148,17 +208,29 @@ public async Task IfRevalidationQueueEmpty_ReturnsRetryLater() var result = await _target.StartNextRevalidationAsync(); _singletonService.Verify(s => s.IsSingletonAsync(), Times.Once); - _stateService.Verify(s => s.IsKillswitchActiveAsync(), Times.Exactly(2)); + _jobState.Verify(s => s.IsKillswitchActiveAsync(), Times.Exactly(2)); _throttler.Verify(s => s.IsThrottledAsync(), Times.Once); _healthService.Verify(h => h.IsHealthyAsync(), Times.Once); - _throttler.Verify(t => t.IncreaseCapacityAsync(), Times.Once); + _jobState.Verify(s => s.IncreaseDesiredPackageEventRateAsync(), Times.Once); _validationEnqueuer.Verify(e => e.StartValidationAsync(It.IsAny()), Times.Never); Assert.Equal(RevalidationResult.RetryLater, result); } + [Fact] + public async Task IfRevalidationQueueNextThrows_ReturnsRetryLater() + { + // Arrange + Setup(nextThrows: true); + + // Act & Assert + var result = await _target.StartNextRevalidationAsync(); + + Assert.Equal(RevalidationResult.RetryLater, result); + } + [Fact] public async Task StartsNextRevalidation() { @@ -174,8 +246,8 @@ public async Task StartsNextRevalidation() .Callback(() => enqueueStep = order++) .Returns(Task.CompletedTask); - _stateService - .Setup(s => s.MarkRevalidationAsEnqueuedAsync(It.IsAny())) + _packageState + .Setup(s => s.MarkPackageRevalidationAsEnqueuedAsync(It.IsAny())) .Callback(() => markStep = order++) .Returns(Task.CompletedTask); @@ -184,11 +256,11 @@ public async Task StartsNextRevalidation() // Assert _singletonService.Verify(s => s.IsSingletonAsync(), Times.Once); - _stateService.Verify(s => s.IsKillswitchActiveAsync(), Times.Exactly(2)); + _jobState.Verify(s => s.IsKillswitchActiveAsync(), Times.Exactly(2)); _throttler.Verify(s => s.IsThrottledAsync(), Times.Once); _healthService.Verify(h => h.IsHealthyAsync(), Times.Once); - _throttler.Verify(t => t.IncreaseCapacityAsync(), Times.Once); + _jobState.Verify(s => s.IncreaseDesiredPackageEventRateAsync(), Times.Once); _validationEnqueuer.Verify( e => e.StartValidationAsync(It.Is(m => @@ -197,7 +269,7 @@ public async Task StartsNextRevalidation() m.ValidationTrackingId == _revalidation.ValidationTrackingId.Value)), Times.Once); - _stateService.Verify(s => s.MarkRevalidationAsEnqueuedAsync(_revalidation), Times.Once); + _packageState.Verify(s => s.MarkPackageRevalidationAsEnqueuedAsync(_revalidation), Times.Once); _telemetryService.Verify(t => t.TrackPackageRevalidationStarted(_revalidation.PackageId, _revalidation.PackageNormalizedVersion)); Assert.Equal(RevalidationResult.RevalidationEnqueued, result); @@ -208,7 +280,8 @@ public async Task StartsNextRevalidation() public class FactsBase { - protected readonly Mock _stateService; + protected readonly Mock _jobState; + protected readonly Mock _packageState; protected readonly Mock _singletonService; protected readonly Mock _throttler; protected readonly Mock _healthService; @@ -223,7 +296,8 @@ public class FactsBase public FactsBase() { - _stateService = new Mock(); + _jobState = new Mock(); + _packageState = new Mock(); _singletonService = new Mock(); _throttler = new Mock(); _healthService = new Mock(); @@ -231,6 +305,14 @@ public FactsBase() _validationEnqueuer = new Mock(); _telemetryService = new Mock(); + _telemetryService + .Setup(t => t.TrackStartNextRevalidationOperation()) + .Returns(new DurationMetric( + Mock.Of(), + "Name", + new StartNextRevalidationOperation(), + Mock.Of>>())); + _config = new RevalidationConfiguration { ShutdownWaitInterval = TimeSpan.MinValue, @@ -244,7 +326,8 @@ public FactsBase() }; _target = new RevalidationService( - _stateService.Object, + _jobState.Object, + _packageState.Object, _singletonService.Object, _throttler.Object, _healthService.Object, @@ -255,13 +338,35 @@ public FactsBase() Mock.Of>()); } - protected void Setup(bool isSingleton = true, bool killswitchActive = false, bool isThrottled = false, bool isHealthy = true, PackageRevalidation next = null) + protected void Setup( + bool isInitialized = true, + bool isSingleton = true, + bool killswitchActive = false, + bool isThrottled = false, + bool isHealthy = true, + PackageRevalidation next = null, + bool initializedThrows = false, + bool singletonThrows = false, + bool killswitchThrows = false, + bool throttledThrows = false, + bool healthyThrows = false, + bool nextThrows = false) { + _jobState.Setup(s => s.IsInitializedAsync()).ReturnsAsync(isInitialized); _singletonService.Setup(s => s.IsSingletonAsync()).ReturnsAsync(isSingleton); - _stateService.Setup(s => s.IsKillswitchActiveAsync()).ReturnsAsync(killswitchActive); + _jobState.Setup(s => s.IsKillswitchActiveAsync()).ReturnsAsync(killswitchActive); _throttler.Setup(t => t.IsThrottledAsync()).ReturnsAsync(isThrottled); _healthService.Setup(t => t.IsHealthyAsync()).ReturnsAsync(isHealthy); _revalidationQueue.Setup(q => q.NextOrNullAsync()).ReturnsAsync(next); + + var exception = new Exception(); + + if (initializedThrows) _jobState.Setup(s => s.IsInitializedAsync()).ThrowsAsync(exception); + if (singletonThrows) _singletonService.Setup(s => s.IsSingletonAsync()).ThrowsAsync(exception); + if (killswitchThrows) _jobState.Setup(s => s.IsKillswitchActiveAsync()).ThrowsAsync(exception); + if (throttledThrows) _throttler.Setup(t => t.IsThrottledAsync()).ThrowsAsync(exception); + if (healthyThrows) _healthService.Setup(t => t.IsHealthyAsync()).ThrowsAsync(exception); + if (nextThrows) _revalidationQueue.Setup(q => q.NextOrNullAsync()).ThrowsAsync(exception); } protected void SetupUnrecoverableErrorResult() diff --git a/tests/NuGet.Services.Revalidate.Tests/Services/RevalidationThrottlerFacts.cs b/tests/NuGet.Services.Revalidate.Tests/Services/RevalidationThrottlerFacts.cs new file mode 100644 index 000000000..7b5da8254 --- /dev/null +++ b/tests/NuGet.Services.Revalidate.Tests/Services/RevalidationThrottlerFacts.cs @@ -0,0 +1,64 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Moq; +using Xunit; + +namespace NuGet.Services.Revalidate.Tests.Services +{ + public class RevalidationThrottlerFacts + { + public class TheIsThrottledAsyncMethod + { + private readonly Mock _settings; + private readonly Mock _state; + private readonly RevalidationConfiguration _config; + + private readonly IRevalidationThrottler _target; + + public TheIsThrottledAsyncMethod() + { + _settings = new Mock(); + _state = new Mock(); + + _config = new RevalidationConfiguration(); + + _target = new RevalidationThrottler( + _settings.Object, + _state.Object, + _config, + Mock.Of>()); + } + + [Fact] + public async Task ReturnsTrueIfRecentRevalidationsMoreThanDesiredRate() + { + // Arrange + _settings.Setup(s => s.GetDesiredPackageEventRateAsync()).ReturnsAsync(50); + _state.Setup(s => s.CountRevalidationsEnqueuedInPastHourAsync()).ReturnsAsync(100); + + // Act & Assert + Assert.True(await _target.IsThrottledAsync()); + + _settings.Verify(s => s.GetDesiredPackageEventRateAsync(), Times.Once); + _state.Verify(s => s.CountRevalidationsEnqueuedInPastHourAsync(), Times.Once); + } + + [Fact] + public async Task ReturnsFalseIfRecentRevalidationsLessThanDesiredRate() + { + // Arrange + _settings.Setup(s => s.GetDesiredPackageEventRateAsync()).ReturnsAsync(100); + _state.Setup(s => s.CountRevalidationsEnqueuedInPastHourAsync()).ReturnsAsync(50); + + // Act & Assert + Assert.False(await _target.IsThrottledAsync()); + + _settings.Verify(s => s.GetDesiredPackageEventRateAsync(), Times.Once); + _state.Verify(s => s.CountRevalidationsEnqueuedInPastHourAsync(), Times.Once); + } + } + } +} From d4e769d093e6d33eeaab3aba81b098d2b340d28f Mon Sep 17 00:00:00 2001 From: Christy Henriksson Date: Tue, 24 Jul 2018 10:21:21 -0700 Subject: [PATCH 2/3] Update jobs to latest SQL AAD (#485) --- src/ArchivePackages/ArchivePackages.Job.cs | 26 ++-- src/ArchivePackages/ArchivePackages.csproj | 3 - .../Gallery.CredentialExpiration.csproj | 3 - .../GalleryCredentialExpiration.cs | 12 +- src/Gallery.CredentialExpiration/Job.cs | 19 ++- .../DeleteExpiredVerificationKeysTask.cs | 5 +- .../Gallery.Maintenance.csproj | 3 - src/Gallery.Maintenance/Job.cs | 10 +- .../SqlConnectionStringBuilderExtensions.cs | 23 --- src/NuGet.Jobs.Common/JobBase.cs | 136 ++++++++++++++++++ .../NuGet.Jobs.Common.csproj | 4 +- .../Job.cs | 23 ++- ...et.Services.Validation.Orchestrator.csproj | 3 - .../Job.cs | 11 +- ...NuGet.SupportRequests.Notifications.csproj | 3 - .../ScheduledTaskFactory.cs | 14 +- .../SupportRequestRepository.cs | 20 +-- .../Tasks/OnCallDailyNotificationTask.cs | 6 +- ...upportRequestsNotificationScheduledTask.cs | 11 +- .../Tasks/WeeklySummaryNotificationTask.cs | 6 +- src/Search.GenerateAuxiliaryData/Job.cs | 51 +++++-- .../NestedJArrayExporter.cs | 12 +- .../RankingsExporter.cs | 6 +- .../Search.GenerateAuxiliaryData.csproj | 3 - .../SqlExporter.cs | 23 +-- .../VerifiedPackagesExporter.cs | 6 +- .../Job.cs | 21 +-- ...tats.AggregateCdnDownloadsInGallery.csproj | 3 - .../DownloadCountReport.cs | 18 +-- .../DownloadsPerToolVersionReport.cs | 18 +-- .../GalleryTotalsReport.cs | 30 ++-- .../Job.cs | 65 +++++---- .../ReportBase.cs | 11 +- .../ReportDataCollector.cs | 27 ++-- ...tats.CreateAzureCdnWarehouseReports.csproj | 3 - .../DataImporter.cs | 11 +- src/Stats.ImportAzureCdnStatistics/Job.cs | 15 +- .../Stats.ImportAzureCdnStatistics.csproj | 3 - .../Warehouse.cs | 24 ++-- .../RefreshClientDimensionJob.cs | 9 +- .../Stats.RefreshClientDimension.csproj | 3 - src/Stats.RollUpDownloadFacts/Job.cs | 9 +- .../Stats.RollUpDownloadFacts.csproj | 3 - .../UpdateLicenseReports.Job.cs | 27 ++-- .../UpdateLicenseReports.csproj | 3 - .../JsonConfigurationJob.cs | 19 +-- .../Validation.Common.Job.csproj | 3 - .../Job.cs | 2 +- ...ion.PackageSigning.ProcessSignature.csproj | 5 - .../NuGet.Services.Revalidate.Tests.csproj | 3 + .../RankingsExporterTests.cs | 10 +- .../VerifiedPackagesExporterTests.cs | 10 +- ...on.PackageSigning.ScanAndSign.Tests.csproj | 3 + 53 files changed, 452 insertions(+), 348 deletions(-) delete mode 100644 src/NuGet.Jobs.Common/Extensions/SqlConnectionStringBuilderExtensions.cs diff --git a/src/ArchivePackages/ArchivePackages.Job.cs b/src/ArchivePackages/ArchivePackages.Job.cs index 97f7528e6..4e036b203 100644 --- a/src/ArchivePackages/ArchivePackages.Job.cs +++ b/src/ArchivePackages/ArchivePackages.Job.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Generic; using System.ComponentModel.Design; +using System.Data.SqlClient; using System.Diagnostics.Tracing; using System.Linq; using System.Threading.Tasks; @@ -12,8 +13,6 @@ using Microsoft.WindowsAzure.Storage.Blob; using Newtonsoft.Json.Linq; using NuGet.Jobs; -using NuGet.Services.KeyVault; -using NuGet.Services.Sql; namespace ArchivePackages { @@ -53,8 +52,6 @@ public class Job : JobBase /// Blob containing the cursor data. Cursor data comprises of cursorDateTime /// public string CursorBlobName { get; set; } - - private ISqlConnectionFactory _packageDbConnectionFactory; protected CloudBlobContainer SourceContainer { get; private set; } @@ -62,13 +59,13 @@ public class Job : JobBase protected CloudBlobContainer SecondaryDestinationContainer { get; private set; } + private SqlConnectionStringBuilder PackageDatabase { get; set; } + public Job() : base(JobEventSource.Log) { } public override void Init(IServiceContainer serviceContainer, IDictionary jobArgsDictionary) { - var secretInjector = (ISecretInjector)serviceContainer.GetService(typeof(ISecretInjector)); - var packageDbConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.PackageDatabase); - _packageDbConnectionFactory = new AzureSqlConnectionFactory(packageDbConnectionString, secretInjector); + PackageDatabase = RegisterDatabase(serviceContainer, jobArgsDictionary, JobArgumentNames.PackageDatabase); Source = CloudStorageAccount.Parse( JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.Source)); @@ -92,13 +89,18 @@ public override void Init(IServiceContainer serviceContainer, IDictionary packages; - using (var connection = await _packageDbConnectionFactory.CreateAsync()) + using (var connection = await OpenSqlConnectionAsync(JobArgumentNames.PackageDatabase)) { packages = (await connection.QueryAsync(@" SELECT pr.Id, p.NormalizedVersion AS Version, p.Hash, p.LastEdited, p.Published @@ -135,7 +138,8 @@ FROM Packages p WHERE Published > @cursorDateTime OR LastEdited > @cursorDateTime", new { cursorDateTime = cursorDateTime })) .ToList(); } - JobEventSourceLog.GatheredPackagesToArchiveFromDb(packages.Count, _packageDbConnectionFactory.DataSource, _packageDbConnectionFactory.InitialCatalog); + + JobEventSourceLog.GatheredPackagesToArchiveFromDb(packages.Count, PackageDatabase.DataSource, PackageDatabase.InitialCatalog); var archiveSet = packages .AsParallel() diff --git a/src/ArchivePackages/ArchivePackages.csproj b/src/ArchivePackages/ArchivePackages.csproj index efb41445d..900001677 100644 --- a/src/ArchivePackages/ArchivePackages.csproj +++ b/src/ArchivePackages/ArchivePackages.csproj @@ -75,9 +75,6 @@ 9.0.1 - - 2.25.0-master-30453 - 4.3.3 diff --git a/src/Gallery.CredentialExpiration/Gallery.CredentialExpiration.csproj b/src/Gallery.CredentialExpiration/Gallery.CredentialExpiration.csproj index 696c6c793..5fc084f77 100644 --- a/src/Gallery.CredentialExpiration/Gallery.CredentialExpiration.csproj +++ b/src/Gallery.CredentialExpiration/Gallery.CredentialExpiration.csproj @@ -92,9 +92,6 @@ 9.0.1 - - 2.25.0-master-30263 - 2.1.3 diff --git a/src/Gallery.CredentialExpiration/GalleryCredentialExpiration.cs b/src/Gallery.CredentialExpiration/GalleryCredentialExpiration.cs index e1021e836..e9db4dc9a 100644 --- a/src/Gallery.CredentialExpiration/GalleryCredentialExpiration.cs +++ b/src/Gallery.CredentialExpiration/GalleryCredentialExpiration.cs @@ -6,7 +6,6 @@ using System.Data.SqlClient; using System.Linq; using System.Threading.Tasks; -using NuGet.Services.Sql; using Gallery.CredentialExpiration.Models; namespace Gallery.CredentialExpiration @@ -14,12 +13,15 @@ namespace Gallery.CredentialExpiration public class GalleryCredentialExpiration : ICredentialExpirationExporter { private readonly CredentialExpirationJobMetadata _jobMetadata; - private readonly ISqlConnectionFactory _galleryDatabase; - public GalleryCredentialExpiration(CredentialExpirationJobMetadata jobMetadata, ISqlConnectionFactory galleryDatabase) + private Func> OpenGallerySqlConnectionAsync { get; } + + public GalleryCredentialExpiration( + CredentialExpirationJobMetadata jobMetadata, + Func> openGallerySqlConnectionAsync) { _jobMetadata = jobMetadata; - _galleryDatabase = galleryDatabase; + OpenGallerySqlConnectionAsync = openGallerySqlConnectionAsync; } /// @@ -51,7 +53,7 @@ public async Task> GetCredentialsAsync(TimeSpan time var minNotificationDate = ConvertToString(GetMinNotificationDate()); // Connect to database - using (var galleryConnection = await _galleryDatabase.CreateAsync()) + using (var galleryConnection = await OpenGallerySqlConnectionAsync()) { // Fetch credentials that expire in _warnDaysBeforeExpiration days // + the user's e-mail address diff --git a/src/Gallery.CredentialExpiration/Job.cs b/src/Gallery.CredentialExpiration/Job.cs index 4290dba1b..e0340eb0a 100644 --- a/src/Gallery.CredentialExpiration/Job.cs +++ b/src/Gallery.CredentialExpiration/Job.cs @@ -2,7 +2,6 @@ // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; -using System.Collections.Concurrent; using System.Collections.Generic; using System.ComponentModel.Design; using System.Data.SqlClient; @@ -15,10 +14,7 @@ using Microsoft.Extensions.Logging; using Microsoft.WindowsAzure.Storage; using Newtonsoft.Json; -using Newtonsoft.Json.Linq; using NuGet.Jobs; -using NuGet.Services.KeyVault; -using NuGet.Services.Sql; using NuGet.Services.Storage; namespace Gallery.CredentialExpiration @@ -34,8 +30,6 @@ public class Job : JobBase private string _galleryBrand; private string _galleryAccountUrl; - private ISqlConnectionFactory _galleryDatabase; - private string _mailFrom; private SmtpClient _smtpClient; @@ -47,9 +41,7 @@ public override void Init(IServiceContainer serviceContainer, IDictionary OpenGallerySqlConnectionAsync() + { + return OpenSqlConnectionAsync(JobArgumentNames.GalleryDatabase); + } + public override async Task Run() { var jobRunTime = DateTimeOffset.UtcNow; // Default values var jobCursor = new JobRunTimeCursor( jobCursorTime: jobRunTime, maxProcessedCredentialsTime: jobRunTime ); - var galleryCredentialExpiration = new GalleryCredentialExpiration(new CredentialExpirationJobMetadata(jobRunTime, _warnDaysBeforeExpiration, jobCursor), _galleryDatabase); + var galleryCredentialExpiration = new GalleryCredentialExpiration(new CredentialExpirationJobMetadata(jobRunTime, _warnDaysBeforeExpiration, jobCursor), OpenGallerySqlConnectionAsync); try { @@ -89,7 +86,7 @@ public override async Task Run() // Load from cursor // Throw if the schema is not correct to ensure that not-intended emails are sent. jobCursor = JsonConvert.DeserializeObject(content, new JsonSerializerSettings() { MissingMemberHandling = MissingMemberHandling.Error }); - galleryCredentialExpiration = new GalleryCredentialExpiration(new CredentialExpirationJobMetadata(jobRunTime, _warnDaysBeforeExpiration, jobCursor), _galleryDatabase); + galleryCredentialExpiration = new GalleryCredentialExpiration(new CredentialExpirationJobMetadata(jobRunTime, _warnDaysBeforeExpiration, jobCursor), OpenGallerySqlConnectionAsync); } // Connect to database diff --git a/src/Gallery.Maintenance/DeleteExpiredVerificationKeysTask.cs b/src/Gallery.Maintenance/DeleteExpiredVerificationKeysTask.cs index dc0d07ca5..14e4d90d9 100644 --- a/src/Gallery.Maintenance/DeleteExpiredVerificationKeysTask.cs +++ b/src/Gallery.Maintenance/DeleteExpiredVerificationKeysTask.cs @@ -9,6 +9,7 @@ using System.Threading.Tasks; using Gallery.Maintenance.Models; using Microsoft.Extensions.Logging; +using NuGet.Jobs; namespace Gallery.Maintenance { @@ -37,7 +38,7 @@ public override async Task RunAsync(Job job) { IEnumerable expiredKeys; - using (var connection = await job.GalleryDatabase.CreateAsync()) + using (var connection = await job.OpenSqlConnectionAsync(JobArgumentNames.GalleryDatabase)) { expiredKeys = await connection.QueryWithRetryAsync( SelectQuery, @@ -59,7 +60,7 @@ public override async Task RunAsync(Job job) if (expectedRowCount > 0) { - using (var connection = await job.GalleryDatabase.CreateAsync()) + using (var connection = await job.OpenSqlConnectionAsync(JobArgumentNames.GalleryDatabase)) { using (var transaction = connection.BeginTransaction()) { diff --git a/src/Gallery.Maintenance/Gallery.Maintenance.csproj b/src/Gallery.Maintenance/Gallery.Maintenance.csproj index b11f12415..e6a3ed08b 100644 --- a/src/Gallery.Maintenance/Gallery.Maintenance.csproj +++ b/src/Gallery.Maintenance/Gallery.Maintenance.csproj @@ -67,9 +67,6 @@ 9.0.1 - - 2.25.0-master-30263 - 4.3.3 diff --git a/src/Gallery.Maintenance/Job.cs b/src/Gallery.Maintenance/Job.cs index 488a49b92..55d64f291 100644 --- a/src/Gallery.Maintenance/Job.cs +++ b/src/Gallery.Maintenance/Job.cs @@ -4,13 +4,11 @@ using System; using System.Collections.Generic; using System.ComponentModel.Design; -using System.Data; using System.Linq; using System.Threading.Tasks; using Microsoft.Extensions.Logging; using NuGet.Jobs; using NuGet.Services.KeyVault; -using NuGet.Services.Sql; namespace Gallery.Maintenance { @@ -19,15 +17,9 @@ namespace Gallery.Maintenance /// public class Job : JobBase { - - public ISqlConnectionFactory GalleryDatabase { get; private set; } - public override void Init(IServiceContainer serviceContainer, IDictionary jobArgsDictionary) { - var secretInjector = (ISecretInjector)serviceContainer.GetService(typeof(ISecretInjector)); - var databaseConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.GalleryDatabase); - - GalleryDatabase = new AzureSqlConnectionFactory(databaseConnectionString, secretInjector); + RegisterDatabase(serviceContainer, jobArgsDictionary, JobArgumentNames.GalleryDatabase); } public override async Task Run() diff --git a/src/NuGet.Jobs.Common/Extensions/SqlConnectionStringBuilderExtensions.cs b/src/NuGet.Jobs.Common/Extensions/SqlConnectionStringBuilderExtensions.cs deleted file mode 100644 index 608481d2c..000000000 --- a/src/NuGet.Jobs.Common/Extensions/SqlConnectionStringBuilderExtensions.cs +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using System.Threading.Tasks; - -// ReSharper disable once CheckNamespace -namespace System.Data.SqlClient -{ - public static class SqlConnectionStringBuilderExtensions - { - public static Task ConnectTo(this SqlConnectionStringBuilder self) - { - return ConnectTo(self.ConnectionString); - } - - private static async Task ConnectTo(string connection) - { - var c = new SqlConnection(connection); - await c.OpenAsync().ConfigureAwait(continueOnCapturedContext: false); - return c; - } - } -} \ No newline at end of file diff --git a/src/NuGet.Jobs.Common/JobBase.cs b/src/NuGet.Jobs.Common/JobBase.cs index e16184854..b16f92846 100644 --- a/src/NuGet.Jobs.Common/JobBase.cs +++ b/src/NuGet.Jobs.Common/JobBase.cs @@ -1,11 +1,18 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +using System; using System.Collections.Generic; using System.ComponentModel.Design; +using System.Data.SqlClient; using System.Diagnostics.Tracing; using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using NuGet.Jobs.Configuration; +using NuGet.Services.KeyVault; +using NuGet.Services.Sql; namespace NuGet.Jobs { @@ -13,6 +20,8 @@ public abstract class JobBase { private readonly EventSource _jobEventSource; + private Dictionary _sqlConnectionFactories; + protected JobBase() : this(null) { @@ -22,6 +31,7 @@ protected JobBase(EventSource jobEventSource) { JobName = GetType().ToString(); _jobEventSource = jobEventSource; + _sqlConnectionFactories = new Dictionary(); } public string JobName { get; private set; } @@ -36,6 +46,132 @@ public void SetLogger(ILoggerFactory loggerFactory, ILogger logger) Logger = logger; } + /// + /// Test connection early to fail fast, and log connection diagnostics. + /// + private async Task TestConnection(ISqlConnectionFactory connectionFactory) + { + try + { + using (var connection = await connectionFactory.OpenAsync()) + using (var cmd = new SqlCommand("SELECT CONCAT(CURRENT_USER, '/', SYSTEM_USER)", connection)) + { + var result = cmd.ExecuteScalar(); + var user = result.ToString(); + Logger.LogInformation("Connected to database {DataSource}/{InitialCatalog} as {User}", + connectionFactory.DataSource, connectionFactory.InitialCatalog, user); + } + } + catch (Exception e) + { + Logger.LogError(0, e, "Failed to connect to database {DataSource}/{InitialCatalog}", + connectionFactory.DataSource, connectionFactory.InitialCatalog); + } + } + + /// + /// Initializes an , for use by validation jobs. + /// + /// ConnectionStringBuilder, used for diagnostics. + public SqlConnectionStringBuilder RegisterDatabase(IServiceProvider serviceProvider) + where T : IDbConfiguration + { + if (serviceProvider == null) + { + throw new ArgumentNullException(nameof(serviceProvider)); + } + + var secretInjector = serviceProvider.GetRequiredService(); + var connectionString = serviceProvider.GetRequiredService>().Value.ConnectionString; + var connectionFactory = new AzureSqlConnectionFactory(connectionString, secretInjector); + + return RegisterDatabase(nameof(T), connectionString, secretInjector); + } + + /// + /// Initializes an , for use by non-validation jobs. + /// + /// ConnectionStringBuilder, used for diagnostics. + public SqlConnectionStringBuilder RegisterDatabase(IServiceContainer serviceContainer, IDictionary jobArgsDictionary, string connectionStringArgName) + { + if (serviceContainer == null) + { + throw new ArgumentNullException(nameof(serviceContainer)); + } + + if (jobArgsDictionary == null) + { + throw new ArgumentNullException(nameof(jobArgsDictionary)); + } + + if (string.IsNullOrEmpty(connectionStringArgName)) + { + throw new ArgumentException("Argument cannot be null or empty.", nameof(connectionStringArgName)); + } + + var secretInjector = (ISecretInjector)serviceContainer.GetService(typeof(ISecretInjector)); + var connectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, connectionStringArgName); + + return RegisterDatabase(connectionStringArgName, connectionString, secretInjector); + } + + /// + /// Register a job database at initialization time. Each call should overwrite any existing + /// registration because calls on every iteration. + /// + /// ConnectionStringBuilder, used for diagnostics. + private SqlConnectionStringBuilder RegisterDatabase(string name, string connectionString, ISecretInjector secretInjector) + { + var connectionFactory = new AzureSqlConnectionFactory(connectionString, secretInjector, Logger); + _sqlConnectionFactories[name] = connectionFactory; + + Task.Run(() => TestConnection(connectionFactory)).Wait(); + + return connectionFactory.SqlConnectionStringBuilder; + } + + /// + /// Create a SqlConnection, for use by validation jobs. + /// + public Task CreateSqlConnectionAsync() + where T : IDbConfiguration + { + var name = nameof(T); + if (!_sqlConnectionFactories.ContainsKey(name)) + { + throw new InvalidOperationException($"Database {name} has not been registered."); + } + + return _sqlConnectionFactories[name].CreateAsync(); + } + + /// + /// Synchronous creation of a SqlConnection, for use by validation jobs. + /// + public SqlConnection CreateSqlConnection() + where T : IDbConfiguration + { + return Task.Run(() => CreateSqlConnectionAsync()).Result; + } + + /// + /// Creates and opens a SqlConnection, for use by non-validation jobs. + /// + public Task OpenSqlConnectionAsync(string connectionStringArgName) + { + if (string.IsNullOrEmpty(connectionStringArgName)) + { + throw new ArgumentException("Argument cannot be null or empty.", nameof(connectionStringArgName)); + } + + if (!_sqlConnectionFactories.ContainsKey(connectionStringArgName)) + { + throw new InvalidOperationException($"Database {connectionStringArgName} has not been registered."); + } + + return _sqlConnectionFactories[connectionStringArgName].OpenAsync(); + } + public abstract void Init(IServiceContainer serviceContainer, IDictionary jobArgsDictionary); public abstract Task Run(); diff --git a/src/NuGet.Jobs.Common/NuGet.Jobs.Common.csproj b/src/NuGet.Jobs.Common/NuGet.Jobs.Common.csproj index 2dd0251bc..1c1527895 100644 --- a/src/NuGet.Jobs.Common/NuGet.Jobs.Common.csproj +++ b/src/NuGet.Jobs.Common/NuGet.Jobs.Common.csproj @@ -49,7 +49,6 @@ - @@ -82,6 +81,9 @@ 2.27.0 + + 2.27.0 + 4.3.3 diff --git a/src/NuGet.Services.Validation.Orchestrator/Job.cs b/src/NuGet.Services.Validation.Orchestrator/Job.cs index 8e6c9f831..b3c5638c3 100644 --- a/src/NuGet.Services.Validation.Orchestrator/Job.cs +++ b/src/NuGet.Services.Validation.Orchestrator/Job.cs @@ -4,7 +4,6 @@ using System; using System.Collections.Generic; using System.ComponentModel.Design; -using System.Data.Common; using System.Net; using System.Net.Http; using System.Reflection; @@ -31,7 +30,6 @@ using NuGet.Services.KeyVault; using NuGet.Services.Logging; using NuGet.Services.ServiceBus; -using NuGet.Services.Sql; using NuGet.Services.Validation.Orchestrator.PackageSigning.ScanAndSign; using NuGet.Services.Validation.Orchestrator.Telemetry; using NuGet.Services.Validation.PackageSigning.ProcessSignature; @@ -86,6 +84,12 @@ public override void Init(IServiceContainer serviceContainer, IDictionary(_serviceProvider); + RegisterDatabase(_serviceProvider); + } ConfigurationValidated = false; } @@ -158,15 +162,6 @@ private void ConfigureLibraries(IServiceCollection services) services.AddLogging(); } - private DbConnection CreateDbConnection(IServiceProvider serviceProvider) where T : IDbConfiguration - { - var connectionString = serviceProvider.GetRequiredService>().Value.ConnectionString; - var connectionFactory = new AzureSqlConnectionFactory(connectionString, - serviceProvider.GetRequiredService()); - - return Task.Run(() => connectionFactory.CreateAsync()).Result; - } - private void ConfigureJobServices(IServiceCollection services, IConfigurationRoot configurationRoot) { services.Configure(configurationRoot.GetSection(ConfigurationSectionName)); @@ -184,15 +179,15 @@ private void ConfigureJobServices(IServiceCollection services, IConfigurationRoo services.AddTransient(); services.AddTransient(); - + services.AddScoped(serviceProvider => new NuGetGallery.EntitiesContext( - CreateDbConnection(serviceProvider), + CreateSqlConnection(), readOnly: false) ); services.AddScoped(serviceProvider => new ValidationEntitiesContext( - CreateDbConnection(serviceProvider))); + CreateSqlConnection())); services.AddScoped(serviceProvider => serviceProvider.GetRequiredService()); diff --git a/src/NuGet.Services.Validation.Orchestrator/NuGet.Services.Validation.Orchestrator.csproj b/src/NuGet.Services.Validation.Orchestrator/NuGet.Services.Validation.Orchestrator.csproj index ebf1fb92c..e7ba23387 100644 --- a/src/NuGet.Services.Validation.Orchestrator/NuGet.Services.Validation.Orchestrator.csproj +++ b/src/NuGet.Services.Validation.Orchestrator/NuGet.Services.Validation.Orchestrator.csproj @@ -123,9 +123,6 @@ 1.2.0 - - 2.26.0-master-34394 - 2.26.0-master-34394 diff --git a/src/NuGet.SupportRequests.Notifications/Job.cs b/src/NuGet.SupportRequests.Notifications/Job.cs index 99de91ce9..8f3292405 100644 --- a/src/NuGet.SupportRequests.Notifications/Job.cs +++ b/src/NuGet.SupportRequests.Notifications/Job.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Generic; using System.ComponentModel.Design; +using System.Data.SqlClient; using System.Threading.Tasks; using NuGet.Jobs; @@ -24,11 +25,19 @@ public override void Init(IServiceContainer serviceContainer, IDictionary OpenSupportSqlConnectionAsync() + { + return OpenSqlConnectionAsync(JobArgumentNames.SourceDatabase); } public override async Task Run() { - var scheduledTask = ScheduledTaskFactory.Create(_serviceContainer, _jobArgsDictionary, LoggerFactory); + var scheduledTask = ScheduledTaskFactory.Create(_serviceContainer, _jobArgsDictionary, + OpenSupportSqlConnectionAsync, LoggerFactory); await scheduledTask.RunAsync(); } diff --git a/src/NuGet.SupportRequests.Notifications/NuGet.SupportRequests.Notifications.csproj b/src/NuGet.SupportRequests.Notifications/NuGet.SupportRequests.Notifications.csproj index a72a38207..44a43935d 100644 --- a/src/NuGet.SupportRequests.Notifications/NuGet.SupportRequests.Notifications.csproj +++ b/src/NuGet.SupportRequests.Notifications/NuGet.SupportRequests.Notifications.csproj @@ -106,9 +106,6 @@ 9.0.1 - - 2.25.0-master-30453 - diff --git a/src/NuGet.SupportRequests.Notifications/ScheduledTaskFactory.cs b/src/NuGet.SupportRequests.Notifications/ScheduledTaskFactory.cs index 67586efb9..42834faf2 100644 --- a/src/NuGet.SupportRequests.Notifications/ScheduledTaskFactory.cs +++ b/src/NuGet.SupportRequests.Notifications/ScheduledTaskFactory.cs @@ -4,6 +4,8 @@ using System; using System.Collections.Generic; using System.ComponentModel.Design; +using System.Data.SqlClient; +using System.Threading.Tasks; using Microsoft.Extensions.Logging; namespace NuGet.SupportRequests.Notifications @@ -12,7 +14,11 @@ internal class ScheduledTaskFactory { private const string _tasksNamespace = "NuGet.SupportRequests.Notifications.Tasks"; - public static IScheduledTask Create(IServiceContainer serviceContainer, IDictionary jobArgsDictionary, ILoggerFactory loggerFactory) + public static IScheduledTask Create( + IServiceContainer serviceContainer, + IDictionary jobArgsDictionary, + Func> openSupportSqlConnectionAsync, + ILoggerFactory loggerFactory) { if (jobArgsDictionary == null) { @@ -25,7 +31,8 @@ public static IScheduledTask Create(IServiceContainer serviceContainer, IDiction } var scheduledTaskName = jobArgsDictionary[JobArgumentNames.ScheduledTask]; - var scheduledTask = GetTaskOfType(scheduledTaskName, serviceContainer, jobArgsDictionary, loggerFactory); + var scheduledTask = GetTaskOfType(scheduledTaskName, serviceContainer, jobArgsDictionary, + openSupportSqlConnectionAsync, loggerFactory); return scheduledTask; } @@ -34,6 +41,7 @@ private static IScheduledTask GetTaskOfType( string taskName, IServiceContainer serviceContainer, IDictionary jobArgsDictionary, + Func> openSupportSqlConnectionAsync, ILoggerFactory loggerFactory) { if (string.IsNullOrEmpty(taskName)) @@ -51,7 +59,7 @@ private static IScheduledTask GetTaskOfType( IScheduledTask scheduledTask; if (scheduledTaskType != null && typeof(IScheduledTask).IsAssignableFrom(scheduledTaskType)) { - var args = new object[] { serviceContainer, jobArgsDictionary, loggerFactory }; + var args = new object[] { serviceContainer, jobArgsDictionary, openSupportSqlConnectionAsync, loggerFactory }; scheduledTask = (IScheduledTask)Activator.CreateInstance(scheduledTaskType, args); } else diff --git a/src/NuGet.SupportRequests.Notifications/SupportRequestRepository.cs b/src/NuGet.SupportRequests.Notifications/SupportRequestRepository.cs index 1dcea245b..d4c3f8f20 100644 --- a/src/NuGet.SupportRequests.Notifications/SupportRequestRepository.cs +++ b/src/NuGet.SupportRequests.Notifications/SupportRequestRepository.cs @@ -8,7 +8,6 @@ using System.Linq; using System.Threading.Tasks; using Microsoft.Extensions.Logging; -using NuGet.Services.Sql; using NuGet.SupportRequests.Notifications.Models; namespace NuGet.SupportRequests.Notifications @@ -20,29 +19,24 @@ internal class SupportRequestRepository private const string _parameterNamePagerDutyUsername = "pagerDutyUserName"; private readonly DateTime _defaultSqlDateTime = new DateTime(1900, 1, 1, 0, 0, 0, DateTimeKind.Utc); private readonly ILogger _logger; - private readonly ISqlConnectionFactory _supportDbConnectionFactory; + private readonly Func> _openSupportSqlConnectionAsync; public SupportRequestRepository( - ILoggerFactory loggerFactory, - ISqlConnectionFactory supportDbConnectionFactory) + Func> openSupportSqlConnectionAsync, + ILoggerFactory loggerFactory) { if (loggerFactory == null) { throw new ArgumentNullException(nameof(loggerFactory)); } - if (supportDbConnectionFactory == null) - { - throw new ArgumentNullException(nameof(supportDbConnectionFactory)); - } - _logger = loggerFactory.CreateLogger(); - _supportDbConnectionFactory = supportDbConnectionFactory; + _openSupportSqlConnectionAsync = openSupportSqlConnectionAsync; } - internal async Task OpenConnectionAsync() + internal async Task OpenSupportSqlConnectionAsync() { - var connection = await _supportDbConnectionFactory.CreateAsync(); + var connection = await _openSupportSqlConnectionAsync(); connection.InfoMessage += OnSqlConnectionInfoMessage; return connection; @@ -181,7 +175,7 @@ private async Task EnsureConnectionOpenAsync(SqlConnection connec { if (connection == null) { - connection = await OpenConnectionAsync(); + connection = await OpenSupportSqlConnectionAsync(); } else if (connection.State != ConnectionState.Open) { diff --git a/src/NuGet.SupportRequests.Notifications/Tasks/OnCallDailyNotificationTask.cs b/src/NuGet.SupportRequests.Notifications/Tasks/OnCallDailyNotificationTask.cs index 35892f72d..82d9b09a1 100644 --- a/src/NuGet.SupportRequests.Notifications/Tasks/OnCallDailyNotificationTask.cs +++ b/src/NuGet.SupportRequests.Notifications/Tasks/OnCallDailyNotificationTask.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Generic; using System.ComponentModel.Design; +using System.Data.SqlClient; using System.Linq; using System.Text; using System.Threading.Tasks; @@ -25,8 +26,9 @@ internal class OnCallDailyNotificationTask public OnCallDailyNotificationTask( IServiceContainer serviceContainer, IDictionary jobArgsDictionary, + Func> openSupportSqlConnectionAsync, ILoggerFactory loggerFactory) - : base(serviceContainer, jobArgsDictionary, loggerFactory) + : base(serviceContainer, jobArgsDictionary, openSupportSqlConnectionAsync, loggerFactory) { var pagerDutyConfiguration = new PagerDutyConfiguration( jobArgsDictionary[_argumentNamePagerDutyAccountName], @@ -44,7 +46,7 @@ protected override async Task BuildNotification( var targetEmailAddress = string.Format(_targetEmailAddressFormat, onCallAlias); List unresolvedIssues; - using (var connection = await supportRequestRepository.OpenConnectionAsync()) + using (var connection = await supportRequestRepository.OpenSupportSqlConnectionAsync()) { unresolvedIssues = await supportRequestRepository.GetUnresolvedIssues(connection, onCallAlias); } diff --git a/src/NuGet.SupportRequests.Notifications/Tasks/SupportRequestsNotificationScheduledTask.cs b/src/NuGet.SupportRequests.Notifications/Tasks/SupportRequestsNotificationScheduledTask.cs index 6976ed0bb..51ffd7d5d 100644 --- a/src/NuGet.SupportRequests.Notifications/Tasks/SupportRequestsNotificationScheduledTask.cs +++ b/src/NuGet.SupportRequests.Notifications/Tasks/SupportRequestsNotificationScheduledTask.cs @@ -4,10 +4,9 @@ using System; using System.Collections.Generic; using System.ComponentModel.Design; +using System.Data.SqlClient; using System.Threading.Tasks; using Microsoft.Extensions.Logging; -using NuGet.Services.KeyVault; -using NuGet.Services.Sql; using NuGet.SupportRequests.Notifications.Notifications; using NuGet.SupportRequests.Notifications.Services; using NuGet.SupportRequests.Notifications.Templates; @@ -24,6 +23,7 @@ internal abstract class SupportRequestsNotificationScheduledTask protected SupportRequestsNotificationScheduledTask( IServiceContainer serviceContainer, IDictionary jobArgsDictionary, + Func> openSupportSqlConnectionAsync, ILoggerFactory loggerFactory) { if (jobArgsDictionary == null) @@ -38,15 +38,12 @@ protected SupportRequestsNotificationScheduledTask( var smtpUri = jobArgsDictionary[JobArgumentNames.SmtpUri]; _messagingService = new MessagingService(loggerFactory, smtpUri); - - var secretInjector = (ISecretInjector)serviceContainer.GetService(typeof(ISecretInjector)); - var supportDbConnectionString = jobArgsDictionary[JobArgumentNames.SourceDatabase]; - var supportDbConnectionFactory = new AzureSqlConnectionFactory(supportDbConnectionString, secretInjector); - _supportRequestRepository = new SupportRequestRepository(loggerFactory, supportDbConnectionFactory); + _supportRequestRepository = new SupportRequestRepository(openSupportSqlConnectionAsync, loggerFactory); } protected abstract Task BuildNotification(SupportRequestRepository supportRequestRepository, DateTime referenceTime); + protected abstract string BuildNotificationBody(string template, TNotification notification); public async Task RunAsync() diff --git a/src/NuGet.SupportRequests.Notifications/Tasks/WeeklySummaryNotificationTask.cs b/src/NuGet.SupportRequests.Notifications/Tasks/WeeklySummaryNotificationTask.cs index 71a0f9ccf..efb3c9518 100644 --- a/src/NuGet.SupportRequests.Notifications/Tasks/WeeklySummaryNotificationTask.cs +++ b/src/NuGet.SupportRequests.Notifications/Tasks/WeeklySummaryNotificationTask.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Generic; using System.ComponentModel.Design; +using System.Data.SqlClient; using System.Linq; using System.Text; using System.Threading.Tasks; @@ -24,8 +25,9 @@ internal class WeeklySummaryNotificationTask public WeeklySummaryNotificationTask( IServiceContainer serviceContainer, IDictionary jobArgsDictionary, + Func> openSupportSqlConnectionAsync, ILoggerFactory loggerFactory) - : base(serviceContainer, jobArgsDictionary, loggerFactory) + : base(serviceContainer, jobArgsDictionary, openSupportSqlConnectionAsync, loggerFactory) { _targetEmailAddress = jobArgsDictionary[_argumentNameTargetEmailAddress]; } @@ -42,7 +44,7 @@ protected override async Task BuildNotification( var startDateUtcLastWeek = referenceTime.AddDays(-7); var startDateUtcPriorWeek = referenceTime.AddDays(-14); - using (var connection = await supportRequestRepository.OpenConnectionAsync()) + using (var connection = await supportRequestRepository.OpenSupportSqlConnectionAsync()) { unresolvedIssues = await supportRequestRepository.GetUnresolvedIssues(connection); diff --git a/src/Search.GenerateAuxiliaryData/Job.cs b/src/Search.GenerateAuxiliaryData/Job.cs index 4d0971f80..84ed5b8b4 100644 --- a/src/Search.GenerateAuxiliaryData/Job.cs +++ b/src/Search.GenerateAuxiliaryData/Job.cs @@ -11,12 +11,10 @@ using Microsoft.WindowsAzure.Storage; using Microsoft.WindowsAzure.Storage.Blob; using NuGet.Jobs; -using NuGet.Services.KeyVault; -using NuGet.Services.Sql; namespace Search.GenerateAuxiliaryData { - internal class Job + public class Job : JobBase { private const string DefaultContainerName = "ng-search-data"; @@ -45,13 +43,8 @@ internal class Job public override void Init(IServiceContainer serviceContainer, IDictionary jobArgsDictionary) { - var secretInjector = (ISecretInjector)serviceContainer.GetService(typeof(ISecretInjector)); - - var packageDbConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.PackageDatabase); - var packageDbConnectionFactory = new AzureSqlConnectionFactory(packageDbConnectionString, secretInjector); - - var statisticsDbConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatisticsDatabase); - var statisticsDbConnectionFactory = new AzureSqlConnectionFactory(statisticsDbConnectionString, secretInjector); + RegisterDatabase(serviceContainer, jobArgsDictionary, JobArgumentNames.PackageDatabase); + RegisterDatabase(serviceContainer, jobArgsDictionary, JobArgumentNames.StatisticsDatabase); var statisticsStorageAccount = CloudStorageAccount.Parse( JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.AzureCdnCloudStorageAccount)); @@ -69,14 +62,42 @@ public override void Init(IServiceContainer serviceContainer, IDictionary { - new VerifiedPackagesExporter(LoggerFactory.CreateLogger(), packageDbConnectionFactory, _destContainer, ScriptVerifiedPackages, OutputNameVerifiedPackages), - new NestedJArrayExporter(LoggerFactory.CreateLogger(), packageDbConnectionFactory, _destContainer, ScriptCuratedFeed, OutputNameCuratedFeed, Col0CuratedFeed, Col1CuratedFeed), - new NestedJArrayExporter(LoggerFactory.CreateLogger(), packageDbConnectionFactory, _destContainer, ScriptOwners, OutputNameOwners, Col0Owners, Col1Owners), - new RankingsExporter(LoggerFactory.CreateLogger(), statisticsDbConnectionFactory, _destContainer, ScriptRankingsTotal, OutputNameRankings), - new BlobStorageExporter(LoggerFactory.CreateLogger(), _statisticsContainer, StatisticsReportName, _destContainer, StatisticsReportName) + new VerifiedPackagesExporter( + OpenGallerySqlConnectionAsync, + LoggerFactory.CreateLogger(), + _destContainer, ScriptVerifiedPackages, OutputNameVerifiedPackages), + + new NestedJArrayExporter( + OpenGallerySqlConnectionAsync, + LoggerFactory.CreateLogger(), + _destContainer, ScriptCuratedFeed, OutputNameCuratedFeed, Col0CuratedFeed, Col1CuratedFeed), + + new NestedJArrayExporter( + OpenGallerySqlConnectionAsync, + LoggerFactory.CreateLogger(), + _destContainer, ScriptOwners, OutputNameOwners, Col0Owners, Col1Owners), + + new RankingsExporter( + OpenStatisticsSqlConnectionAsync, + LoggerFactory.CreateLogger(), + _destContainer, ScriptRankingsTotal, OutputNameRankings), + + new BlobStorageExporter( + LoggerFactory.CreateLogger(), + _statisticsContainer, StatisticsReportName, _destContainer, StatisticsReportName) }; } + public Task OpenGallerySqlConnectionAsync() + { + return OpenSqlConnectionAsync(JobArgumentNames.PackageDatabase); + } + + public Task OpenStatisticsSqlConnectionAsync() + { + return OpenSqlConnectionAsync(JobArgumentNames.StatisticsDatabase); + } + public override async Task Run() { var failedExporters = new List(); diff --git a/src/Search.GenerateAuxiliaryData/NestedJArrayExporter.cs b/src/Search.GenerateAuxiliaryData/NestedJArrayExporter.cs index 83824718a..ff4cd505c 100644 --- a/src/Search.GenerateAuxiliaryData/NestedJArrayExporter.cs +++ b/src/Search.GenerateAuxiliaryData/NestedJArrayExporter.cs @@ -6,10 +6,10 @@ using System.Data; using System.Data.SqlClient; using System.Linq; +using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Microsoft.WindowsAzure.Storage.Blob; using Newtonsoft.Json.Linq; -using NuGet.Services.Sql; namespace Search.GenerateAuxiliaryData { @@ -17,11 +17,17 @@ class NestedJArrayExporter : SqlExporter { public string Col0 { get; } + public string Col1 { get; } + public string SqlScript { get; } - public NestedJArrayExporter(ILogger logger, ISqlConnectionFactory connectionFactory, CloudBlobContainer defaultDestinationContainer, string defaultSqlScript, string defaultName, string defaultCol0, string defaultCol1) - : base(logger, connectionFactory, defaultDestinationContainer, defaultName) + public NestedJArrayExporter( + Func> openGallerySqlConnectionAsync, + ILogger logger, + CloudBlobContainer defaultDestinationContainer, + string defaultSqlScript, string defaultName, string defaultCol0, string defaultCol1) + : base(openGallerySqlConnectionAsync, logger, defaultDestinationContainer, defaultName) { Col0 = defaultCol0; Col1 = defaultCol1; diff --git a/src/Search.GenerateAuxiliaryData/RankingsExporter.cs b/src/Search.GenerateAuxiliaryData/RankingsExporter.cs index 520812ff7..14ddb8bee 100644 --- a/src/Search.GenerateAuxiliaryData/RankingsExporter.cs +++ b/src/Search.GenerateAuxiliaryData/RankingsExporter.cs @@ -4,10 +4,10 @@ using System; using System.Data; using System.Data.SqlClient; +using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Microsoft.WindowsAzure.Storage.Blob; using Newtonsoft.Json.Linq; -using NuGet.Services.Sql; namespace Search.GenerateAuxiliaryData { @@ -20,12 +20,12 @@ public sealed class RankingsExporter : SqlExporter private readonly string _rankingsTotalScript; public RankingsExporter( + Func> openStatisticsSqlConnectionAsync, ILogger logger, - ISqlConnectionFactory connectionFactory, CloudBlobContainer defaultDestinationContainer, string defaultRankingsScript, string defaultName) - : base(logger, connectionFactory, defaultDestinationContainer, defaultName) + : base(openStatisticsSqlConnectionAsync, logger, defaultDestinationContainer, defaultName) { _rankingsTotalScript = defaultRankingsScript; } diff --git a/src/Search.GenerateAuxiliaryData/Search.GenerateAuxiliaryData.csproj b/src/Search.GenerateAuxiliaryData/Search.GenerateAuxiliaryData.csproj index e9d6dddac..d0a100c02 100644 --- a/src/Search.GenerateAuxiliaryData/Search.GenerateAuxiliaryData.csproj +++ b/src/Search.GenerateAuxiliaryData/Search.GenerateAuxiliaryData.csproj @@ -92,9 +92,6 @@ 9.0.1 - - 2.25.0-master-30453 - 7.1.2 diff --git a/src/Search.GenerateAuxiliaryData/SqlExporter.cs b/src/Search.GenerateAuxiliaryData/SqlExporter.cs index fedaa8227..652b1df4f 100644 --- a/src/Search.GenerateAuxiliaryData/SqlExporter.cs +++ b/src/Search.GenerateAuxiliaryData/SqlExporter.cs @@ -1,6 +1,7 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +using System; using System.Collections.Generic; using System.Data; using System.Data.SqlClient; @@ -11,7 +12,6 @@ using Microsoft.WindowsAzure.Storage.Blob; using Newtonsoft.Json; using Newtonsoft.Json.Linq; -using NuGet.Services.Sql; namespace Search.GenerateAuxiliaryData { @@ -20,14 +20,19 @@ public abstract class SqlExporter : Exporter { private static Assembly _executingAssembly = Assembly.GetExecutingAssembly(); private static string _assemblyName = _executingAssembly.GetName().Name; - - public ISqlConnectionFactory ConnectionFactory { get; } - public SqlExporter(ILogger logger, ISqlConnectionFactory connectionFactory, CloudBlobContainer defaultDestinationContainer, string defaultName) + private Func> OpenSqlConnectionAsync { get; } + + public SqlExporter( + Func> openSqlConnectionAsync, + ILogger logger, + CloudBlobContainer defaultDestinationContainer, + string defaultName) : base(logger, defaultDestinationContainer, defaultName) { _logger = logger; - ConnectionFactory = connectionFactory; + + OpenSqlConnectionAsync = openSqlConnectionAsync; } protected static string GetEmbeddedSqlScript(string resourceName) @@ -38,12 +43,12 @@ protected static string GetEmbeddedSqlScript(string resourceName) public override async Task ExportAsync() { - _logger.LogInformation("Generating {ReportName} report from {DataSource}/{InitialCatalog}.", - _name, ConnectionFactory.DataSource, ConnectionFactory.InitialCatalog); - JContainer result; - using (var connection = await ConnectionFactory.CreateAsync()) + using (var connection = await OpenSqlConnectionAsync()) { + _logger.LogInformation("Generating {ReportName} report from {DataSource}/{InitialCatalog}.", + _name, connection.DataSource, connection.Database); + result = GetResultOfQuery(connection); } diff --git a/src/Search.GenerateAuxiliaryData/VerifiedPackagesExporter.cs b/src/Search.GenerateAuxiliaryData/VerifiedPackagesExporter.cs index 20acab67b..04e3bb334 100644 --- a/src/Search.GenerateAuxiliaryData/VerifiedPackagesExporter.cs +++ b/src/Search.GenerateAuxiliaryData/VerifiedPackagesExporter.cs @@ -4,10 +4,10 @@ using System; using System.Data; using System.Data.SqlClient; +using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Microsoft.WindowsAzure.Storage.Blob; using Newtonsoft.Json.Linq; -using NuGet.Services.Sql; namespace Search.GenerateAuxiliaryData { @@ -17,12 +17,12 @@ internal sealed class VerifiedPackagesExporter : SqlExporter private readonly string _verifiedPackagesScript; public VerifiedPackagesExporter( + Func> openGallerySqlConnectionAsync, ILogger logger, - ISqlConnectionFactory connectionFactory, CloudBlobContainer defaultDestinationContainer, string defaultVerifiedPackagesScript, string defaultName) - : base(logger, connectionFactory, defaultDestinationContainer, defaultName) + : base(openGallerySqlConnectionAsync, logger, defaultDestinationContainer, defaultName) { _verifiedPackagesScript = defaultVerifiedPackagesScript; } diff --git a/src/Stats.AggregateCdnDownloadsInGallery/Job.cs b/src/Stats.AggregateCdnDownloadsInGallery/Job.cs index 976b2e208..ee4e750b5 100644 --- a/src/Stats.AggregateCdnDownloadsInGallery/Job.cs +++ b/src/Stats.AggregateCdnDownloadsInGallery/Job.cs @@ -11,8 +11,6 @@ using System.Threading.Tasks; using Microsoft.Extensions.Logging; using NuGet.Jobs; -using NuGet.Services.KeyVault; -using NuGet.Services.Sql; using IPackageIdGroup = System.Linq.IGrouping; namespace Stats.AggregateCdnDownloadsInGallery @@ -55,20 +53,13 @@ GROUP BY Stats.[PackageRegistrationKey] DROP TABLE #AggregateCdnDownloadsInGallery"; private const string _storedProcedureName = "[dbo].[SelectTotalDownloadCountsPerPackageVersion]"; - private ISqlConnectionFactory _statisticsDbConnectionFactory; - private ISqlConnectionFactory _galleryDbConnectionFactory; private int _batchSize; private int _batchSleepSeconds; public override void Init(IServiceContainer serviceContainer, IDictionary jobArgsDictionary) { - var secretInjector = (ISecretInjector)serviceContainer.GetService(typeof(ISecretInjector)); - - var statisticsDbConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatisticsDatabase); - _statisticsDbConnectionFactory = new AzureSqlConnectionFactory(statisticsDbConnectionString, secretInjector); - - var galleryDbConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.DestinationDatabase); - _galleryDbConnectionFactory = new AzureSqlConnectionFactory(galleryDbConnectionString, secretInjector); + RegisterDatabase(serviceContainer, jobArgsDictionary, JobArgumentNames.StatisticsDatabase); + RegisterDatabase(serviceContainer, jobArgsDictionary, JobArgumentNames.DestinationDatabase); _batchSize = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, JobArgumentNames.BatchSize) ?? _defaultBatchSize; _batchSleepSeconds = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, JobArgumentNames.BatchSleepSeconds) ?? _defaultBatchSleepSeconds; @@ -79,12 +70,14 @@ public override async Task Run() // Gather download counts data from statistics warehouse IReadOnlyList downloadData; Logger.LogInformation("Using batch size {BatchSize} and batch sleep seconds {BatchSleepSeconds}.", _batchSize, _batchSleepSeconds); - Logger.LogInformation("Gathering Download Counts from {DataSource}/{InitialCatalog}...", _statisticsDbConnectionFactory.DataSource, _statisticsDbConnectionFactory.InitialCatalog); var stopwatch = Stopwatch.StartNew(); - using (var statisticsDatabase = await _statisticsDbConnectionFactory.CreateAsync()) + using (var statisticsDatabase = await OpenSqlConnectionAsync(JobArgumentNames.StatisticsDatabase)) using (var statisticsDatabaseTransaction = statisticsDatabase.BeginTransaction(IsolationLevel.Snapshot)) { + Logger.LogInformation("Gathering Download Counts from {DataSource}/{InitialCatalog}...", + statisticsDatabase.DataSource, statisticsDatabase.Database); + downloadData = ( await statisticsDatabase.QueryWithRetryAsync( _storedProcedureName, @@ -106,7 +99,7 @@ await statisticsDatabase.QueryWithRetryAsync( return; } - using (var destinationDatabase = await _galleryDbConnectionFactory.CreateAsync()) + using (var destinationDatabase = await OpenSqlConnectionAsync(JobArgumentNames.DestinationDatabase)) { // Fetch package registrations so we can match package ID to package registration key. var packageRegistrationLookup = await GetPackageRegistrations(destinationDatabase); diff --git a/src/Stats.AggregateCdnDownloadsInGallery/Stats.AggregateCdnDownloadsInGallery.csproj b/src/Stats.AggregateCdnDownloadsInGallery/Stats.AggregateCdnDownloadsInGallery.csproj index 3d2de7505..66c03503c 100644 --- a/src/Stats.AggregateCdnDownloadsInGallery/Stats.AggregateCdnDownloadsInGallery.csproj +++ b/src/Stats.AggregateCdnDownloadsInGallery/Stats.AggregateCdnDownloadsInGallery.csproj @@ -75,9 +75,6 @@ 9.0.1 - - 2.25.0-master-30453 - diff --git a/src/Stats.CreateAzureCdnWarehouseReports/DownloadCountReport.cs b/src/Stats.CreateAzureCdnWarehouseReports/DownloadCountReport.cs index 85ae20a62..e7a4bad21 100644 --- a/src/Stats.CreateAzureCdnWarehouseReports/DownloadCountReport.cs +++ b/src/Stats.CreateAzureCdnWarehouseReports/DownloadCountReport.cs @@ -10,7 +10,6 @@ using Microsoft.Extensions.Logging; using Newtonsoft.Json; using Newtonsoft.Json.Linq; -using NuGet.Services.Sql; using NuGet.Versioning; namespace Stats.CreateAzureCdnWarehouseReports @@ -22,25 +21,28 @@ public class DownloadCountReport private readonly TimeSpan _defaultCommandTimeout = TimeSpan.FromMinutes(30); internal const string ReportName = "downloads.v1.json"; + private Func> OpenStatisticsSqlConnectionAsync { get; } + public DownloadCountReport( + Func> openStatisticsSqlConnectionAsync, ILogger logger, - IEnumerable targets, - ISqlConnectionFactory statisticsDbConnectionFactory, - ISqlConnectionFactory galleryDbConnectionFactory) - : base(logger, targets, statisticsDbConnectionFactory, galleryDbConnectionFactory) + IEnumerable targets) + : base(logger, targets) { + OpenStatisticsSqlConnectionAsync = openStatisticsSqlConnectionAsync; } public async Task Run() { // Gather download count data from statistics warehouse IReadOnlyCollection downloadData; - _logger.LogInformation("Gathering Download Counts from {DataSource}/{InitialCatalog}...", - StatisticsDbConnectionFactory.DataSource, StatisticsDbConnectionFactory.InitialCatalog); - using (var connection = await StatisticsDbConnectionFactory.CreateAsync()) + using (var connection = await OpenStatisticsSqlConnectionAsync()) using (var transaction = connection.BeginTransaction(IsolationLevel.Snapshot)) { + _logger.LogInformation("Gathering Download Counts from {DataSource}/{InitialCatalog}...", + connection.DataSource, connection.Database); + downloadData = (await connection.QueryWithRetryAsync( _storedProcedureName, commandType: CommandType.StoredProcedure, transaction: transaction, commandTimeout: _defaultCommandTimeout)).ToList(); } diff --git a/src/Stats.CreateAzureCdnWarehouseReports/DownloadsPerToolVersionReport.cs b/src/Stats.CreateAzureCdnWarehouseReports/DownloadsPerToolVersionReport.cs index 7411526f1..588daf973 100644 --- a/src/Stats.CreateAzureCdnWarehouseReports/DownloadsPerToolVersionReport.cs +++ b/src/Stats.CreateAzureCdnWarehouseReports/DownloadsPerToolVersionReport.cs @@ -11,7 +11,6 @@ using Microsoft.WindowsAzure.Storage; using Newtonsoft.Json; using Newtonsoft.Json.Linq; -using NuGet.Services.Sql; namespace Stats.CreateAzureCdnWarehouseReports { @@ -22,14 +21,14 @@ public class DownloadsPerToolVersionReport private readonly TimeSpan _defaultCommandTimeout = TimeSpan.FromMinutes(30); internal const string ReportName = "tools.v1.json"; + private Func> OpenStatisticsSqlConnectionAsync { get; } + public DownloadsPerToolVersionReport( + Func> openStatisticsSqlConnectionAsync, ILogger logger, CloudStorageAccount cloudStorageAccount, - string statisticsContainerName, - ISqlConnectionFactory statisticsDbConnectionFactory, - ISqlConnectionFactory galleryDbConnectionFactory) - : base(logger, new[] { new StorageContainerTarget(cloudStorageAccount, statisticsContainerName) }, - statisticsDbConnectionFactory, galleryDbConnectionFactory) + string statisticsContainerName) + : base(logger, new[] { new StorageContainerTarget(cloudStorageAccount, statisticsContainerName) }) { } @@ -37,12 +36,13 @@ public async Task Run() { // Gather download count data from statistics warehouse IReadOnlyCollection data; - _logger.LogInformation("Gathering Tools Download Counts from {DataSource}/{InitialCatalog}...", - StatisticsDbConnectionFactory.DataSource, StatisticsDbConnectionFactory.InitialCatalog); - using (var connection = await StatisticsDbConnectionFactory.CreateAsync()) + using (var connection = await OpenStatisticsSqlConnectionAsync()) using (var transaction = connection.BeginTransaction(IsolationLevel.Snapshot)) { + _logger.LogInformation("Gathering Tools Download Counts from {DataSource}/{InitialCatalog}...", + connection.DataSource, connection.Database); + data = (await connection.QueryWithRetryAsync( _storedProcedureName, commandType: CommandType.StoredProcedure, transaction: transaction, commandTimeout: _defaultCommandTimeout)).ToList(); } diff --git a/src/Stats.CreateAzureCdnWarehouseReports/GalleryTotalsReport.cs b/src/Stats.CreateAzureCdnWarehouseReports/GalleryTotalsReport.cs index cdccbb09d..e32a5bd1c 100644 --- a/src/Stats.CreateAzureCdnWarehouseReports/GalleryTotalsReport.cs +++ b/src/Stats.CreateAzureCdnWarehouseReports/GalleryTotalsReport.cs @@ -9,7 +9,6 @@ using Microsoft.Extensions.Logging; using Microsoft.WindowsAzure.Storage; using Newtonsoft.Json; -using NuGet.Services.Sql; namespace Stats.CreateAzureCdnWarehouseReports { @@ -23,27 +22,33 @@ public class GalleryTotalsReport (SELECT COUNT([Key]) FROM Packages WITH (NOLOCK) WHERE Listed = 1 AND Deleted = 0) AS TotalPackages"; internal const string ReportName = "stats-totals.json"; + private Func> OpenGallerySqlConnectionAsync { get; } + + private Func> OpenStatisticsSqlConnectionAsync { get; } + public GalleryTotalsReport( + Func> openGallerySqlConnectionAsync, + Func> openStatisticsSqlConnectionAsync, ILogger logger, CloudStorageAccount cloudStorageAccount, - string statisticsContainerName, - ISqlConnectionFactory statisticsDbConnectionFactory, - ISqlConnectionFactory galleryDbConnectionFactory) - : base(logger, new[] { new StorageContainerTarget(cloudStorageAccount, statisticsContainerName) }, - statisticsDbConnectionFactory, galleryDbConnectionFactory) + string statisticsContainerName) + : base(logger, new[] { new StorageContainerTarget(cloudStorageAccount, statisticsContainerName) }) { + OpenGallerySqlConnectionAsync = openGallerySqlConnectionAsync; + OpenStatisticsSqlConnectionAsync = openStatisticsSqlConnectionAsync; } public async Task Run() { // gather package numbers from gallery database GalleryTotalsData totalsData; - _logger.LogInformation("Gathering Gallery Totals from {GalleryDataSource}/{GalleryInitialCatalog}...", - GalleryDbConnectionFactory.DataSource, GalleryDbConnectionFactory.InitialCatalog); - using (var connection = await GalleryDbConnectionFactory.CreateAsync()) + using (var connection = await OpenGallerySqlConnectionAsync()) using (var transaction = connection.BeginTransaction(IsolationLevel.Snapshot)) { + _logger.LogInformation("Gathering Gallery Totals from {GalleryDataSource}/{GalleryInitialCatalog}...", + connection.DataSource, connection.Database); + totalsData = (await connection.QueryWithRetryAsync( GalleryQuery, commandType: CommandType.Text, transaction: transaction)).First(); } @@ -52,12 +57,13 @@ public async Task Run() _logger.LogInformation("Unique packages: {UniquePackagesCount}", totalsData.UniquePackages); // gather download count data from statistics warehouse - _logger.LogInformation("Gathering Gallery Totals from {StatisticsDataSource}/{StatisticsInitialCatalog}...", - StatisticsDbConnectionFactory.DataSource, StatisticsDbConnectionFactory.InitialCatalog); - using (var connection = await StatisticsDbConnectionFactory.CreateAsync()) + using (var connection = await OpenStatisticsSqlConnectionAsync()) using (var transaction = connection.BeginTransaction(IsolationLevel.Snapshot)) { + _logger.LogInformation("Gathering Gallery Totals from {StatisticsDataSource}/{StatisticsInitialCatalog}...", + connection.DataSource, connection.Database); + totalsData.Downloads = (await connection.ExecuteScalarWithRetryAsync( WarehouseStoredProcedureName, commandType: CommandType.StoredProcedure, diff --git a/src/Stats.CreateAzureCdnWarehouseReports/Job.cs b/src/Stats.CreateAzureCdnWarehouseReports/Job.cs index 611390958..4f8f55254 100644 --- a/src/Stats.CreateAzureCdnWarehouseReports/Job.cs +++ b/src/Stats.CreateAzureCdnWarehouseReports/Job.cs @@ -4,14 +4,13 @@ using System; using System.Collections.Generic; using System.ComponentModel.Design; +using System.Data.SqlClient; using System.Linq; using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Microsoft.WindowsAzure.Storage; using Microsoft.WindowsAzure.Storage.Blob; using NuGet.Jobs; -using NuGet.Services.KeyVault; -using NuGet.Services.Sql; using Stopwatch = System.Diagnostics.Stopwatch; namespace Stats.CreateAzureCdnWarehouseReports @@ -25,13 +24,13 @@ public class Job private CloudStorageAccount _cloudStorageAccount; private CloudStorageAccount _dataStorageAccount; private string _statisticsContainerName; - private ISqlConnectionFactory _statisticsDbConnectionFactory; - private ISqlConnectionFactory _galleryDbConnectionFactory; private string _reportName; private string[] _dataContainerNames; private int _sqlCommandTimeoutSeconds = DefaultSqlCommandTimeoutSeconds; private int _perPackageReportDegreeOfParallelism = DefaultPerPackageReportDegreeOfParallelism; + private SqlConnectionStringBuilder StatisticsDatabase { get; set; } + private static readonly IDictionary _storedProcedures = new Dictionary { {ReportNames.NuGetClientVersion, "[dbo].[DownloadReportNuGetClientVersion]" }, @@ -50,14 +49,10 @@ public class Job public override void Init(IServiceContainer serviceContainer, IDictionary jobArgsDictionary) { - var secretInjector = (ISecretInjector)serviceContainer.GetService(typeof(ISecretInjector)); - _sqlCommandTimeoutSeconds = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, JobArgumentNames.CommandTimeOut) ?? DefaultSqlCommandTimeoutSeconds; - - var statisticsDatabaseConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatisticsDatabase); - _statisticsDbConnectionFactory = new AzureSqlConnectionFactory(statisticsDatabaseConnectionString, secretInjector); + StatisticsDatabase = RegisterDatabase(serviceContainer, jobArgsDictionary, JobArgumentNames.StatisticsDatabase); + RegisterDatabase(serviceContainer, jobArgsDictionary, JobArgumentNames.SourceDatabase); - var galleryDatabaseConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.SourceDatabase); - _galleryDbConnectionFactory = new AzureSqlConnectionFactory(galleryDatabaseConnectionString, secretInjector); + _sqlCommandTimeoutSeconds = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, JobArgumentNames.CommandTimeOut) ?? DefaultSqlCommandTimeoutSeconds; var cloudStorageAccountConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.AzureCdnCloudStorageAccount); var dataStorageAccountConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.DataStorageAccount); @@ -78,12 +73,24 @@ public override void Init(IServiceContainer serviceContainer, IDictionary OpenStatisticsSqlConnectionAsync() + { + return OpenSqlConnectionAsync(JobArgumentNames.StatisticsDatabase); + } + + public Task OpenGallerySqlConnectionAsync() + { + return OpenSqlConnectionAsync(JobArgumentNames.SourceDatabase); + } + public override async Task Run() { var reportGenerationTime = DateTime.UtcNow; var destinationContainer = _cloudStorageAccount.CreateCloudBlobClient().GetContainerReference(_statisticsContainerName); - Logger.LogDebug("Generating reports from {DataSource}/{InitialCatalog} and saving to {AccountName}/{Container}", _statisticsDbConnectionFactory.DataSource, _statisticsDbConnectionFactory.InitialCatalog, _cloudStorageAccount.Credentials.AccountName, destinationContainer.Name); + Logger.LogDebug("Generating reports from {DataSource}/{InitialCatalog} and saving to {AccountName}/{Container}", + StatisticsDatabase.DataSource, StatisticsDatabase.InitialCatalog, + _cloudStorageAccount.Credentials.AccountName, destinationContainer.Name); var reportBuilderLogger = LoggerFactory.CreateLogger(); var reportCollectorLogger = LoggerFactory.CreateLogger(); @@ -93,12 +100,12 @@ public override async Task Run() // generate all reports var reportGenerators = new Dictionary { - { new ReportBuilder(reportBuilderLogger, ReportNames.NuGetClientVersion), new ReportDataCollector(reportCollectorLogger, _storedProcedures[ReportNames.NuGetClientVersion], _statisticsDbConnectionFactory, _sqlCommandTimeoutSeconds) }, - { new ReportBuilder(reportBuilderLogger, ReportNames.Last6Weeks), new ReportDataCollector(reportCollectorLogger, _storedProcedures[ReportNames.Last6Weeks], _statisticsDbConnectionFactory, _sqlCommandTimeoutSeconds) }, - { new ReportBuilder(reportBuilderLogger, ReportNames.RecentCommunityPopularity), new ReportDataCollector(reportCollectorLogger, _storedProcedures[ReportNames.RecentCommunityPopularity], _statisticsDbConnectionFactory, _sqlCommandTimeoutSeconds) }, - { new ReportBuilder(reportBuilderLogger, ReportNames.RecentCommunityPopularityDetail), new ReportDataCollector(reportCollectorLogger, _storedProcedures[ReportNames.RecentCommunityPopularityDetail], _statisticsDbConnectionFactory, _sqlCommandTimeoutSeconds) }, - { new ReportBuilder(reportBuilderLogger, ReportNames.RecentPopularity), new ReportDataCollector(reportCollectorLogger, _storedProcedures[ReportNames.RecentPopularity], _statisticsDbConnectionFactory, _sqlCommandTimeoutSeconds) }, - { new ReportBuilder(reportBuilderLogger, ReportNames.RecentPopularityDetail), new ReportDataCollector(reportCollectorLogger, _storedProcedures[ReportNames.RecentPopularityDetail], _statisticsDbConnectionFactory, _sqlCommandTimeoutSeconds) } + { new ReportBuilder(reportBuilderLogger, ReportNames.NuGetClientVersion), new ReportDataCollector(OpenStatisticsSqlConnectionAsync, reportCollectorLogger, _storedProcedures[ReportNames.NuGetClientVersion], _sqlCommandTimeoutSeconds) }, + { new ReportBuilder(reportBuilderLogger, ReportNames.Last6Weeks), new ReportDataCollector(OpenStatisticsSqlConnectionAsync, reportCollectorLogger, _storedProcedures[ReportNames.Last6Weeks], _sqlCommandTimeoutSeconds) }, + { new ReportBuilder(reportBuilderLogger, ReportNames.RecentCommunityPopularity), new ReportDataCollector(OpenStatisticsSqlConnectionAsync, reportCollectorLogger, _storedProcedures[ReportNames.RecentCommunityPopularity], _sqlCommandTimeoutSeconds) }, + { new ReportBuilder(reportBuilderLogger, ReportNames.RecentCommunityPopularityDetail), new ReportDataCollector(OpenStatisticsSqlConnectionAsync, reportCollectorLogger, _storedProcedures[ReportNames.RecentCommunityPopularityDetail], _sqlCommandTimeoutSeconds) }, + { new ReportBuilder(reportBuilderLogger, ReportNames.RecentPopularity), new ReportDataCollector(OpenStatisticsSqlConnectionAsync, reportCollectorLogger, _storedProcedures[ReportNames.RecentPopularity], _sqlCommandTimeoutSeconds) }, + { new ReportBuilder(reportBuilderLogger, ReportNames.RecentPopularityDetail), new ReportDataCollector(OpenStatisticsSqlConnectionAsync, reportCollectorLogger, _storedProcedures[ReportNames.RecentPopularityDetail], _sqlCommandTimeoutSeconds) } }; foreach (var reportGenerator in reportGenerators) @@ -114,12 +121,14 @@ public override async Task Run() { // generate only the specific report var reportBuilder = new ReportBuilder(reportBuilderLogger, _reportName); - var reportDataCollector = new ReportDataCollector(reportCollectorLogger, _storedProcedures[_reportName], _statisticsDbConnectionFactory, _sqlCommandTimeoutSeconds); + var reportDataCollector = new ReportDataCollector(OpenStatisticsSqlConnectionAsync, reportCollectorLogger, _storedProcedures[_reportName], _sqlCommandTimeoutSeconds); await ProcessReport(LoggerFactory, destinationContainer, reportBuilder, reportDataCollector, reportGenerationTime); } - Logger.LogInformation("Generated reports from {DataSource}/{InitialCatalog} and saving to {AccountName}/{Container}", _statisticsDbConnectionFactory.DataSource, _statisticsDbConnectionFactory.InitialCatalog, _cloudStorageAccount.Credentials.AccountName, destinationContainer.Name); + Logger.LogInformation("Generated reports from {DataSource}/{InitialCatalog} and saving to {AccountName}/{Container}", + StatisticsDatabase.DataSource, StatisticsDatabase.InitialCatalog, + _cloudStorageAccount.Credentials.AccountName, destinationContainer.Name); // totals reports var stopwatch = Stopwatch.StartNew(); @@ -131,7 +140,7 @@ public override async Task Run() { targets.Add(new StorageContainerTarget(_dataStorageAccount, dataContainerName)); } - var downloadCountReport = new DownloadCountReport(LoggerFactory.CreateLogger(), targets, _statisticsDbConnectionFactory, _galleryDbConnectionFactory); + var downloadCountReport = new DownloadCountReport(OpenStatisticsSqlConnectionAsync, LoggerFactory.CreateLogger(), targets); await downloadCountReport.Run(); stopwatch.Stop(); @@ -140,7 +149,7 @@ public override async Task Run() stopwatch.Restart(); // build stats-totals.json - var galleryTotalsReport = new GalleryTotalsReport(LoggerFactory.CreateLogger(), _cloudStorageAccount, _statisticsContainerName, _statisticsDbConnectionFactory, _galleryDbConnectionFactory); + var galleryTotalsReport = new GalleryTotalsReport(OpenGallerySqlConnectionAsync, OpenStatisticsSqlConnectionAsync, LoggerFactory.CreateLogger(), _cloudStorageAccount, _statisticsContainerName); await galleryTotalsReport.Run(); stopwatch.Stop(); @@ -149,7 +158,7 @@ public override async Task Run() // build tools.v1.json - var toolsReport = new DownloadsPerToolVersionReport(LoggerFactory.CreateLogger(), _cloudStorageAccount, _statisticsContainerName, _statisticsDbConnectionFactory, _galleryDbConnectionFactory); + var toolsReport = new DownloadsPerToolVersionReport(OpenStatisticsSqlConnectionAsync, LoggerFactory.CreateLogger(), _cloudStorageAccount, _statisticsContainerName); await toolsReport.Run(); stopwatch.Stop(); @@ -174,14 +183,14 @@ private static async Task ProcessReport(ILoggerFactory loggerFactory, CloudBlobC private async Task RebuildPackageReports(CloudBlobContainer destinationContainer, DateTime reportGenerationTime) { - var dirtyPackageIds = await ReportDataCollector.GetDirtyPackageIds(LoggerFactory.CreateLogger(), _statisticsDbConnectionFactory, reportGenerationTime, _sqlCommandTimeoutSeconds); + var dirtyPackageIds = await ReportDataCollector.GetDirtyPackageIds(OpenStatisticsSqlConnectionAsync, LoggerFactory.CreateLogger(), reportGenerationTime, _sqlCommandTimeoutSeconds); if (!dirtyPackageIds.Any()) return; // first process the top 100 packages var top100 = dirtyPackageIds.Take(100); - var reportDataCollector = new ReportDataCollector(LoggerFactory.CreateLogger(), _storedProceduresPerPackageId[ReportNames.RecentPopularityDetailByPackageId], _statisticsDbConnectionFactory, _sqlCommandTimeoutSeconds); + var reportDataCollector = new ReportDataCollector(OpenStatisticsSqlConnectionAsync, LoggerFactory.CreateLogger(), _storedProceduresPerPackageId[ReportNames.RecentPopularityDetailByPackageId], _sqlCommandTimeoutSeconds); var top100Task = Parallel.ForEach(top100, new ParallelOptions { MaxDegreeOfParallelism = _perPackageReportDegreeOfParallelism }, dirtyPackageId => { var packageId = dirtyPackageId.PackageId.ToLowerInvariant(); @@ -209,9 +218,9 @@ private async Task RebuildPackageReports(CloudBlobContainer destinationContainer "recentpopularity/" + _recentPopularityDetailByPackageReportBaseName + dirtyPackageId.PackageId.ToLowerInvariant()), new ReportDataCollector( + OpenStatisticsSqlConnectionAsync, LoggerFactory.CreateLogger(), _storedProceduresPerPackageId[ReportNames.RecentPopularityDetailByPackageId], - _statisticsDbConnectionFactory, _sqlCommandTimeoutSeconds) } }; @@ -228,7 +237,7 @@ private async Task RebuildPackageReports(CloudBlobContainer destinationContainer if (top100Task.IsCompleted) { var runToCursor = dirtyPackageIds.First().RunToCuror; - await ReportDataCollector.UpdateDirtyPackageIdCursor(_statisticsDbConnectionFactory, runToCursor, _sqlCommandTimeoutSeconds); + await ReportDataCollector.UpdateDirtyPackageIdCursor(OpenStatisticsSqlConnectionAsync, runToCursor, _sqlCommandTimeoutSeconds); } } } @@ -236,7 +245,7 @@ private async Task RebuildPackageReports(CloudBlobContainer destinationContainer private async Task CleanInactiveRecentPopularityDetailByPackageReports(CloudBlobContainer destinationContainer, DateTime reportGenerationTime) { Logger.LogDebug("Getting list of inactive packages."); - var packageIds = await ReportDataCollector.ListInactivePackageIdReports(_statisticsDbConnectionFactory, reportGenerationTime, _sqlCommandTimeoutSeconds); + var packageIds = await ReportDataCollector.ListInactivePackageIdReports(OpenStatisticsSqlConnectionAsync, reportGenerationTime, _sqlCommandTimeoutSeconds); Logger.LogInformation("Found {InactivePackageCount} inactive packages.", packageIds.Count); // Collect the list of reports diff --git a/src/Stats.CreateAzureCdnWarehouseReports/ReportBase.cs b/src/Stats.CreateAzureCdnWarehouseReports/ReportBase.cs index faef0cce4..121fa1ed5 100644 --- a/src/Stats.CreateAzureCdnWarehouseReports/ReportBase.cs +++ b/src/Stats.CreateAzureCdnWarehouseReports/ReportBase.cs @@ -8,7 +8,6 @@ using Microsoft.Extensions.Logging; using Microsoft.WindowsAzure.Storage.Blob; using Microsoft.WindowsAzure.Storage.RetryPolicies; -using NuGet.Services.Sql; namespace Stats.CreateAzureCdnWarehouseReports { @@ -18,20 +17,12 @@ public abstract class ReportBase protected readonly IReadOnlyCollection Targets; - protected readonly ISqlConnectionFactory StatisticsDbConnectionFactory; - - protected ISqlConnectionFactory GalleryDbConnectionFactory; - protected ReportBase( ILogger logger, - IEnumerable targets, - ISqlConnectionFactory statisticsDbConnectionFactory, - ISqlConnectionFactory galleryDbConnectionFactory) + IEnumerable targets) { _logger = logger; Targets = targets.ToList().AsReadOnly(); - StatisticsDbConnectionFactory = statisticsDbConnectionFactory; - GalleryDbConnectionFactory = galleryDbConnectionFactory; } protected async Task GetBlobContainer(StorageContainerTarget target) diff --git a/src/Stats.CreateAzureCdnWarehouseReports/ReportDataCollector.cs b/src/Stats.CreateAzureCdnWarehouseReports/ReportDataCollector.cs index be58465b4..3f4b4a71d 100644 --- a/src/Stats.CreateAzureCdnWarehouseReports/ReportDataCollector.cs +++ b/src/Stats.CreateAzureCdnWarehouseReports/ReportDataCollector.cs @@ -8,7 +8,6 @@ using System.Diagnostics; using System.Threading.Tasks; using Microsoft.Extensions.Logging; -using NuGet.Services.Sql; namespace Stats.CreateAzureCdnWarehouseReports { @@ -16,19 +15,20 @@ internal class ReportDataCollector { private int _commandTimeoutSeconds; private readonly string _procedureName; - private readonly ISqlConnectionFactory _sourceDbConnectionFactory; private ILogger _logger; + private Func> OpenStatisticsSqlConnectionAsync { get; } + public ReportDataCollector( + Func> openStatisticsSqlConnectionAsync, ILogger logger, string procedureName, - ISqlConnectionFactory sourceDbConnectionFactory, int timeout) { + OpenStatisticsSqlConnectionAsync = openStatisticsSqlConnectionAsync; _logger = logger; _procedureName = procedureName; - _sourceDbConnectionFactory = sourceDbConnectionFactory; _commandTimeoutSeconds = timeout; } @@ -47,8 +47,8 @@ public async Task CollectAsync(DateTime reportGenerationTime, params } public static async Task> GetDirtyPackageIds( + Func> openStatisticsSqlConnectionAsync, ILogger logger, - ISqlConnectionFactory sourceDbConnectionFactory, DateTime reportGenerationTime, int commandTimeout) { @@ -57,7 +57,8 @@ public static async Task> GetDirtyPackageIds IReadOnlyCollection packageIds = new List(); // Get the data - await WithRetry(async () => packageIds = await GetDirtyPackageIdsFromWarehouse(sourceDbConnectionFactory, reportGenerationTime, commandTimeout), logger); + await WithRetry(async () => packageIds = await GetDirtyPackageIdsFromWarehouse( + openStatisticsSqlConnectionAsync, reportGenerationTime, commandTimeout), logger); logger.LogInformation("Found {DirtyPackagesCount} dirty packages to update.", packageIds.Count); @@ -65,11 +66,11 @@ public static async Task> GetDirtyPackageIds } public static async Task> ListInactivePackageIdReports( - ISqlConnectionFactory sourceDbConnectionFactory, + Func> openStatisticsSqlConnectionAsync, DateTime reportGenerationTime, int commandTimeout) { - using (var connection = await sourceDbConnectionFactory.CreateAsync()) + using (var connection = await openStatisticsSqlConnectionAsync()) { var command = new SqlCommand("[dbo].[DownloadReportListInactive]", connection); command.CommandType = CommandType.StoredProcedure; @@ -122,7 +123,7 @@ private static async Task WithRetry(Func action, ILogger logger) private async Task ExecuteSql(DateTime reportGenerationTime, params Tuple[] parameters) { - using (var connection = await _sourceDbConnectionFactory.CreateAsync()) + using (var connection = await OpenStatisticsSqlConnectionAsync()) { var command = new SqlCommand(_procedureName, connection); command.CommandType = CommandType.StoredProcedure; @@ -146,11 +147,11 @@ private async Task ExecuteSql(DateTime reportGenerationTime, params T } private static async Task> GetDirtyPackageIdsFromWarehouse( - ISqlConnectionFactory sourceDbConnectionFactory, + Func> openStatisticsSqlConnectionAsync, DateTime reportGenerationTime, int commandTimeout) { - using (var connection = await sourceDbConnectionFactory.CreateAsync()) + using (var connection = await openStatisticsSqlConnectionAsync()) { var command = new SqlCommand("[dbo].[GetDirtyPackageIds]", connection); command.CommandType = CommandType.StoredProcedure; @@ -172,11 +173,11 @@ private static async Task> GetDirtyPackageId } public static async Task UpdateDirtyPackageIdCursor( - ISqlConnectionFactory sourceDbConnectionFactory, + Func> openStatisticsSqlConnectionAsync, DateTime runToCursor, int commandTimeout) { - using (var connection = await sourceDbConnectionFactory.CreateAsync()) + using (var connection = await openStatisticsSqlConnectionAsync()) { var command = new SqlCommand("[dbo].[UpdateDirtyPackageIdCursor]", connection); command.CommandType = CommandType.StoredProcedure; diff --git a/src/Stats.CreateAzureCdnWarehouseReports/Stats.CreateAzureCdnWarehouseReports.csproj b/src/Stats.CreateAzureCdnWarehouseReports/Stats.CreateAzureCdnWarehouseReports.csproj index 0b09db277..5181875fb 100644 --- a/src/Stats.CreateAzureCdnWarehouseReports/Stats.CreateAzureCdnWarehouseReports.csproj +++ b/src/Stats.CreateAzureCdnWarehouseReports/Stats.CreateAzureCdnWarehouseReports.csproj @@ -99,9 +99,6 @@ 2.25.0 - - 2.25.0-master-30453 - 4.3.0-preview1-2524 diff --git a/src/Stats.ImportAzureCdnStatistics/DataImporter.cs b/src/Stats.ImportAzureCdnStatistics/DataImporter.cs index 8f0a9251b..fbe92005c 100644 --- a/src/Stats.ImportAzureCdnStatistics/DataImporter.cs +++ b/src/Stats.ImportAzureCdnStatistics/DataImporter.cs @@ -1,21 +1,22 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +using System; using System.Data; using System.Data.SqlClient; using System.Threading.Tasks; -using NuGet.Services.Sql; namespace Stats.ImportAzureCdnStatistics { internal class DataImporter { - private readonly ISqlConnectionFactory _statisticsDbConnectionFactory; private const string _sqlSelectTop1FromTable = "SELECT TOP 1 * FROM [dbo].[{0}]"; - public DataImporter(ISqlConnectionFactory statisticsDbConnectionFactory) + public Func> OpenStatisticsSqlConnectionAsync { get; } + + public DataImporter(Func> openStatisticsSqlConnectionAsync) { - _statisticsDbConnectionFactory = statisticsDbConnectionFactory; + OpenStatisticsSqlConnectionAsync = openStatisticsSqlConnectionAsync; } public async Task GetDataTableAsync(string tableName) @@ -23,7 +24,7 @@ public async Task GetDataTableAsync(string tableName) var dataTable = new DataTable(); var query = string.Format(_sqlSelectTop1FromTable, tableName); - using (var connection = await _statisticsDbConnectionFactory.CreateAsync()) + using (var connection = await OpenStatisticsSqlConnectionAsync()) { var tableAdapter = new SqlDataAdapter(query, connection) { diff --git a/src/Stats.ImportAzureCdnStatistics/Job.cs b/src/Stats.ImportAzureCdnStatistics/Job.cs index 6b0b18119..5f82c0762 100644 --- a/src/Stats.ImportAzureCdnStatistics/Job.cs +++ b/src/Stats.ImportAzureCdnStatistics/Job.cs @@ -10,9 +10,8 @@ using Microsoft.WindowsAzure.Storage.Blob; using Microsoft.WindowsAzure.Storage.RetryPolicies; using NuGet.Jobs; -using NuGet.Services.KeyVault; -using NuGet.Services.Sql; using Stats.AzureCdnLogs.Common; +using System.Data.SqlClient; namespace Stats.ImportAzureCdnStatistics { @@ -23,16 +22,13 @@ public class Job private string _azureCdnAccountNumber; private string _cloudStorageContainerName; private AzureCdnPlatform _azureCdnPlatform; - private ISqlConnectionFactory _statisticsDbConnectionFactory; private CloudStorageAccount _cloudStorageAccount; private CloudBlobClient _cloudBlobClient; private LogFileProvider _blobLeaseManager; public override void Init(IServiceContainer serviceContainer, IDictionary jobArgsDictionary) { - var secretInjector = (ISecretInjector)serviceContainer.GetService(typeof(ISecretInjector)); - var statisticsDbConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatisticsDatabase); - _statisticsDbConnectionFactory = new AzureSqlConnectionFactory(statisticsDbConnectionString, secretInjector); + RegisterDatabase(serviceContainer, jobArgsDictionary, JobArgumentNames.StatisticsDatabase); var azureCdnPlatform = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.AzureCdnPlatform); var cloudStorageAccountConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.AzureCdnCloudStorageAccount); @@ -54,6 +50,11 @@ public override void Init(IServiceContainer serviceContainer, IDictionary OpenStatisticsSqlConnectionAsync() + { + return OpenSqlConnectionAsync(JobArgumentNames.StatisticsDatabase); + } + public override async Task Run() { // Get the target blob container (for archiving decompressed log files) @@ -65,7 +66,7 @@ public override async Task Run() await deadLetterBlobContainer.CreateIfNotExistsAsync(); // Create a parser - var warehouse = new Warehouse(LoggerFactory, _statisticsDbConnectionFactory); + var warehouse = new Warehouse(OpenStatisticsSqlConnectionAsync, LoggerFactory); var statisticsBlobContainerUtility = new StatisticsBlobContainerUtility( targetBlobContainer, deadLetterBlobContainer, diff --git a/src/Stats.ImportAzureCdnStatistics/Stats.ImportAzureCdnStatistics.csproj b/src/Stats.ImportAzureCdnStatistics/Stats.ImportAzureCdnStatistics.csproj index 6ba8e18cd..47ace8b1a 100644 --- a/src/Stats.ImportAzureCdnStatistics/Stats.ImportAzureCdnStatistics.csproj +++ b/src/Stats.ImportAzureCdnStatistics/Stats.ImportAzureCdnStatistics.csproj @@ -125,9 +125,6 @@ 2.25.0 - - 2.25.0-master-30453 - 4.3.0-preview1-2524 diff --git a/src/Stats.ImportAzureCdnStatistics/Warehouse.cs b/src/Stats.ImportAzureCdnStatistics/Warehouse.cs index 6e4c7bcae..e84718fc1 100644 --- a/src/Stats.ImportAzureCdnStatistics/Warehouse.cs +++ b/src/Stats.ImportAzureCdnStatistics/Warehouse.cs @@ -8,7 +8,6 @@ using System.Linq; using System.Threading.Tasks; using Microsoft.Extensions.Logging; -using NuGet.Services.Sql; using Stats.AzureCdnLogs.Common; using Stopwatch = System.Diagnostics.Stopwatch; @@ -21,7 +20,6 @@ internal class Warehouse private const int _maxRetryCount = 3; private readonly TimeSpan _retryDelay = TimeSpan.FromSeconds(5); private readonly ILogger _logger; - private readonly ISqlConnectionFactory _statisticsDbConnectionFactory; private readonly IDictionary _cachedPackageDimensions = new Dictionary(); private readonly IList _cachedToolDimensions = new List(); private readonly IDictionary _cachedClientDimensions = new Dictionary(); @@ -31,15 +29,17 @@ internal class Warehouse private readonly IDictionary _cachedIpAddressFacts = new Dictionary(); private IReadOnlyCollection _times; - public Warehouse(ILoggerFactory loggerFactory, ISqlConnectionFactory statisticsDbConnectionFactory) + private Func> OpenStatisticsSqlConnectionAsync { get; } + + public Warehouse(Func> openStatisticsSqlConnectionAsync, ILoggerFactory loggerFactory) { if (loggerFactory == null) { throw new ArgumentNullException(nameof(loggerFactory)); } + OpenStatisticsSqlConnectionAsync = openStatisticsSqlConnectionAsync; _logger = loggerFactory.CreateLogger(); - _statisticsDbConnectionFactory = statisticsDbConnectionFactory ?? throw new ArgumentNullException(nameof(statisticsDbConnectionFactory)); } public async Task InsertDownloadFactsAsync(DataTable downloadFactsDataTable, string logFileName) @@ -47,7 +47,7 @@ public async Task InsertDownloadFactsAsync(DataTable downloadFactsDataTable, str _logger.LogDebug("Inserting into facts table..."); var stopwatch = Stopwatch.StartNew(); - using (var connection = await _statisticsDbConnectionFactory.CreateAsync()) + using (var connection = await OpenStatisticsSqlConnectionAsync()) using (var transaction = connection.BeginTransaction(IsolationLevel.Snapshot)) { try @@ -124,7 +124,7 @@ public async Task CreateAsync(IReadOnlyCollection var packages = packagesTask.Result; // create facts data rows by linking source data with dimensions - var dataImporter = new DataImporter(_statisticsDbConnectionFactory); + var dataImporter = new DataImporter(OpenStatisticsSqlConnectionAsync); var factsDataTable = await dataImporter.GetDataTableAsync("Fact_Download"); var knownOperationsAvailable = operations.Any(); @@ -245,7 +245,7 @@ public async Task CreateAsync(IReadOnlyCollection sou var ipAddresses = ipAddressesTask.Result; // create facts data rows by linking source data with dimensions - var dataImporter = new DataImporter(_statisticsDbConnectionFactory); + var dataImporter = new DataImporter(OpenStatisticsSqlConnectionAsync); var dataTable = await dataImporter.GetDataTableAsync("Fact_Dist_Download"); var knownClientsAvailable = clients.Any(); @@ -341,7 +341,7 @@ public async Task StoreLogFileAggregatesAsync(LogFileAggregates logFileAggregate { _logger.LogDebug("Storing log file aggregates..."); - using (var connection = await _statisticsDbConnectionFactory.CreateAsync()) + using (var connection = await OpenStatisticsSqlConnectionAsync()) { try { @@ -375,7 +375,7 @@ public async Task> GetAlreadyAggregatedLogFilesAsync _logger.LogDebug("Retrieving already processed log files..."); var alreadyAggregatedLogFiles = new List(); - using (var connection = await _statisticsDbConnectionFactory.CreateAsync()) + using (var connection = await OpenStatisticsSqlConnectionAsync()) { try { @@ -433,7 +433,7 @@ private async Task HasImportedStatisticsAsync(string logFileName, string c try { - using (var connection = await _statisticsDbConnectionFactory.CreateAsync()) + using (var connection = await OpenStatisticsSqlConnectionAsync()) { var command = connection.CreateCommand(); command.CommandText = commandText; @@ -474,7 +474,7 @@ private async Task> GetDimension(string dimension, stri _logger.LogDebug("Beginning to retrieve dimension '{Dimension}'.", dimension); IDictionary dimensions; - using (var connection = await _statisticsDbConnectionFactory.CreateAsync()) + using (var connection = await OpenStatisticsSqlConnectionAsync()) { dimensions = await retrieve(connection); } @@ -546,7 +546,7 @@ private async Task> GetDimension(string dimension, str _logger.LogDebug("Beginning to retrieve dimension '{Dimension}'.", dimension); IReadOnlyCollection dimensions; - using (var connection = await _statisticsDbConnectionFactory.CreateAsync()) + using (var connection = await OpenStatisticsSqlConnectionAsync()) { dimensions = await retrieve(connection); } diff --git a/src/Stats.RefreshClientDimension/RefreshClientDimensionJob.cs b/src/Stats.RefreshClientDimension/RefreshClientDimensionJob.cs index a10b412c3..5bc3fb1b7 100644 --- a/src/Stats.RefreshClientDimension/RefreshClientDimensionJob.cs +++ b/src/Stats.RefreshClientDimension/RefreshClientDimensionJob.cs @@ -8,23 +8,18 @@ using System.Threading.Tasks; using Microsoft.Extensions.Logging; using NuGet.Jobs; -using NuGet.Services.KeyVault; -using NuGet.Services.Sql; using Stats.ImportAzureCdnStatistics; namespace Stats.RefreshClientDimension { public class RefreshClientDimensionJob : JobBase { - private static ISqlConnectionFactory _statisticsDbConnectionFactory; private static string _targetClientName; private static string _userAgentFilter; public override void Init(IServiceContainer serviceContainer, IDictionary jobArgsDictionary) { - var secretInjector = (ISecretInjector)serviceContainer.GetService(typeof(ISecretInjector)); - var statisticsDbConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatisticsDatabase); - _statisticsDbConnectionFactory = new AzureSqlConnectionFactory(statisticsDbConnectionString, secretInjector); + RegisterDatabase(serviceContainer, jobArgsDictionary, JobArgumentNames.StatisticsDatabase); _targetClientName = JobConfigurationManager.TryGetArgument(jobArgsDictionary, "TargetClientName"); _userAgentFilter = JobConfigurationManager.TryGetArgument(jobArgsDictionary, "UserAgentFilter"); @@ -32,7 +27,7 @@ public override void Init(IServiceContainer serviceContainer, IDictionary> linkedUserAgents; diff --git a/src/Stats.RefreshClientDimension/Stats.RefreshClientDimension.csproj b/src/Stats.RefreshClientDimension/Stats.RefreshClientDimension.csproj index 980174366..1de4656f1 100644 --- a/src/Stats.RefreshClientDimension/Stats.RefreshClientDimension.csproj +++ b/src/Stats.RefreshClientDimension/Stats.RefreshClientDimension.csproj @@ -83,9 +83,6 @@ 9.0.1 - - 2.25.0-master-30453 - 1.2.0 diff --git a/src/Stats.RollUpDownloadFacts/Job.cs b/src/Stats.RollUpDownloadFacts/Job.cs index 76cd61979..51d3dcc40 100644 --- a/src/Stats.RollUpDownloadFacts/Job.cs +++ b/src/Stats.RollUpDownloadFacts/Job.cs @@ -9,8 +9,6 @@ using System.Threading.Tasks; using Microsoft.Extensions.Logging; using NuGet.Jobs; -using NuGet.Services.KeyVault; -using NuGet.Services.Sql; namespace Stats.RollUpDownloadFacts { @@ -21,13 +19,10 @@ public class Job private const string _endTemplateFactDownloadDeletion = " records from [dbo].[Fact_Download]"; private const int DefaultMinAgeInDays = 43; private static int _minAgeInDays; - private static ISqlConnectionFactory _statisticsDbConnectionFactory; public override void Init(IServiceContainer serviceContainer, IDictionary jobArgsDictionary) { - var secretInjector = (ISecretInjector)serviceContainer.GetService(typeof(ISecretInjector)); - var statisticsDbConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatisticsDatabase); - _statisticsDbConnectionFactory = new AzureSqlConnectionFactory(statisticsDbConnectionString, secretInjector); + RegisterDatabase(serviceContainer, jobArgsDictionary, JobArgumentNames.StatisticsDatabase); _minAgeInDays = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, JobArgumentNames.MinAgeInDays) ?? DefaultMinAgeInDays; Logger.LogInformation("Min age in days: {MinAgeInDays}", _minAgeInDays); @@ -35,7 +30,7 @@ public override void Init(IServiceContainer serviceContainer, IDictionary 2.25.0 - - 2.25.0-master-30453 - diff --git a/src/UpdateLicenseReports/UpdateLicenseReports.Job.cs b/src/UpdateLicenseReports/UpdateLicenseReports.Job.cs index 0a3fb2fad..d012fc1de 100644 --- a/src/UpdateLicenseReports/UpdateLicenseReports.Job.cs +++ b/src/UpdateLicenseReports/UpdateLicenseReports.Job.cs @@ -14,8 +14,6 @@ using Newtonsoft.Json.Linq; using Newtonsoft.Json.Schema; using NuGet.Jobs; -using NuGet.Services.KeyVault; -using NuGet.Services.Sql; namespace UpdateLicenseReports { @@ -41,7 +39,6 @@ internal class Job : JobBase private Uri _licenseReportService; private string _licenseReportUser; private string _licenseReportPassword; - private ISqlConnectionFactory _packageDbConnectionFactory; private int? _retryCount; private NetworkCredential _licenseReportCredentials; @@ -61,9 +58,7 @@ private static PackageLicenseReport CreateReport(JObject messageEvent) public override void Init(IServiceContainer serviceContainer, IDictionary jobArgsDictionary) { - var secretInjector = (ISecretInjector)serviceContainer.GetService(typeof(ISecretInjector)); - var dbConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.PackageDatabase); - _packageDbConnectionFactory = new AzureSqlConnectionFactory(dbConnectionString, secretInjector); + RegisterDatabase(serviceContainer, jobArgsDictionary, JobArgumentNames.PackageDatabase); var retryCountString = JobConfigurationManager.TryGetArgument(jobArgsDictionary, JobArgumentNames.RetryCount); if (string.IsNullOrEmpty(retryCountString)) @@ -111,12 +106,12 @@ public override async Task Run() private async Task FetchNextReportUrlAsync() { - Logger.LogInformation("Fetching next report URL from {DataSource}/{InitialCatalog}", - _packageDbConnectionFactory.DataSource, _packageDbConnectionFactory.InitialCatalog); - Uri nextLicenseReport = null; - using (var connection = await _packageDbConnectionFactory.CreateAsync()) + using (var connection = await OpenSqlConnectionAsync(JobArgumentNames.PackageDatabase)) { + Logger.LogInformation("Fetching next report URL from {DataSource}/{InitialCatalog}", + connection.DataSource, connection.Database); + var nextReportUrl = (await connection.QueryAsync( @"SELECT TOP 1 NextLicenseReport FROM GallerySettings")).SingleOrDefault(); @@ -130,11 +125,11 @@ private async Task FetchNextReportUrlAsync() } nextLicenseReport = nextLicenseReport ?? _licenseReportService; - } - Logger.LogInformation("Fetched next report URL '{NextReportUrl}' from {DataSource}/{InitialCatalog}", - (nextLicenseReport == null ? string.Empty : nextLicenseReport.AbsoluteUri), - _packageDbConnectionFactory.DataSource, _packageDbConnectionFactory.InitialCatalog); + Logger.LogInformation("Fetched next report URL '{NextReportUrl}' from {DataSource}/{InitialCatalog}", + (nextLicenseReport == null ? string.Empty : nextLicenseReport.AbsoluteUri), + connection.DataSource, connection.Database); + } return nextLicenseReport; } @@ -242,7 +237,7 @@ private async Task ProcessReportsAsync(Uri nextLicenseReport) Logger.LogInformation("Storing next license report URL: {NextReportUrl}", nextLicenseReport.AbsoluteUri); // Record the next report to the database so we can check it again if we get aborted before finishing. - using (var connection = await _packageDbConnectionFactory.CreateAsync()) + using (var connection = await OpenSqlConnectionAsync(JobArgumentNames.PackageDatabase)) { await connection.QueryAsync(@" UPDATE GallerySettings @@ -274,7 +269,7 @@ UPDATE GallerySettings private async Task StoreReportAsync(PackageLicenseReport report) { - using (var connection = await _packageDbConnectionFactory.CreateAsync()) + using (var connection = await OpenSqlConnectionAsync(JobArgumentNames.PackageDatabase)) using (var command = connection.CreateCommand()) { command.CommandText = "AddPackageLicenseReport2"; diff --git a/src/UpdateLicenseReports/UpdateLicenseReports.csproj b/src/UpdateLicenseReports/UpdateLicenseReports.csproj index 0d9b7210c..6a824cd81 100644 --- a/src/UpdateLicenseReports/UpdateLicenseReports.csproj +++ b/src/UpdateLicenseReports/UpdateLicenseReports.csproj @@ -74,9 +74,6 @@ 2.0.10 - - 2.25.0-master-30453 - diff --git a/src/Validation.Common.Job/JsonConfigurationJob.cs b/src/Validation.Common.Job/JsonConfigurationJob.cs index 9355f050c..7cc47941f 100644 --- a/src/Validation.Common.Job/JsonConfigurationJob.cs +++ b/src/Validation.Common.Job/JsonConfigurationJob.cs @@ -4,12 +4,10 @@ using System; using System.Collections.Generic; using System.ComponentModel.Design; -using System.Data.Common; using System.IO; using System.Net; using System.Net.Http; using System.Reflection; -using System.Threading.Tasks; using Autofac; using Autofac.Extensions.DependencyInjection; using Microsoft.ApplicationInsights; @@ -22,7 +20,6 @@ using NuGet.Services.KeyVault; using NuGet.Services.Logging; using NuGet.Services.ServiceBus; -using NuGet.Services.Sql; using NuGet.Services.Validation; using NuGetGallery; using NuGetGallery.Diagnostics; @@ -64,6 +61,9 @@ public override void Init(IServiceContainer serviceContainer, IDictionary(_serviceProvider); + RegisterDatabase(_serviceProvider); + ServicePointManager.DefaultConnectionLimit = MaximumConnectionsPerServer; } @@ -109,15 +109,6 @@ private IServiceProvider GetServiceProvider(IConfigurationRoot configurationRoot return new AutofacServiceProvider(containerBuilder.Build()); } - protected virtual DbConnection CreateDbConnection(IServiceProvider serviceProvider) where T : IDbConfiguration - { - var connectionString = serviceProvider.GetRequiredService>().Value.ConnectionString; - var connectionFactory = new AzureSqlConnectionFactory(connectionString, - serviceProvider.GetRequiredService()); - - return Task.Run(() => connectionFactory.CreateAsync()).Result; - } - private void ConfigureDefaultJobServices(IServiceCollection services, IConfigurationRoot configurationRoot) { services.Configure(configurationRoot.GetSection(GalleryDbConfigurationSectionName)); @@ -142,12 +133,12 @@ private void ConfigureDefaultJobServices(IServiceCollection services, IConfigura services.AddScoped(p => { - return new ValidationEntitiesContext(CreateDbConnection(p)); + return new ValidationEntitiesContext(CreateSqlConnection()); }); services.AddScoped(p => { - return new EntitiesContext(CreateDbConnection(p), readOnly: true); + return new EntitiesContext(CreateSqlConnection(), readOnly: true); }); services.AddTransient(p => diff --git a/src/Validation.Common.Job/Validation.Common.Job.csproj b/src/Validation.Common.Job/Validation.Common.Job.csproj index fa1a131a8..15b0f761e 100644 --- a/src/Validation.Common.Job/Validation.Common.Job.csproj +++ b/src/Validation.Common.Job/Validation.Common.Job.csproj @@ -102,9 +102,6 @@ 2.27.0 - - 2.27.0 - 2.27.0 diff --git a/src/Validation.PackageSigning.ProcessSignature/Job.cs b/src/Validation.PackageSigning.ProcessSignature/Job.cs index 16997f935..14f9e5bdc 100644 --- a/src/Validation.PackageSigning.ProcessSignature/Job.cs +++ b/src/Validation.PackageSigning.ProcessSignature/Job.cs @@ -35,7 +35,7 @@ protected override void ConfigureJobServices(IServiceCollection services, IConfi services.AddScoped(p => { - return new EntitiesContext(CreateDbConnection(p), readOnly: false); + return new EntitiesContext(CreateSqlConnection(), readOnly: false); }); services.Add(ServiceDescriptor.Transient(typeof(IEntityRepository<>), typeof(EntityRepository<>))); diff --git a/src/Validation.PackageSigning.ProcessSignature/Validation.PackageSigning.ProcessSignature.csproj b/src/Validation.PackageSigning.ProcessSignature/Validation.PackageSigning.ProcessSignature.csproj index 3d331e9f9..1ae9d197a 100644 --- a/src/Validation.PackageSigning.ProcessSignature/Validation.PackageSigning.ProcessSignature.csproj +++ b/src/Validation.PackageSigning.ProcessSignature/Validation.PackageSigning.ProcessSignature.csproj @@ -85,11 +85,6 @@ Validation.PackageSigning.Core - - - 2.27.0 - - ..\..\build diff --git a/tests/NuGet.Services.Revalidate.Tests/NuGet.Services.Revalidate.Tests.csproj b/tests/NuGet.Services.Revalidate.Tests/NuGet.Services.Revalidate.Tests.csproj index a09ec1eea..172a9d5c3 100644 --- a/tests/NuGet.Services.Revalidate.Tests/NuGet.Services.Revalidate.Tests.csproj +++ b/tests/NuGet.Services.Revalidate.Tests/NuGet.Services.Revalidate.Tests.csproj @@ -82,5 +82,8 @@ Tests.ContextHelpers + + + \ No newline at end of file diff --git a/tests/Tests.Search.GenerateAuxiliaryData/RankingsExporterTests.cs b/tests/Tests.Search.GenerateAuxiliaryData/RankingsExporterTests.cs index 9a202115a..9e5cb51d7 100644 --- a/tests/Tests.Search.GenerateAuxiliaryData/RankingsExporterTests.cs +++ b/tests/Tests.Search.GenerateAuxiliaryData/RankingsExporterTests.cs @@ -3,11 +3,12 @@ using System; using System.Data; +using System.Data.SqlClient; +using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Microsoft.WindowsAzure.Storage.Blob; using Moq; using Newtonsoft.Json; -using NuGet.Services.Sql; using Search.GenerateAuxiliaryData; using Xunit; @@ -44,11 +45,16 @@ public void GetRankings_ReturnsRankings() private static RankingsExporter CreateExporter() { return new RankingsExporter( + DoNotOpenSqlConnectionAsync, new LoggerFactory().CreateLogger(), - connectionFactory: new Mock().Object, defaultDestinationContainer: new CloudBlobContainer(new Uri("https://nuget.org")), defaultRankingsScript: "b", defaultName: "c"); } + + public static Task DoNotOpenSqlConnectionAsync() + { + return Task.FromResult((SqlConnection)null); + } } } \ No newline at end of file diff --git a/tests/Tests.Search.GenerateAuxiliaryData/VerifiedPackagesExporterTests.cs b/tests/Tests.Search.GenerateAuxiliaryData/VerifiedPackagesExporterTests.cs index 50b8aea23..c31f36116 100644 --- a/tests/Tests.Search.GenerateAuxiliaryData/VerifiedPackagesExporterTests.cs +++ b/tests/Tests.Search.GenerateAuxiliaryData/VerifiedPackagesExporterTests.cs @@ -3,11 +3,12 @@ using System; using System.Data; +using System.Data.SqlClient; +using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Microsoft.WindowsAzure.Storage.Blob; using Moq; using Newtonsoft.Json; -using NuGet.Services.Sql; using Search.GenerateAuxiliaryData; using Xunit; @@ -43,11 +44,16 @@ public void GetVerifiedPackagesReturnsJsonString() private static VerifiedPackagesExporter CreateExporter() { return new VerifiedPackagesExporter( + DoNotOpenSqlConnectionAsync, new LoggerFactory().CreateLogger(), - connectionFactory: new Mock().Object, defaultDestinationContainer: new CloudBlobContainer(new Uri("https://nuget.org")), defaultVerifiedPackagesScript: "b", defaultName: "c"); } + + public static Task DoNotOpenSqlConnectionAsync() + { + return Task.FromResult((SqlConnection)null); + } } } \ No newline at end of file diff --git a/tests/Validation.PackageSigning.ScanAndSign.Tests/Validation.PackageSigning.ScanAndSign.Tests.csproj b/tests/Validation.PackageSigning.ScanAndSign.Tests/Validation.PackageSigning.ScanAndSign.Tests.csproj index 510501a59..b0662efcb 100644 --- a/tests/Validation.PackageSigning.ScanAndSign.Tests/Validation.PackageSigning.ScanAndSign.Tests.csproj +++ b/tests/Validation.PackageSigning.ScanAndSign.Tests/Validation.PackageSigning.ScanAndSign.Tests.csproj @@ -75,5 +75,8 @@ 2.3.1 + + + \ No newline at end of file From 5b766ba5e6e13fedd8101dccb8a37e11a305e617 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Sharma?= Date: Wed, 25 Jul 2018 08:55:36 -0700 Subject: [PATCH 3/3] [Repository Signing] Add PROD repository signing certificate (#490) Add the PROD repository signing certificate to Process Signature job. --- .../Settings/prod.json | 4 +++- .../SignatureValidator.cs | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/Validation.PackageSigning.ProcessSignature/Settings/prod.json b/src/Validation.PackageSigning.ProcessSignature/Settings/prod.json index 6e6021b58..d195c2287 100644 --- a/src/Validation.PackageSigning.ProcessSignature/Settings/prod.json +++ b/src/Validation.PackageSigning.ProcessSignature/Settings/prod.json @@ -19,7 +19,9 @@ "ConnectionString": "DefaultEndpointsProtocol=https;AccountName=nugetgallery;AccountKey=$$Prod-NuGetGalleryStorage-Key$$" }, "ProcessSignature": { - "AllowedRepositorySigningCertificates": [], + "AllowedRepositorySigningCertificates": [ + "cf7ac17ad047ecd5fdc36822031b12d4ef078b6f2b4c5e6ba41f8ff2cf4bad67" + ], "V3ServiceIndexUrl": "https://api.nuget.org/v3/index.json" }, diff --git a/src/Validation.PackageSigning.ProcessSignature/SignatureValidator.cs b/src/Validation.PackageSigning.ProcessSignature/SignatureValidator.cs index 9016dba69..57b3c872a 100644 --- a/src/Validation.PackageSigning.ProcessSignature/SignatureValidator.cs +++ b/src/Validation.PackageSigning.ProcessSignature/SignatureValidator.cs @@ -420,7 +420,7 @@ private async Task IsValidRepositorySignatureAsync(Context context, T s context.Message.PackageId, context.Message.PackageVersion, context.Message.ValidationId, - signature.V3ServiceIndexUrl.AbsoluteUri); + signature.V3ServiceIndexUrl?.AbsoluteUri); return false; }