diff --git a/src/NuGet.Services.Revalidate/Configuration/RevalidationConfiguration.cs b/src/NuGet.Services.Revalidate/Configuration/RevalidationConfiguration.cs index 671f779b6..e782f79de 100644 --- a/src/NuGet.Services.Revalidate/Configuration/RevalidationConfiguration.cs +++ b/src/NuGet.Services.Revalidate/Configuration/RevalidationConfiguration.cs @@ -1,13 +1,30 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +using System; + namespace NuGet.Services.Revalidate { public class RevalidationConfiguration { + /// + /// The time before the revalidation job restarts itself. + /// + public TimeSpan ShutdownWaitInterval { get; set; } = TimeSpan.FromDays(1); + + /// + /// How long the revalidation job should wait if a revalidation cannot be processed at this time. + /// + public TimeSpan RetryLaterSleep { get; set; } = TimeSpan.FromMinutes(5); + /// /// The configurations used to initialize the revalidation state. /// public InitializationConfiguration Initialization { get; set; } + + /// + /// The configurations used by the in-memory queue of revalidations to start. + /// + public RevalidationQueueConfiguration Queue { get; set; } } } diff --git a/src/NuGet.Services.Revalidate/Configuration/RevalidationQueueConfiguration.cs b/src/NuGet.Services.Revalidate/Configuration/RevalidationQueueConfiguration.cs new file mode 100644 index 000000000..54f8e929d --- /dev/null +++ b/src/NuGet.Services.Revalidate/Configuration/RevalidationQueueConfiguration.cs @@ -0,0 +1,21 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; + +namespace NuGet.Services.Revalidate +{ + public class RevalidationQueueConfiguration + { + /// + /// The maximum times that the should look for a revalidation + /// before giving up. + /// + public int MaximumAttempts { get; set; } = 5; + + /// + /// The time to sleep after an initialized revalidation is deemed completed. + /// + public TimeSpan SleepBetweenAttempts { get; set; } = TimeSpan.FromSeconds(5); + } +} diff --git a/src/NuGet.Services.Revalidate/Initialization/PackageFinder.cs b/src/NuGet.Services.Revalidate/Initialization/PackageFinder.cs index 06db94763..8768f2d12 100644 --- a/src/NuGet.Services.Revalidate/Initialization/PackageFinder.cs +++ b/src/NuGet.Services.Revalidate/Initialization/PackageFinder.cs @@ -189,7 +189,6 @@ private HashSet FindRegistrationKeys(string setName, Expression FindRegistrationKeys(string setName, Expression() + .RunAsync(); + + Logger.LogInformation("Revalidation service finished running"); } } } @@ -85,7 +94,9 @@ public override async Task Run() protected override void ConfigureJobServices(IServiceCollection services, IConfigurationRoot configurationRoot) { services.Configure(configurationRoot.GetSection(JobConfigurationSectionName)); + services.AddSingleton(provider => provider.GetRequiredService>().Value); services.AddSingleton(provider => provider.GetRequiredService>().Value.Initialization); + services.AddSingleton(provider => provider.GetRequiredService>().Value.Queue); services.AddScoped(provider => { @@ -94,9 +105,31 @@ protected override void ConfigureJobServices(IServiceCollection services, IConfi return new GalleryContext(config.ConnectionString, readOnly: false); }); - services.AddScoped(); - services.AddScoped(); - services.AddScoped(); + // Core + services.AddTransient(); + services.AddTransient(); + + services.AddTransient(); + + // Initialization + services.AddTransient(); + services.AddTransient(); + + // Revalidation + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + + services.AddTransient(); + services.AddTransient(); + services.AddTransient(provider => + { + var config = provider.GetRequiredService>().Value; + + return new TopicClientWrapper(config.ConnectionString, config.TopicPath); + }); } protected override void ConfigureAutofacServices(ContainerBuilder containerBuilder) diff --git a/src/NuGet.Services.Revalidate/NuGet.Services.Revalidate.csproj b/src/NuGet.Services.Revalidate/NuGet.Services.Revalidate.csproj index 9bb5fe580..20c23ccfc 100644 --- a/src/NuGet.Services.Revalidate/NuGet.Services.Revalidate.csproj +++ b/src/NuGet.Services.Revalidate/NuGet.Services.Revalidate.csproj @@ -44,16 +44,28 @@ + - - - - + + + + + + + + + + + + + + + diff --git a/src/NuGet.Services.Revalidate/Services/HealthService.cs b/src/NuGet.Services.Revalidate/Services/HealthService.cs new file mode 100644 index 000000000..7d1f0a388 --- /dev/null +++ b/src/NuGet.Services.Revalidate/Services/HealthService.cs @@ -0,0 +1,17 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Threading.Tasks; + +namespace NuGet.Services.Revalidate +{ + public class HealthService : IHealthService + { + public Task IsHealthyAsync() + { + // TODO: + // We are software gods that never make mistakes. + return Task.FromResult(true); + } + } +} diff --git a/src/NuGet.Services.Revalidate/Services/IHealthService.cs b/src/NuGet.Services.Revalidate/Services/IHealthService.cs new file mode 100644 index 000000000..204cd90b8 --- /dev/null +++ b/src/NuGet.Services.Revalidate/Services/IHealthService.cs @@ -0,0 +1,16 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Threading.Tasks; + +namespace NuGet.Services.Revalidate +{ + public interface IHealthService + { + /// + /// Determine whether the NuGet service is healthy. + /// + /// Whether the NuGet service is healthy. + Task IsHealthyAsync(); + } +} diff --git a/src/NuGet.Services.Revalidate/Services/IRevalidationQueue.cs b/src/NuGet.Services.Revalidate/Services/IRevalidationQueue.cs new file mode 100644 index 000000000..7ff6e7fbf --- /dev/null +++ b/src/NuGet.Services.Revalidate/Services/IRevalidationQueue.cs @@ -0,0 +1,17 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Threading.Tasks; +using NuGet.Services.Validation; + +namespace NuGet.Services.Revalidate +{ + public interface IRevalidationQueue + { + /// + /// Fetch the next package to revalidate. + /// + /// The next package to revalidate, or null if there are no packages to revalidate at this time. + Task NextOrNullAsync(); + } +} diff --git a/src/NuGet.Services.Revalidate/ITelemetryService.cs b/src/NuGet.Services.Revalidate/Services/IRevalidationService.cs similarity index 58% rename from src/NuGet.Services.Revalidate/ITelemetryService.cs rename to src/NuGet.Services.Revalidate/Services/IRevalidationService.cs index 9d4101229..220c51d82 100644 --- a/src/NuGet.Services.Revalidate/ITelemetryService.cs +++ b/src/NuGet.Services.Revalidate/Services/IRevalidationService.cs @@ -1,11 +1,14 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. -using System; +using System.Threading.Tasks; namespace NuGet.Services.Revalidate { - public interface ITelemetryService + public interface IRevalidationService { + Task RunAsync(); + + Task StartNextRevalidationAsync(); } } diff --git a/src/NuGet.Services.Revalidate/IRevalidationStateService.cs b/src/NuGet.Services.Revalidate/Services/IRevalidationStateService.cs similarity index 64% rename from src/NuGet.Services.Revalidate/IRevalidationStateService.cs rename to src/NuGet.Services.Revalidate/Services/IRevalidationStateService.cs index 606cf3988..c0180cff6 100644 --- a/src/NuGet.Services.Revalidate/IRevalidationStateService.cs +++ b/src/NuGet.Services.Revalidate/Services/IRevalidationStateService.cs @@ -1,7 +1,6 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. -using System; using System.Collections.Generic; using System.Threading.Tasks; using NuGet.Services.Validation; @@ -10,6 +9,12 @@ namespace NuGet.Services.Revalidate { public interface IRevalidationStateService { + /// + /// Check whether the killswitch has been activated. If it has, all revalidation operations should be halted. + /// + /// Whether the killswitch has been activated. + Task IsKillswitchActiveAsync(); + /// /// Add the new revalidations to the database. /// @@ -27,5 +32,12 @@ public interface IRevalidationStateService /// /// The count of package revalidations in the database. Task PackageRevalidationCountAsync(); + + /// + /// Update the package revalidation and mark is as enqueued. + /// + /// The revalidation to update. + /// A task that completes once the revalidation has been updated. + Task MarkRevalidationAsEnqueuedAsync(PackageRevalidation revalidation); } } diff --git a/src/NuGet.Services.Revalidate/Services/IRevalidationThrottler.cs b/src/NuGet.Services.Revalidate/Services/IRevalidationThrottler.cs new file mode 100644 index 000000000..1d03969e6 --- /dev/null +++ b/src/NuGet.Services.Revalidate/Services/IRevalidationThrottler.cs @@ -0,0 +1,41 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Threading.Tasks; + +namespace NuGet.Services.Revalidate +{ + public interface IRevalidationThrottler + { + /// + /// Check whether the revalidation capacity has been reached. + /// + /// If true, no more revalidations should be performed. + Task IsThrottledAsync(); + + /// + /// Reset the capacity to the configured minimum value. Call this when the service's status is degraded to + /// throttle the revalidations. + /// + /// A task that completes once the capacity theshold has been reset. + Task ResetCapacityAsync(); + + /// + /// Increase the revalidation capacity by one revalidation per minute. + /// + /// A task taht completes once the capacity has been increased. + Task IncreaseCapacityAsync(); + + /// + /// Delay the current task to achieve the desired revalidation rate. + /// + /// Delay the task to ensure the desired revalidation rate. + Task DelayUntilNextRevalidationAsync(); + + /// + /// Delay the current task until when a revalidation can be retried. + /// + /// Delay the task until when revalidations can be retried. + Task DelayUntilRevalidationRetryAsync(); + } +} diff --git a/src/NuGet.Services.Revalidate/Services/ISingletonService.cs b/src/NuGet.Services.Revalidate/Services/ISingletonService.cs new file mode 100644 index 000000000..0db890bf7 --- /dev/null +++ b/src/NuGet.Services.Revalidate/Services/ISingletonService.cs @@ -0,0 +1,19 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Threading.Tasks; + +namespace NuGet.Services.Revalidate +{ + /// + /// Used to ensure that only one instance of this service is running at once. + /// + public interface ISingletonService + { + /// + /// Determines whether this is the only instance of the service running. + /// + /// True if this service is the only instance of the service running. + Task IsSingletonAsync(); + } +} diff --git a/src/NuGet.Services.Revalidate/Services/ITelemetryService.cs b/src/NuGet.Services.Revalidate/Services/ITelemetryService.cs new file mode 100644 index 000000000..01c487039 --- /dev/null +++ b/src/NuGet.Services.Revalidate/Services/ITelemetryService.cs @@ -0,0 +1,16 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; + +namespace NuGet.Services.Revalidate +{ + public interface ITelemetryService + { + IDisposable TrackDurationToStartNextRevalidation(); + + void TrackPackageRevalidationMarkedAsCompleted(string packageId, string normalizedVersion); + + void TrackPackageRevalidationStarted(string packageId, string normalizedVersion); + } +} diff --git a/src/NuGet.Services.Revalidate/Services/RevalidationQueue.cs b/src/NuGet.Services.Revalidate/Services/RevalidationQueue.cs new file mode 100644 index 000000000..679ac81ce --- /dev/null +++ b/src/NuGet.Services.Revalidate/Services/RevalidationQueue.cs @@ -0,0 +1,127 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Data.Entity; +using System.Data.Entity.Infrastructure; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using NuGet.Services.Validation; +using NuGetGallery; + +namespace NuGet.Services.Revalidate +{ + using IGalleryContext = IEntitiesContext; + + public class RevalidationQueue : IRevalidationQueue + { + private readonly IGalleryContext _galleryContext; + private readonly IValidationEntitiesContext _validationContext; + private readonly RevalidationQueueConfiguration _config; + private readonly ITelemetryService _telemetry; + private readonly ILogger _logger; + + public RevalidationQueue( + IGalleryContext galleryContext, + IValidationEntitiesContext validationContext, + RevalidationQueueConfiguration config, + ITelemetryService telemetry, + ILogger logger) + { + _galleryContext = galleryContext ?? throw new ArgumentNullException(nameof(galleryContext)); + _validationContext = validationContext ?? throw new ArgumentNullException(nameof(validationContext)); + _config = config ?? throw new ArgumentNullException(nameof(config)); + _telemetry = telemetry ?? throw new ArgumentNullException(nameof(telemetry)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task NextOrNullAsync() + { + for (var i = 0; i < _config.MaximumAttempts; i++) + { + _logger.LogInformation( + "Attempting to find the next revalidation. Try {Attempt} of {MaxAttempts}", + i + 1, + _config.MaximumAttempts); + + var next = await _validationContext.PackageRevalidations + .Where(r => r.Enqueued == null) + .Where(r => r.Completed == false) + .OrderBy(r => r.Key) + .FirstOrDefaultAsync(); + + if (next == null) + { + _logger.LogWarning("Could not find any incomplete revalidations"); + return null; + } + + // Don't revalidate packages that already have a repository signature or that no longer exist. + if (await HasRepositorySignature(next) || await IsDeleted(next)) + { + await MarkAsCompleted(next); + await Task.Delay(_config.SleepBetweenAttempts); + + continue; + } + + _logger.LogInformation( + "Found revalidation for {PackageId} {PackageNormalizedVersion} after {Attempt} attempts", + next.PackageId, + next.PackageNormalizedVersion, + i + 1); + + return next; + } + + _logger.LogInformation( + "Did not find any revalidations after {MaxAttempts}. Retry later...", + _config.MaximumAttempts); + + return null; + } + + private Task HasRepositorySignature(PackageRevalidation revalidation) + { + return _validationContext.PackageSigningStates + .Where(s => s.PackageId == revalidation.PackageId) + .Where(s => s.PackageNormalizedVersion == revalidation.PackageNormalizedVersion) + .Where(s => s.PackageSignatures.Any(sig => sig.Type == PackageSignatureType.Repository)) + .AnyAsync(); + } + + private async Task IsDeleted(PackageRevalidation revalidation) + { + var packageStatus = await _galleryContext.Set() + .Where(p => p.PackageRegistration.Id == revalidation.PackageId) + .Where(p => p.NormalizedVersion == revalidation.PackageNormalizedVersion) + .Select(p => (PackageStatus?)p.PackageStatusKey) + .FirstOrDefaultAsync(); + + return (packageStatus == null || packageStatus == PackageStatus.Deleted); + } + + private async Task MarkAsCompleted(PackageRevalidation revalidation) + { + _logger.LogInformation( + "Marking package revalidation as completed as it has a repository signature or is deleted for {PackageId} {PackageNormalizedVersion}", + revalidation.PackageId, + revalidation.PackageNormalizedVersion); + + try + { + revalidation.Completed = true; + + await _validationContext.SaveChangesAsync(); + + _telemetry.TrackPackageRevalidationMarkedAsCompleted(revalidation.PackageId, revalidation.PackageNormalizedVersion); + } + catch (DbUpdateConcurrencyException) + { + // Swallow concurrency exceptions. The package will be marked as completed + // on the next iteration of "NextOrNullAsync". + } + } + } +} diff --git a/src/NuGet.Services.Revalidate/Services/RevalidationResult.cs b/src/NuGet.Services.Revalidate/Services/RevalidationResult.cs new file mode 100644 index 000000000..8bed7a7cf --- /dev/null +++ b/src/NuGet.Services.Revalidate/Services/RevalidationResult.cs @@ -0,0 +1,26 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +namespace NuGet.Services.Revalidate +{ + /// + /// The result from + /// + public enum RevalidationResult + { + /// + /// A revalidation was successfully enqueued. + /// + RevalidationEnqueued, + + /// + /// A revalidation could not be enqueued at this time. The revalidation should be retried later. + /// + RetryLater, + + /// + /// This instance of the revalidation job has reached an unrecoverable state and MUST stop. + /// + UnrecoverableError, + } +} diff --git a/src/NuGet.Services.Revalidate/Services/RevalidationService.cs b/src/NuGet.Services.Revalidate/Services/RevalidationService.cs new file mode 100644 index 000000000..46db515de --- /dev/null +++ b/src/NuGet.Services.Revalidate/Services/RevalidationService.cs @@ -0,0 +1,172 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Diagnostics; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using NuGet.Services.Validation; + +namespace NuGet.Services.Revalidate +{ + public class RevalidationService : IRevalidationService + { + private readonly IRevalidationStateService _state; + private readonly ISingletonService _singletonService; + private readonly IRevalidationThrottler _throttler; + private readonly IHealthService _healthService; + private readonly IRevalidationQueue _revalidationQueue; + private readonly IPackageValidationEnqueuer _validationEnqueuer; + private readonly RevalidationConfiguration _config; + private readonly ITelemetryService _telemetryService; + private readonly ILogger _logger; + + public RevalidationService( + IRevalidationStateService state, + ISingletonService singletonService, + IRevalidationThrottler throttler, + IHealthService healthService, + IRevalidationQueue revalidationQueue, + IPackageValidationEnqueuer validationEnqueuer, + RevalidationConfiguration config, + ITelemetryService telemetryService, + ILogger logger) + { + _state = state ?? throw new ArgumentNullException(nameof(state)); + _singletonService = singletonService ?? throw new ArgumentNullException(nameof(singletonService)); + _throttler = throttler ?? throw new ArgumentNullException(nameof(throttler)); + _healthService = healthService ?? throw new ArgumentNullException(nameof(healthService)); + _revalidationQueue = revalidationQueue ?? throw new ArgumentNullException(nameof(revalidationQueue)); + _validationEnqueuer = validationEnqueuer ?? throw new ArgumentNullException(nameof(validationEnqueuer)); + _config = config ?? throw new ArgumentNullException(nameof(config)); + _telemetryService = telemetryService ?? throw new ArgumentNullException(nameof(telemetryService)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task RunAsync() + { + var runTime = Stopwatch.StartNew(); + + do + { + _logger.LogInformation("Starting next revalidation..."); + + var result = await StartNextRevalidationAsync(); + + switch (result) + { + case RevalidationResult.RevalidationEnqueued: + _logger.LogInformation("Successfully enqueued revalidation"); + + await _throttler.DelayUntilNextRevalidationAsync(); + break; + + case RevalidationResult.RetryLater: + _logger.LogInformation("Could not start revalidation, retrying later"); + + await _throttler.DelayUntilRevalidationRetryAsync(); + break; + + case RevalidationResult.UnrecoverableError: + default: + _logger.LogCritical( + "Stopping revalidations due to unrecoverable or unknown result {Result}", + result); + + return; + } + } + while (runTime.Elapsed <= _config.ShutdownWaitInterval); + + _logger.LogInformation("Finished running after {ElapsedTime}", runTime.Elapsed); + } + + public async Task StartNextRevalidationAsync() + { + using (_telemetryService.TrackDurationToStartNextRevalidation()) + { + // Don't start a revalidation if the job has been deactivated, if the ingestion pipeline is unhealthy, + // or if we have reached our quota of desired revalidations. + var checkResult = await CanStartRevalidationAsync(); + if (checkResult != null) + { + _logger.LogInformation( + "Detected that a revalidation should not be started due to result {Result}", + checkResult.Value); + + return checkResult.Value; + } + + // Everything is in tip-top shape! Increase the throttling quota and start the next revalidation. + await _throttler.IncreaseCapacityAsync(); + + var revalidation = await _revalidationQueue.NextOrNullAsync(); + if (revalidation == null) + { + _logger.LogInformation("Could not find a package to revalidate at this time, retry later..."); + + return RevalidationResult.RetryLater; + } + + return await StartRevalidationAsync(revalidation); + } + } + + private async Task CanStartRevalidationAsync() + { + if (!await _singletonService.IsSingletonAsync()) + { + _logger.LogCritical("Detected another instance of the revalidate job, cancelling revalidations!"); + + return RevalidationResult.UnrecoverableError; + } + + if (await _state.IsKillswitchActiveAsync()) + { + _logger.LogWarning("Revalidation killswitch has been activated, retry later..."); + + return RevalidationResult.RetryLater; + } + + if (await _throttler.IsThrottledAsync()) + { + _logger.LogInformation("Revalidations have reached the desired event rate, retry later..."); + + return RevalidationResult.RetryLater; + } + + if (!await _healthService.IsHealthyAsync()) + { + _logger.LogWarning("Service appears to be unhealthy, resetting throttling capacity and retry later..."); + + await _throttler.ResetCapacityAsync(); + + return RevalidationResult.RetryLater; + } + + if (await _state.IsKillswitchActiveAsync()) + { + _logger.LogWarning("Revalidation killswitch has been activated after the throttle and health check, retry later..."); + + return RevalidationResult.RetryLater; + } + + return null; + } + + private async Task StartRevalidationAsync(PackageRevalidation revalidation) + { + var message = new PackageValidationMessageData( + revalidation.PackageId, + revalidation.PackageNormalizedVersion, + revalidation.ValidationTrackingId.Value); + + await _validationEnqueuer.StartValidationAsync(message); + await _state.MarkRevalidationAsEnqueuedAsync(revalidation); + + _telemetryService.TrackPackageRevalidationStarted(revalidation.PackageId, revalidation.PackageNormalizedVersion); + + return RevalidationResult.RevalidationEnqueued; + } + } +} diff --git a/src/NuGet.Services.Revalidate/RevalidationStateService.cs b/src/NuGet.Services.Revalidate/Services/RevalidationStateService.cs similarity index 75% rename from src/NuGet.Services.Revalidate/RevalidationStateService.cs rename to src/NuGet.Services.Revalidate/Services/RevalidationStateService.cs index 0d2e68159..1220486a1 100644 --- a/src/NuGet.Services.Revalidate/RevalidationStateService.cs +++ b/src/NuGet.Services.Revalidate/Services/RevalidationStateService.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Generic; using System.Data.Entity; +using System.Data.Entity.Infrastructure; using System.Linq; using System.Threading.Tasks; using Microsoft.Extensions.Logging; @@ -22,6 +23,12 @@ public RevalidationStateService(IValidationEntitiesContext context, ILogger IsKillswitchActiveAsync() + { + // TODO + return Task.FromResult(false); + } + public async Task AddPackageRevalidationsAsync(IReadOnlyList revalidations) { var validationContext = _context as ValidationEntitiesContext; @@ -69,5 +76,24 @@ public async Task PackageRevalidationCountAsync() { return await _context.PackageRevalidations.CountAsync(); } + + public async Task MarkRevalidationAsEnqueuedAsync(PackageRevalidation revalidation) + { + try + { + revalidation.Enqueued = DateTime.UtcNow; + + await _context.SaveChangesAsync(); + } + catch (DbUpdateConcurrencyException) + { + _logger.LogWarning( + "Failed to update revalidation as enqueued for {PackageId} {PackageNormalizedVersion}", + revalidation.PackageId, + revalidation.PackageNormalizedVersion); + + throw; + } + } } } diff --git a/src/NuGet.Services.Revalidate/Services/RevalidationThrottler.cs b/src/NuGet.Services.Revalidate/Services/RevalidationThrottler.cs new file mode 100644 index 000000000..b736d5863 --- /dev/null +++ b/src/NuGet.Services.Revalidate/Services/RevalidationThrottler.cs @@ -0,0 +1,58 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; + +namespace NuGet.Services.Revalidate +{ + public class RevalidationThrottler : IRevalidationThrottler + { + private readonly RevalidationConfiguration _config; + private readonly ILogger _logger; + + public RevalidationThrottler(RevalidationConfiguration config, ILogger logger) + { + _config = config ?? throw new ArgumentNullException(nameof(config)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public Task IsThrottledAsync() + { + // TODO: + // Calculate desired event rate + // Calculate current event rate (# of revalidations + Gallery actions) + // Compare desired event rate to configured event rate. If configured rate is higher, update desired event rate. + // If current event rate is greater than or equal to desired event rate, return true; + return Task.FromResult(false); + } + + public Task ResetCapacityAsync() + { + return Task.CompletedTask; + } + + public Task IncreaseCapacityAsync() + { + return Task.CompletedTask; + } + + public async Task DelayUntilNextRevalidationAsync() + { + // TODO: Calculate sleep duration to achieve desired event rate. + _logger.LogInformation("Delaying until next revalidation by sleeping for 5 minutes..."); + + await Task.Delay(TimeSpan.FromMinutes(5)); + } + + public async Task DelayUntilRevalidationRetryAsync() + { + _logger.LogInformation( + "Delaying for revalidation retry by sleeping for {SleepDuration}", + _config.RetryLaterSleep); + + await Task.Delay(_config.RetryLaterSleep); + } + } +} diff --git a/src/NuGet.Services.Revalidate/Services/SingletonService.cs b/src/NuGet.Services.Revalidate/Services/SingletonService.cs new file mode 100644 index 000000000..79e92d1ab --- /dev/null +++ b/src/NuGet.Services.Revalidate/Services/SingletonService.cs @@ -0,0 +1,16 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Threading.Tasks; + +namespace NuGet.Services.Revalidate +{ + public class SingletonService : ISingletonService + { + public Task IsSingletonAsync() + { + // TODO + return Task.FromResult(true); + } + } +} diff --git a/src/NuGet.Services.Revalidate/Services/TelemetryService.cs b/src/NuGet.Services.Revalidate/Services/TelemetryService.cs new file mode 100644 index 000000000..9ce792992 --- /dev/null +++ b/src/NuGet.Services.Revalidate/Services/TelemetryService.cs @@ -0,0 +1,57 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using NuGet.Services.Logging; + +namespace NuGet.Services.Revalidate +{ + public class TelemetryService : ITelemetryService + { + private const string RevalidationPrefix = "Revalidation."; + + private const string DurationToStartNextRevalidation = RevalidationPrefix + "DurationToStartNextRevalidation"; + private const string PackageRevalidationMarkedAsCompleted = RevalidationPrefix + "PackageRevalidationMarkedAsCompleted"; + private const string PackageRevalidationStarted = RevalidationPrefix + "PackageRevalidationStarted"; + + private const string PackageId = "PackageId"; + private const string NormalizedVersion = "NormalizedVersion"; + + private readonly ITelemetryClient _client; + + public TelemetryService(ITelemetryClient client) + { + _client = client ?? throw new ArgumentNullException(nameof(client)); + } + + public IDisposable TrackDurationToStartNextRevalidation() + { + return _client.TrackDuration(DurationToStartNextRevalidation); + } + + public void TrackPackageRevalidationMarkedAsCompleted(string packageId, string normalizedVersion) + { + _client.TrackMetric( + PackageRevalidationMarkedAsCompleted, + 1, + new Dictionary + { + { PackageId, packageId }, + { NormalizedVersion, normalizedVersion }, + }); + } + + public void TrackPackageRevalidationStarted(string packageId, string normalizedVersion) + { + _client.TrackMetric( + PackageRevalidationStarted, + 1, + new Dictionary + { + { PackageId, packageId }, + { NormalizedVersion, normalizedVersion }, + }); + } + } +} diff --git a/src/NuGet.Services.Revalidate/Settings/dev.json b/src/NuGet.Services.Revalidate/Settings/dev.json index 573861a73..2964de561 100644 --- a/src/NuGet.Services.Revalidate/Settings/dev.json +++ b/src/NuGet.Services.Revalidate/Settings/dev.json @@ -8,6 +8,10 @@ ], "MaxPackageCreationDate": "2021-03-01T23:52:40.7022034+00:00", // TODO: Update this when repository signing is enabled "SleepDurationBetweenBatches": "00:00:01" + }, + "Queue": { + "MaximumAttempts": 5, + "SleepBetweenAttempts": "00:05:00" } }, @@ -18,9 +22,8 @@ "ConnectionString": "Data Source=tcp:#{Jobs.validation.DatabaseAddress};Initial Catalog=nuget-dev-validation;Integrated Security=False;User ID=$$Dev-ValidationDBWriter-UserName$$;Password=$$Dev-ValidationDBWriter-Password$$;Connect Timeout=30;Encrypt=True" }, "ServiceBus": { - "ConnectionString": "Endpoint=sb://nugetdev.servicebus.windows.net/;SharedAccessKeyName=package-certificates-validator;SharedAccessKey=$$Dev-ServiceBus-SharedAccessKey-Validation-CertificatesValidator$$", - "TopicPath": "validate-certificate", - "SubscriptionName": "validate-certificate" + "ConnectionString": "Endpoint=sb://nugetdev.servicebus.windows.net/;SharedAccessKeyName=gallery;SharedAccessKey=$$Dev-ServiceBus-SharedAccessKey-Validation-GallerySender$$", + "TopicPath": "validation" }, "KeyVault_VaultName": "#{Deployment.Azure.KeyVault.VaultName}", diff --git a/src/NuGet.Services.Revalidate/Settings/int.json b/src/NuGet.Services.Revalidate/Settings/int.json index f0b293e63..25c5abb5c 100644 --- a/src/NuGet.Services.Revalidate/Settings/int.json +++ b/src/NuGet.Services.Revalidate/Settings/int.json @@ -8,6 +8,10 @@ ], "MaxPackageCreationDate": "2021-03-01T23:52:40.7022034+00:00", // TODO: Update this when repository signing is enabled "SleepDurationBetweenBatches": "00:00:30" + }, + "Queue": { + "MaximumAttempts": 5, + "SleepBetweenAttempts": "00:05:00" } }, @@ -18,9 +22,8 @@ "ConnectionString": "Data Source=tcp:#{Jobs.validation.DatabaseAddress};Initial Catalog=nuget-int-validation;Integrated Security=False;User ID=$$Int-ValidationDBWriter-UserName$$;Password=$$Int-ValidationDBWriter-Password$$;Connect Timeout=30;Encrypt=True" }, "ServiceBus": { - "ConnectionString": "Endpoint=sb://nugetint.servicebus.windows.net/;SharedAccessKeyName=package-certificates-validator;SharedAccessKey=$$Int-ServiceBus-SharedAccessKey-Validation-CertificatesValidator$$", - "TopicPath": "validate-certificate", - "SubscriptionName": "validate-certificate" + "ConnectionString": "Endpoint=sb://nugetint.servicebus.windows.net/;SharedAccessKeyName=gallery;SharedAccessKey=$$Int-ServiceBus-SharedAccessKey-Validation-GallerySender$$", + "TopicPath": "validation" }, "KeyVault_VaultName": "#{Deployment.Azure.KeyVault.VaultName}", diff --git a/src/NuGet.Services.Revalidate/Settings/prod.json b/src/NuGet.Services.Revalidate/Settings/prod.json index 69c6316d8..519b9d11f 100644 --- a/src/NuGet.Services.Revalidate/Settings/prod.json +++ b/src/NuGet.Services.Revalidate/Settings/prod.json @@ -8,6 +8,10 @@ ], "MaxPackageCreationDate": "2021-03-01T23:52:40.7022034+00:00", // TODO: Update this when repository signing is enabled "SleepDurationBetweenBatches": "00:00:30" + }, + "Queue": { + "MaximumAttempts": 5, + "SleepBetweenAttempts": "00:05:00" } }, @@ -18,9 +22,8 @@ "ConnectionString": "Data Source=tcp:#{Jobs.validation.DatabaseAddress};Initial Catalog=nuget-prod-validation;Integrated Security=False;User ID=$$Prod-ValidationDBWriter-UserName$$;Password=$$Prod-ValidationDBWriter-Password$$;Connect Timeout=30;Encrypt=True" }, "ServiceBus": { - "ConnectionString": "Endpoint=sb://nugetprod.servicebus.windows.net/;SharedAccessKeyName=package-certificates-validator;SharedAccessKey=$$Prod-ServiceBus-SharedAccessKey-Validation-CertificatesValidator$$", - "TopicPath": "validate-certificate", - "SubscriptionName": "validate-certificate" + "ConnectionString": "Endpoint=sb://nugetprod.servicebus.windows.net/;SharedAccessKeyName=gallery;SharedAccessKey=$$Prod-ServiceBus-SharedAccessKey-Validation-GallerySender$$", + "TopicPath": "validation" }, "KeyVault_VaultName": "#{Deployment.Azure.KeyVault.VaultName}", diff --git a/src/NuGet.Services.Revalidate/TelemetryService.cs b/src/NuGet.Services.Revalidate/TelemetryService.cs deleted file mode 100644 index 354dc2ebf..000000000 --- a/src/NuGet.Services.Revalidate/TelemetryService.cs +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using System; -using NuGet.Services.Logging; - -namespace NuGet.Services.Revalidate -{ - public class TelemetryService : ITelemetryService - { - private readonly ITelemetryClient _client; - - private const string Prefix = "Revalidate."; - - public TelemetryService(ITelemetryClient client) - { - _client = client ?? throw new ArgumentNullException(nameof(client)); - } - } -} diff --git a/src/NuGet.Services.Validation.Orchestrator/settings.json b/src/NuGet.Services.Validation.Orchestrator/settings.json index f98f5cee5..53c714022 100644 --- a/src/NuGet.Services.Validation.Orchestrator/settings.json +++ b/src/NuGet.Services.Validation.Orchestrator/settings.json @@ -4,7 +4,9 @@ { "name": "VcsValidator", "TrackAfter": "1:00:00:00", - "requiredValidations": [ "PackageSigningValidator" ], + "requiredValidations": [ + "PackageSigningValidator" + ], "ShouldStart": true, "FailureBehavior": "MustSucceed" }, @@ -18,14 +20,18 @@ { "name": "PackageSigningValidator2", "TrackAfter": "00:10:00", - "requiredValidations": [ "PackageSigningValidator" ], + "requiredValidations": [ + "PackageSigningValidator" + ], "ShouldStart": true, "FailureBehavior": "AllowedToFail" }, { "name": "PackageCertificatesValidator", "TrackAfter": "00:10:00", - "requiredValidations": [ "PackageSigningValidator2" ], + "requiredValidations": [ + "PackageSigningValidator2" + ], "ShouldStart": true, "FailureBehavior": "MustSucceed" } diff --git a/tests/NuGet.Services.Revalidate.Tests/NuGet.Services.Revalidate.Tests.csproj b/tests/NuGet.Services.Revalidate.Tests/NuGet.Services.Revalidate.Tests.csproj index 6ce4829e5..0fe5be2eb 100644 --- a/tests/NuGet.Services.Revalidate.Tests/NuGet.Services.Revalidate.Tests.csproj +++ b/tests/NuGet.Services.Revalidate.Tests/NuGet.Services.Revalidate.Tests.csproj @@ -67,6 +67,8 @@ + + diff --git a/tests/NuGet.Services.Revalidate.Tests/Services/RevalidationQueueFacts.cs b/tests/NuGet.Services.Revalidate.Tests/Services/RevalidationQueueFacts.cs new file mode 100644 index 000000000..92dc3260b --- /dev/null +++ b/tests/NuGet.Services.Revalidate.Tests/Services/RevalidationQueueFacts.cs @@ -0,0 +1,270 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Moq; +using NuGet.Services.Validation; +using NuGetGallery; +using Tests.ContextHelpers; +using Xunit; + +namespace NuGet.Services.Revalidate.Tests.Services +{ + public class RevalidationQueueFacts + { + public class TheNextOrNullAsyncMethod : FactsBase + { + [Fact] + public async Task SkipsEnqueuedOrCompletedRevalidations() + { + // Arrange + _validationContext.Mock(packageRevalidations: new[] + { + new PackageRevalidation + { + Key = 1, + PackageId = "Enqueued.Package", + PackageNormalizedVersion = "1.0.0", + Enqueued = DateTime.UtcNow, + Completed = false, + }, + new PackageRevalidation + { + Key = 2, + PackageId = "Completed.Package", + PackageNormalizedVersion = "1.0.0", + Enqueued = null, + Completed = true, + }, + new PackageRevalidation + { + Key = 3, + PackageId = "Enqueued.And.Completed.Package", + PackageNormalizedVersion = "1.0.0", + Enqueued = DateTime.UtcNow, + Completed = true, + }, + new PackageRevalidation + { + Key = 4, + PackageId = "Package", + PackageNormalizedVersion = "1.0.0", + Enqueued = null, + Completed = false, + }, + }); + + _galleryContext.Mock(packages: new[] + { + new Package + { + PackageRegistration = new PackageRegistration { Id = "Package" }, + NormalizedVersion = "1.0.0", + PackageStatusKey = PackageStatus.Available, + } + }); + + // Act + var next = await _target.NextOrNullAsync(); + + // Assert + Assert.Equal("Package", next.PackageId); + Assert.Equal("1.0.0", next.PackageNormalizedVersion); + } + + [Fact] + public async Task SkipsRepositorySignedPackages() + { + // Arrange + _validationContext.Mock( + packageRevalidations: new[] + { + new PackageRevalidation + { + Key = 1, + PackageId = "Repository.Signed.Package", + PackageNormalizedVersion = "1.0.0", + Enqueued = null, + Completed = false, + }, + new PackageRevalidation + { + Key = 2, + PackageId = "Package", + PackageNormalizedVersion = "1.0.0", + Enqueued = null, + Completed = false, + }, + }, + packageSigningStates: new[] + { + new PackageSigningState + { + PackageId = "Repository.Signed.Package", + PackageNormalizedVersion = "1.0.0", + + PackageSignatures = new[] + { + new PackageSignature { Type = PackageSignatureType.Repository } + } + } + }); + + _galleryContext.Mock(packages: new[] + { + new Package + { + PackageRegistration = new PackageRegistration { Id = "Package" }, + NormalizedVersion = "1.0.0", + PackageStatusKey = PackageStatus.Available, + } + }); + + // Act + var next = await _target.NextOrNullAsync(); + + // Assert + Assert.Equal("Package", next.PackageId); + Assert.Equal("1.0.0", next.PackageNormalizedVersion); + + _telemetryService.Verify(t => t.TrackPackageRevalidationMarkedAsCompleted("Repository.Signed.Package", "1.0.0"), Times.Once); + } + + [Fact] + public async Task SkipsDeletedPackages() + { + // Arrange + _validationContext.Mock(packageRevalidations: new[] + { + new PackageRevalidation + { + Key = 1, + PackageId = "Soft.Deleted.Package", + PackageNormalizedVersion = "1.0.0", + Enqueued = null, + Completed = false, + }, + new PackageRevalidation + { + Key = 2, + PackageId = "Hard.Deleted.Package", + PackageNormalizedVersion = "1.0.0", + Enqueued = null, + Completed = false, + }, + new PackageRevalidation + { + Key = 3, + PackageId = "Package", + PackageNormalizedVersion = "1.0.0", + Enqueued = null, + Completed = false, + }, + }); + + _galleryContext.Mock(packages: new[] + { + new Package + { + PackageRegistration = new PackageRegistration { Id = "Soft.Deleted.Package" }, + NormalizedVersion = "1.0.0", + PackageStatusKey = PackageStatus.Deleted, + }, + new Package + { + PackageRegistration = new PackageRegistration { Id = "Package" }, + NormalizedVersion = "1.0.0", + PackageStatusKey = PackageStatus.Available, + } + }); + + // Act + var next = await _target.NextOrNullAsync(); + + // Assert + Assert.Equal("Package", next.PackageId); + Assert.Equal("1.0.0", next.PackageNormalizedVersion); + + _telemetryService.Verify(t => t.TrackPackageRevalidationMarkedAsCompleted("Soft.Deleted.Package", "1.0.0"), Times.Once); + _telemetryService.Verify(t => t.TrackPackageRevalidationMarkedAsCompleted("Hard.Deleted.Package", "1.0.0"), Times.Once); + } + + [Fact] + public async Task IfReachesAttemptsThreshold_ReturnsNull() + { + // Arrange + _config.MaximumAttempts = 1; + + _validationContext.Mock(packageRevalidations: new[] + { + new PackageRevalidation + { + Key = 1, + PackageId = "Hard.Deleted.Package", + PackageNormalizedVersion = "1.0.0", + Enqueued = null, + Completed = false, + }, + new PackageRevalidation + { + Key = 2, + PackageId = "Package", + PackageNormalizedVersion = "1.0.0", + Enqueued = null, + Completed = false, + }, + }); + + _galleryContext.Mock(packages: new[] + { + new Package + { + PackageRegistration = new PackageRegistration { Id = "Package" }, + NormalizedVersion = "1.0.0", + PackageStatusKey = PackageStatus.Available, + } + }); + + // Act + var next = await _target.NextOrNullAsync(); + + // Assert + Assert.Null(next); + + _telemetryService.Verify(t => t.TrackPackageRevalidationMarkedAsCompleted("Hard.Deleted.Package", "1.0.0"), Times.Once); + } + } + + public class FactsBase + { + protected readonly Mock _galleryContext; + protected readonly Mock _validationContext; + protected readonly RevalidationQueueConfiguration _config; + protected readonly Mock _telemetryService; + + protected readonly RevalidationQueue _target; + + public FactsBase() + { + _galleryContext = new Mock(); + _validationContext = new Mock(); + _telemetryService = new Mock(); + + _config = new RevalidationQueueConfiguration + { + MaximumAttempts = 5, + SleepBetweenAttempts = TimeSpan.FromSeconds(0) + }; + + _target = new RevalidationQueue( + _galleryContext.Object, + _validationContext.Object, + _config, + _telemetryService.Object, + Mock.Of>()); + } + } + } +} diff --git a/tests/NuGet.Services.Revalidate.Tests/Services/RevalidationServiceFacts.cs b/tests/NuGet.Services.Revalidate.Tests/Services/RevalidationServiceFacts.cs new file mode 100644 index 000000000..839fea7ff --- /dev/null +++ b/tests/NuGet.Services.Revalidate.Tests/Services/RevalidationServiceFacts.cs @@ -0,0 +1,278 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Moq; +using NuGet.Services.Validation; +using Xunit; + +namespace NuGet.Services.Revalidate.Tests.Services +{ + public class RevalidationServiceFacts + { + public class TheRunAsyncMethod : FactsBase + { + [Fact] + public async Task ReturnsUnrecoverableError() + { + /// This test is a sanity check to ensure works as expected. + SetupUnrecoverableErrorResult(); + + Assert.Equal(RevalidationResult.UnrecoverableError, await _target.StartNextRevalidationAsync()); + } + + [Fact] + public async Task OnUnrecoverableError_ShutsDown() + { + // Arrange + // Configure the job to run for a very long time. The unrecoverable error causes the job to end after its first iteration. + _config.ShutdownWaitInterval = TimeSpan.MaxValue; + + SetupUnrecoverableErrorResult(); + + // Act & Assert + await _target.RunAsync(); + + _singletonService.Verify(s => s.IsSingletonAsync(), Times.Once); + } + + [Fact] + public async Task ReturnsRetryLater() + { + /// This test is a sanity check to ensure works as expected. + SetupRetryLaterResult(); + + Assert.Equal(RevalidationResult.RetryLater, await _target.StartNextRevalidationAsync()); + } + + [Fact] + public async Task OnRetryLater_CallsThrottlerCallback() + { + // Arrange + SetupRetryLaterResult(); + + // Act & Assert + await _target.RunAsync(); + + _throttler.Verify(t => t.DelayUntilRevalidationRetryAsync(), Times.Once); + } + + [Fact] + public async Task OnRevalidationEnqueued_CallsThrottlerCallback() + { + // Arrange + Setup(next: _revalidation); + + // Act & Assert + await _target.RunAsync(); + + _throttler.Verify(t => t.DelayUntilNextRevalidationAsync(), Times.Once); + } + } + + public class TheStartNextRevalidationAsyncMethod : FactsBase + { + [Fact] + public async Task IfNotSingleton_ReturnsUnrecoverableError() + { + // Arrange + Setup(isSingleton: false); + + // Act & Assert + var result = await _target.StartNextRevalidationAsync(); + + _singletonService.Verify(s => s.IsSingletonAsync(), Times.Once); + _throttler.Verify(t => t.IncreaseCapacityAsync(), Times.Never); + _validationEnqueuer.Verify(e => e.StartValidationAsync(It.IsAny()), Times.Never); + + Assert.Equal(RevalidationResult.UnrecoverableError, result); + } + + [Fact] + public async Task IfKillswitchActive_ReturnsRetryLater() + { + // Arrange + Setup(killswitchActive: true); + + // Act & Assert + var result = await _target.StartNextRevalidationAsync(); + + _stateService.Verify(s => s.IsKillswitchActiveAsync(), Times.Once); + _throttler.Verify(t => t.IncreaseCapacityAsync(), Times.Never); + _validationEnqueuer.Verify(e => e.StartValidationAsync(It.IsAny()), Times.Never); + + Assert.Equal(RevalidationResult.RetryLater, result); + } + + [Fact] + public async Task IfThrottled_ReturnsRetryLater() + { + // Arrange + Setup(isThrottled: true); + + // Act & Assert + var result = await _target.StartNextRevalidationAsync(); + + _throttler.Verify(s => s.IsThrottledAsync(), Times.Once); + _throttler.Verify(t => t.IncreaseCapacityAsync(), Times.Never); + _validationEnqueuer.Verify(e => e.StartValidationAsync(It.IsAny()), Times.Never); + + Assert.Equal(RevalidationResult.RetryLater, result); + } + + [Fact] + public async Task IfUnhealthy_ReturnsRetryLater() + { + // Arrange + Setup(isHealthy: false); + + // Act & Assert + var result = await _target.StartNextRevalidationAsync(); + + _throttler.Verify(t => t.IncreaseCapacityAsync(), Times.Never); + _healthService.Verify(h => h.IsHealthyAsync(), Times.Once); + _validationEnqueuer.Verify(e => e.StartValidationAsync(It.IsAny()), Times.Never); + + Assert.Equal(RevalidationResult.RetryLater, result); + } + + [Fact] + public async Task IfRevalidationQueueEmpty_ReturnsRetryLater() + { + // Arrange + Setup(next: null); + + // Act & Assert + var result = await _target.StartNextRevalidationAsync(); + + _singletonService.Verify(s => s.IsSingletonAsync(), Times.Once); + _stateService.Verify(s => s.IsKillswitchActiveAsync(), Times.Exactly(2)); + _throttler.Verify(s => s.IsThrottledAsync(), Times.Once); + _healthService.Verify(h => h.IsHealthyAsync(), Times.Once); + + _throttler.Verify(t => t.IncreaseCapacityAsync(), Times.Once); + + _validationEnqueuer.Verify(e => e.StartValidationAsync(It.IsAny()), Times.Never); + + Assert.Equal(RevalidationResult.RetryLater, result); + } + + [Fact] + public async Task StartsNextRevalidation() + { + // Arrange + Setup(next: _revalidation); + + var order = 0; + int enqueueStep = 0; + int markStep = 0; + + _validationEnqueuer + .Setup(e => e.StartValidationAsync(It.IsAny())) + .Callback(() => enqueueStep = order++) + .Returns(Task.CompletedTask); + + _stateService + .Setup(s => s.MarkRevalidationAsEnqueuedAsync(It.IsAny())) + .Callback(() => markStep = order++) + .Returns(Task.CompletedTask); + + // Act + var result = await _target.StartNextRevalidationAsync(); + + // Assert + _singletonService.Verify(s => s.IsSingletonAsync(), Times.Once); + _stateService.Verify(s => s.IsKillswitchActiveAsync(), Times.Exactly(2)); + _throttler.Verify(s => s.IsThrottledAsync(), Times.Once); + _healthService.Verify(h => h.IsHealthyAsync(), Times.Once); + + _throttler.Verify(t => t.IncreaseCapacityAsync(), Times.Once); + + _validationEnqueuer.Verify( + e => e.StartValidationAsync(It.Is(m => + m.PackageId == _revalidation.PackageId && + m.PackageNormalizedVersion == _revalidation.PackageNormalizedVersion && + m.ValidationTrackingId == _revalidation.ValidationTrackingId.Value)), + Times.Once); + + _stateService.Verify(s => s.MarkRevalidationAsEnqueuedAsync(_revalidation), Times.Once); + _telemetryService.Verify(t => t.TrackPackageRevalidationStarted(_revalidation.PackageId, _revalidation.PackageNormalizedVersion)); + + Assert.Equal(RevalidationResult.RevalidationEnqueued, result); + Assert.Equal(2, order); + Assert.True(enqueueStep < markStep); + } + } + + public class FactsBase + { + protected readonly Mock _stateService; + protected readonly Mock _singletonService; + protected readonly Mock _throttler; + protected readonly Mock _healthService; + protected readonly Mock _revalidationQueue; + protected readonly Mock _validationEnqueuer; + protected readonly Mock _telemetryService; + + protected readonly RevalidationConfiguration _config; + protected readonly PackageRevalidation _revalidation; + + public RevalidationService _target; + + public FactsBase() + { + _stateService = new Mock(); + _singletonService = new Mock(); + _throttler = new Mock(); + _healthService = new Mock(); + _revalidationQueue = new Mock(); + _validationEnqueuer = new Mock(); + _telemetryService = new Mock(); + + _config = new RevalidationConfiguration + { + ShutdownWaitInterval = TimeSpan.MinValue, + }; + + _revalidation = new PackageRevalidation + { + PackageId = "Foo.Bar", + PackageNormalizedVersion = "1.2.3", + ValidationTrackingId = Guid.NewGuid() + }; + + _target = new RevalidationService( + _stateService.Object, + _singletonService.Object, + _throttler.Object, + _healthService.Object, + _revalidationQueue.Object, + _validationEnqueuer.Object, + _config, + _telemetryService.Object, + Mock.Of>()); + } + + protected void Setup(bool isSingleton = true, bool killswitchActive = false, bool isThrottled = false, bool isHealthy = true, PackageRevalidation next = null) + { + _singletonService.Setup(s => s.IsSingletonAsync()).ReturnsAsync(isSingleton); + _stateService.Setup(s => s.IsKillswitchActiveAsync()).ReturnsAsync(killswitchActive); + _throttler.Setup(t => t.IsThrottledAsync()).ReturnsAsync(isThrottled); + _healthService.Setup(t => t.IsHealthyAsync()).ReturnsAsync(isHealthy); + _revalidationQueue.Setup(q => q.NextOrNullAsync()).ReturnsAsync(next); + } + + protected void SetupUnrecoverableErrorResult() + { + Setup(isSingleton: false); + } + + protected void SetupRetryLaterResult() + { + Setup(killswitchActive: true, isThrottled: true, isHealthy: false); + } + } + } +}