From f72ea777c8a6008563d95e8f0d1075ccfd420ff0 Mon Sep 17 00:00:00 2001 From: Christy Henriksson Date: Wed, 25 Jul 2018 14:52:39 -0700 Subject: [PATCH 1/2] Revert "Update jobs to latest SQL AAD (#485)" (#492) (#493) This reverts commit d4e769d093e6d33eeaab3aba81b098d2b340d28f. --- src/ArchivePackages/ArchivePackages.Job.cs | 26 ++-- src/ArchivePackages/ArchivePackages.csproj | 3 + .../Gallery.CredentialExpiration.csproj | 3 + .../GalleryCredentialExpiration.cs | 12 +- src/Gallery.CredentialExpiration/Job.cs | 19 +-- .../DeleteExpiredVerificationKeysTask.cs | 5 +- .../Gallery.Maintenance.csproj | 3 + src/Gallery.Maintenance/Job.cs | 10 +- .../SqlConnectionStringBuilderExtensions.cs | 23 +++ src/NuGet.Jobs.Common/JobBase.cs | 136 ------------------ .../NuGet.Jobs.Common.csproj | 4 +- .../Job.cs | 23 +-- ...et.Services.Validation.Orchestrator.csproj | 3 + .../Job.cs | 11 +- ...NuGet.SupportRequests.Notifications.csproj | 3 + .../ScheduledTaskFactory.cs | 14 +- .../SupportRequestRepository.cs | 20 ++- .../Tasks/OnCallDailyNotificationTask.cs | 6 +- ...upportRequestsNotificationScheduledTask.cs | 11 +- .../Tasks/WeeklySummaryNotificationTask.cs | 6 +- src/Search.GenerateAuxiliaryData/Job.cs | 51 ++----- .../NestedJArrayExporter.cs | 12 +- .../RankingsExporter.cs | 6 +- .../Search.GenerateAuxiliaryData.csproj | 3 + .../SqlExporter.cs | 23 ++- .../VerifiedPackagesExporter.cs | 6 +- .../Job.cs | 21 ++- ...tats.AggregateCdnDownloadsInGallery.csproj | 3 + .../DownloadCountReport.cs | 18 ++- .../DownloadsPerToolVersionReport.cs | 18 +-- .../GalleryTotalsReport.cs | 30 ++-- .../Job.cs | 65 ++++----- .../ReportBase.cs | 11 +- .../ReportDataCollector.cs | 27 ++-- ...tats.CreateAzureCdnWarehouseReports.csproj | 3 + .../DataImporter.cs | 11 +- src/Stats.ImportAzureCdnStatistics/Job.cs | 15 +- .../Stats.ImportAzureCdnStatistics.csproj | 3 + .../Warehouse.cs | 24 ++-- .../RefreshClientDimensionJob.cs | 9 +- .../Stats.RefreshClientDimension.csproj | 3 + src/Stats.RollUpDownloadFacts/Job.cs | 9 +- .../Stats.RollUpDownloadFacts.csproj | 3 + .../UpdateLicenseReports.Job.cs | 27 ++-- .../UpdateLicenseReports.csproj | 3 + .../JsonConfigurationJob.cs | 19 ++- .../Validation.Common.Job.csproj | 3 + .../Job.cs | 2 +- ...ion.PackageSigning.ProcessSignature.csproj | 5 + .../NuGet.Services.Revalidate.Tests.csproj | 3 - .../RankingsExporterTests.cs | 10 +- .../VerifiedPackagesExporterTests.cs | 10 +- ...on.PackageSigning.ScanAndSign.Tests.csproj | 3 - 53 files changed, 348 insertions(+), 452 deletions(-) create mode 100644 src/NuGet.Jobs.Common/Extensions/SqlConnectionStringBuilderExtensions.cs diff --git a/src/ArchivePackages/ArchivePackages.Job.cs b/src/ArchivePackages/ArchivePackages.Job.cs index 4e036b203..97f7528e6 100644 --- a/src/ArchivePackages/ArchivePackages.Job.cs +++ b/src/ArchivePackages/ArchivePackages.Job.cs @@ -4,7 +4,6 @@ using System; using System.Collections.Generic; using System.ComponentModel.Design; -using System.Data.SqlClient; using System.Diagnostics.Tracing; using System.Linq; using System.Threading.Tasks; @@ -13,6 +12,8 @@ using Microsoft.WindowsAzure.Storage.Blob; using Newtonsoft.Json.Linq; using NuGet.Jobs; +using NuGet.Services.KeyVault; +using NuGet.Services.Sql; namespace ArchivePackages { @@ -52,6 +53,8 @@ public class Job : JobBase /// Blob containing the cursor data. Cursor data comprises of cursorDateTime /// public string CursorBlobName { get; set; } + + private ISqlConnectionFactory _packageDbConnectionFactory; protected CloudBlobContainer SourceContainer { get; private set; } @@ -59,13 +62,13 @@ public class Job : JobBase protected CloudBlobContainer SecondaryDestinationContainer { get; private set; } - private SqlConnectionStringBuilder PackageDatabase { get; set; } - public Job() : base(JobEventSource.Log) { } public override void Init(IServiceContainer serviceContainer, IDictionary jobArgsDictionary) { - PackageDatabase = RegisterDatabase(serviceContainer, jobArgsDictionary, JobArgumentNames.PackageDatabase); + var secretInjector = (ISecretInjector)serviceContainer.GetService(typeof(ISecretInjector)); + var packageDbConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.PackageDatabase); + _packageDbConnectionFactory = new AzureSqlConnectionFactory(packageDbConnectionString, secretInjector); Source = CloudStorageAccount.Parse( JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.Source)); @@ -89,18 +92,13 @@ public override void Init(IServiceContainer serviceContainer, IDictionary packages; - using (var connection = await OpenSqlConnectionAsync(JobArgumentNames.PackageDatabase)) + using (var connection = await _packageDbConnectionFactory.CreateAsync()) { packages = (await connection.QueryAsync(@" SELECT pr.Id, p.NormalizedVersion AS Version, p.Hash, p.LastEdited, p.Published @@ -138,8 +135,7 @@ FROM Packages p WHERE Published > @cursorDateTime OR LastEdited > @cursorDateTime", new { cursorDateTime = cursorDateTime })) .ToList(); } - - JobEventSourceLog.GatheredPackagesToArchiveFromDb(packages.Count, PackageDatabase.DataSource, PackageDatabase.InitialCatalog); + JobEventSourceLog.GatheredPackagesToArchiveFromDb(packages.Count, _packageDbConnectionFactory.DataSource, _packageDbConnectionFactory.InitialCatalog); var archiveSet = packages .AsParallel() diff --git a/src/ArchivePackages/ArchivePackages.csproj b/src/ArchivePackages/ArchivePackages.csproj index 900001677..efb41445d 100644 --- a/src/ArchivePackages/ArchivePackages.csproj +++ b/src/ArchivePackages/ArchivePackages.csproj @@ -75,6 +75,9 @@ 9.0.1 + + 2.25.0-master-30453 + 4.3.3 diff --git a/src/Gallery.CredentialExpiration/Gallery.CredentialExpiration.csproj b/src/Gallery.CredentialExpiration/Gallery.CredentialExpiration.csproj index 5fc084f77..696c6c793 100644 --- a/src/Gallery.CredentialExpiration/Gallery.CredentialExpiration.csproj +++ b/src/Gallery.CredentialExpiration/Gallery.CredentialExpiration.csproj @@ -92,6 +92,9 @@ 9.0.1 + + 2.25.0-master-30263 + 2.1.3 diff --git a/src/Gallery.CredentialExpiration/GalleryCredentialExpiration.cs b/src/Gallery.CredentialExpiration/GalleryCredentialExpiration.cs index e9db4dc9a..e1021e836 100644 --- a/src/Gallery.CredentialExpiration/GalleryCredentialExpiration.cs +++ b/src/Gallery.CredentialExpiration/GalleryCredentialExpiration.cs @@ -6,6 +6,7 @@ using System.Data.SqlClient; using System.Linq; using System.Threading.Tasks; +using NuGet.Services.Sql; using Gallery.CredentialExpiration.Models; namespace Gallery.CredentialExpiration @@ -13,15 +14,12 @@ namespace Gallery.CredentialExpiration public class GalleryCredentialExpiration : ICredentialExpirationExporter { private readonly CredentialExpirationJobMetadata _jobMetadata; + private readonly ISqlConnectionFactory _galleryDatabase; - private Func> OpenGallerySqlConnectionAsync { get; } - - public GalleryCredentialExpiration( - CredentialExpirationJobMetadata jobMetadata, - Func> openGallerySqlConnectionAsync) + public GalleryCredentialExpiration(CredentialExpirationJobMetadata jobMetadata, ISqlConnectionFactory galleryDatabase) { _jobMetadata = jobMetadata; - OpenGallerySqlConnectionAsync = openGallerySqlConnectionAsync; + _galleryDatabase = galleryDatabase; } /// @@ -53,7 +51,7 @@ public async Task> GetCredentialsAsync(TimeSpan time var minNotificationDate = ConvertToString(GetMinNotificationDate()); // Connect to database - using (var galleryConnection = await OpenGallerySqlConnectionAsync()) + using (var galleryConnection = await _galleryDatabase.CreateAsync()) { // Fetch credentials that expire in _warnDaysBeforeExpiration days // + the user's e-mail address diff --git a/src/Gallery.CredentialExpiration/Job.cs b/src/Gallery.CredentialExpiration/Job.cs index e0340eb0a..4290dba1b 100644 --- a/src/Gallery.CredentialExpiration/Job.cs +++ b/src/Gallery.CredentialExpiration/Job.cs @@ -2,6 +2,7 @@ // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.ComponentModel.Design; using System.Data.SqlClient; @@ -14,7 +15,10 @@ using Microsoft.Extensions.Logging; using Microsoft.WindowsAzure.Storage; using Newtonsoft.Json; +using Newtonsoft.Json.Linq; using NuGet.Jobs; +using NuGet.Services.KeyVault; +using NuGet.Services.Sql; using NuGet.Services.Storage; namespace Gallery.CredentialExpiration @@ -30,6 +34,8 @@ public class Job : JobBase private string _galleryBrand; private string _galleryAccountUrl; + private ISqlConnectionFactory _galleryDatabase; + private string _mailFrom; private SmtpClient _smtpClient; @@ -41,7 +47,9 @@ public override void Init(IServiceContainer serviceContainer, IDictionary OpenGallerySqlConnectionAsync() - { - return OpenSqlConnectionAsync(JobArgumentNames.GalleryDatabase); - } - public override async Task Run() { var jobRunTime = DateTimeOffset.UtcNow; // Default values var jobCursor = new JobRunTimeCursor( jobCursorTime: jobRunTime, maxProcessedCredentialsTime: jobRunTime ); - var galleryCredentialExpiration = new GalleryCredentialExpiration(new CredentialExpirationJobMetadata(jobRunTime, _warnDaysBeforeExpiration, jobCursor), OpenGallerySqlConnectionAsync); + var galleryCredentialExpiration = new GalleryCredentialExpiration(new CredentialExpirationJobMetadata(jobRunTime, _warnDaysBeforeExpiration, jobCursor), _galleryDatabase); try { @@ -86,7 +89,7 @@ public override async Task Run() // Load from cursor // Throw if the schema is not correct to ensure that not-intended emails are sent. jobCursor = JsonConvert.DeserializeObject(content, new JsonSerializerSettings() { MissingMemberHandling = MissingMemberHandling.Error }); - galleryCredentialExpiration = new GalleryCredentialExpiration(new CredentialExpirationJobMetadata(jobRunTime, _warnDaysBeforeExpiration, jobCursor), OpenGallerySqlConnectionAsync); + galleryCredentialExpiration = new GalleryCredentialExpiration(new CredentialExpirationJobMetadata(jobRunTime, _warnDaysBeforeExpiration, jobCursor), _galleryDatabase); } // Connect to database diff --git a/src/Gallery.Maintenance/DeleteExpiredVerificationKeysTask.cs b/src/Gallery.Maintenance/DeleteExpiredVerificationKeysTask.cs index 14e4d90d9..dc0d07ca5 100644 --- a/src/Gallery.Maintenance/DeleteExpiredVerificationKeysTask.cs +++ b/src/Gallery.Maintenance/DeleteExpiredVerificationKeysTask.cs @@ -9,7 +9,6 @@ using System.Threading.Tasks; using Gallery.Maintenance.Models; using Microsoft.Extensions.Logging; -using NuGet.Jobs; namespace Gallery.Maintenance { @@ -38,7 +37,7 @@ public override async Task RunAsync(Job job) { IEnumerable expiredKeys; - using (var connection = await job.OpenSqlConnectionAsync(JobArgumentNames.GalleryDatabase)) + using (var connection = await job.GalleryDatabase.CreateAsync()) { expiredKeys = await connection.QueryWithRetryAsync( SelectQuery, @@ -60,7 +59,7 @@ public override async Task RunAsync(Job job) if (expectedRowCount > 0) { - using (var connection = await job.OpenSqlConnectionAsync(JobArgumentNames.GalleryDatabase)) + using (var connection = await job.GalleryDatabase.CreateAsync()) { using (var transaction = connection.BeginTransaction()) { diff --git a/src/Gallery.Maintenance/Gallery.Maintenance.csproj b/src/Gallery.Maintenance/Gallery.Maintenance.csproj index e6a3ed08b..b11f12415 100644 --- a/src/Gallery.Maintenance/Gallery.Maintenance.csproj +++ b/src/Gallery.Maintenance/Gallery.Maintenance.csproj @@ -67,6 +67,9 @@ 9.0.1 + + 2.25.0-master-30263 + 4.3.3 diff --git a/src/Gallery.Maintenance/Job.cs b/src/Gallery.Maintenance/Job.cs index 55d64f291..488a49b92 100644 --- a/src/Gallery.Maintenance/Job.cs +++ b/src/Gallery.Maintenance/Job.cs @@ -4,11 +4,13 @@ using System; using System.Collections.Generic; using System.ComponentModel.Design; +using System.Data; using System.Linq; using System.Threading.Tasks; using Microsoft.Extensions.Logging; using NuGet.Jobs; using NuGet.Services.KeyVault; +using NuGet.Services.Sql; namespace Gallery.Maintenance { @@ -17,9 +19,15 @@ namespace Gallery.Maintenance /// public class Job : JobBase { + + public ISqlConnectionFactory GalleryDatabase { get; private set; } + public override void Init(IServiceContainer serviceContainer, IDictionary jobArgsDictionary) { - RegisterDatabase(serviceContainer, jobArgsDictionary, JobArgumentNames.GalleryDatabase); + var secretInjector = (ISecretInjector)serviceContainer.GetService(typeof(ISecretInjector)); + var databaseConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.GalleryDatabase); + + GalleryDatabase = new AzureSqlConnectionFactory(databaseConnectionString, secretInjector); } public override async Task Run() diff --git a/src/NuGet.Jobs.Common/Extensions/SqlConnectionStringBuilderExtensions.cs b/src/NuGet.Jobs.Common/Extensions/SqlConnectionStringBuilderExtensions.cs new file mode 100644 index 000000000..608481d2c --- /dev/null +++ b/src/NuGet.Jobs.Common/Extensions/SqlConnectionStringBuilderExtensions.cs @@ -0,0 +1,23 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Threading.Tasks; + +// ReSharper disable once CheckNamespace +namespace System.Data.SqlClient +{ + public static class SqlConnectionStringBuilderExtensions + { + public static Task ConnectTo(this SqlConnectionStringBuilder self) + { + return ConnectTo(self.ConnectionString); + } + + private static async Task ConnectTo(string connection) + { + var c = new SqlConnection(connection); + await c.OpenAsync().ConfigureAwait(continueOnCapturedContext: false); + return c; + } + } +} \ No newline at end of file diff --git a/src/NuGet.Jobs.Common/JobBase.cs b/src/NuGet.Jobs.Common/JobBase.cs index b16f92846..e16184854 100644 --- a/src/NuGet.Jobs.Common/JobBase.cs +++ b/src/NuGet.Jobs.Common/JobBase.cs @@ -1,18 +1,11 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. -using System; using System.Collections.Generic; using System.ComponentModel.Design; -using System.Data.SqlClient; using System.Diagnostics.Tracing; using System.Threading.Tasks; -using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; -using NuGet.Jobs.Configuration; -using NuGet.Services.KeyVault; -using NuGet.Services.Sql; namespace NuGet.Jobs { @@ -20,8 +13,6 @@ public abstract class JobBase { private readonly EventSource _jobEventSource; - private Dictionary _sqlConnectionFactories; - protected JobBase() : this(null) { @@ -31,7 +22,6 @@ protected JobBase(EventSource jobEventSource) { JobName = GetType().ToString(); _jobEventSource = jobEventSource; - _sqlConnectionFactories = new Dictionary(); } public string JobName { get; private set; } @@ -46,132 +36,6 @@ public void SetLogger(ILoggerFactory loggerFactory, ILogger logger) Logger = logger; } - /// - /// Test connection early to fail fast, and log connection diagnostics. - /// - private async Task TestConnection(ISqlConnectionFactory connectionFactory) - { - try - { - using (var connection = await connectionFactory.OpenAsync()) - using (var cmd = new SqlCommand("SELECT CONCAT(CURRENT_USER, '/', SYSTEM_USER)", connection)) - { - var result = cmd.ExecuteScalar(); - var user = result.ToString(); - Logger.LogInformation("Connected to database {DataSource}/{InitialCatalog} as {User}", - connectionFactory.DataSource, connectionFactory.InitialCatalog, user); - } - } - catch (Exception e) - { - Logger.LogError(0, e, "Failed to connect to database {DataSource}/{InitialCatalog}", - connectionFactory.DataSource, connectionFactory.InitialCatalog); - } - } - - /// - /// Initializes an , for use by validation jobs. - /// - /// ConnectionStringBuilder, used for diagnostics. - public SqlConnectionStringBuilder RegisterDatabase(IServiceProvider serviceProvider) - where T : IDbConfiguration - { - if (serviceProvider == null) - { - throw new ArgumentNullException(nameof(serviceProvider)); - } - - var secretInjector = serviceProvider.GetRequiredService(); - var connectionString = serviceProvider.GetRequiredService>().Value.ConnectionString; - var connectionFactory = new AzureSqlConnectionFactory(connectionString, secretInjector); - - return RegisterDatabase(nameof(T), connectionString, secretInjector); - } - - /// - /// Initializes an , for use by non-validation jobs. - /// - /// ConnectionStringBuilder, used for diagnostics. - public SqlConnectionStringBuilder RegisterDatabase(IServiceContainer serviceContainer, IDictionary jobArgsDictionary, string connectionStringArgName) - { - if (serviceContainer == null) - { - throw new ArgumentNullException(nameof(serviceContainer)); - } - - if (jobArgsDictionary == null) - { - throw new ArgumentNullException(nameof(jobArgsDictionary)); - } - - if (string.IsNullOrEmpty(connectionStringArgName)) - { - throw new ArgumentException("Argument cannot be null or empty.", nameof(connectionStringArgName)); - } - - var secretInjector = (ISecretInjector)serviceContainer.GetService(typeof(ISecretInjector)); - var connectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, connectionStringArgName); - - return RegisterDatabase(connectionStringArgName, connectionString, secretInjector); - } - - /// - /// Register a job database at initialization time. Each call should overwrite any existing - /// registration because calls on every iteration. - /// - /// ConnectionStringBuilder, used for diagnostics. - private SqlConnectionStringBuilder RegisterDatabase(string name, string connectionString, ISecretInjector secretInjector) - { - var connectionFactory = new AzureSqlConnectionFactory(connectionString, secretInjector, Logger); - _sqlConnectionFactories[name] = connectionFactory; - - Task.Run(() => TestConnection(connectionFactory)).Wait(); - - return connectionFactory.SqlConnectionStringBuilder; - } - - /// - /// Create a SqlConnection, for use by validation jobs. - /// - public Task CreateSqlConnectionAsync() - where T : IDbConfiguration - { - var name = nameof(T); - if (!_sqlConnectionFactories.ContainsKey(name)) - { - throw new InvalidOperationException($"Database {name} has not been registered."); - } - - return _sqlConnectionFactories[name].CreateAsync(); - } - - /// - /// Synchronous creation of a SqlConnection, for use by validation jobs. - /// - public SqlConnection CreateSqlConnection() - where T : IDbConfiguration - { - return Task.Run(() => CreateSqlConnectionAsync()).Result; - } - - /// - /// Creates and opens a SqlConnection, for use by non-validation jobs. - /// - public Task OpenSqlConnectionAsync(string connectionStringArgName) - { - if (string.IsNullOrEmpty(connectionStringArgName)) - { - throw new ArgumentException("Argument cannot be null or empty.", nameof(connectionStringArgName)); - } - - if (!_sqlConnectionFactories.ContainsKey(connectionStringArgName)) - { - throw new InvalidOperationException($"Database {connectionStringArgName} has not been registered."); - } - - return _sqlConnectionFactories[connectionStringArgName].OpenAsync(); - } - public abstract void Init(IServiceContainer serviceContainer, IDictionary jobArgsDictionary); public abstract Task Run(); diff --git a/src/NuGet.Jobs.Common/NuGet.Jobs.Common.csproj b/src/NuGet.Jobs.Common/NuGet.Jobs.Common.csproj index 1c1527895..2dd0251bc 100644 --- a/src/NuGet.Jobs.Common/NuGet.Jobs.Common.csproj +++ b/src/NuGet.Jobs.Common/NuGet.Jobs.Common.csproj @@ -49,6 +49,7 @@ + @@ -81,9 +82,6 @@ 2.27.0 - - 2.27.0 - 4.3.3 diff --git a/src/NuGet.Services.Validation.Orchestrator/Job.cs b/src/NuGet.Services.Validation.Orchestrator/Job.cs index b3c5638c3..8e6c9f831 100644 --- a/src/NuGet.Services.Validation.Orchestrator/Job.cs +++ b/src/NuGet.Services.Validation.Orchestrator/Job.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Generic; using System.ComponentModel.Design; +using System.Data.Common; using System.Net; using System.Net.Http; using System.Reflection; @@ -30,6 +31,7 @@ using NuGet.Services.KeyVault; using NuGet.Services.Logging; using NuGet.Services.ServiceBus; +using NuGet.Services.Sql; using NuGet.Services.Validation.Orchestrator.PackageSigning.ScanAndSign; using NuGet.Services.Validation.Orchestrator.Telemetry; using NuGet.Services.Validation.PackageSigning.ProcessSignature; @@ -84,12 +86,6 @@ public override void Init(IServiceContainer serviceContainer, IDictionary(_serviceProvider); - RegisterDatabase(_serviceProvider); - } ConfigurationValidated = false; } @@ -162,6 +158,15 @@ private void ConfigureLibraries(IServiceCollection services) services.AddLogging(); } + private DbConnection CreateDbConnection(IServiceProvider serviceProvider) where T : IDbConfiguration + { + var connectionString = serviceProvider.GetRequiredService>().Value.ConnectionString; + var connectionFactory = new AzureSqlConnectionFactory(connectionString, + serviceProvider.GetRequiredService()); + + return Task.Run(() => connectionFactory.CreateAsync()).Result; + } + private void ConfigureJobServices(IServiceCollection services, IConfigurationRoot configurationRoot) { services.Configure(configurationRoot.GetSection(ConfigurationSectionName)); @@ -179,15 +184,15 @@ private void ConfigureJobServices(IServiceCollection services, IConfigurationRoo services.AddTransient(); services.AddTransient(); - + services.AddScoped(serviceProvider => new NuGetGallery.EntitiesContext( - CreateSqlConnection(), + CreateDbConnection(serviceProvider), readOnly: false) ); services.AddScoped(serviceProvider => new ValidationEntitiesContext( - CreateSqlConnection())); + CreateDbConnection(serviceProvider))); services.AddScoped(serviceProvider => serviceProvider.GetRequiredService()); diff --git a/src/NuGet.Services.Validation.Orchestrator/NuGet.Services.Validation.Orchestrator.csproj b/src/NuGet.Services.Validation.Orchestrator/NuGet.Services.Validation.Orchestrator.csproj index e7ba23387..ebf1fb92c 100644 --- a/src/NuGet.Services.Validation.Orchestrator/NuGet.Services.Validation.Orchestrator.csproj +++ b/src/NuGet.Services.Validation.Orchestrator/NuGet.Services.Validation.Orchestrator.csproj @@ -123,6 +123,9 @@ 1.2.0 + + 2.26.0-master-34394 + 2.26.0-master-34394 diff --git a/src/NuGet.SupportRequests.Notifications/Job.cs b/src/NuGet.SupportRequests.Notifications/Job.cs index 8f3292405..99de91ce9 100644 --- a/src/NuGet.SupportRequests.Notifications/Job.cs +++ b/src/NuGet.SupportRequests.Notifications/Job.cs @@ -4,7 +4,6 @@ using System; using System.Collections.Generic; using System.ComponentModel.Design; -using System.Data.SqlClient; using System.Threading.Tasks; using NuGet.Jobs; @@ -25,19 +24,11 @@ public override void Init(IServiceContainer serviceContainer, IDictionary OpenSupportSqlConnectionAsync() - { - return OpenSqlConnectionAsync(JobArgumentNames.SourceDatabase); } public override async Task Run() { - var scheduledTask = ScheduledTaskFactory.Create(_serviceContainer, _jobArgsDictionary, - OpenSupportSqlConnectionAsync, LoggerFactory); + var scheduledTask = ScheduledTaskFactory.Create(_serviceContainer, _jobArgsDictionary, LoggerFactory); await scheduledTask.RunAsync(); } diff --git a/src/NuGet.SupportRequests.Notifications/NuGet.SupportRequests.Notifications.csproj b/src/NuGet.SupportRequests.Notifications/NuGet.SupportRequests.Notifications.csproj index 44a43935d..a72a38207 100644 --- a/src/NuGet.SupportRequests.Notifications/NuGet.SupportRequests.Notifications.csproj +++ b/src/NuGet.SupportRequests.Notifications/NuGet.SupportRequests.Notifications.csproj @@ -106,6 +106,9 @@ 9.0.1 + + 2.25.0-master-30453 + diff --git a/src/NuGet.SupportRequests.Notifications/ScheduledTaskFactory.cs b/src/NuGet.SupportRequests.Notifications/ScheduledTaskFactory.cs index 42834faf2..67586efb9 100644 --- a/src/NuGet.SupportRequests.Notifications/ScheduledTaskFactory.cs +++ b/src/NuGet.SupportRequests.Notifications/ScheduledTaskFactory.cs @@ -4,8 +4,6 @@ using System; using System.Collections.Generic; using System.ComponentModel.Design; -using System.Data.SqlClient; -using System.Threading.Tasks; using Microsoft.Extensions.Logging; namespace NuGet.SupportRequests.Notifications @@ -14,11 +12,7 @@ internal class ScheduledTaskFactory { private const string _tasksNamespace = "NuGet.SupportRequests.Notifications.Tasks"; - public static IScheduledTask Create( - IServiceContainer serviceContainer, - IDictionary jobArgsDictionary, - Func> openSupportSqlConnectionAsync, - ILoggerFactory loggerFactory) + public static IScheduledTask Create(IServiceContainer serviceContainer, IDictionary jobArgsDictionary, ILoggerFactory loggerFactory) { if (jobArgsDictionary == null) { @@ -31,8 +25,7 @@ public static IScheduledTask Create( } var scheduledTaskName = jobArgsDictionary[JobArgumentNames.ScheduledTask]; - var scheduledTask = GetTaskOfType(scheduledTaskName, serviceContainer, jobArgsDictionary, - openSupportSqlConnectionAsync, loggerFactory); + var scheduledTask = GetTaskOfType(scheduledTaskName, serviceContainer, jobArgsDictionary, loggerFactory); return scheduledTask; } @@ -41,7 +34,6 @@ private static IScheduledTask GetTaskOfType( string taskName, IServiceContainer serviceContainer, IDictionary jobArgsDictionary, - Func> openSupportSqlConnectionAsync, ILoggerFactory loggerFactory) { if (string.IsNullOrEmpty(taskName)) @@ -59,7 +51,7 @@ private static IScheduledTask GetTaskOfType( IScheduledTask scheduledTask; if (scheduledTaskType != null && typeof(IScheduledTask).IsAssignableFrom(scheduledTaskType)) { - var args = new object[] { serviceContainer, jobArgsDictionary, openSupportSqlConnectionAsync, loggerFactory }; + var args = new object[] { serviceContainer, jobArgsDictionary, loggerFactory }; scheduledTask = (IScheduledTask)Activator.CreateInstance(scheduledTaskType, args); } else diff --git a/src/NuGet.SupportRequests.Notifications/SupportRequestRepository.cs b/src/NuGet.SupportRequests.Notifications/SupportRequestRepository.cs index d4c3f8f20..1dcea245b 100644 --- a/src/NuGet.SupportRequests.Notifications/SupportRequestRepository.cs +++ b/src/NuGet.SupportRequests.Notifications/SupportRequestRepository.cs @@ -8,6 +8,7 @@ using System.Linq; using System.Threading.Tasks; using Microsoft.Extensions.Logging; +using NuGet.Services.Sql; using NuGet.SupportRequests.Notifications.Models; namespace NuGet.SupportRequests.Notifications @@ -19,24 +20,29 @@ internal class SupportRequestRepository private const string _parameterNamePagerDutyUsername = "pagerDutyUserName"; private readonly DateTime _defaultSqlDateTime = new DateTime(1900, 1, 1, 0, 0, 0, DateTimeKind.Utc); private readonly ILogger _logger; - private readonly Func> _openSupportSqlConnectionAsync; + private readonly ISqlConnectionFactory _supportDbConnectionFactory; public SupportRequestRepository( - Func> openSupportSqlConnectionAsync, - ILoggerFactory loggerFactory) + ILoggerFactory loggerFactory, + ISqlConnectionFactory supportDbConnectionFactory) { if (loggerFactory == null) { throw new ArgumentNullException(nameof(loggerFactory)); } + if (supportDbConnectionFactory == null) + { + throw new ArgumentNullException(nameof(supportDbConnectionFactory)); + } + _logger = loggerFactory.CreateLogger(); - _openSupportSqlConnectionAsync = openSupportSqlConnectionAsync; + _supportDbConnectionFactory = supportDbConnectionFactory; } - internal async Task OpenSupportSqlConnectionAsync() + internal async Task OpenConnectionAsync() { - var connection = await _openSupportSqlConnectionAsync(); + var connection = await _supportDbConnectionFactory.CreateAsync(); connection.InfoMessage += OnSqlConnectionInfoMessage; return connection; @@ -175,7 +181,7 @@ private async Task EnsureConnectionOpenAsync(SqlConnection connec { if (connection == null) { - connection = await OpenSupportSqlConnectionAsync(); + connection = await OpenConnectionAsync(); } else if (connection.State != ConnectionState.Open) { diff --git a/src/NuGet.SupportRequests.Notifications/Tasks/OnCallDailyNotificationTask.cs b/src/NuGet.SupportRequests.Notifications/Tasks/OnCallDailyNotificationTask.cs index 82d9b09a1..35892f72d 100644 --- a/src/NuGet.SupportRequests.Notifications/Tasks/OnCallDailyNotificationTask.cs +++ b/src/NuGet.SupportRequests.Notifications/Tasks/OnCallDailyNotificationTask.cs @@ -4,7 +4,6 @@ using System; using System.Collections.Generic; using System.ComponentModel.Design; -using System.Data.SqlClient; using System.Linq; using System.Text; using System.Threading.Tasks; @@ -26,9 +25,8 @@ internal class OnCallDailyNotificationTask public OnCallDailyNotificationTask( IServiceContainer serviceContainer, IDictionary jobArgsDictionary, - Func> openSupportSqlConnectionAsync, ILoggerFactory loggerFactory) - : base(serviceContainer, jobArgsDictionary, openSupportSqlConnectionAsync, loggerFactory) + : base(serviceContainer, jobArgsDictionary, loggerFactory) { var pagerDutyConfiguration = new PagerDutyConfiguration( jobArgsDictionary[_argumentNamePagerDutyAccountName], @@ -46,7 +44,7 @@ protected override async Task BuildNotification( var targetEmailAddress = string.Format(_targetEmailAddressFormat, onCallAlias); List unresolvedIssues; - using (var connection = await supportRequestRepository.OpenSupportSqlConnectionAsync()) + using (var connection = await supportRequestRepository.OpenConnectionAsync()) { unresolvedIssues = await supportRequestRepository.GetUnresolvedIssues(connection, onCallAlias); } diff --git a/src/NuGet.SupportRequests.Notifications/Tasks/SupportRequestsNotificationScheduledTask.cs b/src/NuGet.SupportRequests.Notifications/Tasks/SupportRequestsNotificationScheduledTask.cs index 51ffd7d5d..6976ed0bb 100644 --- a/src/NuGet.SupportRequests.Notifications/Tasks/SupportRequestsNotificationScheduledTask.cs +++ b/src/NuGet.SupportRequests.Notifications/Tasks/SupportRequestsNotificationScheduledTask.cs @@ -4,9 +4,10 @@ using System; using System.Collections.Generic; using System.ComponentModel.Design; -using System.Data.SqlClient; using System.Threading.Tasks; using Microsoft.Extensions.Logging; +using NuGet.Services.KeyVault; +using NuGet.Services.Sql; using NuGet.SupportRequests.Notifications.Notifications; using NuGet.SupportRequests.Notifications.Services; using NuGet.SupportRequests.Notifications.Templates; @@ -23,7 +24,6 @@ internal abstract class SupportRequestsNotificationScheduledTask protected SupportRequestsNotificationScheduledTask( IServiceContainer serviceContainer, IDictionary jobArgsDictionary, - Func> openSupportSqlConnectionAsync, ILoggerFactory loggerFactory) { if (jobArgsDictionary == null) @@ -38,12 +38,15 @@ protected SupportRequestsNotificationScheduledTask( var smtpUri = jobArgsDictionary[JobArgumentNames.SmtpUri]; _messagingService = new MessagingService(loggerFactory, smtpUri); + + var secretInjector = (ISecretInjector)serviceContainer.GetService(typeof(ISecretInjector)); + var supportDbConnectionString = jobArgsDictionary[JobArgumentNames.SourceDatabase]; + var supportDbConnectionFactory = new AzureSqlConnectionFactory(supportDbConnectionString, secretInjector); - _supportRequestRepository = new SupportRequestRepository(openSupportSqlConnectionAsync, loggerFactory); + _supportRequestRepository = new SupportRequestRepository(loggerFactory, supportDbConnectionFactory); } protected abstract Task BuildNotification(SupportRequestRepository supportRequestRepository, DateTime referenceTime); - protected abstract string BuildNotificationBody(string template, TNotification notification); public async Task RunAsync() diff --git a/src/NuGet.SupportRequests.Notifications/Tasks/WeeklySummaryNotificationTask.cs b/src/NuGet.SupportRequests.Notifications/Tasks/WeeklySummaryNotificationTask.cs index efb3c9518..71a0f9ccf 100644 --- a/src/NuGet.SupportRequests.Notifications/Tasks/WeeklySummaryNotificationTask.cs +++ b/src/NuGet.SupportRequests.Notifications/Tasks/WeeklySummaryNotificationTask.cs @@ -4,7 +4,6 @@ using System; using System.Collections.Generic; using System.ComponentModel.Design; -using System.Data.SqlClient; using System.Linq; using System.Text; using System.Threading.Tasks; @@ -25,9 +24,8 @@ internal class WeeklySummaryNotificationTask public WeeklySummaryNotificationTask( IServiceContainer serviceContainer, IDictionary jobArgsDictionary, - Func> openSupportSqlConnectionAsync, ILoggerFactory loggerFactory) - : base(serviceContainer, jobArgsDictionary, openSupportSqlConnectionAsync, loggerFactory) + : base(serviceContainer, jobArgsDictionary, loggerFactory) { _targetEmailAddress = jobArgsDictionary[_argumentNameTargetEmailAddress]; } @@ -44,7 +42,7 @@ protected override async Task BuildNotification( var startDateUtcLastWeek = referenceTime.AddDays(-7); var startDateUtcPriorWeek = referenceTime.AddDays(-14); - using (var connection = await supportRequestRepository.OpenSupportSqlConnectionAsync()) + using (var connection = await supportRequestRepository.OpenConnectionAsync()) { unresolvedIssues = await supportRequestRepository.GetUnresolvedIssues(connection); diff --git a/src/Search.GenerateAuxiliaryData/Job.cs b/src/Search.GenerateAuxiliaryData/Job.cs index 84ed5b8b4..4d0971f80 100644 --- a/src/Search.GenerateAuxiliaryData/Job.cs +++ b/src/Search.GenerateAuxiliaryData/Job.cs @@ -11,10 +11,12 @@ using Microsoft.WindowsAzure.Storage; using Microsoft.WindowsAzure.Storage.Blob; using NuGet.Jobs; +using NuGet.Services.KeyVault; +using NuGet.Services.Sql; namespace Search.GenerateAuxiliaryData { - public class Job + internal class Job : JobBase { private const string DefaultContainerName = "ng-search-data"; @@ -43,8 +45,13 @@ public class Job public override void Init(IServiceContainer serviceContainer, IDictionary jobArgsDictionary) { - RegisterDatabase(serviceContainer, jobArgsDictionary, JobArgumentNames.PackageDatabase); - RegisterDatabase(serviceContainer, jobArgsDictionary, JobArgumentNames.StatisticsDatabase); + var secretInjector = (ISecretInjector)serviceContainer.GetService(typeof(ISecretInjector)); + + var packageDbConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.PackageDatabase); + var packageDbConnectionFactory = new AzureSqlConnectionFactory(packageDbConnectionString, secretInjector); + + var statisticsDbConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatisticsDatabase); + var statisticsDbConnectionFactory = new AzureSqlConnectionFactory(statisticsDbConnectionString, secretInjector); var statisticsStorageAccount = CloudStorageAccount.Parse( JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.AzureCdnCloudStorageAccount)); @@ -62,42 +69,14 @@ public override void Init(IServiceContainer serviceContainer, IDictionary { - new VerifiedPackagesExporter( - OpenGallerySqlConnectionAsync, - LoggerFactory.CreateLogger(), - _destContainer, ScriptVerifiedPackages, OutputNameVerifiedPackages), - - new NestedJArrayExporter( - OpenGallerySqlConnectionAsync, - LoggerFactory.CreateLogger(), - _destContainer, ScriptCuratedFeed, OutputNameCuratedFeed, Col0CuratedFeed, Col1CuratedFeed), - - new NestedJArrayExporter( - OpenGallerySqlConnectionAsync, - LoggerFactory.CreateLogger(), - _destContainer, ScriptOwners, OutputNameOwners, Col0Owners, Col1Owners), - - new RankingsExporter( - OpenStatisticsSqlConnectionAsync, - LoggerFactory.CreateLogger(), - _destContainer, ScriptRankingsTotal, OutputNameRankings), - - new BlobStorageExporter( - LoggerFactory.CreateLogger(), - _statisticsContainer, StatisticsReportName, _destContainer, StatisticsReportName) + new VerifiedPackagesExporter(LoggerFactory.CreateLogger(), packageDbConnectionFactory, _destContainer, ScriptVerifiedPackages, OutputNameVerifiedPackages), + new NestedJArrayExporter(LoggerFactory.CreateLogger(), packageDbConnectionFactory, _destContainer, ScriptCuratedFeed, OutputNameCuratedFeed, Col0CuratedFeed, Col1CuratedFeed), + new NestedJArrayExporter(LoggerFactory.CreateLogger(), packageDbConnectionFactory, _destContainer, ScriptOwners, OutputNameOwners, Col0Owners, Col1Owners), + new RankingsExporter(LoggerFactory.CreateLogger(), statisticsDbConnectionFactory, _destContainer, ScriptRankingsTotal, OutputNameRankings), + new BlobStorageExporter(LoggerFactory.CreateLogger(), _statisticsContainer, StatisticsReportName, _destContainer, StatisticsReportName) }; } - public Task OpenGallerySqlConnectionAsync() - { - return OpenSqlConnectionAsync(JobArgumentNames.PackageDatabase); - } - - public Task OpenStatisticsSqlConnectionAsync() - { - return OpenSqlConnectionAsync(JobArgumentNames.StatisticsDatabase); - } - public override async Task Run() { var failedExporters = new List(); diff --git a/src/Search.GenerateAuxiliaryData/NestedJArrayExporter.cs b/src/Search.GenerateAuxiliaryData/NestedJArrayExporter.cs index ff4cd505c..83824718a 100644 --- a/src/Search.GenerateAuxiliaryData/NestedJArrayExporter.cs +++ b/src/Search.GenerateAuxiliaryData/NestedJArrayExporter.cs @@ -6,10 +6,10 @@ using System.Data; using System.Data.SqlClient; using System.Linq; -using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Microsoft.WindowsAzure.Storage.Blob; using Newtonsoft.Json.Linq; +using NuGet.Services.Sql; namespace Search.GenerateAuxiliaryData { @@ -17,17 +17,11 @@ class NestedJArrayExporter : SqlExporter { public string Col0 { get; } - public string Col1 { get; } - public string SqlScript { get; } - public NestedJArrayExporter( - Func> openGallerySqlConnectionAsync, - ILogger logger, - CloudBlobContainer defaultDestinationContainer, - string defaultSqlScript, string defaultName, string defaultCol0, string defaultCol1) - : base(openGallerySqlConnectionAsync, logger, defaultDestinationContainer, defaultName) + public NestedJArrayExporter(ILogger logger, ISqlConnectionFactory connectionFactory, CloudBlobContainer defaultDestinationContainer, string defaultSqlScript, string defaultName, string defaultCol0, string defaultCol1) + : base(logger, connectionFactory, defaultDestinationContainer, defaultName) { Col0 = defaultCol0; Col1 = defaultCol1; diff --git a/src/Search.GenerateAuxiliaryData/RankingsExporter.cs b/src/Search.GenerateAuxiliaryData/RankingsExporter.cs index 14ddb8bee..520812ff7 100644 --- a/src/Search.GenerateAuxiliaryData/RankingsExporter.cs +++ b/src/Search.GenerateAuxiliaryData/RankingsExporter.cs @@ -4,10 +4,10 @@ using System; using System.Data; using System.Data.SqlClient; -using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Microsoft.WindowsAzure.Storage.Blob; using Newtonsoft.Json.Linq; +using NuGet.Services.Sql; namespace Search.GenerateAuxiliaryData { @@ -20,12 +20,12 @@ public sealed class RankingsExporter : SqlExporter private readonly string _rankingsTotalScript; public RankingsExporter( - Func> openStatisticsSqlConnectionAsync, ILogger logger, + ISqlConnectionFactory connectionFactory, CloudBlobContainer defaultDestinationContainer, string defaultRankingsScript, string defaultName) - : base(openStatisticsSqlConnectionAsync, logger, defaultDestinationContainer, defaultName) + : base(logger, connectionFactory, defaultDestinationContainer, defaultName) { _rankingsTotalScript = defaultRankingsScript; } diff --git a/src/Search.GenerateAuxiliaryData/Search.GenerateAuxiliaryData.csproj b/src/Search.GenerateAuxiliaryData/Search.GenerateAuxiliaryData.csproj index d0a100c02..e9d6dddac 100644 --- a/src/Search.GenerateAuxiliaryData/Search.GenerateAuxiliaryData.csproj +++ b/src/Search.GenerateAuxiliaryData/Search.GenerateAuxiliaryData.csproj @@ -92,6 +92,9 @@ 9.0.1 + + 2.25.0-master-30453 + 7.1.2 diff --git a/src/Search.GenerateAuxiliaryData/SqlExporter.cs b/src/Search.GenerateAuxiliaryData/SqlExporter.cs index 652b1df4f..fedaa8227 100644 --- a/src/Search.GenerateAuxiliaryData/SqlExporter.cs +++ b/src/Search.GenerateAuxiliaryData/SqlExporter.cs @@ -1,7 +1,6 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. -using System; using System.Collections.Generic; using System.Data; using System.Data.SqlClient; @@ -12,6 +11,7 @@ using Microsoft.WindowsAzure.Storage.Blob; using Newtonsoft.Json; using Newtonsoft.Json.Linq; +using NuGet.Services.Sql; namespace Search.GenerateAuxiliaryData { @@ -20,19 +20,14 @@ public abstract class SqlExporter : Exporter { private static Assembly _executingAssembly = Assembly.GetExecutingAssembly(); private static string _assemblyName = _executingAssembly.GetName().Name; + + public ISqlConnectionFactory ConnectionFactory { get; } - private Func> OpenSqlConnectionAsync { get; } - - public SqlExporter( - Func> openSqlConnectionAsync, - ILogger logger, - CloudBlobContainer defaultDestinationContainer, - string defaultName) + public SqlExporter(ILogger logger, ISqlConnectionFactory connectionFactory, CloudBlobContainer defaultDestinationContainer, string defaultName) : base(logger, defaultDestinationContainer, defaultName) { _logger = logger; - - OpenSqlConnectionAsync = openSqlConnectionAsync; + ConnectionFactory = connectionFactory; } protected static string GetEmbeddedSqlScript(string resourceName) @@ -43,12 +38,12 @@ protected static string GetEmbeddedSqlScript(string resourceName) public override async Task ExportAsync() { + _logger.LogInformation("Generating {ReportName} report from {DataSource}/{InitialCatalog}.", + _name, ConnectionFactory.DataSource, ConnectionFactory.InitialCatalog); + JContainer result; - using (var connection = await OpenSqlConnectionAsync()) + using (var connection = await ConnectionFactory.CreateAsync()) { - _logger.LogInformation("Generating {ReportName} report from {DataSource}/{InitialCatalog}.", - _name, connection.DataSource, connection.Database); - result = GetResultOfQuery(connection); } diff --git a/src/Search.GenerateAuxiliaryData/VerifiedPackagesExporter.cs b/src/Search.GenerateAuxiliaryData/VerifiedPackagesExporter.cs index 04e3bb334..20acab67b 100644 --- a/src/Search.GenerateAuxiliaryData/VerifiedPackagesExporter.cs +++ b/src/Search.GenerateAuxiliaryData/VerifiedPackagesExporter.cs @@ -4,10 +4,10 @@ using System; using System.Data; using System.Data.SqlClient; -using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Microsoft.WindowsAzure.Storage.Blob; using Newtonsoft.Json.Linq; +using NuGet.Services.Sql; namespace Search.GenerateAuxiliaryData { @@ -17,12 +17,12 @@ internal sealed class VerifiedPackagesExporter : SqlExporter private readonly string _verifiedPackagesScript; public VerifiedPackagesExporter( - Func> openGallerySqlConnectionAsync, ILogger logger, + ISqlConnectionFactory connectionFactory, CloudBlobContainer defaultDestinationContainer, string defaultVerifiedPackagesScript, string defaultName) - : base(openGallerySqlConnectionAsync, logger, defaultDestinationContainer, defaultName) + : base(logger, connectionFactory, defaultDestinationContainer, defaultName) { _verifiedPackagesScript = defaultVerifiedPackagesScript; } diff --git a/src/Stats.AggregateCdnDownloadsInGallery/Job.cs b/src/Stats.AggregateCdnDownloadsInGallery/Job.cs index ee4e750b5..976b2e208 100644 --- a/src/Stats.AggregateCdnDownloadsInGallery/Job.cs +++ b/src/Stats.AggregateCdnDownloadsInGallery/Job.cs @@ -11,6 +11,8 @@ using System.Threading.Tasks; using Microsoft.Extensions.Logging; using NuGet.Jobs; +using NuGet.Services.KeyVault; +using NuGet.Services.Sql; using IPackageIdGroup = System.Linq.IGrouping; namespace Stats.AggregateCdnDownloadsInGallery @@ -53,13 +55,20 @@ GROUP BY Stats.[PackageRegistrationKey] DROP TABLE #AggregateCdnDownloadsInGallery"; private const string _storedProcedureName = "[dbo].[SelectTotalDownloadCountsPerPackageVersion]"; + private ISqlConnectionFactory _statisticsDbConnectionFactory; + private ISqlConnectionFactory _galleryDbConnectionFactory; private int _batchSize; private int _batchSleepSeconds; public override void Init(IServiceContainer serviceContainer, IDictionary jobArgsDictionary) { - RegisterDatabase(serviceContainer, jobArgsDictionary, JobArgumentNames.StatisticsDatabase); - RegisterDatabase(serviceContainer, jobArgsDictionary, JobArgumentNames.DestinationDatabase); + var secretInjector = (ISecretInjector)serviceContainer.GetService(typeof(ISecretInjector)); + + var statisticsDbConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatisticsDatabase); + _statisticsDbConnectionFactory = new AzureSqlConnectionFactory(statisticsDbConnectionString, secretInjector); + + var galleryDbConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.DestinationDatabase); + _galleryDbConnectionFactory = new AzureSqlConnectionFactory(galleryDbConnectionString, secretInjector); _batchSize = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, JobArgumentNames.BatchSize) ?? _defaultBatchSize; _batchSleepSeconds = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, JobArgumentNames.BatchSleepSeconds) ?? _defaultBatchSleepSeconds; @@ -70,14 +79,12 @@ public override async Task Run() // Gather download counts data from statistics warehouse IReadOnlyList downloadData; Logger.LogInformation("Using batch size {BatchSize} and batch sleep seconds {BatchSleepSeconds}.", _batchSize, _batchSleepSeconds); + Logger.LogInformation("Gathering Download Counts from {DataSource}/{InitialCatalog}...", _statisticsDbConnectionFactory.DataSource, _statisticsDbConnectionFactory.InitialCatalog); var stopwatch = Stopwatch.StartNew(); - using (var statisticsDatabase = await OpenSqlConnectionAsync(JobArgumentNames.StatisticsDatabase)) + using (var statisticsDatabase = await _statisticsDbConnectionFactory.CreateAsync()) using (var statisticsDatabaseTransaction = statisticsDatabase.BeginTransaction(IsolationLevel.Snapshot)) { - Logger.LogInformation("Gathering Download Counts from {DataSource}/{InitialCatalog}...", - statisticsDatabase.DataSource, statisticsDatabase.Database); - downloadData = ( await statisticsDatabase.QueryWithRetryAsync( _storedProcedureName, @@ -99,7 +106,7 @@ await statisticsDatabase.QueryWithRetryAsync( return; } - using (var destinationDatabase = await OpenSqlConnectionAsync(JobArgumentNames.DestinationDatabase)) + using (var destinationDatabase = await _galleryDbConnectionFactory.CreateAsync()) { // Fetch package registrations so we can match package ID to package registration key. var packageRegistrationLookup = await GetPackageRegistrations(destinationDatabase); diff --git a/src/Stats.AggregateCdnDownloadsInGallery/Stats.AggregateCdnDownloadsInGallery.csproj b/src/Stats.AggregateCdnDownloadsInGallery/Stats.AggregateCdnDownloadsInGallery.csproj index 66c03503c..3d2de7505 100644 --- a/src/Stats.AggregateCdnDownloadsInGallery/Stats.AggregateCdnDownloadsInGallery.csproj +++ b/src/Stats.AggregateCdnDownloadsInGallery/Stats.AggregateCdnDownloadsInGallery.csproj @@ -75,6 +75,9 @@ 9.0.1 + + 2.25.0-master-30453 + diff --git a/src/Stats.CreateAzureCdnWarehouseReports/DownloadCountReport.cs b/src/Stats.CreateAzureCdnWarehouseReports/DownloadCountReport.cs index e7a4bad21..85ae20a62 100644 --- a/src/Stats.CreateAzureCdnWarehouseReports/DownloadCountReport.cs +++ b/src/Stats.CreateAzureCdnWarehouseReports/DownloadCountReport.cs @@ -10,6 +10,7 @@ using Microsoft.Extensions.Logging; using Newtonsoft.Json; using Newtonsoft.Json.Linq; +using NuGet.Services.Sql; using NuGet.Versioning; namespace Stats.CreateAzureCdnWarehouseReports @@ -21,28 +22,25 @@ public class DownloadCountReport private readonly TimeSpan _defaultCommandTimeout = TimeSpan.FromMinutes(30); internal const string ReportName = "downloads.v1.json"; - private Func> OpenStatisticsSqlConnectionAsync { get; } - public DownloadCountReport( - Func> openStatisticsSqlConnectionAsync, ILogger logger, - IEnumerable targets) - : base(logger, targets) + IEnumerable targets, + ISqlConnectionFactory statisticsDbConnectionFactory, + ISqlConnectionFactory galleryDbConnectionFactory) + : base(logger, targets, statisticsDbConnectionFactory, galleryDbConnectionFactory) { - OpenStatisticsSqlConnectionAsync = openStatisticsSqlConnectionAsync; } public async Task Run() { // Gather download count data from statistics warehouse IReadOnlyCollection downloadData; + _logger.LogInformation("Gathering Download Counts from {DataSource}/{InitialCatalog}...", + StatisticsDbConnectionFactory.DataSource, StatisticsDbConnectionFactory.InitialCatalog); - using (var connection = await OpenStatisticsSqlConnectionAsync()) + using (var connection = await StatisticsDbConnectionFactory.CreateAsync()) using (var transaction = connection.BeginTransaction(IsolationLevel.Snapshot)) { - _logger.LogInformation("Gathering Download Counts from {DataSource}/{InitialCatalog}...", - connection.DataSource, connection.Database); - downloadData = (await connection.QueryWithRetryAsync( _storedProcedureName, commandType: CommandType.StoredProcedure, transaction: transaction, commandTimeout: _defaultCommandTimeout)).ToList(); } diff --git a/src/Stats.CreateAzureCdnWarehouseReports/DownloadsPerToolVersionReport.cs b/src/Stats.CreateAzureCdnWarehouseReports/DownloadsPerToolVersionReport.cs index 588daf973..7411526f1 100644 --- a/src/Stats.CreateAzureCdnWarehouseReports/DownloadsPerToolVersionReport.cs +++ b/src/Stats.CreateAzureCdnWarehouseReports/DownloadsPerToolVersionReport.cs @@ -11,6 +11,7 @@ using Microsoft.WindowsAzure.Storage; using Newtonsoft.Json; using Newtonsoft.Json.Linq; +using NuGet.Services.Sql; namespace Stats.CreateAzureCdnWarehouseReports { @@ -21,14 +22,14 @@ public class DownloadsPerToolVersionReport private readonly TimeSpan _defaultCommandTimeout = TimeSpan.FromMinutes(30); internal const string ReportName = "tools.v1.json"; - private Func> OpenStatisticsSqlConnectionAsync { get; } - public DownloadsPerToolVersionReport( - Func> openStatisticsSqlConnectionAsync, ILogger logger, CloudStorageAccount cloudStorageAccount, - string statisticsContainerName) - : base(logger, new[] { new StorageContainerTarget(cloudStorageAccount, statisticsContainerName) }) + string statisticsContainerName, + ISqlConnectionFactory statisticsDbConnectionFactory, + ISqlConnectionFactory galleryDbConnectionFactory) + : base(logger, new[] { new StorageContainerTarget(cloudStorageAccount, statisticsContainerName) }, + statisticsDbConnectionFactory, galleryDbConnectionFactory) { } @@ -36,13 +37,12 @@ public async Task Run() { // Gather download count data from statistics warehouse IReadOnlyCollection data; + _logger.LogInformation("Gathering Tools Download Counts from {DataSource}/{InitialCatalog}...", + StatisticsDbConnectionFactory.DataSource, StatisticsDbConnectionFactory.InitialCatalog); - using (var connection = await OpenStatisticsSqlConnectionAsync()) + using (var connection = await StatisticsDbConnectionFactory.CreateAsync()) using (var transaction = connection.BeginTransaction(IsolationLevel.Snapshot)) { - _logger.LogInformation("Gathering Tools Download Counts from {DataSource}/{InitialCatalog}...", - connection.DataSource, connection.Database); - data = (await connection.QueryWithRetryAsync( _storedProcedureName, commandType: CommandType.StoredProcedure, transaction: transaction, commandTimeout: _defaultCommandTimeout)).ToList(); } diff --git a/src/Stats.CreateAzureCdnWarehouseReports/GalleryTotalsReport.cs b/src/Stats.CreateAzureCdnWarehouseReports/GalleryTotalsReport.cs index e32a5bd1c..cdccbb09d 100644 --- a/src/Stats.CreateAzureCdnWarehouseReports/GalleryTotalsReport.cs +++ b/src/Stats.CreateAzureCdnWarehouseReports/GalleryTotalsReport.cs @@ -9,6 +9,7 @@ using Microsoft.Extensions.Logging; using Microsoft.WindowsAzure.Storage; using Newtonsoft.Json; +using NuGet.Services.Sql; namespace Stats.CreateAzureCdnWarehouseReports { @@ -22,33 +23,27 @@ public class GalleryTotalsReport (SELECT COUNT([Key]) FROM Packages WITH (NOLOCK) WHERE Listed = 1 AND Deleted = 0) AS TotalPackages"; internal const string ReportName = "stats-totals.json"; - private Func> OpenGallerySqlConnectionAsync { get; } - - private Func> OpenStatisticsSqlConnectionAsync { get; } - public GalleryTotalsReport( - Func> openGallerySqlConnectionAsync, - Func> openStatisticsSqlConnectionAsync, ILogger logger, CloudStorageAccount cloudStorageAccount, - string statisticsContainerName) - : base(logger, new[] { new StorageContainerTarget(cloudStorageAccount, statisticsContainerName) }) + string statisticsContainerName, + ISqlConnectionFactory statisticsDbConnectionFactory, + ISqlConnectionFactory galleryDbConnectionFactory) + : base(logger, new[] { new StorageContainerTarget(cloudStorageAccount, statisticsContainerName) }, + statisticsDbConnectionFactory, galleryDbConnectionFactory) { - OpenGallerySqlConnectionAsync = openGallerySqlConnectionAsync; - OpenStatisticsSqlConnectionAsync = openStatisticsSqlConnectionAsync; } public async Task Run() { // gather package numbers from gallery database GalleryTotalsData totalsData; + _logger.LogInformation("Gathering Gallery Totals from {GalleryDataSource}/{GalleryInitialCatalog}...", + GalleryDbConnectionFactory.DataSource, GalleryDbConnectionFactory.InitialCatalog); - using (var connection = await OpenGallerySqlConnectionAsync()) + using (var connection = await GalleryDbConnectionFactory.CreateAsync()) using (var transaction = connection.BeginTransaction(IsolationLevel.Snapshot)) { - _logger.LogInformation("Gathering Gallery Totals from {GalleryDataSource}/{GalleryInitialCatalog}...", - connection.DataSource, connection.Database); - totalsData = (await connection.QueryWithRetryAsync( GalleryQuery, commandType: CommandType.Text, transaction: transaction)).First(); } @@ -57,13 +52,12 @@ public async Task Run() _logger.LogInformation("Unique packages: {UniquePackagesCount}", totalsData.UniquePackages); // gather download count data from statistics warehouse + _logger.LogInformation("Gathering Gallery Totals from {StatisticsDataSource}/{StatisticsInitialCatalog}...", + StatisticsDbConnectionFactory.DataSource, StatisticsDbConnectionFactory.InitialCatalog); - using (var connection = await OpenStatisticsSqlConnectionAsync()) + using (var connection = await StatisticsDbConnectionFactory.CreateAsync()) using (var transaction = connection.BeginTransaction(IsolationLevel.Snapshot)) { - _logger.LogInformation("Gathering Gallery Totals from {StatisticsDataSource}/{StatisticsInitialCatalog}...", - connection.DataSource, connection.Database); - totalsData.Downloads = (await connection.ExecuteScalarWithRetryAsync( WarehouseStoredProcedureName, commandType: CommandType.StoredProcedure, diff --git a/src/Stats.CreateAzureCdnWarehouseReports/Job.cs b/src/Stats.CreateAzureCdnWarehouseReports/Job.cs index 4f8f55254..611390958 100644 --- a/src/Stats.CreateAzureCdnWarehouseReports/Job.cs +++ b/src/Stats.CreateAzureCdnWarehouseReports/Job.cs @@ -4,13 +4,14 @@ using System; using System.Collections.Generic; using System.ComponentModel.Design; -using System.Data.SqlClient; using System.Linq; using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Microsoft.WindowsAzure.Storage; using Microsoft.WindowsAzure.Storage.Blob; using NuGet.Jobs; +using NuGet.Services.KeyVault; +using NuGet.Services.Sql; using Stopwatch = System.Diagnostics.Stopwatch; namespace Stats.CreateAzureCdnWarehouseReports @@ -24,13 +25,13 @@ public class Job private CloudStorageAccount _cloudStorageAccount; private CloudStorageAccount _dataStorageAccount; private string _statisticsContainerName; + private ISqlConnectionFactory _statisticsDbConnectionFactory; + private ISqlConnectionFactory _galleryDbConnectionFactory; private string _reportName; private string[] _dataContainerNames; private int _sqlCommandTimeoutSeconds = DefaultSqlCommandTimeoutSeconds; private int _perPackageReportDegreeOfParallelism = DefaultPerPackageReportDegreeOfParallelism; - private SqlConnectionStringBuilder StatisticsDatabase { get; set; } - private static readonly IDictionary _storedProcedures = new Dictionary { {ReportNames.NuGetClientVersion, "[dbo].[DownloadReportNuGetClientVersion]" }, @@ -49,11 +50,15 @@ public class Job public override void Init(IServiceContainer serviceContainer, IDictionary jobArgsDictionary) { - StatisticsDatabase = RegisterDatabase(serviceContainer, jobArgsDictionary, JobArgumentNames.StatisticsDatabase); - RegisterDatabase(serviceContainer, jobArgsDictionary, JobArgumentNames.SourceDatabase); - + var secretInjector = (ISecretInjector)serviceContainer.GetService(typeof(ISecretInjector)); _sqlCommandTimeoutSeconds = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, JobArgumentNames.CommandTimeOut) ?? DefaultSqlCommandTimeoutSeconds; + var statisticsDatabaseConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatisticsDatabase); + _statisticsDbConnectionFactory = new AzureSqlConnectionFactory(statisticsDatabaseConnectionString, secretInjector); + + var galleryDatabaseConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.SourceDatabase); + _galleryDbConnectionFactory = new AzureSqlConnectionFactory(galleryDatabaseConnectionString, secretInjector); + var cloudStorageAccountConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.AzureCdnCloudStorageAccount); var dataStorageAccountConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.DataStorageAccount); _perPackageReportDegreeOfParallelism = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, JobArgumentNames.PerPackageReportDegreeOfParallelism) ?? DefaultPerPackageReportDegreeOfParallelism; @@ -73,24 +78,12 @@ public override void Init(IServiceContainer serviceContainer, IDictionary OpenStatisticsSqlConnectionAsync() - { - return OpenSqlConnectionAsync(JobArgumentNames.StatisticsDatabase); - } - - public Task OpenGallerySqlConnectionAsync() - { - return OpenSqlConnectionAsync(JobArgumentNames.SourceDatabase); - } - public override async Task Run() { var reportGenerationTime = DateTime.UtcNow; var destinationContainer = _cloudStorageAccount.CreateCloudBlobClient().GetContainerReference(_statisticsContainerName); - Logger.LogDebug("Generating reports from {DataSource}/{InitialCatalog} and saving to {AccountName}/{Container}", - StatisticsDatabase.DataSource, StatisticsDatabase.InitialCatalog, - _cloudStorageAccount.Credentials.AccountName, destinationContainer.Name); + Logger.LogDebug("Generating reports from {DataSource}/{InitialCatalog} and saving to {AccountName}/{Container}", _statisticsDbConnectionFactory.DataSource, _statisticsDbConnectionFactory.InitialCatalog, _cloudStorageAccount.Credentials.AccountName, destinationContainer.Name); var reportBuilderLogger = LoggerFactory.CreateLogger(); var reportCollectorLogger = LoggerFactory.CreateLogger(); @@ -100,12 +93,12 @@ public override async Task Run() // generate all reports var reportGenerators = new Dictionary { - { new ReportBuilder(reportBuilderLogger, ReportNames.NuGetClientVersion), new ReportDataCollector(OpenStatisticsSqlConnectionAsync, reportCollectorLogger, _storedProcedures[ReportNames.NuGetClientVersion], _sqlCommandTimeoutSeconds) }, - { new ReportBuilder(reportBuilderLogger, ReportNames.Last6Weeks), new ReportDataCollector(OpenStatisticsSqlConnectionAsync, reportCollectorLogger, _storedProcedures[ReportNames.Last6Weeks], _sqlCommandTimeoutSeconds) }, - { new ReportBuilder(reportBuilderLogger, ReportNames.RecentCommunityPopularity), new ReportDataCollector(OpenStatisticsSqlConnectionAsync, reportCollectorLogger, _storedProcedures[ReportNames.RecentCommunityPopularity], _sqlCommandTimeoutSeconds) }, - { new ReportBuilder(reportBuilderLogger, ReportNames.RecentCommunityPopularityDetail), new ReportDataCollector(OpenStatisticsSqlConnectionAsync, reportCollectorLogger, _storedProcedures[ReportNames.RecentCommunityPopularityDetail], _sqlCommandTimeoutSeconds) }, - { new ReportBuilder(reportBuilderLogger, ReportNames.RecentPopularity), new ReportDataCollector(OpenStatisticsSqlConnectionAsync, reportCollectorLogger, _storedProcedures[ReportNames.RecentPopularity], _sqlCommandTimeoutSeconds) }, - { new ReportBuilder(reportBuilderLogger, ReportNames.RecentPopularityDetail), new ReportDataCollector(OpenStatisticsSqlConnectionAsync, reportCollectorLogger, _storedProcedures[ReportNames.RecentPopularityDetail], _sqlCommandTimeoutSeconds) } + { new ReportBuilder(reportBuilderLogger, ReportNames.NuGetClientVersion), new ReportDataCollector(reportCollectorLogger, _storedProcedures[ReportNames.NuGetClientVersion], _statisticsDbConnectionFactory, _sqlCommandTimeoutSeconds) }, + { new ReportBuilder(reportBuilderLogger, ReportNames.Last6Weeks), new ReportDataCollector(reportCollectorLogger, _storedProcedures[ReportNames.Last6Weeks], _statisticsDbConnectionFactory, _sqlCommandTimeoutSeconds) }, + { new ReportBuilder(reportBuilderLogger, ReportNames.RecentCommunityPopularity), new ReportDataCollector(reportCollectorLogger, _storedProcedures[ReportNames.RecentCommunityPopularity], _statisticsDbConnectionFactory, _sqlCommandTimeoutSeconds) }, + { new ReportBuilder(reportBuilderLogger, ReportNames.RecentCommunityPopularityDetail), new ReportDataCollector(reportCollectorLogger, _storedProcedures[ReportNames.RecentCommunityPopularityDetail], _statisticsDbConnectionFactory, _sqlCommandTimeoutSeconds) }, + { new ReportBuilder(reportBuilderLogger, ReportNames.RecentPopularity), new ReportDataCollector(reportCollectorLogger, _storedProcedures[ReportNames.RecentPopularity], _statisticsDbConnectionFactory, _sqlCommandTimeoutSeconds) }, + { new ReportBuilder(reportBuilderLogger, ReportNames.RecentPopularityDetail), new ReportDataCollector(reportCollectorLogger, _storedProcedures[ReportNames.RecentPopularityDetail], _statisticsDbConnectionFactory, _sqlCommandTimeoutSeconds) } }; foreach (var reportGenerator in reportGenerators) @@ -121,14 +114,12 @@ public override async Task Run() { // generate only the specific report var reportBuilder = new ReportBuilder(reportBuilderLogger, _reportName); - var reportDataCollector = new ReportDataCollector(OpenStatisticsSqlConnectionAsync, reportCollectorLogger, _storedProcedures[_reportName], _sqlCommandTimeoutSeconds); + var reportDataCollector = new ReportDataCollector(reportCollectorLogger, _storedProcedures[_reportName], _statisticsDbConnectionFactory, _sqlCommandTimeoutSeconds); await ProcessReport(LoggerFactory, destinationContainer, reportBuilder, reportDataCollector, reportGenerationTime); } - Logger.LogInformation("Generated reports from {DataSource}/{InitialCatalog} and saving to {AccountName}/{Container}", - StatisticsDatabase.DataSource, StatisticsDatabase.InitialCatalog, - _cloudStorageAccount.Credentials.AccountName, destinationContainer.Name); + Logger.LogInformation("Generated reports from {DataSource}/{InitialCatalog} and saving to {AccountName}/{Container}", _statisticsDbConnectionFactory.DataSource, _statisticsDbConnectionFactory.InitialCatalog, _cloudStorageAccount.Credentials.AccountName, destinationContainer.Name); // totals reports var stopwatch = Stopwatch.StartNew(); @@ -140,7 +131,7 @@ public override async Task Run() { targets.Add(new StorageContainerTarget(_dataStorageAccount, dataContainerName)); } - var downloadCountReport = new DownloadCountReport(OpenStatisticsSqlConnectionAsync, LoggerFactory.CreateLogger(), targets); + var downloadCountReport = new DownloadCountReport(LoggerFactory.CreateLogger(), targets, _statisticsDbConnectionFactory, _galleryDbConnectionFactory); await downloadCountReport.Run(); stopwatch.Stop(); @@ -149,7 +140,7 @@ public override async Task Run() stopwatch.Restart(); // build stats-totals.json - var galleryTotalsReport = new GalleryTotalsReport(OpenGallerySqlConnectionAsync, OpenStatisticsSqlConnectionAsync, LoggerFactory.CreateLogger(), _cloudStorageAccount, _statisticsContainerName); + var galleryTotalsReport = new GalleryTotalsReport(LoggerFactory.CreateLogger(), _cloudStorageAccount, _statisticsContainerName, _statisticsDbConnectionFactory, _galleryDbConnectionFactory); await galleryTotalsReport.Run(); stopwatch.Stop(); @@ -158,7 +149,7 @@ public override async Task Run() // build tools.v1.json - var toolsReport = new DownloadsPerToolVersionReport(OpenStatisticsSqlConnectionAsync, LoggerFactory.CreateLogger(), _cloudStorageAccount, _statisticsContainerName); + var toolsReport = new DownloadsPerToolVersionReport(LoggerFactory.CreateLogger(), _cloudStorageAccount, _statisticsContainerName, _statisticsDbConnectionFactory, _galleryDbConnectionFactory); await toolsReport.Run(); stopwatch.Stop(); @@ -183,14 +174,14 @@ private static async Task ProcessReport(ILoggerFactory loggerFactory, CloudBlobC private async Task RebuildPackageReports(CloudBlobContainer destinationContainer, DateTime reportGenerationTime) { - var dirtyPackageIds = await ReportDataCollector.GetDirtyPackageIds(OpenStatisticsSqlConnectionAsync, LoggerFactory.CreateLogger(), reportGenerationTime, _sqlCommandTimeoutSeconds); + var dirtyPackageIds = await ReportDataCollector.GetDirtyPackageIds(LoggerFactory.CreateLogger(), _statisticsDbConnectionFactory, reportGenerationTime, _sqlCommandTimeoutSeconds); if (!dirtyPackageIds.Any()) return; // first process the top 100 packages var top100 = dirtyPackageIds.Take(100); - var reportDataCollector = new ReportDataCollector(OpenStatisticsSqlConnectionAsync, LoggerFactory.CreateLogger(), _storedProceduresPerPackageId[ReportNames.RecentPopularityDetailByPackageId], _sqlCommandTimeoutSeconds); + var reportDataCollector = new ReportDataCollector(LoggerFactory.CreateLogger(), _storedProceduresPerPackageId[ReportNames.RecentPopularityDetailByPackageId], _statisticsDbConnectionFactory, _sqlCommandTimeoutSeconds); var top100Task = Parallel.ForEach(top100, new ParallelOptions { MaxDegreeOfParallelism = _perPackageReportDegreeOfParallelism }, dirtyPackageId => { var packageId = dirtyPackageId.PackageId.ToLowerInvariant(); @@ -218,9 +209,9 @@ private async Task RebuildPackageReports(CloudBlobContainer destinationContainer "recentpopularity/" + _recentPopularityDetailByPackageReportBaseName + dirtyPackageId.PackageId.ToLowerInvariant()), new ReportDataCollector( - OpenStatisticsSqlConnectionAsync, LoggerFactory.CreateLogger(), _storedProceduresPerPackageId[ReportNames.RecentPopularityDetailByPackageId], + _statisticsDbConnectionFactory, _sqlCommandTimeoutSeconds) } }; @@ -237,7 +228,7 @@ private async Task RebuildPackageReports(CloudBlobContainer destinationContainer if (top100Task.IsCompleted) { var runToCursor = dirtyPackageIds.First().RunToCuror; - await ReportDataCollector.UpdateDirtyPackageIdCursor(OpenStatisticsSqlConnectionAsync, runToCursor, _sqlCommandTimeoutSeconds); + await ReportDataCollector.UpdateDirtyPackageIdCursor(_statisticsDbConnectionFactory, runToCursor, _sqlCommandTimeoutSeconds); } } } @@ -245,7 +236,7 @@ private async Task RebuildPackageReports(CloudBlobContainer destinationContainer private async Task CleanInactiveRecentPopularityDetailByPackageReports(CloudBlobContainer destinationContainer, DateTime reportGenerationTime) { Logger.LogDebug("Getting list of inactive packages."); - var packageIds = await ReportDataCollector.ListInactivePackageIdReports(OpenStatisticsSqlConnectionAsync, reportGenerationTime, _sqlCommandTimeoutSeconds); + var packageIds = await ReportDataCollector.ListInactivePackageIdReports(_statisticsDbConnectionFactory, reportGenerationTime, _sqlCommandTimeoutSeconds); Logger.LogInformation("Found {InactivePackageCount} inactive packages.", packageIds.Count); // Collect the list of reports diff --git a/src/Stats.CreateAzureCdnWarehouseReports/ReportBase.cs b/src/Stats.CreateAzureCdnWarehouseReports/ReportBase.cs index 121fa1ed5..faef0cce4 100644 --- a/src/Stats.CreateAzureCdnWarehouseReports/ReportBase.cs +++ b/src/Stats.CreateAzureCdnWarehouseReports/ReportBase.cs @@ -8,6 +8,7 @@ using Microsoft.Extensions.Logging; using Microsoft.WindowsAzure.Storage.Blob; using Microsoft.WindowsAzure.Storage.RetryPolicies; +using NuGet.Services.Sql; namespace Stats.CreateAzureCdnWarehouseReports { @@ -17,12 +18,20 @@ public abstract class ReportBase protected readonly IReadOnlyCollection Targets; + protected readonly ISqlConnectionFactory StatisticsDbConnectionFactory; + + protected ISqlConnectionFactory GalleryDbConnectionFactory; + protected ReportBase( ILogger logger, - IEnumerable targets) + IEnumerable targets, + ISqlConnectionFactory statisticsDbConnectionFactory, + ISqlConnectionFactory galleryDbConnectionFactory) { _logger = logger; Targets = targets.ToList().AsReadOnly(); + StatisticsDbConnectionFactory = statisticsDbConnectionFactory; + GalleryDbConnectionFactory = galleryDbConnectionFactory; } protected async Task GetBlobContainer(StorageContainerTarget target) diff --git a/src/Stats.CreateAzureCdnWarehouseReports/ReportDataCollector.cs b/src/Stats.CreateAzureCdnWarehouseReports/ReportDataCollector.cs index 3f4b4a71d..be58465b4 100644 --- a/src/Stats.CreateAzureCdnWarehouseReports/ReportDataCollector.cs +++ b/src/Stats.CreateAzureCdnWarehouseReports/ReportDataCollector.cs @@ -8,6 +8,7 @@ using System.Diagnostics; using System.Threading.Tasks; using Microsoft.Extensions.Logging; +using NuGet.Services.Sql; namespace Stats.CreateAzureCdnWarehouseReports { @@ -15,20 +16,19 @@ internal class ReportDataCollector { private int _commandTimeoutSeconds; private readonly string _procedureName; + private readonly ISqlConnectionFactory _sourceDbConnectionFactory; private ILogger _logger; - private Func> OpenStatisticsSqlConnectionAsync { get; } - public ReportDataCollector( - Func> openStatisticsSqlConnectionAsync, ILogger logger, string procedureName, + ISqlConnectionFactory sourceDbConnectionFactory, int timeout) { - OpenStatisticsSqlConnectionAsync = openStatisticsSqlConnectionAsync; _logger = logger; _procedureName = procedureName; + _sourceDbConnectionFactory = sourceDbConnectionFactory; _commandTimeoutSeconds = timeout; } @@ -47,8 +47,8 @@ public async Task CollectAsync(DateTime reportGenerationTime, params } public static async Task> GetDirtyPackageIds( - Func> openStatisticsSqlConnectionAsync, ILogger logger, + ISqlConnectionFactory sourceDbConnectionFactory, DateTime reportGenerationTime, int commandTimeout) { @@ -57,8 +57,7 @@ public static async Task> GetDirtyPackageIds IReadOnlyCollection packageIds = new List(); // Get the data - await WithRetry(async () => packageIds = await GetDirtyPackageIdsFromWarehouse( - openStatisticsSqlConnectionAsync, reportGenerationTime, commandTimeout), logger); + await WithRetry(async () => packageIds = await GetDirtyPackageIdsFromWarehouse(sourceDbConnectionFactory, reportGenerationTime, commandTimeout), logger); logger.LogInformation("Found {DirtyPackagesCount} dirty packages to update.", packageIds.Count); @@ -66,11 +65,11 @@ await WithRetry(async () => packageIds = await GetDirtyPackageIdsFromWarehouse( } public static async Task> ListInactivePackageIdReports( - Func> openStatisticsSqlConnectionAsync, + ISqlConnectionFactory sourceDbConnectionFactory, DateTime reportGenerationTime, int commandTimeout) { - using (var connection = await openStatisticsSqlConnectionAsync()) + using (var connection = await sourceDbConnectionFactory.CreateAsync()) { var command = new SqlCommand("[dbo].[DownloadReportListInactive]", connection); command.CommandType = CommandType.StoredProcedure; @@ -123,7 +122,7 @@ private static async Task WithRetry(Func action, ILogger logger) private async Task ExecuteSql(DateTime reportGenerationTime, params Tuple[] parameters) { - using (var connection = await OpenStatisticsSqlConnectionAsync()) + using (var connection = await _sourceDbConnectionFactory.CreateAsync()) { var command = new SqlCommand(_procedureName, connection); command.CommandType = CommandType.StoredProcedure; @@ -147,11 +146,11 @@ private async Task ExecuteSql(DateTime reportGenerationTime, params T } private static async Task> GetDirtyPackageIdsFromWarehouse( - Func> openStatisticsSqlConnectionAsync, + ISqlConnectionFactory sourceDbConnectionFactory, DateTime reportGenerationTime, int commandTimeout) { - using (var connection = await openStatisticsSqlConnectionAsync()) + using (var connection = await sourceDbConnectionFactory.CreateAsync()) { var command = new SqlCommand("[dbo].[GetDirtyPackageIds]", connection); command.CommandType = CommandType.StoredProcedure; @@ -173,11 +172,11 @@ private static async Task> GetDirtyPackageId } public static async Task UpdateDirtyPackageIdCursor( - Func> openStatisticsSqlConnectionAsync, + ISqlConnectionFactory sourceDbConnectionFactory, DateTime runToCursor, int commandTimeout) { - using (var connection = await openStatisticsSqlConnectionAsync()) + using (var connection = await sourceDbConnectionFactory.CreateAsync()) { var command = new SqlCommand("[dbo].[UpdateDirtyPackageIdCursor]", connection); command.CommandType = CommandType.StoredProcedure; diff --git a/src/Stats.CreateAzureCdnWarehouseReports/Stats.CreateAzureCdnWarehouseReports.csproj b/src/Stats.CreateAzureCdnWarehouseReports/Stats.CreateAzureCdnWarehouseReports.csproj index 5181875fb..0b09db277 100644 --- a/src/Stats.CreateAzureCdnWarehouseReports/Stats.CreateAzureCdnWarehouseReports.csproj +++ b/src/Stats.CreateAzureCdnWarehouseReports/Stats.CreateAzureCdnWarehouseReports.csproj @@ -99,6 +99,9 @@ 2.25.0 + + 2.25.0-master-30453 + 4.3.0-preview1-2524 diff --git a/src/Stats.ImportAzureCdnStatistics/DataImporter.cs b/src/Stats.ImportAzureCdnStatistics/DataImporter.cs index fbe92005c..8f0a9251b 100644 --- a/src/Stats.ImportAzureCdnStatistics/DataImporter.cs +++ b/src/Stats.ImportAzureCdnStatistics/DataImporter.cs @@ -1,22 +1,21 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. -using System; using System.Data; using System.Data.SqlClient; using System.Threading.Tasks; +using NuGet.Services.Sql; namespace Stats.ImportAzureCdnStatistics { internal class DataImporter { + private readonly ISqlConnectionFactory _statisticsDbConnectionFactory; private const string _sqlSelectTop1FromTable = "SELECT TOP 1 * FROM [dbo].[{0}]"; - public Func> OpenStatisticsSqlConnectionAsync { get; } - - public DataImporter(Func> openStatisticsSqlConnectionAsync) + public DataImporter(ISqlConnectionFactory statisticsDbConnectionFactory) { - OpenStatisticsSqlConnectionAsync = openStatisticsSqlConnectionAsync; + _statisticsDbConnectionFactory = statisticsDbConnectionFactory; } public async Task GetDataTableAsync(string tableName) @@ -24,7 +23,7 @@ public async Task GetDataTableAsync(string tableName) var dataTable = new DataTable(); var query = string.Format(_sqlSelectTop1FromTable, tableName); - using (var connection = await OpenStatisticsSqlConnectionAsync()) + using (var connection = await _statisticsDbConnectionFactory.CreateAsync()) { var tableAdapter = new SqlDataAdapter(query, connection) { diff --git a/src/Stats.ImportAzureCdnStatistics/Job.cs b/src/Stats.ImportAzureCdnStatistics/Job.cs index 5f82c0762..6b0b18119 100644 --- a/src/Stats.ImportAzureCdnStatistics/Job.cs +++ b/src/Stats.ImportAzureCdnStatistics/Job.cs @@ -10,8 +10,9 @@ using Microsoft.WindowsAzure.Storage.Blob; using Microsoft.WindowsAzure.Storage.RetryPolicies; using NuGet.Jobs; +using NuGet.Services.KeyVault; +using NuGet.Services.Sql; using Stats.AzureCdnLogs.Common; -using System.Data.SqlClient; namespace Stats.ImportAzureCdnStatistics { @@ -22,13 +23,16 @@ public class Job private string _azureCdnAccountNumber; private string _cloudStorageContainerName; private AzureCdnPlatform _azureCdnPlatform; + private ISqlConnectionFactory _statisticsDbConnectionFactory; private CloudStorageAccount _cloudStorageAccount; private CloudBlobClient _cloudBlobClient; private LogFileProvider _blobLeaseManager; public override void Init(IServiceContainer serviceContainer, IDictionary jobArgsDictionary) { - RegisterDatabase(serviceContainer, jobArgsDictionary, JobArgumentNames.StatisticsDatabase); + var secretInjector = (ISecretInjector)serviceContainer.GetService(typeof(ISecretInjector)); + var statisticsDbConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatisticsDatabase); + _statisticsDbConnectionFactory = new AzureSqlConnectionFactory(statisticsDbConnectionString, secretInjector); var azureCdnPlatform = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.AzureCdnPlatform); var cloudStorageAccountConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.AzureCdnCloudStorageAccount); @@ -50,11 +54,6 @@ public override void Init(IServiceContainer serviceContainer, IDictionary OpenStatisticsSqlConnectionAsync() - { - return OpenSqlConnectionAsync(JobArgumentNames.StatisticsDatabase); - } - public override async Task Run() { // Get the target blob container (for archiving decompressed log files) @@ -66,7 +65,7 @@ public override async Task Run() await deadLetterBlobContainer.CreateIfNotExistsAsync(); // Create a parser - var warehouse = new Warehouse(OpenStatisticsSqlConnectionAsync, LoggerFactory); + var warehouse = new Warehouse(LoggerFactory, _statisticsDbConnectionFactory); var statisticsBlobContainerUtility = new StatisticsBlobContainerUtility( targetBlobContainer, deadLetterBlobContainer, diff --git a/src/Stats.ImportAzureCdnStatistics/Stats.ImportAzureCdnStatistics.csproj b/src/Stats.ImportAzureCdnStatistics/Stats.ImportAzureCdnStatistics.csproj index 47ace8b1a..6ba8e18cd 100644 --- a/src/Stats.ImportAzureCdnStatistics/Stats.ImportAzureCdnStatistics.csproj +++ b/src/Stats.ImportAzureCdnStatistics/Stats.ImportAzureCdnStatistics.csproj @@ -125,6 +125,9 @@ 2.25.0 + + 2.25.0-master-30453 + 4.3.0-preview1-2524 diff --git a/src/Stats.ImportAzureCdnStatistics/Warehouse.cs b/src/Stats.ImportAzureCdnStatistics/Warehouse.cs index e84718fc1..6e4c7bcae 100644 --- a/src/Stats.ImportAzureCdnStatistics/Warehouse.cs +++ b/src/Stats.ImportAzureCdnStatistics/Warehouse.cs @@ -8,6 +8,7 @@ using System.Linq; using System.Threading.Tasks; using Microsoft.Extensions.Logging; +using NuGet.Services.Sql; using Stats.AzureCdnLogs.Common; using Stopwatch = System.Diagnostics.Stopwatch; @@ -20,6 +21,7 @@ internal class Warehouse private const int _maxRetryCount = 3; private readonly TimeSpan _retryDelay = TimeSpan.FromSeconds(5); private readonly ILogger _logger; + private readonly ISqlConnectionFactory _statisticsDbConnectionFactory; private readonly IDictionary _cachedPackageDimensions = new Dictionary(); private readonly IList _cachedToolDimensions = new List(); private readonly IDictionary _cachedClientDimensions = new Dictionary(); @@ -29,17 +31,15 @@ internal class Warehouse private readonly IDictionary _cachedIpAddressFacts = new Dictionary(); private IReadOnlyCollection _times; - private Func> OpenStatisticsSqlConnectionAsync { get; } - - public Warehouse(Func> openStatisticsSqlConnectionAsync, ILoggerFactory loggerFactory) + public Warehouse(ILoggerFactory loggerFactory, ISqlConnectionFactory statisticsDbConnectionFactory) { if (loggerFactory == null) { throw new ArgumentNullException(nameof(loggerFactory)); } - OpenStatisticsSqlConnectionAsync = openStatisticsSqlConnectionAsync; _logger = loggerFactory.CreateLogger(); + _statisticsDbConnectionFactory = statisticsDbConnectionFactory ?? throw new ArgumentNullException(nameof(statisticsDbConnectionFactory)); } public async Task InsertDownloadFactsAsync(DataTable downloadFactsDataTable, string logFileName) @@ -47,7 +47,7 @@ public async Task InsertDownloadFactsAsync(DataTable downloadFactsDataTable, str _logger.LogDebug("Inserting into facts table..."); var stopwatch = Stopwatch.StartNew(); - using (var connection = await OpenStatisticsSqlConnectionAsync()) + using (var connection = await _statisticsDbConnectionFactory.CreateAsync()) using (var transaction = connection.BeginTransaction(IsolationLevel.Snapshot)) { try @@ -124,7 +124,7 @@ public async Task CreateAsync(IReadOnlyCollection var packages = packagesTask.Result; // create facts data rows by linking source data with dimensions - var dataImporter = new DataImporter(OpenStatisticsSqlConnectionAsync); + var dataImporter = new DataImporter(_statisticsDbConnectionFactory); var factsDataTable = await dataImporter.GetDataTableAsync("Fact_Download"); var knownOperationsAvailable = operations.Any(); @@ -245,7 +245,7 @@ public async Task CreateAsync(IReadOnlyCollection sou var ipAddresses = ipAddressesTask.Result; // create facts data rows by linking source data with dimensions - var dataImporter = new DataImporter(OpenStatisticsSqlConnectionAsync); + var dataImporter = new DataImporter(_statisticsDbConnectionFactory); var dataTable = await dataImporter.GetDataTableAsync("Fact_Dist_Download"); var knownClientsAvailable = clients.Any(); @@ -341,7 +341,7 @@ public async Task StoreLogFileAggregatesAsync(LogFileAggregates logFileAggregate { _logger.LogDebug("Storing log file aggregates..."); - using (var connection = await OpenStatisticsSqlConnectionAsync()) + using (var connection = await _statisticsDbConnectionFactory.CreateAsync()) { try { @@ -375,7 +375,7 @@ public async Task> GetAlreadyAggregatedLogFilesAsync _logger.LogDebug("Retrieving already processed log files..."); var alreadyAggregatedLogFiles = new List(); - using (var connection = await OpenStatisticsSqlConnectionAsync()) + using (var connection = await _statisticsDbConnectionFactory.CreateAsync()) { try { @@ -433,7 +433,7 @@ private async Task HasImportedStatisticsAsync(string logFileName, string c try { - using (var connection = await OpenStatisticsSqlConnectionAsync()) + using (var connection = await _statisticsDbConnectionFactory.CreateAsync()) { var command = connection.CreateCommand(); command.CommandText = commandText; @@ -474,7 +474,7 @@ private async Task> GetDimension(string dimension, stri _logger.LogDebug("Beginning to retrieve dimension '{Dimension}'.", dimension); IDictionary dimensions; - using (var connection = await OpenStatisticsSqlConnectionAsync()) + using (var connection = await _statisticsDbConnectionFactory.CreateAsync()) { dimensions = await retrieve(connection); } @@ -546,7 +546,7 @@ private async Task> GetDimension(string dimension, str _logger.LogDebug("Beginning to retrieve dimension '{Dimension}'.", dimension); IReadOnlyCollection dimensions; - using (var connection = await OpenStatisticsSqlConnectionAsync()) + using (var connection = await _statisticsDbConnectionFactory.CreateAsync()) { dimensions = await retrieve(connection); } diff --git a/src/Stats.RefreshClientDimension/RefreshClientDimensionJob.cs b/src/Stats.RefreshClientDimension/RefreshClientDimensionJob.cs index 5bc3fb1b7..a10b412c3 100644 --- a/src/Stats.RefreshClientDimension/RefreshClientDimensionJob.cs +++ b/src/Stats.RefreshClientDimension/RefreshClientDimensionJob.cs @@ -8,18 +8,23 @@ using System.Threading.Tasks; using Microsoft.Extensions.Logging; using NuGet.Jobs; +using NuGet.Services.KeyVault; +using NuGet.Services.Sql; using Stats.ImportAzureCdnStatistics; namespace Stats.RefreshClientDimension { public class RefreshClientDimensionJob : JobBase { + private static ISqlConnectionFactory _statisticsDbConnectionFactory; private static string _targetClientName; private static string _userAgentFilter; public override void Init(IServiceContainer serviceContainer, IDictionary jobArgsDictionary) { - RegisterDatabase(serviceContainer, jobArgsDictionary, JobArgumentNames.StatisticsDatabase); + var secretInjector = (ISecretInjector)serviceContainer.GetService(typeof(ISecretInjector)); + var statisticsDbConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatisticsDatabase); + _statisticsDbConnectionFactory = new AzureSqlConnectionFactory(statisticsDbConnectionString, secretInjector); _targetClientName = JobConfigurationManager.TryGetArgument(jobArgsDictionary, "TargetClientName"); _userAgentFilter = JobConfigurationManager.TryGetArgument(jobArgsDictionary, "UserAgentFilter"); @@ -27,7 +32,7 @@ public override void Init(IServiceContainer serviceContainer, IDictionary> linkedUserAgents; diff --git a/src/Stats.RefreshClientDimension/Stats.RefreshClientDimension.csproj b/src/Stats.RefreshClientDimension/Stats.RefreshClientDimension.csproj index 1de4656f1..980174366 100644 --- a/src/Stats.RefreshClientDimension/Stats.RefreshClientDimension.csproj +++ b/src/Stats.RefreshClientDimension/Stats.RefreshClientDimension.csproj @@ -83,6 +83,9 @@ 9.0.1 + + 2.25.0-master-30453 + 1.2.0 diff --git a/src/Stats.RollUpDownloadFacts/Job.cs b/src/Stats.RollUpDownloadFacts/Job.cs index 51d3dcc40..76cd61979 100644 --- a/src/Stats.RollUpDownloadFacts/Job.cs +++ b/src/Stats.RollUpDownloadFacts/Job.cs @@ -9,6 +9,8 @@ using System.Threading.Tasks; using Microsoft.Extensions.Logging; using NuGet.Jobs; +using NuGet.Services.KeyVault; +using NuGet.Services.Sql; namespace Stats.RollUpDownloadFacts { @@ -19,10 +21,13 @@ public class Job private const string _endTemplateFactDownloadDeletion = " records from [dbo].[Fact_Download]"; private const int DefaultMinAgeInDays = 43; private static int _minAgeInDays; + private static ISqlConnectionFactory _statisticsDbConnectionFactory; public override void Init(IServiceContainer serviceContainer, IDictionary jobArgsDictionary) { - RegisterDatabase(serviceContainer, jobArgsDictionary, JobArgumentNames.StatisticsDatabase); + var secretInjector = (ISecretInjector)serviceContainer.GetService(typeof(ISecretInjector)); + var statisticsDbConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatisticsDatabase); + _statisticsDbConnectionFactory = new AzureSqlConnectionFactory(statisticsDbConnectionString, secretInjector); _minAgeInDays = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, JobArgumentNames.MinAgeInDays) ?? DefaultMinAgeInDays; Logger.LogInformation("Min age in days: {MinAgeInDays}", _minAgeInDays); @@ -30,7 +35,7 @@ public override void Init(IServiceContainer serviceContainer, IDictionary 2.25.0 + + 2.25.0-master-30453 + diff --git a/src/UpdateLicenseReports/UpdateLicenseReports.Job.cs b/src/UpdateLicenseReports/UpdateLicenseReports.Job.cs index d012fc1de..0a3fb2fad 100644 --- a/src/UpdateLicenseReports/UpdateLicenseReports.Job.cs +++ b/src/UpdateLicenseReports/UpdateLicenseReports.Job.cs @@ -14,6 +14,8 @@ using Newtonsoft.Json.Linq; using Newtonsoft.Json.Schema; using NuGet.Jobs; +using NuGet.Services.KeyVault; +using NuGet.Services.Sql; namespace UpdateLicenseReports { @@ -39,6 +41,7 @@ internal class Job : JobBase private Uri _licenseReportService; private string _licenseReportUser; private string _licenseReportPassword; + private ISqlConnectionFactory _packageDbConnectionFactory; private int? _retryCount; private NetworkCredential _licenseReportCredentials; @@ -58,7 +61,9 @@ private static PackageLicenseReport CreateReport(JObject messageEvent) public override void Init(IServiceContainer serviceContainer, IDictionary jobArgsDictionary) { - RegisterDatabase(serviceContainer, jobArgsDictionary, JobArgumentNames.PackageDatabase); + var secretInjector = (ISecretInjector)serviceContainer.GetService(typeof(ISecretInjector)); + var dbConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.PackageDatabase); + _packageDbConnectionFactory = new AzureSqlConnectionFactory(dbConnectionString, secretInjector); var retryCountString = JobConfigurationManager.TryGetArgument(jobArgsDictionary, JobArgumentNames.RetryCount); if (string.IsNullOrEmpty(retryCountString)) @@ -106,12 +111,12 @@ public override async Task Run() private async Task FetchNextReportUrlAsync() { + Logger.LogInformation("Fetching next report URL from {DataSource}/{InitialCatalog}", + _packageDbConnectionFactory.DataSource, _packageDbConnectionFactory.InitialCatalog); + Uri nextLicenseReport = null; - using (var connection = await OpenSqlConnectionAsync(JobArgumentNames.PackageDatabase)) + using (var connection = await _packageDbConnectionFactory.CreateAsync()) { - Logger.LogInformation("Fetching next report URL from {DataSource}/{InitialCatalog}", - connection.DataSource, connection.Database); - var nextReportUrl = (await connection.QueryAsync( @"SELECT TOP 1 NextLicenseReport FROM GallerySettings")).SingleOrDefault(); @@ -125,12 +130,12 @@ private async Task FetchNextReportUrlAsync() } nextLicenseReport = nextLicenseReport ?? _licenseReportService; - - Logger.LogInformation("Fetched next report URL '{NextReportUrl}' from {DataSource}/{InitialCatalog}", - (nextLicenseReport == null ? string.Empty : nextLicenseReport.AbsoluteUri), - connection.DataSource, connection.Database); } + Logger.LogInformation("Fetched next report URL '{NextReportUrl}' from {DataSource}/{InitialCatalog}", + (nextLicenseReport == null ? string.Empty : nextLicenseReport.AbsoluteUri), + _packageDbConnectionFactory.DataSource, _packageDbConnectionFactory.InitialCatalog); + return nextLicenseReport; } @@ -237,7 +242,7 @@ private async Task ProcessReportsAsync(Uri nextLicenseReport) Logger.LogInformation("Storing next license report URL: {NextReportUrl}", nextLicenseReport.AbsoluteUri); // Record the next report to the database so we can check it again if we get aborted before finishing. - using (var connection = await OpenSqlConnectionAsync(JobArgumentNames.PackageDatabase)) + using (var connection = await _packageDbConnectionFactory.CreateAsync()) { await connection.QueryAsync(@" UPDATE GallerySettings @@ -269,7 +274,7 @@ UPDATE GallerySettings private async Task StoreReportAsync(PackageLicenseReport report) { - using (var connection = await OpenSqlConnectionAsync(JobArgumentNames.PackageDatabase)) + using (var connection = await _packageDbConnectionFactory.CreateAsync()) using (var command = connection.CreateCommand()) { command.CommandText = "AddPackageLicenseReport2"; diff --git a/src/UpdateLicenseReports/UpdateLicenseReports.csproj b/src/UpdateLicenseReports/UpdateLicenseReports.csproj index 6a824cd81..0d9b7210c 100644 --- a/src/UpdateLicenseReports/UpdateLicenseReports.csproj +++ b/src/UpdateLicenseReports/UpdateLicenseReports.csproj @@ -74,6 +74,9 @@ 2.0.10 + + 2.25.0-master-30453 + diff --git a/src/Validation.Common.Job/JsonConfigurationJob.cs b/src/Validation.Common.Job/JsonConfigurationJob.cs index 7cc47941f..9355f050c 100644 --- a/src/Validation.Common.Job/JsonConfigurationJob.cs +++ b/src/Validation.Common.Job/JsonConfigurationJob.cs @@ -4,10 +4,12 @@ using System; using System.Collections.Generic; using System.ComponentModel.Design; +using System.Data.Common; using System.IO; using System.Net; using System.Net.Http; using System.Reflection; +using System.Threading.Tasks; using Autofac; using Autofac.Extensions.DependencyInjection; using Microsoft.ApplicationInsights; @@ -20,6 +22,7 @@ using NuGet.Services.KeyVault; using NuGet.Services.Logging; using NuGet.Services.ServiceBus; +using NuGet.Services.Sql; using NuGet.Services.Validation; using NuGetGallery; using NuGetGallery.Diagnostics; @@ -61,9 +64,6 @@ public override void Init(IServiceContainer serviceContainer, IDictionary(_serviceProvider); - RegisterDatabase(_serviceProvider); - ServicePointManager.DefaultConnectionLimit = MaximumConnectionsPerServer; } @@ -109,6 +109,15 @@ private IServiceProvider GetServiceProvider(IConfigurationRoot configurationRoot return new AutofacServiceProvider(containerBuilder.Build()); } + protected virtual DbConnection CreateDbConnection(IServiceProvider serviceProvider) where T : IDbConfiguration + { + var connectionString = serviceProvider.GetRequiredService>().Value.ConnectionString; + var connectionFactory = new AzureSqlConnectionFactory(connectionString, + serviceProvider.GetRequiredService()); + + return Task.Run(() => connectionFactory.CreateAsync()).Result; + } + private void ConfigureDefaultJobServices(IServiceCollection services, IConfigurationRoot configurationRoot) { services.Configure(configurationRoot.GetSection(GalleryDbConfigurationSectionName)); @@ -133,12 +142,12 @@ private void ConfigureDefaultJobServices(IServiceCollection services, IConfigura services.AddScoped(p => { - return new ValidationEntitiesContext(CreateSqlConnection()); + return new ValidationEntitiesContext(CreateDbConnection(p)); }); services.AddScoped(p => { - return new EntitiesContext(CreateSqlConnection(), readOnly: true); + return new EntitiesContext(CreateDbConnection(p), readOnly: true); }); services.AddTransient(p => diff --git a/src/Validation.Common.Job/Validation.Common.Job.csproj b/src/Validation.Common.Job/Validation.Common.Job.csproj index 15b0f761e..fa1a131a8 100644 --- a/src/Validation.Common.Job/Validation.Common.Job.csproj +++ b/src/Validation.Common.Job/Validation.Common.Job.csproj @@ -102,6 +102,9 @@ 2.27.0 + + 2.27.0 + 2.27.0 diff --git a/src/Validation.PackageSigning.ProcessSignature/Job.cs b/src/Validation.PackageSigning.ProcessSignature/Job.cs index 14f9e5bdc..16997f935 100644 --- a/src/Validation.PackageSigning.ProcessSignature/Job.cs +++ b/src/Validation.PackageSigning.ProcessSignature/Job.cs @@ -35,7 +35,7 @@ protected override void ConfigureJobServices(IServiceCollection services, IConfi services.AddScoped(p => { - return new EntitiesContext(CreateSqlConnection(), readOnly: false); + return new EntitiesContext(CreateDbConnection(p), readOnly: false); }); services.Add(ServiceDescriptor.Transient(typeof(IEntityRepository<>), typeof(EntityRepository<>))); diff --git a/src/Validation.PackageSigning.ProcessSignature/Validation.PackageSigning.ProcessSignature.csproj b/src/Validation.PackageSigning.ProcessSignature/Validation.PackageSigning.ProcessSignature.csproj index 1ae9d197a..3d331e9f9 100644 --- a/src/Validation.PackageSigning.ProcessSignature/Validation.PackageSigning.ProcessSignature.csproj +++ b/src/Validation.PackageSigning.ProcessSignature/Validation.PackageSigning.ProcessSignature.csproj @@ -85,6 +85,11 @@ Validation.PackageSigning.Core + + + 2.27.0 + + ..\..\build diff --git a/tests/NuGet.Services.Revalidate.Tests/NuGet.Services.Revalidate.Tests.csproj b/tests/NuGet.Services.Revalidate.Tests/NuGet.Services.Revalidate.Tests.csproj index 172a9d5c3..a09ec1eea 100644 --- a/tests/NuGet.Services.Revalidate.Tests/NuGet.Services.Revalidate.Tests.csproj +++ b/tests/NuGet.Services.Revalidate.Tests/NuGet.Services.Revalidate.Tests.csproj @@ -82,8 +82,5 @@ Tests.ContextHelpers - - - \ No newline at end of file diff --git a/tests/Tests.Search.GenerateAuxiliaryData/RankingsExporterTests.cs b/tests/Tests.Search.GenerateAuxiliaryData/RankingsExporterTests.cs index 9e5cb51d7..9a202115a 100644 --- a/tests/Tests.Search.GenerateAuxiliaryData/RankingsExporterTests.cs +++ b/tests/Tests.Search.GenerateAuxiliaryData/RankingsExporterTests.cs @@ -3,12 +3,11 @@ using System; using System.Data; -using System.Data.SqlClient; -using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Microsoft.WindowsAzure.Storage.Blob; using Moq; using Newtonsoft.Json; +using NuGet.Services.Sql; using Search.GenerateAuxiliaryData; using Xunit; @@ -45,16 +44,11 @@ public void GetRankings_ReturnsRankings() private static RankingsExporter CreateExporter() { return new RankingsExporter( - DoNotOpenSqlConnectionAsync, new LoggerFactory().CreateLogger(), + connectionFactory: new Mock().Object, defaultDestinationContainer: new CloudBlobContainer(new Uri("https://nuget.org")), defaultRankingsScript: "b", defaultName: "c"); } - - public static Task DoNotOpenSqlConnectionAsync() - { - return Task.FromResult((SqlConnection)null); - } } } \ No newline at end of file diff --git a/tests/Tests.Search.GenerateAuxiliaryData/VerifiedPackagesExporterTests.cs b/tests/Tests.Search.GenerateAuxiliaryData/VerifiedPackagesExporterTests.cs index c31f36116..50b8aea23 100644 --- a/tests/Tests.Search.GenerateAuxiliaryData/VerifiedPackagesExporterTests.cs +++ b/tests/Tests.Search.GenerateAuxiliaryData/VerifiedPackagesExporterTests.cs @@ -3,12 +3,11 @@ using System; using System.Data; -using System.Data.SqlClient; -using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Microsoft.WindowsAzure.Storage.Blob; using Moq; using Newtonsoft.Json; +using NuGet.Services.Sql; using Search.GenerateAuxiliaryData; using Xunit; @@ -44,16 +43,11 @@ public void GetVerifiedPackagesReturnsJsonString() private static VerifiedPackagesExporter CreateExporter() { return new VerifiedPackagesExporter( - DoNotOpenSqlConnectionAsync, new LoggerFactory().CreateLogger(), + connectionFactory: new Mock().Object, defaultDestinationContainer: new CloudBlobContainer(new Uri("https://nuget.org")), defaultVerifiedPackagesScript: "b", defaultName: "c"); } - - public static Task DoNotOpenSqlConnectionAsync() - { - return Task.FromResult((SqlConnection)null); - } } } \ No newline at end of file diff --git a/tests/Validation.PackageSigning.ScanAndSign.Tests/Validation.PackageSigning.ScanAndSign.Tests.csproj b/tests/Validation.PackageSigning.ScanAndSign.Tests/Validation.PackageSigning.ScanAndSign.Tests.csproj index b0662efcb..510501a59 100644 --- a/tests/Validation.PackageSigning.ScanAndSign.Tests/Validation.PackageSigning.ScanAndSign.Tests.csproj +++ b/tests/Validation.PackageSigning.ScanAndSign.Tests/Validation.PackageSigning.ScanAndSign.Tests.csproj @@ -75,8 +75,5 @@ 2.3.1 - - - \ No newline at end of file From a843ac642ed121816ff8b6364f347e467761fa46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Sharma?= Date: Wed, 25 Jul 2018 18:58:29 -0700 Subject: [PATCH 2/2] [Repository Signing] Add option to suppress repository signature extraction (#495) --- .../ProcessSignatureConfiguration.cs | 6 ++ .../Settings/dev.json | 3 +- .../Settings/int.json | 3 +- .../Settings/prod.json | 3 +- .../SignaturePartsExtractor.cs | 10 ++++ .../SignaturePartsExtractorFacts.cs | 60 ++++++++++++++++++- .../SignatureValidatorIntegrationTests.cs | 22 +++---- 7 files changed, 93 insertions(+), 14 deletions(-) diff --git a/src/Validation.PackageSigning.ProcessSignature/ProcessSignatureConfiguration.cs b/src/Validation.PackageSigning.ProcessSignature/ProcessSignatureConfiguration.cs index 4d5bb7d5c..ab0fce92b 100644 --- a/src/Validation.PackageSigning.ProcessSignature/ProcessSignatureConfiguration.cs +++ b/src/Validation.PackageSigning.ProcessSignature/ProcessSignatureConfiguration.cs @@ -20,5 +20,11 @@ public class ProcessSignatureConfiguration /// repository signature is removed. /// public string V3ServiceIndexUrl { get; set; } + + /// + /// Whether repository signatures should be persisted to the database. Disable this if repository signing + /// is in test mode and repository signed packages are not published. + /// + public bool CommitRepositorySignatures { get; set; } } } diff --git a/src/Validation.PackageSigning.ProcessSignature/Settings/dev.json b/src/Validation.PackageSigning.ProcessSignature/Settings/dev.json index 11728b8a0..ecbe7450d 100644 --- a/src/Validation.PackageSigning.ProcessSignature/Settings/dev.json +++ b/src/Validation.PackageSigning.ProcessSignature/Settings/dev.json @@ -22,7 +22,8 @@ "AllowedRepositorySigningCertificates": [ "cf6ce6768ef858a3a667be1af8aa524d386c7f59a34542713f5dfb0d79acf3dd" ], - "V3ServiceIndexUrl": "https://apidev.nugettest.org/v3/index.json" + "V3ServiceIndexUrl": "https://apidev.nugettest.org/v3/index.json", + "CommitRepositorySignatures": false }, "PackageDownloadTimeout": "00:10:00", diff --git a/src/Validation.PackageSigning.ProcessSignature/Settings/int.json b/src/Validation.PackageSigning.ProcessSignature/Settings/int.json index 1180c0816..3dd7a70fd 100644 --- a/src/Validation.PackageSigning.ProcessSignature/Settings/int.json +++ b/src/Validation.PackageSigning.ProcessSignature/Settings/int.json @@ -22,7 +22,8 @@ "AllowedRepositorySigningCertificates": [ "cf6ce6768ef858a3a667be1af8aa524d386c7f59a34542713f5dfb0d79acf3dd" ], - "V3ServiceIndexUrl": "https://apiint.nugettest.org/v3/index.json" + "V3ServiceIndexUrl": "https://apiint.nugettest.org/v3/index.json", + "CommitRepositorySignatures": false }, "PackageDownloadTimeout": "00:10:00", diff --git a/src/Validation.PackageSigning.ProcessSignature/Settings/prod.json b/src/Validation.PackageSigning.ProcessSignature/Settings/prod.json index d195c2287..2836e7501 100644 --- a/src/Validation.PackageSigning.ProcessSignature/Settings/prod.json +++ b/src/Validation.PackageSigning.ProcessSignature/Settings/prod.json @@ -22,7 +22,8 @@ "AllowedRepositorySigningCertificates": [ "cf7ac17ad047ecd5fdc36822031b12d4ef078b6f2b4c5e6ba41f8ff2cf4bad67" ], - "V3ServiceIndexUrl": "https://api.nuget.org/v3/index.json" + "V3ServiceIndexUrl": "https://api.nuget.org/v3/index.json", + "CommitRepositorySignatures": false }, "PackageDownloadTimeout": "00:10:00", diff --git a/src/Validation.PackageSigning.ProcessSignature/SignaturePartsExtractor.cs b/src/Validation.PackageSigning.ProcessSignature/SignaturePartsExtractor.cs index 3eccf8bca..7fb6056bd 100644 --- a/src/Validation.PackageSigning.ProcessSignature/SignaturePartsExtractor.cs +++ b/src/Validation.PackageSigning.ProcessSignature/SignaturePartsExtractor.cs @@ -8,6 +8,7 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; using NuGet.Jobs.Validation.PackageSigning.Storage; using NuGet.Packaging.Signing; using NuGet.Services.Validation; @@ -18,15 +19,18 @@ public class SignaturePartsExtractor : ISignaturePartsExtractor { private readonly ICertificateStore _certificateStore; private readonly IValidationEntitiesContext _entitiesContext; + private readonly IOptionsSnapshot _configuration; private readonly ILogger _logger; public SignaturePartsExtractor( ICertificateStore certificateStore, IValidationEntitiesContext entitiesContext, + IOptionsSnapshot configuration, ILogger logger) { _certificateStore = certificateStore ?? throw new ArgumentNullException(nameof(certificateStore)); _entitiesContext = entitiesContext ?? throw new ArgumentNullException(nameof(entitiesContext)); + _configuration = configuration ?? throw new ArgumentNullException(nameof(configuration)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); } @@ -227,6 +231,12 @@ private async Task InitializePackageSignatureAndTrustedTimestampAsync( return; } + if (type == PackageSignatureType.Repository && !_configuration.Value.CommitRepositorySignatures) + { + _logger.LogInformation("Skipping initialization of repository signature due to configuration!"); + return; + } + // Initialize the package signature record. var packageSignature = await InitializePackageSignatureAsync( packageKey, diff --git a/tests/Validation.PackageSigning.ProcessSignature.Tests/SignaturePartsExtractorFacts.cs b/tests/Validation.PackageSigning.ProcessSignature.Tests/SignaturePartsExtractorFacts.cs index 0b4077061..885f83c4c 100644 --- a/tests/Validation.PackageSigning.ProcessSignature.Tests/SignaturePartsExtractorFacts.cs +++ b/tests/Validation.PackageSigning.ProcessSignature.Tests/SignaturePartsExtractorFacts.cs @@ -10,7 +10,9 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; using Moq; +using NuGet.Jobs.Validation.PackageSigning; using NuGet.Jobs.Validation.PackageSigning.ProcessSignature; using NuGet.Jobs.Validation.PackageSigning.Storage; using NuGet.Packaging.Signing; @@ -62,7 +64,7 @@ public class SignaturePartsExtractorFacts "CN=NUGET_DO_NOT_TRUST.root.test.test, OU=Test Organizational Unit Name, O=Test Organization Name, L=Redmond, S=WA, C=US", TestResources.RootThumbprint), }, - TimestampEndCertificate = new SubjectAndThumbprint( + TimestampEndCertificate = new SubjectAndThumbprint( "CN=Symantec SHA256 TimeStamping Signer - G2, OU=Symantec Trust Network, O=Symantec Corporation, C=US", TestResources.Leaf1TimestampThumbprint), TimestampParentCertificates = new[] @@ -116,6 +118,8 @@ public class ExtractAsync private readonly Mock _certificateStore; private readonly List _savedCertificates; private readonly Mock _entitiesContext; + private readonly Mock> _configAccessor; + private readonly ProcessSignatureConfiguration _config; private readonly Mock> _logger; private readonly SignaturePartsExtractor _target; @@ -152,11 +156,20 @@ public ExtractAsync() .Setup(x => x.TrustedTimestamps) .Returns(DbSetMockFactory.Create()); + _configAccessor = new Mock>(); + _config = new ProcessSignatureConfiguration + { + CommitRepositorySignatures = true + }; + + _configAccessor.Setup(a => a.Value).Returns(_config); + _logger = new Mock>(); _target = new SignaturePartsExtractor( _certificateStore.Object, _entitiesContext.Object, + _configAccessor.Object, _logger.Object); } @@ -616,6 +629,51 @@ public async Task IgnoreExtraCertificates() signature.SignedCms.Certificates.Count + signature.Timestamps.Sum(x => x.SignedCms.Certificates.Count)); } + [Fact] + public async Task IfRepositorySignatureExtractionIsDisabled_IgnoresRepositorySignatureOnRepositorySignedPackage() + { + // Arrange + var signature = await TestResources.LoadPrimarySignatureAsync(TestResources.RepoSignedPackageLeaf1); + + _entitiesContext + .Setup(x => x.PackageSignatures) + .Returns(DbSetMockFactory.Create()); + + _config.CommitRepositorySignatures = false; + + // Act + await _target.ExtractAsync(_packageKey, signature, _token); + + // Assert + Assert.Equal(0, _entitiesContext.Object.PackageSignatures.Count()); + + // The repository signature's certificate is still stored on blob storage. + VerifyStoredCertificates(Leaf1Certificates); + } + + [Fact] + public async Task IfRepositorySignatureExtractionIsDisabled_IgnoresRepositorySignatureOnRepositoryCounterSignedPackage() + { + // Arrange + var signature = await TestResources.LoadPrimarySignatureAsync(TestResources.AuthorAndRepoSignedPackageLeaf1); + + _entitiesContext + .Setup(x => x.PackageSignatures) + .Returns(DbSetMockFactory.Create()); + + _config.CommitRepositorySignatures = false; + + // Act + await _target.ExtractAsync(_packageKey, signature, _token); + + // Assert + Assert.Equal(1, _entitiesContext.Object.PackageSignatures.Count()); + Assert.Equal(PackageSignatureType.Author, _entitiesContext.Object.PackageSignatures.First().Type); + + // The repository signature's certificate is still stored on blob storage. + VerifyStoredCertificates(AuthorAndRepoSignedCertificates); + } + private void AssignIds() { var endCertificates = _entitiesContext.Object.EndCertificates.AsQueryable().ToList(); diff --git a/tests/Validation.PackageSigning.ProcessSignature.Tests/SignatureValidatorIntegrationTests.cs b/tests/Validation.PackageSigning.ProcessSignature.Tests/SignatureValidatorIntegrationTests.cs index 742f2d98c..a4d899a8a 100644 --- a/tests/Validation.PackageSigning.ProcessSignature.Tests/SignatureValidatorIntegrationTests.cs +++ b/tests/Validation.PackageSigning.ProcessSignature.Tests/SignatureValidatorIntegrationTests.cs @@ -119,9 +119,21 @@ public SignatureValidatorIntegrationTests(CertificateIntegrationTestFixture fixt _certificateStore = new Mock(); + // These dependencies are concrete. + _configuration = new ProcessSignatureConfiguration + { + AllowedRepositorySigningCertificates = new List { "fake-thumbprint" }, + V3ServiceIndexUrl = TestResources.V3ServiceIndexUrl, + CommitRepositorySignatures = true, + }; + _optionsSnapshot = new Mock>(); + _optionsSnapshot.Setup(x => x.Value).Returns(() => _configuration); + _formatValidator = new SignatureFormatValidator(_optionsSnapshot.Object); + _signaturePartsExtractor = new SignaturePartsExtractor( _certificateStore.Object, _validationEntitiesContext.Object, + _optionsSnapshot.Object, loggerFactory.CreateLogger()); _packageFileService = new Mock(); @@ -144,16 +156,6 @@ public SignatureValidatorIntegrationTests(CertificateIntegrationTestFixture fixt _corePackageService = new Mock(); - // These dependencies are concrete. - _configuration = new ProcessSignatureConfiguration - { - AllowedRepositorySigningCertificates = new List { "fake-thumbprint" }, - V3ServiceIndexUrl = TestResources.V3ServiceIndexUrl, - }; - _optionsSnapshot = new Mock>(); - _optionsSnapshot.Setup(x => x.Value).Returns(() => _configuration); - _formatValidator = new SignatureFormatValidator(_optionsSnapshot.Object); - _telemetryClient = new Mock(); _telemetryService = new TelemetryService(_telemetryClient.Object);