From c8ae1435f7982e7ba56c78de45212ddc472191cf Mon Sep 17 00:00:00 2001 From: Christy Henriksson Date: Tue, 24 Jul 2018 10:21:21 -0700 Subject: [PATCH] Update jobs to latest SQL AAD (#485) --- src/ArchivePackages/ArchivePackages.Job.cs | 26 ++-- src/ArchivePackages/ArchivePackages.csproj | 3 - .../Gallery.CredentialExpiration.csproj | 3 - .../GalleryCredentialExpiration.cs | 12 +- src/Gallery.CredentialExpiration/Job.cs | 19 ++- .../DeleteExpiredVerificationKeysTask.cs | 5 +- .../Gallery.Maintenance.csproj | 3 - src/Gallery.Maintenance/Job.cs | 10 +- .../SqlConnectionStringBuilderExtensions.cs | 23 --- src/NuGet.Jobs.Common/JobBase.cs | 136 ++++++++++++++++++ .../NuGet.Jobs.Common.csproj | 4 +- .../Job.cs | 23 ++- ...et.Services.Validation.Orchestrator.csproj | 3 - .../Job.cs | 11 +- ...NuGet.SupportRequests.Notifications.csproj | 3 - .../ScheduledTaskFactory.cs | 14 +- .../SupportRequestRepository.cs | 20 +-- .../Tasks/OnCallDailyNotificationTask.cs | 6 +- ...upportRequestsNotificationScheduledTask.cs | 11 +- .../Tasks/WeeklySummaryNotificationTask.cs | 6 +- src/Search.GenerateAuxiliaryData/Job.cs | 51 +++++-- .../NestedJArrayExporter.cs | 12 +- .../RankingsExporter.cs | 6 +- .../Search.GenerateAuxiliaryData.csproj | 3 - .../SqlExporter.cs | 23 +-- .../VerifiedPackagesExporter.cs | 6 +- .../Job.cs | 21 +-- ...tats.AggregateCdnDownloadsInGallery.csproj | 3 - .../DownloadCountReport.cs | 18 +-- .../DownloadsPerToolVersionReport.cs | 18 +-- .../GalleryTotalsReport.cs | 30 ++-- .../Job.cs | 65 +++++---- .../ReportBase.cs | 11 +- .../ReportDataCollector.cs | 27 ++-- ...tats.CreateAzureCdnWarehouseReports.csproj | 3 - .../DataImporter.cs | 11 +- src/Stats.ImportAzureCdnStatistics/Job.cs | 15 +- .../Stats.ImportAzureCdnStatistics.csproj | 3 - .../Warehouse.cs | 24 ++-- .../RefreshClientDimensionJob.cs | 9 +- .../Stats.RefreshClientDimension.csproj | 3 - src/Stats.RollUpDownloadFacts/Job.cs | 9 +- .../Stats.RollUpDownloadFacts.csproj | 3 - .../UpdateLicenseReports.Job.cs | 27 ++-- .../UpdateLicenseReports.csproj | 3 - .../JsonConfigurationJob.cs | 19 +-- .../Validation.Common.Job.csproj | 3 - .../Job.cs | 2 +- ...ion.PackageSigning.ProcessSignature.csproj | 5 - .../NuGet.Services.Revalidate.Tests.csproj | 3 + .../RankingsExporterTests.cs | 10 +- .../VerifiedPackagesExporterTests.cs | 10 +- ...on.PackageSigning.ScanAndSign.Tests.csproj | 3 + 53 files changed, 452 insertions(+), 348 deletions(-) delete mode 100644 src/NuGet.Jobs.Common/Extensions/SqlConnectionStringBuilderExtensions.cs diff --git a/src/ArchivePackages/ArchivePackages.Job.cs b/src/ArchivePackages/ArchivePackages.Job.cs index 97f7528e6..4e036b203 100644 --- a/src/ArchivePackages/ArchivePackages.Job.cs +++ b/src/ArchivePackages/ArchivePackages.Job.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Generic; using System.ComponentModel.Design; +using System.Data.SqlClient; using System.Diagnostics.Tracing; using System.Linq; using System.Threading.Tasks; @@ -12,8 +13,6 @@ using Microsoft.WindowsAzure.Storage.Blob; using Newtonsoft.Json.Linq; using NuGet.Jobs; -using NuGet.Services.KeyVault; -using NuGet.Services.Sql; namespace ArchivePackages { @@ -53,8 +52,6 @@ public class Job : JobBase /// Blob containing the cursor data. Cursor data comprises of cursorDateTime /// public string CursorBlobName { get; set; } - - private ISqlConnectionFactory _packageDbConnectionFactory; protected CloudBlobContainer SourceContainer { get; private set; } @@ -62,13 +59,13 @@ public class Job : JobBase protected CloudBlobContainer SecondaryDestinationContainer { get; private set; } + private SqlConnectionStringBuilder PackageDatabase { get; set; } + public Job() : base(JobEventSource.Log) { } public override void Init(IServiceContainer serviceContainer, IDictionary jobArgsDictionary) { - var secretInjector = (ISecretInjector)serviceContainer.GetService(typeof(ISecretInjector)); - var packageDbConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.PackageDatabase); - _packageDbConnectionFactory = new AzureSqlConnectionFactory(packageDbConnectionString, secretInjector); + PackageDatabase = RegisterDatabase(serviceContainer, jobArgsDictionary, JobArgumentNames.PackageDatabase); Source = CloudStorageAccount.Parse( JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.Source)); @@ -92,13 +89,18 @@ public override void Init(IServiceContainer serviceContainer, IDictionary packages; - using (var connection = await _packageDbConnectionFactory.CreateAsync()) + using (var connection = await OpenSqlConnectionAsync(JobArgumentNames.PackageDatabase)) { packages = (await connection.QueryAsync(@" SELECT pr.Id, p.NormalizedVersion AS Version, p.Hash, p.LastEdited, p.Published @@ -135,7 +138,8 @@ FROM Packages p WHERE Published > @cursorDateTime OR LastEdited > @cursorDateTime", new { cursorDateTime = cursorDateTime })) .ToList(); } - JobEventSourceLog.GatheredPackagesToArchiveFromDb(packages.Count, _packageDbConnectionFactory.DataSource, _packageDbConnectionFactory.InitialCatalog); + + JobEventSourceLog.GatheredPackagesToArchiveFromDb(packages.Count, PackageDatabase.DataSource, PackageDatabase.InitialCatalog); var archiveSet = packages .AsParallel() diff --git a/src/ArchivePackages/ArchivePackages.csproj b/src/ArchivePackages/ArchivePackages.csproj index efb41445d..900001677 100644 --- a/src/ArchivePackages/ArchivePackages.csproj +++ b/src/ArchivePackages/ArchivePackages.csproj @@ -75,9 +75,6 @@ 9.0.1 - - 2.25.0-master-30453 - 4.3.3 diff --git a/src/Gallery.CredentialExpiration/Gallery.CredentialExpiration.csproj b/src/Gallery.CredentialExpiration/Gallery.CredentialExpiration.csproj index 696c6c793..5fc084f77 100644 --- a/src/Gallery.CredentialExpiration/Gallery.CredentialExpiration.csproj +++ b/src/Gallery.CredentialExpiration/Gallery.CredentialExpiration.csproj @@ -92,9 +92,6 @@ 9.0.1 - - 2.25.0-master-30263 - 2.1.3 diff --git a/src/Gallery.CredentialExpiration/GalleryCredentialExpiration.cs b/src/Gallery.CredentialExpiration/GalleryCredentialExpiration.cs index e1021e836..e9db4dc9a 100644 --- a/src/Gallery.CredentialExpiration/GalleryCredentialExpiration.cs +++ b/src/Gallery.CredentialExpiration/GalleryCredentialExpiration.cs @@ -6,7 +6,6 @@ using System.Data.SqlClient; using System.Linq; using System.Threading.Tasks; -using NuGet.Services.Sql; using Gallery.CredentialExpiration.Models; namespace Gallery.CredentialExpiration @@ -14,12 +13,15 @@ namespace Gallery.CredentialExpiration public class GalleryCredentialExpiration : ICredentialExpirationExporter { private readonly CredentialExpirationJobMetadata _jobMetadata; - private readonly ISqlConnectionFactory _galleryDatabase; - public GalleryCredentialExpiration(CredentialExpirationJobMetadata jobMetadata, ISqlConnectionFactory galleryDatabase) + private Func> OpenGallerySqlConnectionAsync { get; } + + public GalleryCredentialExpiration( + CredentialExpirationJobMetadata jobMetadata, + Func> openGallerySqlConnectionAsync) { _jobMetadata = jobMetadata; - _galleryDatabase = galleryDatabase; + OpenGallerySqlConnectionAsync = openGallerySqlConnectionAsync; } /// @@ -51,7 +53,7 @@ public async Task> GetCredentialsAsync(TimeSpan time var minNotificationDate = ConvertToString(GetMinNotificationDate()); // Connect to database - using (var galleryConnection = await _galleryDatabase.CreateAsync()) + using (var galleryConnection = await OpenGallerySqlConnectionAsync()) { // Fetch credentials that expire in _warnDaysBeforeExpiration days // + the user's e-mail address diff --git a/src/Gallery.CredentialExpiration/Job.cs b/src/Gallery.CredentialExpiration/Job.cs index 4290dba1b..e0340eb0a 100644 --- a/src/Gallery.CredentialExpiration/Job.cs +++ b/src/Gallery.CredentialExpiration/Job.cs @@ -2,7 +2,6 @@ // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; -using System.Collections.Concurrent; using System.Collections.Generic; using System.ComponentModel.Design; using System.Data.SqlClient; @@ -15,10 +14,7 @@ using Microsoft.Extensions.Logging; using Microsoft.WindowsAzure.Storage; using Newtonsoft.Json; -using Newtonsoft.Json.Linq; using NuGet.Jobs; -using NuGet.Services.KeyVault; -using NuGet.Services.Sql; using NuGet.Services.Storage; namespace Gallery.CredentialExpiration @@ -34,8 +30,6 @@ public class Job : JobBase private string _galleryBrand; private string _galleryAccountUrl; - private ISqlConnectionFactory _galleryDatabase; - private string _mailFrom; private SmtpClient _smtpClient; @@ -47,9 +41,7 @@ public override void Init(IServiceContainer serviceContainer, IDictionary OpenGallerySqlConnectionAsync() + { + return OpenSqlConnectionAsync(JobArgumentNames.GalleryDatabase); + } + public override async Task Run() { var jobRunTime = DateTimeOffset.UtcNow; // Default values var jobCursor = new JobRunTimeCursor( jobCursorTime: jobRunTime, maxProcessedCredentialsTime: jobRunTime ); - var galleryCredentialExpiration = new GalleryCredentialExpiration(new CredentialExpirationJobMetadata(jobRunTime, _warnDaysBeforeExpiration, jobCursor), _galleryDatabase); + var galleryCredentialExpiration = new GalleryCredentialExpiration(new CredentialExpirationJobMetadata(jobRunTime, _warnDaysBeforeExpiration, jobCursor), OpenGallerySqlConnectionAsync); try { @@ -89,7 +86,7 @@ public override async Task Run() // Load from cursor // Throw if the schema is not correct to ensure that not-intended emails are sent. jobCursor = JsonConvert.DeserializeObject(content, new JsonSerializerSettings() { MissingMemberHandling = MissingMemberHandling.Error }); - galleryCredentialExpiration = new GalleryCredentialExpiration(new CredentialExpirationJobMetadata(jobRunTime, _warnDaysBeforeExpiration, jobCursor), _galleryDatabase); + galleryCredentialExpiration = new GalleryCredentialExpiration(new CredentialExpirationJobMetadata(jobRunTime, _warnDaysBeforeExpiration, jobCursor), OpenGallerySqlConnectionAsync); } // Connect to database diff --git a/src/Gallery.Maintenance/DeleteExpiredVerificationKeysTask.cs b/src/Gallery.Maintenance/DeleteExpiredVerificationKeysTask.cs index dc0d07ca5..14e4d90d9 100644 --- a/src/Gallery.Maintenance/DeleteExpiredVerificationKeysTask.cs +++ b/src/Gallery.Maintenance/DeleteExpiredVerificationKeysTask.cs @@ -9,6 +9,7 @@ using System.Threading.Tasks; using Gallery.Maintenance.Models; using Microsoft.Extensions.Logging; +using NuGet.Jobs; namespace Gallery.Maintenance { @@ -37,7 +38,7 @@ public override async Task RunAsync(Job job) { IEnumerable expiredKeys; - using (var connection = await job.GalleryDatabase.CreateAsync()) + using (var connection = await job.OpenSqlConnectionAsync(JobArgumentNames.GalleryDatabase)) { expiredKeys = await connection.QueryWithRetryAsync( SelectQuery, @@ -59,7 +60,7 @@ public override async Task RunAsync(Job job) if (expectedRowCount > 0) { - using (var connection = await job.GalleryDatabase.CreateAsync()) + using (var connection = await job.OpenSqlConnectionAsync(JobArgumentNames.GalleryDatabase)) { using (var transaction = connection.BeginTransaction()) { diff --git a/src/Gallery.Maintenance/Gallery.Maintenance.csproj b/src/Gallery.Maintenance/Gallery.Maintenance.csproj index b11f12415..e6a3ed08b 100644 --- a/src/Gallery.Maintenance/Gallery.Maintenance.csproj +++ b/src/Gallery.Maintenance/Gallery.Maintenance.csproj @@ -67,9 +67,6 @@ 9.0.1 - - 2.25.0-master-30263 - 4.3.3 diff --git a/src/Gallery.Maintenance/Job.cs b/src/Gallery.Maintenance/Job.cs index 488a49b92..55d64f291 100644 --- a/src/Gallery.Maintenance/Job.cs +++ b/src/Gallery.Maintenance/Job.cs @@ -4,13 +4,11 @@ using System; using System.Collections.Generic; using System.ComponentModel.Design; -using System.Data; using System.Linq; using System.Threading.Tasks; using Microsoft.Extensions.Logging; using NuGet.Jobs; using NuGet.Services.KeyVault; -using NuGet.Services.Sql; namespace Gallery.Maintenance { @@ -19,15 +17,9 @@ namespace Gallery.Maintenance /// public class Job : JobBase { - - public ISqlConnectionFactory GalleryDatabase { get; private set; } - public override void Init(IServiceContainer serviceContainer, IDictionary jobArgsDictionary) { - var secretInjector = (ISecretInjector)serviceContainer.GetService(typeof(ISecretInjector)); - var databaseConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.GalleryDatabase); - - GalleryDatabase = new AzureSqlConnectionFactory(databaseConnectionString, secretInjector); + RegisterDatabase(serviceContainer, jobArgsDictionary, JobArgumentNames.GalleryDatabase); } public override async Task Run() diff --git a/src/NuGet.Jobs.Common/Extensions/SqlConnectionStringBuilderExtensions.cs b/src/NuGet.Jobs.Common/Extensions/SqlConnectionStringBuilderExtensions.cs deleted file mode 100644 index 608481d2c..000000000 --- a/src/NuGet.Jobs.Common/Extensions/SqlConnectionStringBuilderExtensions.cs +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using System.Threading.Tasks; - -// ReSharper disable once CheckNamespace -namespace System.Data.SqlClient -{ - public static class SqlConnectionStringBuilderExtensions - { - public static Task ConnectTo(this SqlConnectionStringBuilder self) - { - return ConnectTo(self.ConnectionString); - } - - private static async Task ConnectTo(string connection) - { - var c = new SqlConnection(connection); - await c.OpenAsync().ConfigureAwait(continueOnCapturedContext: false); - return c; - } - } -} \ No newline at end of file diff --git a/src/NuGet.Jobs.Common/JobBase.cs b/src/NuGet.Jobs.Common/JobBase.cs index e16184854..b16f92846 100644 --- a/src/NuGet.Jobs.Common/JobBase.cs +++ b/src/NuGet.Jobs.Common/JobBase.cs @@ -1,11 +1,18 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +using System; using System.Collections.Generic; using System.ComponentModel.Design; +using System.Data.SqlClient; using System.Diagnostics.Tracing; using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using NuGet.Jobs.Configuration; +using NuGet.Services.KeyVault; +using NuGet.Services.Sql; namespace NuGet.Jobs { @@ -13,6 +20,8 @@ public abstract class JobBase { private readonly EventSource _jobEventSource; + private Dictionary _sqlConnectionFactories; + protected JobBase() : this(null) { @@ -22,6 +31,7 @@ protected JobBase(EventSource jobEventSource) { JobName = GetType().ToString(); _jobEventSource = jobEventSource; + _sqlConnectionFactories = new Dictionary(); } public string JobName { get; private set; } @@ -36,6 +46,132 @@ public void SetLogger(ILoggerFactory loggerFactory, ILogger logger) Logger = logger; } + /// + /// Test connection early to fail fast, and log connection diagnostics. + /// + private async Task TestConnection(ISqlConnectionFactory connectionFactory) + { + try + { + using (var connection = await connectionFactory.OpenAsync()) + using (var cmd = new SqlCommand("SELECT CONCAT(CURRENT_USER, '/', SYSTEM_USER)", connection)) + { + var result = cmd.ExecuteScalar(); + var user = result.ToString(); + Logger.LogInformation("Connected to database {DataSource}/{InitialCatalog} as {User}", + connectionFactory.DataSource, connectionFactory.InitialCatalog, user); + } + } + catch (Exception e) + { + Logger.LogError(0, e, "Failed to connect to database {DataSource}/{InitialCatalog}", + connectionFactory.DataSource, connectionFactory.InitialCatalog); + } + } + + /// + /// Initializes an , for use by validation jobs. + /// + /// ConnectionStringBuilder, used for diagnostics. + public SqlConnectionStringBuilder RegisterDatabase(IServiceProvider serviceProvider) + where T : IDbConfiguration + { + if (serviceProvider == null) + { + throw new ArgumentNullException(nameof(serviceProvider)); + } + + var secretInjector = serviceProvider.GetRequiredService(); + var connectionString = serviceProvider.GetRequiredService>().Value.ConnectionString; + var connectionFactory = new AzureSqlConnectionFactory(connectionString, secretInjector); + + return RegisterDatabase(nameof(T), connectionString, secretInjector); + } + + /// + /// Initializes an , for use by non-validation jobs. + /// + /// ConnectionStringBuilder, used for diagnostics. + public SqlConnectionStringBuilder RegisterDatabase(IServiceContainer serviceContainer, IDictionary jobArgsDictionary, string connectionStringArgName) + { + if (serviceContainer == null) + { + throw new ArgumentNullException(nameof(serviceContainer)); + } + + if (jobArgsDictionary == null) + { + throw new ArgumentNullException(nameof(jobArgsDictionary)); + } + + if (string.IsNullOrEmpty(connectionStringArgName)) + { + throw new ArgumentException("Argument cannot be null or empty.", nameof(connectionStringArgName)); + } + + var secretInjector = (ISecretInjector)serviceContainer.GetService(typeof(ISecretInjector)); + var connectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, connectionStringArgName); + + return RegisterDatabase(connectionStringArgName, connectionString, secretInjector); + } + + /// + /// Register a job database at initialization time. Each call should overwrite any existing + /// registration because calls on every iteration. + /// + /// ConnectionStringBuilder, used for diagnostics. + private SqlConnectionStringBuilder RegisterDatabase(string name, string connectionString, ISecretInjector secretInjector) + { + var connectionFactory = new AzureSqlConnectionFactory(connectionString, secretInjector, Logger); + _sqlConnectionFactories[name] = connectionFactory; + + Task.Run(() => TestConnection(connectionFactory)).Wait(); + + return connectionFactory.SqlConnectionStringBuilder; + } + + /// + /// Create a SqlConnection, for use by validation jobs. + /// + public Task CreateSqlConnectionAsync() + where T : IDbConfiguration + { + var name = nameof(T); + if (!_sqlConnectionFactories.ContainsKey(name)) + { + throw new InvalidOperationException($"Database {name} has not been registered."); + } + + return _sqlConnectionFactories[name].CreateAsync(); + } + + /// + /// Synchronous creation of a SqlConnection, for use by validation jobs. + /// + public SqlConnection CreateSqlConnection() + where T : IDbConfiguration + { + return Task.Run(() => CreateSqlConnectionAsync()).Result; + } + + /// + /// Creates and opens a SqlConnection, for use by non-validation jobs. + /// + public Task OpenSqlConnectionAsync(string connectionStringArgName) + { + if (string.IsNullOrEmpty(connectionStringArgName)) + { + throw new ArgumentException("Argument cannot be null or empty.", nameof(connectionStringArgName)); + } + + if (!_sqlConnectionFactories.ContainsKey(connectionStringArgName)) + { + throw new InvalidOperationException($"Database {connectionStringArgName} has not been registered."); + } + + return _sqlConnectionFactories[connectionStringArgName].OpenAsync(); + } + public abstract void Init(IServiceContainer serviceContainer, IDictionary jobArgsDictionary); public abstract Task Run(); diff --git a/src/NuGet.Jobs.Common/NuGet.Jobs.Common.csproj b/src/NuGet.Jobs.Common/NuGet.Jobs.Common.csproj index 2dd0251bc..1c1527895 100644 --- a/src/NuGet.Jobs.Common/NuGet.Jobs.Common.csproj +++ b/src/NuGet.Jobs.Common/NuGet.Jobs.Common.csproj @@ -49,7 +49,6 @@ - @@ -82,6 +81,9 @@ 2.27.0 + + 2.27.0 + 4.3.3 diff --git a/src/NuGet.Services.Validation.Orchestrator/Job.cs b/src/NuGet.Services.Validation.Orchestrator/Job.cs index 8e6c9f831..b3c5638c3 100644 --- a/src/NuGet.Services.Validation.Orchestrator/Job.cs +++ b/src/NuGet.Services.Validation.Orchestrator/Job.cs @@ -4,7 +4,6 @@ using System; using System.Collections.Generic; using System.ComponentModel.Design; -using System.Data.Common; using System.Net; using System.Net.Http; using System.Reflection; @@ -31,7 +30,6 @@ using NuGet.Services.KeyVault; using NuGet.Services.Logging; using NuGet.Services.ServiceBus; -using NuGet.Services.Sql; using NuGet.Services.Validation.Orchestrator.PackageSigning.ScanAndSign; using NuGet.Services.Validation.Orchestrator.Telemetry; using NuGet.Services.Validation.PackageSigning.ProcessSignature; @@ -86,6 +84,12 @@ public override void Init(IServiceContainer serviceContainer, IDictionary(_serviceProvider); + RegisterDatabase(_serviceProvider); + } ConfigurationValidated = false; } @@ -158,15 +162,6 @@ private void ConfigureLibraries(IServiceCollection services) services.AddLogging(); } - private DbConnection CreateDbConnection(IServiceProvider serviceProvider) where T : IDbConfiguration - { - var connectionString = serviceProvider.GetRequiredService>().Value.ConnectionString; - var connectionFactory = new AzureSqlConnectionFactory(connectionString, - serviceProvider.GetRequiredService()); - - return Task.Run(() => connectionFactory.CreateAsync()).Result; - } - private void ConfigureJobServices(IServiceCollection services, IConfigurationRoot configurationRoot) { services.Configure(configurationRoot.GetSection(ConfigurationSectionName)); @@ -184,15 +179,15 @@ private void ConfigureJobServices(IServiceCollection services, IConfigurationRoo services.AddTransient(); services.AddTransient(); - + services.AddScoped(serviceProvider => new NuGetGallery.EntitiesContext( - CreateDbConnection(serviceProvider), + CreateSqlConnection(), readOnly: false) ); services.AddScoped(serviceProvider => new ValidationEntitiesContext( - CreateDbConnection(serviceProvider))); + CreateSqlConnection())); services.AddScoped(serviceProvider => serviceProvider.GetRequiredService()); diff --git a/src/NuGet.Services.Validation.Orchestrator/NuGet.Services.Validation.Orchestrator.csproj b/src/NuGet.Services.Validation.Orchestrator/NuGet.Services.Validation.Orchestrator.csproj index ebf1fb92c..e7ba23387 100644 --- a/src/NuGet.Services.Validation.Orchestrator/NuGet.Services.Validation.Orchestrator.csproj +++ b/src/NuGet.Services.Validation.Orchestrator/NuGet.Services.Validation.Orchestrator.csproj @@ -123,9 +123,6 @@ 1.2.0 - - 2.26.0-master-34394 - 2.26.0-master-34394 diff --git a/src/NuGet.SupportRequests.Notifications/Job.cs b/src/NuGet.SupportRequests.Notifications/Job.cs index 99de91ce9..8f3292405 100644 --- a/src/NuGet.SupportRequests.Notifications/Job.cs +++ b/src/NuGet.SupportRequests.Notifications/Job.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Generic; using System.ComponentModel.Design; +using System.Data.SqlClient; using System.Threading.Tasks; using NuGet.Jobs; @@ -24,11 +25,19 @@ public override void Init(IServiceContainer serviceContainer, IDictionary OpenSupportSqlConnectionAsync() + { + return OpenSqlConnectionAsync(JobArgumentNames.SourceDatabase); } public override async Task Run() { - var scheduledTask = ScheduledTaskFactory.Create(_serviceContainer, _jobArgsDictionary, LoggerFactory); + var scheduledTask = ScheduledTaskFactory.Create(_serviceContainer, _jobArgsDictionary, + OpenSupportSqlConnectionAsync, LoggerFactory); await scheduledTask.RunAsync(); } diff --git a/src/NuGet.SupportRequests.Notifications/NuGet.SupportRequests.Notifications.csproj b/src/NuGet.SupportRequests.Notifications/NuGet.SupportRequests.Notifications.csproj index a72a38207..44a43935d 100644 --- a/src/NuGet.SupportRequests.Notifications/NuGet.SupportRequests.Notifications.csproj +++ b/src/NuGet.SupportRequests.Notifications/NuGet.SupportRequests.Notifications.csproj @@ -106,9 +106,6 @@ 9.0.1 - - 2.25.0-master-30453 - diff --git a/src/NuGet.SupportRequests.Notifications/ScheduledTaskFactory.cs b/src/NuGet.SupportRequests.Notifications/ScheduledTaskFactory.cs index 67586efb9..42834faf2 100644 --- a/src/NuGet.SupportRequests.Notifications/ScheduledTaskFactory.cs +++ b/src/NuGet.SupportRequests.Notifications/ScheduledTaskFactory.cs @@ -4,6 +4,8 @@ using System; using System.Collections.Generic; using System.ComponentModel.Design; +using System.Data.SqlClient; +using System.Threading.Tasks; using Microsoft.Extensions.Logging; namespace NuGet.SupportRequests.Notifications @@ -12,7 +14,11 @@ internal class ScheduledTaskFactory { private const string _tasksNamespace = "NuGet.SupportRequests.Notifications.Tasks"; - public static IScheduledTask Create(IServiceContainer serviceContainer, IDictionary jobArgsDictionary, ILoggerFactory loggerFactory) + public static IScheduledTask Create( + IServiceContainer serviceContainer, + IDictionary jobArgsDictionary, + Func> openSupportSqlConnectionAsync, + ILoggerFactory loggerFactory) { if (jobArgsDictionary == null) { @@ -25,7 +31,8 @@ public static IScheduledTask Create(IServiceContainer serviceContainer, IDiction } var scheduledTaskName = jobArgsDictionary[JobArgumentNames.ScheduledTask]; - var scheduledTask = GetTaskOfType(scheduledTaskName, serviceContainer, jobArgsDictionary, loggerFactory); + var scheduledTask = GetTaskOfType(scheduledTaskName, serviceContainer, jobArgsDictionary, + openSupportSqlConnectionAsync, loggerFactory); return scheduledTask; } @@ -34,6 +41,7 @@ private static IScheduledTask GetTaskOfType( string taskName, IServiceContainer serviceContainer, IDictionary jobArgsDictionary, + Func> openSupportSqlConnectionAsync, ILoggerFactory loggerFactory) { if (string.IsNullOrEmpty(taskName)) @@ -51,7 +59,7 @@ private static IScheduledTask GetTaskOfType( IScheduledTask scheduledTask; if (scheduledTaskType != null && typeof(IScheduledTask).IsAssignableFrom(scheduledTaskType)) { - var args = new object[] { serviceContainer, jobArgsDictionary, loggerFactory }; + var args = new object[] { serviceContainer, jobArgsDictionary, openSupportSqlConnectionAsync, loggerFactory }; scheduledTask = (IScheduledTask)Activator.CreateInstance(scheduledTaskType, args); } else diff --git a/src/NuGet.SupportRequests.Notifications/SupportRequestRepository.cs b/src/NuGet.SupportRequests.Notifications/SupportRequestRepository.cs index 1dcea245b..d4c3f8f20 100644 --- a/src/NuGet.SupportRequests.Notifications/SupportRequestRepository.cs +++ b/src/NuGet.SupportRequests.Notifications/SupportRequestRepository.cs @@ -8,7 +8,6 @@ using System.Linq; using System.Threading.Tasks; using Microsoft.Extensions.Logging; -using NuGet.Services.Sql; using NuGet.SupportRequests.Notifications.Models; namespace NuGet.SupportRequests.Notifications @@ -20,29 +19,24 @@ internal class SupportRequestRepository private const string _parameterNamePagerDutyUsername = "pagerDutyUserName"; private readonly DateTime _defaultSqlDateTime = new DateTime(1900, 1, 1, 0, 0, 0, DateTimeKind.Utc); private readonly ILogger _logger; - private readonly ISqlConnectionFactory _supportDbConnectionFactory; + private readonly Func> _openSupportSqlConnectionAsync; public SupportRequestRepository( - ILoggerFactory loggerFactory, - ISqlConnectionFactory supportDbConnectionFactory) + Func> openSupportSqlConnectionAsync, + ILoggerFactory loggerFactory) { if (loggerFactory == null) { throw new ArgumentNullException(nameof(loggerFactory)); } - if (supportDbConnectionFactory == null) - { - throw new ArgumentNullException(nameof(supportDbConnectionFactory)); - } - _logger = loggerFactory.CreateLogger(); - _supportDbConnectionFactory = supportDbConnectionFactory; + _openSupportSqlConnectionAsync = openSupportSqlConnectionAsync; } - internal async Task OpenConnectionAsync() + internal async Task OpenSupportSqlConnectionAsync() { - var connection = await _supportDbConnectionFactory.CreateAsync(); + var connection = await _openSupportSqlConnectionAsync(); connection.InfoMessage += OnSqlConnectionInfoMessage; return connection; @@ -181,7 +175,7 @@ private async Task EnsureConnectionOpenAsync(SqlConnection connec { if (connection == null) { - connection = await OpenConnectionAsync(); + connection = await OpenSupportSqlConnectionAsync(); } else if (connection.State != ConnectionState.Open) { diff --git a/src/NuGet.SupportRequests.Notifications/Tasks/OnCallDailyNotificationTask.cs b/src/NuGet.SupportRequests.Notifications/Tasks/OnCallDailyNotificationTask.cs index 35892f72d..82d9b09a1 100644 --- a/src/NuGet.SupportRequests.Notifications/Tasks/OnCallDailyNotificationTask.cs +++ b/src/NuGet.SupportRequests.Notifications/Tasks/OnCallDailyNotificationTask.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Generic; using System.ComponentModel.Design; +using System.Data.SqlClient; using System.Linq; using System.Text; using System.Threading.Tasks; @@ -25,8 +26,9 @@ internal class OnCallDailyNotificationTask public OnCallDailyNotificationTask( IServiceContainer serviceContainer, IDictionary jobArgsDictionary, + Func> openSupportSqlConnectionAsync, ILoggerFactory loggerFactory) - : base(serviceContainer, jobArgsDictionary, loggerFactory) + : base(serviceContainer, jobArgsDictionary, openSupportSqlConnectionAsync, loggerFactory) { var pagerDutyConfiguration = new PagerDutyConfiguration( jobArgsDictionary[_argumentNamePagerDutyAccountName], @@ -44,7 +46,7 @@ protected override async Task BuildNotification( var targetEmailAddress = string.Format(_targetEmailAddressFormat, onCallAlias); List unresolvedIssues; - using (var connection = await supportRequestRepository.OpenConnectionAsync()) + using (var connection = await supportRequestRepository.OpenSupportSqlConnectionAsync()) { unresolvedIssues = await supportRequestRepository.GetUnresolvedIssues(connection, onCallAlias); } diff --git a/src/NuGet.SupportRequests.Notifications/Tasks/SupportRequestsNotificationScheduledTask.cs b/src/NuGet.SupportRequests.Notifications/Tasks/SupportRequestsNotificationScheduledTask.cs index 6976ed0bb..51ffd7d5d 100644 --- a/src/NuGet.SupportRequests.Notifications/Tasks/SupportRequestsNotificationScheduledTask.cs +++ b/src/NuGet.SupportRequests.Notifications/Tasks/SupportRequestsNotificationScheduledTask.cs @@ -4,10 +4,9 @@ using System; using System.Collections.Generic; using System.ComponentModel.Design; +using System.Data.SqlClient; using System.Threading.Tasks; using Microsoft.Extensions.Logging; -using NuGet.Services.KeyVault; -using NuGet.Services.Sql; using NuGet.SupportRequests.Notifications.Notifications; using NuGet.SupportRequests.Notifications.Services; using NuGet.SupportRequests.Notifications.Templates; @@ -24,6 +23,7 @@ internal abstract class SupportRequestsNotificationScheduledTask protected SupportRequestsNotificationScheduledTask( IServiceContainer serviceContainer, IDictionary jobArgsDictionary, + Func> openSupportSqlConnectionAsync, ILoggerFactory loggerFactory) { if (jobArgsDictionary == null) @@ -38,15 +38,12 @@ protected SupportRequestsNotificationScheduledTask( var smtpUri = jobArgsDictionary[JobArgumentNames.SmtpUri]; _messagingService = new MessagingService(loggerFactory, smtpUri); - - var secretInjector = (ISecretInjector)serviceContainer.GetService(typeof(ISecretInjector)); - var supportDbConnectionString = jobArgsDictionary[JobArgumentNames.SourceDatabase]; - var supportDbConnectionFactory = new AzureSqlConnectionFactory(supportDbConnectionString, secretInjector); - _supportRequestRepository = new SupportRequestRepository(loggerFactory, supportDbConnectionFactory); + _supportRequestRepository = new SupportRequestRepository(openSupportSqlConnectionAsync, loggerFactory); } protected abstract Task BuildNotification(SupportRequestRepository supportRequestRepository, DateTime referenceTime); + protected abstract string BuildNotificationBody(string template, TNotification notification); public async Task RunAsync() diff --git a/src/NuGet.SupportRequests.Notifications/Tasks/WeeklySummaryNotificationTask.cs b/src/NuGet.SupportRequests.Notifications/Tasks/WeeklySummaryNotificationTask.cs index 71a0f9ccf..efb3c9518 100644 --- a/src/NuGet.SupportRequests.Notifications/Tasks/WeeklySummaryNotificationTask.cs +++ b/src/NuGet.SupportRequests.Notifications/Tasks/WeeklySummaryNotificationTask.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Generic; using System.ComponentModel.Design; +using System.Data.SqlClient; using System.Linq; using System.Text; using System.Threading.Tasks; @@ -24,8 +25,9 @@ internal class WeeklySummaryNotificationTask public WeeklySummaryNotificationTask( IServiceContainer serviceContainer, IDictionary jobArgsDictionary, + Func> openSupportSqlConnectionAsync, ILoggerFactory loggerFactory) - : base(serviceContainer, jobArgsDictionary, loggerFactory) + : base(serviceContainer, jobArgsDictionary, openSupportSqlConnectionAsync, loggerFactory) { _targetEmailAddress = jobArgsDictionary[_argumentNameTargetEmailAddress]; } @@ -42,7 +44,7 @@ protected override async Task BuildNotification( var startDateUtcLastWeek = referenceTime.AddDays(-7); var startDateUtcPriorWeek = referenceTime.AddDays(-14); - using (var connection = await supportRequestRepository.OpenConnectionAsync()) + using (var connection = await supportRequestRepository.OpenSupportSqlConnectionAsync()) { unresolvedIssues = await supportRequestRepository.GetUnresolvedIssues(connection); diff --git a/src/Search.GenerateAuxiliaryData/Job.cs b/src/Search.GenerateAuxiliaryData/Job.cs index 4d0971f80..84ed5b8b4 100644 --- a/src/Search.GenerateAuxiliaryData/Job.cs +++ b/src/Search.GenerateAuxiliaryData/Job.cs @@ -11,12 +11,10 @@ using Microsoft.WindowsAzure.Storage; using Microsoft.WindowsAzure.Storage.Blob; using NuGet.Jobs; -using NuGet.Services.KeyVault; -using NuGet.Services.Sql; namespace Search.GenerateAuxiliaryData { - internal class Job + public class Job : JobBase { private const string DefaultContainerName = "ng-search-data"; @@ -45,13 +43,8 @@ internal class Job public override void Init(IServiceContainer serviceContainer, IDictionary jobArgsDictionary) { - var secretInjector = (ISecretInjector)serviceContainer.GetService(typeof(ISecretInjector)); - - var packageDbConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.PackageDatabase); - var packageDbConnectionFactory = new AzureSqlConnectionFactory(packageDbConnectionString, secretInjector); - - var statisticsDbConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatisticsDatabase); - var statisticsDbConnectionFactory = new AzureSqlConnectionFactory(statisticsDbConnectionString, secretInjector); + RegisterDatabase(serviceContainer, jobArgsDictionary, JobArgumentNames.PackageDatabase); + RegisterDatabase(serviceContainer, jobArgsDictionary, JobArgumentNames.StatisticsDatabase); var statisticsStorageAccount = CloudStorageAccount.Parse( JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.AzureCdnCloudStorageAccount)); @@ -69,14 +62,42 @@ public override void Init(IServiceContainer serviceContainer, IDictionary { - new VerifiedPackagesExporter(LoggerFactory.CreateLogger(), packageDbConnectionFactory, _destContainer, ScriptVerifiedPackages, OutputNameVerifiedPackages), - new NestedJArrayExporter(LoggerFactory.CreateLogger(), packageDbConnectionFactory, _destContainer, ScriptCuratedFeed, OutputNameCuratedFeed, Col0CuratedFeed, Col1CuratedFeed), - new NestedJArrayExporter(LoggerFactory.CreateLogger(), packageDbConnectionFactory, _destContainer, ScriptOwners, OutputNameOwners, Col0Owners, Col1Owners), - new RankingsExporter(LoggerFactory.CreateLogger(), statisticsDbConnectionFactory, _destContainer, ScriptRankingsTotal, OutputNameRankings), - new BlobStorageExporter(LoggerFactory.CreateLogger(), _statisticsContainer, StatisticsReportName, _destContainer, StatisticsReportName) + new VerifiedPackagesExporter( + OpenGallerySqlConnectionAsync, + LoggerFactory.CreateLogger(), + _destContainer, ScriptVerifiedPackages, OutputNameVerifiedPackages), + + new NestedJArrayExporter( + OpenGallerySqlConnectionAsync, + LoggerFactory.CreateLogger(), + _destContainer, ScriptCuratedFeed, OutputNameCuratedFeed, Col0CuratedFeed, Col1CuratedFeed), + + new NestedJArrayExporter( + OpenGallerySqlConnectionAsync, + LoggerFactory.CreateLogger(), + _destContainer, ScriptOwners, OutputNameOwners, Col0Owners, Col1Owners), + + new RankingsExporter( + OpenStatisticsSqlConnectionAsync, + LoggerFactory.CreateLogger(), + _destContainer, ScriptRankingsTotal, OutputNameRankings), + + new BlobStorageExporter( + LoggerFactory.CreateLogger(), + _statisticsContainer, StatisticsReportName, _destContainer, StatisticsReportName) }; } + public Task OpenGallerySqlConnectionAsync() + { + return OpenSqlConnectionAsync(JobArgumentNames.PackageDatabase); + } + + public Task OpenStatisticsSqlConnectionAsync() + { + return OpenSqlConnectionAsync(JobArgumentNames.StatisticsDatabase); + } + public override async Task Run() { var failedExporters = new List(); diff --git a/src/Search.GenerateAuxiliaryData/NestedJArrayExporter.cs b/src/Search.GenerateAuxiliaryData/NestedJArrayExporter.cs index 83824718a..ff4cd505c 100644 --- a/src/Search.GenerateAuxiliaryData/NestedJArrayExporter.cs +++ b/src/Search.GenerateAuxiliaryData/NestedJArrayExporter.cs @@ -6,10 +6,10 @@ using System.Data; using System.Data.SqlClient; using System.Linq; +using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Microsoft.WindowsAzure.Storage.Blob; using Newtonsoft.Json.Linq; -using NuGet.Services.Sql; namespace Search.GenerateAuxiliaryData { @@ -17,11 +17,17 @@ class NestedJArrayExporter : SqlExporter { public string Col0 { get; } + public string Col1 { get; } + public string SqlScript { get; } - public NestedJArrayExporter(ILogger logger, ISqlConnectionFactory connectionFactory, CloudBlobContainer defaultDestinationContainer, string defaultSqlScript, string defaultName, string defaultCol0, string defaultCol1) - : base(logger, connectionFactory, defaultDestinationContainer, defaultName) + public NestedJArrayExporter( + Func> openGallerySqlConnectionAsync, + ILogger logger, + CloudBlobContainer defaultDestinationContainer, + string defaultSqlScript, string defaultName, string defaultCol0, string defaultCol1) + : base(openGallerySqlConnectionAsync, logger, defaultDestinationContainer, defaultName) { Col0 = defaultCol0; Col1 = defaultCol1; diff --git a/src/Search.GenerateAuxiliaryData/RankingsExporter.cs b/src/Search.GenerateAuxiliaryData/RankingsExporter.cs index 520812ff7..14ddb8bee 100644 --- a/src/Search.GenerateAuxiliaryData/RankingsExporter.cs +++ b/src/Search.GenerateAuxiliaryData/RankingsExporter.cs @@ -4,10 +4,10 @@ using System; using System.Data; using System.Data.SqlClient; +using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Microsoft.WindowsAzure.Storage.Blob; using Newtonsoft.Json.Linq; -using NuGet.Services.Sql; namespace Search.GenerateAuxiliaryData { @@ -20,12 +20,12 @@ public sealed class RankingsExporter : SqlExporter private readonly string _rankingsTotalScript; public RankingsExporter( + Func> openStatisticsSqlConnectionAsync, ILogger logger, - ISqlConnectionFactory connectionFactory, CloudBlobContainer defaultDestinationContainer, string defaultRankingsScript, string defaultName) - : base(logger, connectionFactory, defaultDestinationContainer, defaultName) + : base(openStatisticsSqlConnectionAsync, logger, defaultDestinationContainer, defaultName) { _rankingsTotalScript = defaultRankingsScript; } diff --git a/src/Search.GenerateAuxiliaryData/Search.GenerateAuxiliaryData.csproj b/src/Search.GenerateAuxiliaryData/Search.GenerateAuxiliaryData.csproj index e9d6dddac..d0a100c02 100644 --- a/src/Search.GenerateAuxiliaryData/Search.GenerateAuxiliaryData.csproj +++ b/src/Search.GenerateAuxiliaryData/Search.GenerateAuxiliaryData.csproj @@ -92,9 +92,6 @@ 9.0.1 - - 2.25.0-master-30453 - 7.1.2 diff --git a/src/Search.GenerateAuxiliaryData/SqlExporter.cs b/src/Search.GenerateAuxiliaryData/SqlExporter.cs index fedaa8227..652b1df4f 100644 --- a/src/Search.GenerateAuxiliaryData/SqlExporter.cs +++ b/src/Search.GenerateAuxiliaryData/SqlExporter.cs @@ -1,6 +1,7 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +using System; using System.Collections.Generic; using System.Data; using System.Data.SqlClient; @@ -11,7 +12,6 @@ using Microsoft.WindowsAzure.Storage.Blob; using Newtonsoft.Json; using Newtonsoft.Json.Linq; -using NuGet.Services.Sql; namespace Search.GenerateAuxiliaryData { @@ -20,14 +20,19 @@ public abstract class SqlExporter : Exporter { private static Assembly _executingAssembly = Assembly.GetExecutingAssembly(); private static string _assemblyName = _executingAssembly.GetName().Name; - - public ISqlConnectionFactory ConnectionFactory { get; } - public SqlExporter(ILogger logger, ISqlConnectionFactory connectionFactory, CloudBlobContainer defaultDestinationContainer, string defaultName) + private Func> OpenSqlConnectionAsync { get; } + + public SqlExporter( + Func> openSqlConnectionAsync, + ILogger logger, + CloudBlobContainer defaultDestinationContainer, + string defaultName) : base(logger, defaultDestinationContainer, defaultName) { _logger = logger; - ConnectionFactory = connectionFactory; + + OpenSqlConnectionAsync = openSqlConnectionAsync; } protected static string GetEmbeddedSqlScript(string resourceName) @@ -38,12 +43,12 @@ protected static string GetEmbeddedSqlScript(string resourceName) public override async Task ExportAsync() { - _logger.LogInformation("Generating {ReportName} report from {DataSource}/{InitialCatalog}.", - _name, ConnectionFactory.DataSource, ConnectionFactory.InitialCatalog); - JContainer result; - using (var connection = await ConnectionFactory.CreateAsync()) + using (var connection = await OpenSqlConnectionAsync()) { + _logger.LogInformation("Generating {ReportName} report from {DataSource}/{InitialCatalog}.", + _name, connection.DataSource, connection.Database); + result = GetResultOfQuery(connection); } diff --git a/src/Search.GenerateAuxiliaryData/VerifiedPackagesExporter.cs b/src/Search.GenerateAuxiliaryData/VerifiedPackagesExporter.cs index 20acab67b..04e3bb334 100644 --- a/src/Search.GenerateAuxiliaryData/VerifiedPackagesExporter.cs +++ b/src/Search.GenerateAuxiliaryData/VerifiedPackagesExporter.cs @@ -4,10 +4,10 @@ using System; using System.Data; using System.Data.SqlClient; +using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Microsoft.WindowsAzure.Storage.Blob; using Newtonsoft.Json.Linq; -using NuGet.Services.Sql; namespace Search.GenerateAuxiliaryData { @@ -17,12 +17,12 @@ internal sealed class VerifiedPackagesExporter : SqlExporter private readonly string _verifiedPackagesScript; public VerifiedPackagesExporter( + Func> openGallerySqlConnectionAsync, ILogger logger, - ISqlConnectionFactory connectionFactory, CloudBlobContainer defaultDestinationContainer, string defaultVerifiedPackagesScript, string defaultName) - : base(logger, connectionFactory, defaultDestinationContainer, defaultName) + : base(openGallerySqlConnectionAsync, logger, defaultDestinationContainer, defaultName) { _verifiedPackagesScript = defaultVerifiedPackagesScript; } diff --git a/src/Stats.AggregateCdnDownloadsInGallery/Job.cs b/src/Stats.AggregateCdnDownloadsInGallery/Job.cs index 976b2e208..ee4e750b5 100644 --- a/src/Stats.AggregateCdnDownloadsInGallery/Job.cs +++ b/src/Stats.AggregateCdnDownloadsInGallery/Job.cs @@ -11,8 +11,6 @@ using System.Threading.Tasks; using Microsoft.Extensions.Logging; using NuGet.Jobs; -using NuGet.Services.KeyVault; -using NuGet.Services.Sql; using IPackageIdGroup = System.Linq.IGrouping; namespace Stats.AggregateCdnDownloadsInGallery @@ -55,20 +53,13 @@ GROUP BY Stats.[PackageRegistrationKey] DROP TABLE #AggregateCdnDownloadsInGallery"; private const string _storedProcedureName = "[dbo].[SelectTotalDownloadCountsPerPackageVersion]"; - private ISqlConnectionFactory _statisticsDbConnectionFactory; - private ISqlConnectionFactory _galleryDbConnectionFactory; private int _batchSize; private int _batchSleepSeconds; public override void Init(IServiceContainer serviceContainer, IDictionary jobArgsDictionary) { - var secretInjector = (ISecretInjector)serviceContainer.GetService(typeof(ISecretInjector)); - - var statisticsDbConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatisticsDatabase); - _statisticsDbConnectionFactory = new AzureSqlConnectionFactory(statisticsDbConnectionString, secretInjector); - - var galleryDbConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.DestinationDatabase); - _galleryDbConnectionFactory = new AzureSqlConnectionFactory(galleryDbConnectionString, secretInjector); + RegisterDatabase(serviceContainer, jobArgsDictionary, JobArgumentNames.StatisticsDatabase); + RegisterDatabase(serviceContainer, jobArgsDictionary, JobArgumentNames.DestinationDatabase); _batchSize = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, JobArgumentNames.BatchSize) ?? _defaultBatchSize; _batchSleepSeconds = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, JobArgumentNames.BatchSleepSeconds) ?? _defaultBatchSleepSeconds; @@ -79,12 +70,14 @@ public override async Task Run() // Gather download counts data from statistics warehouse IReadOnlyList downloadData; Logger.LogInformation("Using batch size {BatchSize} and batch sleep seconds {BatchSleepSeconds}.", _batchSize, _batchSleepSeconds); - Logger.LogInformation("Gathering Download Counts from {DataSource}/{InitialCatalog}...", _statisticsDbConnectionFactory.DataSource, _statisticsDbConnectionFactory.InitialCatalog); var stopwatch = Stopwatch.StartNew(); - using (var statisticsDatabase = await _statisticsDbConnectionFactory.CreateAsync()) + using (var statisticsDatabase = await OpenSqlConnectionAsync(JobArgumentNames.StatisticsDatabase)) using (var statisticsDatabaseTransaction = statisticsDatabase.BeginTransaction(IsolationLevel.Snapshot)) { + Logger.LogInformation("Gathering Download Counts from {DataSource}/{InitialCatalog}...", + statisticsDatabase.DataSource, statisticsDatabase.Database); + downloadData = ( await statisticsDatabase.QueryWithRetryAsync( _storedProcedureName, @@ -106,7 +99,7 @@ await statisticsDatabase.QueryWithRetryAsync( return; } - using (var destinationDatabase = await _galleryDbConnectionFactory.CreateAsync()) + using (var destinationDatabase = await OpenSqlConnectionAsync(JobArgumentNames.DestinationDatabase)) { // Fetch package registrations so we can match package ID to package registration key. var packageRegistrationLookup = await GetPackageRegistrations(destinationDatabase); diff --git a/src/Stats.AggregateCdnDownloadsInGallery/Stats.AggregateCdnDownloadsInGallery.csproj b/src/Stats.AggregateCdnDownloadsInGallery/Stats.AggregateCdnDownloadsInGallery.csproj index 3d2de7505..66c03503c 100644 --- a/src/Stats.AggregateCdnDownloadsInGallery/Stats.AggregateCdnDownloadsInGallery.csproj +++ b/src/Stats.AggregateCdnDownloadsInGallery/Stats.AggregateCdnDownloadsInGallery.csproj @@ -75,9 +75,6 @@ 9.0.1 - - 2.25.0-master-30453 - diff --git a/src/Stats.CreateAzureCdnWarehouseReports/DownloadCountReport.cs b/src/Stats.CreateAzureCdnWarehouseReports/DownloadCountReport.cs index 85ae20a62..e7a4bad21 100644 --- a/src/Stats.CreateAzureCdnWarehouseReports/DownloadCountReport.cs +++ b/src/Stats.CreateAzureCdnWarehouseReports/DownloadCountReport.cs @@ -10,7 +10,6 @@ using Microsoft.Extensions.Logging; using Newtonsoft.Json; using Newtonsoft.Json.Linq; -using NuGet.Services.Sql; using NuGet.Versioning; namespace Stats.CreateAzureCdnWarehouseReports @@ -22,25 +21,28 @@ public class DownloadCountReport private readonly TimeSpan _defaultCommandTimeout = TimeSpan.FromMinutes(30); internal const string ReportName = "downloads.v1.json"; + private Func> OpenStatisticsSqlConnectionAsync { get; } + public DownloadCountReport( + Func> openStatisticsSqlConnectionAsync, ILogger logger, - IEnumerable targets, - ISqlConnectionFactory statisticsDbConnectionFactory, - ISqlConnectionFactory galleryDbConnectionFactory) - : base(logger, targets, statisticsDbConnectionFactory, galleryDbConnectionFactory) + IEnumerable targets) + : base(logger, targets) { + OpenStatisticsSqlConnectionAsync = openStatisticsSqlConnectionAsync; } public async Task Run() { // Gather download count data from statistics warehouse IReadOnlyCollection downloadData; - _logger.LogInformation("Gathering Download Counts from {DataSource}/{InitialCatalog}...", - StatisticsDbConnectionFactory.DataSource, StatisticsDbConnectionFactory.InitialCatalog); - using (var connection = await StatisticsDbConnectionFactory.CreateAsync()) + using (var connection = await OpenStatisticsSqlConnectionAsync()) using (var transaction = connection.BeginTransaction(IsolationLevel.Snapshot)) { + _logger.LogInformation("Gathering Download Counts from {DataSource}/{InitialCatalog}...", + connection.DataSource, connection.Database); + downloadData = (await connection.QueryWithRetryAsync( _storedProcedureName, commandType: CommandType.StoredProcedure, transaction: transaction, commandTimeout: _defaultCommandTimeout)).ToList(); } diff --git a/src/Stats.CreateAzureCdnWarehouseReports/DownloadsPerToolVersionReport.cs b/src/Stats.CreateAzureCdnWarehouseReports/DownloadsPerToolVersionReport.cs index 7411526f1..588daf973 100644 --- a/src/Stats.CreateAzureCdnWarehouseReports/DownloadsPerToolVersionReport.cs +++ b/src/Stats.CreateAzureCdnWarehouseReports/DownloadsPerToolVersionReport.cs @@ -11,7 +11,6 @@ using Microsoft.WindowsAzure.Storage; using Newtonsoft.Json; using Newtonsoft.Json.Linq; -using NuGet.Services.Sql; namespace Stats.CreateAzureCdnWarehouseReports { @@ -22,14 +21,14 @@ public class DownloadsPerToolVersionReport private readonly TimeSpan _defaultCommandTimeout = TimeSpan.FromMinutes(30); internal const string ReportName = "tools.v1.json"; + private Func> OpenStatisticsSqlConnectionAsync { get; } + public DownloadsPerToolVersionReport( + Func> openStatisticsSqlConnectionAsync, ILogger logger, CloudStorageAccount cloudStorageAccount, - string statisticsContainerName, - ISqlConnectionFactory statisticsDbConnectionFactory, - ISqlConnectionFactory galleryDbConnectionFactory) - : base(logger, new[] { new StorageContainerTarget(cloudStorageAccount, statisticsContainerName) }, - statisticsDbConnectionFactory, galleryDbConnectionFactory) + string statisticsContainerName) + : base(logger, new[] { new StorageContainerTarget(cloudStorageAccount, statisticsContainerName) }) { } @@ -37,12 +36,13 @@ public async Task Run() { // Gather download count data from statistics warehouse IReadOnlyCollection data; - _logger.LogInformation("Gathering Tools Download Counts from {DataSource}/{InitialCatalog}...", - StatisticsDbConnectionFactory.DataSource, StatisticsDbConnectionFactory.InitialCatalog); - using (var connection = await StatisticsDbConnectionFactory.CreateAsync()) + using (var connection = await OpenStatisticsSqlConnectionAsync()) using (var transaction = connection.BeginTransaction(IsolationLevel.Snapshot)) { + _logger.LogInformation("Gathering Tools Download Counts from {DataSource}/{InitialCatalog}...", + connection.DataSource, connection.Database); + data = (await connection.QueryWithRetryAsync( _storedProcedureName, commandType: CommandType.StoredProcedure, transaction: transaction, commandTimeout: _defaultCommandTimeout)).ToList(); } diff --git a/src/Stats.CreateAzureCdnWarehouseReports/GalleryTotalsReport.cs b/src/Stats.CreateAzureCdnWarehouseReports/GalleryTotalsReport.cs index cdccbb09d..e32a5bd1c 100644 --- a/src/Stats.CreateAzureCdnWarehouseReports/GalleryTotalsReport.cs +++ b/src/Stats.CreateAzureCdnWarehouseReports/GalleryTotalsReport.cs @@ -9,7 +9,6 @@ using Microsoft.Extensions.Logging; using Microsoft.WindowsAzure.Storage; using Newtonsoft.Json; -using NuGet.Services.Sql; namespace Stats.CreateAzureCdnWarehouseReports { @@ -23,27 +22,33 @@ public class GalleryTotalsReport (SELECT COUNT([Key]) FROM Packages WITH (NOLOCK) WHERE Listed = 1 AND Deleted = 0) AS TotalPackages"; internal const string ReportName = "stats-totals.json"; + private Func> OpenGallerySqlConnectionAsync { get; } + + private Func> OpenStatisticsSqlConnectionAsync { get; } + public GalleryTotalsReport( + Func> openGallerySqlConnectionAsync, + Func> openStatisticsSqlConnectionAsync, ILogger logger, CloudStorageAccount cloudStorageAccount, - string statisticsContainerName, - ISqlConnectionFactory statisticsDbConnectionFactory, - ISqlConnectionFactory galleryDbConnectionFactory) - : base(logger, new[] { new StorageContainerTarget(cloudStorageAccount, statisticsContainerName) }, - statisticsDbConnectionFactory, galleryDbConnectionFactory) + string statisticsContainerName) + : base(logger, new[] { new StorageContainerTarget(cloudStorageAccount, statisticsContainerName) }) { + OpenGallerySqlConnectionAsync = openGallerySqlConnectionAsync; + OpenStatisticsSqlConnectionAsync = openStatisticsSqlConnectionAsync; } public async Task Run() { // gather package numbers from gallery database GalleryTotalsData totalsData; - _logger.LogInformation("Gathering Gallery Totals from {GalleryDataSource}/{GalleryInitialCatalog}...", - GalleryDbConnectionFactory.DataSource, GalleryDbConnectionFactory.InitialCatalog); - using (var connection = await GalleryDbConnectionFactory.CreateAsync()) + using (var connection = await OpenGallerySqlConnectionAsync()) using (var transaction = connection.BeginTransaction(IsolationLevel.Snapshot)) { + _logger.LogInformation("Gathering Gallery Totals from {GalleryDataSource}/{GalleryInitialCatalog}...", + connection.DataSource, connection.Database); + totalsData = (await connection.QueryWithRetryAsync( GalleryQuery, commandType: CommandType.Text, transaction: transaction)).First(); } @@ -52,12 +57,13 @@ public async Task Run() _logger.LogInformation("Unique packages: {UniquePackagesCount}", totalsData.UniquePackages); // gather download count data from statistics warehouse - _logger.LogInformation("Gathering Gallery Totals from {StatisticsDataSource}/{StatisticsInitialCatalog}...", - StatisticsDbConnectionFactory.DataSource, StatisticsDbConnectionFactory.InitialCatalog); - using (var connection = await StatisticsDbConnectionFactory.CreateAsync()) + using (var connection = await OpenStatisticsSqlConnectionAsync()) using (var transaction = connection.BeginTransaction(IsolationLevel.Snapshot)) { + _logger.LogInformation("Gathering Gallery Totals from {StatisticsDataSource}/{StatisticsInitialCatalog}...", + connection.DataSource, connection.Database); + totalsData.Downloads = (await connection.ExecuteScalarWithRetryAsync( WarehouseStoredProcedureName, commandType: CommandType.StoredProcedure, diff --git a/src/Stats.CreateAzureCdnWarehouseReports/Job.cs b/src/Stats.CreateAzureCdnWarehouseReports/Job.cs index 611390958..4f8f55254 100644 --- a/src/Stats.CreateAzureCdnWarehouseReports/Job.cs +++ b/src/Stats.CreateAzureCdnWarehouseReports/Job.cs @@ -4,14 +4,13 @@ using System; using System.Collections.Generic; using System.ComponentModel.Design; +using System.Data.SqlClient; using System.Linq; using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Microsoft.WindowsAzure.Storage; using Microsoft.WindowsAzure.Storage.Blob; using NuGet.Jobs; -using NuGet.Services.KeyVault; -using NuGet.Services.Sql; using Stopwatch = System.Diagnostics.Stopwatch; namespace Stats.CreateAzureCdnWarehouseReports @@ -25,13 +24,13 @@ public class Job private CloudStorageAccount _cloudStorageAccount; private CloudStorageAccount _dataStorageAccount; private string _statisticsContainerName; - private ISqlConnectionFactory _statisticsDbConnectionFactory; - private ISqlConnectionFactory _galleryDbConnectionFactory; private string _reportName; private string[] _dataContainerNames; private int _sqlCommandTimeoutSeconds = DefaultSqlCommandTimeoutSeconds; private int _perPackageReportDegreeOfParallelism = DefaultPerPackageReportDegreeOfParallelism; + private SqlConnectionStringBuilder StatisticsDatabase { get; set; } + private static readonly IDictionary _storedProcedures = new Dictionary { {ReportNames.NuGetClientVersion, "[dbo].[DownloadReportNuGetClientVersion]" }, @@ -50,14 +49,10 @@ public class Job public override void Init(IServiceContainer serviceContainer, IDictionary jobArgsDictionary) { - var secretInjector = (ISecretInjector)serviceContainer.GetService(typeof(ISecretInjector)); - _sqlCommandTimeoutSeconds = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, JobArgumentNames.CommandTimeOut) ?? DefaultSqlCommandTimeoutSeconds; - - var statisticsDatabaseConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatisticsDatabase); - _statisticsDbConnectionFactory = new AzureSqlConnectionFactory(statisticsDatabaseConnectionString, secretInjector); + StatisticsDatabase = RegisterDatabase(serviceContainer, jobArgsDictionary, JobArgumentNames.StatisticsDatabase); + RegisterDatabase(serviceContainer, jobArgsDictionary, JobArgumentNames.SourceDatabase); - var galleryDatabaseConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.SourceDatabase); - _galleryDbConnectionFactory = new AzureSqlConnectionFactory(galleryDatabaseConnectionString, secretInjector); + _sqlCommandTimeoutSeconds = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, JobArgumentNames.CommandTimeOut) ?? DefaultSqlCommandTimeoutSeconds; var cloudStorageAccountConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.AzureCdnCloudStorageAccount); var dataStorageAccountConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.DataStorageAccount); @@ -78,12 +73,24 @@ public override void Init(IServiceContainer serviceContainer, IDictionary OpenStatisticsSqlConnectionAsync() + { + return OpenSqlConnectionAsync(JobArgumentNames.StatisticsDatabase); + } + + public Task OpenGallerySqlConnectionAsync() + { + return OpenSqlConnectionAsync(JobArgumentNames.SourceDatabase); + } + public override async Task Run() { var reportGenerationTime = DateTime.UtcNow; var destinationContainer = _cloudStorageAccount.CreateCloudBlobClient().GetContainerReference(_statisticsContainerName); - Logger.LogDebug("Generating reports from {DataSource}/{InitialCatalog} and saving to {AccountName}/{Container}", _statisticsDbConnectionFactory.DataSource, _statisticsDbConnectionFactory.InitialCatalog, _cloudStorageAccount.Credentials.AccountName, destinationContainer.Name); + Logger.LogDebug("Generating reports from {DataSource}/{InitialCatalog} and saving to {AccountName}/{Container}", + StatisticsDatabase.DataSource, StatisticsDatabase.InitialCatalog, + _cloudStorageAccount.Credentials.AccountName, destinationContainer.Name); var reportBuilderLogger = LoggerFactory.CreateLogger(); var reportCollectorLogger = LoggerFactory.CreateLogger(); @@ -93,12 +100,12 @@ public override async Task Run() // generate all reports var reportGenerators = new Dictionary { - { new ReportBuilder(reportBuilderLogger, ReportNames.NuGetClientVersion), new ReportDataCollector(reportCollectorLogger, _storedProcedures[ReportNames.NuGetClientVersion], _statisticsDbConnectionFactory, _sqlCommandTimeoutSeconds) }, - { new ReportBuilder(reportBuilderLogger, ReportNames.Last6Weeks), new ReportDataCollector(reportCollectorLogger, _storedProcedures[ReportNames.Last6Weeks], _statisticsDbConnectionFactory, _sqlCommandTimeoutSeconds) }, - { new ReportBuilder(reportBuilderLogger, ReportNames.RecentCommunityPopularity), new ReportDataCollector(reportCollectorLogger, _storedProcedures[ReportNames.RecentCommunityPopularity], _statisticsDbConnectionFactory, _sqlCommandTimeoutSeconds) }, - { new ReportBuilder(reportBuilderLogger, ReportNames.RecentCommunityPopularityDetail), new ReportDataCollector(reportCollectorLogger, _storedProcedures[ReportNames.RecentCommunityPopularityDetail], _statisticsDbConnectionFactory, _sqlCommandTimeoutSeconds) }, - { new ReportBuilder(reportBuilderLogger, ReportNames.RecentPopularity), new ReportDataCollector(reportCollectorLogger, _storedProcedures[ReportNames.RecentPopularity], _statisticsDbConnectionFactory, _sqlCommandTimeoutSeconds) }, - { new ReportBuilder(reportBuilderLogger, ReportNames.RecentPopularityDetail), new ReportDataCollector(reportCollectorLogger, _storedProcedures[ReportNames.RecentPopularityDetail], _statisticsDbConnectionFactory, _sqlCommandTimeoutSeconds) } + { new ReportBuilder(reportBuilderLogger, ReportNames.NuGetClientVersion), new ReportDataCollector(OpenStatisticsSqlConnectionAsync, reportCollectorLogger, _storedProcedures[ReportNames.NuGetClientVersion], _sqlCommandTimeoutSeconds) }, + { new ReportBuilder(reportBuilderLogger, ReportNames.Last6Weeks), new ReportDataCollector(OpenStatisticsSqlConnectionAsync, reportCollectorLogger, _storedProcedures[ReportNames.Last6Weeks], _sqlCommandTimeoutSeconds) }, + { new ReportBuilder(reportBuilderLogger, ReportNames.RecentCommunityPopularity), new ReportDataCollector(OpenStatisticsSqlConnectionAsync, reportCollectorLogger, _storedProcedures[ReportNames.RecentCommunityPopularity], _sqlCommandTimeoutSeconds) }, + { new ReportBuilder(reportBuilderLogger, ReportNames.RecentCommunityPopularityDetail), new ReportDataCollector(OpenStatisticsSqlConnectionAsync, reportCollectorLogger, _storedProcedures[ReportNames.RecentCommunityPopularityDetail], _sqlCommandTimeoutSeconds) }, + { new ReportBuilder(reportBuilderLogger, ReportNames.RecentPopularity), new ReportDataCollector(OpenStatisticsSqlConnectionAsync, reportCollectorLogger, _storedProcedures[ReportNames.RecentPopularity], _sqlCommandTimeoutSeconds) }, + { new ReportBuilder(reportBuilderLogger, ReportNames.RecentPopularityDetail), new ReportDataCollector(OpenStatisticsSqlConnectionAsync, reportCollectorLogger, _storedProcedures[ReportNames.RecentPopularityDetail], _sqlCommandTimeoutSeconds) } }; foreach (var reportGenerator in reportGenerators) @@ -114,12 +121,14 @@ public override async Task Run() { // generate only the specific report var reportBuilder = new ReportBuilder(reportBuilderLogger, _reportName); - var reportDataCollector = new ReportDataCollector(reportCollectorLogger, _storedProcedures[_reportName], _statisticsDbConnectionFactory, _sqlCommandTimeoutSeconds); + var reportDataCollector = new ReportDataCollector(OpenStatisticsSqlConnectionAsync, reportCollectorLogger, _storedProcedures[_reportName], _sqlCommandTimeoutSeconds); await ProcessReport(LoggerFactory, destinationContainer, reportBuilder, reportDataCollector, reportGenerationTime); } - Logger.LogInformation("Generated reports from {DataSource}/{InitialCatalog} and saving to {AccountName}/{Container}", _statisticsDbConnectionFactory.DataSource, _statisticsDbConnectionFactory.InitialCatalog, _cloudStorageAccount.Credentials.AccountName, destinationContainer.Name); + Logger.LogInformation("Generated reports from {DataSource}/{InitialCatalog} and saving to {AccountName}/{Container}", + StatisticsDatabase.DataSource, StatisticsDatabase.InitialCatalog, + _cloudStorageAccount.Credentials.AccountName, destinationContainer.Name); // totals reports var stopwatch = Stopwatch.StartNew(); @@ -131,7 +140,7 @@ public override async Task Run() { targets.Add(new StorageContainerTarget(_dataStorageAccount, dataContainerName)); } - var downloadCountReport = new DownloadCountReport(LoggerFactory.CreateLogger(), targets, _statisticsDbConnectionFactory, _galleryDbConnectionFactory); + var downloadCountReport = new DownloadCountReport(OpenStatisticsSqlConnectionAsync, LoggerFactory.CreateLogger(), targets); await downloadCountReport.Run(); stopwatch.Stop(); @@ -140,7 +149,7 @@ public override async Task Run() stopwatch.Restart(); // build stats-totals.json - var galleryTotalsReport = new GalleryTotalsReport(LoggerFactory.CreateLogger(), _cloudStorageAccount, _statisticsContainerName, _statisticsDbConnectionFactory, _galleryDbConnectionFactory); + var galleryTotalsReport = new GalleryTotalsReport(OpenGallerySqlConnectionAsync, OpenStatisticsSqlConnectionAsync, LoggerFactory.CreateLogger(), _cloudStorageAccount, _statisticsContainerName); await galleryTotalsReport.Run(); stopwatch.Stop(); @@ -149,7 +158,7 @@ public override async Task Run() // build tools.v1.json - var toolsReport = new DownloadsPerToolVersionReport(LoggerFactory.CreateLogger(), _cloudStorageAccount, _statisticsContainerName, _statisticsDbConnectionFactory, _galleryDbConnectionFactory); + var toolsReport = new DownloadsPerToolVersionReport(OpenStatisticsSqlConnectionAsync, LoggerFactory.CreateLogger(), _cloudStorageAccount, _statisticsContainerName); await toolsReport.Run(); stopwatch.Stop(); @@ -174,14 +183,14 @@ private static async Task ProcessReport(ILoggerFactory loggerFactory, CloudBlobC private async Task RebuildPackageReports(CloudBlobContainer destinationContainer, DateTime reportGenerationTime) { - var dirtyPackageIds = await ReportDataCollector.GetDirtyPackageIds(LoggerFactory.CreateLogger(), _statisticsDbConnectionFactory, reportGenerationTime, _sqlCommandTimeoutSeconds); + var dirtyPackageIds = await ReportDataCollector.GetDirtyPackageIds(OpenStatisticsSqlConnectionAsync, LoggerFactory.CreateLogger(), reportGenerationTime, _sqlCommandTimeoutSeconds); if (!dirtyPackageIds.Any()) return; // first process the top 100 packages var top100 = dirtyPackageIds.Take(100); - var reportDataCollector = new ReportDataCollector(LoggerFactory.CreateLogger(), _storedProceduresPerPackageId[ReportNames.RecentPopularityDetailByPackageId], _statisticsDbConnectionFactory, _sqlCommandTimeoutSeconds); + var reportDataCollector = new ReportDataCollector(OpenStatisticsSqlConnectionAsync, LoggerFactory.CreateLogger(), _storedProceduresPerPackageId[ReportNames.RecentPopularityDetailByPackageId], _sqlCommandTimeoutSeconds); var top100Task = Parallel.ForEach(top100, new ParallelOptions { MaxDegreeOfParallelism = _perPackageReportDegreeOfParallelism }, dirtyPackageId => { var packageId = dirtyPackageId.PackageId.ToLowerInvariant(); @@ -209,9 +218,9 @@ private async Task RebuildPackageReports(CloudBlobContainer destinationContainer "recentpopularity/" + _recentPopularityDetailByPackageReportBaseName + dirtyPackageId.PackageId.ToLowerInvariant()), new ReportDataCollector( + OpenStatisticsSqlConnectionAsync, LoggerFactory.CreateLogger(), _storedProceduresPerPackageId[ReportNames.RecentPopularityDetailByPackageId], - _statisticsDbConnectionFactory, _sqlCommandTimeoutSeconds) } }; @@ -228,7 +237,7 @@ private async Task RebuildPackageReports(CloudBlobContainer destinationContainer if (top100Task.IsCompleted) { var runToCursor = dirtyPackageIds.First().RunToCuror; - await ReportDataCollector.UpdateDirtyPackageIdCursor(_statisticsDbConnectionFactory, runToCursor, _sqlCommandTimeoutSeconds); + await ReportDataCollector.UpdateDirtyPackageIdCursor(OpenStatisticsSqlConnectionAsync, runToCursor, _sqlCommandTimeoutSeconds); } } } @@ -236,7 +245,7 @@ private async Task RebuildPackageReports(CloudBlobContainer destinationContainer private async Task CleanInactiveRecentPopularityDetailByPackageReports(CloudBlobContainer destinationContainer, DateTime reportGenerationTime) { Logger.LogDebug("Getting list of inactive packages."); - var packageIds = await ReportDataCollector.ListInactivePackageIdReports(_statisticsDbConnectionFactory, reportGenerationTime, _sqlCommandTimeoutSeconds); + var packageIds = await ReportDataCollector.ListInactivePackageIdReports(OpenStatisticsSqlConnectionAsync, reportGenerationTime, _sqlCommandTimeoutSeconds); Logger.LogInformation("Found {InactivePackageCount} inactive packages.", packageIds.Count); // Collect the list of reports diff --git a/src/Stats.CreateAzureCdnWarehouseReports/ReportBase.cs b/src/Stats.CreateAzureCdnWarehouseReports/ReportBase.cs index faef0cce4..121fa1ed5 100644 --- a/src/Stats.CreateAzureCdnWarehouseReports/ReportBase.cs +++ b/src/Stats.CreateAzureCdnWarehouseReports/ReportBase.cs @@ -8,7 +8,6 @@ using Microsoft.Extensions.Logging; using Microsoft.WindowsAzure.Storage.Blob; using Microsoft.WindowsAzure.Storage.RetryPolicies; -using NuGet.Services.Sql; namespace Stats.CreateAzureCdnWarehouseReports { @@ -18,20 +17,12 @@ public abstract class ReportBase protected readonly IReadOnlyCollection Targets; - protected readonly ISqlConnectionFactory StatisticsDbConnectionFactory; - - protected ISqlConnectionFactory GalleryDbConnectionFactory; - protected ReportBase( ILogger logger, - IEnumerable targets, - ISqlConnectionFactory statisticsDbConnectionFactory, - ISqlConnectionFactory galleryDbConnectionFactory) + IEnumerable targets) { _logger = logger; Targets = targets.ToList().AsReadOnly(); - StatisticsDbConnectionFactory = statisticsDbConnectionFactory; - GalleryDbConnectionFactory = galleryDbConnectionFactory; } protected async Task GetBlobContainer(StorageContainerTarget target) diff --git a/src/Stats.CreateAzureCdnWarehouseReports/ReportDataCollector.cs b/src/Stats.CreateAzureCdnWarehouseReports/ReportDataCollector.cs index be58465b4..3f4b4a71d 100644 --- a/src/Stats.CreateAzureCdnWarehouseReports/ReportDataCollector.cs +++ b/src/Stats.CreateAzureCdnWarehouseReports/ReportDataCollector.cs @@ -8,7 +8,6 @@ using System.Diagnostics; using System.Threading.Tasks; using Microsoft.Extensions.Logging; -using NuGet.Services.Sql; namespace Stats.CreateAzureCdnWarehouseReports { @@ -16,19 +15,20 @@ internal class ReportDataCollector { private int _commandTimeoutSeconds; private readonly string _procedureName; - private readonly ISqlConnectionFactory _sourceDbConnectionFactory; private ILogger _logger; + private Func> OpenStatisticsSqlConnectionAsync { get; } + public ReportDataCollector( + Func> openStatisticsSqlConnectionAsync, ILogger logger, string procedureName, - ISqlConnectionFactory sourceDbConnectionFactory, int timeout) { + OpenStatisticsSqlConnectionAsync = openStatisticsSqlConnectionAsync; _logger = logger; _procedureName = procedureName; - _sourceDbConnectionFactory = sourceDbConnectionFactory; _commandTimeoutSeconds = timeout; } @@ -47,8 +47,8 @@ public async Task CollectAsync(DateTime reportGenerationTime, params } public static async Task> GetDirtyPackageIds( + Func> openStatisticsSqlConnectionAsync, ILogger logger, - ISqlConnectionFactory sourceDbConnectionFactory, DateTime reportGenerationTime, int commandTimeout) { @@ -57,7 +57,8 @@ public static async Task> GetDirtyPackageIds IReadOnlyCollection packageIds = new List(); // Get the data - await WithRetry(async () => packageIds = await GetDirtyPackageIdsFromWarehouse(sourceDbConnectionFactory, reportGenerationTime, commandTimeout), logger); + await WithRetry(async () => packageIds = await GetDirtyPackageIdsFromWarehouse( + openStatisticsSqlConnectionAsync, reportGenerationTime, commandTimeout), logger); logger.LogInformation("Found {DirtyPackagesCount} dirty packages to update.", packageIds.Count); @@ -65,11 +66,11 @@ public static async Task> GetDirtyPackageIds } public static async Task> ListInactivePackageIdReports( - ISqlConnectionFactory sourceDbConnectionFactory, + Func> openStatisticsSqlConnectionAsync, DateTime reportGenerationTime, int commandTimeout) { - using (var connection = await sourceDbConnectionFactory.CreateAsync()) + using (var connection = await openStatisticsSqlConnectionAsync()) { var command = new SqlCommand("[dbo].[DownloadReportListInactive]", connection); command.CommandType = CommandType.StoredProcedure; @@ -122,7 +123,7 @@ private static async Task WithRetry(Func action, ILogger logger) private async Task ExecuteSql(DateTime reportGenerationTime, params Tuple[] parameters) { - using (var connection = await _sourceDbConnectionFactory.CreateAsync()) + using (var connection = await OpenStatisticsSqlConnectionAsync()) { var command = new SqlCommand(_procedureName, connection); command.CommandType = CommandType.StoredProcedure; @@ -146,11 +147,11 @@ private async Task ExecuteSql(DateTime reportGenerationTime, params T } private static async Task> GetDirtyPackageIdsFromWarehouse( - ISqlConnectionFactory sourceDbConnectionFactory, + Func> openStatisticsSqlConnectionAsync, DateTime reportGenerationTime, int commandTimeout) { - using (var connection = await sourceDbConnectionFactory.CreateAsync()) + using (var connection = await openStatisticsSqlConnectionAsync()) { var command = new SqlCommand("[dbo].[GetDirtyPackageIds]", connection); command.CommandType = CommandType.StoredProcedure; @@ -172,11 +173,11 @@ private static async Task> GetDirtyPackageId } public static async Task UpdateDirtyPackageIdCursor( - ISqlConnectionFactory sourceDbConnectionFactory, + Func> openStatisticsSqlConnectionAsync, DateTime runToCursor, int commandTimeout) { - using (var connection = await sourceDbConnectionFactory.CreateAsync()) + using (var connection = await openStatisticsSqlConnectionAsync()) { var command = new SqlCommand("[dbo].[UpdateDirtyPackageIdCursor]", connection); command.CommandType = CommandType.StoredProcedure; diff --git a/src/Stats.CreateAzureCdnWarehouseReports/Stats.CreateAzureCdnWarehouseReports.csproj b/src/Stats.CreateAzureCdnWarehouseReports/Stats.CreateAzureCdnWarehouseReports.csproj index 0b09db277..5181875fb 100644 --- a/src/Stats.CreateAzureCdnWarehouseReports/Stats.CreateAzureCdnWarehouseReports.csproj +++ b/src/Stats.CreateAzureCdnWarehouseReports/Stats.CreateAzureCdnWarehouseReports.csproj @@ -99,9 +99,6 @@ 2.25.0 - - 2.25.0-master-30453 - 4.3.0-preview1-2524 diff --git a/src/Stats.ImportAzureCdnStatistics/DataImporter.cs b/src/Stats.ImportAzureCdnStatistics/DataImporter.cs index 8f0a9251b..fbe92005c 100644 --- a/src/Stats.ImportAzureCdnStatistics/DataImporter.cs +++ b/src/Stats.ImportAzureCdnStatistics/DataImporter.cs @@ -1,21 +1,22 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +using System; using System.Data; using System.Data.SqlClient; using System.Threading.Tasks; -using NuGet.Services.Sql; namespace Stats.ImportAzureCdnStatistics { internal class DataImporter { - private readonly ISqlConnectionFactory _statisticsDbConnectionFactory; private const string _sqlSelectTop1FromTable = "SELECT TOP 1 * FROM [dbo].[{0}]"; - public DataImporter(ISqlConnectionFactory statisticsDbConnectionFactory) + public Func> OpenStatisticsSqlConnectionAsync { get; } + + public DataImporter(Func> openStatisticsSqlConnectionAsync) { - _statisticsDbConnectionFactory = statisticsDbConnectionFactory; + OpenStatisticsSqlConnectionAsync = openStatisticsSqlConnectionAsync; } public async Task GetDataTableAsync(string tableName) @@ -23,7 +24,7 @@ public async Task GetDataTableAsync(string tableName) var dataTable = new DataTable(); var query = string.Format(_sqlSelectTop1FromTable, tableName); - using (var connection = await _statisticsDbConnectionFactory.CreateAsync()) + using (var connection = await OpenStatisticsSqlConnectionAsync()) { var tableAdapter = new SqlDataAdapter(query, connection) { diff --git a/src/Stats.ImportAzureCdnStatistics/Job.cs b/src/Stats.ImportAzureCdnStatistics/Job.cs index 6b0b18119..5f82c0762 100644 --- a/src/Stats.ImportAzureCdnStatistics/Job.cs +++ b/src/Stats.ImportAzureCdnStatistics/Job.cs @@ -10,9 +10,8 @@ using Microsoft.WindowsAzure.Storage.Blob; using Microsoft.WindowsAzure.Storage.RetryPolicies; using NuGet.Jobs; -using NuGet.Services.KeyVault; -using NuGet.Services.Sql; using Stats.AzureCdnLogs.Common; +using System.Data.SqlClient; namespace Stats.ImportAzureCdnStatistics { @@ -23,16 +22,13 @@ public class Job private string _azureCdnAccountNumber; private string _cloudStorageContainerName; private AzureCdnPlatform _azureCdnPlatform; - private ISqlConnectionFactory _statisticsDbConnectionFactory; private CloudStorageAccount _cloudStorageAccount; private CloudBlobClient _cloudBlobClient; private LogFileProvider _blobLeaseManager; public override void Init(IServiceContainer serviceContainer, IDictionary jobArgsDictionary) { - var secretInjector = (ISecretInjector)serviceContainer.GetService(typeof(ISecretInjector)); - var statisticsDbConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatisticsDatabase); - _statisticsDbConnectionFactory = new AzureSqlConnectionFactory(statisticsDbConnectionString, secretInjector); + RegisterDatabase(serviceContainer, jobArgsDictionary, JobArgumentNames.StatisticsDatabase); var azureCdnPlatform = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.AzureCdnPlatform); var cloudStorageAccountConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.AzureCdnCloudStorageAccount); @@ -54,6 +50,11 @@ public override void Init(IServiceContainer serviceContainer, IDictionary OpenStatisticsSqlConnectionAsync() + { + return OpenSqlConnectionAsync(JobArgumentNames.StatisticsDatabase); + } + public override async Task Run() { // Get the target blob container (for archiving decompressed log files) @@ -65,7 +66,7 @@ public override async Task Run() await deadLetterBlobContainer.CreateIfNotExistsAsync(); // Create a parser - var warehouse = new Warehouse(LoggerFactory, _statisticsDbConnectionFactory); + var warehouse = new Warehouse(OpenStatisticsSqlConnectionAsync, LoggerFactory); var statisticsBlobContainerUtility = new StatisticsBlobContainerUtility( targetBlobContainer, deadLetterBlobContainer, diff --git a/src/Stats.ImportAzureCdnStatistics/Stats.ImportAzureCdnStatistics.csproj b/src/Stats.ImportAzureCdnStatistics/Stats.ImportAzureCdnStatistics.csproj index 6ba8e18cd..47ace8b1a 100644 --- a/src/Stats.ImportAzureCdnStatistics/Stats.ImportAzureCdnStatistics.csproj +++ b/src/Stats.ImportAzureCdnStatistics/Stats.ImportAzureCdnStatistics.csproj @@ -125,9 +125,6 @@ 2.25.0 - - 2.25.0-master-30453 - 4.3.0-preview1-2524 diff --git a/src/Stats.ImportAzureCdnStatistics/Warehouse.cs b/src/Stats.ImportAzureCdnStatistics/Warehouse.cs index 6e4c7bcae..e84718fc1 100644 --- a/src/Stats.ImportAzureCdnStatistics/Warehouse.cs +++ b/src/Stats.ImportAzureCdnStatistics/Warehouse.cs @@ -8,7 +8,6 @@ using System.Linq; using System.Threading.Tasks; using Microsoft.Extensions.Logging; -using NuGet.Services.Sql; using Stats.AzureCdnLogs.Common; using Stopwatch = System.Diagnostics.Stopwatch; @@ -21,7 +20,6 @@ internal class Warehouse private const int _maxRetryCount = 3; private readonly TimeSpan _retryDelay = TimeSpan.FromSeconds(5); private readonly ILogger _logger; - private readonly ISqlConnectionFactory _statisticsDbConnectionFactory; private readonly IDictionary _cachedPackageDimensions = new Dictionary(); private readonly IList _cachedToolDimensions = new List(); private readonly IDictionary _cachedClientDimensions = new Dictionary(); @@ -31,15 +29,17 @@ internal class Warehouse private readonly IDictionary _cachedIpAddressFacts = new Dictionary(); private IReadOnlyCollection _times; - public Warehouse(ILoggerFactory loggerFactory, ISqlConnectionFactory statisticsDbConnectionFactory) + private Func> OpenStatisticsSqlConnectionAsync { get; } + + public Warehouse(Func> openStatisticsSqlConnectionAsync, ILoggerFactory loggerFactory) { if (loggerFactory == null) { throw new ArgumentNullException(nameof(loggerFactory)); } + OpenStatisticsSqlConnectionAsync = openStatisticsSqlConnectionAsync; _logger = loggerFactory.CreateLogger(); - _statisticsDbConnectionFactory = statisticsDbConnectionFactory ?? throw new ArgumentNullException(nameof(statisticsDbConnectionFactory)); } public async Task InsertDownloadFactsAsync(DataTable downloadFactsDataTable, string logFileName) @@ -47,7 +47,7 @@ public async Task InsertDownloadFactsAsync(DataTable downloadFactsDataTable, str _logger.LogDebug("Inserting into facts table..."); var stopwatch = Stopwatch.StartNew(); - using (var connection = await _statisticsDbConnectionFactory.CreateAsync()) + using (var connection = await OpenStatisticsSqlConnectionAsync()) using (var transaction = connection.BeginTransaction(IsolationLevel.Snapshot)) { try @@ -124,7 +124,7 @@ public async Task CreateAsync(IReadOnlyCollection var packages = packagesTask.Result; // create facts data rows by linking source data with dimensions - var dataImporter = new DataImporter(_statisticsDbConnectionFactory); + var dataImporter = new DataImporter(OpenStatisticsSqlConnectionAsync); var factsDataTable = await dataImporter.GetDataTableAsync("Fact_Download"); var knownOperationsAvailable = operations.Any(); @@ -245,7 +245,7 @@ public async Task CreateAsync(IReadOnlyCollection sou var ipAddresses = ipAddressesTask.Result; // create facts data rows by linking source data with dimensions - var dataImporter = new DataImporter(_statisticsDbConnectionFactory); + var dataImporter = new DataImporter(OpenStatisticsSqlConnectionAsync); var dataTable = await dataImporter.GetDataTableAsync("Fact_Dist_Download"); var knownClientsAvailable = clients.Any(); @@ -341,7 +341,7 @@ public async Task StoreLogFileAggregatesAsync(LogFileAggregates logFileAggregate { _logger.LogDebug("Storing log file aggregates..."); - using (var connection = await _statisticsDbConnectionFactory.CreateAsync()) + using (var connection = await OpenStatisticsSqlConnectionAsync()) { try { @@ -375,7 +375,7 @@ public async Task> GetAlreadyAggregatedLogFilesAsync _logger.LogDebug("Retrieving already processed log files..."); var alreadyAggregatedLogFiles = new List(); - using (var connection = await _statisticsDbConnectionFactory.CreateAsync()) + using (var connection = await OpenStatisticsSqlConnectionAsync()) { try { @@ -433,7 +433,7 @@ private async Task HasImportedStatisticsAsync(string logFileName, string c try { - using (var connection = await _statisticsDbConnectionFactory.CreateAsync()) + using (var connection = await OpenStatisticsSqlConnectionAsync()) { var command = connection.CreateCommand(); command.CommandText = commandText; @@ -474,7 +474,7 @@ private async Task> GetDimension(string dimension, stri _logger.LogDebug("Beginning to retrieve dimension '{Dimension}'.", dimension); IDictionary dimensions; - using (var connection = await _statisticsDbConnectionFactory.CreateAsync()) + using (var connection = await OpenStatisticsSqlConnectionAsync()) { dimensions = await retrieve(connection); } @@ -546,7 +546,7 @@ private async Task> GetDimension(string dimension, str _logger.LogDebug("Beginning to retrieve dimension '{Dimension}'.", dimension); IReadOnlyCollection dimensions; - using (var connection = await _statisticsDbConnectionFactory.CreateAsync()) + using (var connection = await OpenStatisticsSqlConnectionAsync()) { dimensions = await retrieve(connection); } diff --git a/src/Stats.RefreshClientDimension/RefreshClientDimensionJob.cs b/src/Stats.RefreshClientDimension/RefreshClientDimensionJob.cs index a10b412c3..5bc3fb1b7 100644 --- a/src/Stats.RefreshClientDimension/RefreshClientDimensionJob.cs +++ b/src/Stats.RefreshClientDimension/RefreshClientDimensionJob.cs @@ -8,23 +8,18 @@ using System.Threading.Tasks; using Microsoft.Extensions.Logging; using NuGet.Jobs; -using NuGet.Services.KeyVault; -using NuGet.Services.Sql; using Stats.ImportAzureCdnStatistics; namespace Stats.RefreshClientDimension { public class RefreshClientDimensionJob : JobBase { - private static ISqlConnectionFactory _statisticsDbConnectionFactory; private static string _targetClientName; private static string _userAgentFilter; public override void Init(IServiceContainer serviceContainer, IDictionary jobArgsDictionary) { - var secretInjector = (ISecretInjector)serviceContainer.GetService(typeof(ISecretInjector)); - var statisticsDbConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatisticsDatabase); - _statisticsDbConnectionFactory = new AzureSqlConnectionFactory(statisticsDbConnectionString, secretInjector); + RegisterDatabase(serviceContainer, jobArgsDictionary, JobArgumentNames.StatisticsDatabase); _targetClientName = JobConfigurationManager.TryGetArgument(jobArgsDictionary, "TargetClientName"); _userAgentFilter = JobConfigurationManager.TryGetArgument(jobArgsDictionary, "UserAgentFilter"); @@ -32,7 +27,7 @@ public override void Init(IServiceContainer serviceContainer, IDictionary> linkedUserAgents; diff --git a/src/Stats.RefreshClientDimension/Stats.RefreshClientDimension.csproj b/src/Stats.RefreshClientDimension/Stats.RefreshClientDimension.csproj index 980174366..1de4656f1 100644 --- a/src/Stats.RefreshClientDimension/Stats.RefreshClientDimension.csproj +++ b/src/Stats.RefreshClientDimension/Stats.RefreshClientDimension.csproj @@ -83,9 +83,6 @@ 9.0.1 - - 2.25.0-master-30453 - 1.2.0 diff --git a/src/Stats.RollUpDownloadFacts/Job.cs b/src/Stats.RollUpDownloadFacts/Job.cs index 76cd61979..51d3dcc40 100644 --- a/src/Stats.RollUpDownloadFacts/Job.cs +++ b/src/Stats.RollUpDownloadFacts/Job.cs @@ -9,8 +9,6 @@ using System.Threading.Tasks; using Microsoft.Extensions.Logging; using NuGet.Jobs; -using NuGet.Services.KeyVault; -using NuGet.Services.Sql; namespace Stats.RollUpDownloadFacts { @@ -21,13 +19,10 @@ public class Job private const string _endTemplateFactDownloadDeletion = " records from [dbo].[Fact_Download]"; private const int DefaultMinAgeInDays = 43; private static int _minAgeInDays; - private static ISqlConnectionFactory _statisticsDbConnectionFactory; public override void Init(IServiceContainer serviceContainer, IDictionary jobArgsDictionary) { - var secretInjector = (ISecretInjector)serviceContainer.GetService(typeof(ISecretInjector)); - var statisticsDbConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatisticsDatabase); - _statisticsDbConnectionFactory = new AzureSqlConnectionFactory(statisticsDbConnectionString, secretInjector); + RegisterDatabase(serviceContainer, jobArgsDictionary, JobArgumentNames.StatisticsDatabase); _minAgeInDays = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, JobArgumentNames.MinAgeInDays) ?? DefaultMinAgeInDays; Logger.LogInformation("Min age in days: {MinAgeInDays}", _minAgeInDays); @@ -35,7 +30,7 @@ public override void Init(IServiceContainer serviceContainer, IDictionary 2.25.0 - - 2.25.0-master-30453 - diff --git a/src/UpdateLicenseReports/UpdateLicenseReports.Job.cs b/src/UpdateLicenseReports/UpdateLicenseReports.Job.cs index 0a3fb2fad..d012fc1de 100644 --- a/src/UpdateLicenseReports/UpdateLicenseReports.Job.cs +++ b/src/UpdateLicenseReports/UpdateLicenseReports.Job.cs @@ -14,8 +14,6 @@ using Newtonsoft.Json.Linq; using Newtonsoft.Json.Schema; using NuGet.Jobs; -using NuGet.Services.KeyVault; -using NuGet.Services.Sql; namespace UpdateLicenseReports { @@ -41,7 +39,6 @@ internal class Job : JobBase private Uri _licenseReportService; private string _licenseReportUser; private string _licenseReportPassword; - private ISqlConnectionFactory _packageDbConnectionFactory; private int? _retryCount; private NetworkCredential _licenseReportCredentials; @@ -61,9 +58,7 @@ private static PackageLicenseReport CreateReport(JObject messageEvent) public override void Init(IServiceContainer serviceContainer, IDictionary jobArgsDictionary) { - var secretInjector = (ISecretInjector)serviceContainer.GetService(typeof(ISecretInjector)); - var dbConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.PackageDatabase); - _packageDbConnectionFactory = new AzureSqlConnectionFactory(dbConnectionString, secretInjector); + RegisterDatabase(serviceContainer, jobArgsDictionary, JobArgumentNames.PackageDatabase); var retryCountString = JobConfigurationManager.TryGetArgument(jobArgsDictionary, JobArgumentNames.RetryCount); if (string.IsNullOrEmpty(retryCountString)) @@ -111,12 +106,12 @@ public override async Task Run() private async Task FetchNextReportUrlAsync() { - Logger.LogInformation("Fetching next report URL from {DataSource}/{InitialCatalog}", - _packageDbConnectionFactory.DataSource, _packageDbConnectionFactory.InitialCatalog); - Uri nextLicenseReport = null; - using (var connection = await _packageDbConnectionFactory.CreateAsync()) + using (var connection = await OpenSqlConnectionAsync(JobArgumentNames.PackageDatabase)) { + Logger.LogInformation("Fetching next report URL from {DataSource}/{InitialCatalog}", + connection.DataSource, connection.Database); + var nextReportUrl = (await connection.QueryAsync( @"SELECT TOP 1 NextLicenseReport FROM GallerySettings")).SingleOrDefault(); @@ -130,11 +125,11 @@ private async Task FetchNextReportUrlAsync() } nextLicenseReport = nextLicenseReport ?? _licenseReportService; - } - Logger.LogInformation("Fetched next report URL '{NextReportUrl}' from {DataSource}/{InitialCatalog}", - (nextLicenseReport == null ? string.Empty : nextLicenseReport.AbsoluteUri), - _packageDbConnectionFactory.DataSource, _packageDbConnectionFactory.InitialCatalog); + Logger.LogInformation("Fetched next report URL '{NextReportUrl}' from {DataSource}/{InitialCatalog}", + (nextLicenseReport == null ? string.Empty : nextLicenseReport.AbsoluteUri), + connection.DataSource, connection.Database); + } return nextLicenseReport; } @@ -242,7 +237,7 @@ private async Task ProcessReportsAsync(Uri nextLicenseReport) Logger.LogInformation("Storing next license report URL: {NextReportUrl}", nextLicenseReport.AbsoluteUri); // Record the next report to the database so we can check it again if we get aborted before finishing. - using (var connection = await _packageDbConnectionFactory.CreateAsync()) + using (var connection = await OpenSqlConnectionAsync(JobArgumentNames.PackageDatabase)) { await connection.QueryAsync(@" UPDATE GallerySettings @@ -274,7 +269,7 @@ UPDATE GallerySettings private async Task StoreReportAsync(PackageLicenseReport report) { - using (var connection = await _packageDbConnectionFactory.CreateAsync()) + using (var connection = await OpenSqlConnectionAsync(JobArgumentNames.PackageDatabase)) using (var command = connection.CreateCommand()) { command.CommandText = "AddPackageLicenseReport2"; diff --git a/src/UpdateLicenseReports/UpdateLicenseReports.csproj b/src/UpdateLicenseReports/UpdateLicenseReports.csproj index 0d9b7210c..6a824cd81 100644 --- a/src/UpdateLicenseReports/UpdateLicenseReports.csproj +++ b/src/UpdateLicenseReports/UpdateLicenseReports.csproj @@ -74,9 +74,6 @@ 2.0.10 - - 2.25.0-master-30453 - diff --git a/src/Validation.Common.Job/JsonConfigurationJob.cs b/src/Validation.Common.Job/JsonConfigurationJob.cs index 9355f050c..7cc47941f 100644 --- a/src/Validation.Common.Job/JsonConfigurationJob.cs +++ b/src/Validation.Common.Job/JsonConfigurationJob.cs @@ -4,12 +4,10 @@ using System; using System.Collections.Generic; using System.ComponentModel.Design; -using System.Data.Common; using System.IO; using System.Net; using System.Net.Http; using System.Reflection; -using System.Threading.Tasks; using Autofac; using Autofac.Extensions.DependencyInjection; using Microsoft.ApplicationInsights; @@ -22,7 +20,6 @@ using NuGet.Services.KeyVault; using NuGet.Services.Logging; using NuGet.Services.ServiceBus; -using NuGet.Services.Sql; using NuGet.Services.Validation; using NuGetGallery; using NuGetGallery.Diagnostics; @@ -64,6 +61,9 @@ public override void Init(IServiceContainer serviceContainer, IDictionary(_serviceProvider); + RegisterDatabase(_serviceProvider); + ServicePointManager.DefaultConnectionLimit = MaximumConnectionsPerServer; } @@ -109,15 +109,6 @@ private IServiceProvider GetServiceProvider(IConfigurationRoot configurationRoot return new AutofacServiceProvider(containerBuilder.Build()); } - protected virtual DbConnection CreateDbConnection(IServiceProvider serviceProvider) where T : IDbConfiguration - { - var connectionString = serviceProvider.GetRequiredService>().Value.ConnectionString; - var connectionFactory = new AzureSqlConnectionFactory(connectionString, - serviceProvider.GetRequiredService()); - - return Task.Run(() => connectionFactory.CreateAsync()).Result; - } - private void ConfigureDefaultJobServices(IServiceCollection services, IConfigurationRoot configurationRoot) { services.Configure(configurationRoot.GetSection(GalleryDbConfigurationSectionName)); @@ -142,12 +133,12 @@ private void ConfigureDefaultJobServices(IServiceCollection services, IConfigura services.AddScoped(p => { - return new ValidationEntitiesContext(CreateDbConnection(p)); + return new ValidationEntitiesContext(CreateSqlConnection()); }); services.AddScoped(p => { - return new EntitiesContext(CreateDbConnection(p), readOnly: true); + return new EntitiesContext(CreateSqlConnection(), readOnly: true); }); services.AddTransient(p => diff --git a/src/Validation.Common.Job/Validation.Common.Job.csproj b/src/Validation.Common.Job/Validation.Common.Job.csproj index fa1a131a8..15b0f761e 100644 --- a/src/Validation.Common.Job/Validation.Common.Job.csproj +++ b/src/Validation.Common.Job/Validation.Common.Job.csproj @@ -102,9 +102,6 @@ 2.27.0 - - 2.27.0 - 2.27.0 diff --git a/src/Validation.PackageSigning.ProcessSignature/Job.cs b/src/Validation.PackageSigning.ProcessSignature/Job.cs index 16997f935..14f9e5bdc 100644 --- a/src/Validation.PackageSigning.ProcessSignature/Job.cs +++ b/src/Validation.PackageSigning.ProcessSignature/Job.cs @@ -35,7 +35,7 @@ protected override void ConfigureJobServices(IServiceCollection services, IConfi services.AddScoped(p => { - return new EntitiesContext(CreateDbConnection(p), readOnly: false); + return new EntitiesContext(CreateSqlConnection(), readOnly: false); }); services.Add(ServiceDescriptor.Transient(typeof(IEntityRepository<>), typeof(EntityRepository<>))); diff --git a/src/Validation.PackageSigning.ProcessSignature/Validation.PackageSigning.ProcessSignature.csproj b/src/Validation.PackageSigning.ProcessSignature/Validation.PackageSigning.ProcessSignature.csproj index 3d331e9f9..1ae9d197a 100644 --- a/src/Validation.PackageSigning.ProcessSignature/Validation.PackageSigning.ProcessSignature.csproj +++ b/src/Validation.PackageSigning.ProcessSignature/Validation.PackageSigning.ProcessSignature.csproj @@ -85,11 +85,6 @@ Validation.PackageSigning.Core - - - 2.27.0 - - ..\..\build diff --git a/tests/NuGet.Services.Revalidate.Tests/NuGet.Services.Revalidate.Tests.csproj b/tests/NuGet.Services.Revalidate.Tests/NuGet.Services.Revalidate.Tests.csproj index a09ec1eea..172a9d5c3 100644 --- a/tests/NuGet.Services.Revalidate.Tests/NuGet.Services.Revalidate.Tests.csproj +++ b/tests/NuGet.Services.Revalidate.Tests/NuGet.Services.Revalidate.Tests.csproj @@ -82,5 +82,8 @@ Tests.ContextHelpers + + + \ No newline at end of file diff --git a/tests/Tests.Search.GenerateAuxiliaryData/RankingsExporterTests.cs b/tests/Tests.Search.GenerateAuxiliaryData/RankingsExporterTests.cs index 9a202115a..9e5cb51d7 100644 --- a/tests/Tests.Search.GenerateAuxiliaryData/RankingsExporterTests.cs +++ b/tests/Tests.Search.GenerateAuxiliaryData/RankingsExporterTests.cs @@ -3,11 +3,12 @@ using System; using System.Data; +using System.Data.SqlClient; +using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Microsoft.WindowsAzure.Storage.Blob; using Moq; using Newtonsoft.Json; -using NuGet.Services.Sql; using Search.GenerateAuxiliaryData; using Xunit; @@ -44,11 +45,16 @@ public void GetRankings_ReturnsRankings() private static RankingsExporter CreateExporter() { return new RankingsExporter( + DoNotOpenSqlConnectionAsync, new LoggerFactory().CreateLogger(), - connectionFactory: new Mock().Object, defaultDestinationContainer: new CloudBlobContainer(new Uri("https://nuget.org")), defaultRankingsScript: "b", defaultName: "c"); } + + public static Task DoNotOpenSqlConnectionAsync() + { + return Task.FromResult((SqlConnection)null); + } } } \ No newline at end of file diff --git a/tests/Tests.Search.GenerateAuxiliaryData/VerifiedPackagesExporterTests.cs b/tests/Tests.Search.GenerateAuxiliaryData/VerifiedPackagesExporterTests.cs index 50b8aea23..c31f36116 100644 --- a/tests/Tests.Search.GenerateAuxiliaryData/VerifiedPackagesExporterTests.cs +++ b/tests/Tests.Search.GenerateAuxiliaryData/VerifiedPackagesExporterTests.cs @@ -3,11 +3,12 @@ using System; using System.Data; +using System.Data.SqlClient; +using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Microsoft.WindowsAzure.Storage.Blob; using Moq; using Newtonsoft.Json; -using NuGet.Services.Sql; using Search.GenerateAuxiliaryData; using Xunit; @@ -43,11 +44,16 @@ public void GetVerifiedPackagesReturnsJsonString() private static VerifiedPackagesExporter CreateExporter() { return new VerifiedPackagesExporter( + DoNotOpenSqlConnectionAsync, new LoggerFactory().CreateLogger(), - connectionFactory: new Mock().Object, defaultDestinationContainer: new CloudBlobContainer(new Uri("https://nuget.org")), defaultVerifiedPackagesScript: "b", defaultName: "c"); } + + public static Task DoNotOpenSqlConnectionAsync() + { + return Task.FromResult((SqlConnection)null); + } } } \ No newline at end of file diff --git a/tests/Validation.PackageSigning.ScanAndSign.Tests/Validation.PackageSigning.ScanAndSign.Tests.csproj b/tests/Validation.PackageSigning.ScanAndSign.Tests/Validation.PackageSigning.ScanAndSign.Tests.csproj index 510501a59..b0662efcb 100644 --- a/tests/Validation.PackageSigning.ScanAndSign.Tests/Validation.PackageSigning.ScanAndSign.Tests.csproj +++ b/tests/Validation.PackageSigning.ScanAndSign.Tests/Validation.PackageSigning.ScanAndSign.Tests.csproj @@ -75,5 +75,8 @@ 2.3.1 + + + \ No newline at end of file