diff --git a/src/Stats.CreateAzureCdnWarehouseReports/Configuration/CreateAzureCdnWarehouseReportsConfiguration.cs b/src/Stats.CreateAzureCdnWarehouseReports/Configuration/CreateAzureCdnWarehouseReportsConfiguration.cs new file mode 100644 index 000000000..642eeaa96 --- /dev/null +++ b/src/Stats.CreateAzureCdnWarehouseReports/Configuration/CreateAzureCdnWarehouseReportsConfiguration.cs @@ -0,0 +1,22 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +namespace Stats.CreateAzureCdnWarehouseReports +{ + public class CreateAzureCdnWarehouseReportsConfiguration + { + public string AzureCdnCloudStorageAccount { get; set; } + + public string AzureCdnCloudStorageContainerName { get; set; } + + public string DataStorageAccount { get; set; } + + public string DataContainerName { get; set; } + + public int? CommandTimeOut { get; set; } + + public int? PerPackageReportDegreeOfParallelism { get; set; } + + public string ReportName { get; set; } + } +} diff --git a/src/Stats.CreateAzureCdnWarehouseReports/Job.cs b/src/Stats.CreateAzureCdnWarehouseReports/CreateAzureCdnWarehouseReportsJob.cs similarity index 61% rename from src/Stats.CreateAzureCdnWarehouseReports/Job.cs rename to src/Stats.CreateAzureCdnWarehouseReports/CreateAzureCdnWarehouseReportsJob.cs index 611390958..636d1a7eb 100644 --- a/src/Stats.CreateAzureCdnWarehouseReports/Job.cs +++ b/src/Stats.CreateAzureCdnWarehouseReports/CreateAzureCdnWarehouseReportsJob.cs @@ -4,29 +4,31 @@ using System; using System.Collections.Generic; using System.ComponentModel.Design; +using System.Data.SqlClient; using System.Linq; using System.Threading.Tasks; +using Autofac; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; using Microsoft.WindowsAzure.Storage; using Microsoft.WindowsAzure.Storage.Blob; using NuGet.Jobs; -using NuGet.Services.KeyVault; -using NuGet.Services.Sql; +using NuGet.Jobs.Configuration; using Stopwatch = System.Diagnostics.Stopwatch; namespace Stats.CreateAzureCdnWarehouseReports { - public class Job - : JobBase + public class CreateAzureCdnWarehouseReportsJob : JsonConfigurationJob { private const int DefaultPerPackageReportDegreeOfParallelism = 8; // Generate private const int DefaultSqlCommandTimeoutSeconds = 1800; // 30 minute SQL command timeout by default private const string _recentPopularityDetailByPackageReportBaseName = "recentpopularitydetail_"; + private CloudStorageAccount _cloudStorageAccount; private CloudStorageAccount _dataStorageAccount; private string _statisticsContainerName; - private ISqlConnectionFactory _statisticsDbConnectionFactory; - private ISqlConnectionFactory _galleryDbConnectionFactory; private string _reportName; private string[] _dataContainerNames; private int _sqlCommandTimeoutSeconds = DefaultSqlCommandTimeoutSeconds; @@ -34,45 +36,51 @@ public class Job private static readonly IDictionary _storedProcedures = new Dictionary { - {ReportNames.NuGetClientVersion, "[dbo].[DownloadReportNuGetClientVersion]" }, - {ReportNames.Last6Weeks, "[dbo].[DownloadReportLast6Weeks]" }, - {ReportNames.RecentCommunityPopularity, "[dbo].[DownloadReportRecentCommunityPopularity]" }, - {ReportNames.RecentCommunityPopularityDetail, "[dbo].[DownloadReportRecentCommunityPopularityDetail]" }, - {ReportNames.RecentPopularity, "[dbo].[DownloadReportRecentPopularity]" }, - {ReportNames.RecentPopularityDetail, "[dbo].[DownloadReportRecentPopularityDetail]" }, + { ReportNames.NuGetClientVersion, "[dbo].[DownloadReportNuGetClientVersion]" }, + { ReportNames.Last6Weeks, "[dbo].[DownloadReportLast6Weeks]" }, + { ReportNames.RecentCommunityPopularity, "[dbo].[DownloadReportRecentCommunityPopularity]" }, + { ReportNames.RecentCommunityPopularityDetail, "[dbo].[DownloadReportRecentCommunityPopularityDetail]" }, + { ReportNames.RecentPopularity, "[dbo].[DownloadReportRecentPopularity]" }, + { ReportNames.RecentPopularityDetail, "[dbo].[DownloadReportRecentPopularityDetail]" }, }; private static readonly IDictionary _storedProceduresPerPackageId = new Dictionary { - {ReportNames.RecentPopularityDetailByPackageId, "[dbo].[DownloadReportRecentPopularityDetailByPackage]" } + { ReportNames.RecentPopularityDetailByPackageId, "[dbo].[DownloadReportRecentPopularityDetailByPackage]" } }; public override void Init(IServiceContainer serviceContainer, IDictionary jobArgsDictionary) { - var secretInjector = (ISecretInjector)serviceContainer.GetService(typeof(ISecretInjector)); - _sqlCommandTimeoutSeconds = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, JobArgumentNames.CommandTimeOut) ?? DefaultSqlCommandTimeoutSeconds; + base.Init(serviceContainer, jobArgsDictionary); + + var configuration = _serviceProvider.GetRequiredService>().Value; + + _sqlCommandTimeoutSeconds = configuration.CommandTimeOut ?? DefaultSqlCommandTimeoutSeconds; + + _perPackageReportDegreeOfParallelism = configuration.PerPackageReportDegreeOfParallelism ?? DefaultPerPackageReportDegreeOfParallelism; - var statisticsDatabaseConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatisticsDatabase); - _statisticsDbConnectionFactory = new AzureSqlConnectionFactory(statisticsDatabaseConnectionString, secretInjector); + _cloudStorageAccount = ValidateAzureCloudStorageAccount( + configuration.AzureCdnCloudStorageAccount, + nameof(configuration.AzureCdnCloudStorageAccount)); - var galleryDatabaseConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.SourceDatabase); - _galleryDbConnectionFactory = new AzureSqlConnectionFactory(galleryDatabaseConnectionString, secretInjector); + _statisticsContainerName = ValidateAzureContainerName( + configuration.AzureCdnCloudStorageContainerName, + nameof(configuration.AzureCdnCloudStorageContainerName)); - var cloudStorageAccountConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.AzureCdnCloudStorageAccount); - var dataStorageAccountConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.DataStorageAccount); - _perPackageReportDegreeOfParallelism = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, JobArgumentNames.PerPackageReportDegreeOfParallelism) ?? DefaultPerPackageReportDegreeOfParallelism; + _dataStorageAccount = ValidateAzureCloudStorageAccount( + configuration.DataStorageAccount, + nameof(configuration.DataStorageAccount)); - _cloudStorageAccount = ValidateAzureCloudStorageAccount(cloudStorageAccountConnectionString, JobArgumentNames.AzureCdnCloudStorageAccount); - _statisticsContainerName = ValidateAzureContainerName(JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.AzureCdnCloudStorageContainerName), JobArgumentNames.AzureCdnCloudStorageContainerName); - _dataStorageAccount = ValidateAzureCloudStorageAccount(dataStorageAccountConnectionString, JobArgumentNames.DataStorageAccount); - _reportName = ValidateReportName(JobConfigurationManager.TryGetArgument(jobArgsDictionary, JobArgumentNames.WarehouseReportName)); + _reportName = ValidateReportName( + configuration.ReportName, + nameof(configuration.ReportName)); - var containerNames = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.DataContainerName) + var containerNames = configuration.DataContainerName .Split(new[] { '|' }, StringSplitOptions.RemoveEmptyEntries); foreach (var containerName in containerNames) { - ValidateAzureContainerName(containerName, JobArgumentNames.DataContainerName); + ValidateAzureContainerName(containerName, nameof(configuration.DataContainerName)); } _dataContainerNames = containerNames; @@ -80,10 +88,13 @@ public override void Init(IServiceContainer serviceContainer, IDictionary(); + var reportGenerationTime = DateTime.UtcNow; var destinationContainer = _cloudStorageAccount.CreateCloudBlobClient().GetContainerReference(_statisticsContainerName); - Logger.LogDebug("Generating reports from {DataSource}/{InitialCatalog} and saving to {AccountName}/{Container}", _statisticsDbConnectionFactory.DataSource, _statisticsDbConnectionFactory.InitialCatalog, _cloudStorageAccount.Credentials.AccountName, destinationContainer.Name); + Logger.LogDebug("Generating reports from {DataSource}/{InitialCatalog} and saving to {AccountName}/{Container}", + statisticsDatabase.DataSource, statisticsDatabase.InitialCatalog, _cloudStorageAccount.Credentials.AccountName, destinationContainer.Name); var reportBuilderLogger = LoggerFactory.CreateLogger(); var reportCollectorLogger = LoggerFactory.CreateLogger(); @@ -92,13 +103,35 @@ public override async Task Run() { // generate all reports var reportGenerators = new Dictionary + { + { + new ReportBuilder(reportBuilderLogger, ReportNames.NuGetClientVersion), + new ReportDataCollector(reportCollectorLogger, _storedProcedures[ReportNames.NuGetClientVersion], OpenSqlConnectionAsync, _sqlCommandTimeoutSeconds) + }, + + { + new ReportBuilder(reportBuilderLogger, ReportNames.Last6Weeks), + new ReportDataCollector(reportCollectorLogger, _storedProcedures[ReportNames.Last6Weeks], OpenSqlConnectionAsync, _sqlCommandTimeoutSeconds) + }, + + { + new ReportBuilder(reportBuilderLogger, ReportNames.RecentCommunityPopularity), + new ReportDataCollector(reportCollectorLogger, _storedProcedures[ReportNames.RecentCommunityPopularity], OpenSqlConnectionAsync, _sqlCommandTimeoutSeconds) + }, + + { + new ReportBuilder(reportBuilderLogger, ReportNames.RecentCommunityPopularityDetail), + new ReportDataCollector(reportCollectorLogger, _storedProcedures[ReportNames.RecentCommunityPopularityDetail], OpenSqlConnectionAsync, _sqlCommandTimeoutSeconds) + }, + { - { new ReportBuilder(reportBuilderLogger, ReportNames.NuGetClientVersion), new ReportDataCollector(reportCollectorLogger, _storedProcedures[ReportNames.NuGetClientVersion], _statisticsDbConnectionFactory, _sqlCommandTimeoutSeconds) }, - { new ReportBuilder(reportBuilderLogger, ReportNames.Last6Weeks), new ReportDataCollector(reportCollectorLogger, _storedProcedures[ReportNames.Last6Weeks], _statisticsDbConnectionFactory, _sqlCommandTimeoutSeconds) }, - { new ReportBuilder(reportBuilderLogger, ReportNames.RecentCommunityPopularity), new ReportDataCollector(reportCollectorLogger, _storedProcedures[ReportNames.RecentCommunityPopularity], _statisticsDbConnectionFactory, _sqlCommandTimeoutSeconds) }, - { new ReportBuilder(reportBuilderLogger, ReportNames.RecentCommunityPopularityDetail), new ReportDataCollector(reportCollectorLogger, _storedProcedures[ReportNames.RecentCommunityPopularityDetail], _statisticsDbConnectionFactory, _sqlCommandTimeoutSeconds) }, - { new ReportBuilder(reportBuilderLogger, ReportNames.RecentPopularity), new ReportDataCollector(reportCollectorLogger, _storedProcedures[ReportNames.RecentPopularity], _statisticsDbConnectionFactory, _sqlCommandTimeoutSeconds) }, - { new ReportBuilder(reportBuilderLogger, ReportNames.RecentPopularityDetail), new ReportDataCollector(reportCollectorLogger, _storedProcedures[ReportNames.RecentPopularityDetail], _statisticsDbConnectionFactory, _sqlCommandTimeoutSeconds) } + new ReportBuilder(reportBuilderLogger, ReportNames.RecentPopularity), + new ReportDataCollector(reportCollectorLogger, _storedProcedures[ReportNames.RecentPopularity], OpenSqlConnectionAsync, _sqlCommandTimeoutSeconds) + }, + + { + new ReportBuilder(reportBuilderLogger, ReportNames.RecentPopularityDetail), + new ReportDataCollector(reportCollectorLogger, _storedProcedures[ReportNames.RecentPopularityDetail], OpenSqlConnectionAsync, _sqlCommandTimeoutSeconds) } }; foreach (var reportGenerator in reportGenerators) @@ -114,12 +147,13 @@ public override async Task Run() { // generate only the specific report var reportBuilder = new ReportBuilder(reportBuilderLogger, _reportName); - var reportDataCollector = new ReportDataCollector(reportCollectorLogger, _storedProcedures[_reportName], _statisticsDbConnectionFactory, _sqlCommandTimeoutSeconds); + var reportDataCollector = new ReportDataCollector(reportCollectorLogger, _storedProcedures[_reportName], OpenSqlConnectionAsync, _sqlCommandTimeoutSeconds); await ProcessReport(LoggerFactory, destinationContainer, reportBuilder, reportDataCollector, reportGenerationTime); } - Logger.LogInformation("Generated reports from {DataSource}/{InitialCatalog} and saving to {AccountName}/{Container}", _statisticsDbConnectionFactory.DataSource, _statisticsDbConnectionFactory.InitialCatalog, _cloudStorageAccount.Credentials.AccountName, destinationContainer.Name); + Logger.LogInformation("Generated reports from {DataSource}/{InitialCatalog} and saving to {AccountName}/{Container}", + statisticsDatabase.DataSource, statisticsDatabase.InitialCatalog, _cloudStorageAccount.Credentials.AccountName, destinationContainer.Name); // totals reports var stopwatch = Stopwatch.StartNew(); @@ -131,7 +165,12 @@ public override async Task Run() { targets.Add(new StorageContainerTarget(_dataStorageAccount, dataContainerName)); } - var downloadCountReport = new DownloadCountReport(LoggerFactory.CreateLogger(), targets, _statisticsDbConnectionFactory, _galleryDbConnectionFactory); + + var downloadCountReport = new DownloadCountReport( + LoggerFactory.CreateLogger(), + targets, + OpenSqlConnectionAsync, + OpenSqlConnectionAsync); await downloadCountReport.Run(); stopwatch.Stop(); @@ -140,7 +179,12 @@ public override async Task Run() stopwatch.Restart(); // build stats-totals.json - var galleryTotalsReport = new GalleryTotalsReport(LoggerFactory.CreateLogger(), _cloudStorageAccount, _statisticsContainerName, _statisticsDbConnectionFactory, _galleryDbConnectionFactory); + var galleryTotalsReport = new GalleryTotalsReport( + LoggerFactory.CreateLogger(), + _cloudStorageAccount, + _statisticsContainerName, + OpenSqlConnectionAsync, + OpenSqlConnectionAsync); await galleryTotalsReport.Run(); stopwatch.Stop(); @@ -149,7 +193,12 @@ public override async Task Run() // build tools.v1.json - var toolsReport = new DownloadsPerToolVersionReport(LoggerFactory.CreateLogger(), _cloudStorageAccount, _statisticsContainerName, _statisticsDbConnectionFactory, _galleryDbConnectionFactory); + var toolsReport = new DownloadsPerToolVersionReport( + LoggerFactory.CreateLogger(), + _cloudStorageAccount, + _statisticsContainerName, + OpenSqlConnectionAsync, + OpenSqlConnectionAsync); await toolsReport.Run(); stopwatch.Stop(); @@ -158,7 +207,8 @@ public override async Task Run() stopwatch.Restart(); } - private static async Task ProcessReport(ILoggerFactory loggerFactory, CloudBlobContainer destinationContainer, ReportBuilder reportBuilder, ReportDataCollector reportDataCollector, DateTime reportGenerationTime, params Tuple[] parameters) + private static async Task ProcessReport(ILoggerFactory loggerFactory, CloudBlobContainer destinationContainer, ReportBuilder reportBuilder, + ReportDataCollector reportDataCollector, DateTime reportGenerationTime, params Tuple[] parameters) { var dataTable = await reportDataCollector.CollectAsync(reportGenerationTime, parameters); if (dataTable.Rows.Count == 0) @@ -174,18 +224,32 @@ private static async Task ProcessReport(ILoggerFactory loggerFactory, CloudBlobC private async Task RebuildPackageReports(CloudBlobContainer destinationContainer, DateTime reportGenerationTime) { - var dirtyPackageIds = await ReportDataCollector.GetDirtyPackageIds(LoggerFactory.CreateLogger(), _statisticsDbConnectionFactory, reportGenerationTime, _sqlCommandTimeoutSeconds); + var dirtyPackageIds = await ReportDataCollector.GetDirtyPackageIds( + LoggerFactory.CreateLogger(), + OpenSqlConnectionAsync, + reportGenerationTime, + _sqlCommandTimeoutSeconds); if (!dirtyPackageIds.Any()) + { return; + } // first process the top 100 packages var top100 = dirtyPackageIds.Take(100); - var reportDataCollector = new ReportDataCollector(LoggerFactory.CreateLogger(), _storedProceduresPerPackageId[ReportNames.RecentPopularityDetailByPackageId], _statisticsDbConnectionFactory, _sqlCommandTimeoutSeconds); + var reportDataCollector = new ReportDataCollector( + LoggerFactory.CreateLogger(), + _storedProceduresPerPackageId[ReportNames.RecentPopularityDetailByPackageId], + OpenSqlConnectionAsync, + _sqlCommandTimeoutSeconds); + var top100Task = Parallel.ForEach(top100, new ParallelOptions { MaxDegreeOfParallelism = _perPackageReportDegreeOfParallelism }, dirtyPackageId => { var packageId = dirtyPackageId.PackageId.ToLowerInvariant(); - var reportBuilder = new RecentPopularityDetailByPackageReportBuilder(LoggerFactory.CreateLogger(), ReportNames.RecentPopularityDetailByPackageId, "recentpopularity/" + _recentPopularityDetailByPackageReportBaseName + packageId); + var reportBuilder = new RecentPopularityDetailByPackageReportBuilder( + LoggerFactory.CreateLogger(), + ReportNames.RecentPopularityDetailByPackageId, + "recentpopularity/" + _recentPopularityDetailByPackageReportBaseName + packageId); ProcessReport(LoggerFactory, destinationContainer, reportBuilder, reportDataCollector, reportGenerationTime, Tuple.Create("@PackageId", 128, dirtyPackageId.PackageId)).Wait(); ApplicationInsightsHelper.TrackReportProcessed(reportBuilder.ReportName + " report", packageId); @@ -211,7 +275,7 @@ private async Task RebuildPackageReports(CloudBlobContainer destinationContainer new ReportDataCollector( LoggerFactory.CreateLogger(), _storedProceduresPerPackageId[ReportNames.RecentPopularityDetailByPackageId], - _statisticsDbConnectionFactory, + OpenSqlConnectionAsync, _sqlCommandTimeoutSeconds) } }; @@ -228,7 +292,7 @@ private async Task RebuildPackageReports(CloudBlobContainer destinationContainer if (top100Task.IsCompleted) { var runToCursor = dirtyPackageIds.First().RunToCuror; - await ReportDataCollector.UpdateDirtyPackageIdCursor(_statisticsDbConnectionFactory, runToCursor, _sqlCommandTimeoutSeconds); + await ReportDataCollector.UpdateDirtyPackageIdCursor(OpenSqlConnectionAsync, runToCursor, _sqlCommandTimeoutSeconds); } } } @@ -236,7 +300,11 @@ private async Task RebuildPackageReports(CloudBlobContainer destinationContainer private async Task CleanInactiveRecentPopularityDetailByPackageReports(CloudBlobContainer destinationContainer, DateTime reportGenerationTime) { Logger.LogDebug("Getting list of inactive packages."); - var packageIds = await ReportDataCollector.ListInactivePackageIdReports(_statisticsDbConnectionFactory, reportGenerationTime, _sqlCommandTimeoutSeconds); + var packageIds = await ReportDataCollector.ListInactivePackageIdReports( + OpenSqlConnectionAsync, + reportGenerationTime, + _sqlCommandTimeoutSeconds); + Logger.LogInformation("Found {InactivePackageCount} inactive packages.", packageIds.Count); // Collect the list of reports @@ -265,11 +333,11 @@ private async Task CleanInactiveRecentPopularityDetailByPackageReports(CloudBlob }); } - private static CloudStorageAccount ValidateAzureCloudStorageAccount(string cloudStorageAccount, string parameterName) + private static CloudStorageAccount ValidateAzureCloudStorageAccount(string cloudStorageAccount, string configurationName) { if (string.IsNullOrEmpty(cloudStorageAccount)) { - throw new ArgumentException($"Job parameter {parameterName} is not defined."); + throw new ArgumentException($"Job configuration {configurationName} is not defined."); } CloudStorageAccount account; @@ -278,20 +346,20 @@ private static CloudStorageAccount ValidateAzureCloudStorageAccount(string cloud return account; } - throw new ArgumentException($"Job parameter {parameterName} is invalid."); + throw new ArgumentException($"Job configuration {configurationName} is invalid."); } - private static string ValidateAzureContainerName(string containerName, string parameterName) + private static string ValidateAzureContainerName(string containerName, string configurationName) { if (string.IsNullOrWhiteSpace(containerName)) { - throw new ArgumentException($"Job parameter {parameterName} is not defined."); + throw new ArgumentException($"Job configuration {configurationName} is not defined."); } return containerName; } - private static string ValidateReportName(string reportName) + private static string ValidateReportName(string reportName, string configurationName) { if (string.IsNullOrWhiteSpace(reportName)) { @@ -300,12 +368,21 @@ private static string ValidateReportName(string reportName) if (!_storedProcedures.ContainsKey(reportName.ToLowerInvariant())) { - throw new ArgumentException("Job parameter ReportName contains unknown report name."); + throw new ArgumentException($"Job configuration {configurationName} contains unknown report name."); } return reportName; } + protected override void ConfigureAutofacServices(ContainerBuilder containerBuilder) + { + } + + protected override void ConfigureJobServices(IServiceCollection services, IConfigurationRoot configurationRoot) + { + ConfigureInitializationSection(services, configurationRoot); + } + private static class ReportNames { public const string NuGetClientVersion = "nugetclientversion"; diff --git a/src/Stats.CreateAzureCdnWarehouseReports/DownloadCountReport.cs b/src/Stats.CreateAzureCdnWarehouseReports/DownloadCountReport.cs index 85ae20a62..bc9deae7e 100644 --- a/src/Stats.CreateAzureCdnWarehouseReports/DownloadCountReport.cs +++ b/src/Stats.CreateAzureCdnWarehouseReports/DownloadCountReport.cs @@ -10,7 +10,6 @@ using Microsoft.Extensions.Logging; using Newtonsoft.Json; using Newtonsoft.Json.Linq; -using NuGet.Services.Sql; using NuGet.Versioning; namespace Stats.CreateAzureCdnWarehouseReports @@ -25,9 +24,9 @@ public class DownloadCountReport public DownloadCountReport( ILogger logger, IEnumerable targets, - ISqlConnectionFactory statisticsDbConnectionFactory, - ISqlConnectionFactory galleryDbConnectionFactory) - : base(logger, targets, statisticsDbConnectionFactory, galleryDbConnectionFactory) + Func> openStatisticsSqlConnectionAsync, + Func> openGallerySqlConnectionAsync) + : base(logger, targets, openStatisticsSqlConnectionAsync, openGallerySqlConnectionAsync) { } @@ -35,12 +34,13 @@ public async Task Run() { // Gather download count data from statistics warehouse IReadOnlyCollection downloadData; - _logger.LogInformation("Gathering Download Counts from {DataSource}/{InitialCatalog}...", - StatisticsDbConnectionFactory.DataSource, StatisticsDbConnectionFactory.InitialCatalog); - using (var connection = await StatisticsDbConnectionFactory.CreateAsync()) + using (var connection = await OpenStatisticsSqlConnectionAsync()) using (var transaction = connection.BeginTransaction(IsolationLevel.Snapshot)) { + _logger.LogInformation("Gathering Download Counts from {DataSource}/{InitialCatalog}...", + connection.DataSource, connection.Database); + downloadData = (await connection.QueryWithRetryAsync( _storedProcedureName, commandType: CommandType.StoredProcedure, transaction: transaction, commandTimeout: _defaultCommandTimeout)).ToList(); } diff --git a/src/Stats.CreateAzureCdnWarehouseReports/DownloadsPerToolVersionReport.cs b/src/Stats.CreateAzureCdnWarehouseReports/DownloadsPerToolVersionReport.cs index 7411526f1..df9e0ccc9 100644 --- a/src/Stats.CreateAzureCdnWarehouseReports/DownloadsPerToolVersionReport.cs +++ b/src/Stats.CreateAzureCdnWarehouseReports/DownloadsPerToolVersionReport.cs @@ -11,7 +11,6 @@ using Microsoft.WindowsAzure.Storage; using Newtonsoft.Json; using Newtonsoft.Json.Linq; -using NuGet.Services.Sql; namespace Stats.CreateAzureCdnWarehouseReports { @@ -26,10 +25,10 @@ public DownloadsPerToolVersionReport( ILogger logger, CloudStorageAccount cloudStorageAccount, string statisticsContainerName, - ISqlConnectionFactory statisticsDbConnectionFactory, - ISqlConnectionFactory galleryDbConnectionFactory) + Func> openStatisticsSqlConnectionAsync, + Func> openGallerySqlConnectionAsync) : base(logger, new[] { new StorageContainerTarget(cloudStorageAccount, statisticsContainerName) }, - statisticsDbConnectionFactory, galleryDbConnectionFactory) + openStatisticsSqlConnectionAsync, openGallerySqlConnectionAsync) { } @@ -37,12 +36,13 @@ public async Task Run() { // Gather download count data from statistics warehouse IReadOnlyCollection data; - _logger.LogInformation("Gathering Tools Download Counts from {DataSource}/{InitialCatalog}...", - StatisticsDbConnectionFactory.DataSource, StatisticsDbConnectionFactory.InitialCatalog); - using (var connection = await StatisticsDbConnectionFactory.CreateAsync()) + using (var connection = await OpenStatisticsSqlConnectionAsync()) using (var transaction = connection.BeginTransaction(IsolationLevel.Snapshot)) { + _logger.LogInformation("Gathering Tools Download Counts from {DataSource}/{InitialCatalog}...", + connection.DataSource, connection.Database); + data = (await connection.QueryWithRetryAsync( _storedProcedureName, commandType: CommandType.StoredProcedure, transaction: transaction, commandTimeout: _defaultCommandTimeout)).ToList(); } diff --git a/src/Stats.CreateAzureCdnWarehouseReports/GalleryTotalsReport.cs b/src/Stats.CreateAzureCdnWarehouseReports/GalleryTotalsReport.cs index cdccbb09d..92919f973 100644 --- a/src/Stats.CreateAzureCdnWarehouseReports/GalleryTotalsReport.cs +++ b/src/Stats.CreateAzureCdnWarehouseReports/GalleryTotalsReport.cs @@ -9,7 +9,6 @@ using Microsoft.Extensions.Logging; using Microsoft.WindowsAzure.Storage; using Newtonsoft.Json; -using NuGet.Services.Sql; namespace Stats.CreateAzureCdnWarehouseReports { @@ -27,10 +26,10 @@ public GalleryTotalsReport( ILogger logger, CloudStorageAccount cloudStorageAccount, string statisticsContainerName, - ISqlConnectionFactory statisticsDbConnectionFactory, - ISqlConnectionFactory galleryDbConnectionFactory) + Func> openStatisticsSqlConnectionAsync, + Func> openGallerySqlConnectionAsync) : base(logger, new[] { new StorageContainerTarget(cloudStorageAccount, statisticsContainerName) }, - statisticsDbConnectionFactory, galleryDbConnectionFactory) + openStatisticsSqlConnectionAsync, openGallerySqlConnectionAsync) { } @@ -38,12 +37,13 @@ public async Task Run() { // gather package numbers from gallery database GalleryTotalsData totalsData; - _logger.LogInformation("Gathering Gallery Totals from {GalleryDataSource}/{GalleryInitialCatalog}...", - GalleryDbConnectionFactory.DataSource, GalleryDbConnectionFactory.InitialCatalog); - using (var connection = await GalleryDbConnectionFactory.CreateAsync()) + using (var connection = await OpenGallerySqlConnectionAsync()) using (var transaction = connection.BeginTransaction(IsolationLevel.Snapshot)) { + _logger.LogInformation("Gathering Gallery Totals from {GalleryDataSource}/{GalleryInitialCatalog}...", + connection.DataSource, connection.Database); + totalsData = (await connection.QueryWithRetryAsync( GalleryQuery, commandType: CommandType.Text, transaction: transaction)).First(); } @@ -52,12 +52,12 @@ public async Task Run() _logger.LogInformation("Unique packages: {UniquePackagesCount}", totalsData.UniquePackages); // gather download count data from statistics warehouse - _logger.LogInformation("Gathering Gallery Totals from {StatisticsDataSource}/{StatisticsInitialCatalog}...", - StatisticsDbConnectionFactory.DataSource, StatisticsDbConnectionFactory.InitialCatalog); - - using (var connection = await StatisticsDbConnectionFactory.CreateAsync()) + using (var connection = await OpenStatisticsSqlConnectionAsync()) using (var transaction = connection.BeginTransaction(IsolationLevel.Snapshot)) { + _logger.LogInformation("Gathering Gallery Totals from {StatisticsDataSource}/{StatisticsInitialCatalog}...", + connection.DataSource, connection.Database); + totalsData.Downloads = (await connection.ExecuteScalarWithRetryAsync( WarehouseStoredProcedureName, commandType: CommandType.StoredProcedure, diff --git a/src/Stats.CreateAzureCdnWarehouseReports/Program.cs b/src/Stats.CreateAzureCdnWarehouseReports/Program.cs index f16d9e031..e5c658e9e 100644 --- a/src/Stats.CreateAzureCdnWarehouseReports/Program.cs +++ b/src/Stats.CreateAzureCdnWarehouseReports/Program.cs @@ -9,7 +9,7 @@ class Program { static void Main(string[] args) { - var job = new Job(); + var job = new CreateAzureCdnWarehouseReportsJob(); JobRunner.Run(job, args).Wait(); } } diff --git a/src/Stats.CreateAzureCdnWarehouseReports/ReportBase.cs b/src/Stats.CreateAzureCdnWarehouseReports/ReportBase.cs index faef0cce4..b01f5b186 100644 --- a/src/Stats.CreateAzureCdnWarehouseReports/ReportBase.cs +++ b/src/Stats.CreateAzureCdnWarehouseReports/ReportBase.cs @@ -3,12 +3,12 @@ using System; using System.Collections.Generic; +using System.Data.SqlClient; using System.Linq; using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Microsoft.WindowsAzure.Storage.Blob; using Microsoft.WindowsAzure.Storage.RetryPolicies; -using NuGet.Services.Sql; namespace Stats.CreateAzureCdnWarehouseReports { @@ -18,20 +18,20 @@ public abstract class ReportBase protected readonly IReadOnlyCollection Targets; - protected readonly ISqlConnectionFactory StatisticsDbConnectionFactory; + protected readonly Func> OpenStatisticsSqlConnectionAsync; - protected ISqlConnectionFactory GalleryDbConnectionFactory; + protected Func> OpenGallerySqlConnectionAsync; protected ReportBase( ILogger logger, IEnumerable targets, - ISqlConnectionFactory statisticsDbConnectionFactory, - ISqlConnectionFactory galleryDbConnectionFactory) + Func> openStatisticsSqlConnectionAsync, + Func> openGallerySqlConnectionAsync) { _logger = logger; Targets = targets.ToList().AsReadOnly(); - StatisticsDbConnectionFactory = statisticsDbConnectionFactory; - GalleryDbConnectionFactory = galleryDbConnectionFactory; + OpenStatisticsSqlConnectionAsync = openStatisticsSqlConnectionAsync; + OpenGallerySqlConnectionAsync = openGallerySqlConnectionAsync; } protected async Task GetBlobContainer(StorageContainerTarget target) diff --git a/src/Stats.CreateAzureCdnWarehouseReports/ReportDataCollector.cs b/src/Stats.CreateAzureCdnWarehouseReports/ReportDataCollector.cs index be58465b4..7472f515c 100644 --- a/src/Stats.CreateAzureCdnWarehouseReports/ReportDataCollector.cs +++ b/src/Stats.CreateAzureCdnWarehouseReports/ReportDataCollector.cs @@ -8,7 +8,6 @@ using System.Diagnostics; using System.Threading.Tasks; using Microsoft.Extensions.Logging; -using NuGet.Services.Sql; namespace Stats.CreateAzureCdnWarehouseReports { @@ -16,19 +15,19 @@ internal class ReportDataCollector { private int _commandTimeoutSeconds; private readonly string _procedureName; - private readonly ISqlConnectionFactory _sourceDbConnectionFactory; + private readonly Func> _openGallerySqlConnectionAsync; private ILogger _logger; public ReportDataCollector( ILogger logger, string procedureName, - ISqlConnectionFactory sourceDbConnectionFactory, + Func> openGallerySqlConnectionAsync, int timeout) { _logger = logger; _procedureName = procedureName; - _sourceDbConnectionFactory = sourceDbConnectionFactory; + _openGallerySqlConnectionAsync = openGallerySqlConnectionAsync; _commandTimeoutSeconds = timeout; } @@ -48,7 +47,7 @@ public async Task CollectAsync(DateTime reportGenerationTime, params public static async Task> GetDirtyPackageIds( ILogger logger, - ISqlConnectionFactory sourceDbConnectionFactory, + Func> openGallerySqlConnectionAsync, DateTime reportGenerationTime, int commandTimeout) { @@ -57,7 +56,7 @@ public static async Task> GetDirtyPackageIds IReadOnlyCollection packageIds = new List(); // Get the data - await WithRetry(async () => packageIds = await GetDirtyPackageIdsFromWarehouse(sourceDbConnectionFactory, reportGenerationTime, commandTimeout), logger); + await WithRetry(async () => packageIds = await GetDirtyPackageIdsFromWarehouse(openGallerySqlConnectionAsync, reportGenerationTime, commandTimeout), logger); logger.LogInformation("Found {DirtyPackagesCount} dirty packages to update.", packageIds.Count); @@ -65,11 +64,11 @@ public static async Task> GetDirtyPackageIds } public static async Task> ListInactivePackageIdReports( - ISqlConnectionFactory sourceDbConnectionFactory, + Func> openGallerySqlConnectionAsync, DateTime reportGenerationTime, int commandTimeout) { - using (var connection = await sourceDbConnectionFactory.CreateAsync()) + using (var connection = await openGallerySqlConnectionAsync()) { var command = new SqlCommand("[dbo].[DownloadReportListInactive]", connection); command.CommandType = CommandType.StoredProcedure; @@ -122,7 +121,7 @@ private static async Task WithRetry(Func action, ILogger logger) private async Task ExecuteSql(DateTime reportGenerationTime, params Tuple[] parameters) { - using (var connection = await _sourceDbConnectionFactory.CreateAsync()) + using (var connection = await _openGallerySqlConnectionAsync()) { var command = new SqlCommand(_procedureName, connection); command.CommandType = CommandType.StoredProcedure; @@ -146,11 +145,11 @@ private async Task ExecuteSql(DateTime reportGenerationTime, params T } private static async Task> GetDirtyPackageIdsFromWarehouse( - ISqlConnectionFactory sourceDbConnectionFactory, + Func> openGallerySqlConnectionAsync, DateTime reportGenerationTime, int commandTimeout) { - using (var connection = await sourceDbConnectionFactory.CreateAsync()) + using (var connection = await openGallerySqlConnectionAsync()) { var command = new SqlCommand("[dbo].[GetDirtyPackageIds]", connection); command.CommandType = CommandType.StoredProcedure; @@ -172,11 +171,11 @@ private static async Task> GetDirtyPackageId } public static async Task UpdateDirtyPackageIdCursor( - ISqlConnectionFactory sourceDbConnectionFactory, + Func> openGallerySqlConnectionAsync, DateTime runToCursor, int commandTimeout) { - using (var connection = await sourceDbConnectionFactory.CreateAsync()) + using (var connection = await openGallerySqlConnectionAsync()) { var command = new SqlCommand("[dbo].[UpdateDirtyPackageIdCursor]", connection); command.CommandType = CommandType.StoredProcedure; diff --git a/src/Stats.CreateAzureCdnWarehouseReports/Scripts/Stats.CreateAzureCdnWarehouseReports.cmd b/src/Stats.CreateAzureCdnWarehouseReports/Scripts/Stats.CreateAzureCdnWarehouseReports.cmd index b4fe1d9e7..bf9c90725 100644 --- a/src/Stats.CreateAzureCdnWarehouseReports/Scripts/Stats.CreateAzureCdnWarehouseReports.cmd +++ b/src/Stats.CreateAzureCdnWarehouseReports/Scripts/Stats.CreateAzureCdnWarehouseReports.cmd @@ -3,26 +3,16 @@ cd bin :Top - echo "Starting job - #{Jobs.stats.createazurecdnwarehousereports.Title}" +echo "Starting job - #{Jobs.stats.createazurecdnwarehousereports.Title}" - title #{Jobs.stats.createazurecdnwarehousereports.Title} +title #{Jobs.stats.createazurecdnwarehousereports.Title} - start /w stats.createazurecdnwarehousereports.exe ^ - -VaultName "#{Deployment.Azure.KeyVault.VaultName}" ^ - -ClientId "#{Deployment.Azure.KeyVault.ClientId}" ^ - -CertificateThumbprint "#{Deployment.Azure.KeyVault.CertificateThumbprint}" ^ - -AzureCdnCloudStorageAccount "#{Jobs.stats.createazurecdnwarehousereports.AzureCdn.CloudStorageAccount}" ^ - -AzureCdnCloudStorageContainerName "#{Jobs.stats.createazurecdnwarehousereports.AzureCdn.CloudStorageContainerName}" ^ - -StatisticsDatabase "#{Jobs.stats.createazurecdnwarehousereports.StatisticsDatabase}" ^ - -SourceDatabase "#{Jobs.stats.createazurecdnwarehousereports.SourceDatabase}" ^ - -DataStorageAccount "#{Jobs.stats.createazurecdnwarehousereports.DataStorageAccount}" ^ - -InstrumentationKey "#{Jobs.stats.createazurecdnwarehousereports.InstrumentationKey}" ^ - -DataContainerName "#{Jobs.stats.createazurecdnwarehousereports.DataContainerName}" ^ - -CommandTimeOut "#{Jobs.stats.createazurecdnwarehousereports.CommandTimeOut}" ^ - -PerPackageReportDegreeOfParallelism "#{Jobs.stats.createazurecdnwarehousereports.PerPackageReportDegreeOfParallelism}" ^ - -verbose true ^ - -Interval #{Jobs.stats.createazurecdnwarehousereports.Interval} +start /w stats.createazurecdnwarehousereports.exe ^ + -Configuration "#{Jobs.stats.createazurecdnwarehousereports.Configuration}" + -InstrumentationKey "#{Jobs.stats.createazurecdnwarehousereports.InstrumentationKey}" ^ + -Interval #{Jobs.stats.createazurecdnwarehousereports.Interval} ^ + -verbose true - echo "Finished #{Jobs.stats.createazurecdnwarehousereports.Title}" +echo "Finished #{Jobs.stats.createazurecdnwarehousereports.Title}" - goto Top +goto Top diff --git a/src/Stats.CreateAzureCdnWarehouseReports/Settings/dev.json b/src/Stats.CreateAzureCdnWarehouseReports/Settings/dev.json new file mode 100644 index 000000000..3355925bd --- /dev/null +++ b/src/Stats.CreateAzureCdnWarehouseReports/Settings/dev.json @@ -0,0 +1,25 @@ +{ + "Initialization": { + "AzureCdnCloudStorageAccount": "DefaultEndpointsProtocol=https;AccountName=nugetdevlegacy;AccountKey=$$Dev-NuGetDevLegacyStorage-Key$$", + "AzureCdnCloudStorageContainerName": "nuget-cdnstats", + "DataStorageAccount": "DefaultEndpointsProtocol=https;AccountName=nugetdev0;AccountKey=$$Dev-NuGetDev0Storage-Key$$", + "DataContainerName": "ng-search-data", + "CommandTimeOut": "7200", + "PerPackageReportDegreeOfParallelism": "64" + }, + + "GalleryDb": { + "ConnectionString": "Data Source=tcp:#{Deployment.Azure.Sql.GalleryDatabaseAddress};Initial Catalog=nuget-dev-0-v2gallery;User ID=$$Dev-GalleryDBReadOnly-UserName$$;Password=$$Dev-GalleryDBReadOnly-Password$$;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;" + }, + + "StatisticsDb": { + "ConnectionString": "Data Source=tcp:#{Deployment.Azure.Sql.StatisticsDatabaseAddress};Initial Catalog=nuget-dev-statistics;User ID=$$Dev-StatisticsDBWriter-UserName$$;Password=$$Dev-StatisticsDBWriter-Password$$;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;" + }, + + "KeyVault_VaultName": "#{Deployment.Azure.KeyVault.VaultName}", + "KeyVault_ClientId": "#{Deployment.Azure.KeyVault.ClientId}", + "KeyVault_CertificateThumbprint": "#{Deployment.Azure.KeyVault.CertificateThumbprint}", + "KeyVault_ValidateCertificate": true, + "KeyVault_StoreName": "My", + "KeyVault_StoreLocation": "LocalMachine" +} \ No newline at end of file diff --git a/src/Stats.CreateAzureCdnWarehouseReports/Settings/int.json b/src/Stats.CreateAzureCdnWarehouseReports/Settings/int.json new file mode 100644 index 000000000..fe47aa6ba --- /dev/null +++ b/src/Stats.CreateAzureCdnWarehouseReports/Settings/int.json @@ -0,0 +1,25 @@ +{ + "Initialization": { + "AzureCdnCloudStorageAccount": "DefaultEndpointsProtocol=https;AccountName=nugetint0;AccountKey=$$Int-NuGetInt0Storage-Key$$", + "AzureCdnCloudStorageContainerName": "nuget-cdnstats", + "DataStorageAccount": "DefaultEndpointsProtocol=https;AccountName=nugetint0;AccountKey=$$Int-NuGetInt0Storage-Key$$", + "DataContainerName": "ng-search-data", + "CommandTimeOut": "7200", + "PerPackageReportDegreeOfParallelism": "64" + }, + + "GalleryDb": { + "ConnectionString": "Data Source=tcp:#{Deployment.Azure.Sql.GalleryDatabaseAddress};Initial Catalog=nuget-int-0-v2gallery;User ID=$$Int-GalleryDBReadOnly-UserName$$;Password=$$Int-GalleryDBReadOnly-Password$$;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;" + }, + + "StatisticsDb": { + "ConnectionString": "Data Source=tcp:#{Deployment.Azure.Sql.StatisticsDatabaseAddress};Initial Catalog=nuget-int-statistics;User ID=$$Int-StatisticsDBWriter-UserName$$;Password=$$Int-StatisticsDBWriter-Password$$;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;" + }, + + "KeyVault_VaultName": "#{Deployment.Azure.KeyVault.VaultName}", + "KeyVault_ClientId": "#{Deployment.Azure.KeyVault.ClientId}", + "KeyVault_CertificateThumbprint": "#{Deployment.Azure.KeyVault.CertificateThumbprint}", + "KeyVault_ValidateCertificate": true, + "KeyVault_StoreName": "My", + "KeyVault_StoreLocation": "LocalMachine" +} \ No newline at end of file diff --git a/src/Stats.CreateAzureCdnWarehouseReports/Settings/prod.json b/src/Stats.CreateAzureCdnWarehouseReports/Settings/prod.json new file mode 100644 index 000000000..47b14d6e8 --- /dev/null +++ b/src/Stats.CreateAzureCdnWarehouseReports/Settings/prod.json @@ -0,0 +1,25 @@ +{ + "Initialization": { + "AzureCdnCloudStorageAccount": "DefaultEndpointsProtocol=https;AccountName=nugetgallery;AccountKey=$$Prod-NuGetGalleryStorage-Key$$", + "AzureCdnCloudStorageContainerName": "nuget-cdnstats", + "DataStorageAccount": "DefaultEndpointsProtocol=https;AccountName=nugetprod0;AccountKey=$$Prod-NuGetProd0Storage-Key$$", + "DataContainerName": "ng-search-data", + "CommandTimeOut": "7200", + "PerPackageReportDegreeOfParallelism": "64" + }, + + "GalleryDb": { + "ConnectionString": "Data Source=tcp:#{Deployment.Azure.Sql.GalleryDatabaseAddress};Initial Catalog=NuGetGallery;User ID=$$Prod-GalleryDBReadOnly-UserName$$;Password=$$Prod-GalleryDBReadOnly-Password$$;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;" + }, + + "StatisticsDb": { + "ConnectionString": "Data Source=tcp:#{Deployment.Azure.Sql.StatisticsDatabaseAddress};Initial Catalog=nuget-prod-statistics;User ID=$$Prod-StatisticsDBWriter-UserName$$;Password=$$Prod-StatisticsDBWriter-Password$$;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;" + }, + + "KeyVault_VaultName": "#{Deployment.Azure.KeyVault.VaultName}", + "KeyVault_ClientId": "#{Deployment.Azure.KeyVault.ClientId}", + "KeyVault_CertificateThumbprint": "#{Deployment.Azure.KeyVault.CertificateThumbprint}", + "KeyVault_ValidateCertificate": true, + "KeyVault_StoreName": "My", + "KeyVault_StoreLocation": "LocalMachine" +} \ No newline at end of file diff --git a/src/Stats.CreateAzureCdnWarehouseReports/Stats.CreateAzureCdnWarehouseReports.csproj b/src/Stats.CreateAzureCdnWarehouseReports/Stats.CreateAzureCdnWarehouseReports.csproj index c92a9ef7b..a559d7432 100644 --- a/src/Stats.CreateAzureCdnWarehouseReports/Stats.CreateAzureCdnWarehouseReports.csproj +++ b/src/Stats.CreateAzureCdnWarehouseReports/Stats.CreateAzureCdnWarehouseReports.csproj @@ -41,6 +41,7 @@ + @@ -48,7 +49,7 @@ - + @@ -62,6 +63,9 @@ + + + @@ -96,12 +100,6 @@ 9.0.1 - - 2.25.0 - - - 2.27.0 - 4.3.0-preview1-2524 diff --git a/src/Stats.CreateAzureCdnWarehouseReports/Stats.CreateAzureCdnWarehouseReports.nuspec b/src/Stats.CreateAzureCdnWarehouseReports/Stats.CreateAzureCdnWarehouseReports.nuspec index 850276ed1..851685278 100644 --- a/src/Stats.CreateAzureCdnWarehouseReports/Stats.CreateAzureCdnWarehouseReports.nuspec +++ b/src/Stats.CreateAzureCdnWarehouseReports/Stats.CreateAzureCdnWarehouseReports.nuspec @@ -18,5 +18,7 @@ + + \ No newline at end of file diff --git a/src/Stats.RollUpDownloadFacts/Configuration/RollUpDownloadFactsConfiguration.cs b/src/Stats.RollUpDownloadFacts/Configuration/RollUpDownloadFactsConfiguration.cs new file mode 100644 index 000000000..ac9d0c82d --- /dev/null +++ b/src/Stats.RollUpDownloadFacts/Configuration/RollUpDownloadFactsConfiguration.cs @@ -0,0 +1,10 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +namespace Stats.RollUpDownloadFacts +{ + public class RollUpDownloadFactsConfiguration + { + public int? MinAgeInDays { get; set; } + } +} diff --git a/src/Stats.RollUpDownloadFacts/Program.cs b/src/Stats.RollUpDownloadFacts/Program.cs index 1550c66ef..10b96b312 100644 --- a/src/Stats.RollUpDownloadFacts/Program.cs +++ b/src/Stats.RollUpDownloadFacts/Program.cs @@ -9,7 +9,7 @@ public class Program { public static void Main(string[] args) { - var job = new Job(); + var job = new RollUpDownloadFactsJob(); JobRunner.Run(job, args).Wait(); } } diff --git a/src/Stats.RollUpDownloadFacts/Job.cs b/src/Stats.RollUpDownloadFacts/RollUpDownloadFactsJob.cs similarity index 70% rename from src/Stats.RollUpDownloadFacts/Job.cs rename to src/Stats.RollUpDownloadFacts/RollUpDownloadFactsJob.cs index 76cd61979..ce2727b88 100644 --- a/src/Stats.RollUpDownloadFacts/Job.cs +++ b/src/Stats.RollUpDownloadFacts/RollUpDownloadFactsJob.cs @@ -7,35 +7,38 @@ using System.Data.SqlClient; using System.Linq; using System.Threading.Tasks; +using Autofac; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; using NuGet.Jobs; -using NuGet.Services.KeyVault; -using NuGet.Services.Sql; +using NuGet.Jobs.Configuration; namespace Stats.RollUpDownloadFacts { - public class Job - : JobBase + public class RollUpDownloadFactsJob : JsonConfigurationJob { private const string _startTemplateRecordsDeletion = "Package Dimension ID "; private const string _endTemplateFactDownloadDeletion = " records from [dbo].[Fact_Download]"; private const int DefaultMinAgeInDays = 43; - private static int _minAgeInDays; - private static ISqlConnectionFactory _statisticsDbConnectionFactory; + + private RollUpDownloadFactsConfiguration _configuration; + private int _minAgeInDays; public override void Init(IServiceContainer serviceContainer, IDictionary jobArgsDictionary) { - var secretInjector = (ISecretInjector)serviceContainer.GetService(typeof(ISecretInjector)); - var statisticsDbConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatisticsDatabase); - _statisticsDbConnectionFactory = new AzureSqlConnectionFactory(statisticsDbConnectionString, secretInjector); + base.Init(serviceContainer, jobArgsDictionary); + + _configuration = _serviceProvider.GetRequiredService>().Value; - _minAgeInDays = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, JobArgumentNames.MinAgeInDays) ?? DefaultMinAgeInDays; + _minAgeInDays = _configuration.MinAgeInDays ?? DefaultMinAgeInDays; Logger.LogInformation("Min age in days: {MinAgeInDays}", _minAgeInDays); } public override async Task Run() { - using (var connection = await _statisticsDbConnectionFactory.CreateAsync()) + using (var connection = await OpenSqlConnectionAsync()) { connection.InfoMessage -= OnSqlConnectionInfoMessage; connection.InfoMessage += OnSqlConnectionInfoMessage; @@ -72,5 +75,14 @@ private void OnSqlConnectionInfoMessage(object sender, SqlInfoMessageEventArgs e ApplicationInsightsHelper.TrackRollUpMetric("Download Facts Deleted", value, packageDimensionId); } } + + protected override void ConfigureAutofacServices(ContainerBuilder containerBuilder) + { + } + + protected override void ConfigureJobServices(IServiceCollection services, IConfigurationRoot configurationRoot) + { + ConfigureInitializationSection(services, configurationRoot); + } } } \ No newline at end of file diff --git a/src/Stats.RollUpDownloadFacts/Scripts/Stats.RollUpDownloadFacts.cmd b/src/Stats.RollUpDownloadFacts/Scripts/Stats.RollUpDownloadFacts.cmd index c49025f34..a67f091c4 100644 --- a/src/Stats.RollUpDownloadFacts/Scripts/Stats.RollUpDownloadFacts.cmd +++ b/src/Stats.RollUpDownloadFacts/Scripts/Stats.RollUpDownloadFacts.cmd @@ -3,12 +3,16 @@ cd bin :Top - echo "Starting job - #{Jobs.stats.rollupdownloadfacts.Title}" +echo "Starting job - #{Jobs.stats.rollupdownloadfacts.Title}" - title #{Jobs.stats.rollupdownloadfacts.Title} +title #{Jobs.stats.rollupdownloadfacts.Title} - start /w stats.rollupdownloadfacts.exe -VaultName "#{Deployment.Azure.KeyVault.VaultName}" -ClientId "#{Deployment.Azure.KeyVault.ClientId}" -CertificateThumbprint "#{Deployment.Azure.KeyVault.CertificateThumbprint}" -MinAgeInDays "#{Jobs.stats.rollupdownloadfacts.MinAgeInDays}" -StatisticsDatabase "#{Jobs.stats.rollupdownloadfacts.StatisticsDatabase}" -InstrumentationKey "#{Jobs.stats.rollupdownloadfacts.InstrumentationKey}" -verbose true -Interval #{Jobs.stats.rollupdownloadfacts.Interval} +start /w stats.rollupdownloadfacts.exe ^ + -Configuration "#{Jobs.stats.rollupdownloadfacts.Configuration}" ^ + -InstrumentationKey "#{Jobs.stats.rollupdownloadfacts.InstrumentationKey}" ^ + -Interval #{Jobs.stats.rollupdownloadfacts.Interval} ^ + -verbose true - echo "Finished #{Jobs.stats.rollupdownloadfacts.Title}" +echo "Finished #{Jobs.stats.rollupdownloadfacts.Title}" - goto Top \ No newline at end of file +goto Top \ No newline at end of file diff --git a/src/Stats.RollUpDownloadFacts/Settings/dev.json b/src/Stats.RollUpDownloadFacts/Settings/dev.json new file mode 100644 index 000000000..08dfe6c64 --- /dev/null +++ b/src/Stats.RollUpDownloadFacts/Settings/dev.json @@ -0,0 +1,16 @@ +{ + "Initialization": { + "MinAgeInDays": "43" + }, + + "StatisticsDb": { + "ConnectionString": "Data Source=tcp:#{Deployment.Azure.Sql.StatisticsDatabaseAddress};Initial Catalog=nuget-dev-statistics;User ID=$$Dev-StatisticsDBWriter-UserName$$;Password=$$Dev-StatisticsDBWriter-Password$$;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;" + }, + + "KeyVault_VaultName": "#{Deployment.Azure.KeyVault.VaultName}", + "KeyVault_ClientId": "#{Deployment.Azure.KeyVault.ClientId}", + "KeyVault_CertificateThumbprint": "#{Deployment.Azure.KeyVault.CertificateThumbprint}", + "KeyVault_ValidateCertificate": true, + "KeyVault_StoreName": "My", + "KeyVault_StoreLocation": "LocalMachine" +} \ No newline at end of file diff --git a/src/Stats.RollUpDownloadFacts/Settings/int.json b/src/Stats.RollUpDownloadFacts/Settings/int.json new file mode 100644 index 000000000..7a920ed55 --- /dev/null +++ b/src/Stats.RollUpDownloadFacts/Settings/int.json @@ -0,0 +1,16 @@ +{ + "Initialization": { + "MinAgeInDays": "43" + }, + + "StatisticsDb": { + "ConnectionString": "Data Source=tcp:#{Deployment.Azure.Sql.StatisticsDatabaseAddress};Initial Catalog=nuget-int-statistics;User ID=$$Int-StatisticsDBWriter-UserName$$;Password=$$Int-StatisticsDBWriter-Password$$;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;" + }, + + "KeyVault_VaultName": "#{Deployment.Azure.KeyVault.VaultName}", + "KeyVault_ClientId": "#{Deployment.Azure.KeyVault.ClientId}", + "KeyVault_CertificateThumbprint": "#{Deployment.Azure.KeyVault.CertificateThumbprint}", + "KeyVault_ValidateCertificate": true, + "KeyVault_StoreName": "My", + "KeyVault_StoreLocation": "LocalMachine" +} \ No newline at end of file diff --git a/src/Stats.RollUpDownloadFacts/Settings/prod.json b/src/Stats.RollUpDownloadFacts/Settings/prod.json new file mode 100644 index 000000000..23e6bba77 --- /dev/null +++ b/src/Stats.RollUpDownloadFacts/Settings/prod.json @@ -0,0 +1,16 @@ +{ + "Initialization": { + "MinAgeInDays": "43" + }, + + "StatisticsDb": { + "ConnectionString": "Data Source=tcp:#{Deployment.Azure.Sql.StatisticsDatabaseAddress};Initial Catalog=nuget-prod-statistics;User ID=$$Prod-StatisticsDBWriter-UserName$$;Password=$$Prod-StatisticsDBWriter-Password$$;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;" + }, + + "KeyVault_VaultName": "#{Deployment.Azure.KeyVault.VaultName}", + "KeyVault_ClientId": "#{Deployment.Azure.KeyVault.ClientId}", + "KeyVault_CertificateThumbprint": "#{Deployment.Azure.KeyVault.CertificateThumbprint}", + "KeyVault_ValidateCertificate": true, + "KeyVault_StoreName": "My", + "KeyVault_StoreLocation": "LocalMachine" +} \ No newline at end of file diff --git a/src/Stats.RollUpDownloadFacts/Stats.RollUpDownloadFacts.csproj b/src/Stats.RollUpDownloadFacts/Stats.RollUpDownloadFacts.csproj index ce168dc80..54a710af6 100644 --- a/src/Stats.RollUpDownloadFacts/Stats.RollUpDownloadFacts.csproj +++ b/src/Stats.RollUpDownloadFacts/Stats.RollUpDownloadFacts.csproj @@ -43,13 +43,17 @@ - + + + + + @@ -81,12 +85,6 @@ 1.0.0 - - 2.25.0 - - - 2.27.0 - diff --git a/src/Stats.RollUpDownloadFacts/Stats.RollUpDownloadFacts.nuspec b/src/Stats.RollUpDownloadFacts/Stats.RollUpDownloadFacts.nuspec index 3df47101c..1dbd510b8 100644 --- a/src/Stats.RollUpDownloadFacts/Stats.RollUpDownloadFacts.nuspec +++ b/src/Stats.RollUpDownloadFacts/Stats.RollUpDownloadFacts.nuspec @@ -18,5 +18,7 @@ + + \ No newline at end of file