Migrate CreateAzureCdnWarehouseReports to use JsonConfig (#526)

NuGet · Aug 8, 2018 · cbd9bae · cbd9bae
1 parent 64f9076
commit cbd9bae
Show file tree

Hide file tree

Showing 23 changed files with 387 additions and 150 deletions.
diff --git a/...eateAzureCdnWarehouseReports/Configuration/CreateAzureCdnWarehouseReportsConfiguration.cs b/...eateAzureCdnWarehouseReports/Configuration/CreateAzureCdnWarehouseReportsConfiguration.cs
@@ -0,0 +1,22 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+namespace Stats.CreateAzureCdnWarehouseReports
+{
+    public class CreateAzureCdnWarehouseReportsConfiguration
+    {
+        public string AzureCdnCloudStorageAccount { get; set; }
+
+        public string AzureCdnCloudStorageContainerName { get; set; }
+
+        public string DataStorageAccount { get; set; }
+
+        public string DataContainerName { get; set; }
+
+        public int? CommandTimeOut { get; set; }
+
+        public int? PerPackageReportDegreeOfParallelism { get; set; }
+
+        public string ReportName { get; set; }
+    }
+}
diff --git a/...ats.CreateAzureCdnWarehouseReports/Job.cs → ...orts/CreateAzureCdnWarehouseReportsJob.cs b/...ats.CreateAzureCdnWarehouseReports/Job.cs → ...orts/CreateAzureCdnWarehouseReportsJob.cs
diff --git a/src/Stats.CreateAzureCdnWarehouseReports/DownloadCountReport.cs b/src/Stats.CreateAzureCdnWarehouseReports/DownloadCountReport.cs
@@ -10,7 +10,6 @@
 using Microsoft.Extensions.Logging;
 using Newtonsoft.Json;
 using Newtonsoft.Json.Linq;
-using NuGet.Services.Sql;
 using NuGet.Versioning;
 
 namespace Stats.CreateAzureCdnWarehouseReports
@@ -25,22 +24,23 @@ public class DownloadCountReport
         public DownloadCountReport(
             ILogger<DownloadCountReport> logger,
             IEnumerable<StorageContainerTarget> targets,
-            ISqlConnectionFactory statisticsDbConnectionFactory,
-            ISqlConnectionFactory galleryDbConnectionFactory)
-            : base(logger, targets, statisticsDbConnectionFactory, galleryDbConnectionFactory)
+            Func<Task<SqlConnection>> openStatisticsSqlConnectionAsync,
+            Func<Task<SqlConnection>> openGallerySqlConnectionAsync)
+            : base(logger, targets, openStatisticsSqlConnectionAsync, openGallerySqlConnectionAsync)
         {
         }
 
         public async Task Run()
         {
             // Gather download count data from statistics warehouse
             IReadOnlyCollection<DownloadCountData> downloadData;
-            _logger.LogInformation("Gathering Download Counts from {DataSource}/{InitialCatalog}...",
-                StatisticsDbConnectionFactory.DataSource, StatisticsDbConnectionFactory.InitialCatalog);
 
-            using (var connection = await StatisticsDbConnectionFactory.CreateAsync())
+            using (var connection = await OpenStatisticsSqlConnectionAsync())
             using (var transaction = connection.BeginTransaction(IsolationLevel.Snapshot))
             {
+                _logger.LogInformation("Gathering Download Counts from {DataSource}/{InitialCatalog}...",
+                    connection.DataSource, connection.Database);
+
                 downloadData = (await connection.QueryWithRetryAsync<DownloadCountData>(
                     _storedProcedureName, commandType: CommandType.StoredProcedure, transaction: transaction, commandTimeout: _defaultCommandTimeout)).ToList();
             }

diff --git a/src/Stats.CreateAzureCdnWarehouseReports/DownloadsPerToolVersionReport.cs b/src/Stats.CreateAzureCdnWarehouseReports/DownloadsPerToolVersionReport.cs
@@ -11,7 +11,6 @@
 using Microsoft.WindowsAzure.Storage;
 using Newtonsoft.Json;
 using Newtonsoft.Json.Linq;
-using NuGet.Services.Sql;
 
 namespace Stats.CreateAzureCdnWarehouseReports
 {
@@ -26,23 +25,24 @@ public DownloadsPerToolVersionReport(
             ILogger<DownloadsPerToolVersionReport> logger,
             CloudStorageAccount cloudStorageAccount,
             string statisticsContainerName,
-            ISqlConnectionFactory statisticsDbConnectionFactory,
-            ISqlConnectionFactory galleryDbConnectionFactory)
+            Func<Task<SqlConnection>> openStatisticsSqlConnectionAsync,
+            Func<Task<SqlConnection>> openGallerySqlConnectionAsync)
             : base(logger, new[] { new StorageContainerTarget(cloudStorageAccount, statisticsContainerName) },
-                  statisticsDbConnectionFactory, galleryDbConnectionFactory)
+                  openStatisticsSqlConnectionAsync, openGallerySqlConnectionAsync)
         {
         }
 
         public async Task Run()
         {
             // Gather download count data from statistics warehouse
             IReadOnlyCollection<ToolDownloadCountData> data;
-            _logger.LogInformation("Gathering Tools Download Counts from {DataSource}/{InitialCatalog}...",
-                StatisticsDbConnectionFactory.DataSource, StatisticsDbConnectionFactory.InitialCatalog);
 
-            using (var connection = await StatisticsDbConnectionFactory.CreateAsync())
+            using (var connection = await OpenStatisticsSqlConnectionAsync())
             using (var transaction = connection.BeginTransaction(IsolationLevel.Snapshot))
             {
+                _logger.LogInformation("Gathering Tools Download Counts from {DataSource}/{InitialCatalog}...",
+                    connection.DataSource, connection.Database);
+
                 data = (await connection.QueryWithRetryAsync<ToolDownloadCountData>(
                     _storedProcedureName, commandType: CommandType.StoredProcedure, transaction: transaction, commandTimeout: _defaultCommandTimeout)).ToList();
             }

diff --git a/src/Stats.CreateAzureCdnWarehouseReports/GalleryTotalsReport.cs b/src/Stats.CreateAzureCdnWarehouseReports/GalleryTotalsReport.cs
@@ -9,7 +9,6 @@
 using Microsoft.Extensions.Logging;
 using Microsoft.WindowsAzure.Storage;
 using Newtonsoft.Json;
-using NuGet.Services.Sql;
 
 namespace Stats.CreateAzureCdnWarehouseReports
 {
@@ -27,23 +26,24 @@ public GalleryTotalsReport(
             ILogger<GalleryTotalsReport> logger,
             CloudStorageAccount cloudStorageAccount,
             string statisticsContainerName,
-            ISqlConnectionFactory statisticsDbConnectionFactory,
-            ISqlConnectionFactory galleryDbConnectionFactory)
+            Func<Task<SqlConnection>> openStatisticsSqlConnectionAsync,
+            Func<Task<SqlConnection>> openGallerySqlConnectionAsync)
             : base(logger, new[] { new StorageContainerTarget(cloudStorageAccount, statisticsContainerName) },
-                  statisticsDbConnectionFactory, galleryDbConnectionFactory)
+                  openStatisticsSqlConnectionAsync, openGallerySqlConnectionAsync)
         {
         }
 
         public async Task Run()
         {
             // gather package numbers from gallery database
             GalleryTotalsData totalsData;
-            _logger.LogInformation("Gathering Gallery Totals from {GalleryDataSource}/{GalleryInitialCatalog}...",
-                GalleryDbConnectionFactory.DataSource, GalleryDbConnectionFactory.InitialCatalog);
 
-            using (var connection = await GalleryDbConnectionFactory.CreateAsync())
+            using (var connection = await OpenGallerySqlConnectionAsync())
             using (var transaction = connection.BeginTransaction(IsolationLevel.Snapshot))
             {
+                _logger.LogInformation("Gathering Gallery Totals from {GalleryDataSource}/{GalleryInitialCatalog}...",
+                    connection.DataSource, connection.Database);
+
                 totalsData = (await connection.QueryWithRetryAsync<GalleryTotalsData>(
                     GalleryQuery, commandType: CommandType.Text, transaction: transaction)).First();
             }
@@ -52,12 +52,12 @@ public async Task Run()
             _logger.LogInformation("Unique packages: {UniquePackagesCount}", totalsData.UniquePackages);
 
             // gather download count data from statistics warehouse
-            _logger.LogInformation("Gathering Gallery Totals from {StatisticsDataSource}/{StatisticsInitialCatalog}...",
-                StatisticsDbConnectionFactory.DataSource, StatisticsDbConnectionFactory.InitialCatalog);
-
-            using (var connection = await StatisticsDbConnectionFactory.CreateAsync())
+            using (var connection = await OpenStatisticsSqlConnectionAsync())
             using (var transaction = connection.BeginTransaction(IsolationLevel.Snapshot))
             {
+                _logger.LogInformation("Gathering Gallery Totals from {StatisticsDataSource}/{StatisticsInitialCatalog}...",
+                    connection.DataSource, connection.Database);
+
                 totalsData.Downloads = (await connection.ExecuteScalarWithRetryAsync<long>(
                     WarehouseStoredProcedureName,
                     commandType: CommandType.StoredProcedure,

diff --git a/src/Stats.CreateAzureCdnWarehouseReports/Program.cs b/src/Stats.CreateAzureCdnWarehouseReports/Program.cs
@@ -9,7 +9,7 @@ class Program
     {
         static void Main(string[] args)
         {
-            var job = new Job();
+            var job = new CreateAzureCdnWarehouseReportsJob();
             JobRunner.Run(job, args).Wait();
         }
     }

diff --git a/src/Stats.CreateAzureCdnWarehouseReports/ReportBase.cs b/src/Stats.CreateAzureCdnWarehouseReports/ReportBase.cs
@@ -3,12 +3,12 @@
 
 using System;
 using System.Collections.Generic;
+using System.Data.SqlClient;
 using System.Linq;
 using System.Threading.Tasks;
 using Microsoft.Extensions.Logging;
 using Microsoft.WindowsAzure.Storage.Blob;
 using Microsoft.WindowsAzure.Storage.RetryPolicies;
-using NuGet.Services.Sql;
 
 namespace Stats.CreateAzureCdnWarehouseReports
 {
@@ -18,20 +18,20 @@ public abstract class ReportBase
 
         protected readonly IReadOnlyCollection<StorageContainerTarget> Targets;
 
-        protected readonly ISqlConnectionFactory StatisticsDbConnectionFactory;
+        protected readonly Func<Task<SqlConnection>> OpenStatisticsSqlConnectionAsync;
 
-        protected ISqlConnectionFactory GalleryDbConnectionFactory;
+        protected Func<Task<SqlConnection>> OpenGallerySqlConnectionAsync;
 
         protected ReportBase(
             ILogger<ReportBase> logger,
             IEnumerable<StorageContainerTarget> targets,
-            ISqlConnectionFactory statisticsDbConnectionFactory,
-            ISqlConnectionFactory galleryDbConnectionFactory)
+            Func<Task<SqlConnection>> openStatisticsSqlConnectionAsync,
+            Func<Task<SqlConnection>> openGallerySqlConnectionAsync)
         {
             _logger = logger;
             Targets = targets.ToList().AsReadOnly();
-            StatisticsDbConnectionFactory = statisticsDbConnectionFactory;
-            GalleryDbConnectionFactory = galleryDbConnectionFactory;
+            OpenStatisticsSqlConnectionAsync = openStatisticsSqlConnectionAsync;
+            OpenGallerySqlConnectionAsync = openGallerySqlConnectionAsync;
         }
 
         protected async Task<CloudBlobContainer> GetBlobContainer(StorageContainerTarget target)

diff --git a/src/Stats.CreateAzureCdnWarehouseReports/ReportDataCollector.cs b/src/Stats.CreateAzureCdnWarehouseReports/ReportDataCollector.cs
@@ -8,27 +8,26 @@
 using System.Diagnostics;
 using System.Threading.Tasks;
 using Microsoft.Extensions.Logging;
-using NuGet.Services.Sql;
 
 namespace Stats.CreateAzureCdnWarehouseReports
 {
     internal class ReportDataCollector
     {
         private int _commandTimeoutSeconds;
         private readonly string _procedureName;
-        private readonly ISqlConnectionFactory _sourceDbConnectionFactory;
+        private readonly Func<Task<SqlConnection>> _openGallerySqlConnectionAsync;
 
         private ILogger<ReportDataCollector> _logger;
 
         public ReportDataCollector(
             ILogger<ReportDataCollector> logger,
             string procedureName,
-            ISqlConnectionFactory sourceDbConnectionFactory,
+            Func<Task<SqlConnection>> openGallerySqlConnectionAsync,
             int timeout)
         {
             _logger = logger;
             _procedureName = procedureName;
-            _sourceDbConnectionFactory = sourceDbConnectionFactory;
+            _openGallerySqlConnectionAsync = openGallerySqlConnectionAsync;
             _commandTimeoutSeconds = timeout;
         }
 
@@ -48,7 +47,7 @@ public async Task<DataTable> CollectAsync(DateTime reportGenerationTime, params
 
         public static async Task<IReadOnlyCollection<DirtyPackageId>> GetDirtyPackageIds(
             ILogger logger,
-            ISqlConnectionFactory sourceDbConnectionFactory,
+            Func<Task<SqlConnection>> openGallerySqlConnectionAsync,
             DateTime reportGenerationTime,
             int commandTimeout)
         {
@@ -57,19 +56,19 @@ public static async Task<IReadOnlyCollection<DirtyPackageId>> GetDirtyPackageIds
             IReadOnlyCollection<DirtyPackageId> packageIds = new List<DirtyPackageId>();
 
             // Get the data
-            await WithRetry(async () => packageIds = await GetDirtyPackageIdsFromWarehouse(sourceDbConnectionFactory, reportGenerationTime, commandTimeout), logger);
+            await WithRetry(async () => packageIds = await GetDirtyPackageIdsFromWarehouse(openGallerySqlConnectionAsync, reportGenerationTime, commandTimeout), logger);
 
             logger.LogInformation("Found {DirtyPackagesCount} dirty packages to update.", packageIds.Count);
 
             return packageIds;
         }
 
         public static async Task<IReadOnlyCollection<string>> ListInactivePackageIdReports(
-            ISqlConnectionFactory sourceDbConnectionFactory,
+            Func<Task<SqlConnection>> openGallerySqlConnectionAsync,
             DateTime reportGenerationTime,
             int commandTimeout)
         {
-            using (var connection = await sourceDbConnectionFactory.CreateAsync())
+            using (var connection = await openGallerySqlConnectionAsync())
             {
                 var command = new SqlCommand("[dbo].[DownloadReportListInactive]", connection);
                 command.CommandType = CommandType.StoredProcedure;
@@ -122,7 +121,7 @@ private static async Task WithRetry(Func<Task> action, ILogger logger)
 
         private async Task<DataTable> ExecuteSql(DateTime reportGenerationTime, params Tuple<string, int, string>[] parameters)
         {
-            using (var connection = await _sourceDbConnectionFactory.CreateAsync())
+            using (var connection = await _openGallerySqlConnectionAsync())
             {
                 var command = new SqlCommand(_procedureName, connection);
                 command.CommandType = CommandType.StoredProcedure;
@@ -146,11 +145,11 @@ private async Task<DataTable> ExecuteSql(DateTime reportGenerationTime, params T
         }
 
         private static async Task<IReadOnlyCollection<DirtyPackageId>> GetDirtyPackageIdsFromWarehouse(
-            ISqlConnectionFactory sourceDbConnectionFactory,
+            Func<Task<SqlConnection>> openGallerySqlConnectionAsync,
             DateTime reportGenerationTime,
             int commandTimeout)
         {
-            using (var connection = await sourceDbConnectionFactory.CreateAsync())
+            using (var connection = await openGallerySqlConnectionAsync())
             {
                 var command = new SqlCommand("[dbo].[GetDirtyPackageIds]", connection);
                 command.CommandType = CommandType.StoredProcedure;
@@ -172,11 +171,11 @@ private static async Task<IReadOnlyCollection<DirtyPackageId>> GetDirtyPackageId
         }
 
         public static async Task UpdateDirtyPackageIdCursor(
-            ISqlConnectionFactory sourceDbConnectionFactory,
+            Func<Task<SqlConnection>> openGallerySqlConnectionAsync,
             DateTime runToCursor,
             int commandTimeout)
         {
-            using (var connection = await sourceDbConnectionFactory.CreateAsync())
+            using (var connection = await openGallerySqlConnectionAsync())
             {
                 var command = new SqlCommand("[dbo].[UpdateDirtyPackageIdCursor]", connection);
                 command.CommandType = CommandType.StoredProcedure;

diff --git a/src/Stats.CreateAzureCdnWarehouseReports/Scripts/Stats.CreateAzureCdnWarehouseReports.cmd b/src/Stats.CreateAzureCdnWarehouseReports/Scripts/Stats.CreateAzureCdnWarehouseReports.cmd
@@ -3,26 +3,16 @@
 cd bin
 
 :Top
-	echo "Starting job - #{Jobs.stats.createazurecdnwarehousereports.Title}"
+echo "Starting job - #{Jobs.stats.createazurecdnwarehousereports.Title}"
 
-	title #{Jobs.stats.createazurecdnwarehousereports.Title}
+title #{Jobs.stats.createazurecdnwarehousereports.Title}
 
-	start /w stats.createazurecdnwarehousereports.exe ^
-		-VaultName "#{Deployment.Azure.KeyVault.VaultName}" ^
-		-ClientId "#{Deployment.Azure.KeyVault.ClientId}" ^
-		-CertificateThumbprint "#{Deployment.Azure.KeyVault.CertificateThumbprint}" ^
-		-AzureCdnCloudStorageAccount "#{Jobs.stats.createazurecdnwarehousereports.AzureCdn.CloudStorageAccount}" ^
-		-AzureCdnCloudStorageContainerName "#{Jobs.stats.createazurecdnwarehousereports.AzureCdn.CloudStorageContainerName}" ^
-		-StatisticsDatabase "#{Jobs.stats.createazurecdnwarehousereports.StatisticsDatabase}" ^
-		-SourceDatabase "#{Jobs.stats.createazurecdnwarehousereports.SourceDatabase}" ^
-		-DataStorageAccount "#{Jobs.stats.createazurecdnwarehousereports.DataStorageAccount}" ^
-		-InstrumentationKey "#{Jobs.stats.createazurecdnwarehousereports.InstrumentationKey}" ^
-		-DataContainerName "#{Jobs.stats.createazurecdnwarehousereports.DataContainerName}" ^
-		-CommandTimeOut "#{Jobs.stats.createazurecdnwarehousereports.CommandTimeOut}" ^
-		-PerPackageReportDegreeOfParallelism "#{Jobs.stats.createazurecdnwarehousereports.PerPackageReportDegreeOfParallelism}" ^
-		-verbose true ^
-		-Interval #{Jobs.stats.createazurecdnwarehousereports.Interval} 
+start /w stats.createazurecdnwarehousereports.exe ^
+    -Configuration "#{Jobs.stats.createazurecdnwarehousereports.Configuration}"
+	-InstrumentationKey "#{Jobs.stats.createazurecdnwarehousereports.InstrumentationKey}" ^
+	-Interval #{Jobs.stats.createazurecdnwarehousereports.Interval}  ^
+	-verbose true
 
-	echo "Finished #{Jobs.stats.createazurecdnwarehousereports.Title}"
+echo "Finished #{Jobs.stats.createazurecdnwarehousereports.Title}"
 
-	goto Top
+goto Top
diff --git a/src/Stats.CreateAzureCdnWarehouseReports/Settings/dev.json b/src/Stats.CreateAzureCdnWarehouseReports/Settings/dev.json
@@ -0,0 +1,25 @@
+{
+  "Initialization": {
+    "AzureCdnCloudStorageAccount": "DefaultEndpointsProtocol=https;AccountName=nugetdevlegacy;AccountKey=$$Dev-NuGetDevLegacyStorage-Key$$",
+    "AzureCdnCloudStorageContainerName": "nuget-cdnstats",
+    "DataStorageAccount": "DefaultEndpointsProtocol=https;AccountName=nugetdev0;AccountKey=$$Dev-NuGetDev0Storage-Key$$",
+    "DataContainerName": "ng-search-data",
+    "CommandTimeOut": "7200",
+    "PerPackageReportDegreeOfParallelism": "64"
+  },
+
+  "GalleryDb": {
+    "ConnectionString": "Data Source=tcp:#{Deployment.Azure.Sql.GalleryDatabaseAddress};Initial Catalog=nuget-dev-0-v2gallery;User ID=$$Dev-GalleryDBReadOnly-UserName$$;Password=$$Dev-GalleryDBReadOnly-Password$$;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;"
+  },
+
+  "StatisticsDb": {
+    "ConnectionString": "Data Source=tcp:#{Deployment.Azure.Sql.StatisticsDatabaseAddress};Initial Catalog=nuget-dev-statistics;User ID=$$Dev-StatisticsDBWriter-UserName$$;Password=$$Dev-StatisticsDBWriter-Password$$;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;"
+  },
+
+  "KeyVault_VaultName": "#{Deployment.Azure.KeyVault.VaultName}",
+  "KeyVault_ClientId": "#{Deployment.Azure.KeyVault.ClientId}",
+  "KeyVault_CertificateThumbprint": "#{Deployment.Azure.KeyVault.CertificateThumbprint}",
+  "KeyVault_ValidateCertificate": true,
+  "KeyVault_StoreName": "My",
+  "KeyVault_StoreLocation": "LocalMachine"
+}
diff --git a/src/Stats.CreateAzureCdnWarehouseReports/Settings/int.json b/src/Stats.CreateAzureCdnWarehouseReports/Settings/int.json
@@ -0,0 +1,25 @@
+{
+  "Initialization": {
+    "AzureCdnCloudStorageAccount": "DefaultEndpointsProtocol=https;AccountName=nugetint0;AccountKey=$$Int-NuGetInt0Storage-Key$$",
+    "AzureCdnCloudStorageContainerName": "nuget-cdnstats",
+    "DataStorageAccount": "DefaultEndpointsProtocol=https;AccountName=nugetint0;AccountKey=$$Int-NuGetInt0Storage-Key$$",
+    "DataContainerName": "ng-search-data",
+    "CommandTimeOut": "7200",
+    "PerPackageReportDegreeOfParallelism": "64"
+  },
+
+  "GalleryDb": {
+    "ConnectionString": "Data Source=tcp:#{Deployment.Azure.Sql.GalleryDatabaseAddress};Initial Catalog=nuget-int-0-v2gallery;User ID=$$Int-GalleryDBReadOnly-UserName$$;Password=$$Int-GalleryDBReadOnly-Password$$;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;"
+  },
+
+  "StatisticsDb": {
+    "ConnectionString": "Data Source=tcp:#{Deployment.Azure.Sql.StatisticsDatabaseAddress};Initial Catalog=nuget-int-statistics;User ID=$$Int-StatisticsDBWriter-UserName$$;Password=$$Int-StatisticsDBWriter-Password$$;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;"
+  },
+
+  "KeyVault_VaultName": "#{Deployment.Azure.KeyVault.VaultName}",
+  "KeyVault_ClientId": "#{Deployment.Azure.KeyVault.ClientId}",
+  "KeyVault_CertificateThumbprint": "#{Deployment.Azure.KeyVault.CertificateThumbprint}",
+  "KeyVault_ValidateCertificate": true,
+  "KeyVault_StoreName": "My",
+  "KeyVault_StoreLocation": "LocalMachine"
+}