Migrate ImportCdnStats to use JsonConfig (#524)

NuGet · Aug 8, 2018 · 64f9076 · 64f9076
1 parent ffb3d42
commit 64f9076
Show file tree

Hide file tree

Showing 12 changed files with 146 additions and 64 deletions.
diff --git a/src/Stats.ImportAzureCdnStatistics/Configuration/ImportAzureCdnStatisticsConfiguration.cs b/src/Stats.ImportAzureCdnStatistics/Configuration/ImportAzureCdnStatisticsConfiguration.cs
@@ -0,0 +1,18 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+namespace Stats.ImportAzureCdnStatistics
+{
+    public class ImportAzureCdnStatisticsConfiguration
+    {
+        public string AzureCdnCloudStorageAccount { get; set; }
+
+        public string AzureCdnCloudStorageContainerName { get; set; }
+
+        public string AzureCdnPlatform { get; set; }
+
+        public string AzureCdnAccountNumber { get; set; }
+
+        public bool AggregatesOnly { get; set; }
+    }
+}
diff --git a/src/Stats.ImportAzureCdnStatistics/DataImporter.cs b/src/Stats.ImportAzureCdnStatistics/DataImporter.cs
@@ -1,29 +1,29 @@
 // Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
+using System;
 using System.Data;
 using System.Data.SqlClient;
 using System.Threading.Tasks;
-using NuGet.Services.Sql;
 
 namespace Stats.ImportAzureCdnStatistics
 {
     internal class DataImporter
     {
-        private readonly ISqlConnectionFactory _statisticsDbConnectionFactory;
+        private readonly Func<Task<SqlConnection>> _openStatisticsSqlConnectionAsync;
         private const string _sqlSelectTop1FromTable = "SELECT TOP 1 * FROM [dbo].[{0}]";
 
-        public DataImporter(ISqlConnectionFactory statisticsDbConnectionFactory)
+        public DataImporter(Func<Task<SqlConnection>> openStatisticsSqlConnectionAsync)
         {
-            _statisticsDbConnectionFactory = statisticsDbConnectionFactory;
+            _openStatisticsSqlConnectionAsync = openStatisticsSqlConnectionAsync;
         }
 
         public async Task<DataTable> GetDataTableAsync(string tableName)
         {
             var dataTable = new DataTable();
             var query = string.Format(_sqlSelectTop1FromTable, tableName);
 
-            using (var connection = await _statisticsDbConnectionFactory.CreateAsync())
+            using (var connection = await _openStatisticsSqlConnectionAsync())
             {
                 var tableAdapter = new SqlDataAdapter(query, connection)
                 {

diff --git a/src/Stats.ImportAzureCdnStatistics/Job.cs → ...Statistics/ImportAzureCdnStatisticsJob.cs b/src/Stats.ImportAzureCdnStatistics/Job.cs → ...Statistics/ImportAzureCdnStatisticsJob.cs
@@ -6,66 +6,57 @@
 using System.ComponentModel.Design;
 using System.Globalization;
 using System.Threading.Tasks;
+using Autofac;
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Options;
 using Microsoft.WindowsAzure.Storage;
 using Microsoft.WindowsAzure.Storage.Blob;
 using Microsoft.WindowsAzure.Storage.RetryPolicies;
 using NuGet.Jobs;
-using NuGet.Services.KeyVault;
-using NuGet.Services.Sql;
+using NuGet.Jobs.Configuration;
 using Stats.AzureCdnLogs.Common;
 
 namespace Stats.ImportAzureCdnStatistics
 {
-    public class Job
-        : JobBase
+    public class ImportAzureCdnStatisticsJob : JsonConfigurationJob
     {
-        private bool _aggregatesOnly;
-        private string _azureCdnAccountNumber;
-        private string _cloudStorageContainerName;
+        private ImportAzureCdnStatisticsConfiguration _configuration;
         private AzureCdnPlatform _azureCdnPlatform;
-        private ISqlConnectionFactory _statisticsDbConnectionFactory;
-        private CloudStorageAccount _cloudStorageAccount;
         private CloudBlobClient _cloudBlobClient;
         private LogFileProvider _blobLeaseManager;
 
         public override void Init(IServiceContainer serviceContainer, IDictionary<string, string> jobArgsDictionary)
         {
-            var secretInjector = (ISecretInjector)serviceContainer.GetService(typeof(ISecretInjector));
-            var statisticsDbConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.StatisticsDatabase);
-            _statisticsDbConnectionFactory = new AzureSqlConnectionFactory(statisticsDbConnectionString, secretInjector);
+            base.Init(serviceContainer, jobArgsDictionary);
 
-            var azureCdnPlatform = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.AzureCdnPlatform);
-            var cloudStorageAccountConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.AzureCdnCloudStorageAccount);
-            _cloudStorageAccount = ValidateAzureCloudStorageAccount(cloudStorageAccountConnectionString);
+            _configuration = _serviceProvider.GetRequiredService<IOptionsSnapshot<ImportAzureCdnStatisticsConfiguration>>().Value;
 
-            _azureCdnAccountNumber = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.AzureCdnAccountNumber);
-            _azureCdnPlatform = ValidateAzureCdnPlatform(azureCdnPlatform);
-            _cloudStorageContainerName = ValidateAzureContainerName(JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.AzureCdnCloudStorageContainerName));
+            _azureCdnPlatform = ValidateAzureCdnPlatform(_configuration.AzureCdnPlatform);
 
-            _aggregatesOnly = JobConfigurationManager.TryGetBoolArgument(jobArgsDictionary, JobArgumentNames.AggregatesOnly);
-
-            // construct a cloud blob client for the configured storage account
-            _cloudBlobClient = _cloudStorageAccount.CreateCloudBlobClient();
+            var cloudStorageAccount = ValidateAzureCloudStorageAccount(_configuration.AzureCdnCloudStorageAccount);
+            _cloudBlobClient = cloudStorageAccount.CreateCloudBlobClient();
             _cloudBlobClient.DefaultRequestOptions.RetryPolicy = new ExponentialRetry(TimeSpan.FromSeconds(10), 5);
 
-            // Get the source blob container (containing compressed log files)
-            // and construct a log source (fetching raw logs from the source blob container)
-            var sourceBlobContainer = _cloudBlobClient.GetContainerReference(_cloudStorageContainerName);
-            _blobLeaseManager = new LogFileProvider(sourceBlobContainer, LoggerFactory);
+            _blobLeaseManager = new LogFileProvider(
+                _cloudBlobClient.GetContainerReference(_configuration.AzureCdnCloudStorageContainerName),
+                LoggerFactory);
         }
 
         public override async Task Run()
         {
             // Get the target blob container (for archiving decompressed log files)
-            var targetBlobContainer = _cloudBlobClient.GetContainerReference(_cloudStorageContainerName + "-archive");
+            var targetBlobContainer = _cloudBlobClient.GetContainerReference(
+                _configuration.AzureCdnCloudStorageContainerName + "-archive");
             await targetBlobContainer.CreateIfNotExistsAsync();
 
             // Get the dead-letter table (corrupted or failed blobs will end up there)
-            var deadLetterBlobContainer = _cloudBlobClient.GetContainerReference(_cloudStorageContainerName + "-deadletter");
+            var deadLetterBlobContainer = _cloudBlobClient.GetContainerReference(
+                _configuration.AzureCdnCloudStorageContainerName + "-deadletter");
             await deadLetterBlobContainer.CreateIfNotExistsAsync();
 
             // Create a parser
-            var warehouse = new Warehouse(LoggerFactory, _statisticsDbConnectionFactory);
+            var warehouse = new Warehouse(LoggerFactory, OpenSqlConnectionAsync<StatisticsDbConfiguration>);
             var statisticsBlobContainerUtility = new StatisticsBlobContainerUtility(
                 targetBlobContainer,
                 deadLetterBlobContainer,
@@ -74,11 +65,13 @@ public override async Task Run()
             var logProcessor = new LogFileProcessor(statisticsBlobContainerUtility, LoggerFactory, warehouse);
 
             // Get the next to-be-processed raw log file using the cdn raw log file name prefix
-            var prefix = string.Format(CultureInfo.InvariantCulture, "{0}_{1}_", _azureCdnPlatform.GetRawLogFilePrefix(), _azureCdnAccountNumber);
+            var prefix = string.Format(CultureInfo.InvariantCulture, "{0}_{1}_",
+                _azureCdnPlatform.GetRawLogFilePrefix(),
+                _configuration.AzureCdnAccountNumber);
 
             // Get next raw log file to be processed
             IReadOnlyCollection<string> alreadyAggregatedLogFiles = null;
-            if (_aggregatesOnly)
+            if (_configuration.AggregatesOnly)
             {
                 // We only want to process aggregates for the log files.
                 // Get the list of files we already processed so we can skip them.
@@ -90,9 +83,9 @@ public override async Task Run()
             {
                 var packageTranslator = new PackageTranslator("packagetranslations.json");
                 var packageStatisticsParser = new PackageStatisticsParser(packageTranslator, LoggerFactory);
-                await logProcessor.ProcessLogFileAsync(leasedLogFile, packageStatisticsParser, _aggregatesOnly);
+                await logProcessor.ProcessLogFileAsync(leasedLogFile, packageStatisticsParser, _configuration.AggregatesOnly);
 
-                if (_aggregatesOnly)
+                if (_configuration.AggregatesOnly)
                 {
                     _blobLeaseManager.TrackLastProcessedBlobUri(leasedLogFile.Uri);
                 }
@@ -139,5 +132,14 @@ private static string ValidateAzureContainerName(string containerName)
             }
             return containerName;
         }
+
+        protected override void ConfigureAutofacServices(ContainerBuilder containerBuilder)
+        {
+        }
+
+        protected override void ConfigureJobServices(IServiceCollection services, IConfigurationRoot configurationRoot)
+        {
+            ConfigureInitializationSection<ImportAzureCdnStatisticsConfiguration>(services, configurationRoot);
+        }
     }
 }
diff --git a/src/Stats.ImportAzureCdnStatistics/LogFileProcessor.cs b/src/Stats.ImportAzureCdnStatistics/LogFileProcessor.cs
@@ -31,7 +31,7 @@ public LogFileProcessor(
 
             _warehouse = warehouse ?? throw new ArgumentNullException(nameof(warehouse));
             _statisticsBlobContainerUtility = statisticsBlobContainerUtility ?? throw new ArgumentNullException(nameof(statisticsBlobContainerUtility));
-            _logger = loggerFactory.CreateLogger<Job>();
+            _logger = loggerFactory.CreateLogger<ImportAzureCdnStatisticsJob>();
         }
 
         public async Task ProcessLogFileAsync(ILeasedLogFile logFile, IPackageStatisticsParser packageStatisticsParser, bool aggregatesOnly = false)

diff --git a/src/Stats.ImportAzureCdnStatistics/Program.cs b/src/Stats.ImportAzureCdnStatistics/Program.cs
@@ -9,7 +9,7 @@ public class Program
     {
         public static void Main(string[] args)
         {
-            var job = new Job();
+            var job = new ImportAzureCdnStatisticsJob();
             JobRunner.Run(job, args).Wait();
         }
     }

diff --git a/src/Stats.ImportAzureCdnStatistics/Scripts/Stats.ImportAzureCdnStatistics.cmd b/src/Stats.ImportAzureCdnStatistics/Scripts/Stats.ImportAzureCdnStatistics.cmd
@@ -3,12 +3,16 @@
 cd bin
 
 :Top
-	echo "Starting job - #{Jobs.stats.importazurecdnstatistics.Title}"
+echo "Starting job - #{Jobs.stats.importazurecdnstatistics.Title}"
 
-	title #{Jobs.stats.importazurecdnstatistics.Title}
+title #{Jobs.stats.importazurecdnstatistics.Title}
 
-	start /w stats.importazurecdnstatistics.exe -VaultName "#{Deployment.Azure.KeyVault.VaultName}" -ClientId "#{Deployment.Azure.KeyVault.ClientId}" -CertificateThumbprint "#{Deployment.Azure.KeyVault.CertificateThumbprint}" -AzureCdnCloudStorageAccount "#{Jobs.stats.importazurecdnstatistics.AzureCdn.CloudStorageAccount}" -AzureCdnCloudStorageContainerName "#{Jobs.stats.importazurecdnstatistics.AzureCdn.CloudStorageContainerName}" -AzureCdnPlatform "#{Jobs.stats.importazurecdnstatistics.AzureCdn.Platform}" -AzureCdnAccountNumber "#{Jobs.stats.importazurecdnstatistics.AzureCdn.AccountNumber}" -StatisticsDatabase "#{Jobs.stats.importazurecdnstatistics.StatisticsDatabase}" -InstrumentationKey "#{Jobs.stats.importazurecdnstatistics.InstrumentationKey}" -verbose true -Interval #{Jobs.stats.importazurecdnstatistics.Interval} 
+start /w stats.importazurecdnstatistics.exe ^
+    -Configuration "#{Jobs.stats.importazurecdnstatistics.Configuration}" ^
+    -InstrumentationKey "#{Jobs.stats.importazurecdnstatistics.InstrumentationKey}" ^
+    -Interval #{Jobs.stats.importazurecdnstatistics.Interval} ^
+    -verbose true
 
-	echo "Finished #{Jobs.stats.importazurecdnstatistics.Title}"
+echo "Finished #{Jobs.stats.importazurecdnstatistics.Title}"
 
-	goto Top
+goto Top
diff --git a/src/Stats.ImportAzureCdnStatistics/Settings/dev.json b/src/Stats.ImportAzureCdnStatistics/Settings/dev.json
@@ -0,0 +1,19 @@
+{
+  "Initialization": {
+    "AzureCdnCloudStorageAccount": "DefaultEndpointsProtocol=https;AccountName=nugetdevlegacy;AccountKey=$$Dev-NuGetDevLegacyStorage-Key$$",
+    "AzureCdnCloudStorageContainerName": "nuget-cdnlogs-raw",
+    "AzureCdnPlatform": "HttpLargeObject",
+    "AzureCdnAccountNumber": "$$Dev-AzureCdn-AccountNumber$$"
+  },
+
+  "StatisticsDb": {
+    "ConnectionString": "Data Source=tcp:#{Deployment.Azure.Sql.StatisticsDatabaseAddress};Initial Catalog=nuget-dev-statistics;User ID=$$Dev-StatisticsDBWriter-UserName$$;Password=$$Dev-StatisticsDBWriter-Password$$;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;"
+  },
+
+  "KeyVault_VaultName": "#{Deployment.Azure.KeyVault.VaultName}",
+  "KeyVault_ClientId": "#{Deployment.Azure.KeyVault.ClientId}",
+  "KeyVault_CertificateThumbprint": "#{Deployment.Azure.KeyVault.CertificateThumbprint}",
+  "KeyVault_ValidateCertificate": true,
+  "KeyVault_StoreName": "My",
+  "KeyVault_StoreLocation": "LocalMachine"
+}
diff --git a/src/Stats.ImportAzureCdnStatistics/Settings/int.json b/src/Stats.ImportAzureCdnStatistics/Settings/int.json
@@ -0,0 +1,19 @@
+{
+  "Initialization": {
+    "AzureCdnCloudStorageAccount": "DefaultEndpointsProtocol=https;AccountName=nugetint0;AccountKey=$$Int-NuGetInt0Storage-Key$$",
+    "AzureCdnCloudStorageContainerName": "nuget-cdnlogs-raw",
+    "AzureCdnPlatform": "HttpLargeObject",
+    "AzureCdnAccountNumber": "$$Int-AzureCdn-AccountNumber$$"
+  },
+
+  "StatisticsDb": {
+    "ConnectionString": "Data Source=tcp:#{Deployment.Azure.Sql.StatisticsDatabaseAddress};Initial Catalog=nuget-int-statistics;User ID=$$Int-StatisticsDBWriter-UserName$$;Password=$$Int-StatisticsDBWriter-Password$$;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;"
+  },
+
+  "KeyVault_VaultName": "#{Deployment.Azure.KeyVault.VaultName}",
+  "KeyVault_ClientId": "#{Deployment.Azure.KeyVault.ClientId}",
+  "KeyVault_CertificateThumbprint": "#{Deployment.Azure.KeyVault.CertificateThumbprint}",
+  "KeyVault_ValidateCertificate": true,
+  "KeyVault_StoreName": "My",
+  "KeyVault_StoreLocation": "LocalMachine"
+}
diff --git a/src/Stats.ImportAzureCdnStatistics/Settings/prod.json b/src/Stats.ImportAzureCdnStatistics/Settings/prod.json
@@ -0,0 +1,19 @@
+{
+  "Initialization": {
+    "AzureCdnCloudStorageAccount": "DefaultEndpointsProtocol=https;AccountName=nugetgallery;AccountKey=$$Prod-NuGetGalleryStorage-Key$$",
+    "AzureCdnCloudStorageContainerName": "nuget-cdnlogs-raw",
+    "AzureCdnPlatform": "HttpLargeObject",
+    "AzureCdnAccountNumber": "$$Prod-AzureCdn-AccountNumber$$"
+  },
+
+  "StatisticsDb": {
+    "ConnectionString": "Data Source=tcp:#{Deployment.Azure.Sql.StatisticsDatabaseAddress};Initial Catalog=nuget-prod-statistics;User ID=$$Prod-StatisticsDBWriter-UserName$$;Password=$$Prod-StatisticsDBWriter-Password$$;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;"
+  },
+
+  "KeyVault_VaultName": "#{Deployment.Azure.KeyVault.VaultName}",
+  "KeyVault_ClientId": "#{Deployment.Azure.KeyVault.ClientId}",
+  "KeyVault_CertificateThumbprint": "#{Deployment.Azure.KeyVault.CertificateThumbprint}",
+  "KeyVault_ValidateCertificate": true,
+  "KeyVault_StoreName": "My",
+  "KeyVault_StoreLocation": "LocalMachine"
+}
diff --git a/src/Stats.ImportAzureCdnStatistics/Stats.ImportAzureCdnStatistics.csproj b/src/Stats.ImportAzureCdnStatistics/Stats.ImportAzureCdnStatistics.csproj
@@ -49,6 +49,7 @@
   </ItemGroup>
   <ItemGroup>
     <Compile Include="ApplicationInsightsHelper.cs" />
+    <Compile Include="Configuration\ImportAzureCdnStatisticsConfiguration.cs" />
     <Compile Include="IPackageStatisticsParser.cs" />
     <Compile Include="IStatisticsBlobContainerUtility.cs" />
     <Compile Include="PackageDefinition.cs" />
@@ -70,7 +71,7 @@
     <Compile Include="Dimensions\PlatformDimension.cs" />
     <Compile Include="DataImporter.cs" />
     <Compile Include="Dimensions\DateDimension.cs" />
-    <Compile Include="Job.cs" />
+    <Compile Include="ImportAzureCdnStatisticsJob.cs" />
     <Compile Include="Dimensions\PackageDimension.cs" />
     <Compile Include="Program.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
@@ -91,6 +92,9 @@
       <CopyToOutputDirectory>Always</CopyToOutputDirectory>
     </None>
     <None Include="Scripts\*" />
+    <None Include="Settings\dev.json" />
+    <None Include="Settings\int.json" />
+    <None Include="Settings\prod.json" />
     <None Include="Stats.ImportAzureCdnStatistics.nuspec" />
   </ItemGroup>
   <ItemGroup>
@@ -122,12 +126,6 @@
     <PackageReference Include="Microsoft.Extensions.Logging.Abstractions">
       <Version>1.0.0</Version>
     </PackageReference>
-    <PackageReference Include="NuGet.Services.Logging">
-      <Version>2.25.0</Version>
-    </PackageReference>
-    <PackageReference Include="NuGet.Services.Sql">
-      <Version>2.27.0</Version>
-    </PackageReference>
     <PackageReference Include="NuGet.Versioning">
       <Version>4.3.0-preview1-2524</Version>
     </PackageReference>

diff --git a/src/Stats.ImportAzureCdnStatistics/Stats.ImportAzureCdnStatistics.nuspec b/src/Stats.ImportAzureCdnStatistics/Stats.ImportAzureCdnStatistics.nuspec
@@ -18,5 +18,7 @@
     <file src="Scripts\PreDeploy.ps1" />
     <file src="Scripts\PostDeploy.ps1" />
     <file src="Scripts\nssm.exe" />
+
+    <file src="Settings\*.json" target="bin" />
   </files>
 </package>