Merge pull request #685 from NuGet/chenriks-legacy

Convert CollectCdnLogs jobs to JsonConfig
NuGet · Nov 29, 2018 · b09cd63 · b09cd63
2 parents 6d9b8e4 + e521773
commit b09cd63
Show file tree

Hide file tree

Showing 23 changed files with 411 additions and 192 deletions.
diff --git a/src/Stats.AzureCdnLogs.Common/Collect/AzureStatsLogDestination.cs b/src/Stats.AzureCdnLogs.Common/Collect/AzureStatsLogDestination.cs
@@ -24,9 +24,9 @@ public class AzureStatsLogDestination : ILogDestination
         private CloudBlobClient _cloudBlobClient;
         private CloudBlobContainer _cloudBlobContainer;
 
-        public AzureStatsLogDestination(string connectionString, string containerName)
+        public AzureStatsLogDestination(CloudStorageAccount storageAccount, string containerName)
         {
-            _azureAccount = CloudStorageAccount.Parse(connectionString);
+            _azureAccount = storageAccount;
             _cloudBlobClient = _azureAccount.CreateCloudBlobClient();
             _cloudBlobContainer = _cloudBlobClient.GetContainerReference(containerName);
             _cloudBlobContainer.CreateIfNotExists();

diff --git a/src/Stats.AzureCdnLogs.Common/Collect/AzureStatsLogSource.cs b/src/Stats.AzureCdnLogs.Common/Collect/AzureStatsLogSource.cs
@@ -33,9 +33,9 @@ public class AzureStatsLogSource : ILogSource
         /// </summary>
         /// <param name="connectionString">The connection string for the Azure account.</param>
         /// <param name="containerName">The container name.</param>
-        public AzureStatsLogSource(string connectionString, string containerName, int azureServerTimeoutInSeconds)
+        public AzureStatsLogSource(CloudStorageAccount storageAccount, string containerName, int azureServerTimeoutInSeconds)
         {
-            _azureAccount = CloudStorageAccount.Parse(connectionString);
+            _azureAccount = storageAccount;
             _blobClient = _azureAccount.CreateCloudBlobClient();
             _container = _blobClient.GetContainerReference(containerName);
             _blobRequestOptions = new BlobRequestOptions();

diff --git a/src/Stats.CollectAzureCdnLogs/Configuration/CollectAzureCdnLogsConfiguration.cs b/src/Stats.CollectAzureCdnLogs/Configuration/CollectAzureCdnLogsConfiguration.cs
@@ -0,0 +1,22 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+namespace Stats.CollectAzureCdnLogs
+{
+    public class CollectAzureCdnLogsConfiguration
+    {
+        public string AzureCdnAccountNumber { get; set; }
+
+        public string AzureCdnCloudStorageAccount { get; set; }
+
+        public string AzureCdnCloudStorageContainerName { get; set; }
+
+        public string AzureCdnPlatform { get; set; }
+
+        public string FtpSourceUri { get; set; }
+
+        public string FtpSourceUsername { get; set; }
+
+        public string FtpSourcePassword { get; set; }
+    }
+}
diff --git a/src/Stats.CollectAzureCdnLogs/Job.cs b/src/Stats.CollectAzureCdnLogs/Job.cs
@@ -9,9 +9,13 @@
 using System.Text;
 using System.Text.RegularExpressions;
 using System.Threading.Tasks;
+using Autofac;
 using ICSharpCode.SharpZipLib;
 using ICSharpCode.SharpZipLib.GZip;
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
 using Microsoft.WindowsAzure.Storage;
 using NuGet.Jobs;
 using Stats.AzureCdnLogs.Common;
@@ -20,31 +24,48 @@
 
 namespace Stats.CollectAzureCdnLogs
 {
-    public class Job
-         : JobBase
+    public class Job : JsonConfigurationJob
     {
         private static readonly DateTime _unixTimestamp = new DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc);
         private Uri _ftpServerUri;
-        private string _ftpUsername;
-        private string _ftpPassword;
-        private string _azureCdnAccountNumber;
         private AzureCdnPlatform _azureCdnPlatform;
         private CloudStorageAccount _cloudStorageAccount;
-        private string _cloudStorageContainerName;
+
+        private CollectAzureCdnLogsConfiguration _configuration;
 
         public override void Init(IServiceContainer serviceContainer, IDictionary<string, string> jobArgsDictionary)
         {
-            var ftpLogFolder = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.FtpSourceUri);
-            var azureCdnPlatform = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.AzureCdnPlatform);
-            var cloudStorageAccount = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.AzureCdnCloudStorageAccount);
-            _cloudStorageContainerName = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.AzureCdnCloudStorageContainerName);
-            _azureCdnAccountNumber = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.AzureCdnAccountNumber);
-            _ftpUsername = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.FtpSourceUsername);
-            _ftpPassword = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.FtpSourcePassword);
-
-            _ftpServerUri = ValidateFtpUri(ftpLogFolder);
-            _azureCdnPlatform = ValidateAzureCdnPlatform(azureCdnPlatform);
-            _cloudStorageAccount = ValidateAzureCloudStorageAccount(cloudStorageAccount);
+            base.Init(serviceContainer, jobArgsDictionary);
+
+            InitializeJobConfiguration(_serviceProvider);
+        }
+
+        public void InitializeJobConfiguration(IServiceProvider serviceProvider)
+        {
+            _configuration = serviceProvider.GetRequiredService<IOptionsSnapshot<CollectAzureCdnLogsConfiguration>>().Value;
+
+            if (string.IsNullOrEmpty(_configuration.AzureCdnAccountNumber))
+            {
+                throw new ArgumentException("Configuration 'AzureCdnAccountNumber' is required", nameof(_configuration));
+            }
+
+            if (string.IsNullOrEmpty(_configuration.AzureCdnCloudStorageContainerName))
+            {
+                throw new ArgumentException("Configuration 'AzureCdnCloudStorageContainerName' is required", nameof(_configuration));
+            }
+
+            if (string.IsNullOrEmpty(_configuration.FtpSourceUsername)) {
+                throw new ArgumentException("Configuration 'FtpSourceUsername' is required", nameof(_configuration));
+            }
+
+            if (string.IsNullOrEmpty(_configuration.FtpSourcePassword))
+            {
+                throw new ArgumentException("Configuration 'FtpSourcePassword' is required", nameof(_configuration));
+            }
+
+            _cloudStorageAccount = ValidateAzureCloudStorageAccount(_configuration.AzureCdnCloudStorageAccount);
+            _azureCdnPlatform = ValidateAzureCdnPlatform(_configuration.AzureCdnPlatform);
+            _ftpServerUri = ValidateFtpUri(_configuration.FtpSourceUri);
         }
 
         private static CloudStorageAccount ValidateAzureCloudStorageAccount(string cloudStorageAccount)
@@ -110,14 +131,14 @@ private static Uri ValidateFtpUri(string serverUrl)
 
         public override async Task Run()
         {
-            var ftpClient = new FtpRawLogClient(LoggerFactory, _ftpUsername, _ftpPassword);
+            var ftpClient = new FtpRawLogClient(LoggerFactory, _configuration.FtpSourceUsername, _configuration.FtpSourcePassword);
             var azureClient = new CloudBlobRawLogClient(LoggerFactory, _cloudStorageAccount);
 
             // Collect directory listing.
             var rawLogFileUris = await ftpClient.GetRawLogFileUris(_ftpServerUri);
 
             // Prepare cloud storage blob container.
-            var cloudBlobContainer = await azureClient.CreateContainerIfNotExistsAsync(_cloudStorageContainerName);
+            var cloudBlobContainer = await azureClient.CreateContainerIfNotExistsAsync(_configuration.AzureCdnCloudStorageContainerName);
 
             foreach (var rawLogFileUri in rawLogFileUris)
             {
@@ -126,7 +147,7 @@ public override async Task Run()
                     var rawLogFile = new RawLogFileInfo(rawLogFileUri);
 
                     if (_azureCdnPlatform != rawLogFile.AzureCdnPlatform
-                        || !_azureCdnAccountNumber.Equals(rawLogFile.AzureCdnAccountNumber, StringComparison.InvariantCultureIgnoreCase))
+                        || !_configuration.AzureCdnAccountNumber.Equals(rawLogFile.AzureCdnAccountNumber, StringComparison.InvariantCultureIgnoreCase))
                     {
                         // Only process the raw log files matching the target CDN platform and account number.
                         continue;
@@ -367,5 +388,14 @@ private static string ToUnixTimeStamp(DateTime dateTime)
             var secondsPastEpoch = (dateTime - _unixTimestamp).TotalSeconds;
             return secondsPastEpoch.ToString(CultureInfo.InvariantCulture);
         }
+
+        protected override void ConfigureAutofacServices(ContainerBuilder containerBuilder)
+        {
+        }
+
+        protected override void ConfigureJobServices(IServiceCollection services, IConfigurationRoot configurationRoot)
+        {
+            ConfigureInitializationSection<CollectAzureCdnLogsConfiguration>(services, configurationRoot);
+        }
     }
 }
diff --git a/src/Stats.CollectAzureCdnLogs/Scripts/Stats.CollectAzureCdnLogs.cmd b/src/Stats.CollectAzureCdnLogs/Scripts/Stats.CollectAzureCdnLogs.cmd
@@ -3,12 +3,16 @@
 cd bin
 
 :Top
-	echo "Starting job - #{Jobs.stats.collectazurecdnlogs.Title}"
+echo "Starting job - #{Jobs.stats.collectazurecdnlogs.Title}"
 
-	title #{Jobs.stats.collectazurecdnlogs.Title}
+title #{Jobs.stats.collectazurecdnlogs.Title}
 
-    start /w stats.collectazurecdnlogs.exe -VaultName "#{Deployment.Azure.KeyVault.VaultName}" -ClientId "#{Deployment.Azure.KeyVault.ClientId}" -CertificateThumbprint "#{Deployment.Azure.KeyVault.CertificateThumbprint}" -FtpSourceUri "#{Jobs.stats.collectazurecdnlogs.FtpSource.Uri}" -FtpSourceUsername "#{Jobs.stats.collectazurecdnlogs.FtpSource.Username}" -FtpSourcePassword "#{Jobs.stats.collectazurecdnlogs.FtpSource.Password}" -AzureCdnAccountNumber "#{Jobs.stats.collectazurecdnlogs.AzureCdn.AccountNumber}" -AzureCdnPlatform "#{Jobs.stats.collectazurecdnlogs.AzureCdn.Platform}" -AzureCdnCloudStorageAccount "#{Jobs.stats.collectazurecdnlogs.AzureCdn.CloudStorageAccount}" -AzureCdnCloudStorageContainerName "#{Jobs.stats.collectazurecdnlogs.AzureCdn.CloudStorageContainerName}" -InstrumentationKey "#{Jobs.stats.collectazurecdnlogs.InstrumentationKey}" -verbose true -Interval #{Jobs.stats.collectazurecdnlogs.Interval}
+start /w stats.collectazurecdnlogs.exe ^
+    -Configuration "#{Jobs.stats.collectazurecdnlogs.Configuration}" ^
+    -InstrumentationKey "#{Jobs.stats.collectazurecdnlogs.InstrumentationKey}" ^
+    -verbose true ^
+    -Interval #{Jobs.stats.collectazurecdnlogs.Interval}
 
-	echo "Finished #{Jobs.stats.collectazurecdnlogs.Title}"
+echo "Finished #{Jobs.stats.collectazurecdnlogs.Title}"
 
-	goto Top
+goto Top
diff --git a/src/Stats.CollectAzureCdnLogs/Settings/dev.json b/src/Stats.CollectAzureCdnLogs/Settings/dev.json
@@ -0,0 +1,18 @@
+{
+  "Initialization": {
+    "FtpSourceUri": "#{Jobs.stats.collectazurecdnlogs.FtpSource.Uri}",
+    "FtpSourceUsername": "#{Jobs.stats.collectazurecdnlogs.FtpSource.Username}",
+    "FtpSourcePassword": "#{Jobs.stats.collectazurecdnlogs.FtpSource.Password}",
+    "AzureCdnAccountNumber": "#{Jobs.stats.collectazurecdnlogs.AzureCdn.AccountNumber}",
+    "AzureCdnPlatform": "#{Jobs.stats.collectazurecdnlogs.AzureCdn.Platform}",
+    "AzureCdnCloudStorageAccount": "#{Jobs.stats.collectazurecdnlogs.AzureCdn.CloudStorageAccount}",
+    "AzureCdnCloudStorageContainerName": "#{Jobs.stats.collectazurecdnlogs.AzureCdn.CloudStorageContainerName}"
+  },
+
+  "KeyVault_VaultName": "#{Deployment.Azure.KeyVault.VaultName}",
+  "KeyVault_ClientId": "#{Deployment.Azure.KeyVault.ClientId}",
+  "KeyVault_CertificateThumbprint": "#{Deployment.Azure.KeyVault.CertificateThumbprint}",
+  "KeyVault_ValidateCertificate": true,
+  "KeyVault_StoreName": "My",
+  "KeyVault_StoreLocation": "LocalMachine"
+}
diff --git a/src/Stats.CollectAzureCdnLogs/Settings/int.json b/src/Stats.CollectAzureCdnLogs/Settings/int.json
@@ -0,0 +1,18 @@
+{
+  "Initialization": {
+    "FtpSourceUri": "#{Jobs.stats.collectazurecdnlogs.FtpSource.Uri}",
+    "FtpSourceUsername": "#{Jobs.stats.collectazurecdnlogs.FtpSource.Username}",
+    "FtpSourcePassword": "#{Jobs.stats.collectazurecdnlogs.FtpSource.Password}",
+    "AzureCdnAccountNumber": "#{Jobs.stats.collectazurecdnlogs.AzureCdn.AccountNumber}",
+    "AzureCdnPlatform": "#{Jobs.stats.collectazurecdnlogs.AzureCdn.Platform}",
+    "AzureCdnCloudStorageAccount": "#{Jobs.stats.collectazurecdnlogs.AzureCdn.CloudStorageAccount}",
+    "AzureCdnCloudStorageContainerName": "#{Jobs.stats.collectazurecdnlogs.AzureCdn.CloudStorageContainerName}"
+  },
+
+  "KeyVault_VaultName": "#{Deployment.Azure.KeyVault.VaultName}",
+  "KeyVault_ClientId": "#{Deployment.Azure.KeyVault.ClientId}",
+  "KeyVault_CertificateThumbprint": "#{Deployment.Azure.KeyVault.CertificateThumbprint}",
+  "KeyVault_ValidateCertificate": true,
+  "KeyVault_StoreName": "My",
+  "KeyVault_StoreLocation": "LocalMachine"
+}
diff --git a/src/Stats.CollectAzureCdnLogs/Settings/prod.json b/src/Stats.CollectAzureCdnLogs/Settings/prod.json
@@ -0,0 +1,18 @@
+{
+  "Initialization": {
+    "FtpSourceUri": "#{Jobs.stats.collectazurecdnlogs.FtpSource.Uri}",
+    "FtpSourceUsername": "#{Jobs.stats.collectazurecdnlogs.FtpSource.Username}",
+    "FtpSourcePassword": "#{Jobs.stats.collectazurecdnlogs.FtpSource.Password}",
+    "AzureCdnAccountNumber": "#{Jobs.stats.collectazurecdnlogs.AzureCdn.AccountNumber}",
+    "AzureCdnPlatform": "#{Jobs.stats.collectazurecdnlogs.AzureCdn.Platform}",
+    "AzureCdnCloudStorageAccount": "#{Jobs.stats.collectazurecdnlogs.AzureCdn.CloudStorageAccount}",
+    "AzureCdnCloudStorageContainerName": "#{Jobs.stats.collectazurecdnlogs.AzureCdn.CloudStorageContainerName}"
+  },
+
+  "KeyVault_VaultName": "#{Deployment.Azure.KeyVault.VaultName}",
+  "KeyVault_ClientId": "#{Deployment.Azure.KeyVault.ClientId}",
+  "KeyVault_CertificateThumbprint": "#{Deployment.Azure.KeyVault.CertificateThumbprint}",
+  "KeyVault_ValidateCertificate": true,
+  "KeyVault_StoreName": "My",
+  "KeyVault_StoreLocation": "LocalMachine"
+}
diff --git a/src/Stats.CollectAzureCdnLogs/Stats.CollectAzureCdnLogs.csproj b/src/Stats.CollectAzureCdnLogs/Stats.CollectAzureCdnLogs.csproj
@@ -49,6 +49,7 @@
   </ItemGroup>
   <ItemGroup>
     <Compile Include="Blob\CloudBlobRawLogClient.cs" />
+    <Compile Include="Configuration\CollectAzureCdnLogsConfiguration.cs" />
     <Compile Include="FileExtensions.cs" />
     <Compile Include="Ftp\FtpDownloadStream.cs" />
     <Compile Include="Ftp\FtpRawLogClient.cs" />
@@ -70,6 +71,9 @@
       <SubType>Designer</SubType>
     </None>
     <None Include="Scripts\*" />
+    <None Include="Settings\dev.json" />
+    <None Include="Settings\int.json" />
+    <None Include="Settings\prod.json" />
     <None Include="Stats.CollectAzureCdnLogs.nuspec" />
   </ItemGroup>
   <ItemGroup>
@@ -110,6 +114,7 @@
       <Version>7.1.2</Version>
     </PackageReference>
   </ItemGroup>
+  <ItemGroup />
   <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
   <PropertyGroup>
     <SignPath>..\..\build</SignPath>

diff --git a/src/Stats.CollectAzureCdnLogs/Stats.CollectAzureCdnLogs.nuspec b/src/Stats.CollectAzureCdnLogs/Stats.CollectAzureCdnLogs.nuspec
@@ -18,5 +18,7 @@
     <file src="Scripts\PreDeploy.ps1" />
     <file src="Scripts\PostDeploy.ps1" />
     <file src="Scripts\nssm.exe" />
+
+    <file src="Settings\*.json" target="bin" />
   </files>
 </package>
diff --git a/src/Stats.CollectAzureChinaCDNLogs/ArgumentNames.cs b/src/Stats.CollectAzureChinaCDNLogs/ArgumentNames.cs
diff --git a/src/Stats.CollectAzureChinaCDNLogs/Configuration/CollectAzureChinaCdnLogsConfiguration.cs b/src/Stats.CollectAzureChinaCDNLogs/Configuration/CollectAzureChinaCdnLogsConfiguration.cs
@@ -0,0 +1,20 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+namespace Stats.CollectAzureChinaCDNLogs
+{
+    public class CollectAzureChinaCdnLogsConfiguration
+    {
+        public string AzureAccountConnectionStringSource { get; set; }
+
+        public string AzureAccountConnectionStringDestination { get; set; }
+
+        public string AzureContainerNameSource { get; set; }
+
+        public string AzureContainerNameDestination { get; set; }
+
+        public string DestinationFilePrefix { get; set; }
+
+        public int? ExecutionTimeoutInSeconds { get; set; }
+    }
+}