diff --git a/src/ArchivePackages/ArchivePackages.Job.cs b/src/ArchivePackages/ArchivePackages.Job.cs index 97f7528e6..a2ac8c672 100644 --- a/src/ArchivePackages/ArchivePackages.Job.cs +++ b/src/ArchivePackages/ArchivePackages.Job.cs @@ -4,20 +4,22 @@ using System; using System.Collections.Generic; using System.ComponentModel.Design; -using System.Diagnostics.Tracing; +using System.Data.SqlClient; using System.Linq; using System.Threading.Tasks; +using Autofac; using Dapper; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; using Microsoft.WindowsAzure.Storage; using Microsoft.WindowsAzure.Storage.Blob; using Newtonsoft.Json.Linq; using NuGet.Jobs; -using NuGet.Services.KeyVault; -using NuGet.Services.Sql; +using NuGet.Jobs.Configuration; namespace ArchivePackages { - public class Job : JobBase + public class Job : JsonConfigurationJob { private readonly JobEventSource JobEventSourceLog = JobEventSource.Log; private const string ContentTypeJson = "application/json"; @@ -27,6 +29,8 @@ public class Job : JobBase private const string DefaultPackagesArchiveContainerName = "ng-backups"; private const string DefaultCursorBlobName = "cursor.json"; + private InitializationConfiguration Configuration { get; set; } + /// /// Gets or sets an Azure Storage Uri referring to a container to use as the source for package blobs /// @@ -44,6 +48,7 @@ public class Job : JobBase /// DestinationContainerName should be same as the primary destination /// public CloudStorageAccount SecondaryDestination { get; set; } + /// /// Destination Container name for both Primary and Secondary destinations. Also, for the cursor blob /// @@ -53,8 +58,11 @@ public class Job : JobBase /// Blob containing the cursor data. Cursor data comprises of cursorDateTime /// public string CursorBlobName { get; set; } - - private ISqlConnectionFactory _packageDbConnectionFactory; + + /// + /// Gallery database registration, for diagnostics. + /// + private SqlConnectionStringBuilder GalleryDatabase { get; set; } protected CloudBlobContainer SourceContainer { get; private set; } @@ -66,33 +74,34 @@ public Job() : base(JobEventSource.Log) { } public override void Init(IServiceContainer serviceContainer, IDictionary jobArgsDictionary) { - var secretInjector = (ISecretInjector)serviceContainer.GetService(typeof(ISecretInjector)); - var packageDbConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.PackageDatabase); - _packageDbConnectionFactory = new AzureSqlConnectionFactory(packageDbConnectionString, secretInjector); + base.Init(serviceContainer, jobArgsDictionary); - Source = CloudStorageAccount.Parse( - JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.Source)); + Configuration = _serviceProvider.GetRequiredService(); - PrimaryDestination = CloudStorageAccount.Parse( - JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.PrimaryDestination)); + GalleryDatabase = GetDatabaseRegistration(); - var secondaryDestinationCstr = JobConfigurationManager.TryGetArgument(jobArgsDictionary, JobArgumentNames.SecondaryDestination); - SecondaryDestination = string.IsNullOrEmpty(secondaryDestinationCstr) ? null : CloudStorageAccount.Parse(secondaryDestinationCstr); + Source = CloudStorageAccount.Parse(Configuration.Source); - SourceContainerName = JobConfigurationManager.TryGetArgument(jobArgsDictionary, JobArgumentNames.SourceContainerName) ?? DefaultPackagesContainerName; + PrimaryDestination = CloudStorageAccount.Parse(Configuration.PrimaryDestination); - DestinationContainerName = JobConfigurationManager.TryGetArgument(jobArgsDictionary, JobArgumentNames.DestinationContainerName) ?? DefaultPackagesArchiveContainerName; + if (!string.IsNullOrEmpty(Configuration.SecondaryDestination)) + { + SecondaryDestination = CloudStorageAccount.Parse(Configuration.SecondaryDestination); + } + + SourceContainerName = Configuration.SourceContainerName ?? DefaultPackagesContainerName; + DestinationContainerName = Configuration.DestinationContainerName ?? DefaultPackagesArchiveContainerName; SourceContainer = Source.CreateCloudBlobClient().GetContainerReference(SourceContainerName); PrimaryDestinationContainer = PrimaryDestination.CreateCloudBlobClient().GetContainerReference(DestinationContainerName); SecondaryDestinationContainer = SecondaryDestination?.CreateCloudBlobClient().GetContainerReference(DestinationContainerName); - CursorBlobName = JobConfigurationManager.TryGetArgument(jobArgsDictionary, JobArgumentNames.CursorBlob) ?? DefaultCursorBlobName; + CursorBlobName = Configuration.CursorBlob ?? DefaultCursorBlobName; } public override async Task Run() { - JobEventSourceLog.PreparingToArchive(Source.Credentials.AccountName, SourceContainer.Name, PrimaryDestination.Credentials.AccountName, PrimaryDestinationContainer.Name, _packageDbConnectionFactory.DataSource, _packageDbConnectionFactory.InitialCatalog); + JobEventSourceLog.PreparingToArchive(Source.Credentials.AccountName, SourceContainer.Name, PrimaryDestination.Credentials.AccountName, PrimaryDestinationContainer.Name, GalleryDatabase.DataSource, GalleryDatabase.InitialCatalog); await Archive(PrimaryDestinationContainer); // todo: consider reusing package query for primary and secondary archives @@ -124,9 +133,9 @@ private async Task Archive(CloudBlobContainer destinationContainer) JobEventSourceLog.CursorData(cursorDateTime.ToString(DateTimeFormatSpecifier)); - JobEventSourceLog.GatheringPackagesToArchiveFromDb(_packageDbConnectionFactory.DataSource, _packageDbConnectionFactory.InitialCatalog); + JobEventSourceLog.GatheringPackagesToArchiveFromDb(GalleryDatabase.DataSource, GalleryDatabase.InitialCatalog); List packages; - using (var connection = await _packageDbConnectionFactory.CreateAsync()) + using (var connection = await OpenSqlConnectionAsync()) { packages = (await connection.QueryAsync(@" SELECT pr.Id, p.NormalizedVersion AS Version, p.Hash, p.LastEdited, p.Published @@ -135,7 +144,7 @@ FROM Packages p WHERE Published > @cursorDateTime OR LastEdited > @cursorDateTime", new { cursorDateTime = cursorDateTime })) .ToList(); } - JobEventSourceLog.GatheredPackagesToArchiveFromDb(packages.Count, _packageDbConnectionFactory.DataSource, _packageDbConnectionFactory.InitialCatalog); + JobEventSourceLog.GatheredPackagesToArchiveFromDb(packages.Count, GalleryDatabase.DataSource, GalleryDatabase.InitialCatalog); var archiveSet = packages .AsParallel() @@ -193,129 +202,14 @@ private async Task ArchivePackage(string sourceBlobName, string destinationBlobN JobEventSourceLog.StartedCopy(sourceBlob.Name, destBlob.Name); } } - } - [EventSource(Name = "Outercurve-NuGet-Jobs-ArchivePackages")] - public class JobEventSource : EventSource - { - public static readonly JobEventSource Log = new JobEventSource(); - - private JobEventSource() { } - - [Event( - eventId: 1, - Level = EventLevel.Informational, - Message = "Preparing to archive packages from {0}/{1} to primary destination {2}/{3} using package data from {4}/{5}")] - public void PreparingToArchive(string sourceAccount, string sourceContainer, string destAccount, string destContainer, string dbServer, string dbName) { WriteEvent(1, sourceAccount, sourceContainer, destAccount, destContainer, dbServer, dbName); } - - [Event( - eventId: 2, - Level = EventLevel.Informational, - Message = "Preparing to archive packages to secondary destination {0}/{1}")] - public void PreparingToArchive2(string destAccount, string destContainer) { WriteEvent(2, destAccount, destContainer); } - - [Event( - eventId: 3, - Level = EventLevel.Informational, - Message = "Cursor data: CursorDateTime is {0}")] - public void CursorData(string cursorDateTime) { WriteEvent(3, cursorDateTime); } - - [Event( - eventId: 4, - Level = EventLevel.Informational, - Task = Tasks.GatheringDbPackages, - Opcode = EventOpcode.Start, - Message = "Gathering list of packages to archive from {0}/{1}")] - public void GatheringPackagesToArchiveFromDb(string dbServer, string dbName) { WriteEvent(4, dbServer, dbName); } - - [Event( - eventId: 5, - Level = EventLevel.Informational, - Task = Tasks.GatheringDbPackages, - Opcode = EventOpcode.Stop, - Message = "Gathered {0} packages to archive from {1}/{2}")] - public void GatheredPackagesToArchiveFromDb(int gathered, string dbServer, string dbName) { WriteEvent(5, gathered, dbServer, dbName); } - - [Event( - eventId: 6, - Level = EventLevel.Informational, - Task = Tasks.ArchivingPackages, - Opcode = EventOpcode.Start, - Message = "Starting archive of {0} packages.")] - public void StartingArchive(int count) { WriteEvent(6, count); } - - [Event( - eventId: 7, - Level = EventLevel.Informational, - Task = Tasks.ArchivingPackages, - Opcode = EventOpcode.Stop, - Message = "Started archive.")] - public void StartedArchive() { WriteEvent(7); } - - [Event( - eventId: 8, - Level = EventLevel.Informational, - Message = "Archive already exists: {0}")] - public void ArchiveExists(string blobName) { WriteEvent(8, blobName); } - - [Event( - eventId: 9, - Level = EventLevel.Warning, - Message = "Source Blob does not exist: {0}")] - public void SourceBlobMissing(string blobName) { WriteEvent(9, blobName); } - - [Event( - eventId: 12, - Level = EventLevel.Informational, - Task = Tasks.StartingPackageCopy, - Opcode = EventOpcode.Start, - Message = "Starting copy of {0} to {1}.")] - public void StartingCopy(string source, string dest) { WriteEvent(12, source, dest); } - - [Event( - eventId: 13, - Level = EventLevel.Informational, - Task = Tasks.StartingPackageCopy, - Opcode = EventOpcode.Stop, - Message = "Started copy of {0} to {1}.")] - public void StartedCopy(string source, string dest) { WriteEvent(13, source, dest); } - - [Event( - eventId: 14, - Level = EventLevel.Informational, - Message = "NewCursor data: CursorDateTime is {0}")] - public void NewCursorData(string cursorDateTime) { WriteEvent(14, cursorDateTime); } - } - - public static class Tasks - { - public const EventTask GatheringDbPackages = (EventTask)0x1; - public const EventTask ArchivingPackages = (EventTask)0x2; - public const EventTask StartingPackageCopy = (EventTask)0x3; - } - - public class PackageRef - { - public PackageRef(string id, string version, string hash) - { - Id = id; - Version = version; - Hash = hash; - } - public PackageRef(string id, string version, string hash, DateTime lastEdited) - : this(id, version, hash) + protected override void ConfigureAutofacServices(ContainerBuilder containerBuilder) { - LastEdited = lastEdited; } - public PackageRef(string id, string version, string hash, DateTime lastEdited, DateTime published) - : this(id, version, hash, lastEdited) + + protected override void ConfigureJobServices(IServiceCollection services, IConfigurationRoot configurationRoot) { - Published = published; + ConfigureInitializationSection(services, configurationRoot); } - public string Id { get; set; } - public string Version { get; set; } - public string Hash { get; set; } - public DateTime? LastEdited { get; set; } - public DateTime? Published { get; set; } } } diff --git a/src/ArchivePackages/ArchivePackages.csproj b/src/ArchivePackages/ArchivePackages.csproj index 5bc7bfd8c..390248939 100644 --- a/src/ArchivePackages/ArchivePackages.csproj +++ b/src/ArchivePackages/ArchivePackages.csproj @@ -44,6 +44,10 @@ + + + + @@ -52,6 +56,9 @@ + + + @@ -75,9 +82,6 @@ 9.0.1 - - 2.27.0 - 4.3.3 @@ -85,6 +89,7 @@ 7.1.2 + \ No newline at end of file diff --git a/src/ArchivePackages/Configuration/InitializationConfiguration.cs b/src/ArchivePackages/Configuration/InitializationConfiguration.cs new file mode 100644 index 000000000..37d01393c --- /dev/null +++ b/src/ArchivePackages/Configuration/InitializationConfiguration.cs @@ -0,0 +1,48 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +namespace ArchivePackages +{ + public class InitializationConfiguration + { + /// + /// Source storage account. + /// + public string Source { get; set; } + + /// + /// Source storage container name. + /// + public string SourceContainerName { get; set; } + + /// + /// Primary archive destination. + /// + public string PrimaryDestination { get; set; } + + /// + /// Secondary archive destination. + /// + public string SecondaryDestination { get; set; } + + /// + /// Source storage container name. + /// + public string DestinationContainerName { get; set; } + + /// + /// Cursor blob name. + /// + public string CursorBlob { get; set; } + + /// + /// Sleep interval between job run iterations. + /// + public int Sleep { get; set; } + + /// + /// Application insights instrumentation key. + /// + public string InstrumentationKey { get; set; } + } +} diff --git a/src/ArchivePackages/JobEventSource.cs b/src/ArchivePackages/JobEventSource.cs new file mode 100644 index 000000000..b5c8a6c0f --- /dev/null +++ b/src/ArchivePackages/JobEventSource.cs @@ -0,0 +1,99 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Diagnostics.Tracing; + +namespace ArchivePackages +{ + [EventSource(Name = "Outercurve-NuGet-Jobs-ArchivePackages")] + public class JobEventSource : EventSource + { + public static readonly JobEventSource Log = new JobEventSource(); + + private JobEventSource() { } + + [Event( + eventId: 1, + Level = EventLevel.Informational, + Message = "Preparing to archive packages from {0}/{1} to primary destination {2}/{3} using package data from {4}/{5}")] + public void PreparingToArchive(string sourceAccount, string sourceContainer, string destAccount, string destContainer, string dbServer, string dbName) { WriteEvent(1, sourceAccount, sourceContainer, destAccount, destContainer, dbServer, dbName); } + + [Event( + eventId: 2, + Level = EventLevel.Informational, + Message = "Preparing to archive packages to secondary destination {0}/{1}")] + public void PreparingToArchive2(string destAccount, string destContainer) { WriteEvent(2, destAccount, destContainer); } + + [Event( + eventId: 3, + Level = EventLevel.Informational, + Message = "Cursor data: CursorDateTime is {0}")] + public void CursorData(string cursorDateTime) { WriteEvent(3, cursorDateTime); } + + [Event( + eventId: 4, + Level = EventLevel.Informational, + Task = JobTasks.GatheringDbPackages, + Opcode = EventOpcode.Start, + Message = "Gathering list of packages to archive from {0}/{1}")] + public void GatheringPackagesToArchiveFromDb(string dbServer, string dbName) { WriteEvent(4, dbServer, dbName); } + + [Event( + eventId: 5, + Level = EventLevel.Informational, + Task = JobTasks.GatheringDbPackages, + Opcode = EventOpcode.Stop, + Message = "Gathered {0} packages to archive from {1}/{2}")] + public void GatheredPackagesToArchiveFromDb(int gathered, string dbServer, string dbName) { WriteEvent(5, gathered, dbServer, dbName); } + + [Event( + eventId: 6, + Level = EventLevel.Informational, + Task = JobTasks.ArchivingPackages, + Opcode = EventOpcode.Start, + Message = "Starting archive of {0} packages.")] + public void StartingArchive(int count) { WriteEvent(6, count); } + + [Event( + eventId: 7, + Level = EventLevel.Informational, + Task = JobTasks.ArchivingPackages, + Opcode = EventOpcode.Stop, + Message = "Started archive.")] + public void StartedArchive() { WriteEvent(7); } + + [Event( + eventId: 8, + Level = EventLevel.Informational, + Message = "Archive already exists: {0}")] + public void ArchiveExists(string blobName) { WriteEvent(8, blobName); } + + [Event( + eventId: 9, + Level = EventLevel.Warning, + Message = "Source Blob does not exist: {0}")] + public void SourceBlobMissing(string blobName) { WriteEvent(9, blobName); } + + [Event( + eventId: 12, + Level = EventLevel.Informational, + Task = JobTasks.StartingPackageCopy, + Opcode = EventOpcode.Start, + Message = "Starting copy of {0} to {1}.")] + public void StartingCopy(string source, string dest) { WriteEvent(12, source, dest); } + + [Event( + eventId: 13, + Level = EventLevel.Informational, + Task = JobTasks.StartingPackageCopy, + Opcode = EventOpcode.Stop, + Message = "Started copy of {0} to {1}.")] + public void StartedCopy(string source, string dest) { WriteEvent(13, source, dest); } + + [Event( + eventId: 14, + Level = EventLevel.Informational, + Message = "NewCursor data: CursorDateTime is {0}")] + public void NewCursorData(string cursorDateTime) { WriteEvent(14, cursorDateTime); } + } +} \ No newline at end of file diff --git a/src/ArchivePackages/JobTasks.cs b/src/ArchivePackages/JobTasks.cs new file mode 100644 index 000000000..2af93d804 --- /dev/null +++ b/src/ArchivePackages/JobTasks.cs @@ -0,0 +1,16 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Diagnostics.Tracing; + +namespace ArchivePackages +{ + public static class JobTasks + { + public const EventTask GatheringDbPackages = (EventTask)0x1; + + public const EventTask ArchivingPackages = (EventTask)0x2; + + public const EventTask StartingPackageCopy = (EventTask)0x3; + } +} diff --git a/src/ArchivePackages/PackageRef.cs b/src/ArchivePackages/PackageRef.cs new file mode 100644 index 000000000..448105da3 --- /dev/null +++ b/src/ArchivePackages/PackageRef.cs @@ -0,0 +1,39 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; + +namespace ArchivePackages +{ + public class PackageRef + { + public PackageRef(string id, string version, string hash) + { + Id = id; + Version = version; + Hash = hash; + } + + public PackageRef(string id, string version, string hash, DateTime lastEdited) + : this(id, version, hash) + { + LastEdited = lastEdited; + } + + public PackageRef(string id, string version, string hash, DateTime lastEdited, DateTime published) + : this(id, version, hash, lastEdited) + { + Published = published; + } + + public string Id { get; set; } + + public string Version { get; set; } + + public string Hash { get; set; } + + public DateTime? LastEdited { get; set; } + + public DateTime? Published { get; set; } + } +} diff --git a/src/ArchivePackages/Scripts/ArchivePackages.cmd b/src/ArchivePackages/Scripts/ArchivePackages.cmd index 864f297e1..d87da1a5a 100644 --- a/src/ArchivePackages/Scripts/ArchivePackages.cmd +++ b/src/ArchivePackages/Scripts/ArchivePackages.cmd @@ -7,7 +7,7 @@ cd bin title #{Jobs.archivepackages.Title} - start /w archivepackages.exe -VaultName "#{Deployment.Azure.KeyVault.VaultName}" -ClientId "#{Deployment.Azure.KeyVault.ClientId}" -CertificateThumbprint "#{Deployment.Azure.KeyVault.CertificateThumbprint}" -PackageDatabase "#{Jobs.archivepackages.PackageDatabase}" -Source "#{Jobs.archivepackages.Source}" -PrimaryDestination "#{Jobs.archivepackages.PrimaryDestination}" -SecondaryDestination "#{Jobs.archivepackages.SecondaryDestination}" -Sleep "#{Jobs.archivepackages.Sleep}" -InstrumentationKey "#{Jobs.archivepackages.ApplicationInsightsInstrumentationKey}" + start /w archivepackages.exe -Configuration "#{Jobs.archivepackages.Configuration}" echo "Finished #{Jobs.archivepackages.Title}" diff --git a/src/ArchivePackages/Settings/dev.json b/src/ArchivePackages/Settings/dev.json new file mode 100644 index 000000000..c1b10d5e0 --- /dev/null +++ b/src/ArchivePackages/Settings/dev.json @@ -0,0 +1,20 @@ +{ + "Initialization": { + "Source": "DefaultEndpointsProtocol=https;AccountName=nugetdevlegacy;AccountKey=$$Dev-NuGetDevLegacyStorage-Key$$", + "PrimaryDestination": "DefaultEndpointsProtocol=https;AccountName=nugetdevlegacy;AccountKey=$$Dev-NuGetDevLegacyStorage-Key$$", + "SecondaryDestination": "DefaultEndpointsProtocol=https;AccountName=nugetdev1;AccountKey=$$Dev-NuGetDev1Storage-Key$$", + "Sleep": "600000", + "InstrumentationKey": "$$Dev-ApplicationInsightsV2Gallery-InstrumentationKey$$" + }, + + "GalleryDb": { + "ConnectionString": "Data Source=tcp:#{Jobs.validation.GalleryDatabaseAddress};Initial Catalog=nuget-dev-0-v2gallery;Integrated Security=False;User ID=$$Dev-GalleryDBReadOnly-UserName$$;Password=$$Dev-GalleryDBReadOnly-Password$$;Connect Timeout=30;Encrypt=True" + }, + + "KeyVault_VaultName": "#{Deployment.Azure.KeyVault.VaultName}", + "KeyVault_ClientId": "#{Deployment.Azure.KeyVault.ClientId}", + "KeyVault_CertificateThumbprint": "#{Deployment.Azure.KeyVault.CertificateThumbprint}", + "KeyVault_ValidateCertificate": true, + "KeyVault_StoreName": "My", + "KeyVault_StoreLocation": "LocalMachine" +} \ No newline at end of file diff --git a/src/ArchivePackages/Settings/int.json b/src/ArchivePackages/Settings/int.json new file mode 100644 index 000000000..5d7b3fe1f --- /dev/null +++ b/src/ArchivePackages/Settings/int.json @@ -0,0 +1,20 @@ +{ + "Initialization": { + "Source": "DefaultEndpointsProtocol=https;AccountName=nugetint0;AccountKey=$$Int-NuGetInt0Storage-Key$$", + "PrimaryDestination": "DefaultEndpointsProtocol=https;AccountName=nugetint0;AccountKey=$$Int-NuGetInt0Storage-Key$$", + "SecondaryDestination": "DefaultEndpointsProtocol=https;AccountName=nugetint1;AccountKey=$$Int-NuGetInt1Storage-Key$$", + "Sleep": "600000", + "InstrumentationKey": "$$Int-ApplicationInsightsV2Gallery-InstrumentationKey$$" + }, + + "GalleryDb": { + "ConnectionString": "Data Source=tcp:jvhowicii4.database.windows.net;Initial Catalog=nuget-int-0-v2gallery;User ID=$$Int-GalleryDBReadOnly-UserName$$;Password=$$Int-GalleryDBReadOnly-Password$$;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30" + }, + + "KeyVault_VaultName": "#{Deployment.Azure.KeyVault.VaultName}", + "KeyVault_ClientId": "#{Deployment.Azure.KeyVault.ClientId}", + "KeyVault_CertificateThumbprint": "#{Deployment.Azure.KeyVault.CertificateThumbprint}", + "KeyVault_ValidateCertificate": true, + "KeyVault_StoreName": "My", + "KeyVault_StoreLocation": "LocalMachine" +} \ No newline at end of file diff --git a/src/ArchivePackages/Settings/prod.json b/src/ArchivePackages/Settings/prod.json new file mode 100644 index 000000000..2ab04a6da --- /dev/null +++ b/src/ArchivePackages/Settings/prod.json @@ -0,0 +1,20 @@ +{ + "Initialization": { + "Source": "DefaultEndpointsProtocol=https;AccountName=nugetgallery;AccountKey=$$Prod-NuGetGalleryStorage-Key$$", + "PrimaryDestination": "DefaultEndpointsProtocol=https;AccountName=nugetgallery;AccountKey=$$Prod-NuGetGalleryStorage-Key$$", + "SecondaryDestination": "DefaultEndpointsProtocol=https;AccountName=nugetprod1;AccountKey=$$Prod-NuGetProd1Storage-Key$$", + "Sleep": "600000", + "InstrumentationKey": "$$Prod-ApplicationInsightsV2Gallery-InstrumentationKey$$" + }, + + "GalleryDb": { + "ConnectionString": "Data Source=tcp:amejn8fzzh.database.windows.net;Initial Catalog=NuGetGallery;User ID=$$Prod-GalleryDBReadOnly-UserName$$;Password=$$Prod-GalleryDBReadOnly-Password$$;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;" + }, + + "KeyVault_VaultName": "#{Deployment.Azure.KeyVault.VaultName}", + "KeyVault_ClientId": "#{Deployment.Azure.KeyVault.ClientId}", + "KeyVault_CertificateThumbprint": "#{Deployment.Azure.KeyVault.CertificateThumbprint}", + "KeyVault_ValidateCertificate": true, + "KeyVault_StoreName": "My", + "KeyVault_StoreLocation": "LocalMachine" +} \ No newline at end of file diff --git a/src/Monitoring.RebootSearchInstance/Job.cs b/src/Monitoring.RebootSearchInstance/Job.cs index 3bbe5643c..b177b123e 100644 --- a/src/Monitoring.RebootSearchInstance/Job.cs +++ b/src/Monitoring.RebootSearchInstance/Job.cs @@ -17,7 +17,7 @@ namespace NuGet.Monitoring.RebootSearchInstance { - public class Job : JsonConfigurationJob + public class Job : ValidationJobBase { private const string AzureManagementSectionName = "AzureManagement"; private const string MonitorConfigurationSectionName = "MonitorConfiguration"; diff --git a/src/NuGet.Jobs.Common/JobBase.cs b/src/NuGet.Jobs.Common/JobBase.cs index b57b5348a..02937ed24 100644 --- a/src/NuGet.Jobs.Common/JobBase.cs +++ b/src/NuGet.Jobs.Common/JobBase.cs @@ -196,6 +196,21 @@ public SqlConnection CreateSqlConnection() return Task.Run(() => CreateSqlConnectionAsync()).Result; } + /// + /// Opens a SqlConnection. + /// + public Task OpenSqlConnectionAsync() + where T : IDbConfiguration + { + var name = GetDatabaseKey(); + if (!SqlConnectionFactories.ContainsKey(name)) + { + throw new InvalidOperationException($"Database {name} has not been registered."); + } + + return SqlConnectionFactories[name].OpenAsync(); + } + /// /// Creates and opens a SqlConnection, for use by non-validation jobs. /// diff --git a/src/Validation.Common.Job/JsonConfigurationJob.cs b/src/NuGet.Jobs.Common/JsonConfigurationJob.cs similarity index 62% rename from src/Validation.Common.Job/JsonConfigurationJob.cs rename to src/NuGet.Jobs.Common/JsonConfigurationJob.cs index 9355f050c..eb19fd8c2 100644 --- a/src/Validation.Common.Job/JsonConfigurationJob.cs +++ b/src/NuGet.Jobs.Common/JsonConfigurationJob.cs @@ -4,12 +4,8 @@ using System; using System.Collections.Generic; using System.ComponentModel.Design; -using System.Data.Common; +using System.Diagnostics.Tracing; using System.IO; -using System.Net; -using System.Net.Http; -using System.Reflection; -using System.Threading.Tasks; using Autofac; using Autofac.Extensions.DependencyInjection; using Microsoft.ApplicationInsights; @@ -21,26 +17,26 @@ using NuGet.Services.Configuration; using NuGet.Services.KeyVault; using NuGet.Services.Logging; -using NuGet.Services.ServiceBus; -using NuGet.Services.Sql; -using NuGet.Services.Validation; -using NuGetGallery; -using NuGetGallery.Diagnostics; -namespace NuGet.Jobs.Validation +namespace NuGet.Jobs { public abstract class JsonConfigurationJob : JobBase { + private const string InitializationConfigurationSectionName = "Initialization"; private const string GalleryDbConfigurationSectionName = "GalleryDb"; private const string ValidationDbConfigurationSectionName = "ValidationDb"; private const string ServiceBusConfigurationSectionName = "ServiceBus"; private const string ValidationStorageConfigurationSectionName = "ValidationStorage"; - private const string PackageDownloadTimeoutName = "PackageDownloadTimeout"; - /// - /// The maximum number of concurrent connections that can be established to a single server. - /// - private const int MaximumConnectionsPerServer = 64; + public JsonConfigurationJob() + : this(null) + { + } + + public JsonConfigurationJob(EventSource jobEventSource) + : base(jobEventSource) + { + } /// /// The argument this job uses to determine the configuration file's path. @@ -64,7 +60,7 @@ public override void Init(IServiceContainer serviceContainer, IDictionary(IServiceProvider serviceProvider) where T : IDbConfiguration - { - var connectionString = serviceProvider.GetRequiredService>().Value.ConnectionString; - var connectionFactory = new AzureSqlConnectionFactory(connectionString, - serviceProvider.GetRequiredService()); - - return Task.Run(() => connectionFactory.CreateAsync()).Result; - } - - private void ConfigureDefaultJobServices(IServiceCollection services, IConfigurationRoot configurationRoot) + protected virtual void ConfigureDefaultJobServices(IServiceCollection services, IConfigurationRoot configurationRoot) { services.Configure(configurationRoot.GetSection(GalleryDbConfigurationSectionName)); services.Configure(configurationRoot.GetSection(ValidationDbConfigurationSectionName)); @@ -127,53 +114,6 @@ private void ConfigureDefaultJobServices(IServiceCollection services, IConfigura services.AddSingleton(new TelemetryClient()); services.AddTransient(); - services.AddTransient(); - services.AddTransient(); - services.AddTransient(); - - services.AddTransient(c => - { - var configurationAccessor = c.GetRequiredService>(); - return new CloudBlobClientWrapper( - configurationAccessor.Value.ConnectionString, - readAccessGeoRedundant: false); - }); - services.AddTransient(); - - services.AddScoped(p => - { - return new ValidationEntitiesContext(CreateDbConnection(p)); - }); - - services.AddScoped(p => - { - return new EntitiesContext(CreateDbConnection(p), readOnly: true); - }); - - services.AddTransient(p => - { - var config = p.GetRequiredService>().Value; - - return new SubscriptionClientWrapper(config.ConnectionString, config.TopicPath, config.SubscriptionName); - }); - - services.AddSingleton(p => - { - var assembly = Assembly.GetEntryAssembly(); - var assemblyName = assembly.GetName().Name; - var assemblyVersion = assembly.GetCustomAttribute()?.InformationalVersion ?? "0.0.0"; - - var client = new HttpClient(new WebRequestHandler - { - AllowPipelining = true, - AutomaticDecompression = (DecompressionMethods.GZip | DecompressionMethods.Deflate), - }); - - client.Timeout = configurationRoot.GetValue(PackageDownloadTimeoutName); - client.DefaultRequestHeaders.Add("User-Agent", $"{assemblyName}/{assemblyVersion}"); - - return client; - }); } private void ConfigureLibraries(IServiceCollection services) @@ -184,6 +124,30 @@ private void ConfigureLibraries(IServiceCollection services) services.AddLogging(); } + protected virtual void RegisterDatabases(IServiceProvider serviceProvider) + { + var galleryDb = serviceProvider.GetRequiredService>(); + if (!string.IsNullOrEmpty(galleryDb.Value?.ConnectionString)) + { + RegisterDatabase(serviceProvider); + } + + var validationDb = serviceProvider.GetRequiredService>(); + if (!string.IsNullOrEmpty(validationDb.Value?.ConnectionString)) + { + RegisterDatabase(serviceProvider); + } + } + + protected virtual void ConfigureInitializationSection( + IServiceCollection services, + IConfigurationRoot configurationRoot) + where TConfiguration : class + { + services.Configure(configurationRoot.GetSection(InitializationConfigurationSectionName)); + services.AddSingleton(provider => provider.GetRequiredService>().Value); + } + /// /// Method to be implemented in derived classes to provide Autofac-specific configuration for /// that specific job (like setting up keyed resolution). diff --git a/src/NuGet.Jobs.Common/NuGet.Jobs.Common.csproj b/src/NuGet.Jobs.Common/NuGet.Jobs.Common.csproj index ebf00a3bd..671dc41c7 100644 --- a/src/NuGet.Jobs.Common/NuGet.Jobs.Common.csproj +++ b/src/NuGet.Jobs.Common/NuGet.Jobs.Common.csproj @@ -51,6 +51,7 @@ + @@ -70,9 +71,21 @@ + + 4.6.2 + + + 4.2.0 + 1.50.2 + + 1.1.1 + + + 1.1.2 + 2.27.0 diff --git a/src/NuGet.Services.Revalidate/Job.cs b/src/NuGet.Services.Revalidate/Job.cs index 0dd289326..43f15ae56 100644 --- a/src/NuGet.Services.Revalidate/Job.cs +++ b/src/NuGet.Services.Revalidate/Job.cs @@ -26,7 +26,7 @@ namespace NuGet.Services.Revalidate using GalleryContext = EntitiesContext; using IGalleryContext = IEntitiesContext; - public class Job : JsonConfigurationJob + public class Job : ValidationJobBase { private const string RebuildPreinstalledSetArgumentName = "RebuildPreinstalledSet"; private const string InitializeArgumentName = "Initialize"; diff --git a/src/PackageHash/Job.cs b/src/PackageHash/Job.cs index 58cdcfb5d..714990328 100644 --- a/src/PackageHash/Job.cs +++ b/src/PackageHash/Job.cs @@ -18,7 +18,7 @@ namespace NuGet.Services.PackageHash { - public class Job : JsonConfigurationJob + public class Job : ValidationJobBase { private const string PackageHashConfigurationSectionName = "PackageHash"; diff --git a/src/Validation.Common.Job/SubcriptionProcessorJob.cs b/src/Validation.Common.Job/SubcriptionProcessorJob.cs index 38d24c525..f05231b56 100644 --- a/src/Validation.Common.Job/SubcriptionProcessorJob.cs +++ b/src/Validation.Common.Job/SubcriptionProcessorJob.cs @@ -12,7 +12,7 @@ namespace NuGet.Jobs.Validation { - public abstract class SubcriptionProcessorJob : JsonConfigurationJob + public abstract class SubcriptionProcessorJob : ValidationJobBase { private const string SubscriptionProcessorConfigurationSectionName = "ServiceBus"; diff --git a/src/Validation.Common.Job/Validation.Common.Job.csproj b/src/Validation.Common.Job/Validation.Common.Job.csproj index ccf242593..af94ffb50 100644 --- a/src/Validation.Common.Job/Validation.Common.Job.csproj +++ b/src/Validation.Common.Job/Validation.Common.Job.csproj @@ -61,7 +61,7 @@ - + diff --git a/src/Validation.Common.Job/ValidationJobBase.cs b/src/Validation.Common.Job/ValidationJobBase.cs new file mode 100644 index 000000000..898e726d0 --- /dev/null +++ b/src/Validation.Common.Job/ValidationJobBase.cs @@ -0,0 +1,92 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.ComponentModel.Design; +using System.Net; +using System.Net.Http; +using System.Reflection; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using NuGet.Jobs.Configuration; +using NuGet.Services.ServiceBus; +using NuGet.Services.Validation; +using NuGetGallery; +using NuGetGallery.Diagnostics; + +namespace NuGet.Jobs.Validation +{ + public abstract class ValidationJobBase : JsonConfigurationJob + { + private const string PackageDownloadTimeoutName = "PackageDownloadTimeout"; + + /// + /// The maximum number of concurrent connections that can be established to a single server. + /// + private const int MaximumConnectionsPerServer = 64; + + public override void Init(IServiceContainer serviceContainer, IDictionary jobArgsDictionary) + { + base.Init(serviceContainer, jobArgsDictionary); + + ServicePointManager.DefaultConnectionLimit = MaximumConnectionsPerServer; + } + + protected override void ConfigureDefaultJobServices(IServiceCollection services, IConfigurationRoot configurationRoot) + { + base.ConfigureDefaultJobServices(services, configurationRoot); + + //services.AddSingleton(new TelemetryClient()); + //services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + + services.AddTransient(c => + { + var configurationAccessor = c.GetRequiredService>(); + return new CloudBlobClientWrapper( + configurationAccessor.Value.ConnectionString, + readAccessGeoRedundant: false); + }); + services.AddTransient(); + + services.AddScoped(p => + { + return new ValidationEntitiesContext(CreateSqlConnection()); + }); + + services.AddScoped(p => + { + return new EntitiesContext(CreateSqlConnection(), readOnly: true); + }); + + services.AddTransient(p => + { + var config = p.GetRequiredService>().Value; + + return new SubscriptionClientWrapper(config.ConnectionString, config.TopicPath, config.SubscriptionName); + }); + + services.AddSingleton(p => + { + var assembly = Assembly.GetEntryAssembly(); + var assemblyName = assembly.GetName().Name; + var assemblyVersion = assembly.GetCustomAttribute()?.InformationalVersion ?? "0.0.0"; + + var client = new HttpClient(new WebRequestHandler + { + AllowPipelining = true, + AutomaticDecompression = (DecompressionMethods.GZip | DecompressionMethods.Deflate), + }); + + client.Timeout = configurationRoot.GetValue(PackageDownloadTimeoutName); + client.DefaultRequestHeaders.Add("User-Agent", $"{assemblyName}/{assemblyVersion}"); + + return client; + }); + } + } +} diff --git a/src/Validation.PackageSigning.ProcessSignature/Job.cs b/src/Validation.PackageSigning.ProcessSignature/Job.cs index 16997f935..14f9e5bdc 100644 --- a/src/Validation.PackageSigning.ProcessSignature/Job.cs +++ b/src/Validation.PackageSigning.ProcessSignature/Job.cs @@ -35,7 +35,7 @@ protected override void ConfigureJobServices(IServiceCollection services, IConfi services.AddScoped(p => { - return new EntitiesContext(CreateDbConnection(p), readOnly: false); + return new EntitiesContext(CreateSqlConnection(), readOnly: false); }); services.Add(ServiceDescriptor.Transient(typeof(IEntityRepository<>), typeof(EntityRepository<>))); diff --git a/src/Validation.PackageSigning.RevalidateCertificate/Job.cs b/src/Validation.PackageSigning.RevalidateCertificate/Job.cs index 44d49fb6e..ec5e49107 100644 --- a/src/Validation.PackageSigning.RevalidateCertificate/Job.cs +++ b/src/Validation.PackageSigning.RevalidateCertificate/Job.cs @@ -15,7 +15,7 @@ namespace Validation.PackageSigning.RevalidateCertificate { - public class Job : JsonConfigurationJob + public class Job : ValidationJobBase { private const string RevalidationConfigurationSectionName = "RevalidateJob";