Skip to content
This repository has been archived by the owner on Jul 30, 2024. It is now read-only.
/ NuGet.Jobs Public archive

Commit

Permalink
Revert "Update jobs to latest SQL AAD (#485)"
Browse files Browse the repository at this point in the history
This reverts commit d4e769d.
  • Loading branch information
chenriksson committed Jul 25, 2018
1 parent 4837967 commit 7e87d9d
Show file tree
Hide file tree
Showing 53 changed files with 348 additions and 452 deletions.
26 changes: 11 additions & 15 deletions src/ArchivePackages/ArchivePackages.Job.cs
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@
using System;
using System.Collections.Generic;
using System.ComponentModel.Design;
using System.Data.SqlClient;
using System.Diagnostics.Tracing;
using System.Linq;
using System.Threading.Tasks;
Expand All @@ -13,6 +12,8 @@
using Microsoft.WindowsAzure.Storage.Blob;
using Newtonsoft.Json.Linq;
using NuGet.Jobs;
using NuGet.Services.KeyVault;
using NuGet.Services.Sql;

namespace ArchivePackages
{
Expand Down Expand Up @@ -52,20 +53,22 @@ public class Job : JobBase
/// Blob containing the cursor data. Cursor data comprises of cursorDateTime
/// </summary>
public string CursorBlobName { get; set; }

private ISqlConnectionFactory _packageDbConnectionFactory;

protected CloudBlobContainer SourceContainer { get; private set; }

protected CloudBlobContainer PrimaryDestinationContainer { get; private set; }

protected CloudBlobContainer SecondaryDestinationContainer { get; private set; }

private SqlConnectionStringBuilder PackageDatabase { get; set; }

public Job() : base(JobEventSource.Log) { }

public override void Init(IServiceContainer serviceContainer, IDictionary<string, string> jobArgsDictionary)
{
PackageDatabase = RegisterDatabase(serviceContainer, jobArgsDictionary, JobArgumentNames.PackageDatabase);
var secretInjector = (ISecretInjector)serviceContainer.GetService(typeof(ISecretInjector));
var packageDbConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.PackageDatabase);
_packageDbConnectionFactory = new AzureSqlConnectionFactory(packageDbConnectionString, secretInjector);

Source = CloudStorageAccount.Parse(
JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.Source));
Expand All @@ -89,18 +92,13 @@ public override void Init(IServiceContainer serviceContainer, IDictionary<string

public override async Task Run()
{
JobEventSourceLog.PreparingToArchive(
Source.Credentials.AccountName, SourceContainer.Name,
PrimaryDestination.Credentials.AccountName, PrimaryDestinationContainer.Name,
PackageDatabase.DataSource, PackageDatabase.InitialCatalog);

JobEventSourceLog.PreparingToArchive(Source.Credentials.AccountName, SourceContainer.Name, PrimaryDestination.Credentials.AccountName, PrimaryDestinationContainer.Name, _packageDbConnectionFactory.DataSource, _packageDbConnectionFactory.InitialCatalog);
await Archive(PrimaryDestinationContainer);

// todo: consider reusing package query for primary and secondary archives
if (SecondaryDestinationContainer != null)
{
JobEventSourceLog.PreparingToArchive2(SecondaryDestination.Credentials.AccountName, SecondaryDestinationContainer.Name);

await Archive(SecondaryDestinationContainer);
}
}
Expand All @@ -126,10 +124,9 @@ private async Task Archive(CloudBlobContainer destinationContainer)

JobEventSourceLog.CursorData(cursorDateTime.ToString(DateTimeFormatSpecifier));

JobEventSourceLog.GatheringPackagesToArchiveFromDb(PackageDatabase.DataSource, PackageDatabase.InitialCatalog);

JobEventSourceLog.GatheringPackagesToArchiveFromDb(_packageDbConnectionFactory.DataSource, _packageDbConnectionFactory.InitialCatalog);
List<PackageRef> packages;
using (var connection = await OpenSqlConnectionAsync(JobArgumentNames.PackageDatabase))
using (var connection = await _packageDbConnectionFactory.CreateAsync())
{
packages = (await connection.QueryAsync<PackageRef>(@"
SELECT pr.Id, p.NormalizedVersion AS Version, p.Hash, p.LastEdited, p.Published
Expand All @@ -138,8 +135,7 @@ FROM Packages p
WHERE Published > @cursorDateTime OR LastEdited > @cursorDateTime", new { cursorDateTime = cursorDateTime }))
.ToList();
}

JobEventSourceLog.GatheredPackagesToArchiveFromDb(packages.Count, PackageDatabase.DataSource, PackageDatabase.InitialCatalog);
JobEventSourceLog.GatheredPackagesToArchiveFromDb(packages.Count, _packageDbConnectionFactory.DataSource, _packageDbConnectionFactory.InitialCatalog);

var archiveSet = packages
.AsParallel()
Expand Down
3 changes: 3 additions & 0 deletions src/ArchivePackages/ArchivePackages.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,9 @@
<PackageReference Include="Newtonsoft.Json">
<Version>9.0.1</Version>
</PackageReference>
<PackageReference Include="NuGet.Services.Sql">
<Version>2.25.0-master-30453</Version>
</PackageReference>
<PackageReference Include="System.Net.Http">
<Version>4.3.3</Version>
</PackageReference>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,9 @@
<PackageReference Include="Newtonsoft.Json">
<Version>9.0.1</Version>
</PackageReference>
<PackageReference Include="NuGet.Services.Sql">
<Version>2.25.0-master-30263</Version>
</PackageReference>
<PackageReference Include="NuGet.Services.Storage">
<Version>2.1.3</Version>
</PackageReference>
Expand Down
12 changes: 5 additions & 7 deletions src/Gallery.CredentialExpiration/GalleryCredentialExpiration.cs
Original file line number Diff line number Diff line change
Expand Up @@ -6,22 +6,20 @@
using System.Data.SqlClient;
using System.Linq;
using System.Threading.Tasks;
using NuGet.Services.Sql;
using Gallery.CredentialExpiration.Models;

namespace Gallery.CredentialExpiration
{
public class GalleryCredentialExpiration : ICredentialExpirationExporter
{
private readonly CredentialExpirationJobMetadata _jobMetadata;
private readonly ISqlConnectionFactory _galleryDatabase;

private Func<Task<SqlConnection>> OpenGallerySqlConnectionAsync { get; }

public GalleryCredentialExpiration(
CredentialExpirationJobMetadata jobMetadata,
Func<Task<SqlConnection>> openGallerySqlConnectionAsync)
public GalleryCredentialExpiration(CredentialExpirationJobMetadata jobMetadata, ISqlConnectionFactory galleryDatabase)
{
_jobMetadata = jobMetadata;
OpenGallerySqlConnectionAsync = openGallerySqlConnectionAsync;
_galleryDatabase = galleryDatabase;
}

/// <summary>
Expand Down Expand Up @@ -53,7 +51,7 @@ public async Task<List<ExpiredCredentialData>> GetCredentialsAsync(TimeSpan time
var minNotificationDate = ConvertToString(GetMinNotificationDate());

// Connect to database
using (var galleryConnection = await OpenGallerySqlConnectionAsync())
using (var galleryConnection = await _galleryDatabase.CreateAsync())
{
// Fetch credentials that expire in _warnDaysBeforeExpiration days
// + the user's e-mail address
Expand Down
19 changes: 11 additions & 8 deletions src/Gallery.CredentialExpiration/Job.cs
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.

using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.ComponentModel.Design;
using System.Data.SqlClient;
Expand All @@ -14,7 +15,10 @@
using Microsoft.Extensions.Logging;
using Microsoft.WindowsAzure.Storage;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using NuGet.Jobs;
using NuGet.Services.KeyVault;
using NuGet.Services.Sql;
using NuGet.Services.Storage;

namespace Gallery.CredentialExpiration
Expand All @@ -30,6 +34,8 @@ public class Job : JobBase
private string _galleryBrand;
private string _galleryAccountUrl;

private ISqlConnectionFactory _galleryDatabase;

private string _mailFrom;
private SmtpClient _smtpClient;

Expand All @@ -41,7 +47,9 @@ public override void Init(IServiceContainer serviceContainer, IDictionary<string
{
_whatIf = JobConfigurationManager.TryGetBoolArgument(jobArgsDictionary, JobArgumentNames.WhatIf);

RegisterDatabase(serviceContainer, jobArgsDictionary, JobArgumentNames.GalleryDatabase);
var secretInjector = (ISecretInjector)serviceContainer.GetService(typeof(ISecretInjector));
var databaseConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.GalleryDatabase);
_galleryDatabase = new AzureSqlConnectionFactory(databaseConnectionString, secretInjector);

_galleryBrand = JobConfigurationManager.GetArgument(jobArgsDictionary, MyJobArgumentNames.GalleryBrand);
_galleryAccountUrl = JobConfigurationManager.GetArgument(jobArgsDictionary, MyJobArgumentNames.GalleryAccountUrl);
Expand All @@ -63,17 +71,12 @@ public override void Init(IServiceContainer serviceContainer, IDictionary<string
_storage = storageFactory.Create();
}

public Task<SqlConnection> OpenGallerySqlConnectionAsync()
{
return OpenSqlConnectionAsync(JobArgumentNames.GalleryDatabase);
}

public override async Task Run()
{
var jobRunTime = DateTimeOffset.UtcNow;
// Default values
var jobCursor = new JobRunTimeCursor( jobCursorTime: jobRunTime, maxProcessedCredentialsTime: jobRunTime );
var galleryCredentialExpiration = new GalleryCredentialExpiration(new CredentialExpirationJobMetadata(jobRunTime, _warnDaysBeforeExpiration, jobCursor), OpenGallerySqlConnectionAsync);
var galleryCredentialExpiration = new GalleryCredentialExpiration(new CredentialExpirationJobMetadata(jobRunTime, _warnDaysBeforeExpiration, jobCursor), _galleryDatabase);

try
{
Expand All @@ -86,7 +89,7 @@ public override async Task Run()
// Load from cursor
// Throw if the schema is not correct to ensure that not-intended emails are sent.
jobCursor = JsonConvert.DeserializeObject<JobRunTimeCursor>(content, new JsonSerializerSettings() { MissingMemberHandling = MissingMemberHandling.Error });
galleryCredentialExpiration = new GalleryCredentialExpiration(new CredentialExpirationJobMetadata(jobRunTime, _warnDaysBeforeExpiration, jobCursor), OpenGallerySqlConnectionAsync);
galleryCredentialExpiration = new GalleryCredentialExpiration(new CredentialExpirationJobMetadata(jobRunTime, _warnDaysBeforeExpiration, jobCursor), _galleryDatabase);
}

// Connect to database
Expand Down
5 changes: 2 additions & 3 deletions src/Gallery.Maintenance/DeleteExpiredVerificationKeysTask.cs
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@
using System.Threading.Tasks;
using Gallery.Maintenance.Models;
using Microsoft.Extensions.Logging;
using NuGet.Jobs;

namespace Gallery.Maintenance
{
Expand Down Expand Up @@ -38,7 +37,7 @@ public override async Task RunAsync(Job job)
{
IEnumerable<PackageVerificationKey> expiredKeys;

using (var connection = await job.OpenSqlConnectionAsync(JobArgumentNames.GalleryDatabase))
using (var connection = await job.GalleryDatabase.CreateAsync())
{
expiredKeys = await connection.QueryWithRetryAsync<PackageVerificationKey>(
SelectQuery,
Expand All @@ -60,7 +59,7 @@ public override async Task RunAsync(Job job)

if (expectedRowCount > 0)
{
using (var connection = await job.OpenSqlConnectionAsync(JobArgumentNames.GalleryDatabase))
using (var connection = await job.GalleryDatabase.CreateAsync())
{
using (var transaction = connection.BeginTransaction())
{
Expand Down
3 changes: 3 additions & 0 deletions src/Gallery.Maintenance/Gallery.Maintenance.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,9 @@
<PackageReference Include="Newtonsoft.Json">
<Version>9.0.1</Version>
</PackageReference>
<PackageReference Include="NuGet.Services.Sql">
<Version>2.25.0-master-30263</Version>
</PackageReference>
<PackageReference Include="System.Net.Http">
<Version>4.3.3</Version>
</PackageReference>
Expand Down
10 changes: 9 additions & 1 deletion src/Gallery.Maintenance/Job.cs
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,13 @@
using System;
using System.Collections.Generic;
using System.ComponentModel.Design;
using System.Data;
using System.Linq;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using NuGet.Jobs;
using NuGet.Services.KeyVault;
using NuGet.Services.Sql;

namespace Gallery.Maintenance
{
Expand All @@ -17,9 +19,15 @@ namespace Gallery.Maintenance
/// </summary>
public class Job : JobBase
{

public ISqlConnectionFactory GalleryDatabase { get; private set; }

public override void Init(IServiceContainer serviceContainer, IDictionary<string, string> jobArgsDictionary)
{
RegisterDatabase(serviceContainer, jobArgsDictionary, JobArgumentNames.GalleryDatabase);
var secretInjector = (ISecretInjector)serviceContainer.GetService(typeof(ISecretInjector));
var databaseConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.GalleryDatabase);

GalleryDatabase = new AzureSqlConnectionFactory(databaseConnectionString, secretInjector);
}

public override async Task Run()
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.

using System.Threading.Tasks;

// ReSharper disable once CheckNamespace
namespace System.Data.SqlClient
{
public static class SqlConnectionStringBuilderExtensions
{
public static Task<SqlConnection> ConnectTo(this SqlConnectionStringBuilder self)
{
return ConnectTo(self.ConnectionString);
}

private static async Task<SqlConnection> ConnectTo(string connection)
{
var c = new SqlConnection(connection);
await c.OpenAsync().ConfigureAwait(continueOnCapturedContext: false);
return c;
}
}
}
Loading

0 comments on commit 7e87d9d

Please sign in to comment.