Skip to content
This repository has been archived by the owner on Jul 30, 2024. It is now read-only.
/ NuGet.Jobs Public archive

Commit

Permalink
Feed2Catalog should throttle its Azure storage requests (#520)
Browse files Browse the repository at this point in the history
  • Loading branch information
Scott Bommarito authored May 16, 2019
1 parent 6e20830 commit 62358c0
Show file tree
Hide file tree
Showing 9 changed files with 82 additions and 39 deletions.
3 changes: 3 additions & 0 deletions src/Catalog/NuGet.Services.Metadata.Catalog.csproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -254,6 +254,9 @@
<PackageReference Include="NuGet.Packaging">
<Version>5.0.0-preview1.5665</Version>
</PackageReference>
<PackageReference Include="NuGet.Protocol">
<Version>5.0.0-preview1.5665</Version>
</PackageReference>
<PackageReference Include="NuGet.StrongName.json-ld.net">
<Version>1.0.6</Version>
</PackageReference>
Expand Down
32 changes: 10 additions & 22 deletions src/Catalog/Persistence/AzureStorage.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,39 +14,20 @@
using Microsoft.WindowsAzure.Storage.Blob;
using Microsoft.WindowsAzure.Storage.DataMovement;
using Microsoft.WindowsAzure.Storage.RetryPolicies;
using NuGet.Protocol;

namespace NuGet.Services.Metadata.Catalog.Persistence
{
public class AzureStorage : Storage, IAzureStorage
{
private readonly bool _compressContent;
private readonly IThrottle _throttle;
private readonly CloudBlobDirectory _directory;
private readonly bool _useServerSideCopy;

public static readonly TimeSpan DefaultServerTimeout = TimeSpan.FromSeconds(30);
public static readonly TimeSpan DefaultMaxExecutionTime = TimeSpan.FromMinutes(10);

public AzureStorage(
CloudStorageAccount account,
string containerName,
string path,
Uri baseAddress,
bool useServerSideCopy,
bool compressContent,
bool verbose) : this(
account,
containerName,
path,
baseAddress,
DefaultMaxExecutionTime,
DefaultServerTimeout,
useServerSideCopy,
compressContent,
verbose,
initializeContainer: true)
{
}

public AzureStorage(
CloudStorageAccount account,
string containerName,
Expand All @@ -57,7 +38,8 @@ public AzureStorage(
bool useServerSideCopy,
bool compressContent,
bool verbose,
bool initializeContainer) : this(
bool initializeContainer,
IThrottle throttle) : this(
account.CreateCloudBlobClient().GetContainerReference(containerName).GetDirectoryReference(path),
baseAddress,
maxExecutionTime,
Expand All @@ -66,6 +48,7 @@ public AzureStorage(
{
_useServerSideCopy = useServerSideCopy;
_compressContent = compressContent;
_throttle = throttle ?? NullThrottle.Instance;
Verbose = verbose;
}

Expand Down Expand Up @@ -309,6 +292,7 @@ protected override async Task<StorageContent> OnLoadAsync(Uri resourceUri, Cance

CloudBlockBlob blob = GetBlockBlobReference(name);

await _throttle.WaitAsync();
try
{
string content;
Expand Down Expand Up @@ -349,6 +333,10 @@ protected override async Task<StorageContent> OnLoadAsync(Uri resourceUri, Cance

return null;
}
finally
{
_throttle.Release();
}
}

protected override async Task OnDeleteAsync(Uri resourceUri, CancellationToken cancellationToken)
Expand Down
7 changes: 5 additions & 2 deletions src/Catalog/Persistence/AzureStorageFactory.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@

using System;
using Microsoft.WindowsAzure.Storage;
using NuGet.Protocol;

namespace NuGet.Services.Metadata.Catalog.Persistence
{
Expand All @@ -27,7 +28,8 @@ public AzureStorageFactory(
bool useServerSideCopy,
bool compressContent,
bool verbose,
bool initializeContainer)
bool initializeContainer,
IThrottle throttle) : base(throttle)
{
_account = account;
_containerName = containerName;
Expand Down Expand Up @@ -101,7 +103,8 @@ public override Storage Create(string name = null)
_useServerSideCopy,
CompressContent,
Verbose,
_initializeContainer);
_initializeContainer,
Throttle);
}
}
}
12 changes: 12 additions & 0 deletions src/Catalog/Persistence/StorageFactory.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,21 @@
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.

using System;
using NuGet.Protocol;

namespace NuGet.Services.Metadata.Catalog.Persistence
{
public abstract class StorageFactory : IStorageFactory
{
public StorageFactory() : this(throttle: null)
{
}

public StorageFactory(IThrottle throttle)
{
Throttle = throttle ?? NullThrottle.Instance;
}

public abstract Storage Create(string name = null);

public Uri BaseAddress { get; protected set; }
Expand All @@ -16,6 +26,8 @@ public abstract class StorageFactory : IStorageFactory

public bool Verbose { get; protected set; }

public IThrottle Throttle { get; }

public override string ToString()
{
return BaseAddress.ToString();
Expand Down
39 changes: 30 additions & 9 deletions src/Ng/CommandHelpers.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
using Microsoft.Extensions.Logging;
using Microsoft.WindowsAzure.Storage;
using Microsoft.WindowsAzure.Storage.Auth;
using NuGet.Protocol;
using NuGet.Services.Configuration;
using NuGet.Services.KeyVault;
using NuGet.Services.Metadata.Catalog;
Expand Down Expand Up @@ -127,7 +128,10 @@ public static RegistrationStorageFactories CreateRegistrationStorageFactories(ID
return new RegistrationStorageFactories(legacyStorageFactory, semVer2StorageFactory);
}

public static CatalogStorageFactory CreateStorageFactory(IDictionary<string, string> arguments, bool verbose)
public static CatalogStorageFactory CreateStorageFactory(
IDictionary<string, string> arguments,
bool verbose,
IThrottle throttle = null)
{
IDictionary<string, string> names = new Dictionary<string, string>
{
Expand All @@ -142,7 +146,12 @@ public static CatalogStorageFactory CreateStorageFactory(IDictionary<string, str
{ Arguments.StorageServerTimeoutInSeconds, Arguments.StorageServerTimeoutInSeconds }
};

return CreateStorageFactoryImpl(arguments, names, verbose, compressed: false);
return CreateStorageFactoryImpl(
arguments,
names,
verbose,
compressed: false,
throttle: throttle);
}

public static CatalogStorageFactory CreateCompressedStorageFactory(IDictionary<string, string> arguments, bool verbose)
Expand Down Expand Up @@ -191,7 +200,11 @@ public static CatalogStorageFactory CreateSemVer2StorageFactory(IDictionary<stri
return CreateStorageFactoryImpl(arguments, names, verbose, compressed: true);
}

public static CatalogStorageFactory CreateSuffixedStorageFactory(string suffix, IDictionary<string, string> arguments, bool verbose)
public static CatalogStorageFactory CreateSuffixedStorageFactory(
string suffix,
IDictionary<string, string> arguments,
bool verbose,
IThrottle throttle = null)
{
if (string.IsNullOrEmpty(suffix))
{
Expand All @@ -211,13 +224,20 @@ public static CatalogStorageFactory CreateSuffixedStorageFactory(string suffix,
{ Arguments.StorageServerTimeoutInSeconds, Arguments.StorageServerTimeoutInSeconds }
};

return CreateStorageFactoryImpl(arguments, names, verbose, compressed: false);
return CreateStorageFactoryImpl(
arguments,
names,
verbose,
compressed: false,
throttle: throttle);
}

private static CatalogStorageFactory CreateStorageFactoryImpl(IDictionary<string, string> arguments,
IDictionary<string, string> argumentNameMap,
bool verbose,
bool compressed)
private static CatalogStorageFactory CreateStorageFactoryImpl(
IDictionary<string, string> arguments,
IDictionary<string, string> argumentNameMap,
bool verbose,
bool compressed,
IThrottle throttle = null)
{
Uri storageBaseAddress = null;
var storageBaseAddressStr = arguments.GetOrDefault<string>(argumentNameMap[Arguments.StorageBaseAddress]);
Expand Down Expand Up @@ -270,7 +290,8 @@ private static CatalogStorageFactory CreateStorageFactoryImpl(IDictionary<string
storageUseServerSideCopy,
compressed,
verbose,
initializeContainer: true);
initializeContainer: true,
throttle: throttle ?? NullThrottle.Instance);
}
throw new ArgumentException($"Unrecognized storageType \"{storageType}\"");
}
Expand Down
20 changes: 17 additions & 3 deletions src/Ng/Jobs/Feed2CatalogJob.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,12 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using NuGet.Protocol;
using NuGet.Services.Configuration;
using NuGet.Services.Metadata.Catalog;
using NuGet.Services.Metadata.Catalog.Helpers;
Expand Down Expand Up @@ -83,11 +85,23 @@ protected override void Init(IDictionary<string, string> arguments, Cancellation

if (preferAlternatePackageSourceStorage)
{
preferredPackageSourceStorageFactory = CommandHelpers.CreateSuffixedStorageFactory("PreferredPackageSourceStorage", arguments, Verbose);
preferredPackageSourceStorageFactory = CommandHelpers.CreateSuffixedStorageFactory(
"PreferredPackageSourceStorage",
arguments,
Verbose,
new SemaphoreSlimThrottle(new SemaphoreSlim(ServicePointManager.DefaultConnectionLimit)));
}

var catalogStorageFactory = CommandHelpers.CreateStorageFactory(arguments, Verbose);
var auditingStorageFactory = CommandHelpers.CreateSuffixedStorageFactory("Auditing", arguments, Verbose);
var catalogStorageFactory = CommandHelpers.CreateStorageFactory(
arguments,
Verbose,
new SemaphoreSlimThrottle(new SemaphoreSlim(ServicePointManager.DefaultConnectionLimit)));

var auditingStorageFactory = CommandHelpers.CreateSuffixedStorageFactory(
"Auditing",
arguments,
Verbose,
new SemaphoreSlimThrottle(new SemaphoreSlim(ServicePointManager.DefaultConnectionLimit)));

Logger.LogInformation("CONFIG source: \"{ConfigSource}\" storage: \"{Storage}\" preferred package source storage: \"{PreferredPackageSourceStorage}\"",
Gallery,
Expand Down
2 changes: 1 addition & 1 deletion src/Ng/Ng.csproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -172,7 +172,7 @@
<PrivateAssets>all</PrivateAssets>
</PackageReference>
<PackageReference Include="NuGet.Protocol">
<Version>4.8.0</Version>
<Version>5.0.0-preview1.5665</Version>
</PackageReference>
<PackageReference Include="NuGet.Services.Configuration">
<Version>2.2.2</Version>
Expand Down
4 changes: 3 additions & 1 deletion src/NuGet.Services.AzureSearch/DependencyInjectionExtensions.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
using Microsoft.Extensions.Options;
using Microsoft.WindowsAzure.Storage;
using NuGet.Jobs.Validation;
using NuGet.Protocol;
using NuGet.Protocol.Catalog;
using NuGet.Protocol.Registration;
using NuGet.Services.AzureSearch.Catalog2AzureSearch;
Expand Down Expand Up @@ -128,7 +129,8 @@ private static void RegisterAzureSearchJobStorageServices(ContainerBuilder conta
useServerSideCopy: true,
compressContent: false,
verbose: true,
initializeContainer: false);
initializeContainer: false,
throttle: NullThrottle.Instance);
})
.Keyed<IStorageFactory>(key);

Expand Down
2 changes: 1 addition & 1 deletion tests/NgTests/NgTests.csproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -197,7 +197,7 @@
<Version>4.7.25</Version>
</PackageReference>
<PackageReference Include="NuGet.Protocol">
<Version>4.8.0</Version>
<Version>5.0.0-preview1.5665</Version>
</PackageReference>
<PackageReference Include="NuGet.Services.Configuration">
<Version>2.28.0</Version>
Expand Down

0 comments on commit 62358c0

Please sign in to comment.