Orchestrator extracts license file to flatcontainer (#687)

* Added PackageStatusProcessor for package-specific code. * Added optional OnBeforeUpdateDatabaseToMakePackageAvailable to EntityStatusProcessor. * Added telemetry for license file operations. * ServerCommon version upgrade.
NuGet · Dec 6, 2018 · 15d62a1 · 15d62a1
1 parent 8a3b27d
commit 15d62a1
Show file tree

Hide file tree

Showing 19 changed files with 533 additions and 277 deletions.
diff --git a/src/NuGet.Jobs.Common/NuGet.Jobs.Common.csproj b/src/NuGet.Jobs.Common/NuGet.Jobs.Common.csproj
@@ -105,13 +105,13 @@
       <PrivateAssets>all</PrivateAssets>
     </PackageReference>
     <PackageReference Include="NuGet.Services.Configuration">
-      <Version>2.39.0</Version>
+      <Version>2.40.0</Version>
     </PackageReference>
     <PackageReference Include="NuGet.Services.Logging">
-      <Version>2.39.0</Version>
+      <Version>2.40.0</Version>
     </PackageReference>
     <PackageReference Include="NuGet.Services.Sql">
-      <Version>2.39.0</Version>
+      <Version>2.40.0</Version>
     </PackageReference>
     <PackageReference Include="System.Net.Http">
       <Version>4.3.3</Version>

diff --git a/src/NuGet.Services.Revalidate/NuGet.Services.Revalidate.csproj b/src/NuGet.Services.Revalidate/NuGet.Services.Revalidate.csproj
@@ -123,7 +123,7 @@
       <PrivateAssets>all</PrivateAssets>
     </PackageReference>
     <PackageReference Include="NuGet.Services.Status">
-      <Version>2.39.0</Version>
+      <Version>2.40.0</Version>
     </PackageReference>
   </ItemGroup>
   <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />

diff --git a/src/NuGet.Services.Validation.Orchestrator/Configuration/FlatContainerConfiguration.cs b/src/NuGet.Services.Validation.Orchestrator/Configuration/FlatContainerConfiguration.cs
@@ -0,0 +1,10 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+namespace NuGet.Services.Validation.Orchestrator
+{
+    public class FlatContainerConfiguration
+    {
+        public string ConnectionString { get; set; }
+    }
+}
diff --git a/src/NuGet.Services.Validation.Orchestrator/EntityStatusProcessor.cs b/src/NuGet.Services.Validation.Orchestrator/EntityStatusProcessor.cs
diff --git a/src/NuGet.Services.Validation.Orchestrator/Job.cs b/src/NuGet.Services.Validation.Orchestrator/Job.cs
@@ -61,9 +61,9 @@ public class Job : JobBase
         private const string GalleryDbConfigurationSectionName = "GalleryDb";
         private const string ValidationDbConfigurationSectionName = "ValidationDb";
         private const string ServiceBusConfigurationSectionName = "ServiceBus";
-        private const string SmtpConfigurationSectionName = "Smtp";
         private const string EmailConfigurationSectionName = "Email";
         private const string PackageDownloadTimeoutName = "PackageDownloadTimeout";
+        private const string FlatContainerConfigurationSectionName = "FlatContainer";
 
         private const string EmailBindingKey = EmailConfigurationSectionName;
         private const string PackageVerificationTopicClientBindingKey = "PackageVerificationTopicClient";
@@ -74,6 +74,7 @@ public class Job : JobBase
         private const string ScanBindingKey = "Scan";
         private const string ValidationStorageBindingKey = "ValidationStorage";
         private const string OrchestratorBindingKey = "Orchestrator";
+        private const string CoreLicenseFileServiceBindingKey = "CoreLicenseFileService";
 
         private const string SymbolsValidatorSectionName = "SymbolsValidator";
         private const string SymbolsValidationBindingKey = SymbolsValidatorSectionName;
@@ -193,6 +194,7 @@ private void ConfigureJobServices(IServiceCollection services, IConfigurationRoo
             services.Configure<ScanAndSignConfiguration>(configurationRoot.GetSection(ScanAndSignSectionName));
             services.Configure<SymbolScanOnlyConfiguration>(configurationRoot.GetSection(SymbolScanOnlySectionName));
             services.Configure<ScanAndSignEnqueuerConfiguration>(configurationRoot.GetSection(ScanAndSignSectionName));
+            services.Configure<FlatContainerConfiguration>(configurationRoot.GetSection(FlatContainerConfigurationSectionName));
 
             services.Configure<SymbolsValidationConfiguration>(configurationRoot.GetSection(SymbolsValidatorSectionName));
             services.Configure<SymbolsIngesterConfiguration>(configurationRoot.GetSection(SymbolsIngesterSectionName));
@@ -243,7 +245,7 @@ private void ConfigureJobServices(IServiceCollection services, IConfigurationRoo
                 });
             services.AddTransient<ICoreFileStorageService, CloudBlobCoreFileStorageService>();
             services.AddTransient<IFileDownloader, PackageDownloader>();
-            services.AddTransient<IStatusProcessor<Package>, EntityStatusProcessor<Package>>();
+            services.AddTransient<IStatusProcessor<Package>, PackageStatusProcessor>();
             services.AddTransient<IValidationSetProvider<Package>, ValidationSetProvider<Package>>();
             services.AddTransient<IValidationSetProcessor, ValidationSetProcessor>();
             services.AddTransient<IBrokeredMessageSerializer<SignatureValidationMessage>, SignatureValidationMessageSerializer>();
@@ -349,6 +351,7 @@ private static IServiceProvider CreateProvider(IServiceCollection services, ICon
                     ConfigurePackageCertificatesValidator(containerBuilder);
                     ConfigureScanAndSignProcessor(containerBuilder);
                     ConfigureScanValidator(containerBuilder);
+                    ConfigureFlatContainer(containerBuilder);
                     break;
                 case ValidatingType.SymbolPackage:
                     ConfigureSymbolScanValidator(containerBuilder);
@@ -507,6 +510,33 @@ private static void ConfigureScanValidator(ContainerBuilder builder)
                 .AsSelf();
         }
 
+        private static void ConfigureFlatContainer(ContainerBuilder builder)
+        {
+            builder
+                .Register<CloudBlobClientWrapper>(c =>
+                {
+                    var configurationAccessor = c.Resolve<IOptionsSnapshot<FlatContainerConfiguration>>();
+                    return new CloudBlobClientWrapper(
+                        configurationAccessor.Value.ConnectionString,
+                        readAccessGeoRedundant: false);
+                })
+                .Keyed<ICloudBlobClient>(CoreLicenseFileServiceBindingKey);
+
+            builder
+                .RegisterType<CloudBlobCoreFileStorageService>()
+                .WithKeyedParameter(typeof(ICloudBlobClient), CoreLicenseFileServiceBindingKey)
+                .Keyed<ICoreFileStorageService>(CoreLicenseFileServiceBindingKey);
+
+            builder
+                .RegisterType<OrchestratorContentFileMetadataService>()
+                .As<IContentFileMetadataService>();
+
+            builder
+                .RegisterType<CoreLicenseFileService>()
+                .WithKeyedParameter(typeof(ICoreFileStorageService), CoreLicenseFileServiceBindingKey)
+                .As<ICoreLicenseFileService>();
+        }
+
         private static void ConfigureOrchestratorMessageHandler(IServiceCollection services, IConfigurationRoot configurationRoot)
         {
             var validatingType = configurationRoot

diff --git a/src/NuGet.Services.Validation.Orchestrator/NuGet.Services.Validation.Orchestrator.csproj b/src/NuGet.Services.Validation.Orchestrator/NuGet.Services.Validation.Orchestrator.csproj
@@ -45,12 +45,16 @@
   <ItemGroup>
     <Compile Include="BaseValidator.cs" />
     <Compile Include="Configuration\ConfigurationValidator.cs" />
+    <Compile Include="Configuration\FlatContainerConfiguration.cs" />
     <Compile Include="Configuration\TopologicalSort.cs" />
     <Compile Include="ContainerBuilderExtensions.cs" />
     <Compile Include="Configuration\CoreMessageServiceConfiguration.cs" />
     <Compile Include="Configuration\EmailConfiguration.cs" />
+    <Compile Include="EntityStatusProcessor.cs" />
     <Compile Include="Error.cs" />
+    <Compile Include="PackageStatusProcessor.cs" />
     <Compile Include="Services\MessageServiceConfiguration.cs" />
+    <Compile Include="Services\OrchestratorContentFileMetadataService.cs" />
     <Compile Include="Services\SymbolsMessageService.cs" />
     <Compile Include="SymbolsStatusProcessor.cs" />
     <Compile Include="Symbols\ISymbolsIngesterMessageEnqueuer.cs" />
@@ -92,7 +96,6 @@
     <Compile Include="PackageSigning\ValidateCertificate\ValidateCertificateConfiguration.cs" />
     <Compile Include="PackageSigning\ValidateCertificate\ValidateCertificateEnqueuer.cs" />
     <Compile Include="PackageSigning\ProcessSignature\PackageSignatureValidator.cs" />
-    <Compile Include="PackageStatusProcessor.cs" />
     <Compile Include="PackageValidationMessageDataSerializer.cs" />
     <Compile Include="Program.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
@@ -160,7 +163,7 @@
       <PrivateAssets>all</PrivateAssets>
     </PackageReference>
     <PackageReference Include="NuGet.Services.Messaging.Email">
-      <Version>2.39.0</Version>
+      <Version>2.40.0</Version>
     </PackageReference>
   </ItemGroup>
   <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />