Add Db2Monitoring to verify that all packages in the database have be…

…en monitored (#555)

build.ps1

@@ -118,6 +118,7 @@ Invoke-BuildStep 'Creating artifacts' {
             "src\Ng\Catalog2Monitoring.nuspec", `
             "src\Ng\Catalog2Registration.nuspec", `
             "src\Ng\Db2Catalog.nuspec", `
+            "src\Ng\Db2Monitoring.nuspec", `
             "src\Ng\Monitoring2Monitoring.nuspec", `
             "src\Ng\MonitoringProcessor.nuspec", `
             "src\Ng\Ng.Operations.nuspec", `

src/Catalog/Helpers/Db2CatalogProjection.cs

@@ -34,6 +34,7 @@ public FeedPackageDetails ReadFeedPackageDetailsFromDataReader(DbDataReader data
 
             var packageId = dataReader[Db2CatalogProjectionColumnNames.PackageId].ToString();
             var normalizedPackageVersion = dataReader[Db2CatalogProjectionColumnNames.NormalizedVersion].ToString();
+            var fullPackageVersion = dataReader[Db2CatalogProjectionColumnNames.FullVersion].ToString();
             var listed = dataReader.GetBoolean(dataReader.GetOrdinal(Db2CatalogProjectionColumnNames.Listed));
             var hideLicenseReport = dataReader.GetBoolean(dataReader.GetOrdinal(Db2CatalogProjectionColumnNames.HideLicenseReport));
 
@@ -47,6 +48,7 @@ public FeedPackageDetails ReadFeedPackageDetailsFromDataReader(DbDataReader data
                 listed ? dataReader.ReadDateTime(Db2CatalogProjectionColumnNames.Published).ForceUtc() : Constants.UnpublishedDate,
                 packageId,
                 normalizedPackageVersion,
+                fullPackageVersion,
                 hideLicenseReport ? null : dataReader[Db2CatalogProjectionColumnNames.LicenseNames]?.ToString(),
                 hideLicenseReport ? null : dataReader[Db2CatalogProjectionColumnNames.LicenseReportUrl]?.ToString(),
                 deprecationInfo,

src/Catalog/Helpers/Db2CatalogProjectionColumnNames.cs

@@ -10,6 +10,7 @@ public static class Db2CatalogProjectionColumnNames
     {
         public const string PackageId = "Id";
         public const string NormalizedVersion = "NormalizedVersion";
+        public const string FullVersion = "Version";
         public const string Listed = "Listed";
         public const string HideLicenseReport = "HideLicenseReport";
         public const string Created = "Created";

src/Catalog/Helpers/FeedPackageDetails.cs

@@ -12,7 +12,8 @@ public sealed class FeedPackageDetails
         public DateTime LastEditedDate { get; }
         public DateTime PublishedDate { get; }
         public string PackageId { get; }
-        public string PackageVersion { get; }
+        public string PackageNormalizedVersion { get; }
+        public string PackageFullVersion { get; }
         public string LicenseNames { get; }
         public string LicenseReportUrl { get; }
         public bool RequiresLicenseAcceptance { get; }
@@ -26,14 +27,16 @@ public FeedPackageDetails(
             DateTime lastEditedDate,
             DateTime publishedDate,
             string packageId,
-            string packageVersion)
+            string packageNormalizedVersion,
+            string packageFullVersion)
             : this(
                 contentUri,
                 createdDate,
                 lastEditedDate,
                 publishedDate,
                 packageId,
-                packageVersion,
+                packageNormalizedVersion,
+                packageFullVersion,
                 licenseNames: null,
                 licenseReportUrl: null,
                 deprecationInfo: null,
@@ -47,7 +50,8 @@ public FeedPackageDetails(
             DateTime lastEditedDate,
             DateTime publishedDate,
             string packageId,
-            string packageVersion,
+            string packageNormalizedVersion,
+            string packageFullVersion,
             string licenseNames,
             string licenseReportUrl,
             PackageDeprecationItem deprecationInfo,
@@ -58,7 +62,8 @@ public FeedPackageDetails(
             LastEditedDate = lastEditedDate;
             PublishedDate = publishedDate;
             PackageId = packageId;
-            PackageVersion = packageVersion;
+            PackageNormalizedVersion = packageNormalizedVersion;
+            PackageFullVersion = packageFullVersion;
             LicenseNames = licenseNames;
             LicenseReportUrl = licenseReportUrl;
             DeprecationInfo = deprecationInfo;

src/Catalog/Helpers/GalleryDatabaseQueryService.cs

@@ -22,6 +22,7 @@ public class GalleryDatabaseQueryService : IGalleryDatabaseQueryService
 
         private static readonly string Db2CatalogSqlSubQuery = $@" PR.[Id],	
                         P.[NormalizedVersion],	
+                        P.[Version],
                         P.[Created],	
                         P.[LastEdited],
                         P.[Published],

src/Catalog/PackageCatalogItemCreator.cs

@@ -72,7 +72,7 @@ public async Task<PackageCatalogItem> CreateAsync(
             _logger.LogInformation(
                 "Creating package catalog item for {Id} {Version}",
                 packageItem.PackageId,
-                packageItem.PackageVersion);
+                packageItem.PackageNormalizedVersion);
 
             if (_storage != null)
             {
@@ -87,7 +87,7 @@ public async Task<PackageCatalogItem> CreateAsync(
             _logger.LogInformation(
                 "Finished creating package catalog item for {Id} {Version}",
                 packageItem.PackageId,
-                packageItem.PackageVersion);
+                packageItem.PackageNormalizedVersion);
 
             return item;
         }
@@ -98,8 +98,8 @@ private async Task<PackageCatalogItem> GetPackageViaStorageAsync(
         {
             PackageCatalogItem item = null;
             var packageId = packageItem.PackageId.ToLowerInvariant();
-            var packageVersion = packageItem.PackageVersion.ToLowerInvariant();
-            var packageFileName = PackageUtility.GetPackageFileName(packageId, packageVersion);
+            var packageNormalizedVersion = packageItem.PackageNormalizedVersion.ToLowerInvariant();
+            var packageFileName = PackageUtility.GetPackageFileName(packageId, packageNormalizedVersion);
             var blobUri = _storage.ResolveUri(packageFileName);
             var blob = await _storage.GetCloudBlockBlobReferenceAsync(blobUri);
 
@@ -108,14 +108,14 @@ private async Task<PackageCatalogItem> GetPackageViaStorageAsync(
                 _telemetryService.TrackMetric(
                     TelemetryConstants.NonExistentBlob,
                     metric: 1,
-                    properties: GetProperties(packageId, packageVersion, blob));
+                    properties: GetProperties(packageId, packageNormalizedVersion, blob));
 
                 return item;
             }
 
             using (_telemetryService.TrackDuration(
                 TelemetryConstants.PackageBlobReadSeconds,
-                GetProperties(packageId, packageVersion, blob: null)))
+                GetProperties(packageId, packageNormalizedVersion, blob: null)))
             {
                 await blob.FetchAttributesAsync(cancellationToken);
 
@@ -162,7 +162,7 @@ private async Task<PackageCatalogItem> GetPackageViaStorageAsync(
                             _telemetryService.TrackMetric(
                                 TelemetryConstants.BlobModified,
                                 metric: 1,
-                                properties: GetProperties(packageId, packageVersion, blob));
+                                properties: GetProperties(packageId, packageNormalizedVersion, blob));
                         }
                     }
                 }
@@ -171,7 +171,7 @@ private async Task<PackageCatalogItem> GetPackageViaStorageAsync(
                     _telemetryService.TrackMetric(
                         TelemetryConstants.NonExistentPackageHash,
                         metric: 1,
-                        properties: GetProperties(packageId, packageVersion, blob));
+                        properties: GetProperties(packageId, packageNormalizedVersion, blob));
                 }
             }
 
@@ -243,16 +243,16 @@ private static Dictionary<string, string> GetProperties(FeedPackageDetails packa
         {
             return GetProperties(
                 packageItem.PackageId.ToLowerInvariant(),
-                packageItem.PackageVersion.ToLowerInvariant(),
+                packageItem.PackageNormalizedVersion.ToLowerInvariant(),
                 blob);
         }
 
-        private static Dictionary<string, string> GetProperties(string packageId, string packageVersion, ICloudBlockBlob blob)
+        private static Dictionary<string, string> GetProperties(string packageId, string packageNormalizedVersion, ICloudBlockBlob blob)
         {
             var properties = new Dictionary<string, string>()
             {
                 { TelemetryConstants.Id, packageId },
-                { TelemetryConstants.Version, packageVersion }
+                { TelemetryConstants.Version, packageNormalizedVersion }
             };
 
             if (blob != null)

src/Ng/Db2Monitoring.nuspec

@@ -0,0 +1,16 @@
+<?xml version="1.0"?>
+<package >
+  <metadata>
+    <id>Db2Monitoring</id>
+    <version>$version$</version>
+    <authors>.NET Foundation</authors>
+    <owners>.NET Foundation</owners>
+    <description>The Db2Monitoring job.</description>
+    <copyright>Copyright .NET Foundation</copyright>
+  </metadata>
+  <files>
+    <file src="bin\$configuration$\*.*" target="Ng"/>
+    <file src="Scripts\*.ps1" />
+    <file src="Scripts\nssm.exe" />
+  </files>
+</package>

src/Ng/Jobs/Catalog2MonitoringJob.cs

@@ -26,25 +26,20 @@ public Catalog2MonitoringJob(ITelemetryService telemetryService, ILoggerFactory
 
         protected override void Init(IDictionary<string, string> arguments, CancellationToken cancellationToken)
         {
-            var index = arguments.GetOrThrow<string>(Arguments.Index);
             var source = arguments.GetOrThrow<string>(Arguments.Source);
             var verbose = arguments.GetOrDefault(Arguments.Verbose, false);
 
             CommandHelpers.AssertAzureStorage(arguments);
 
             var monitoringStorageFactory = CommandHelpers.CreateStorageFactory(arguments, verbose);
-
             var endpointConfiguration = CommandHelpers.GetEndpointConfiguration(arguments);
-
             var messageHandlerFactory = CommandHelpers.GetHttpMessageHandlerFactory(TelemetryService, verbose);
-
             var statusService = CommandHelpers.GetPackageMonitoringStatusService(arguments, monitoringStorageFactory, LoggerFactory);
-
             var queue = CommandHelpers.CreateStorageQueue<PackageValidatorContext>(arguments, PackageValidatorContext.Version);
 
             Logger.LogInformation(
-                "CONFIG index: {Index} storage: {Storage} registration cursor uri: {RegistrationCursorUri} flat-container cursor uri: {FlatContainerCursorUri}",
-                index, monitoringStorageFactory, endpointConfiguration.RegistrationCursorUri, endpointConfiguration.FlatContainerCursorUri);
+                "CONFIG storage: {Storage} registration cursor uri: {RegistrationCursorUri} flat-container cursor uri: {FlatContainerCursorUri}",
+                monitoringStorageFactory, endpointConfiguration.RegistrationCursorUri, endpointConfiguration.FlatContainerCursorUri);
 
             _enqueuer = ValidationFactory.CreatePackageValidatorContextEnqueuer(
                 queue,