Update PackageLagMonitor for AzureSearch (#798)

* Update instance generation to understand azureSearch type. Pipe serviceType through.
* Support AzureSearchDiagResponse. Pipe ServiceType through to telemetry.
* Fix tests.
* Fixed committimestamp to be pessimistic.
* Add test for commit time stamp. PR feedback.
* Async test.

src/PackageLagMonitor/AzureSearchDiagnosticResponse.cs

@@ -0,0 +1,23 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+
+namespace NuGet.Jobs.Monitoring.PackageLag
+{
+    public class AzureSearchDiagnosticResponse
+    {
+        public IndexInformation SearchIndex { get; set; }
+    }
+
+    public class IndexInformation
+    {
+        public string Name { get; set; }
+
+        public long DocumentCount { get; set; }
+
+        public DateTimeOffset LastCommitTimestamp { get; set; }
+
+        public TimeSpan LastCommitTimestampDuration { get; set; }
+     }
+}

src/PackageLagMonitor/Instance.cs

@@ -8,19 +8,21 @@ namespace NuGet.Jobs.Monitoring.PackageLag
 {
     public class Instance
     {
-        public Instance(string slot, int index, string diagUrl, string baseQueryUrl, string region)
+        public Instance(string slot, int index, string diagUrl, string baseQueryUrl, string region, ServiceType serviceType)
         {
             Slot = slot ?? throw new ArgumentNullException(nameof(slot));
             Index = index;
             DiagUrl = diagUrl ?? throw new ArgumentNullException(nameof(diagUrl));
             BaseQueryUrl = baseQueryUrl ?? throw new ArgumentNullException(nameof(baseQueryUrl));
             Region = region ?? throw new ArgumentNullException(nameof(region));
+            ServiceType = serviceType;
         }
 
         public string Slot { get; }
         public int Index { get; }
         public string DiagUrl { get; }
         public string BaseQueryUrl { get; }
         public string Region { get; }
+        public ServiceType ServiceType { get; }
     }
 }

src/PackageLagMonitor/Monitoring.PackageLag.csproj

@@ -48,6 +48,7 @@
   </PropertyGroup>
   <ItemGroup>
     <Compile Include="AzureManagementAPIWrapperConfiguration.cs" />
+    <Compile Include="AzureSearchDiagnosticResponse.cs" />
     <Compile Include="HttpWrappers\HttpClientWrapper.cs" />
     <Compile Include="HttpResponseException.cs" />
     <Compile Include="HttpWrappers\IHttpClientWrapper.cs" />
@@ -67,6 +68,7 @@
     <Compile Include="SearchResultResponse.cs" />
     <Compile Include="SearchServiceClient.cs" />
     <Compile Include="SearchServiceConfiguration.cs" />
+    <Compile Include="ServiceType.cs" />
     <Compile Include="Telemetry\IPackageLagTelemetryService.cs" />
     <Compile Include="Telemetry\PackageLagTelemetryService.cs" />
   </ItemGroup>

src/PackageLagMonitor/PackageLagCatalogLeafProcessor.cs

@@ -4,11 +4,9 @@
 using System;
 using System.Collections.Generic;
 using System.Linq;
-using System.Net.Http;
 using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.Extensions.Logging;
-using Newtonsoft.Json;
 using NuGet.Jobs.Monitoring.PackageLag.Telemetry;
 using NuGet.Protocol.Catalog;
 
@@ -171,7 +169,7 @@ public Task<bool> ProcessPackageDetailsAsync(PackageDetailsCatalogLeaf leaf)
                     if (shouldRetry)
                     {
                         ++retryCount;
-                        _logger.LogInformation("Waiting for {RetryTime} seconds before retrying {PackageId} {PackageVersion} against {SearchBaseUrl}", WaitBetweenPolls.TotalSeconds, packageId, packageVersion, instance.BaseQueryUrl);
+                        _logger.LogInformation("{ServiceType}: Waiting for {RetryTime} seconds before retrying {PackageId} {PackageVersion} against {SearchBaseUrl}", instance.ServiceType, WaitBetweenPolls.TotalSeconds, packageId, packageVersion, instance.BaseQueryUrl);
                         await Task.Delay(WaitBetweenPolls);
                     }
                 } while (shouldRetry && retryCount < RetryLimit);
@@ -187,8 +185,8 @@ public Task<bool> ProcessPackageDetailsAsync(PackageDetailsCatalogLeaf leaf)
                     var timeStamp = (isListOperation ? lastEdited : created);
 
                     // We log both of these values here as they will differ if a package went through validation pipline.
-                    _logger.LogInformation("Lag {Timestamp}:{PackageId} {PackageVersion} SearchInstance:{Region}{Instance} Created: {CreatedLag} V3: {V3Lag}", timeStamp, packageId, packageVersion, instance.Region, instance.Index, createdDelay, v3Delay);
-                    _logger.LogInformation("LastReload:{LastReloadTimestamp} LastEdited:{LastEditedTimestamp} Created:{CreatedTimestamp} ", lastReloadTime, lastEdited, created);
+                    _logger.LogInformation("{ServiceType}: Lag {Timestamp}:{PackageId} {PackageVersion} SearchInstance:{Region}{Instance} Created: {CreatedLag} V3: {V3Lag}", instance.ServiceType, timeStamp, packageId, packageVersion, instance.Region, instance.Index, createdDelay, v3Delay);
+                    _logger.LogInformation("{ServiceType}: LastReload:{LastReloadTimestamp} LastEdited:{LastEditedTimestamp} Created:{CreatedTimestamp} ", instance.ServiceType, lastReloadTime, lastEdited, created);
                     if (!isListOperation)
                     {
                         _telemetryService.TrackPackageCreationLag(timeStamp, instance, packageId, packageVersion, createdDelay);
@@ -204,12 +202,12 @@ public Task<bool> ProcessPackageDetailsAsync(PackageDetailsCatalogLeaf leaf)
                 }
                 else
                 {
-                    _logger.LogInformation("Lag check for {PackageId} {PackageVersion} was abandoned. Retry limit reached.", packageId, packageVersion);
+                    _logger.LogInformation("{ServiceType}: Lag check for {PackageId} {PackageVersion} was abandoned. Retry limit reached.", instance.ServiceType, packageId, packageVersion);
                 }
             }
             catch (Exception e)
             {
-                _logger.LogError("Failed to compute lag for {PackageId} {PackageVersion}. {Exception}", packageId, packageVersion, e);
+                _logger.LogError("{ServiceType}: Failed to compute lag for {PackageId} {PackageVersion}. {Exception}", instance.ServiceType, packageId, packageVersion, e);
             }
 
             return null;

src/PackageLagMonitor/RegionInformation.cs

@@ -1,7 +1,6 @@
 // Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
-
 namespace NuGet.Jobs.Monitoring.PackageLag
 {
     public class RegionInformation
@@ -11,5 +10,15 @@ public class RegionInformation
         public string ServiceName { get; set; }
 
         public string Region { get; set; }
+
+        /// <summary>
+        /// Base url to use for queries. Used for AzureSearch case.
+        /// </summary>
+        public string BaseUrl { get; set; }
+
+        /// <summary>
+        /// One of LuceneSearch or AzureSearch depending on what type of search service this information defines.
+        /// </summary>
+        public ServiceType ServiceType { get; set; }
     }
 }

src/PackageLagMonitor/SearchDiagnosticResponse.cs

@@ -7,19 +7,12 @@ namespace NuGet.Jobs.Monitoring.PackageLag
 {
     public class SearchDiagnosticResponse
     {
-        public long NumDocs { get; set; }
-        public string IndexName { get; set; }
-        public long LastIndexReloadDurationInMilliseconds { get; set; }
         public DateTimeOffset LastIndexReloadTime { get; set; }
-        public DateTimeOffset LastReopen { get; set; }
         public CommitUserData CommitUserData { get; set; }
     }
 
     public class CommitUserData
     {
         public string CommitTimeStamp { get; set; }
-        public string Description { get; set; }
-        public string Count { get; set; }
-        public string Trace { get; set; }
     }
 }

src/PackageLagMonitor/SearchServiceClient.cs

@@ -86,7 +86,17 @@ public async Task<SearchDiagnosticResponse> GetSearchDiagnosticResponseAsync(
 
                     var diagContent = diagResponse.Content;
                     var searchDiagResultRaw = await diagContent.ReadAsStringAsync();
-                    var response = JsonConvert.DeserializeObject<SearchDiagnosticResponse>(searchDiagResultRaw);
+                    SearchDiagnosticResponse response = null;
+                    switch (instance.ServiceType)
+                    {
+                        case ServiceType.LuceneSearch:
+                            response = JsonConvert.DeserializeObject<SearchDiagnosticResponse>(searchDiagResultRaw);
+                            break;
+                        case ServiceType.AzureSearch:
+                            var tempResponse = JsonConvert.DeserializeObject<AzureSearchDiagnosticResponse>(searchDiagResultRaw);
+                            response = ConvertAzureSearchResponse(tempResponse);
+                            break;
+                    }
 
                     return response;
                 }
@@ -107,18 +117,26 @@ public async Task<IReadOnlyList<Instance>> GetSearchEndpointsAsync(
             RegionInformation regionInformation,
             CancellationToken token)
         {
-            var result = await _azureManagementApiWrapper.GetCloudServicePropertiesAsync(
-                _configuration.Value.Subscription,
-                regionInformation.ResourceGroup,
-                regionInformation.ServiceName,
-                ProductionSlot,
-                token);
+            switch (regionInformation.ServiceType)
+            {
+                case ServiceType.LuceneSearch:
+                    var result = await _azureManagementApiWrapper.GetCloudServicePropertiesAsync(
+                        _configuration.Value.Subscription,
+                        regionInformation.ResourceGroup,
+                        regionInformation.ServiceName,
+                        ProductionSlot,
+                        token);
 
-            var cloudService = AzureHelper.ParseCloudServiceProperties(result);
+                    var cloudService = AzureHelper.ParseCloudServiceProperties(result);
 
-            var instances = GetInstances(cloudService.Uri, cloudService.InstanceCount, regionInformation);
+                    var instances = GetInstances(cloudService.Uri, cloudService.InstanceCount, regionInformation, ServiceType.LuceneSearch);
 
-            return instances;
+                    return instances;
+                case ServiceType.AzureSearch:
+                    return GetInstances(new Uri(regionInformation.BaseUrl), instanceCount: 1, regionInformation: regionInformation, serviceType: ServiceType.AzureSearch);
+                default:
+                    throw new NotImplementedException($"Unknown ServiceType: {regionInformation.ServiceType}");
+            }
         }
 
         public async Task<SearchResultResponse> GetSearchResultAsync(Instance instance, string query, CancellationToken token)
@@ -160,15 +178,26 @@ public async Task<SearchResultResponse> GetSearchResultAsync(Instance instance,
             throw new NotImplementedException();
         }
 
-        private List<Instance> GetInstances(Uri endpointUri, int instanceCount, RegionInformation regionInformation)
+        private List<Instance> GetInstances(Uri endpointUri, int instanceCount, RegionInformation regionInformation, ServiceType serviceType)
         {
             var instancePortMinimum = _configuration.Value.InstancePortMinimum;
-
-            _logger.LogInformation(
-                "Testing {InstanceCount} instances, starting at port {InstancePortMinimum} for region {Region}.",
-                instanceCount,
-                instancePortMinimum,
-                regionInformation.Region);
+            switch (serviceType)
+            {
+                case ServiceType.LuceneSearch:
+                    _logger.LogInformation(
+                        "{ServiceType}: Testing {InstanceCount} instances, starting at port {InstancePortMinimum} for region {Region}.",
+                        ServiceType.LuceneSearch,
+                        instanceCount,
+                        instancePortMinimum,
+                        regionInformation.Region);
+                    break;
+                case ServiceType.AzureSearch:
+                    _logger.LogInformation(
+                        "{ServiceType}: Testing for region {Region}.",
+                        ServiceType.AzureSearch,
+                        regionInformation.Region);
+                    break;
+            }
 
             return Enumerable
                 .Range(0, instanceCount)
@@ -177,23 +206,46 @@ private List<Instance> GetInstances(Uri endpointUri, int instanceCount, RegionIn
                     var diagUriBuilder = new UriBuilder(endpointUri);
 
                     diagUriBuilder.Scheme = "https";
-                    diagUriBuilder.Port = instancePortMinimum + i;
+                    if (serviceType == ServiceType.LuceneSearch)
+                    {
+                        diagUriBuilder.Port = instancePortMinimum + i;
+                    }
                     diagUriBuilder.Path = "search/diag";
 
                     var queryBaseUriBuilder = new UriBuilder(endpointUri);
 
                     queryBaseUriBuilder.Scheme = "https";
-                    queryBaseUriBuilder.Port = instancePortMinimum + i;
+                    if (serviceType == ServiceType.LuceneSearch)
+                    {
+                        queryBaseUriBuilder.Port = instancePortMinimum + i;
+                    }
                     queryBaseUriBuilder.Path = "search/query";
 
                     return new Instance(
                         ProductionSlot,
                         i,
                         diagUriBuilder.Uri.ToString(),
                         queryBaseUriBuilder.Uri.ToString(),
-                        regionInformation.Region);
+                        regionInformation.Region,
+                        serviceType);
                 })
                 .ToList();
         }
+
+        private SearchDiagnosticResponse ConvertAzureSearchResponse(AzureSearchDiagnosticResponse azureSearchDiagnosticResponse)
+        {
+            var result = new SearchDiagnosticResponse
+            {
+                // We will use UtcNow here since AzureSearch diagnostic endpoint doesn't currently have last reloaded information.
+                // See https://github.com/NuGet/Engineering/issues/2651 for more information
+                LastIndexReloadTime = DateTimeOffset.UtcNow,
+                CommitUserData = new CommitUserData
+                {
+                    CommitTimeStamp = azureSearchDiagnosticResponse.SearchIndex.LastCommitTimestamp.ToString()
+                }
+            };
+
+            return result;
+        }
     }
 }

src/PackageLagMonitor/ServiceType.cs

@@ -0,0 +1,11 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+namespace NuGet.Jobs.Monitoring.PackageLag
+{
+    public enum ServiceType
+    {
+        LuceneSearch,
+        AzureSearch
+    }
+}

src/PackageLagMonitor/Telemetry/PackageLagTelemetryService.cs

@@ -16,6 +16,7 @@ public class PackageLagTelemetryService : IPackageLagTelemetryService
         private const string Region = "Region";
         private const string Subscription = "Subscription";
         private const string InstanceIndex = "InstanceIndex";
+        private const string ServiceType = "ServiceType";
 
         private const string CreatedLagName = "PackageCreationLagInSeconds";
         private const string V3LagName = "V3LagInSeconds";
@@ -36,7 +37,8 @@ public void TrackPackageCreationLag(DateTimeOffset eventTime, Instance instance,
                 { PackageVersion, packageVersion },
                 { Region, instance.Region },
                 { Subscription, _subscription },
-                { InstanceIndex, instance.Index.ToString() }
+                { InstanceIndex, instance.Index.ToString() },
+                { ServiceType, instance.ServiceType.ToString() }
             });
         }
 
@@ -48,7 +50,8 @@ public void TrackV3Lag(DateTimeOffset eventTime, Instance instance, string packa
                 { PackageVersion, packageVersion },
                 { Region,  instance.Region },
                 { Subscription, _subscription },
-                { InstanceIndex, instance.Index.ToString() }
+                { InstanceIndex, instance.Index.ToString() },
+                { ServiceType, instance.ServiceType.ToString() }
             });
         }
     }

tests/Monitoring.PackageLag.Tests/Monitoring.PackageLag.Tests.csproj

@@ -42,6 +42,7 @@
   <ItemGroup>
     <Compile Include="Properties\AssemblyInfo.cs" />
     <Compile Include="PackageLagCatalogLeafProcessorFacts.cs" />
+    <Compile Include="SearchServiceClientFacts.cs" />
     <Compile Include="TestHelpers.cs" />
   </ItemGroup>
   <ItemGroup>

tests/Monitoring.PackageLag.Tests/PackageLagCatalogLeafProcessorFacts.cs

@@ -79,13 +79,15 @@ public PackageLagCatalogLeafProcessorFacts(ITestOutputHelper output)
                     0,
                     "http://localhost:801/search/diag",
                     "http://localhost:801/query",
-                    _region),
+                    _region,
+                    ServiceType.LuceneSearch),
                 new Instance(
                     _slot,
                     1,
                     "http://localhost:802/search/diag",
                     "http://localhost:802/query",
-                    _region)
+                    _region,
+                    ServiceType.AzureSearch)
             };
             _feedTimestamp = new DateTimeOffset(2018, 1, 1, 8, 0, 0, TimeSpan.Zero);