diff --git a/src/PackageLagMonitor/AzureSearchDiagnosticResponse.cs b/src/PackageLagMonitor/AzureSearchDiagnosticResponse.cs new file mode 100644 index 000000000..b3a0c6d73 --- /dev/null +++ b/src/PackageLagMonitor/AzureSearchDiagnosticResponse.cs @@ -0,0 +1,23 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; + +namespace NuGet.Jobs.Monitoring.PackageLag +{ + public class AzureSearchDiagnosticResponse + { + public IndexInformation SearchIndex { get; set; } + } + + public class IndexInformation + { + public string Name { get; set; } + + public long DocumentCount { get; set; } + + public DateTimeOffset LastCommitTimestamp { get; set; } + + public TimeSpan LastCommitTimestampDuration { get; set; } + } +} diff --git a/src/PackageLagMonitor/Instance.cs b/src/PackageLagMonitor/Instance.cs index de1d7608b..678ab78ec 100644 --- a/src/PackageLagMonitor/Instance.cs +++ b/src/PackageLagMonitor/Instance.cs @@ -8,13 +8,14 @@ namespace NuGet.Jobs.Monitoring.PackageLag { public class Instance { - public Instance(string slot, int index, string diagUrl, string baseQueryUrl, string region) + public Instance(string slot, int index, string diagUrl, string baseQueryUrl, string region, ServiceType serviceType) { Slot = slot ?? throw new ArgumentNullException(nameof(slot)); Index = index; DiagUrl = diagUrl ?? throw new ArgumentNullException(nameof(diagUrl)); BaseQueryUrl = baseQueryUrl ?? throw new ArgumentNullException(nameof(baseQueryUrl)); Region = region ?? throw new ArgumentNullException(nameof(region)); + ServiceType = serviceType; } public string Slot { get; } @@ -22,5 +23,6 @@ public Instance(string slot, int index, string diagUrl, string baseQueryUrl, str public string DiagUrl { get; } public string BaseQueryUrl { get; } public string Region { get; } + public ServiceType ServiceType { get; } } } diff --git a/src/PackageLagMonitor/Monitoring.PackageLag.csproj b/src/PackageLagMonitor/Monitoring.PackageLag.csproj index 62471bf5b..6abd8ec97 100644 --- a/src/PackageLagMonitor/Monitoring.PackageLag.csproj +++ b/src/PackageLagMonitor/Monitoring.PackageLag.csproj @@ -48,6 +48,7 @@ + @@ -67,6 +68,7 @@ + diff --git a/src/PackageLagMonitor/PackageLagCatalogLeafProcessor.cs b/src/PackageLagMonitor/PackageLagCatalogLeafProcessor.cs index 89442c507..8b8de857f 100644 --- a/src/PackageLagMonitor/PackageLagCatalogLeafProcessor.cs +++ b/src/PackageLagMonitor/PackageLagCatalogLeafProcessor.cs @@ -4,11 +4,9 @@ using System; using System.Collections.Generic; using System.Linq; -using System.Net.Http; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; -using Newtonsoft.Json; using NuGet.Jobs.Monitoring.PackageLag.Telemetry; using NuGet.Protocol.Catalog; @@ -171,7 +169,7 @@ public Task ProcessPackageDetailsAsync(PackageDetailsCatalogLeaf leaf) if (shouldRetry) { ++retryCount; - _logger.LogInformation("Waiting for {RetryTime} seconds before retrying {PackageId} {PackageVersion} against {SearchBaseUrl}", WaitBetweenPolls.TotalSeconds, packageId, packageVersion, instance.BaseQueryUrl); + _logger.LogInformation("{ServiceType}: Waiting for {RetryTime} seconds before retrying {PackageId} {PackageVersion} against {SearchBaseUrl}", instance.ServiceType, WaitBetweenPolls.TotalSeconds, packageId, packageVersion, instance.BaseQueryUrl); await Task.Delay(WaitBetweenPolls); } } while (shouldRetry && retryCount < RetryLimit); @@ -187,8 +185,8 @@ public Task ProcessPackageDetailsAsync(PackageDetailsCatalogLeaf leaf) var timeStamp = (isListOperation ? lastEdited : created); // We log both of these values here as they will differ if a package went through validation pipline. - _logger.LogInformation("Lag {Timestamp}:{PackageId} {PackageVersion} SearchInstance:{Region}{Instance} Created: {CreatedLag} V3: {V3Lag}", timeStamp, packageId, packageVersion, instance.Region, instance.Index, createdDelay, v3Delay); - _logger.LogInformation("LastReload:{LastReloadTimestamp} LastEdited:{LastEditedTimestamp} Created:{CreatedTimestamp} ", lastReloadTime, lastEdited, created); + _logger.LogInformation("{ServiceType}: Lag {Timestamp}:{PackageId} {PackageVersion} SearchInstance:{Region}{Instance} Created: {CreatedLag} V3: {V3Lag}", instance.ServiceType, timeStamp, packageId, packageVersion, instance.Region, instance.Index, createdDelay, v3Delay); + _logger.LogInformation("{ServiceType}: LastReload:{LastReloadTimestamp} LastEdited:{LastEditedTimestamp} Created:{CreatedTimestamp} ", instance.ServiceType, lastReloadTime, lastEdited, created); if (!isListOperation) { _telemetryService.TrackPackageCreationLag(timeStamp, instance, packageId, packageVersion, createdDelay); @@ -204,12 +202,12 @@ public Task ProcessPackageDetailsAsync(PackageDetailsCatalogLeaf leaf) } else { - _logger.LogInformation("Lag check for {PackageId} {PackageVersion} was abandoned. Retry limit reached.", packageId, packageVersion); + _logger.LogInformation("{ServiceType}: Lag check for {PackageId} {PackageVersion} was abandoned. Retry limit reached.", instance.ServiceType, packageId, packageVersion); } } catch (Exception e) { - _logger.LogError("Failed to compute lag for {PackageId} {PackageVersion}. {Exception}", packageId, packageVersion, e); + _logger.LogError("{ServiceType}: Failed to compute lag for {PackageId} {PackageVersion}. {Exception}", instance.ServiceType, packageId, packageVersion, e); } return null; diff --git a/src/PackageLagMonitor/RegionInformation.cs b/src/PackageLagMonitor/RegionInformation.cs index 5c07f6b91..790f0b13e 100644 --- a/src/PackageLagMonitor/RegionInformation.cs +++ b/src/PackageLagMonitor/RegionInformation.cs @@ -1,7 +1,6 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - namespace NuGet.Jobs.Monitoring.PackageLag { public class RegionInformation @@ -11,5 +10,15 @@ public class RegionInformation public string ServiceName { get; set; } public string Region { get; set; } + + /// + /// Base url to use for queries. Used for AzureSearch case. + /// + public string BaseUrl { get; set; } + + /// + /// One of LuceneSearch or AzureSearch depending on what type of search service this information defines. + /// + public ServiceType ServiceType { get; set; } } } diff --git a/src/PackageLagMonitor/SearchDiagnosticResponse.cs b/src/PackageLagMonitor/SearchDiagnosticResponse.cs index f5bba44e3..384211191 100644 --- a/src/PackageLagMonitor/SearchDiagnosticResponse.cs +++ b/src/PackageLagMonitor/SearchDiagnosticResponse.cs @@ -7,19 +7,12 @@ namespace NuGet.Jobs.Monitoring.PackageLag { public class SearchDiagnosticResponse { - public long NumDocs { get; set; } - public string IndexName { get; set; } - public long LastIndexReloadDurationInMilliseconds { get; set; } public DateTimeOffset LastIndexReloadTime { get; set; } - public DateTimeOffset LastReopen { get; set; } public CommitUserData CommitUserData { get; set; } } public class CommitUserData { public string CommitTimeStamp { get; set; } - public string Description { get; set; } - public string Count { get; set; } - public string Trace { get; set; } } } diff --git a/src/PackageLagMonitor/SearchServiceClient.cs b/src/PackageLagMonitor/SearchServiceClient.cs index 0685e7655..a7597f182 100644 --- a/src/PackageLagMonitor/SearchServiceClient.cs +++ b/src/PackageLagMonitor/SearchServiceClient.cs @@ -86,7 +86,17 @@ public async Task GetSearchDiagnosticResponseAsync( var diagContent = diagResponse.Content; var searchDiagResultRaw = await diagContent.ReadAsStringAsync(); - var response = JsonConvert.DeserializeObject(searchDiagResultRaw); + SearchDiagnosticResponse response = null; + switch (instance.ServiceType) + { + case ServiceType.LuceneSearch: + response = JsonConvert.DeserializeObject(searchDiagResultRaw); + break; + case ServiceType.AzureSearch: + var tempResponse = JsonConvert.DeserializeObject(searchDiagResultRaw); + response = ConvertAzureSearchResponse(tempResponse); + break; + } return response; } @@ -107,18 +117,26 @@ public async Task> GetSearchEndpointsAsync( RegionInformation regionInformation, CancellationToken token) { - var result = await _azureManagementApiWrapper.GetCloudServicePropertiesAsync( - _configuration.Value.Subscription, - regionInformation.ResourceGroup, - regionInformation.ServiceName, - ProductionSlot, - token); + switch (regionInformation.ServiceType) + { + case ServiceType.LuceneSearch: + var result = await _azureManagementApiWrapper.GetCloudServicePropertiesAsync( + _configuration.Value.Subscription, + regionInformation.ResourceGroup, + regionInformation.ServiceName, + ProductionSlot, + token); - var cloudService = AzureHelper.ParseCloudServiceProperties(result); + var cloudService = AzureHelper.ParseCloudServiceProperties(result); - var instances = GetInstances(cloudService.Uri, cloudService.InstanceCount, regionInformation); + var instances = GetInstances(cloudService.Uri, cloudService.InstanceCount, regionInformation, ServiceType.LuceneSearch); - return instances; + return instances; + case ServiceType.AzureSearch: + return GetInstances(new Uri(regionInformation.BaseUrl), instanceCount: 1, regionInformation: regionInformation, serviceType: ServiceType.AzureSearch); + default: + throw new NotImplementedException($"Unknown ServiceType: {regionInformation.ServiceType}"); + } } public async Task GetSearchResultAsync(Instance instance, string query, CancellationToken token) @@ -160,15 +178,26 @@ public async Task GetSearchResultAsync(Instance instance, throw new NotImplementedException(); } - private List GetInstances(Uri endpointUri, int instanceCount, RegionInformation regionInformation) + private List GetInstances(Uri endpointUri, int instanceCount, RegionInformation regionInformation, ServiceType serviceType) { var instancePortMinimum = _configuration.Value.InstancePortMinimum; - - _logger.LogInformation( - "Testing {InstanceCount} instances, starting at port {InstancePortMinimum} for region {Region}.", - instanceCount, - instancePortMinimum, - regionInformation.Region); + switch (serviceType) + { + case ServiceType.LuceneSearch: + _logger.LogInformation( + "{ServiceType}: Testing {InstanceCount} instances, starting at port {InstancePortMinimum} for region {Region}.", + ServiceType.LuceneSearch, + instanceCount, + instancePortMinimum, + regionInformation.Region); + break; + case ServiceType.AzureSearch: + _logger.LogInformation( + "{ServiceType}: Testing for region {Region}.", + ServiceType.AzureSearch, + regionInformation.Region); + break; + } return Enumerable .Range(0, instanceCount) @@ -177,13 +206,19 @@ private List GetInstances(Uri endpointUri, int instanceCount, RegionIn var diagUriBuilder = new UriBuilder(endpointUri); diagUriBuilder.Scheme = "https"; - diagUriBuilder.Port = instancePortMinimum + i; + if (serviceType == ServiceType.LuceneSearch) + { + diagUriBuilder.Port = instancePortMinimum + i; + } diagUriBuilder.Path = "search/diag"; var queryBaseUriBuilder = new UriBuilder(endpointUri); queryBaseUriBuilder.Scheme = "https"; - queryBaseUriBuilder.Port = instancePortMinimum + i; + if (serviceType == ServiceType.LuceneSearch) + { + queryBaseUriBuilder.Port = instancePortMinimum + i; + } queryBaseUriBuilder.Path = "search/query"; return new Instance( @@ -191,9 +226,26 @@ private List GetInstances(Uri endpointUri, int instanceCount, RegionIn i, diagUriBuilder.Uri.ToString(), queryBaseUriBuilder.Uri.ToString(), - regionInformation.Region); + regionInformation.Region, + serviceType); }) .ToList(); } + + private SearchDiagnosticResponse ConvertAzureSearchResponse(AzureSearchDiagnosticResponse azureSearchDiagnosticResponse) + { + var result = new SearchDiagnosticResponse + { + // We will use UtcNow here since AzureSearch diagnostic endpoint doesn't currently have last reloaded information. + // See https://github.com/NuGet/Engineering/issues/2651 for more information + LastIndexReloadTime = DateTimeOffset.UtcNow, + CommitUserData = new CommitUserData + { + CommitTimeStamp = azureSearchDiagnosticResponse.SearchIndex.LastCommitTimestamp.ToString() + } + }; + + return result; + } } } diff --git a/src/PackageLagMonitor/ServiceType.cs b/src/PackageLagMonitor/ServiceType.cs new file mode 100644 index 000000000..94e8ef2f9 --- /dev/null +++ b/src/PackageLagMonitor/ServiceType.cs @@ -0,0 +1,11 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +namespace NuGet.Jobs.Monitoring.PackageLag +{ + public enum ServiceType + { + LuceneSearch, + AzureSearch + } +} diff --git a/src/PackageLagMonitor/Telemetry/PackageLagTelemetryService.cs b/src/PackageLagMonitor/Telemetry/PackageLagTelemetryService.cs index 4386ff4de..81c550d8c 100644 --- a/src/PackageLagMonitor/Telemetry/PackageLagTelemetryService.cs +++ b/src/PackageLagMonitor/Telemetry/PackageLagTelemetryService.cs @@ -16,6 +16,7 @@ public class PackageLagTelemetryService : IPackageLagTelemetryService private const string Region = "Region"; private const string Subscription = "Subscription"; private const string InstanceIndex = "InstanceIndex"; + private const string ServiceType = "ServiceType"; private const string CreatedLagName = "PackageCreationLagInSeconds"; private const string V3LagName = "V3LagInSeconds"; @@ -36,7 +37,8 @@ public void TrackPackageCreationLag(DateTimeOffset eventTime, Instance instance, { PackageVersion, packageVersion }, { Region, instance.Region }, { Subscription, _subscription }, - { InstanceIndex, instance.Index.ToString() } + { InstanceIndex, instance.Index.ToString() }, + { ServiceType, instance.ServiceType.ToString() } }); } @@ -48,7 +50,8 @@ public void TrackV3Lag(DateTimeOffset eventTime, Instance instance, string packa { PackageVersion, packageVersion }, { Region, instance.Region }, { Subscription, _subscription }, - { InstanceIndex, instance.Index.ToString() } + { InstanceIndex, instance.Index.ToString() }, + { ServiceType, instance.ServiceType.ToString() } }); } } diff --git a/tests/Monitoring.PackageLag.Tests/Monitoring.PackageLag.Tests.csproj b/tests/Monitoring.PackageLag.Tests/Monitoring.PackageLag.Tests.csproj index dfeae0603..2f5242bb1 100644 --- a/tests/Monitoring.PackageLag.Tests/Monitoring.PackageLag.Tests.csproj +++ b/tests/Monitoring.PackageLag.Tests/Monitoring.PackageLag.Tests.csproj @@ -42,6 +42,7 @@ + diff --git a/tests/Monitoring.PackageLag.Tests/PackageLagCatalogLeafProcessorFacts.cs b/tests/Monitoring.PackageLag.Tests/PackageLagCatalogLeafProcessorFacts.cs index a70bd6a84..b1d16d75b 100644 --- a/tests/Monitoring.PackageLag.Tests/PackageLagCatalogLeafProcessorFacts.cs +++ b/tests/Monitoring.PackageLag.Tests/PackageLagCatalogLeafProcessorFacts.cs @@ -79,13 +79,15 @@ public PackageLagCatalogLeafProcessorFacts(ITestOutputHelper output) 0, "http://localhost:801/search/diag", "http://localhost:801/query", - _region), + _region, + ServiceType.LuceneSearch), new Instance( _slot, 1, "http://localhost:802/search/diag", "http://localhost:802/query", - _region) + _region, + ServiceType.AzureSearch) }; _feedTimestamp = new DateTimeOffset(2018, 1, 1, 8, 0, 0, TimeSpan.Zero); diff --git a/tests/Monitoring.PackageLag.Tests/SearchServiceClientFacts.cs b/tests/Monitoring.PackageLag.Tests/SearchServiceClientFacts.cs new file mode 100644 index 000000000..cbb3f38a8 --- /dev/null +++ b/tests/Monitoring.PackageLag.Tests/SearchServiceClientFacts.cs @@ -0,0 +1,107 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Moq; +using Newtonsoft.Json; +using NuGet.Jobs.Monitoring.PackageLag; +using NuGet.Services.AzureManagement; +using System; +using System.Net; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using Xunit; + +namespace NuGet.Monitoring.PackageLag.Tests +{ + public class SearchServiceClientFacts + { + private Instance _luceneInstance; + private Instance _azureSearchInstance; + private IAzureManagementAPIWrapper _azureApiWrapper; + private IOptionsSnapshot _options; + private ILogger _logger; + + public SearchServiceClientFacts() + { + _luceneInstance = new Instance("production", 0, "Lucene-DiagUrl", "Lucene-BaseQueryUrl", "USNC", ServiceType.LuceneSearch); + _azureSearchInstance = new Instance("production", 0, "Azure-DiagUrl", "Azure-BaseQueryUrl", "USNC", ServiceType.AzureSearch); + + + var azureApiMock = new Mock(); + var configMock = new Mock>(); + var loggerMock = new Mock>(); + + configMock.Setup(cm => cm.Value) + .Returns(new SearchServiceConfiguration + { + InstancePortMinimum = 100 + }); + + _azureApiWrapper = azureApiMock.Object; + _options = configMock.Object; + _logger = loggerMock.Object; + } + + [Fact] + public async Task CommitTimeStampDataIsCorrect() + { + // Arrange + var token = new CancellationToken(); + var httpClientMock = new Mock(); + var luceneExpectedTicks = 5; + httpClientMock.Setup(hcm => hcm.GetAsync(It.Is(it => it.Equals("Lucene-DiagUrl")), HttpCompletionOption.ResponseContentRead, It.IsAny())) + .Returns(Task.FromResult((IHttpResponseMessageWrapper)new TestHttpResponseMessage(HttpStatusCode.OK, JsonConvert.SerializeObject(new SearchDiagnosticResponse + { + CommitUserData = new CommitUserData + { + CommitTimeStamp = new DateTimeOffset(luceneExpectedTicks, new TimeSpan(0)).ToString() + }, + LastIndexReloadTime = new DateTimeOffset(luceneExpectedTicks, new TimeSpan(0)) + })))); + httpClientMock.Setup(hcm => hcm.GetAsync(It.Is(it => it.Equals("Azure-DiagUrl")), HttpCompletionOption.ResponseContentRead, It.IsAny())) + .Returns(Task.FromResult((IHttpResponseMessageWrapper)new TestHttpResponseMessage(HttpStatusCode.OK, JsonConvert.SerializeObject(new AzureSearchDiagnosticResponse + { + SearchIndex = new IndexInformation + { + LastCommitTimestamp = new DateTimeOffset() + } + })))); + var searchClient = new SearchServiceClient(_azureApiWrapper, httpClientMock.Object, _options, _logger); + + var azureStartTimestamp = DateTime.UtcNow; + + var luceneResponse = await searchClient.GetIndexLastReloadTimeAsync(_luceneInstance, token); + var azureResponse = await searchClient.GetIndexLastReloadTimeAsync(_azureSearchInstance, token); + + var azureStopTimestamp = DateTime.UtcNow; + + Assert.InRange(azureResponse, azureStartTimestamp, azureStopTimestamp); + Assert.Equal(luceneResponse, new DateTimeOffset(luceneExpectedTicks, new TimeSpan(0))); + } + + private class TestHttpResponseMessage : IHttpResponseMessageWrapper + { + public TestHttpResponseMessage(HttpStatusCode response, string content) + { + StatusCode = response; + var contentMock = new Mock(); + contentMock.Setup(cm => cm.ReadAsStringAsync()) + .Returns(Task.FromResult(content)); + + Content = contentMock.Object; + } + + public bool IsSuccessStatusCode { get { return StatusCode.Equals(HttpStatusCode.OK); } set { } } + + public string ReasonPhrase { get; set; } + public HttpStatusCode StatusCode { get; set; } + + public IHttpContentWrapper Content { get; set; } + + public void Dispose() { } + } + } +} diff --git a/tests/Monitoring.RebootSearchInstance.Tests/SearchInstanceRebooterFacts.cs b/tests/Monitoring.RebootSearchInstance.Tests/SearchInstanceRebooterFacts.cs index 3e668075f..158ba1f0d 100644 --- a/tests/Monitoring.RebootSearchInstance.Tests/SearchInstanceRebooterFacts.cs +++ b/tests/Monitoring.RebootSearchInstance.Tests/SearchInstanceRebooterFacts.cs @@ -77,19 +77,22 @@ public SearchInstanceRebooterFacts(ITestOutputHelper output) 0, "http://localhost:801/search/diag", "http://localhost:801/query", - _region), + _region, + ServiceType.LuceneSearch), new Instance( _slot, 1, "http://localhost:802/search/diag", "http://localhost:802/query", - _region), + _region, + ServiceType.LuceneSearch), new Instance( _slot, 2, "http://localhost:803/search/diag", "http://localhost:803/query", - _region), + _region, + ServiceType.LuceneSearch), }; _feedTimestamp = new DateTimeOffset(2018, 1, 1, 8, 0, 0, TimeSpan.Zero);