This repository has been archived by the owner on Jul 30, 2024. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 21
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
82 changed files
with
2,571 additions
and
137 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -20,6 +20,13 @@ NuGet.Jobs | |
| 7. Also, add settings.job file to mark the job as singleton, if the job will be run as a webjob, and it be a continuously running singleton | ||
| ## Specific instructions | ||
| This repository has a lot of jobs used for completely different purposes. For this reason, each should should have its | ||
| own documentation. This specific documentation is far from complete... but you have to start somewhere! | ||
| - [Monitoring.RebootSearchInstance](src/Monitoring.RebootSearchInstance/README.md) - check each region for stuck search instances and restart them | ||
| Open Source Code of Conduct | ||
| =================== | ||
| This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or contact [[email protected]](mailto:[email protected]) with any additional questions or comments. | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,6 @@ | ||
| <?xml version="1.0" encoding="utf-8" ?> | ||
| <configuration> | ||
| <startup> | ||
| <supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.6.2" /> | ||
| </startup> | ||
| </configuration> |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,73 @@ | ||
| // Copyright (c) .NET Foundation. All rights reserved. | ||
| // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. | ||
|
|
||
| using System; | ||
| using System.Collections.Generic; | ||
| using System.Globalization; | ||
| using System.Linq; | ||
| using System.Net.Http; | ||
| using System.Threading.Tasks; | ||
| using System.Xml.Linq; | ||
| using Autofac; | ||
| using Microsoft.Extensions.Logging; | ||
| using Microsoft.Extensions.Options; | ||
|
|
||
| namespace NuGet.Monitoring.RebootSearchInstance | ||
| { | ||
| public class FeedClient : IFeedClient | ||
| { | ||
| private const string FeedRelativeUrl = | ||
| "api/v2/Packages?" + | ||
| "$filter={0}%20ne%20null&" + | ||
| "$top=1&" + | ||
| "$orderby={0}%20desc&" + | ||
| "$select={0}"; | ||
|
|
||
| private static readonly IEnumerable<string> TimeStampProperties = new string[] | ||
| { | ||
| "Created", | ||
| "LastEdited", | ||
| }; | ||
|
|
||
| private readonly HttpClient _httpClient; | ||
| private readonly IOptionsSnapshot<MonitorConfiguration> _configuration; | ||
| private readonly ILogger<FeedClient> _logger; | ||
|
|
||
| public FeedClient(HttpClient httpClient, IOptionsSnapshot<MonitorConfiguration> configuration, ILogger<FeedClient> logger) | ||
| { | ||
| _httpClient = httpClient; | ||
| _configuration = configuration; | ||
| _logger = logger; | ||
| } | ||
|
|
||
| public async Task<DateTimeOffset> GetLatestFeedTimeStampAsync() | ||
| { | ||
| var tasks = TimeStampProperties | ||
| .Select(GetLatestFeedTimeStampAsync) | ||
| .ToList(); | ||
|
|
||
| await Task.WhenAll(tasks); | ||
|
|
||
| return tasks.Max(t => t.Result); | ||
| } | ||
|
|
||
| private async Task<DateTimeOffset> GetLatestFeedTimeStampAsync(string propertyName) | ||
| { | ||
| var galleryBaseUrl = _configuration.Value.FeedUrl; | ||
| var feedUrl = new Uri(galleryBaseUrl).GetLeftPart(UriPartial.Authority); | ||
| var url = $"{feedUrl}/{string.Format(FeedRelativeUrl, propertyName)}"; | ||
| using (var stream = await _httpClient.GetStreamAsync(url)) | ||
| { | ||
| var packagesResultObject = XDocument.Load(stream); | ||
| var propertyValue = DateTimeOffset.Parse( | ||
| packagesResultObject.Descendants() | ||
| .Where(a => a.Name.LocalName == propertyName) | ||
| .FirstOrDefault().Value, | ||
| formatProvider: null, | ||
| styles: DateTimeStyles.AssumeUniversal); | ||
| _logger.LogInformation("Feed at {GalleryBaseUrl} has {TimeStampPropertyName} cursor at {TimeStampPropertyValue}", galleryBaseUrl, propertyName, propertyValue); | ||
| return propertyValue; | ||
| } | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,13 @@ | ||
| // Copyright (c) .NET Foundation. All rights reserved. | ||
| // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. | ||
|
|
||
| using System; | ||
| using System.Threading.Tasks; | ||
|
|
||
| namespace NuGet.Monitoring.RebootSearchInstance | ||
| { | ||
| public interface IFeedClient | ||
| { | ||
| Task<DateTimeOffset> GetLatestFeedTimeStampAsync(); | ||
| } | ||
| } |
13 changes: 13 additions & 0 deletions
13
src/Monitoring.RebootSearchInstance/ISearchInstanceRebooter.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,13 @@ | ||
| // Copyright (c) .NET Foundation. All rights reserved. | ||
| // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. | ||
|
|
||
| using System.Threading; | ||
| using System.Threading.Tasks; | ||
|
|
||
| namespace NuGet.Monitoring.RebootSearchInstance | ||
| { | ||
| public interface ISearchInstanceRebooter | ||
| { | ||
| Task RunAsync(CancellationToken token); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,17 @@ | ||
| // Copyright (c) .NET Foundation. All rights reserved. | ||
| // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. | ||
|
|
||
| using System; | ||
|
|
||
| namespace NuGet.Monitoring.RebootSearchInstance | ||
| { | ||
| public interface ITelemetryService | ||
| { | ||
| void TrackHealthyInstanceCount(string region, int count); | ||
| void TrackInstanceCount(string region, int count); | ||
| void TrackInstanceReboot(string region, int index); | ||
| void TrackInstanceRebootDuration(string region, int index, TimeSpan duration, InstanceHealth health); | ||
| void TrackUnhealthyInstanceCount(string region, int count); | ||
| void TrackUnknownInstanceCount(string region, int count); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,12 @@ | ||
| // Copyright (c) .NET Foundation. All rights reserved. | ||
| // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. | ||
|
|
||
| namespace NuGet.Monitoring.RebootSearchInstance | ||
| { | ||
| public enum InstanceHealth | ||
| { | ||
| Healthy, | ||
| Unhealthy, | ||
| Unknown, | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,80 @@ | ||
| // Copyright (c) .NET Foundation. All rights reserved. | ||
| // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. | ||
|
|
||
| using System; | ||
| using System.Net; | ||
| using System.Net.Http; | ||
| using System.Reflection; | ||
| using System.Threading; | ||
| using System.Threading.Tasks; | ||
| using Autofac; | ||
| using Microsoft.Extensions.Configuration; | ||
| using Microsoft.Extensions.DependencyInjection; | ||
| using Microsoft.Extensions.Options; | ||
| using NuGet.Jobs.Montoring.PackageLag; | ||
| using NuGet.Jobs.Validation; | ||
| using NuGet.Services.AzureManagement; | ||
|
|
||
| namespace NuGet.Monitoring.RebootSearchInstance | ||
| { | ||
| public class Job : JsonConfigurationJob | ||
| { | ||
| private const string AzureManagementSectionName = "AzureManagement"; | ||
| private const string MonitorConfigurationSectionName = "MonitorConfiguration"; | ||
|
|
||
| public override async Task Run() | ||
| { | ||
| using (var scope = _serviceProvider.CreateScope()) | ||
| { | ||
| var rebooter = scope.ServiceProvider.GetRequiredService<ISearchInstanceRebooter>(); | ||
| await rebooter.RunAsync(CancellationToken.None); | ||
| } | ||
| } | ||
|
|
||
| protected override void ConfigureJobServices(IServiceCollection services, IConfigurationRoot configurationRoot) | ||
| { | ||
| services.Configure<MonitorConfiguration>(configurationRoot.GetSection(MonitorConfigurationSectionName)); | ||
| services.Configure<SearchServiceConfiguration>(configurationRoot.GetSection(MonitorConfigurationSectionName)); | ||
| services.Configure<AzureManagementAPIWrapperConfiguration>(configurationRoot.GetSection(AzureManagementSectionName)); | ||
|
|
||
| services.AddTransient<ITelemetryService, TelemetryService>(); | ||
| services.AddTransient<ISearchInstanceRebooter, SearchInstanceRebooter>(); | ||
| services.AddTransient<IFeedClient, FeedClient>(); | ||
| services.AddTransient<ISearchServiceClient, SearchServiceClient>(); | ||
| services.AddSingleton<IAzureManagementAPIWrapper, AzureManagementAPIWrapper>(); | ||
| services.AddTransient<IAzureManagementAPIWrapperConfiguration>(p => p.GetService<IOptionsSnapshot<AzureManagementAPIWrapperConfiguration>>().Value); | ||
|
|
||
| services.AddSingleton(p => | ||
| { | ||
| var assembly = Assembly.GetEntryAssembly(); | ||
| var assemblyName = assembly.GetName().Name; | ||
| var assemblyVersion = assembly.GetCustomAttribute<AssemblyInformationalVersionAttribute>()?.InformationalVersion ?? "0.0.0"; | ||
|
|
||
| var client = new HttpClient(new WebRequestHandler | ||
| { | ||
| AllowPipelining = true, | ||
| AutomaticDecompression = (DecompressionMethods.GZip | DecompressionMethods.Deflate), | ||
| ServerCertificateCustomValidationCallback = | ||
| (httpRequestMessage, cert, cetChain, policyErrors) => | ||
| { | ||
| if (policyErrors == System.Net.Security.SslPolicyErrors.RemoteCertificateNameMismatch || policyErrors == System.Net.Security.SslPolicyErrors.None) | ||
| { | ||
| return true; | ||
| } | ||
|
|
||
| return false; | ||
| }, | ||
| }); | ||
|
|
||
| client.DefaultRequestHeaders.Add("User-Agent", $"{assemblyName}/{assemblyVersion}"); | ||
| client.Timeout = TimeSpan.FromSeconds(10); | ||
|
|
||
| return client; | ||
| }); | ||
| } | ||
|
|
||
| protected override void ConfigureAutofacServices(ContainerBuilder containerBuilder) | ||
| { | ||
| } | ||
| } | ||
| } |
47 changes: 47 additions & 0 deletions
47
src/Monitoring.RebootSearchInstance/MonitorConfiguration.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,47 @@ | ||
| // Copyright (c) .NET Foundation. All rights reserved. | ||
| // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. | ||
|
|
||
| using System; | ||
| using NuGet.Jobs.Montoring.PackageLag; | ||
|
|
||
| namespace NuGet.Monitoring.RebootSearchInstance | ||
| { | ||
| public class MonitorConfiguration : SearchServiceConfiguration | ||
| { | ||
| public string FeedUrl { get; set; } | ||
| public string Role { get; set; } | ||
| public string RoleInstanceFormat { get; set; } | ||
|
|
||
| /// <summary> | ||
| /// This configuration is in seconds to match the monitor configuration. If the search instance lag is less | ||
| /// than this duration, that instance is considered healthy. | ||
| /// </summary> | ||
| public int HealthyThresholdInSeconds { get; set; } | ||
|
|
||
| /// <summary> | ||
| /// This configuration is in seconds to match the monitor configuration. If the search instance lag is more | ||
| /// than this duration, that instance is considered unhealthy. | ||
| /// </summary> | ||
| public int UnhealthyThresholdInSeconds { get; set; } | ||
|
|
||
| /// <summary> | ||
| /// The time to wait for a restarted instance to become healthy before moving on. | ||
| /// </summary> | ||
| public TimeSpan WaitForHealthyDuration { get; set; } | ||
|
|
||
| /// <summary> | ||
| /// The time to wait before checking a region for unhealthy instances again. | ||
| /// </summary> | ||
| public TimeSpan SleepDuration { get; set; } | ||
|
|
||
| /// <summary> | ||
| /// How long the process should run before ending (allowing the caller to restart the process as desired). | ||
| /// </summary> | ||
| public TimeSpan ProcessLifetime { get; set; } | ||
|
|
||
| /// <summary> | ||
| /// How frequently to poll an instance that was just restarted to check for it's new health status. | ||
| /// </summary> | ||
| public TimeSpan InstancePollFrequency { get; set; } | ||
| } | ||
| } |
Oops, something went wrong.