[GitHub Indexer] Search for popular GitHub repositories

NuGet.Jobs.sln

@@ -1,7 +1,7 @@
 
 Microsoft Visual Studio Solution File, Format Version 12.00
-# Visual Studio Version 16
-VisualStudioVersion = 16.0.28902.138
+# Visual Studio 15
+VisualStudioVersion = 15.0.28307.645
 MinimumVisualStudioVersion = 10.0.40219.1
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NuGet.Jobs.Common", "src\NuGet.Jobs.Common\NuGet.Jobs.Common.csproj", "{4B4B1EFB-8F33-42E6-B79F-54E7F3293D31}"
 EndProject
@@ -151,6 +151,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NuGet.Jobs.GitHubIndexer",
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "TestUtil", "tests\TestUtil\TestUtil.csproj", "{C3F84BAD-ACFA-4AE3-8286-D12F5A5BBC62}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NuGet.Jobs.GitHubIndexer.Tests", "tests\NuGet.Jobs.GitHubIndexer.Tests\NuGet.Jobs.GitHubIndexer.Tests.csproj", "{4A64FEB4-198C-445B-835F-A5B68EFBFDA7}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -399,6 +401,10 @@ Global
 		{C3F84BAD-ACFA-4AE3-8286-D12F5A5BBC62}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{C3F84BAD-ACFA-4AE3-8286-D12F5A5BBC62}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{C3F84BAD-ACFA-4AE3-8286-D12F5A5BBC62}.Release|Any CPU.Build.0 = Release|Any CPU
+		{4A64FEB4-198C-445B-835F-A5B68EFBFDA7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{4A64FEB4-198C-445B-835F-A5B68EFBFDA7}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{4A64FEB4-198C-445B-835F-A5B68EFBFDA7}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{4A64FEB4-198C-445B-835F-A5B68EFBFDA7}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -464,6 +470,7 @@ Global
 		{D3F1711A-25AC-4EC9-9971-4F838BCD2A07} = {6A776396-02B1-475D-A104-26940ADB04AB}
 		{42B1EB66-58F9-4D9A-8E23-FF12CBF5D643} = {FA5644B5-4F08-43F6-86B3-039374312A47}
 		{C3F84BAD-ACFA-4AE3-8286-D12F5A5BBC62} = {6A776396-02B1-475D-A104-26940ADB04AB}
+		{4A64FEB4-198C-445B-835F-A5B68EFBFDA7} = {6A776396-02B1-475D-A104-26940ADB04AB}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {284A7AC3-FB43-4F1F-9C9C-2AF0E1F46C2B}

src/NuGet.Jobs.GitHubIndexer/GitHubSearcherConfiguration.cs

@@ -0,0 +1,23 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+namespace NuGet.Jobs.GitHubIndexer
+{
+    public class GitHubSearcherConfiguration
+    {
+        /// <summary>
+        /// Minimum number of stars that a GitHub Repo needs to have to be included in the indexing
+        /// </summary>
+        public int MinStars { get; set; } = 100;
+
+        /// <summary>
+        /// The number of results that would be shown per page. This is currently limited to 100 (limit verified on 6/24/2019)
+        /// </summary>
+        public int ResultsPerPage { get; set; } = 100;
+
+        /// <summary>
+        /// The limit of results that a single search query can show. This is currently limited to 1000 (limit verified on 6/24/2019)
+        /// </summary>
+        public int MaxGitHubResultsPerQuery { get; set; } = 1000;
+    }
+}

src/NuGet.Jobs.GitHubIndexer/GitRepoSearchers/GitHub/GitHubSearchApiResponse.cs

@@ -0,0 +1,23 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Collections.Generic;
+using NuGetGallery;
+
+namespace NuGet.Jobs.GitHubIndexer
+{
+    public class GitHubSearchApiResponse
+    {
+        public GitHubSearchApiResponse(IReadOnlyList<RepositoryInformation> result, DateTimeOffset date, DateTimeOffset throttleResetTime)
+        {
+            Result = result ?? throw new ArgumentNullException(nameof(result));
+            Date = date;
+            ThrottleResetTime = throttleResetTime;
+        }
+
+        public IReadOnlyList<RepositoryInformation> Result { get; }
+        public DateTimeOffset Date { get; }
+        public DateTimeOffset ThrottleResetTime { get; }
+    }
+}

src/NuGet.Jobs.GitHubIndexer/GitRepoSearchers/GitHub/GitHubSearchWrapper.cs

@@ -0,0 +1,55 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Globalization;
+using System.IO;
+using System.Linq;
+using System.Threading.Tasks;
+using NuGetGallery;
+using Octokit;
+
+namespace NuGet.Jobs.GitHubIndexer
+{
+    public class GitHubSearchWrapper : IGitHubSearchWrapper
+    {
+        private readonly IGitHubClient _client;
+
+        public GitHubSearchWrapper(IGitHubClient client)
+        {
+            _client = client ?? throw new ArgumentNullException(nameof(client));
+        }
+
+        public int? GetRemainingRequestCount()
+        {
+            var apiInfo = _client.GetLastApiInfo();
+            return apiInfo?.RateLimit.Remaining;
+        }
+
+        public async Task<GitHubSearchApiResponse> GetResponse(SearchRepositoriesRequest request)
+        {
+            var apiResponse = await _client.Connection.Get<SearchRepositoryResult>(ApiUrls.SearchRepositories(), request.Parameters, null);
+            if (!apiResponse.HttpResponse.Headers.TryGetValue("Date", out var ghStrDate)
+                || !DateTime.TryParseExact(ghStrDate, "ddd',' dd MMM yyyy HH:mm:ss 'GMT'", CultureInfo.InvariantCulture, DateTimeStyles.None, out var ghTime))
+            {
+                throw new InvalidDataException("Date is required to compute the throttling time.");
+            }
+
+            if (!apiResponse.HttpResponse.Headers.TryGetValue("X-RateLimit-Reset", out var ghStrResetLimit)
+                || !long.TryParse(ghStrResetLimit, out var ghResetTime))
+            {
+                throw new InvalidDataException("X-RateLimit-Reset is required to compute the throttling time.");
+            }
+
+            return new GitHubSearchApiResponse(
+                apiResponse.Body.Items
+                    .Select(repo => new RepositoryInformation(
+                        $"{repo.Owner.Login}/{repo.Name}",
+                        repo.HtmlUrl,
+                        repo.StargazersCount,
+                        Array.Empty<string>())).ToList(),
+                ghTime.ToLocalTime(),
+                DateTimeOffset.FromUnixTimeSeconds(ghResetTime).ToLocalTime());
+        }
+    }
+}

src/NuGet.Jobs.GitHubIndexer/GitRepoSearchers/GitHub/GitHubSearcher.cs

@@ -0,0 +1,160 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+using NuGetGallery;
+using Octokit;
+
+namespace NuGet.Jobs.GitHubIndexer
+{
+    public class GitHubSearcher : IGitRepoSearcher
+    {
+        private static readonly TimeSpan LimitExceededRetryTime = TimeSpan.FromSeconds(5);
+
+        private readonly ILogger<GitHubSearcher> _logger;
+        private readonly IOptionsSnapshot<GitHubSearcherConfiguration> _configuration;
+        private readonly IGitHubSearchWrapper _searchApiRequester;
+
+        private DateTimeOffset _throttleResetTime;
+
+        public GitHubSearcher(
+            IGitHubSearchWrapper searchApiRequester,
+            ILogger<GitHubSearcher> logger,
+            IOptionsSnapshot<GitHubSearcherConfiguration> configuration)
+        {
+            _logger = logger ?? throw new ArgumentNullException(nameof(logger));
+            _configuration = configuration ?? throw new ArgumentNullException(nameof(configuration));
+            _searchApiRequester = searchApiRequester ?? throw new ArgumentNullException(nameof(searchApiRequester));
+        }
+
+        private int _minStars => _configuration.Value.MinStars;
+        private int _resultsPerPage => _configuration.Value.ResultsPerPage;
+        private int _maxGithubResultPerQuery => _configuration.Value.MaxGitHubResultsPerQuery;
+
+        /// <summary>
+        /// Searches for all the C# repos that have more than 100 stars on GitHub, orders them in Descending order and returns them.
+        /// </summary>
+        /// <returns>List of C# repos on GitHub that have more than 100 stars</returns>
+        public async Task<IReadOnlyList<RepositoryInformation>> GetPopularRepositories()
+        {
+            _logger.LogInformation("Starting search on GitHub...");
+            var result = await GetResultsFromGitHub();
+            return result
+                .GroupBy(x => x.Id) // Used to remove duplicate repos (since the GH Search API may return a result that we already had in memory)
+                .Select(g => g.First())
+                .OrderByDescending(x => x.Stars)
+                .ToList();
+        }
+
+        private async Task CheckThrottle()
+        {
+            if (_searchApiRequester.GetRemainingRequestCount() == 0)
+            {
+                var sleepTime = _throttleResetTime - DateTimeOffset.Now;
+                _throttleResetTime = DateTimeOffset.Now;
+                if (sleepTime.TotalSeconds > 0)
+                {
+                    _logger.LogInformation("Waiting {TotalSeconds} seconds to cooldown.", sleepTime.TotalSeconds);
+                    await Task.Delay(sleepTime);
+                }
+
+                _logger.LogInformation("Resuming search.");
+            }
+        }
+
+        private async Task<IReadOnlyList<RepositoryInformation>> SearchRepo(SearchRepositoriesRequest request)
+        {
+            _logger.LogInformation("Requesting page {Page} for stars {Stars}", request.Page, request.Stars);
+
+            bool? error = null;
+            GitHubSearchApiResponse response = null;
+            while (!error.HasValue || error.Value)
+            {
+                try
+                {
+                    response = await _searchApiRequester.GetResponse(request);
+                    error = false;
+                }
+                catch (RateLimitExceededException)
+                {
+                    _logger.LogError("Exceeded GitHub RateLimit! Waiting for {time} before retrying.", LimitExceededRetryTime);
+                    await Task.Delay(LimitExceededRetryTime);
+                }
+            }
+
+            if (_throttleResetTime < DateTimeOffset.Now)
+            {
+                var timeToWait = response.ThrottleResetTime - response.Date;
+                _throttleResetTime = DateTimeOffset.Now + timeToWait;
+            }
+
+            return response.Result;
+        }
+
+        private async Task<List<RepositoryInformation>> GetResultsFromGitHub()
+        {
+            _throttleResetTime = DateTimeOffset.Now;
+            var upperStarBound = int.MaxValue;
+            var resultList = new List<RepositoryInformation>();
+            var lastPage = Math.Ceiling(_maxGithubResultPerQuery / (double)_resultsPerPage);
+
+            while (upperStarBound >= _minStars)
+            {
+                var page = 0;
+                while (page < lastPage)
+                {
+                    await CheckThrottle();
+
+                    var request = new SearchRepositoriesRequest
+                    {
+                        Stars = new Range(_minStars, upperStarBound),
+                        Language = Language.CSharp,
+                        SortField = RepoSearchSort.Stars,
+                        Order = SortDirection.Descending,
+                        PerPage = _resultsPerPage,
+                        Page = page + 1
+                    };
+
+                    var response = await SearchRepo(request);
+
+                    if (response == null || !response.Any())
+                    {
+                        _logger.LogWarning("Search request didn't return any item. Page: {Page} {ConfigInfo}", request.Page, GetConfigInfo());
+                        return resultList;
+                    }
+
+                    // TODO: Block unwanted repos (https://github.com/NuGet/NuGetGallery/issues/7298)
+                    resultList.AddRange(response);
+                    page++;
+
+                    if (page == lastPage && response.First().Stars == response.Last().Stars)
+                    {
+                        // This may result in missing data since more the entire result set produced by a query has the same star count, we can't produce
+                        // an "all the repos with the same star count but that are not these ones" query
+                        _logger.LogWarning("Last page results have the same star count! This may result in missing data. StarCount: {Stars} {ConfigInfo}",
+                            response.First().Stars,
+                            GetConfigInfo());
+
+                        return resultList;
+                    }
+                }
+
+                upperStarBound = resultList.Last().Stars;
+            }
+
+            return resultList;
+        }
+
+        private string GetConfigInfo()
+        {
+            return $"MinStars: {_minStars}\n" +
+               $"ResultsPerPage: {_resultsPerPage}\n" +
+               $"MaxGithubResultPerQuery: {_maxGithubResultPerQuery}\n";
+        }
+    }
+}

src/NuGet.Jobs.GitHubIndexer/GitRepoSearchers/GitHub/IGitHubSearchWrapper.cs

@@ -0,0 +1,24 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System.Threading.Tasks;
+using Octokit;
+
+namespace NuGet.Jobs.GitHubIndexer
+{
+    public interface IGitHubSearchWrapper
+    {
+        /// <summary>
+        /// Queries the GitHub Repo Search Api and returns its reponse
+        /// </summary>
+        /// <param name="request">Request to be made to the GitHub Repo Search Api</param>
+        /// <returns>Parsed reponse of the GitHub Repo Search Api</returns>
+        Task<GitHubSearchApiResponse> GetResponse(SearchRepositoriesRequest request);
+
+        /// <summary>
+        /// Returns the number of remaining requests before the search gets throttled
+        /// </summary>
+        /// <returns>Returns the number of remaining requests or null if no info is available (no request has been done yet)</returns>
+        int? GetRemainingRequestCount();
+    }
+}

src/NuGet.Jobs.GitHubIndexer/GitRepoSearchers/IGitRepoSearcher.cs

@@ -0,0 +1,18 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System.Collections.Generic;
+using System.Threading.Tasks;
+using NuGetGallery;
+
+namespace NuGet.Jobs.GitHubIndexer
+{
+    public interface IGitRepoSearcher
+    {
+        /// <summary>
+        /// Searches for all popular C# repos, orders them in Descending order and returns a list containing their basic information
+        /// </summary>
+        /// <returns>List of popular C# repositories</returns>
+        Task<IReadOnlyList<RepositoryInformation>> GetPopularRepositories();
+    }
+}

src/NuGet.Jobs.GitHubIndexer/Job.cs

@@ -1,32 +1,40 @@
 // Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
+using System.IO;
 using System.Threading.Tasks;
 using Autofac;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
+using Newtonsoft.Json;
+using Octokit;
 
 namespace NuGet.Jobs.GitHubIndexer
 {
     public class Job : JsonConfigurationJob
     {
-        public Job()
-        {
-        }
+        private const string GitHubIndexerUserAgent = "NuGet-NuGet.Jobs-GitHubIndexer";
+        private const string GitHubSearcherConfigurationSectionName = "GitHubSearcher";
 
-        public override Task Run()
+        public override async Task Run()
         {
-            throw new System.NotImplementedException();
+            var searcher = _serviceProvider.GetRequiredService<IGitRepoSearcher>();
+            var repos = await searcher.GetPopularRepositories();
+
+            File.WriteAllText("Repos.json", JsonConvert.SerializeObject(repos));
         }
 
-        protected override void ConfigureAutofacServices(ContainerBuilder containerBuilder)
+        protected override void ConfigureJobServices(IServiceCollection services, IConfigurationRoot configurationRoot)
         {
-            throw new System.NotImplementedException();
+            services.AddTransient<IGitRepoSearcher, GitHubSearcher>();
+            services.AddSingleton<IGitHubClient>(provider => new GitHubClient(new ProductHeaderValue(GitHubIndexerUserAgent)));
+            services.AddSingleton<IGitHubSearchWrapper, GitHubSearchWrapper>();
+
+            services.Configure<GitHubSearcherConfiguration>(configurationRoot.GetSection(GitHubSearcherConfigurationSectionName));
         }
 
-        protected override void ConfigureJobServices(IServiceCollection services, IConfigurationRoot configurationRoot)
+        protected override void ConfigureAutofacServices(ContainerBuilder containerBuilder)
         {
-            throw new System.NotImplementedException();
         }
     }
 }