Fix #1970 by adding a perf log for the external search service

src/NuGetGallery.Cloud/NuGetGalleryContent/diagnostics.wadcfg

@@ -5,6 +5,11 @@
     <IISLogs container="wad-iis-requestlogs" directoryQuotaInMB="256" />
     <FailedRequestLogs container="wad-iis-failedrequestlogs" directoryQuotaInMB="256" />
     <CrashDumps container="wad-crash-dumps" directoryQuotaInMB="256" />
+    <DataSources>
+      <DirectoryConfiguration container="ng-v2gallery-logs" directoryQuotaInMB="1024">
+        <LocalResource name="Logs" relativePath="." />
+      </DirectoryConfiguration>
+    </DataSources>
   </Directories>
   <Logs bufferQuotaInMB="512" scheduledTransferPeriod="PT1M" scheduledTransferLogLevelFilter="Verbose" />
   <PerformanceCounters bufferQuotaInMB="512" scheduledTransferPeriod="PT1M">

src/NuGetGallery/App_Start/AppActivator.cs

@@ -18,6 +18,7 @@
 using Ninject.Web.Common;
 using NuGetGallery;
 using NuGetGallery.Configuration;
+using NuGetGallery.Diagnostics;
 using NuGetGallery.Infrastructure;
 using NuGetGallery.Infrastructure.Jobs;
 using NuGetGallery.Jobs;
@@ -37,6 +38,8 @@ public static class AppActivator
 
         public static void PreStart()
         {
+            MessageQueue.Enable(maxPerQueue: 1000);
+
             AntiForgeryConfig.UniqueClaimTypeIdentifier = ClaimTypes.NameIdentifier;
 
             ViewEngines.Engines.Clear();
@@ -192,6 +195,10 @@ private static void BackgroundJobsPostStart(IAppConfiguration configuration)
                         () => new EntitiesContext(configuration.SqlConnectionString, readOnly: false), 
                         timeout: TimeSpan.FromMinutes(5)));
             }
+            if (configuration.CollectPerfLogs)
+            {
+                jobs.Add(CreateLogFlushJob());
+            }
 
             if (jobs.AnySafe())
             {
@@ -205,6 +212,31 @@ private static void BackgroundJobsPostStart(IAppConfiguration configuration)
             }
         }
 
+        private static ProcessPerfEvents CreateLogFlushJob()
+        {
+            var logDirectory = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "App_Data", "Logs");
+            try
+            {
+                if (RoleEnvironment.IsAvailable)
+                {
+                    var resource = RoleEnvironment.GetLocalResource("Logs");
+                    if (resource != null)
+                    {
+                        logDirectory = Path.Combine(resource.RootPath);
+                    }
+                }
+            }
+            catch (Exception)
+            {
+                // Meh, so Azure isn't available...
+            }
+            return new ProcessPerfEvents(
+                TimeSpan.FromSeconds(10),
+                logDirectory,
+                new[] { "ExternalSearchService" },
+                timeout: TimeSpan.FromSeconds(10));
+        }
+
         private static void BackgroundJobsStop()
         {
             if (_jobManager != null)

src/NuGetGallery/Areas/Admin/Controllers/LuceneController.cs

@@ -6,6 +6,8 @@
 using System.Web.Mvc;
 using NuGetGallery.Areas.Admin.Models;
 using NuGetGallery.Configuration;
+using NuGetGallery.Diagnostics;
+using NuGetGallery.Infrastructure.Lucene;
 
 namespace NuGetGallery.Areas.Admin.Controllers
 {
@@ -33,7 +35,8 @@ public virtual async Task<ActionResult> Index()
                 IndexSize = await IndexingService.GetIndexSizeInBytes(),
                 Directory = IndexingService.IndexPath,
                 IsLocal = IndexingService.IsLocal,
-                Location = Config.LuceneIndexLocation
+                Location = Config.LuceneIndexLocation,
+                QueryStats = PerfCounters.GetStats(ExternalSearchService.SearchRTTPerfCounter)
             });
         }
 

src/NuGetGallery/Areas/Admin/Models/LuceneInfoModel.cs

@@ -4,6 +4,7 @@
 using System.Linq;
 using System.Text;
 using NuGetGallery.Configuration;
+using NuGetGallery.Diagnostics;
 
 namespace NuGetGallery.Areas.Admin.Models
 {
@@ -15,6 +16,7 @@ public class LuceneInfoModel
         public long IndexSize { get; set; }
         public bool IsLocal { get; set; }
         public LuceneIndexLocation Location { get; set; }
+        public PerfCounters.PerfStats QueryStats { get; set; }
 
         public string FormatIndexSize()
         {

src/NuGetGallery/Areas/Admin/Views/Home/Index.cshtml

@@ -33,7 +33,7 @@
         </li>
         <li>
             <h2>
-                <a class="action-menu-link" href="@Url.Action(MVC.Admin.Lucene.Index())">
+                <a class="action-menu-link" href="@Url.Action("Index", "Lucene")">
                     <i class="icon-search action-menu-icon"></i>
                     <span class="action-menu-text">Lucene Index Maintainance</span>
                 </a>

src/NuGetGallery/Areas/Admin/Views/Lucene/Index.cshtml

@@ -44,4 +44,18 @@ else
 else 
 {
     <p>The Lucene Index is located on an external service and cannot be rebuilt.</p>
+}
+
+<h3>Query Statistics</h3>
+@if (Model.QueryStats != null)
+{
+    <p>
+        Average Round Trip Time: @Model.QueryStats.Average.ToString("0.00")ms<br/>
+        Maximum Round Trip Time: @Model.QueryStats.Max.ToString("0.00")ms<br/>
+        Minimum Round Trip Time: @Model.QueryStats.Min.ToString("0.00")ms<br/>
+        Based on the most recent @Model.QueryStats.Samples samples<br/>
+    </p>
+}
+else {
+    <p>Query statistics are not available or no queries have been made since the last server restart</p>
 }

src/NuGetGallery/Configuration/AppConfiguration.cs

@@ -113,5 +113,10 @@ public class AppConfiguration : IAppConfiguration
         /// Gets the Google Analytics Property ID being used, if any.
         /// </summary>
         public string GoogleAnalyticsPropertyId { get; set; }
+
+        /// <summary>
+        /// Gets a boolean indicating if perf logs should be collected
+        /// </summary>
+        public bool CollectPerfLogs { get; set; }
     }
 }

src/NuGetGallery/Configuration/IAppConfiguration.cs

@@ -98,5 +98,10 @@ public interface IAppConfiguration
         /// Gets the Google Analytics Property ID being used, if any.
         /// </summary>
         string GoogleAnalyticsPropertyId { get; set; }
+
+        /// <summary>
+        /// Gets a boolean indicating if perf logs should be collected
+        /// </summary>
+        bool CollectPerfLogs { get; set; }
     }
 }

src/NuGetGallery/Controllers/ApiController.cs

@@ -439,7 +439,7 @@ public virtual async Task<ActionResult> GetStatsDownloads(int? count)
         public virtual async Task<ActionResult> GetTypeahead(string q)
         {
             // Query the search service
-            var results = await SearchService.Search(new SearchFilter()
+            var results = await SearchService.Search(new SearchFilter(SearchFilter.UITypeaheadContext)
             {
                 SearchTerm = q, /* For typeahead we want wildcard matching */
                 IncludePrerelease = true,

src/NuGetGallery/Controllers/CuratedFeedsController.cs

@@ -64,7 +64,7 @@ public virtual async Task<ActionResult> ListPackages(string curatedFeedName, str
 
             q = (q ?? "").Trim();
 
-            var searchFilter = SearchAdaptor.GetSearchFilter(q, page, sortOrder: null);
+            var searchFilter = SearchAdaptor.GetSearchFilter(q, page, sortOrder: null, context: SearchFilter.UISearchContext);
             searchFilter.CuratedFeed = CuratedFeedService.GetFeedByName(curatedFeedName, includePackages: false);
             if (searchFilter.CuratedFeed == null)
             {

src/NuGetGallery/Controllers/PackagesController.cs

@@ -292,7 +292,7 @@ public virtual async Task<ActionResult> ListPackages(string q, int page = 1)
 
             q = (q ?? "").Trim();
 
-            var searchFilter = SearchAdaptor.GetSearchFilter(q, page, sortOrder: null);
+            var searchFilter = SearchAdaptor.GetSearchFilter(q, page, sortOrder: null, context: SearchFilter.UISearchContext);
             var results = await _searchService.Search(searchFilter);
             int totalHits = results.Hits;
             if (page == 1 && !results.Data.Any())

src/NuGetGallery/DataServices/SearchAdaptor.cs

@@ -16,9 +16,9 @@ public static class SearchAdaptor
         /// </summary>
         internal const int MaxPageSize = 40;
 
-        public static SearchFilter GetSearchFilter(string q, int page, string sortOrder)
+        public static SearchFilter GetSearchFilter(string q, int page, string sortOrder, string context)
         {
-            var searchFilter = new SearchFilter
+            var searchFilter = new SearchFilter(context)
             {
                 SearchTerm = q,
                 Skip = (page - 1) * Constants.DefaultPackageListPageSize, // pages are 1-based. 
@@ -71,9 +71,10 @@ public static async Task<IQueryable<Package>> SearchCore(
             CuratedFeed curatedFeed)
         {
             SearchFilter searchFilter;
-            // We can only use Lucene if the client queries for the latest versions (IsLatest \ IsLatestStable) versions of a package
-            // and specific sort orders that we have in the index.
-            if (TryReadSearchFilter(request.RawUrl, out searchFilter))
+            // We can only use Lucene if:
+            //  a) We are looking for the latest version of a package OR the Index contains all versions of each package
+            //  b) The sort order is something Lucene can handle
+            if (TryReadSearchFilter(searchService.ContainsAllVersions, request.RawUrl, out searchFilter))
             {
                 searchFilter.SearchTerm = searchTerm;
                 searchFilter.IncludePrerelease = includePrerelease;
@@ -94,7 +95,7 @@ public static async Task<IQueryable<Package>> SearchCore(
             return packages.Search(searchTerm);
         }
 
-        private static bool TryReadSearchFilter(string url, out SearchFilter searchFilter)
+        private static bool TryReadSearchFilter(bool allVersionsInIndex, string url, out SearchFilter searchFilter)
         {
             if (url == null)
             {
@@ -103,7 +104,6 @@ private static bool TryReadSearchFilter(string url, out SearchFilter searchFilte
             }
 
             int indexOfQuestionMark = url.IndexOf('?');
-
             if (indexOfQuestionMark == -1)
             {
                 searchFilter = null;
@@ -119,7 +119,7 @@ private static bool TryReadSearchFilter(string url, out SearchFilter searchFilte
                 return false;
             }
 
-            searchFilter = new SearchFilter
+            searchFilter = new SearchFilter(SearchFilter.ODataSearchContext)
             {
                 // The way the default paging works is WCF attempts to read up to the MaxPageSize elements. If it finds as many, it'll assume there 
                 // are more elements to be paged and generate a continuation link. Consequently we'll always ask to pull MaxPageSize elements so WCF generates the 
@@ -144,6 +144,7 @@ private static bool TryReadSearchFilter(string url, out SearchFilter searchFilte
 
             // We'll only use the index if we the query searches for latest \ latest-stable packages
 
+
             string filter;
             if (queryTerms.TryGetValue("$filter", out filter))
             {
@@ -153,7 +154,7 @@ private static bool TryReadSearchFilter(string url, out SearchFilter searchFilte
                     return false;
                 }
             }
-            else
+            else if(!allVersionsInIndex)
             {
                 searchFilter = null;
                 return false;

src/NuGetGallery/DataServices/SearchHijacker.cs

@@ -155,7 +155,7 @@ private static bool IsSelectV2FeedPackage(MethodCallExpression expr)
             private IQueryable Hijack(IList<Tuple<Target, string>> comparisons)
             {
                 // Perform the search using the search service and just return the result.
-                return SearchService.RawSearch(new SearchFilter()
+                return SearchService.RawSearch(new SearchFilter(SearchFilter.ODataInterceptContext)
                 {
                     SearchTerm = BuildQuery(comparisons),
                     IncludePrerelease = true,

src/NuGetGallery/Diagnostics/DiagnosticsGlimpseTab.cs

@@ -0,0 +1,24 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Web;
+using Glimpse.Core.Extensibility;
+
+namespace NuGetGallery.Diagnostics
+{
+    public class DiagnosticsGlimpseTab : TabBase
+    {
+        public override object GetData(ITabContext context)
+        {
+            return new
+            {
+                MessageQueues = MessageQueue.GetQueueStats()
+            };
+        }
+
+        public override string Name
+        {
+            get { return "Diag"; }
+        }
+    }
+}

src/NuGetGallery/Diagnostics/IDiagnosticsSource.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Collections.Generic;
 using System.Diagnostics;
 using System.Globalization;
 using System.Runtime.CompilerServices;
@@ -10,6 +11,8 @@ namespace NuGetGallery.Diagnostics
     public interface IDiagnosticsSource
     {
         void TraceEvent(TraceEventType type, int id, string message, [CallerMemberName] string member = null, [CallerFilePath] string file = null, [CallerLineNumber] int line = 0);
+
+        void PerfEvent(string name, TimeSpan time, IEnumerable<KeyValuePair<string, object>> payload);
     }
 
     public static class DiagnosticsSourceExtensions

src/NuGetGallery/Diagnostics/NullDiagnosticsSource.cs

@@ -17,5 +17,10 @@ public void TraceEvent(System.Diagnostics.TraceEventType type, int id, string me
         {
             // No-op!
         }
+
+        public void PerfEvent(string name, TimeSpan time, IEnumerable<KeyValuePair<string, object>> payload)
+        {
+            // No-op!
+        }
     }
 }