Merge pull request #1962 from NuGet/anurse/1960-searchservice

Integrated Search Service into the Gallery

nuget.config

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="utf-8"?>
+<configuration>
+  <packageSources>
+    <add key="nugetbuild" value="https://www.myget.org/F/nugetbuild/" />
+  </packageSources>
+  <activePackageSource>
+    <add key="All" value="(Aggregate source)"  />
+  </activePackageSource>
+</configuration>

src/NuGetGallery.Cloud/ServiceConfiguration.Local.cscfg

@@ -22,6 +22,7 @@
       <Setting name="Gallery.GoogleAnalyticsPropertyId" value="" />
       <Setting name="Gallery.AzureCdnHost" value="" />
       <Setting name="Gallery.SiteRoot" value="nuget.localtest.me" />
+      <Setting name="Gallery.SearchServiceUri" value="" />
       <Setting name="Auth.MicrosoftAccount.Enabled" value="false" />
       <Setting name="Auth.MicrosoftAccount.ClientId" value="" />
       <Setting name="Auth.MicrosoftAccount.ClientSecret" value="" />

src/NuGetGallery.Cloud/ServiceDefinition.csdef

@@ -20,6 +20,7 @@
       <Setting name="Gallery.LuceneIndexLocation" />
       <Setting name="Gallery.GoogleAnalyticsPropertyId" />
       <Setting name="Gallery.HasWorker" />
+      <Setting name="Gallery.SearchServiceUri" />
       <Setting name="Auth.MicrosoftAccount.Enabled" />
       <Setting name="Auth.MicrosoftAccount.ClientId" />
       <Setting name="Auth.MicrosoftAccount.ClientSecret" />

src/NuGetGallery.Operations/app.config

@@ -1,3 +1,27 @@
-<?xml version="1.0" encoding="utf-8" ?>
+<?xml version="1.0" encoding="utf-8"?>
 <configuration>
+  <runtime>
+    <assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">
+      <dependentAssembly>
+        <assemblyIdentity name="Microsoft.Data.OData" publicKeyToken="31bf3856ad364e35" culture="neutral" />
+        <bindingRedirect oldVersion="0.0.0.0-5.6.0.0" newVersion="5.6.0.0" />
+      </dependentAssembly>
+      <dependentAssembly>
+        <assemblyIdentity name="EntityFramework" publicKeyToken="b77a5c561934e089" culture="neutral" />
+        <bindingRedirect oldVersion="0.0.0.0-5.0.0.0" newVersion="5.0.0.0" />
+      </dependentAssembly>
+      <dependentAssembly>
+        <assemblyIdentity name="System.Web.Mvc" publicKeyToken="31bf3856ad364e35" culture="neutral" />
+        <bindingRedirect oldVersion="0.0.0.0-4.0.0.0" newVersion="4.0.0.0" />
+      </dependentAssembly>
+      <dependentAssembly>
+        <assemblyIdentity name="Newtonsoft.Json" publicKeyToken="30ad4fe6b2a6aeed" culture="neutral" />
+        <bindingRedirect oldVersion="0.0.0.0-4.5.0.0" newVersion="4.5.0.0" />
+      </dependentAssembly>
+      <dependentAssembly>
+        <assemblyIdentity name="Microsoft.WindowsAzure.Storage" publicKeyToken="31bf3856ad364e35" culture="neutral" />
+        <bindingRedirect oldVersion="0.0.0.0-2.1.0.3" newVersion="2.1.0.3" />
+      </dependentAssembly>
+    </assemblyBinding>
+  </runtime>
 </configuration>

src/NuGetGallery/Admin.LuceneController.generated.cs

@@ -44,8 +44,6 @@ protected RedirectToRouteResult RedirectToAction(ActionResult result) {
         public ActionNamesClass ActionNames { get { return s_actions; } }
         [GeneratedCode("T4MVC", "2.0"), DebuggerNonUserCode]
         public class ActionNamesClass {
-            public readonly string Index = "Index";
-            public readonly string Rebuild = "Rebuild";
         }
 
 
@@ -62,16 +60,6 @@ public class ViewNames {
     public class T4MVC_LuceneController: NuGetGallery.Areas.Admin.Controllers.LuceneController {
         public T4MVC_LuceneController() : base(Dummy.Instance) { }
 
-        public override System.Web.Mvc.ActionResult Index() {
-            var callInfo = new T4MVC_ActionResult(Area, Name, ActionNames.Index);
-            return callInfo;
-        }
-
-        public override System.Web.Mvc.ActionResult Rebuild() {
-            var callInfo = new T4MVC_ActionResult(Area, Name, ActionNames.Rebuild);
-            return callInfo;
-        }
-
     }
 }
 

src/NuGetGallery/App_Start/AppActivator.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Collections.Generic;
 using System.Data.Entity;
 using System.Diagnostics;
 using System.IO;
@@ -127,6 +128,7 @@ private static void BundlingPostStart()
                 .Include("~/Scripts/jquery-{version}.js")
                 .Include("~/Scripts/jquery.validate.js")
                 .Include("~/Scripts/jquery.validate.unobtrusive.js")
+                .Include("~/Scripts/typeahead.bundle.js")
                 .Include("~/Scripts/nugetgallery.js")
                 .Include("~/Scripts/stats.js");
             BundleTable.Bundles.Add(scriptBundle);
@@ -177,40 +179,38 @@ private static void AppPostStart()
 
         private static void BackgroundJobsPostStart(IAppConfiguration configuration)
         {
-            var jobs = configuration.HasWorker ?
-                new IJob[]
-                {
-                    new LuceneIndexingJob(
-                        TimeSpan.FromMinutes(10), 
-                        () => new EntitiesContext(configuration.SqlConnectionString, readOnly: true), 
-                        timeout: TimeSpan.FromMinutes(2), 
-                        location: configuration.LuceneIndexLocation)
-                }                
-                    :
-                new IJob[]
-                {
-                    // readonly: false workaround - let statistics background job write to DB in read-only mode since we don't care too much about losing that data
+            var indexer = Container.Kernel.TryGet<IIndexingService>();
+            var jobs = new List<IJob>();
+            if (indexer != null)
+            {
+                indexer.RegisterBackgroundJobs(jobs, configuration);
+            }
+            if (!configuration.HasWorker)
+            {
+                jobs.Add(
                     new UpdateStatisticsJob(TimeSpan.FromMinutes(5), 
                         () => new EntitiesContext(configuration.SqlConnectionString, readOnly: false), 
-                        timeout: TimeSpan.FromMinutes(5)),
-                    new LuceneIndexingJob(
-                        TimeSpan.FromMinutes(10), 
-                        () => new EntitiesContext(configuration.SqlConnectionString, readOnly: true), 
-                        timeout: TimeSpan.FromMinutes(2), 
-                        location: configuration.LuceneIndexLocation)
-                };
-            var jobCoordinator = new NuGetJobCoordinator();
-            _jobManager = new JobManager(jobs, jobCoordinator)
-                {
-                    RestartSchedulerOnFailure = true
-                };
-            _jobManager.Fail(e => ErrorLog.GetDefault(null).Log(new Error(e)));
-            _jobManager.Start();
+                        timeout: TimeSpan.FromMinutes(5)));
+            }
+
+            if (jobs.AnySafe())
+            {
+                var jobCoordinator = new NuGetJobCoordinator();
+                _jobManager = new JobManager(jobs, jobCoordinator)
+                    {
+                        RestartSchedulerOnFailure = true
+                    };
+                _jobManager.Fail(e => ErrorLog.GetDefault(null).Log(new Error(e)));
+                _jobManager.Start();
+            }
         }
 
         private static void BackgroundJobsStop()
         {
-            _jobManager.Dispose();
+            if (_jobManager != null)
+            {
+                _jobManager.Dispose();
+            }
         }
 
         private static void NinjectPreStart()

src/NuGetGallery/App_Start/ContainerBindings.cs

@@ -15,6 +15,7 @@
 using NuGetGallery.Infrastructure;
 using System.Diagnostics;
 using NuGetGallery.Auditing;
+using NuGetGallery.Infrastructure.Lucene;
 
 namespace NuGetGallery
 {
@@ -35,9 +36,7 @@ public override void Load()
                 .ToMethod(_ => LuceneCommon.GetDirectory(configuration.Current.LuceneIndexLocation))
                 .InSingletonScope();
 
-            Bind<ISearchService>()
-                .To<LuceneSearchService>()
-                .InRequestScope();
+            ConfigureSearch(configuration);
 
             if (!String.IsNullOrEmpty(configuration.Current.AzureStorageConnectionString))
             {
@@ -118,10 +117,6 @@ public override void Load()
                 .To<NuGetControllerFactory>()
                 .InRequestScope();
 
-            Bind<IIndexingService>()
-                .To<LuceneIndexingService>()
-                .InRequestScope();
-
             Bind<INuGetExeDownloaderService>()
                 .To<NuGetExeDownloaderService>()
                 .InRequestScope();
@@ -219,6 +214,28 @@ public override void Load()
                 .InRequestScope();
         }
 
+        private void ConfigureSearch(ConfigurationService configuration)
+        {
+            if (configuration.Current.SearchServiceUri == null)
+            {
+                Bind<ISearchService>()
+                    .To<LuceneSearchService>()
+                    .InRequestScope();
+                Bind<IIndexingService>()
+                    .To<LuceneIndexingService>()
+                    .InRequestScope();
+            }
+            else
+            {
+                Bind<ISearchService>()
+                    .To<ExternalSearchService>()
+                    .InRequestScope();
+                Bind<IIndexingService>()
+                    .To<ExternalSearchService>()
+                    .InRequestScope();
+            }
+        }
+
         private void ConfigureForLocalFileSystem()
         {
             Bind<IFileStorageService>()

src/NuGetGallery/App_Start/Routes.cs

@@ -69,7 +69,7 @@ public static void RegisterRoutes(RouteCollection routes)
             var packageListRoute = routes.MapRoute(
                 RouteName.ListPackages,
                 "packages",
-                MVC.Packages.ListPackages());
+                new { controller = MVC.Packages.Name, action = "ListPackages" });
 
             var uploadPackageRoute = routes.MapRoute(
                 RouteName.UploadPackage,
@@ -207,27 +207,27 @@ public static void RegisterRoutes(RouteCollection routes)
             routes.MapRoute(
                 RouteName.CuratedFeed,
                 "curated-feeds/{name}",
-                new { controller = CuratedFeedsController.ControllerName, action = "CuratedFeed" });
+                new { controller = MVC.CuratedFeeds.Name, action = "CuratedFeed" });
 
             routes.MapRoute(
                 RouteName.CuratedFeedListPackages,
                 "curated-feeds/{curatedFeedName}/packages",
-                MVC.CuratedFeeds.ListPackages());
+                new { controller = MVC.CuratedFeeds.Name, action = "ListPackages" });
 
             routes.MapRoute(
                 RouteName.CreateCuratedPackageForm,
                 "forms/add-package-to-curated-feed",
-                new { controller = CuratedPackagesController.ControllerName, action = "CreateCuratedPackageForm" });
+                new { controller = MVC.CuratedPackages.Name, action = "CreateCuratedPackageForm" });
 
             routes.MapRoute(
                 RouteName.CuratedPackage,
                 "curated-feeds/{curatedFeedName}/curated-packages/{curatedPackageId}",
-                new { controller = CuratedPackagesController.ControllerName, action = "CuratedPackage" });
+                new { controller = MVC.CuratedPackages.Name, action = "CuratedPackage" });
 
             routes.MapRoute(
                 RouteName.CuratedPackages,
                 "curated-feeds/{curatedFeedName}/curated-packages",
-                new { controller = CuratedPackagesController.ControllerName, action = "CuratedPackages" });
+                new { controller = MVC.CuratedPackages.Name, action = "CuratedPackages" });
 
             // TODO : Most of the routes are essentially of the format api/v{x}/*. We should refactor the code to vary them by the version.
             // V1 Routes
@@ -321,6 +321,12 @@ public static void RegisterRoutes(RouteCollection routes)
                 defaults: new { controller = MVC.Api.Name, action = "ServiceAlert" },
                 constraints: new { httpMethod = new HttpMethodConstraint("GET") });
 
+            routes.MapRoute(
+                RouteName.TypeaheadApi,
+                "api/v2/typeahead",
+                defaults: new { controller = MVC.Api.Name, action = "GetTypeaheadApi" },
+                constraints: new { httpMethod = new HttpMethodConstraint("GET") });
+
             routes.MapRoute(
                 RouteName.DownloadNuGetExe,
                 "nuget.exe",
@@ -350,7 +356,7 @@ public static void RegisterRoutes(RouteCollection routes)
                 r => r.MapRoute(
                     RouteName.ListPackages,
                     "List/Packages",
-                    MVC.Packages.ListPackages()),
+                    new { controller = MVC.Packages.Name, action = "ListPackages" }),
                 permanent: true).To(packageListRoute);
 
             routes.Redirect(

src/NuGetGallery/Areas/Admin/Controllers/LuceneController.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Collections.Generic;
 using System.Linq;
+using System.Threading.Tasks;
 using System.Web;
 using System.Web.Mvc;
 using NuGetGallery.Areas.Admin.Models;
@@ -23,20 +24,21 @@ public LuceneController(IIndexingService indexingService, IAppConfiguration conf
         //
         // GET: /Admin/Lucene/
 
-        public virtual ActionResult Index()
+        public virtual async Task<ActionResult> Index()
         {
             return View("Index", new LuceneInfoModel()
             {
-                LastUpdated = IndexingService.GetLastWriteTime(),
-                DocumentCount = IndexingService.GetDocumentCount(),
-                IndexSize = IndexingService.GetIndexSizeInBytes(),
+                LastUpdated = await IndexingService.GetLastWriteTime(),
+                DocumentCount = await IndexingService.GetDocumentCount(),
+                IndexSize = await IndexingService.GetIndexSizeInBytes(),
                 Directory = IndexingService.IndexPath,
+                IsLocal = IndexingService.IsLocal,
                 Location = Config.LuceneIndexLocation
             });
         }
 
         [HttpPost]
-        public virtual ActionResult Rebuild()
+        public virtual Task<ActionResult> Rebuild()
         {
             IndexingService.UpdateIndex(forceRefresh: true);
             return Index();

src/NuGetGallery/Areas/Admin/Models/LuceneInfoModel.cs

@@ -10,10 +10,11 @@ namespace NuGetGallery.Areas.Admin.Models
     public class LuceneInfoModel
     {
         public DateTime? LastUpdated { get; set; }
-        public LuceneIndexLocation Location { get; set; }
         public string Directory { get; set; }
         public int DocumentCount { get; set; }
         public long IndexSize { get; set; }
+        public bool IsLocal { get; set; }
+        public LuceneIndexLocation Location { get; set; }
 
         public string FormatIndexSize()
         {

src/NuGetGallery/Areas/Admin/Views/Lucene/Index.cshtml

@@ -19,14 +19,29 @@ else
     </p>
     <p>
         There are @Model.DocumentCount documents in the index.<br />
-        The index consumes @Model.FormatIndexSize() of space and is stored in @Model.Directory (@Model.Location).
+        The index consumes @Model.FormatIndexSize() of space.<br />
+        The index is located on @Model.Directory
+        @if(Model.IsLocal) {
+            @:(@Model.Location)
+        } 
+        else 
+        {
+            @:(External)
+        }
     </p>
 }
 
-@using (Html.BeginForm(MVC.Admin.Lucene.Rebuild()))
+@if(Model.IsLocal) 
 {
-    <fieldset class="form">
-        <input type="submit" value="Rebuild Now" />
-        <p>NOTE: This will only affect the current instance!</p>
-    </fieldset>
+    using (Html.BeginForm(MVC.Admin.Lucene.Rebuild()))
+    {
+        <fieldset class="form">
+            <input type="submit" value="Rebuild Now" />
+            <p>NOTE: This will only affect the current instance!</p>
+        </fieldset>
+    }
 }
+else 
+{
+    <p>The Lucene Index is located on an external service and cannot be rebuilt.</p>
+}

src/NuGetGallery/Configuration/AppConfiguration.cs

@@ -35,6 +35,11 @@ public class AppConfiguration : IAppConfiguration
         /// </summary>
         public string AzureStorageConnectionString { get; set; }
 
+        /// <summary>
+        /// Gets the URI to the search service
+        /// </summary>
+        public Uri SearchServiceUri { get; set; }
+
         /// <summary>
         /// Gets a boolean indicating if the site requires that email addresses be confirmed
         /// </summary>

src/NuGetGallery/Configuration/IAppConfiguration.cs

@@ -34,6 +34,11 @@ public interface IAppConfiguration
         /// </summary>
         string AzureStorageConnectionString { get; set; }
 
+        /// <summary>
+        /// Gets the URI to the search service
+        /// </summary>
+        Uri SearchServiceUri { get; set; }
+
         /// <summary>
         /// Gets a boolean indicating if the site requires that email addresses be confirmed
         /// </summary>