Fix NuGet/NuGetGallery#1940 by adding search service jobs

src/JobHost/Program.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Diagnostics;
 using System.Diagnostics.Tracing;
 using System.Linq;
 using System.Text;
@@ -13,6 +14,12 @@ class Program
     {
         static void Main(string[] args)
         {
+            if (args.Length > 0 && String.Equals(args[0], "dbg", StringComparison.OrdinalIgnoreCase))
+            {
+                args = args.Skip(1).ToArray();
+                Debugger.Launch();
+            }
+
             Arguments parsed;
             try
             {

src/NuGet.Services.Work/Jobs/Bases/SearchIndexJobHandlerBase.cs

@@ -17,6 +17,7 @@ public abstract class SearchIndexJobHandlerBase<T> : JobHandler<T>
         public SqlConnectionStringBuilder PackageDatabase { get; set; }
         public CloudStorageAccount StorageAccount { get; set; }
         public string StorageContainerName { get; set; }
+        public string LocalIndexFolder { get; set; }
 
         protected ConfigurationHub Config { get; set; }
 

src/NuGet.Services.Work/Jobs/GenerateSearchRankingsJob.cs

@@ -9,6 +9,7 @@
 using Dapper;
 using Microsoft.WindowsAzure.Storage;
 using Microsoft.WindowsAzure.Storage.Blob;
+using Newtonsoft.Json;
 using Newtonsoft.Json.Linq;
 using NuGet.Services.Client;
 using NuGet.Services.Configuration;
@@ -56,8 +57,10 @@ protected internal override async Task Execute()
             }
 
             // Gather overall rankings
+            JObject report = new JObject();
             Log.GatheringOverallRankings(WarehouseConnection.DataSource, WarehouseConnection.InitialCatalog);
             var overallData = await GatherOverallRankings();
+            report.Add("Rank", overallData);
             Log.GatheredOverallRankings(overallData.Count);
 
             // Get project types
@@ -66,27 +69,18 @@ protected internal override async Task Execute()
             Log.GotAvailableProjectTypes(projectTypes.Count);
 
             // Gather data by project type
-            IDictionary<string, IList<SearchRankingEntry>> byProjectType = new Dictionary<string, IList<SearchRankingEntry>>();
             int count = 0;
             Log.GatheringProjectTypeRankings(WarehouseConnection.DataSource, WarehouseConnection.InitialCatalog);
             foreach (var projectType in projectTypes)
             {
                 Log.GatheringProjectTypeRanking(WarehouseConnection.DataSource, WarehouseConnection.InitialCatalog, projectType);
                 var data = await GatherProjectTypeRanking(projectType);
+                report.Add(projectType, data);
                 Log.GatheredProjectTypeRanking(data.Count, projectType);
                 count += data.Count;
-
-                byProjectType.Add(projectType, data);
             }
             Log.GatheredProjectTypeRankings(count);
 
-            // Generate the report
-            var report = new SearchRankingReport()
-            {
-                Overall = overallData,
-                ByProjectType = byProjectType
-            };
-
             // Write the JSON blob
             if (!String.IsNullOrEmpty(OutputDirectory))
             {
@@ -99,7 +93,7 @@ protected internal override async Task Execute()
             }
         }
 
-        private async Task WriteToFile(SearchRankingReport report)
+        private async Task WriteToFile(JObject report)
         {
             string fullPath = Path.Combine(OutputDirectory, ReportName);
             string parentDir = Path.GetDirectoryName(fullPath);
@@ -116,20 +110,20 @@ private async Task WriteToFile(SearchRankingReport report)
                 }
                 using (var writer = new StreamWriter(File.OpenWrite(fullPath)))
                 {
-                    await writer.WriteAsync(JsonFormat.Serialize(report));
+                    await writer.WriteAsync(report.ToString(Formatting.Indented));
                 }
             }
             Log.WroteReportBlob(fullPath);
         }
 
-        private async Task WriteToBlob(SearchRankingReport report)
+        private async Task WriteToBlob(JObject report)
         {
             var blob = DestinationContainer.GetBlockBlobReference(ReportName);
             Log.WritingReportBlob(blob.Uri.AbsoluteUri);
             if (!WhatIf)
             {
                 blob.Properties.ContentType = MimeTypes.Json;
-                await blob.UploadTextAsync(JsonFormat.Serialize(report));
+                await blob.UploadTextAsync(report.ToString(Formatting.Indented));
             }
             Log.WroteReportBlob(blob.Uri.AbsoluteUri);
         }
@@ -145,15 +139,17 @@ private void LoadDefaults()
             }
         }
 
-        private async Task<IList<SearchRankingEntry>> GatherOverallRankings()
+        private async Task<JArray> GatherOverallRankings()
         {
             using (var connection = await WarehouseConnection.ConnectTo())
             {
                 // Get the script
                 var script = await ResourceHelpers.ReadResourceFile("NuGet.Services.Work.Jobs.Scripts.SearchRanking_Overall.sql");
 
                 // Execute it and return the results
-                return (await connection.QueryAsync<SearchRankingEntry>(script)).ToList();
+                return new JArray(
+                    (await connection.QueryAsync<SearchRankingEntry>(script))
+                        .Select(e => e.PackageId));
             }
         }
 
@@ -166,15 +162,17 @@ private async Task<IList<string>> GetProjectTypes()
             }
         }
 
-        private async Task<IList<SearchRankingEntry>> GatherProjectTypeRanking(string projectType)
+        private async Task<JArray> GatherProjectTypeRanking(string projectType)
         {
             using (var connection = await WarehouseConnection.ConnectTo())
             {
                 // Get the script
                 var script = await ResourceHelpers.ReadResourceFile("NuGet.Services.Work.Jobs.Scripts.SearchRanking_ByProjectType.sql");
 
                 // Execute it and return the results
-                return (await connection.QueryAsync<SearchRankingEntry>(script, new { ProjectGuid = projectType })).ToList();
+                return new JArray(
+                    (await connection.QueryAsync<SearchRankingEntry>(script, new { ProjectGuid = projectType }))
+                        .Select(e => e.PackageId));
             }
         }
     }

src/NuGet.Services.Work/Jobs/RebuildSearchIndexJob.cs

@@ -18,29 +18,63 @@ public class RebuildSearchIndexJob : SearchIndexJobHandlerBase<RebuildSearchInde
     {
         public RebuildSearchIndexJob(ConfigurationHub config) : base(config) { }
 
-        protected internal override Task Execute()
+        protected internal override async Task Execute()
         {
+            // This job can take a long time and is run manually. Make the job timeout long
+            await Extend(TimeSpan.FromHours(12));
+
             // Run the task
+            Log.BeginningIndex(
+                PackageDatabase.DataSource + "/" + PackageDatabase.InitialCatalog,
+                String.IsNullOrEmpty(LocalIndexFolder) ?
+                    (StorageAccount.Credentials.AccountName + "/" + StorageContainerName) :
+                    LocalIndexFolder);
             FullBuildTask task = new FullBuildTask()
             {
                 SqlConnectionString = PackageDatabase.ConnectionString,
                 StorageAccount = StorageAccount,
-                Container = StorageContainerName ?? "ng-search",
+                Container = String.IsNullOrEmpty(LocalIndexFolder) ?
+                    StorageContainerName :
+                    null,
+                Folder = LocalIndexFolder,
                 Log = new EventSourceWriter(Log.IndexingTrace)
             };
             task.Execute();
-
-            return Task.FromResult(0);
+            Log.FinishedIndex();
         }
     }
 
     [EventSource(Name="Outercurve-NuGet-Jobs-RebuildSearchIndex")]
     public class RebuildSearchIndexEventSource : EventSource
     {
+        public static readonly RebuildSearchIndexEventSource Log = new RebuildSearchIndexEventSource();
+        private RebuildSearchIndexEventSource() { }
+
         [Event(
             eventId: 1,
             Level = EventLevel.Informational,
             Message = "Indexing Trace: {0}")]
         public void IndexingTrace(string message) { WriteEvent(1, message); }
+
+        [Event(
+            eventId: 2,
+            Level = EventLevel.Informational,
+            Message = "Beginning Index Rebuild from {0} to {1}",
+            Task = Tasks.Indexing,
+            Opcode = EventOpcode.Start)]
+        public void BeginningIndex(string source, string destination) { WriteEvent(2, source, destination); }
+
+        [Event(
+            eventId: 3,
+            Level = EventLevel.Informational,
+            Message = "Finished Index Rebuild",
+            Task = Tasks.Indexing,
+            Opcode = EventOpcode.Stop)]
+        public void FinishedIndex() { WriteEvent(3); }
+
+        public static class Tasks
+        {
+            public const EventTask Indexing = (EventTask)0x1;
+        }
     }
 }

src/NuGet.Services.Work/Jobs/UpdateSearchIndexJob.cs

@@ -22,7 +22,10 @@ protected internal override Task Execute()
             {                
                 SqlConnectionString = PackageDatabase.ConnectionString,
                 StorageAccount = StorageAccount,
-                Container = StorageContainerName ?? "ng-search",
+                Container = String.IsNullOrEmpty(LocalIndexFolder) ? 
+                    (StorageContainerName ?? "ng-search") :
+                    null,
+                Folder = LocalIndexFolder,
                 Log = new EventSourceWriter(Log.IndexingTrace),
             };
             task.Execute();
@@ -34,6 +37,9 @@ protected internal override Task Execute()
     [EventSource(Name="Outercurve-NuGet-Jobs-UpdateSearchIndex")]
     public class UpdateSearchIndexEventSource : EventSource
     {
+        public static readonly UpdateSearchIndexEventSource Log = new UpdateSearchIndexEventSource();
+        private UpdateSearchIndexEventSource() { }
+
         [Event(
             eventId: 1,
             Level = EventLevel.Informational,

src/NuGet.Services.Work/Monitoring/EventSourceWriter.cs

@@ -33,7 +33,7 @@ public override void Write(char value)
 
         public override void Flush()
         {
-            _receiver(_buffer.ToString());
+            _receiver(_buffer.ToString().Trim());
             _buffer.Clear();
         }
 

src/NuGet.Services.Work/NuGet.Services.Work.csproj

@@ -137,8 +137,9 @@
       <SpecificVersion>False</SpecificVersion>
       <HintPath>..\..\packages\Nuget.Core.2.8.0\lib\net40-Client\NuGet.Core.dll</HintPath>
     </Reference>
-    <Reference Include="NuGet.Indexing">
-      <HintPath>..\..\packages\NuGet.Indexing.3.0.2-alpha-4\lib\net45\NuGet.Indexing.dll</HintPath>
+    <Reference Include="NuGet.Indexing, Version=3.0.0.0, Culture=neutral, processorArchitecture=MSIL">
+      <SpecificVersion>False</SpecificVersion>
+      <HintPath>..\..\packages\NuGet.Indexing.3.0.2-alpha-9\lib\net45\NuGet.Indexing.dll</HintPath>
     </Reference>
     <Reference Include="NuGet.Services.Platform, Version=3.0.0.0, Culture=neutral, processorArchitecture=MSIL">
       <SpecificVersion>False</SpecificVersion>
@@ -181,9 +182,6 @@
     <Reference Include="System.Reactive.Linq">
       <HintPath>..\..\packages\Rx-Linq.2.2.2\lib\net45\System.Reactive.Linq.dll</HintPath>
     </Reference>
-    <Reference Include="System.Reactive.PlatformServices">
-      <HintPath>..\..\packages\Rx-PlatformServices.2.2.2\lib\net45\System.Reactive.PlatformServices.dll</HintPath>
-    </Reference>
     <Reference Include="System.Runtime.Serialization" />
     <Reference Include="System.ServiceModel" />
     <Reference Include="System.Spatial, Version=5.6.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
@@ -252,7 +250,6 @@
     <Compile Include="Jobs\Models\PackageEdit.cs" />
     <Compile Include="Jobs\Models\PackageRef.cs" />
     <Compile Include="Jobs\Models\SearchRankingEntry.cs" />
-    <Compile Include="Jobs\Models\SearchRankingReport.cs" />
     <Compile Include="Jobs\PurgeCompletedInvocationsJob.cs" />
     <Compile Include="Jobs\PurgePackageStatisticsJob.cs" />
     <Compile Include="Jobs\RebuildSearchIndexJob.cs" />
@@ -336,4 +333,4 @@
   <Target Name="AfterBuild">
   </Target>
   -->
-</Project>
+</Project>

src/NuGet.Services.Work/WorkService.cs

@@ -210,22 +210,24 @@ public IObservable<EventEntry> RunJob(string job, string payload)
                         QueuedAt = DateTime.UtcNow,
                         NextVisibleAt = DateTime.UtcNow + TimeSpan.FromMinutes(5)
                     });
-            var buffer = new ReplaySubject<EventEntry>();
-            var capture = new InvocationLogCapture(invocation);
-            capture.Subscribe(buffer.OnNext, buffer.OnError);
-            runner.Dispatch(invocation, capture, CancellationToken.None).ContinueWith(t =>
+            return Observable.Create<EventEntry>(observer =>
             {
-                if (t.IsFaulted)
+                var capture = new InvocationLogCapture(invocation);
+                capture.Subscribe(e => observer.OnNext(e), ex => observer.OnError(ex));
+                runner.Dispatch(invocation, capture, CancellationToken.None).ContinueWith(t =>
                 {
-                    buffer.OnError(t.Exception);
-                }
-                else
-                {
-                    buffer.OnCompleted();
-                }
-                return t;
+                    if (t.IsFaulted)
+                    {
+                        observer.OnError(t.Exception);
+                    }
+                    else
+                    {
+                        observer.OnCompleted();
+                    }
+                    return t;
+                });
+                return () => { }; // No action on unsubscribe
             });
-            return buffer;
         }
 
         protected override void ConfigureAttributeRouting(DefaultInlineConstraintResolver resolver)

src/NuGet.Services.Work/packages.config

@@ -27,7 +27,7 @@
   <package id="Microsoft.WindowsAzure.ConfigurationManager" version="2.0.3" targetFramework="net45" />
   <package id="Newtonsoft.Json" version="6.0.1" targetFramework="net45" />
   <package id="Nuget.Core" version="2.8.0" targetFramework="net45" />
-  <package id="NuGet.Indexing" version="3.0.2-alpha-4" targetFramework="net45" />
+  <package id="NuGet.Indexing" version="3.0.2-alpha-9" targetFramework="net45" />
   <package id="NuGet.Services.Platform" version="3.0.1-rel-3" targetFramework="net45" />
   <package id="NuGet.Services.Platform.Client" version="3.0.1-rel-3" targetFramework="net45" />
   <package id="NuGetGallery.Core" version="2.0.0-alpha-118" targetFramework="net45" />