Merge pull request #8 from NuGet/anurse/1940-searchjobs

Fix NuGet/NuGetGallery#1940 by adding search maintenance tasks

src/JobHost/Program.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Diagnostics;
 using System.Diagnostics.Tracing;
 using System.Linq;
 using System.Text;
@@ -13,6 +14,12 @@ class Program
     {
         static void Main(string[] args)
         {
+            if (args.Length > 0 && String.Equals(args[0], "dbg", StringComparison.OrdinalIgnoreCase))
+            {
+                args = args.Skip(1).ToArray();
+                Debugger.Launch();
+            }
+
             Arguments parsed;
             try
             {

src/NuGet.Services.Work.Cloud/ServiceConfiguration.Local.cscfg

@@ -12,7 +12,7 @@
             <!-- SQL Connections -->
             <Setting name="Sql.Primary" value="Data Source=(LocalDB)\v11.0;Initial Catalog=nuget-local-0;Integrated Security=SSPI" />
             <Setting name="Sql.Legacy" value="Data Source=(LocalDB)\v11.0;Initial Catalog=NuGetGallery;Integrated Security=SSPI" />
-            <Setting name="Sql.Warehouse" value="Data Source=(LocalDB)\v11.0;Initial Catalog=NuGetWarehouse;Integrated Security=SSPI" />
+            <Setting name="Sql.Warehouse" value="Data Source=(LocalDB)\v11.0;Initial Catalog=nuget-local-0-warehouse;Integrated Security=SSPI" />
 
             <Setting name="Work.PollInterval" value="00:00:15" />
             <Setting name="Work.WorkersPerCore" value="2" />

src/NuGet.Services.Work/Jobs/Bases/SearchIndexJobHandlerBase.cs

@@ -0,0 +1,41 @@
+using System;
+using System.Collections.Generic;
+using System.Data.SqlClient;
+using System.Diagnostics.Tracing;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Microsoft.WindowsAzure.Storage;
+using NuGet.Indexing;
+using NuGet.Services.Configuration;
+
+namespace NuGet.Services.Work.Jobs.Bases
+{
+    public abstract class SearchIndexJobHandlerBase<T> : JobHandler<T>
+        where T : EventSource
+    {
+        public SqlConnectionStringBuilder PackageDatabase { get; set; }
+        public CloudStorageAccount StorageAccount { get; set; }
+        public string StorageContainerName { get; set; }
+        public string LocalIndexFolder { get; set; }
+
+        protected ConfigurationHub Config { get; set; }
+
+        public SearchIndexJobHandlerBase(ConfigurationHub config)
+        {
+            Config = config;
+        }
+
+        protected override InvocationResult BindContext(InvocationContext context)
+        {
+            var result = base.BindContext(context);
+
+            // Load default values
+            PackageDatabase = PackageDatabase ?? Config.Sql.Legacy;
+            StorageAccount = StorageAccount ?? Config.Storage.Primary;
+            StorageContainerName = StorageContainerName ?? "ng-search";
+
+            return result;
+        }
+    }
+}

src/NuGet.Services.Work/Jobs/GenerateSearchRankingsJob.cs

@@ -9,6 +9,7 @@
 using Dapper;
 using Microsoft.WindowsAzure.Storage;
 using Microsoft.WindowsAzure.Storage.Blob;
+using Newtonsoft.Json;
 using Newtonsoft.Json.Linq;
 using NuGet.Services.Client;
 using NuGet.Services.Configuration;
@@ -56,8 +57,10 @@ protected internal override async Task Execute()
             }
 
             // Gather overall rankings
+            JObject report = new JObject();
             Log.GatheringOverallRankings(WarehouseConnection.DataSource, WarehouseConnection.InitialCatalog);
             var overallData = await GatherOverallRankings();
+            report.Add("Rank", overallData);
             Log.GatheredOverallRankings(overallData.Count);
 
             // Get project types
@@ -66,27 +69,18 @@ protected internal override async Task Execute()
             Log.GotAvailableProjectTypes(projectTypes.Count);
 
             // Gather data by project type
-            IDictionary<string, IList<SearchRankingEntry>> byProjectType = new Dictionary<string, IList<SearchRankingEntry>>();
             int count = 0;
             Log.GatheringProjectTypeRankings(WarehouseConnection.DataSource, WarehouseConnection.InitialCatalog);
             foreach (var projectType in projectTypes)
             {
                 Log.GatheringProjectTypeRanking(WarehouseConnection.DataSource, WarehouseConnection.InitialCatalog, projectType);
                 var data = await GatherProjectTypeRanking(projectType);
+                report.Add(projectType, data);
                 Log.GatheredProjectTypeRanking(data.Count, projectType);
                 count += data.Count;
-
-                byProjectType.Add(projectType, data);
             }
             Log.GatheredProjectTypeRankings(count);
 
-            // Generate the report
-            var report = new SearchRankingReport()
-            {
-                Overall = overallData,
-                ByProjectType = byProjectType
-            };
-
             // Write the JSON blob
             if (!String.IsNullOrEmpty(OutputDirectory))
             {
@@ -99,7 +93,7 @@ protected internal override async Task Execute()
             }
         }
 
-        private async Task WriteToFile(SearchRankingReport report)
+        private async Task WriteToFile(JObject report)
         {
             string fullPath = Path.Combine(OutputDirectory, ReportName);
             string parentDir = Path.GetDirectoryName(fullPath);
@@ -116,20 +110,20 @@ private async Task WriteToFile(SearchRankingReport report)
                 }
                 using (var writer = new StreamWriter(File.OpenWrite(fullPath)))
                 {
-                    await writer.WriteAsync(JsonFormat.Serialize(report));
+                    await writer.WriteAsync(report.ToString(Formatting.Indented));
                 }
             }
             Log.WroteReportBlob(fullPath);
         }
 
-        private async Task WriteToBlob(SearchRankingReport report)
+        private async Task WriteToBlob(JObject report)
         {
             var blob = DestinationContainer.GetBlockBlobReference(ReportName);
             Log.WritingReportBlob(blob.Uri.AbsoluteUri);
             if (!WhatIf)
             {
                 blob.Properties.ContentType = MimeTypes.Json;
-                await blob.UploadTextAsync(JsonFormat.Serialize(report));
+                await blob.UploadTextAsync(report.ToString(Formatting.Indented));
             }
             Log.WroteReportBlob(blob.Uri.AbsoluteUri);
         }
@@ -145,15 +139,17 @@ private void LoadDefaults()
             }
         }
 
-        private async Task<IList<SearchRankingEntry>> GatherOverallRankings()
+        private async Task<JArray> GatherOverallRankings()
         {
             using (var connection = await WarehouseConnection.ConnectTo())
             {
                 // Get the script
                 var script = await ResourceHelpers.ReadResourceFile("NuGet.Services.Work.Jobs.Scripts.SearchRanking_Overall.sql");
 
                 // Execute it and return the results
-                return (await connection.QueryAsync<SearchRankingEntry>(script)).ToList();
+                return new JArray(
+                    (await connection.QueryAsync<SearchRankingEntry>(script))
+                        .Select(e => e.PackageId));
             }
         }
 
@@ -166,15 +162,17 @@ private async Task<IList<string>> GetProjectTypes()
             }
         }
 
-        private async Task<IList<SearchRankingEntry>> GatherProjectTypeRanking(string projectType)
+        private async Task<JArray> GatherProjectTypeRanking(string projectType)
         {
             using (var connection = await WarehouseConnection.ConnectTo())
             {
                 // Get the script
                 var script = await ResourceHelpers.ReadResourceFile("NuGet.Services.Work.Jobs.Scripts.SearchRanking_ByProjectType.sql");
 
                 // Execute it and return the results
-                return (await connection.QueryAsync<SearchRankingEntry>(script, new { ProjectGuid = projectType })).ToList();
+                return new JArray(
+                    (await connection.QueryAsync<SearchRankingEntry>(script, new { ProjectGuid = projectType }))
+                        .Select(e => e.PackageId));
             }
         }
     }

src/NuGet.Services.Work/Jobs/RebuildSearchIndexJob.cs

@@ -0,0 +1,80 @@
+using System;
+using System.Collections.Generic;
+using System.Data.SqlClient;
+using System.Diagnostics.Tracing;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Microsoft.WindowsAzure.Storage;
+using Microsoft.WindowsAzure.Storage.Blob;
+using NuGet.Indexing;
+using NuGet.Services.Configuration;
+using NuGet.Services.Work.Jobs.Bases;
+using NuGet.Services.Work.Monitoring;
+
+namespace NuGet.Services.Work.Jobs
+{
+    public class RebuildSearchIndexJob : SearchIndexJobHandlerBase<RebuildSearchIndexEventSource>
+    {
+        public RebuildSearchIndexJob(ConfigurationHub config) : base(config) { }
+
+        protected internal override async Task Execute()
+        {
+            // This job can take a long time and is run manually. Make the job timeout long
+            await Extend(TimeSpan.FromHours(12));
+
+            // Run the task
+            Log.BeginningIndex(
+                PackageDatabase.DataSource + "/" + PackageDatabase.InitialCatalog,
+                String.IsNullOrEmpty(LocalIndexFolder) ?
+                    (StorageAccount.Credentials.AccountName + "/" + StorageContainerName) :
+                    LocalIndexFolder);
+            FullBuildTask task = new FullBuildTask()
+            {
+                SqlConnectionString = PackageDatabase.ConnectionString,
+                StorageAccount = StorageAccount,
+                Container = String.IsNullOrEmpty(LocalIndexFolder) ?
+                    StorageContainerName :
+                    null,
+                Folder = LocalIndexFolder,
+                Log = new EventSourceWriter(Log.IndexingTrace)
+            };
+            task.Execute();
+            Log.FinishedIndex();
+        }
+    }
+
+    [EventSource(Name="Outercurve-NuGet-Jobs-RebuildSearchIndex")]
+    public class RebuildSearchIndexEventSource : EventSource
+    {
+        public static readonly RebuildSearchIndexEventSource Log = new RebuildSearchIndexEventSource();
+        private RebuildSearchIndexEventSource() { }
+
+        [Event(
+            eventId: 1,
+            Level = EventLevel.Informational,
+            Message = "Indexing Trace: {0}")]
+        public void IndexingTrace(string message) { WriteEvent(1, message); }
+
+        [Event(
+            eventId: 2,
+            Level = EventLevel.Informational,
+            Message = "Beginning Index Rebuild from {0} to {1}",
+            Task = Tasks.Indexing,
+            Opcode = EventOpcode.Start)]
+        public void BeginningIndex(string source, string destination) { WriteEvent(2, source, destination); }
+
+        [Event(
+            eventId: 3,
+            Level = EventLevel.Informational,
+            Message = "Finished Index Rebuild",
+            Task = Tasks.Indexing,
+            Opcode = EventOpcode.Stop)]
+        public void FinishedIndex() { WriteEvent(3); }
+
+        public static class Tasks
+        {
+            public const EventTask Indexing = (EventTask)0x1;
+        }
+    }
+}

src/NuGet.Services.Work/Jobs/UpdateSearchIndexJob.cs

@@ -0,0 +1,49 @@
+using System;
+using System.Collections.Generic;
+using System.Diagnostics.Tracing;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using NuGet.Indexing;
+using NuGet.Services.Configuration;
+using NuGet.Services.Work.Jobs.Bases;
+using NuGet.Services.Work.Monitoring;
+
+namespace NuGet.Services.Work.Jobs
+{
+    public class UpdateSearchIndexJob : SearchIndexJobHandlerBase<UpdateSearchIndexEventSource>
+    {
+        public UpdateSearchIndexJob(ConfigurationHub config) : base(config) { }
+
+        protected internal override Task Execute()
+        {
+            // Run the task
+            UpdateIndexTask task = new UpdateIndexTask()
+            {                
+                SqlConnectionString = PackageDatabase.ConnectionString,
+                StorageAccount = StorageAccount,
+                Container = String.IsNullOrEmpty(LocalIndexFolder) ? 
+                    (StorageContainerName ?? "ng-search") :
+                    null,
+                Folder = LocalIndexFolder,
+                Log = new EventSourceWriter(Log.IndexingTrace),
+            };
+            task.Execute();
+
+            return Task.FromResult(0);
+        }
+    }
+
+    [EventSource(Name="Outercurve-NuGet-Jobs-UpdateSearchIndex")]
+    public class UpdateSearchIndexEventSource : EventSource
+    {
+        public static readonly UpdateSearchIndexEventSource Log = new UpdateSearchIndexEventSource();
+        private UpdateSearchIndexEventSource() { }
+
+        [Event(
+            eventId: 1,
+            Level = EventLevel.Informational,
+            Message = "Indexing Trace: {0}")]
+        public void IndexingTrace(string message) { WriteEvent(1, message); }
+    }
+}

src/NuGet.Services.Work/Monitoring/EventSourceWriter.cs

@@ -0,0 +1,49 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace NuGet.Services.Work.Monitoring
+{
+    /// <summary>
+    /// Writes lines to a specified EventSource method
+    /// </summary>
+    public class EventSourceWriter : TextWriter
+    {
+        private Action<string> _receiver;
+        private StringBuilder _buffer = new StringBuilder();
+
+        public override Encoding Encoding
+        {
+            get { return Encoding.Default; }
+        }
+
+        public EventSourceWriter(Action<string> receiver)
+        {
+            _receiver = receiver;
+        }
+
+        public override void Write(char value)
+        {
+            _buffer.Append(value);
+            CheckFlushLine();
+        }
+
+        public override void Flush()
+        {
+            _receiver(_buffer.ToString().Trim());
+            _buffer.Clear();
+        }
+
+        private void CheckFlushLine()
+        {
+            if (_buffer.ToString().EndsWith(NewLine))
+            {
+                // Flush the buffer
+                Flush();
+            }
+        }
+    }
+}