From 730ad9b3f62169999b5123516baa6a5d954e11c6 Mon Sep 17 00:00:00 2001 From: anurse Date: Tue, 4 Mar 2014 12:43:41 -0800 Subject: [PATCH 1/2] Added Search Index Maintainance Jobs --- .../ServiceConfiguration.Local.cscfg | 2 +- .../Jobs/Bases/SearchIndexJobHandlerBase.cs | 40 +++++ .../Jobs/RebuildSearchIndexJob.cs | 46 ++++++ .../Jobs/UpdateSearchIndexJob.cs | 43 ++++++ .../Monitoring/EventSourceWriter.cs | 49 +++++++ .../NuGet.Services.Work.csproj | 65 ++++++++- src/NuGet.Services.Work/app.config | 137 ++++++++++-------- src/NuGet.Services.Work/packages.config | 13 +- 8 files changed, 323 insertions(+), 72 deletions(-) create mode 100644 src/NuGet.Services.Work/Jobs/Bases/SearchIndexJobHandlerBase.cs create mode 100644 src/NuGet.Services.Work/Jobs/RebuildSearchIndexJob.cs create mode 100644 src/NuGet.Services.Work/Jobs/UpdateSearchIndexJob.cs create mode 100644 src/NuGet.Services.Work/Monitoring/EventSourceWriter.cs diff --git a/src/NuGet.Services.Work.Cloud/ServiceConfiguration.Local.cscfg b/src/NuGet.Services.Work.Cloud/ServiceConfiguration.Local.cscfg index 8980178..051b193 100644 --- a/src/NuGet.Services.Work.Cloud/ServiceConfiguration.Local.cscfg +++ b/src/NuGet.Services.Work.Cloud/ServiceConfiguration.Local.cscfg @@ -12,7 +12,7 @@ - + diff --git a/src/NuGet.Services.Work/Jobs/Bases/SearchIndexJobHandlerBase.cs b/src/NuGet.Services.Work/Jobs/Bases/SearchIndexJobHandlerBase.cs new file mode 100644 index 0000000..f6e64ef --- /dev/null +++ b/src/NuGet.Services.Work/Jobs/Bases/SearchIndexJobHandlerBase.cs @@ -0,0 +1,40 @@ +using System; +using System.Collections.Generic; +using System.Data.SqlClient; +using System.Diagnostics.Tracing; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Microsoft.WindowsAzure.Storage; +using NuGet.Indexing; +using NuGet.Services.Configuration; + +namespace NuGet.Services.Work.Jobs.Bases +{ + public abstract class SearchIndexJobHandlerBase : JobHandler + where T : EventSource + { + public SqlConnectionStringBuilder PackageDatabase { get; set; } + public CloudStorageAccount StorageAccount { get; set; } + public string StorageContainerName { get; set; } + + protected ConfigurationHub Config { get; set; } + + public SearchIndexJobHandlerBase(ConfigurationHub config) + { + Config = config; + } + + protected override InvocationResult BindContext(InvocationContext context) + { + var result = base.BindContext(context); + + // Load default values + PackageDatabase = PackageDatabase ?? Config.Sql.Legacy; + StorageAccount = StorageAccount ?? Config.Storage.Primary; + StorageContainerName = StorageContainerName ?? "ng-search"; + + return result; + } + } +} diff --git a/src/NuGet.Services.Work/Jobs/RebuildSearchIndexJob.cs b/src/NuGet.Services.Work/Jobs/RebuildSearchIndexJob.cs new file mode 100644 index 0000000..6651c4e --- /dev/null +++ b/src/NuGet.Services.Work/Jobs/RebuildSearchIndexJob.cs @@ -0,0 +1,46 @@ +using System; +using System.Collections.Generic; +using System.Data.SqlClient; +using System.Diagnostics.Tracing; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Microsoft.WindowsAzure.Storage; +using Microsoft.WindowsAzure.Storage.Blob; +using NuGet.Indexing; +using NuGet.Services.Configuration; +using NuGet.Services.Work.Jobs.Bases; +using NuGet.Services.Work.Monitoring; + +namespace NuGet.Services.Work.Jobs +{ + public class RebuildSearchIndexJob : SearchIndexJobHandlerBase + { + public RebuildSearchIndexJob(ConfigurationHub config) : base(config) { } + + protected internal override Task Execute() + { + // Run the task + FullBuildTask task = new FullBuildTask() + { + SqlConnectionString = PackageDatabase.ConnectionString, + StorageAccount = StorageAccount, + Container = StorageContainerName ?? "ng-search", + Log = new EventSourceWriter(Log.IndexingTrace) + }; + task.Execute(); + + return Task.FromResult(0); + } + } + + [EventSource(Name="Outercurve-NuGet-Jobs-RebuildSearchIndex")] + public class RebuildSearchIndexEventSource : EventSource + { + [Event( + eventId: 1, + Level = EventLevel.Informational, + Message = "Indexing Trace: {0}")] + public void IndexingTrace(string message) { WriteEvent(1, message); } + } +} diff --git a/src/NuGet.Services.Work/Jobs/UpdateSearchIndexJob.cs b/src/NuGet.Services.Work/Jobs/UpdateSearchIndexJob.cs new file mode 100644 index 0000000..157ffcf --- /dev/null +++ b/src/NuGet.Services.Work/Jobs/UpdateSearchIndexJob.cs @@ -0,0 +1,43 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics.Tracing; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using NuGet.Indexing; +using NuGet.Services.Configuration; +using NuGet.Services.Work.Jobs.Bases; +using NuGet.Services.Work.Monitoring; + +namespace NuGet.Services.Work.Jobs +{ + public class UpdateSearchIndexJob : SearchIndexJobHandlerBase + { + public UpdateSearchIndexJob(ConfigurationHub config) : base(config) { } + + protected internal override Task Execute() + { + // Run the task + UpdateIndexTask task = new UpdateIndexTask() + { + SqlConnectionString = PackageDatabase.ConnectionString, + StorageAccount = StorageAccount, + Container = StorageContainerName ?? "ng-search", + Log = new EventSourceWriter(Log.IndexingTrace), + }; + task.Execute(); + + return Task.FromResult(0); + } + } + + [EventSource(Name="Outercurve-NuGet-Jobs-UpdateSearchIndex")] + public class UpdateSearchIndexEventSource : EventSource + { + [Event( + eventId: 1, + Level = EventLevel.Informational, + Message = "Indexing Trace: {0}")] + public void IndexingTrace(string message) { WriteEvent(1, message); } + } +} diff --git a/src/NuGet.Services.Work/Monitoring/EventSourceWriter.cs b/src/NuGet.Services.Work/Monitoring/EventSourceWriter.cs new file mode 100644 index 0000000..3c9afc4 --- /dev/null +++ b/src/NuGet.Services.Work/Monitoring/EventSourceWriter.cs @@ -0,0 +1,49 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace NuGet.Services.Work.Monitoring +{ + /// + /// Writes lines to a specified EventSource method + /// + public class EventSourceWriter : TextWriter + { + private Action _receiver; + private StringBuilder _buffer = new StringBuilder(); + + public override Encoding Encoding + { + get { return Encoding.Default; } + } + + public EventSourceWriter(Action receiver) + { + _receiver = receiver; + } + + public override void Write(char value) + { + _buffer.Append(value); + CheckFlushLine(); + } + + public override void Flush() + { + _receiver(_buffer.ToString()); + _buffer.Clear(); + } + + private void CheckFlushLine() + { + if (_buffer.ToString().EndsWith(NewLine)) + { + // Flush the buffer + Flush(); + } + } + } +} diff --git a/src/NuGet.Services.Work/NuGet.Services.Work.csproj b/src/NuGet.Services.Work/NuGet.Services.Work.csproj index 89ef179..cbadff5 100644 --- a/src/NuGet.Services.Work/NuGet.Services.Work.csproj +++ b/src/NuGet.Services.Work/NuGet.Services.Work.csproj @@ -44,6 +44,48 @@ ..\..\packages\Dapper.1.13\lib\net45\Dapper.dll + + ..\..\packages\EntityFramework.5.0.0\lib\net45\EntityFramework.dll + + + ..\..\packages\SharpZipLib.0.86.0\lib\20\ICSharpCode.SharpZipLib.dll + + + ..\..\packages\Lucene.Net.3.0.3\lib\NET40\Lucene.Net.dll + + + ..\..\packages\Lucene.Net.Contrib.3.0.3\lib\net40\Lucene.Net.Contrib.Analyzers.dll + + + ..\..\packages\Lucene.Net.Contrib.3.0.3\lib\net40\Lucene.Net.Contrib.Core.dll + + + ..\..\packages\Lucene.Net.Contrib.3.0.3\lib\net40\Lucene.Net.Contrib.FastVectorHighlighter.dll + + + ..\..\packages\Lucene.Net.Contrib.3.0.3\lib\net40\Lucene.Net.Contrib.Highlighter.dll + + + ..\..\packages\Lucene.Net.Contrib.3.0.3\lib\net40\Lucene.Net.Contrib.Memory.dll + + + ..\..\packages\Lucene.Net.Contrib.3.0.3\lib\net40\Lucene.Net.Contrib.Queries.dll + + + ..\..\packages\Lucene.Net.Contrib.3.0.3\lib\net40\Lucene.Net.Contrib.Regex.dll + + + ..\..\packages\Lucene.Net.Contrib.3.0.3\lib\net40\Lucene.Net.Contrib.SimpleFacetedSearch.dll + + + ..\..\packages\Lucene.Net.Contrib.3.0.3\lib\net40\Lucene.Net.Contrib.Snowball.dll + + + ..\..\packages\Lucene.Net.Contrib.3.0.3\lib\net40\Lucene.Net.Contrib.SpellChecker.dll + + + ..\..\packages\Lucene.Net.Store.Azure.2.0.4937.26631\lib\net40\Lucene.Net.Store.Azure.dll + ..\..\packages\Microsoft.Data.Edm.5.6.0\lib\net40\Microsoft.Data.Edm.dll @@ -95,11 +137,19 @@ False ..\..\packages\Nuget.Core.2.8.0\lib\net40-Client\NuGet.Core.dll - - ..\..\packages\NuGet.Services.Platform.3.0.1-rel\lib\net45\NuGet.Services.Platform.dll + + ..\..\packages\NuGet.Indexing.3.0.2-alpha-4\lib\net45\NuGet.Indexing.dll + + + False + ..\..\packages\NuGet.Services.Platform.3.0.1-rel-3\lib\net45\NuGet.Services.Platform.dll + + + False + ..\..\packages\NuGet.Services.Platform.Client.3.0.1-rel-3\lib\portable-net45+wp80+win\NuGet.Services.Platform.Client.dll - - ..\..\packages\NuGet.Services.Platform.Client.3.0.1-rel\lib\portable-net45+wp80+win\NuGet.Services.Platform.Client.dll + + ..\..\packages\NuGetGallery.Core.2.0.0-alpha-118\lib\net45\NuGetGallery.Core.dll ..\..\packages\Owin.1.0\lib\net40\Owin.dll @@ -107,6 +157,7 @@ + @@ -190,6 +241,7 @@ + @@ -203,6 +255,7 @@ + @@ -213,7 +266,9 @@ + + @@ -281,4 +336,4 @@ --> - \ No newline at end of file + diff --git a/src/NuGet.Services.Work/app.config b/src/NuGet.Services.Work/app.config index b00b86a..fe616c5 100644 --- a/src/NuGet.Services.Work/app.config +++ b/src/NuGet.Services.Work/app.config @@ -1,65 +1,76 @@  - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/NuGet.Services.Work/packages.config b/src/NuGet.Services.Work/packages.config index 69b3123..9bfa3a2 100644 --- a/src/NuGet.Services.Work/packages.config +++ b/src/NuGet.Services.Work/packages.config @@ -5,6 +5,10 @@ + + + + @@ -23,14 +27,17 @@ - - + + + + + - \ No newline at end of file + From 84b7629831a1b5cc490f05b8ec3ca75197d59c39 Mon Sep 17 00:00:00 2001 From: anurse Date: Wed, 5 Mar 2014 11:56:01 -0800 Subject: [PATCH 2/2] Fix NuGet/NuGetGallery#1940 by adding search service jobs --- src/JobHost/Program.cs | 7 ++++ .../Jobs/Bases/SearchIndexJobHandlerBase.cs | 1 + .../Jobs/GenerateSearchRankingsJob.cs | 34 +++++++-------- .../Jobs/Models/SearchRankingReport.cs | 20 --------- .../Jobs/RebuildSearchIndexJob.cs | 42 +++++++++++++++++-- .../Jobs/UpdateSearchIndexJob.cs | 8 +++- .../Monitoring/EventSourceWriter.cs | 2 +- .../NuGet.Services.Work.csproj | 11 ++--- src/NuGet.Services.Work/WorkService.cs | 28 +++++++------ src/NuGet.Services.Work/packages.config | 2 +- 10 files changed, 90 insertions(+), 65 deletions(-) delete mode 100644 src/NuGet.Services.Work/Jobs/Models/SearchRankingReport.cs diff --git a/src/JobHost/Program.cs b/src/JobHost/Program.cs index 20421b4..3f38401 100644 --- a/src/JobHost/Program.cs +++ b/src/JobHost/Program.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.Diagnostics.Tracing; using System.Linq; using System.Text; @@ -13,6 +14,12 @@ class Program { static void Main(string[] args) { + if (args.Length > 0 && String.Equals(args[0], "dbg", StringComparison.OrdinalIgnoreCase)) + { + args = args.Skip(1).ToArray(); + Debugger.Launch(); + } + Arguments parsed; try { diff --git a/src/NuGet.Services.Work/Jobs/Bases/SearchIndexJobHandlerBase.cs b/src/NuGet.Services.Work/Jobs/Bases/SearchIndexJobHandlerBase.cs index f6e64ef..9a80efd 100644 --- a/src/NuGet.Services.Work/Jobs/Bases/SearchIndexJobHandlerBase.cs +++ b/src/NuGet.Services.Work/Jobs/Bases/SearchIndexJobHandlerBase.cs @@ -17,6 +17,7 @@ public abstract class SearchIndexJobHandlerBase : JobHandler public SqlConnectionStringBuilder PackageDatabase { get; set; } public CloudStorageAccount StorageAccount { get; set; } public string StorageContainerName { get; set; } + public string LocalIndexFolder { get; set; } protected ConfigurationHub Config { get; set; } diff --git a/src/NuGet.Services.Work/Jobs/GenerateSearchRankingsJob.cs b/src/NuGet.Services.Work/Jobs/GenerateSearchRankingsJob.cs index 6159741..8044282 100644 --- a/src/NuGet.Services.Work/Jobs/GenerateSearchRankingsJob.cs +++ b/src/NuGet.Services.Work/Jobs/GenerateSearchRankingsJob.cs @@ -9,6 +9,7 @@ using Dapper; using Microsoft.WindowsAzure.Storage; using Microsoft.WindowsAzure.Storage.Blob; +using Newtonsoft.Json; using Newtonsoft.Json.Linq; using NuGet.Services.Client; using NuGet.Services.Configuration; @@ -56,8 +57,10 @@ protected internal override async Task Execute() } // Gather overall rankings + JObject report = new JObject(); Log.GatheringOverallRankings(WarehouseConnection.DataSource, WarehouseConnection.InitialCatalog); var overallData = await GatherOverallRankings(); + report.Add("Rank", overallData); Log.GatheredOverallRankings(overallData.Count); // Get project types @@ -66,27 +69,18 @@ protected internal override async Task Execute() Log.GotAvailableProjectTypes(projectTypes.Count); // Gather data by project type - IDictionary> byProjectType = new Dictionary>(); int count = 0; Log.GatheringProjectTypeRankings(WarehouseConnection.DataSource, WarehouseConnection.InitialCatalog); foreach (var projectType in projectTypes) { Log.GatheringProjectTypeRanking(WarehouseConnection.DataSource, WarehouseConnection.InitialCatalog, projectType); var data = await GatherProjectTypeRanking(projectType); + report.Add(projectType, data); Log.GatheredProjectTypeRanking(data.Count, projectType); count += data.Count; - - byProjectType.Add(projectType, data); } Log.GatheredProjectTypeRankings(count); - // Generate the report - var report = new SearchRankingReport() - { - Overall = overallData, - ByProjectType = byProjectType - }; - // Write the JSON blob if (!String.IsNullOrEmpty(OutputDirectory)) { @@ -99,7 +93,7 @@ protected internal override async Task Execute() } } - private async Task WriteToFile(SearchRankingReport report) + private async Task WriteToFile(JObject report) { string fullPath = Path.Combine(OutputDirectory, ReportName); string parentDir = Path.GetDirectoryName(fullPath); @@ -116,20 +110,20 @@ private async Task WriteToFile(SearchRankingReport report) } using (var writer = new StreamWriter(File.OpenWrite(fullPath))) { - await writer.WriteAsync(JsonFormat.Serialize(report)); + await writer.WriteAsync(report.ToString(Formatting.Indented)); } } Log.WroteReportBlob(fullPath); } - private async Task WriteToBlob(SearchRankingReport report) + private async Task WriteToBlob(JObject report) { var blob = DestinationContainer.GetBlockBlobReference(ReportName); Log.WritingReportBlob(blob.Uri.AbsoluteUri); if (!WhatIf) { blob.Properties.ContentType = MimeTypes.Json; - await blob.UploadTextAsync(JsonFormat.Serialize(report)); + await blob.UploadTextAsync(report.ToString(Formatting.Indented)); } Log.WroteReportBlob(blob.Uri.AbsoluteUri); } @@ -145,7 +139,7 @@ private void LoadDefaults() } } - private async Task> GatherOverallRankings() + private async Task GatherOverallRankings() { using (var connection = await WarehouseConnection.ConnectTo()) { @@ -153,7 +147,9 @@ private async Task> GatherOverallRankings() var script = await ResourceHelpers.ReadResourceFile("NuGet.Services.Work.Jobs.Scripts.SearchRanking_Overall.sql"); // Execute it and return the results - return (await connection.QueryAsync(script)).ToList(); + return new JArray( + (await connection.QueryAsync(script)) + .Select(e => e.PackageId)); } } @@ -166,7 +162,7 @@ private async Task> GetProjectTypes() } } - private async Task> GatherProjectTypeRanking(string projectType) + private async Task GatherProjectTypeRanking(string projectType) { using (var connection = await WarehouseConnection.ConnectTo()) { @@ -174,7 +170,9 @@ private async Task> GatherProjectTypeRanking(string pr var script = await ResourceHelpers.ReadResourceFile("NuGet.Services.Work.Jobs.Scripts.SearchRanking_ByProjectType.sql"); // Execute it and return the results - return (await connection.QueryAsync(script, new { ProjectGuid = projectType })).ToList(); + return new JArray( + (await connection.QueryAsync(script, new { ProjectGuid = projectType })) + .Select(e => e.PackageId)); } } } diff --git a/src/NuGet.Services.Work/Jobs/Models/SearchRankingReport.cs b/src/NuGet.Services.Work/Jobs/Models/SearchRankingReport.cs deleted file mode 100644 index fac82a9..0000000 --- a/src/NuGet.Services.Work/Jobs/Models/SearchRankingReport.cs +++ /dev/null @@ -1,20 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -namespace NuGet.Services.Work.Jobs.Models -{ - public class SearchRankingReport - { - public IList Overall { get; set; } - public IDictionary> ByProjectType { get; set; } - - public SearchRankingReport() - { - Overall = new List(); - ByProjectType = new Dictionary>(); - } - } -} diff --git a/src/NuGet.Services.Work/Jobs/RebuildSearchIndexJob.cs b/src/NuGet.Services.Work/Jobs/RebuildSearchIndexJob.cs index 6651c4e..bca4e29 100644 --- a/src/NuGet.Services.Work/Jobs/RebuildSearchIndexJob.cs +++ b/src/NuGet.Services.Work/Jobs/RebuildSearchIndexJob.cs @@ -18,29 +18,63 @@ public class RebuildSearchIndexJob : SearchIndexJobHandlerBaseFalse ..\..\packages\Nuget.Core.2.8.0\lib\net40-Client\NuGet.Core.dll - - ..\..\packages\NuGet.Indexing.3.0.2-alpha-4\lib\net45\NuGet.Indexing.dll + + False + ..\..\packages\NuGet.Indexing.3.0.2-alpha-9\lib\net45\NuGet.Indexing.dll False @@ -181,9 +182,6 @@ ..\..\packages\Rx-Linq.2.2.2\lib\net45\System.Reactive.Linq.dll - - ..\..\packages\Rx-PlatformServices.2.2.2\lib\net45\System.Reactive.PlatformServices.dll - @@ -252,7 +250,6 @@ - @@ -336,4 +333,4 @@ --> - + \ No newline at end of file diff --git a/src/NuGet.Services.Work/WorkService.cs b/src/NuGet.Services.Work/WorkService.cs index 6c8fca2..ab1df46 100644 --- a/src/NuGet.Services.Work/WorkService.cs +++ b/src/NuGet.Services.Work/WorkService.cs @@ -210,22 +210,24 @@ public IObservable RunJob(string job, string payload) QueuedAt = DateTime.UtcNow, NextVisibleAt = DateTime.UtcNow + TimeSpan.FromMinutes(5) }); - var buffer = new ReplaySubject(); - var capture = new InvocationLogCapture(invocation); - capture.Subscribe(buffer.OnNext, buffer.OnError); - runner.Dispatch(invocation, capture, CancellationToken.None).ContinueWith(t => + return Observable.Create(observer => { - if (t.IsFaulted) + var capture = new InvocationLogCapture(invocation); + capture.Subscribe(e => observer.OnNext(e), ex => observer.OnError(ex)); + runner.Dispatch(invocation, capture, CancellationToken.None).ContinueWith(t => { - buffer.OnError(t.Exception); - } - else - { - buffer.OnCompleted(); - } - return t; + if (t.IsFaulted) + { + observer.OnError(t.Exception); + } + else + { + observer.OnCompleted(); + } + return t; + }); + return () => { }; // No action on unsubscribe }); - return buffer; } protected override void ConfigureAttributeRouting(DefaultInlineConstraintResolver resolver) diff --git a/src/NuGet.Services.Work/packages.config b/src/NuGet.Services.Work/packages.config index 9bfa3a2..c83f187 100644 --- a/src/NuGet.Services.Work/packages.config +++ b/src/NuGet.Services.Work/packages.config @@ -27,7 +27,7 @@ - +